From 0587256733a1f9df8deb4ac4c6f63a2ded10e274 Mon Sep 17 00:00:00 2001
From: Yazeed Al Oyoun <yazeed.aloyoun@gmail.com>
Date: Thu, 5 Mar 2020 21:57:01 +0100
Subject: [PATCH 001/151] minor create_trade() optimization

---
 freqtrade/freqtradebot.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 4cbacdb1e..e5ae9043a 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -246,7 +246,7 @@ class FreqtradeBot:
         if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False):
             logger.info(
                 f"Getting price from order book {bid_strategy['price_side'].capitalize()} side."
-                )
+            )
             order_book_top = bid_strategy.get('order_book_top', 1)
             order_book = self.exchange.get_order_book(pair, order_book_top)
             logger.debug('order_book %s', order_book)
@@ -394,21 +394,21 @@ class FreqtradeBot:
             logger.info(f"Pair {pair} is currently locked.")
             return False
 
+        if not self.get_free_open_trades():
+            logger.debug(f"Can't open a new trade for {pair}: max number of trades is reached.")
+            return False
+
+        stake_amount = self.get_trade_stake_amount(pair)
+        if not stake_amount:
+            logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
+            return False
+
         # running get_signal on historical data fetched
         (buy, sell) = self.strategy.get_signal(
             pair, self.strategy.ticker_interval,
             self.dataprovider.ohlcv(pair, self.strategy.ticker_interval))
 
         if buy and not sell:
-            if not self.get_free_open_trades():
-                logger.debug("Can't open a new trade: max number of trades is reached.")
-                return False
-
-            stake_amount = self.get_trade_stake_amount(pair)
-            if not stake_amount:
-                logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
-                return False
-
             logger.info(f"Buy signal found: about create a new trade with stake_amount: "
                         f"{stake_amount} ...")
 

From 3a8b68c0fd721094c606286b4dce6a86d89442d9 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Tue, 10 Mar 2020 20:30:36 +0100
Subject: [PATCH 002/151] Initial work on progressbar

---
 freqtrade/loggers.py           |  9 ++++++---
 freqtrade/optimize/hyperopt.py | 34 ++++++++++++++++++++++++++++++----
 requirements-hyperopt.txt      |  1 +
 3 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py
index c69388430..609604ff2 100644
--- a/freqtrade/loggers.py
+++ b/freqtrade/loggers.py
@@ -4,6 +4,7 @@ import sys
 from logging import Formatter
 from logging.handlers import RotatingFileHandler, SysLogHandler
 from typing import Any, Dict, List
+import progressbar
 
 from freqtrade.exceptions import OperationalException
 
@@ -18,13 +19,13 @@ def _set_loggers(verbosity: int = 0) -> None:
     """
 
     logging.getLogger('requests').setLevel(
-            logging.INFO if verbosity <= 1 else logging.DEBUG
+        logging.INFO if verbosity <= 1 else logging.DEBUG
     )
     logging.getLogger("urllib3").setLevel(
-            logging.INFO if verbosity <= 1 else logging.DEBUG
+        logging.INFO if verbosity <= 1 else logging.DEBUG
     )
     logging.getLogger('ccxt.base.exchange').setLevel(
-            logging.INFO if verbosity <= 2 else logging.DEBUG
+        logging.INFO if verbosity <= 2 else logging.DEBUG
     )
     logging.getLogger('telegram').setLevel(logging.INFO)
 
@@ -36,6 +37,8 @@ def setup_logging(config: Dict[str, Any]) -> None:
     # Log level
     verbosity = config['verbosity']
 
+    progressbar.streams.wrap_stderr()
+
     # Log to stderr
     log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)]
 
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 4c32a0543..93fc3c558 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -7,7 +7,6 @@ This module contains the hyperopt logic
 import locale
 import logging
 import random
-import sys
 import warnings
 from math import ceil
 from collections import OrderedDict
@@ -22,8 +21,9 @@ from colorama import init as colorama_init
 from joblib import (Parallel, cpu_count, delayed, dump, load,
                     wrap_non_picklable_objects)
 from pandas import DataFrame, json_normalize, isna
+import progressbar
 import tabulate
-from os import path
+from os import path, popen
 import io
 
 from freqtrade.data.converter import trim_dataframe
@@ -270,7 +270,6 @@ class Hyperopt:
             # Print '\n' after each 100th epoch to separate dots from the log messages.
             # Otherwise output is messy on a terminal.
             print('.', end='' if results['current_epoch'] % 100 != 0 else None)  # type: ignore
-            sys.stdout.flush()
 
         if self.print_all or is_best:
             if not self.print_all:
@@ -622,6 +621,10 @@ class Hyperopt:
     def _set_random_state(self, random_state: Optional[int]) -> int:
         return random_state or random.randint(1, 2**16 - 1)
 
+    def _get_height(self) -> int:
+        rows = int((popen('stty size', 'r').read().split())[0])
+        return rows
+
     def start(self) -> None:
         self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
         logger.info(f"Using optimizer random state: {self.random_state}")
@@ -629,7 +632,6 @@ class Hyperopt:
         data, timerange = self.backtesting.load_bt_data()
 
         preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
-
         # Trim startup period from analyzed dataframe
         for pair, df in preprocessed.items():
             preprocessed[pair] = trim_dataframe(df, timerange)
@@ -659,6 +661,14 @@ class Hyperopt:
 
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
+                self.progress_bar = progressbar.ProgressBar(
+                    min_value=0,
+                    max_value=self.total_epochs,
+                    initial_value=0,
+                    line_breaks=True,
+                    enable_colors=self.print_colorized
+                )
+                self.progress_bar.start()
                 jobs = parallel._effective_n_jobs()
                 logger.info(f'Effective number of parallel workers used: {jobs}')
                 EVALS = ceil(self.total_epochs / jobs)
@@ -673,6 +683,9 @@ class Hyperopt:
                     self.opt.tell(asked, [v['loss'] for v in f_val])
                     self.fix_optimizer_models_list()
 
+                    # Calculate progressbar outputs
+                    pbar_line = ceil(self._get_height() / 2)
+
                     for j, val in enumerate(f_val):
                         # Use human-friendly indexes here (starting from 1)
                         current = i * jobs + j + 1
@@ -689,12 +702,19 @@ class Hyperopt:
 
                         self.print_results(val)
 
+                        if pbar_line <= current:
+                            self.progress_bar.update(current)
+                            pbar_line = current + ceil(self._get_height() / 2)
+
                         if is_best:
                             self.current_best_loss = val['loss']
                         self.trials.append(val)
                         # Save results after each best epoch and every 100 epochs
                         if is_best or current % 100 == 0:
                             self.save_trials()
+                self.progress_bar.finish()
+
+                # self.progress_bar.update(current)
         except KeyboardInterrupt:
             print('User interrupted..')
 
@@ -708,3 +728,9 @@ class Hyperopt:
             # This is printed when Ctrl+C is pressed quickly, before first epochs have
             # a chance to be evaluated.
             print("No epochs evaluated yet, no best result.")
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        del state['trials']
+        del state['progress_bar']
+        return state
diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt
index c7e586a33..3c500839e 100644
--- a/requirements-hyperopt.txt
+++ b/requirements-hyperopt.txt
@@ -7,3 +7,4 @@ scikit-learn==0.22.2.post1
 scikit-optimize==0.7.4
 filelock==3.0.12
 joblib==0.14.1
+progressbar2==3.50.0

From 81cbb925568dbae6260caf88ced5c90f7abc1a3b Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Wed, 11 Mar 2020 22:30:36 +0100
Subject: [PATCH 003/151] Switch to TQDM

---
 freqtrade/commands/hyperopt_commands.py |  4 +-
 freqtrade/optimize/hyperopt.py          | 71 +++++++++++++++++++------
 2 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py
index 5b2388252..3f61ea66c 100755
--- a/freqtrade/commands/hyperopt_commands.py
+++ b/freqtrade/commands/hyperopt_commands.py
@@ -52,8 +52,8 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
 
     if not export_csv:
         try:
-            Hyperopt.print_result_table(config, trials, total_epochs,
-                                        not filteroptions['only_best'], print_colorized, 0)
+            print(Hyperopt.get_result_table(config, trials, total_epochs,
+                                            not filteroptions['only_best'], print_colorized, 0))
         except KeyboardInterrupt:
             print('User interrupted..')
 
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 93fc3c558..e110f7e9c 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -21,7 +21,7 @@ from colorama import init as colorama_init
 from joblib import (Parallel, cpu_count, delayed, dump, load,
                     wrap_non_picklable_objects)
 from pandas import DataFrame, json_normalize, isna
-import progressbar
+from tqdm import tqdm
 import tabulate
 from os import path, popen
 import io
@@ -275,11 +275,37 @@ class Hyperopt:
             if not self.print_all:
                 # Separate the results explanation string from dots
                 print("\n")
-            self.print_result_table(self.config, results, self.total_epochs,
-                                    self.print_all, self.print_colorized,
-                                    self.hyperopt_table_header)
+            print(self.get_result_table(
+                    self.config, results, self.total_epochs,
+                    self.print_all, self.print_colorized,
+                    self.hyperopt_table_header
+                )
+            )
             self.hyperopt_table_header = 2
 
+    def get_results(self, results) -> str:
+        """
+        Log results if it is better than any previous evaluation
+        """
+        output = ''
+        is_best = results['is_best']
+        # if not self.print_all:
+        #    Print '\n' after each 100th epoch to separate dots from the log messages.
+        #    Otherwise output is messy on a terminal.
+        #    return '.', end='' if results['current_epoch'] % 100 != 0 else None  # type: ignore
+
+        if self.print_all or is_best:
+            # if not self.print_all:
+            #    Separate the results explanation string from dots
+            #    print("\n")
+            output = self.get_result_table(
+                self.config, results, self.total_epochs,
+                self.print_all, self.print_colorized,
+                self.hyperopt_table_header
+            )
+            self.hyperopt_table_header = 2
+        return output
+
     @staticmethod
     def print_results_explanation(results, total_epochs, highlight_best: bool,
                                   print_colorized: bool) -> None:
@@ -303,13 +329,13 @@ class Hyperopt:
                 f"Objective: {results['loss']:.5f}")
 
     @staticmethod
-    def print_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
-                           print_colorized: bool, remove_header: int) -> None:
+    def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
+                         print_colorized: bool, remove_header: int) -> str:
         """
         Log result table
         """
         if not results:
-            return
+            return ''
 
         tabulate.PRESERVE_WHITESPACE = True
 
@@ -380,7 +406,7 @@ class Hyperopt:
                 trials.to_dict(orient='list'), tablefmt='psql',
                 headers='keys', stralign="right"
             )
-        print(table)
+        return table
 
     @staticmethod
     def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool,
@@ -661,6 +687,7 @@ class Hyperopt:
 
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
+                """
                 self.progress_bar = progressbar.ProgressBar(
                     min_value=0,
                     max_value=self.total_epochs,
@@ -668,9 +695,17 @@ class Hyperopt:
                     line_breaks=True,
                     enable_colors=self.print_colorized
                 )
-                self.progress_bar.start()
+                """
                 jobs = parallel._effective_n_jobs()
                 logger.info(f'Effective number of parallel workers used: {jobs}')
+
+                # Define progressbar
+                self.progress_bar = tqdm(
+                    total=self.total_epochs, ncols=108, unit=' Epoch',
+                    bar_format='Epoch {n_fmt}/{total_fmt} ({percentage:3.0f}%)|{bar}|'
+                    ' [{elapsed}<{remaining} {rate_fmt}{postfix}]'
+                )
+
                 EVALS = ceil(self.total_epochs / jobs)
                 for i in range(EVALS):
                     # Correct the number of epochs to be processed for the last
@@ -684,8 +719,7 @@ class Hyperopt:
                     self.fix_optimizer_models_list()
 
                     # Calculate progressbar outputs
-                    pbar_line = ceil(self._get_height() / 2)
-
+                    # pbar_line = ceil(self._get_height() / 2)
                     for j, val in enumerate(f_val):
                         # Use human-friendly indexes here (starting from 1)
                         current = i * jobs + j + 1
@@ -699,20 +733,25 @@ class Hyperopt:
                         # evaluations can take different time. Here they are aligned in the
                         # order they will be shown to the user.
                         val['is_best'] = is_best
-
-                        self.print_results(val)
-
+                        # print(current)
+                        output = self.get_results(val)
+                        self.progress_bar.write(output)
+                        # self.progress_bar.write(str(len(output.split('\n')[0])))
+                        self.progress_bar.ncols = len(output.split('\n')[0])
+                        self.progress_bar.update(1)
+                        """
                         if pbar_line <= current:
                             self.progress_bar.update(current)
                             pbar_line = current + ceil(self._get_height() / 2)
-
+                        """
                         if is_best:
                             self.current_best_loss = val['loss']
                         self.trials.append(val)
                         # Save results after each best epoch and every 100 epochs
                         if is_best or current % 100 == 0:
                             self.save_trials()
-                self.progress_bar.finish()
+                self.progress_bar.ncols = 108
+                self.progress_bar.close()
 
                 # self.progress_bar.update(current)
         except KeyboardInterrupt:

From 755763ec42b51370c7326e4e90f5c35921ba2125 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Wed, 11 Mar 2020 22:43:27 +0100
Subject: [PATCH 004/151] Update requirements

---
 requirements-hyperopt.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt
index 3c500839e..7469674cd 100644
--- a/requirements-hyperopt.txt
+++ b/requirements-hyperopt.txt
@@ -7,4 +7,4 @@ scikit-learn==0.22.2.post1
 scikit-optimize==0.7.4
 filelock==3.0.12
 joblib==0.14.1
-progressbar2==3.50.0
+tqdm==4.43.0

From 40a413c524b9eb2b2d13e222ac1e81173a1d5b1e Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Wed, 11 Mar 2020 22:50:23 +0100
Subject: [PATCH 005/151] More remove of progressbar2

---
 freqtrade/loggers.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py
index 609604ff2..153ce8c80 100644
--- a/freqtrade/loggers.py
+++ b/freqtrade/loggers.py
@@ -4,7 +4,6 @@ import sys
 from logging import Formatter
 from logging.handlers import RotatingFileHandler, SysLogHandler
 from typing import Any, Dict, List
-import progressbar
 
 from freqtrade.exceptions import OperationalException
 
@@ -37,8 +36,6 @@ def setup_logging(config: Dict[str, Any]) -> None:
     # Log level
     verbosity = config['verbosity']
 
-    progressbar.streams.wrap_stderr()
-
     # Log to stderr
     log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)]
 

From 9387ed923c97a2dffebfa1ca57662745d97a0f44 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Thu, 12 Mar 2020 02:07:50 +0100
Subject: [PATCH 006/151] fix for empty lines

---
 freqtrade/optimize/hyperopt.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index e110f7e9c..48a89e1d6 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -289,15 +289,8 @@ class Hyperopt:
         """
         output = ''
         is_best = results['is_best']
-        # if not self.print_all:
-        #    Print '\n' after each 100th epoch to separate dots from the log messages.
-        #    Otherwise output is messy on a terminal.
-        #    return '.', end='' if results['current_epoch'] % 100 != 0 else None  # type: ignore
 
         if self.print_all or is_best:
-            # if not self.print_all:
-            #    Separate the results explanation string from dots
-            #    print("\n")
             output = self.get_result_table(
                 self.config, results, self.total_epochs,
                 self.print_all, self.print_colorized,
@@ -735,9 +728,10 @@ class Hyperopt:
                         val['is_best'] = is_best
                         # print(current)
                         output = self.get_results(val)
-                        self.progress_bar.write(output)
+                        if output:
+                            self.progress_bar.write(output)
                         # self.progress_bar.write(str(len(output.split('\n')[0])))
-                        self.progress_bar.ncols = len(output.split('\n')[0])
+                        self.progress_bar.ncols = 108
                         self.progress_bar.update(1)
                         """
                         if pbar_line <= current:

From df1ae565dc09c93b046b2ba79cb5a1ce647c1d09 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Thu, 12 Mar 2020 02:26:41 +0100
Subject: [PATCH 007/151] clean-up

---
 freqtrade/optimize/hyperopt.py | 32 +++-----------------------------
 1 file changed, 3 insertions(+), 29 deletions(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 48a89e1d6..a6da8c4b8 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -23,7 +23,7 @@ from joblib import (Parallel, cpu_count, delayed, dump, load,
 from pandas import DataFrame, json_normalize, isna
 from tqdm import tqdm
 import tabulate
-from os import path, popen
+from os import path
 import io
 
 from freqtrade.data.converter import trim_dataframe
@@ -640,10 +640,6 @@ class Hyperopt:
     def _set_random_state(self, random_state: Optional[int]) -> int:
         return random_state or random.randint(1, 2**16 - 1)
 
-    def _get_height(self) -> int:
-        rows = int((popen('stty size', 'r').read().split())[0])
-        return rows
-
     def start(self) -> None:
         self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
         logger.info(f"Using optimizer random state: {self.random_state}")
@@ -680,15 +676,6 @@ class Hyperopt:
 
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
-                """
-                self.progress_bar = progressbar.ProgressBar(
-                    min_value=0,
-                    max_value=self.total_epochs,
-                    initial_value=0,
-                    line_breaks=True,
-                    enable_colors=self.print_colorized
-                )
-                """
                 jobs = parallel._effective_n_jobs()
                 logger.info(f'Effective number of parallel workers used: {jobs}')
 
@@ -712,7 +699,6 @@ class Hyperopt:
                     self.fix_optimizer_models_list()
 
                     # Calculate progressbar outputs
-                    # pbar_line = ceil(self._get_height() / 2)
                     for j, val in enumerate(f_val):
                         # Use human-friendly indexes here (starting from 1)
                         current = i * jobs + j + 1
@@ -726,18 +712,12 @@ class Hyperopt:
                         # evaluations can take different time. Here they are aligned in the
                         # order they will be shown to the user.
                         val['is_best'] = is_best
-                        # print(current)
                         output = self.get_results(val)
                         if output:
                             self.progress_bar.write(output)
-                        # self.progress_bar.write(str(len(output.split('\n')[0])))
                         self.progress_bar.ncols = 108
                         self.progress_bar.update(1)
-                        """
-                        if pbar_line <= current:
-                            self.progress_bar.update(current)
-                            pbar_line = current + ceil(self._get_height() / 2)
-                        """
+
                         if is_best:
                             self.current_best_loss = val['loss']
                         self.trials.append(val)
@@ -747,8 +727,8 @@ class Hyperopt:
                 self.progress_bar.ncols = 108
                 self.progress_bar.close()
 
-                # self.progress_bar.update(current)
         except KeyboardInterrupt:
+            self.progress_bar.close()
             print('User interrupted..')
 
         self.save_trials(final=True)
@@ -761,9 +741,3 @@ class Hyperopt:
             # This is printed when Ctrl+C is pressed quickly, before first epochs have
             # a chance to be evaluated.
             print("No epochs evaluated yet, no best result.")
-
-    def __getstate__(self):
-        state = self.__dict__.copy()
-        del state['trials']
-        del state['progress_bar']
-        return state

From 1a59fc11be8312c19cd5723f641ac66128b05117 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Thu, 12 Mar 2020 02:36:18 +0100
Subject: [PATCH 008/151] doh

---
 freqtrade/optimize/hyperopt.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index a6da8c4b8..9ff4a1ead 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -741,3 +741,9 @@ class Hyperopt:
             # This is printed when Ctrl+C is pressed quickly, before first epochs have
             # a chance to be evaluated.
             print("No epochs evaluated yet, no best result.")
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        del state['trials']
+        del state['progress_bar']
+        return state

From 5737139979351eab8482cbe230b111310e11981b Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Thu, 12 Mar 2020 16:47:09 +0100
Subject: [PATCH 009/151] Small fix

---
 freqtrade/optimize/hyperopt.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 9ff4a1ead..a5921703f 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -7,6 +7,7 @@ This module contains the hyperopt logic
 import locale
 import logging
 import random
+import sys
 import warnings
 from math import ceil
 from collections import OrderedDict
@@ -270,6 +271,7 @@ class Hyperopt:
             # Print '\n' after each 100th epoch to separate dots from the log messages.
             # Otherwise output is messy on a terminal.
             print('.', end='' if results['current_epoch'] % 100 != 0 else None)  # type: ignore
+            sys.stdout.flush()
 
         if self.print_all or is_best:
             if not self.print_all:

From 27faf12fdeef84f404aa7bf32f27e7591cc1a910 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Sat, 14 Mar 2020 22:15:03 +0100
Subject: [PATCH 010/151] Fix if no file exists

---
 freqtrade/commands/arguments.py   |  2 +-
 freqtrade/commands/cli_options.py |  5 +++++
 freqtrade/data/btanalysis.py      |  8 ++++++--
 freqtrade/plot/plotting.py        | 19 +++++++++++++++----
 tests/data/test_btanalysis.py     |  5 ++++-
 5 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py
index 1a8cca72b..66fa0b038 100644
--- a/freqtrade/commands/arguments.py
+++ b/freqtrade/commands/arguments.py
@@ -59,7 +59,7 @@ ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchang
 
 ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
                        "db_url", "trade_source", "export", "exportfilename",
-                       "timerange", "ticker_interval"]
+                       "timerange", "ticker_interval", "skip_trades"]
 
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
                     "trade_source", "ticker_interval"]
diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py
index 42c697d56..187e0a424 100644
--- a/freqtrade/commands/cli_options.py
+++ b/freqtrade/commands/cli_options.py
@@ -413,6 +413,11 @@ AVAILABLE_CLI_OPTIONS = {
         metavar='INT',
         default=750,
     ),
+    "skip_trades": Arg(
+        '--skip-trades',
+        help='Skip using trades file from backtesting and DB.',
+        action='store_true',
+    ),
     "trade_source": Arg(
         '--trade-source',
         help='Specify the source for trades (Can be DB or file (backtest file)) '
diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py
index 681bf6734..fc938ad7e 100644
--- a/freqtrade/data/btanalysis.py
+++ b/freqtrade/data/btanalysis.py
@@ -111,7 +111,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
                             t.calc_profit(), t.calc_profit_ratio(),
                             t.open_rate, t.close_rate, t.amount,
                             (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)
-                                if t.close_date else None),
+                             if t.close_date else None),
                             t.sell_reason,
                             t.fee_open, t.fee_close,
                             t.open_rate_requested,
@@ -129,12 +129,16 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
     return trades
 
 
-def load_trades(source: str, db_url: str, exportfilename: str) -> pd.DataFrame:
+def load_trades(source: str, db_url: str, exportfilename: str, skip_trades: bool) -> pd.DataFrame:
     """
     Based on configuration option "trade_source":
     * loads data from DB (using `db_url`)
     * loads data from backtestfile (using `exportfilename`)
     """
+    if skip_trades:
+        df = pd.DataFrame(columns=BT_DATA_COLUMNS)
+        return df
+
     if source == "DB":
         return load_trades_from_db(db_url)
     elif source == "file":
diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py
index be7be2de0..cb2686878 100644
--- a/freqtrade/plot/plotting.py
+++ b/freqtrade/plot/plotting.py
@@ -3,6 +3,7 @@ from pathlib import Path
 from typing import Any, Dict, List
 
 import pandas as pd
+from os.path import isfile
 
 from freqtrade.configuration import TimeRange
 from freqtrade.data.btanalysis import (calculate_max_drawdown,
@@ -48,11 +49,21 @@ def init_plotscript(config):
         data_format=config.get('dataformat_ohlcv', 'json'),
     )
 
-    trades = load_trades(config['trade_source'],
-                         db_url=config.get('db_url'),
-                         exportfilename=config.get('exportfilename'),
-                         )
+    skip_trades = False
+    if not isfile(config.get('exportfilename')) and config['trade_source'] == 'file':
+        logger.info("Backtest file is missing skipping trades.")
+        skip_trades = True
+    elif config.get('skip_trades', False):
+        skip_trades = True
+
+    trades = load_trades(
+        config['trade_source'],
+        db_url=config.get('db_url'),
+        exportfilename=config.get('exportfilename'),
+        skip_trades=skip_trades
+    )
     trades = trim_dataframe(trades, timerange, 'open_time')
+
     return {"ohlcv": data,
             "trades": trades,
             "pairs": pairs,
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index da5d225b9..44e9c1072 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -104,6 +104,7 @@ def test_load_trades(default_conf, mocker):
     load_trades("DB",
                 db_url=default_conf.get('db_url'),
                 exportfilename=default_conf.get('exportfilename'),
+                skip_trades=False
                 )
 
     assert db_mock.call_count == 1
@@ -114,7 +115,9 @@ def test_load_trades(default_conf, mocker):
     default_conf['exportfilename'] = "testfile.json"
     load_trades("file",
                 db_url=default_conf.get('db_url'),
-                exportfilename=default_conf.get('exportfilename'),)
+                exportfilename=default_conf.get('exportfilename'),
+                skip_trades=False
+                )
 
     assert db_mock.call_count == 0
     assert bt_mock.call_count == 1

From cf7e80f45d5d664c1dcb1cc92335cffb67186d12 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Sat, 14 Mar 2020 23:55:13 +0100
Subject: [PATCH 011/151] Docs and logging

---
 docs/plotting.md           | 62 +++++++++++++++++++++++++-------------
 freqtrade/plot/plotting.py |  2 +-
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/docs/plotting.md b/docs/plotting.md
index 3eef8f8e7..ef28f0fe3 100644
--- a/docs/plotting.md
+++ b/docs/plotting.md
@@ -23,44 +23,64 @@ The `freqtrade plot-dataframe` subcommand shows an interactive graph with three
 Possible arguments:
 
 ```
-usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME]
-                                [--strategy-path PATH] [-p PAIRS [PAIRS ...]] [--indicators1 INDICATORS1 [INDICATORS1 ...]]
-                                [--indicators2 INDICATORS2 [INDICATORS2 ...]] [--plot-limit INT] [--db-url PATH]
-                                [--trade-source {DB,file}] [--export EXPORT] [--export-filename PATH] [--timerange TIMERANGE]
-                                [-i TICKER_INTERVAL]
+usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+                                [-d PATH] [--userdir PATH] [-s NAME]
+                                [--strategy-path PATH] [-p PAIRS [PAIRS ...]]
+                                [--indicators1 INDICATORS1 [INDICATORS1 ...]]
+                                [--indicators2 INDICATORS2 [INDICATORS2 ...]]
+                                [--plot-limit INT] [--db-url PATH]
+                                [--trade-source {DB,file}] [--export EXPORT]
+                                [--export-filename PATH]
+                                [--timerange TIMERANGE] [-i TICKER_INTERVAL]
+                                [--skip-trades]
 
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
-                        Show profits for only these pairs. Pairs are space-separated.
+                        Show profits for only these pairs. Pairs are space-
+                        separated.
   --indicators1 INDICATORS1 [INDICATORS1 ...]
-                        Set indicators from your strategy you want in the first row of the graph. Space-separated list. Example:
+                        Set indicators from your strategy you want in the
+                        first row of the graph. Space-separated list. Example:
                         `ema3 ema5`. Default: `['sma', 'ema3', 'ema5']`.
   --indicators2 INDICATORS2 [INDICATORS2 ...]
-                        Set indicators from your strategy you want in the third row of the graph. Space-separated list. Example:
+                        Set indicators from your strategy you want in the
+                        third row of the graph. Space-separated list. Example:
                         `fastd fastk`. Default: `['macd', 'macdsignal']`.
-  --plot-limit INT      Specify tick limit for plotting. Notice: too high values cause huge files. Default: 750.
-  --db-url PATH         Override trades database URL, this is useful in custom deployments (default: `sqlite:///tradesv3.sqlite`
-                        for Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for Dry Run).
+  --plot-limit INT      Specify tick limit for plotting. Notice: too high
+                        values cause huge files. Default: 750.
+  --db-url PATH         Override trades database URL, this is useful in custom
+                        deployments (default: `sqlite:///tradesv3.sqlite` for
+                        Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for
+                        Dry Run).
   --trade-source {DB,file}
-                        Specify the source for trades (Can be DB or file (backtest file)) Default: file
-  --export EXPORT       Export backtest results, argument are: trades. Example: `--export=trades`
+                        Specify the source for trades (Can be DB or file
+                        (backtest file)) Default: file
+  --export EXPORT       Export backtest results, argument are: trades.
+                        Example: `--export=trades`
   --export-filename PATH
-                        Save backtest results to the file with this filename. Requires `--export` to be set as well. Example:
-                        `--export-filename=user_data/backtest_results/backtest_today.json`
+                        Save backtest results to the file with this filename.
+                        Requires `--export` to be set as well. Example:
+                        `--export-filename=user_data/backtest_results/backtest
+                        _today.json`
   --timerange TIMERANGE
                         Specify what timerange of data to use.
   -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL
-                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`).
+                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
+                        `1d`).
+  --skip-trades         Skip using trades file from backtesting and DB.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
                         details.
   -V, --version         show program's version number and exit
   -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to
-                        `-` to read config from stdin.
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
   -d PATH, --datadir PATH
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
@@ -68,9 +88,9 @@ Common arguments:
 
 Strategy arguments:
   -s NAME, --strategy NAME
-                        Specify strategy class name which will be used by the bot.
+                        Specify strategy class name which will be used by the
+                        bot.
   --strategy-path PATH  Specify additional strategy lookup path.
-
 ```
 
 Example:
diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py
index cb2686878..8da61597f 100644
--- a/freqtrade/plot/plotting.py
+++ b/freqtrade/plot/plotting.py
@@ -51,7 +51,7 @@ def init_plotscript(config):
 
     skip_trades = False
     if not isfile(config.get('exportfilename')) and config['trade_source'] == 'file':
-        logger.info("Backtest file is missing skipping trades.")
+        logger.warning("Backtest file is missing skipping trades.")
         skip_trades = True
     elif config.get('skip_trades', False):
         skip_trades = True

From 2c0980aa3ab2af40e46411819f2e53b76eae6fa4 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Sun, 15 Mar 2020 00:09:08 +0100
Subject: [PATCH 012/151] Tests

---
 tests/data/test_btanalysis.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 44e9c1072..95f371d7f 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -122,6 +122,18 @@ def test_load_trades(default_conf, mocker):
     assert db_mock.call_count == 0
     assert bt_mock.call_count == 1
 
+    db_mock.reset_mock()
+    bt_mock.reset_mock()
+    default_conf['exportfilename'] = "testfile.json"
+    load_trades("file",
+                db_url=default_conf.get('db_url'),
+                exportfilename=default_conf.get('exportfilename'),
+                skip_trades=True
+                )
+
+    assert db_mock.call_count == 0
+    assert bt_mock.call_count == 0
+
 
 def test_combine_dataframes_with_mean(testdatadir):
     pairs = ["ETH/BTC", "ADA/BTC"]

From 328dbd3930981103d2646e27e53156f104e59745 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 15 Mar 2020 15:04:48 +0100
Subject: [PATCH 013/151] Remove unnecessary parameter to
 generate_text_table_sell_reason

---
 freqtrade/optimize/backtesting.py       | 7 +++----
 freqtrade/optimize/optimize_reports.py  | 8 ++++----
 tests/optimize/test_optimize_reports.py | 6 ++----
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 40e6590f7..5ee5a058f 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -7,7 +7,7 @@ import logging
 from copy import deepcopy
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Any, Dict, List, NamedTuple, Optional
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple
 
 import arrow
 from pandas import DataFrame
@@ -108,7 +108,7 @@ class Backtesting:
         # And the regular "stoploss" function would not apply to that case
         self.strategy.order_types['stoploss_on_exchange'] = False
 
-    def load_bt_data(self):
+    def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
         timerange = TimeRange.parse_timerange(None if self.config.get(
             'timerange') is None else str(self.config.get('timerange')))
 
@@ -432,8 +432,7 @@ class Backtesting:
                 print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
             print(table)
 
-            table = generate_text_table_sell_reason(data,
-                                                    stake_currency=self.config['stake_currency'],
+            table = generate_text_table_sell_reason(stake_currency=self.config['stake_currency'],
                                                     max_open_trades=self.config['max_open_trades'],
                                                     results=results)
             if isinstance(table, str):
diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py
index 39bde50a8..ee29aa283 100644
--- a/freqtrade/optimize/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports.py
@@ -69,12 +69,12 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")  # type: ignore
 
 
-def generate_text_table_sell_reason(
-    data: Dict[str, Dict], stake_currency: str, max_open_trades: int, results: DataFrame
-) -> str:
+def generate_text_table_sell_reason(stake_currency: str, max_open_trades: int,
+                                    results: DataFrame) -> str:
     """
     Generate small table outlining Backtest results
-    :param data: Dict of <pair: dataframe> containing data that was used during backtesting.
+    :param stake_currency: Stakecurrency used
+    :param max_open_trades: Max_open_trades parameter
     :param results: Dataframe containing the backtest results
     :return: pretty printed table with tabulate as string
     """
diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py
index 285ecaa02..5e68a5ef8 100644
--- a/tests/optimize/test_optimize_reports.py
+++ b/tests/optimize/test_optimize_reports.py
@@ -61,10 +61,8 @@ def test_generate_text_table_sell_reason(default_conf, mocker):
         '|     stop_loss |       1 |      0 |       0 |        1 |'
         '            -10 |            -10 |             -0.2 |             -5 |'
     )
-    assert generate_text_table_sell_reason(
-        data={'ETH/BTC': {}},
-        stake_currency='BTC', max_open_trades=2,
-        results=results) == result_str
+    assert generate_text_table_sell_reason(stake_currency='BTC', max_open_trades=2,
+                                           results=results) == result_str
 
 
 def test_generate_text_table_strategy(default_conf, mocker):

From 6106d59e1a0122863849e03e671db698f02c5c96 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 15 Mar 2020 15:17:35 +0100
Subject: [PATCH 014/151] Move store_backtest_results to optimize_reports

---
 freqtrade/optimize/backtesting.py      | 18 ------------------
 freqtrade/optimize/optimize_reports.py | 26 +++++++++++++++++++++++++-
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 5ee5a058f..9dd1b8429 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -6,7 +6,6 @@ This module contains the backtesting logic
 import logging
 from copy import deepcopy
 from datetime import datetime, timedelta
-from pathlib import Path
 from typing import Any, Dict, List, NamedTuple, Optional, Tuple
 
 import arrow
@@ -134,23 +133,6 @@ class Backtesting:
 
         return data, timerange
 
-    def _store_backtest_result(self, recordfilename: Path, results: DataFrame,
-                               strategyname: Optional[str] = None) -> None:
-
-        records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
-                    t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
-                    t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value)
-                   for index, t in results.iterrows()]
-
-        if records:
-            if strategyname:
-                # Inject strategyname to filename
-                recordfilename = Path.joinpath(
-                    recordfilename.parent,
-                    f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix)
-            logger.info(f'Dumping backtest results to {recordfilename}')
-            file_dump_json(recordfilename, records)
-
     def _get_ohlcv_as_lists(self, processed: Dict) -> Dict[str, DataFrame]:
         """
         Helper function to convert a processed dataframes into lists for performance reasons.
diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py
index ee29aa283..fb407f681 100644
--- a/freqtrade/optimize/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports.py
@@ -1,9 +1,33 @@
+import logging
 from datetime import timedelta
-from typing import Dict
+from pathlib import Path
+from typing import Dict, Optional
 
 from pandas import DataFrame
 from tabulate import tabulate
 
+from freqtrade.misc import file_dump_json
+
+logger = logging.getLogger(__name__)
+
+
+def store_backtest_result(recordfilename: Path, results: DataFrame,
+                          strategyname: Optional[str] = None) -> None:
+
+    records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
+                t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
+                t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value)
+               for index, t in results.iterrows()]
+
+    if records:
+        if strategyname:
+            # Inject strategyname to filename
+            recordfilename = Path.joinpath(
+                recordfilename.parent,
+                f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix)
+        logger.info(f'Dumping backtest results to {recordfilename}')
+        file_dump_json(recordfilename, records)
+
 
 def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_trades: int,
                         results: DataFrame, skip_nan: bool = False) -> str:

From a13d581658173fb764c110ae9966767f17c85e7d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 15 Mar 2020 15:17:53 +0100
Subject: [PATCH 015/151] Move backtest-result visualization out of backtesting
 class

---
 freqtrade/optimize/backtesting.py      | 48 ++------------------------
 freqtrade/optimize/optimize_reports.py | 44 +++++++++++++++++++++++
 2 files changed, 47 insertions(+), 45 deletions(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 9dd1b8429..323331bc6 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -18,10 +18,7 @@ from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
-from freqtrade.misc import file_dump_json
-from freqtrade.optimize.optimize_reports import (
-    generate_text_table, generate_text_table_sell_reason,
-    generate_text_table_strategy)
+from freqtrade.optimize.optimize_reports import show_backtest_results
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.state import RunMode
@@ -399,44 +396,5 @@ class Backtesting:
                 max_open_trades=max_open_trades,
                 position_stacking=position_stacking,
             )
-
-        for strategy, results in all_results.items():
-
-            if self.config.get('export', False):
-                self._store_backtest_result(self.config['exportfilename'], results,
-                                            strategy if len(self.strategylist) > 1 else None)
-
-            print(f"Result for strategy {strategy}")
-            table = generate_text_table(data, stake_currency=self.config['stake_currency'],
-                                        max_open_trades=self.config['max_open_trades'],
-                                        results=results)
-            if isinstance(table, str):
-                print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
-            print(table)
-
-            table = generate_text_table_sell_reason(stake_currency=self.config['stake_currency'],
-                                                    max_open_trades=self.config['max_open_trades'],
-                                                    results=results)
-            if isinstance(table, str):
-                print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
-            print(table)
-
-            table = generate_text_table(data,
-                                        stake_currency=self.config['stake_currency'],
-                                        max_open_trades=self.config['max_open_trades'],
-                                        results=results.loc[results.open_at_end], skip_nan=True)
-            if isinstance(table, str):
-                print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
-            print(table)
-            if isinstance(table, str):
-                print('=' * len(table.splitlines()[0]))
-            print()
-        if len(all_results) > 1:
-            # Print Strategy summary table
-            table = generate_text_table_strategy(self.config['stake_currency'],
-                                                 self.config['max_open_trades'],
-                                                 all_results=all_results)
-            print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
-            print(table)
-            print('=' * len(table.splitlines()[0]))
-            print('\nFor more details, please look at the detail tables above')
+        # Show backtest results
+        show_backtest_results(self.config, data, all_results)
diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py
index fb407f681..fef0accb0 100644
--- a/freqtrade/optimize/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports.py
@@ -197,3 +197,47 @@ def generate_edge_table(results: dict) -> str:
     # Ignore type as floatfmt does allow tuples but mypy does not know that
     return tabulate(tabular_data, headers=headers,
                     floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")  # type: ignore
+
+
+def show_backtest_results(config: Dict, btdata: Dict[str, DataFrame],
+                          all_results: Dict[str, DataFrame]):
+    for strategy, results in all_results.items():
+
+        if config.get('export', False):
+            store_backtest_result(config['exportfilename'], results,
+                                  strategy if len(all_results) > 1 else None)
+
+        print(f"Result for strategy {strategy}")
+        table = generate_text_table(btdata, stake_currency=config['stake_currency'],
+                                    max_open_trades=config['max_open_trades'],
+                                    results=results)
+        if isinstance(table, str):
+            print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
+        print(table)
+
+        table = generate_text_table_sell_reason(stake_currency=config['stake_currency'],
+                                                max_open_trades=config['max_open_trades'],
+                                                results=results)
+        if isinstance(table, str):
+            print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
+        print(table)
+
+        table = generate_text_table(btdata,
+                                    stake_currency=config['stake_currency'],
+                                    max_open_trades=config['max_open_trades'],
+                                    results=results.loc[results.open_at_end], skip_nan=True)
+        if isinstance(table, str):
+            print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
+        print(table)
+        if isinstance(table, str):
+            print('=' * len(table.splitlines()[0]))
+        print()
+    if len(all_results) > 1:
+        # Print Strategy summary table
+        table = generate_text_table_strategy(config['stake_currency'],
+                                                config['max_open_trades'],
+                                                all_results=all_results)
+        print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
+        print(table)
+        print('=' * len(table.splitlines()[0]))
+        print('\nFor more details, please look at the detail tables above')

From e95665cecaeec2337dee47660674401535b406a7 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 15 Mar 2020 15:36:23 +0100
Subject: [PATCH 016/151] Make backtestresult storing independent from printing

---
 freqtrade/optimize/backtesting.py      |  6 ++++-
 freqtrade/optimize/optimize_reports.py | 36 ++++++++++++--------------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 323331bc6..1725a7d13 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -18,7 +18,8 @@ from freqtrade.data.converter import trim_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
-from freqtrade.optimize.optimize_reports import show_backtest_results
+from freqtrade.optimize.optimize_reports import (show_backtest_results,
+                                                 store_backtest_result)
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.state import RunMode
@@ -396,5 +397,8 @@ class Backtesting:
                 max_open_trades=max_open_trades,
                 position_stacking=position_stacking,
             )
+
+        if self.config.get('export', False):
+            store_backtest_result(self.config['exportfilename'], all_results)
         # Show backtest results
         show_backtest_results(self.config, data, all_results)
diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py
index fef0accb0..701432071 100644
--- a/freqtrade/optimize/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports.py
@@ -11,22 +11,22 @@ from freqtrade.misc import file_dump_json
 logger = logging.getLogger(__name__)
 
 
-def store_backtest_result(recordfilename: Path, results: DataFrame,
-                          strategyname: Optional[str] = None) -> None:
+def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame]) -> None:
 
-    records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
-                t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
-                t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value)
-               for index, t in results.iterrows()]
+    for strategy, results in all_results.items():
+        records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
+                    t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
+                    t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value)
+                   for index, t in results.iterrows()]
 
-    if records:
-        if strategyname:
-            # Inject strategyname to filename
-            recordfilename = Path.joinpath(
-                recordfilename.parent,
-                f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix)
-        logger.info(f'Dumping backtest results to {recordfilename}')
-        file_dump_json(recordfilename, records)
+        if records:
+            if len(all_results) > 1:
+                # Inject strategy to filename
+                recordfilename = Path.joinpath(
+                    recordfilename.parent,
+                    f'{recordfilename.stem}-{strategy}').with_suffix(recordfilename.suffix)
+            logger.info(f'Dumping backtest results to {recordfilename}')
+            file_dump_json(recordfilename, records)
 
 
 def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_trades: int,
@@ -203,10 +203,6 @@ def show_backtest_results(config: Dict, btdata: Dict[str, DataFrame],
                           all_results: Dict[str, DataFrame]):
     for strategy, results in all_results.items():
 
-        if config.get('export', False):
-            store_backtest_result(config['exportfilename'], results,
-                                  strategy if len(all_results) > 1 else None)
-
         print(f"Result for strategy {strategy}")
         table = generate_text_table(btdata, stake_currency=config['stake_currency'],
                                     max_open_trades=config['max_open_trades'],
@@ -235,8 +231,8 @@ def show_backtest_results(config: Dict, btdata: Dict[str, DataFrame],
     if len(all_results) > 1:
         # Print Strategy summary table
         table = generate_text_table_strategy(config['stake_currency'],
-                                                config['max_open_trades'],
-                                                all_results=all_results)
+                                             config['max_open_trades'],
+                                             all_results=all_results)
         print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
         print(table)
         print('=' * len(table.splitlines()[0]))

From fe50a0f3a13e0d0f92084314af081ce1230acaac Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 15 Mar 2020 15:36:53 +0100
Subject: [PATCH 017/151] Move test for store_bt_results to optimize_reports

---
 tests/optimize/test_backtesting.py      | 89 +++----------------------
 tests/optimize/test_optimize_reports.py | 76 ++++++++++++++++++++-
 2 files changed, 83 insertions(+), 82 deletions(-)

diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py
index da23a9af4..1c4d3b16a 100644
--- a/tests/optimize/test_backtesting.py
+++ b/tests/optimize/test_backtesting.py
@@ -331,8 +331,8 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
 
     mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
     patch_exchange(mocker)
-    mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
-    mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock(return_value=1))
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest')
+    mocker.patch('freqtrade.optimize.backtesting.show_backtest_results')
 
     default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
     default_conf['ticker_interval'] = '1m'
@@ -361,8 +361,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
                  MagicMock(return_value=pd.DataFrame()))
     mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
     patch_exchange(mocker)
-    mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
-    mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock(return_value=1))
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest')
+    mocker.patch('freqtrade.optimize.backtesting.show_backtest_results')
 
     default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
     default_conf['ticker_interval'] = "1m"
@@ -507,7 +507,6 @@ def test_backtest_only_sell(mocker, default_conf, testdatadir):
 
 def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir):
     mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
-    mocker.patch('freqtrade.optimize.backtesting.file_dump_json', MagicMock())
     backtest_conf = _make_backtest_conf(mocker, conf=default_conf,
                                         pair='UNITTEST/BTC', datadir=testdatadir)
     default_conf['ticker_interval'] = '1m'
@@ -515,7 +514,6 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir):
     backtesting.strategy.advise_buy = _trend_alternate  # Override
     backtesting.strategy.advise_sell = _trend_alternate  # Override
     results = backtesting.backtest(**backtest_conf)
-    backtesting._store_backtest_result("test_.json", results)
     # 200 candles in backtest data
     # won't buy on first (shifted by 1)
     # 100 buys signals
@@ -586,84 +584,12 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
     assert len(evaluate_result_multi(results, '5m', 1)) == 0
 
 
-def test_backtest_record(default_conf, fee, mocker):
-    names = []
-    records = []
-    patch_exchange(mocker)
-    mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
-    mocker.patch(
-        'freqtrade.optimize.backtesting.file_dump_json',
-        new=lambda n, r: (names.append(n), records.append(r))
-    )
-
-    backtesting = Backtesting(default_conf)
-    results = pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC",
-                                     "UNITTEST/BTC", "UNITTEST/BTC"],
-                            "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780],
-                            "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
-                            "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime,
-                                          Arrow(2017, 11, 14, 21, 36, 00).datetime,
-                                          Arrow(2017, 11, 14, 22, 12, 00).datetime,
-                                          Arrow(2017, 11, 14, 22, 44, 00).datetime],
-                            "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime,
-                                           Arrow(2017, 11, 14, 22, 10, 00).datetime,
-                                           Arrow(2017, 11, 14, 22, 43, 00).datetime,
-                                           Arrow(2017, 11, 14, 22, 58, 00).datetime],
-                            "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
-                            "close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
-                            "open_index": [1, 119, 153, 185],
-                            "close_index": [118, 151, 184, 199],
-                            "trade_duration": [123, 34, 31, 14],
-                            "open_at_end": [False, False, False, True],
-                            "sell_reason": [SellType.ROI, SellType.STOP_LOSS,
-                                            SellType.ROI, SellType.FORCE_SELL]
-                            })
-    backtesting._store_backtest_result("backtest-result.json", results)
-    assert len(results) == 4
-    # Assert file_dump_json was only called once
-    assert names == ['backtest-result.json']
-    records = records[0]
-    # Ensure records are of correct type
-    assert len(records) == 4
-
-    # reset test to test with strategy name
-    names = []
-    records = []
-    backtesting._store_backtest_result(Path("backtest-result.json"), results, "DefStrat")
-    assert len(results) == 4
-    # Assert file_dump_json was only called once
-    assert names == [Path('backtest-result-DefStrat.json')]
-    records = records[0]
-    # Ensure records are of correct type
-    assert len(records) == 4
-
-    # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117)
-    # Below follows just a typecheck of the schema/type of trade-records
-    oix = None
-    for (pair, profit, date_buy, date_sell, buy_index, dur,
-         openr, closer, open_at_end, sell_reason) in records:
-        assert pair == 'UNITTEST/BTC'
-        assert isinstance(profit, float)
-        # FIX: buy/sell should be converted to ints
-        assert isinstance(date_buy, float)
-        assert isinstance(date_sell, float)
-        assert isinstance(openr, float)
-        assert isinstance(closer, float)
-        assert isinstance(open_at_end, bool)
-        assert isinstance(sell_reason, str)
-        isinstance(buy_index, pd._libs.tslib.Timestamp)
-        if oix:
-            assert buy_index > oix
-        oix = buy_index
-        assert dur > 0
-
-
 def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
     default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
 
     patch_exchange(mocker)
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
-    mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.show_backtest_results', MagicMock())
 
     patched_configuration_load_config_file(mocker, default_conf)
 
@@ -705,9 +631,10 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
     backtestmock = MagicMock()
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
     gen_table_mock = MagicMock()
-    mocker.patch('freqtrade.optimize.backtesting.generate_text_table', gen_table_mock)
+    mocker.patch('freqtrade.optimize.optimize_reports.generate_text_table', gen_table_mock)
     gen_strattable_mock = MagicMock()
-    mocker.patch('freqtrade.optimize.backtesting.generate_text_table_strategy', gen_strattable_mock)
+    mocker.patch('freqtrade.optimize.optimize_reports.generate_text_table_strategy',
+                 gen_strattable_mock)
     patched_configuration_load_config_file(mocker, default_conf)
 
     args = [
diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py
index 5e68a5ef8..36c9a93b3 100644
--- a/tests/optimize/test_optimize_reports.py
+++ b/tests/optimize/test_optimize_reports.py
@@ -1,10 +1,14 @@
+from pathlib import Path
+
 import pandas as pd
+from arrow import Arrow
 
 from freqtrade.edge import PairInfo
 from freqtrade.optimize.optimize_reports import (
     generate_edge_table, generate_text_table, generate_text_table_sell_reason,
-    generate_text_table_strategy)
+    generate_text_table_strategy, store_backtest_result)
 from freqtrade.strategy.interface import SellType
+from tests.conftest import patch_exchange
 
 
 def test_generate_text_table(default_conf, mocker):
@@ -113,3 +117,73 @@ def test_generate_edge_table(edge_conf, mocker):
     assert generate_edge_table(results).count('| ETH/BTC |') == 1
     assert generate_edge_table(results).count(
         '|   Risk Reward Ratio |   Required Risk Reward |   Expectancy |') == 1
+
+
+def test_backtest_record(default_conf, fee, mocker):
+    names = []
+    records = []
+    patch_exchange(mocker)
+    mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
+    mocker.patch(
+        'freqtrade.optimize.optimize_reports.file_dump_json',
+        new=lambda n, r: (names.append(n), records.append(r))
+    )
+
+    results = {'DefStrat': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC",
+                                               "UNITTEST/BTC", "UNITTEST/BTC"],
+                                      "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780],
+                                      "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
+                                      "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime,
+                                                    Arrow(2017, 11, 14, 21, 36, 00).datetime,
+                                                    Arrow(2017, 11, 14, 22, 12, 00).datetime,
+                                                    Arrow(2017, 11, 14, 22, 44, 00).datetime],
+                                      "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime,
+                                                     Arrow(2017, 11, 14, 22, 10, 00).datetime,
+                                                     Arrow(2017, 11, 14, 22, 43, 00).datetime,
+                                                     Arrow(2017, 11, 14, 22, 58, 00).datetime],
+                                      "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
+                                      "close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
+                                      "open_index": [1, 119, 153, 185],
+                                      "close_index": [118, 151, 184, 199],
+                                      "trade_duration": [123, 34, 31, 14],
+                                      "open_at_end": [False, False, False, True],
+                                      "sell_reason": [SellType.ROI, SellType.STOP_LOSS,
+                                                      SellType.ROI, SellType.FORCE_SELL]
+                                      })}
+    store_backtest_result(Path("backtest-result.json"), results)
+    # Assert file_dump_json was only called once
+    assert names == [Path('backtest-result.json')]
+    records = records[0]
+    # Ensure records are of correct type
+    assert len(records) == 4
+
+    # reset test to test with strategy name
+    names = []
+    records = []
+    results['Strat'] = pd.DataFrame()
+    store_backtest_result(Path("backtest-result.json"), results)
+    # Assert file_dump_json was only called once
+    assert names == [Path('backtest-result-DefStrat.json')]
+    records = records[0]
+    # Ensure records are of correct type
+    assert len(records) == 4
+
+    # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117)
+    # Below follows just a typecheck of the schema/type of trade-records
+    oix = None
+    for (pair, profit, date_buy, date_sell, buy_index, dur,
+         openr, closer, open_at_end, sell_reason) in records:
+        assert pair == 'UNITTEST/BTC'
+        assert isinstance(profit, float)
+        # FIX: buy/sell should be converted to ints
+        assert isinstance(date_buy, float)
+        assert isinstance(date_sell, float)
+        assert isinstance(openr, float)
+        assert isinstance(closer, float)
+        assert isinstance(open_at_end, bool)
+        assert isinstance(sell_reason, str)
+        isinstance(buy_index, pd._libs.tslib.Timestamp)
+        if oix:
+            assert buy_index > oix
+        oix = buy_index
+        assert dur > 0

From e1b08ad76caf8482ee712d782bad482c643b0307 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 15 Mar 2020 15:38:26 +0100
Subject: [PATCH 018/151] Add docstring to store_backtest_result

---
 freqtrade/optimize/optimize_reports.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py
index 701432071..abfbaf1d8 100644
--- a/freqtrade/optimize/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports.py
@@ -12,7 +12,11 @@ logger = logging.getLogger(__name__)
 
 
 def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame]) -> None:
-
+    """
+    Stores backtest results to file (one file per strategy)
+    :param recordfilename: Destination filename
+    :param all_results: Dict of Dataframes, one results dataframe per strategy
+    """
     for strategy, results in all_results.items():
         records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
                     t.close_time.timestamp(), t.open_index - 1, t.trade_duration,

From 3d4664c2a6da70027189d73a04b75bdd1821f79e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 15 Mar 2020 15:40:12 +0100
Subject: [PATCH 019/151] Remove unnecessary import

---
 freqtrade/optimize/optimize_reports.py  |  2 +-
 tests/optimize/test_optimize_reports.py | 40 ++++++++++++-------------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py
index abfbaf1d8..251da9159 100644
--- a/freqtrade/optimize/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports.py
@@ -1,7 +1,7 @@
 import logging
 from datetime import timedelta
 from pathlib import Path
-from typing import Dict, Optional
+from typing import Dict
 
 from pandas import DataFrame
 from tabulate import tabulate
diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py
index 36c9a93b3..f19668459 100644
--- a/tests/optimize/test_optimize_reports.py
+++ b/tests/optimize/test_optimize_reports.py
@@ -130,26 +130,26 @@ def test_backtest_record(default_conf, fee, mocker):
     )
 
     results = {'DefStrat': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC",
-                                               "UNITTEST/BTC", "UNITTEST/BTC"],
-                                      "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780],
-                                      "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
-                                      "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime,
-                                                    Arrow(2017, 11, 14, 21, 36, 00).datetime,
-                                                    Arrow(2017, 11, 14, 22, 12, 00).datetime,
-                                                    Arrow(2017, 11, 14, 22, 44, 00).datetime],
-                                      "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime,
-                                                     Arrow(2017, 11, 14, 22, 10, 00).datetime,
-                                                     Arrow(2017, 11, 14, 22, 43, 00).datetime,
-                                                     Arrow(2017, 11, 14, 22, 58, 00).datetime],
-                                      "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
-                                      "close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
-                                      "open_index": [1, 119, 153, 185],
-                                      "close_index": [118, 151, 184, 199],
-                                      "trade_duration": [123, 34, 31, 14],
-                                      "open_at_end": [False, False, False, True],
-                                      "sell_reason": [SellType.ROI, SellType.STOP_LOSS,
-                                                      SellType.ROI, SellType.FORCE_SELL]
-                                      })}
+                                                  "UNITTEST/BTC", "UNITTEST/BTC"],
+                                         "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780],
+                                         "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
+                                         "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime,
+                                                       Arrow(2017, 11, 14, 21, 36, 00).datetime,
+                                                       Arrow(2017, 11, 14, 22, 12, 00).datetime,
+                                                       Arrow(2017, 11, 14, 22, 44, 00).datetime],
+                                         "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime,
+                                                        Arrow(2017, 11, 14, 22, 10, 00).datetime,
+                                                        Arrow(2017, 11, 14, 22, 43, 00).datetime,
+                                                        Arrow(2017, 11, 14, 22, 58, 00).datetime],
+                                         "open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
+                                         "close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
+                                         "open_index": [1, 119, 153, 185],
+                                         "close_index": [118, 151, 184, 199],
+                                         "trade_duration": [123, 34, 31, 14],
+                                         "open_at_end": [False, False, False, True],
+                                         "sell_reason": [SellType.ROI, SellType.STOP_LOSS,
+                                                         SellType.ROI, SellType.FORCE_SELL]
+                                         })}
     store_backtest_result(Path("backtest-result.json"), results)
     # Assert file_dump_json was only called once
     assert names == [Path('backtest-result.json')]

From 8c33e07dc660853eee0eee0d10109063f52d11cf Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Sun, 15 Mar 2020 21:20:32 +0100
Subject: [PATCH 020/151] Update based on comments

---
 docs/plotting.md                  |  2 +-
 freqtrade/commands/arguments.py   |  2 +-
 freqtrade/commands/cli_options.py |  6 +++---
 freqtrade/data/btanalysis.py      |  6 +++---
 freqtrade/plot/plotting.py        | 13 ++++++-------
 tests/data/test_btanalysis.py     |  6 +++---
 6 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/docs/plotting.md b/docs/plotting.md
index ef28f0fe3..56906ebec 100644
--- a/docs/plotting.md
+++ b/docs/plotting.md
@@ -68,7 +68,7 @@ optional arguments:
   -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL
                         Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
                         `1d`).
-  --skip-trades         Skip using trades file from backtesting and DB.
+  --no-trades           Skip using trades from backtesting file and DB.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py
index 66fa0b038..8c64c5857 100644
--- a/freqtrade/commands/arguments.py
+++ b/freqtrade/commands/arguments.py
@@ -59,7 +59,7 @@ ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchang
 
 ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
                        "db_url", "trade_source", "export", "exportfilename",
-                       "timerange", "ticker_interval", "skip_trades"]
+                       "timerange", "ticker_interval", "no_trades"]
 
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
                     "trade_source", "ticker_interval"]
diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py
index 187e0a424..5cf1b7fce 100644
--- a/freqtrade/commands/cli_options.py
+++ b/freqtrade/commands/cli_options.py
@@ -413,9 +413,9 @@ AVAILABLE_CLI_OPTIONS = {
         metavar='INT',
         default=750,
     ),
-    "skip_trades": Arg(
-        '--skip-trades',
-        help='Skip using trades file from backtesting and DB.',
+    "no_trades": Arg(
+        '--no-trades',
+        help='Skip using trades from backtesting file and DB.',
         action='store_true',
     ),
     "trade_source": Arg(
diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py
index 8670f30c6..17b243f56 100644
--- a/freqtrade/data/btanalysis.py
+++ b/freqtrade/data/btanalysis.py
@@ -129,7 +129,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
     return trades
 
 
-def load_trades(source: str, db_url: str, exportfilename: Path, skip_trades: bool) -> pd.DataFrame:
+def load_trades(source: str, db_url: str, exportfilename: Path, no_trades: bool) -> pd.DataFrame:
     """
     Based on configuration option "trade_source":
     * loads data from DB (using `db_url`)
@@ -137,10 +137,10 @@ def load_trades(source: str, db_url: str, exportfilename: Path, skip_trades: boo
     :param source: "DB" or "file" - specify source to load from
     :param db_url: sqlalchemy formatted url to a database
     :param exportfilename: Json file generated by backtesting
-    :param skip_trades: Skip using trades, only return backtesting data columns
+    :param no_trades: Skip using trades, only return backtesting data columns
     :return: DataFrame containing trades
     """
-    if skip_trades:
+    if no_trades:
         df = pd.DataFrame(columns=BT_DATA_COLUMNS)
         return df
 
diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py
index 8da61597f..fc8f25612 100644
--- a/freqtrade/plot/plotting.py
+++ b/freqtrade/plot/plotting.py
@@ -3,7 +3,6 @@ from pathlib import Path
 from typing import Any, Dict, List
 
 import pandas as pd
-from os.path import isfile
 
 from freqtrade.configuration import TimeRange
 from freqtrade.data.btanalysis import (calculate_max_drawdown,
@@ -49,18 +48,18 @@ def init_plotscript(config):
         data_format=config.get('dataformat_ohlcv', 'json'),
     )
 
-    skip_trades = False
-    if not isfile(config.get('exportfilename')) and config['trade_source'] == 'file':
+    no_trades = False
+    if config.get('no_trades', False):
+        no_trades = True
+    elif not config['exportfilename'].is_file() and config['trade_source'] == 'file':
         logger.warning("Backtest file is missing skipping trades.")
-        skip_trades = True
-    elif config.get('skip_trades', False):
-        skip_trades = True
+        no_trades = True
 
     trades = load_trades(
         config['trade_source'],
         db_url=config.get('db_url'),
         exportfilename=config.get('exportfilename'),
-        skip_trades=skip_trades
+        no_trades=no_trades
     )
     trades = trim_dataframe(trades, timerange, 'open_time')
 
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 8561f2b84..4b47faeee 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -105,7 +105,7 @@ def test_load_trades(default_conf, mocker):
     load_trades("DB",
                 db_url=default_conf.get('db_url'),
                 exportfilename=default_conf.get('exportfilename'),
-                skip_trades=False
+                no_trades=False
                 )
 
     assert db_mock.call_count == 1
@@ -117,7 +117,7 @@ def test_load_trades(default_conf, mocker):
     load_trades("file",
                 db_url=default_conf.get('db_url'),
                 exportfilename=default_conf.get('exportfilename'),
-                skip_trades=False
+                no_trades=False
                 )
 
     assert db_mock.call_count == 0
@@ -129,7 +129,7 @@ def test_load_trades(default_conf, mocker):
     load_trades("file",
                 db_url=default_conf.get('db_url'),
                 exportfilename=default_conf.get('exportfilename'),
-                skip_trades=True
+                no_trades=True
                 )
 
     assert db_mock.call_count == 0

From 06198c00283eeaef44f0dda391a3cc769acce6ae Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Sun, 15 Mar 2020 21:27:45 +0100
Subject: [PATCH 021/151] Missed configuration.py

---
 freqtrade/configuration/configuration.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py
index ce2101441..e5515670d 100644
--- a/freqtrade/configuration/configuration.py
+++ b/freqtrade/configuration/configuration.py
@@ -359,6 +359,9 @@ class Configuration:
         self._args_to_config(config, argname='erase',
                              logstring='Erase detected. Deleting existing data.')
 
+        self._args_to_config(config, argname='no_trades',
+                             logstring='Parameter --no-trades detected.')
+
         self._args_to_config(config, argname='timeframes',
                              logstring='timeframes --timeframes: {}')
 

From e8a92cb3136971bbcd96257e4dc036e7b701da26 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2020 08:17:21 +0000
Subject: [PATCH 022/151] Bump pandas from 1.0.1 to 1.0.2

Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.0.1 to 1.0.2.
- [Release notes](https://github.com/pandas-dev/pandas/releases)
- [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md)
- [Commits](https://github.com/pandas-dev/pandas/compare/v1.0.1...v1.0.2)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 68024f587..12d7336aa 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@
 -r requirements-common.txt
 
 numpy==1.18.1
-pandas==1.0.1
+pandas==1.0.2

From 0b341757525c23f1242e7e9b189d97b76f5b54b2 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2020 08:17:53 +0000
Subject: [PATCH 023/151] Bump ccxt from 1.23.81 to 1.24.31

Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.23.81 to 1.24.31.
- [Release notes](https://github.com/ccxt/ccxt/releases)
- [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst)
- [Commits](https://github.com/ccxt/ccxt/compare/1.23.81...1.24.31)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 784eef93c..f3f7d8a07 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.23.81
+ccxt==1.24.31
 SQLAlchemy==1.3.13
 python-telegram-bot==12.4.2
 arrow==0.15.5

From 8cfff40fcfb2ba7145c12b165f75e34debcedae6 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2020 08:18:07 +0000
Subject: [PATCH 024/151] Bump plotly from 4.5.3 to 4.5.4

Bumps [plotly](https://github.com/plotly/plotly.py) from 4.5.3 to 4.5.4.
- [Release notes](https://github.com/plotly/plotly.py/releases)
- [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md)
- [Commits](https://github.com/plotly/plotly.py/compare/v4.5.3...v4.5.4)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-plot.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-plot.txt b/requirements-plot.txt
index 381334a66..7a5b21e2d 100644
--- a/requirements-plot.txt
+++ b/requirements-plot.txt
@@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==4.5.3
+plotly==4.5.4
 

From 3eee0c43a727fe91bad3481e1dab0551793afa4e Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2020 08:18:33 +0000
Subject: [PATCH 025/151] Bump pytest from 5.3.5 to 5.4.1

Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.3.5 to 5.4.1.
- [Release notes](https://github.com/pytest-dev/pytest/releases)
- [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest/compare/5.3.5...5.4.1)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-dev.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 1e58ae6e0..f2801c15b 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -8,7 +8,7 @@ flake8==3.7.9
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==4.0.0
 mypy==0.761
-pytest==5.3.5
+pytest==5.4.1
 pytest-asyncio==0.10.0
 pytest-cov==2.8.1
 pytest-mock==2.0.0

From 7a7530d57d4ef7d39ff715ccee7cf9ddf2a820a0 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2020 08:53:20 +0000
Subject: [PATCH 026/151] Bump sqlalchemy from 1.3.13 to 1.3.15

Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.3.13 to 1.3.15.
- [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases)
- [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES)
- [Commits](https://github.com/sqlalchemy/sqlalchemy/commits)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index f3f7d8a07..7c2ad4283 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,7 +1,7 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
 ccxt==1.24.31
-SQLAlchemy==1.3.13
+SQLAlchemy==1.3.15
 python-telegram-bot==12.4.2
 arrow==0.15.5
 cachetools==4.0.0

From 62d449251cc8a02182a46015bc2a36e71d549c85 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2020 08:54:12 +0000
Subject: [PATCH 027/151] Bump mypy from 0.761 to 0.770

Bumps [mypy](https://github.com/python/mypy) from 0.761 to 0.770.
- [Release notes](https://github.com/python/mypy/releases)
- [Commits](https://github.com/python/mypy/compare/v0.761...v0.770)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-dev.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index f2801c15b..a4d83eb4f 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -7,7 +7,7 @@ coveralls==1.11.1
 flake8==3.7.9
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==4.0.0
-mypy==0.761
+mypy==0.770
 pytest==5.4.1
 pytest-asyncio==0.10.0
 pytest-cov==2.8.1

From 4f46fb9bf5cb0ae22e3b365e552f62da712bd99f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 17 Mar 2020 19:33:18 +0100
Subject: [PATCH 028/151] Add template and jupyter files to release

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 7529152a0..c67f5258f 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,3 +2,4 @@ include LICENSE
 include README.md
 include config.json.example
 recursive-include freqtrade *.py
+recursive-include freqtrade/templates/ *.j2 *.ipynb

From 05250ba6611278b794677780a2e00febdb14f476 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Wed, 18 Mar 2020 11:00:33 +0100
Subject: [PATCH 029/151] Update docs/plotting.md

Co-Authored-By: Matthias <xmatthias@outlook.com>
---
 docs/plotting.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plotting.md b/docs/plotting.md
index 56906ebec..be83065a6 100644
--- a/docs/plotting.md
+++ b/docs/plotting.md
@@ -32,7 +32,7 @@ usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                 [--trade-source {DB,file}] [--export EXPORT]
                                 [--export-filename PATH]
                                 [--timerange TIMERANGE] [-i TICKER_INTERVAL]
-                                [--skip-trades]
+                                [--no-trades]
 
 optional arguments:
   -h, --help            show this help message and exit

From 0920d6fce4d35ebd3f99ed321131764fde628206 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Wed, 18 Mar 2020 11:01:09 +0100
Subject: [PATCH 030/151] Update freqtrade/data/btanalysis.py

Co-Authored-By: Matthias <xmatthias@outlook.com>
---
 freqtrade/data/btanalysis.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py
index 17b243f56..898072748 100644
--- a/freqtrade/data/btanalysis.py
+++ b/freqtrade/data/btanalysis.py
@@ -129,7 +129,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
     return trades
 
 
-def load_trades(source: str, db_url: str, exportfilename: Path, no_trades: bool) -> pd.DataFrame:
+def load_trades(source: str, db_url: str, exportfilename: Path, no_trades: bool = False) -> pd.DataFrame:
     """
     Based on configuration option "trade_source":
     * loads data from DB (using `db_url`)

From 3e1bef888ab1344f8763e130cdef835b0b030d6b Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 18 Mar 2020 11:42:42 +0100
Subject: [PATCH 031/151] Fix flake8 error

---
 freqtrade/data/btanalysis.py  | 3 ++-
 tests/data/test_btanalysis.py | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py
index 898072748..23a9f720c 100644
--- a/freqtrade/data/btanalysis.py
+++ b/freqtrade/data/btanalysis.py
@@ -129,7 +129,8 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
     return trades
 
 
-def load_trades(source: str, db_url: str, exportfilename: Path, no_trades: bool = False) -> pd.DataFrame:
+def load_trades(source: str, db_url: str, exportfilename: Path,
+                no_trades: bool = False) -> pd.DataFrame:
     """
     Based on configuration option "trade_source":
     * loads data from DB (using `db_url`)
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 4b47faeee..463e5ae36 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -117,7 +117,6 @@ def test_load_trades(default_conf, mocker):
     load_trades("file",
                 db_url=default_conf.get('db_url'),
                 exportfilename=default_conf.get('exportfilename'),
-                no_trades=False
                 )
 
     assert db_mock.call_count == 0

From ecf3a3e07076e3101e804e5216ec38ff236a1686 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 19 Mar 2020 19:42:51 +0100
Subject: [PATCH 032/151] Add test validating different return values

---
 tests/test_freqtradebot.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index 9f6e5ef0c..e37270bd3 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -2228,10 +2228,16 @@ def test_handle_timedout_limit_buy(mocker, default_conf, limit_buy_order) -> Non
     assert cancel_order_mock.call_count == 1
 
 
-def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_order) -> None:
+@pytest.mark.parametrize('cancelorder', [
+    {},
+    'String Return value',
+    123
+])
+def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_order,
+                                                cancelorder) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
-    cancel_order_mock = MagicMock(return_value={})
+    cancel_order_mock = MagicMock(return_value=cancelorder)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         cancel_order=cancel_order_mock

From 5e702f6891c76a066aae85833026780cc01ebee5 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 19 Mar 2020 19:43:19 +0100
Subject: [PATCH 033/151] Verify cancel_order returnvalue is a dictionary

---
 freqtrade/freqtradebot.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 9897b39b4..bc3078a5c 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -891,6 +891,9 @@ class FreqtradeBot:
         if order['status'] != 'canceled':
             reason = "cancelled due to timeout"
             corder = self.exchange.cancel_order(trade.open_order_id, trade.pair)
+            # Some exchanges don't return a dict here.
+            if not isinstance(corder, dict):
+                corder = {}
             logger.info('Buy order %s for %s.', reason, trade)
         else:
             # Order was cancelled already, so we can reuse the existing dict

From 5f9479b39fe4c31ac0171501c28186ef0b1065b9 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Fri, 20 Mar 2020 02:10:44 +0300
Subject: [PATCH 034/151] Edge import cosmetics

---
 freqtrade/edge/edge_positioning.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py
index d196ab4b3..5305e23cf 100644
--- a/freqtrade/edge/edge_positioning.py
+++ b/freqtrade/edge/edge_positioning.py
@@ -8,10 +8,10 @@ import numpy as np
 import utils_find_1st as utf1st
 from pandas import DataFrame
 
-from freqtrade import constants
 from freqtrade.configuration import TimeRange
-from freqtrade.data import history
+from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.exceptions import OperationalException
+from freqtrade.data.history import get_timerange, load_data, refresh_data
 from freqtrade.strategy.interface import SellType
 
 logger = logging.getLogger(__name__)
@@ -54,7 +54,7 @@ class Edge:
         if self.config['max_open_trades'] != float('inf'):
             logger.critical('max_open_trades should be -1 in config !')
 
-        if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT:
+        if self.config['stake_amount'] != UNLIMITED_STAKE_AMOUNT:
             raise OperationalException('Edge works only with unlimited stake amount')
 
         # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future.
@@ -96,7 +96,7 @@ class Edge:
         logger.info('Using local backtesting data (using whitelist in given config) ...')
 
         if self._refresh_pairs:
-            history.refresh_data(
+            refresh_data(
                 datadir=self.config['datadir'],
                 pairs=pairs,
                 exchange=self.exchange,
@@ -104,7 +104,7 @@ class Edge:
                 timerange=self._timerange,
             )
 
-        data = history.load_data(
+        data = load_data(
             datadir=self.config['datadir'],
             pairs=pairs,
             timeframe=self.strategy.ticker_interval,
@@ -122,7 +122,7 @@ class Edge:
         preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
 
         # Print timeframe
-        min_date, max_date = history.get_timerange(preprocessed)
+        min_date, max_date = get_timerange(preprocessed)
         logger.info(
             'Measuring data from %s up to %s (%s days) ...',
             min_date.isoformat(),

From 3e0ffdce75aa6795caa830bfeb4c614a1750c207 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Fri, 20 Mar 2020 04:21:17 +0300
Subject: [PATCH 035/151] Adjust tests

---
 tests/edge/test_edge.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py
index 3bebeee65..2304c53c2 100644
--- a/tests/edge/test_edge.py
+++ b/tests/edge/test_edge.py
@@ -292,8 +292,8 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m',
 def test_edge_process_downloaded_data(mocker, edge_conf):
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
-    mocker.patch('freqtrade.data.history.refresh_data', MagicMock())
-    mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
+    mocker.patch('freqtrade.edge.edge_positioning.refresh_data', MagicMock())
+    mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data)
     edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
 
     assert edge.calculate()
@@ -304,8 +304,8 @@ def test_edge_process_downloaded_data(mocker, edge_conf):
 def test_edge_process_no_data(mocker, edge_conf, caplog):
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
-    mocker.patch('freqtrade.data.history.refresh_data', MagicMock())
-    mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={}))
+    mocker.patch('freqtrade.edge.edge_positioning.refresh_data', MagicMock())
+    mocker.patch('freqtrade.edge.edge_positioning.load_data', MagicMock(return_value={}))
     edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)
 
     assert not edge.calculate()
@@ -317,8 +317,8 @@ def test_edge_process_no_data(mocker, edge_conf, caplog):
 def test_edge_process_no_trades(mocker, edge_conf, caplog):
     freqtrade = get_patched_freqtradebot(mocker, edge_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001))
-    mocker.patch('freqtrade.data.history.refresh_data', MagicMock())
-    mocker.patch('freqtrade.data.history.load_data', mocked_load_data)
+    mocker.patch('freqtrade.edge.edge_positioning.refresh_data', MagicMock())
+    mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data)
     # Return empty
     mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', MagicMock(return_value=[]))
     edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy)

From 942792f1233f35eba31d51d96ecbc92163376af1 Mon Sep 17 00:00:00 2001
From: Yazeed Al Oyoun <yazeed.aloyoun@gmail.com>
Date: Fri, 20 Mar 2020 05:48:53 +0100
Subject: [PATCH 036/151] updated as suggested

---
 freqtrade/freqtradebot.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index e5ae9043a..378252c29 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -394,21 +394,23 @@ class FreqtradeBot:
             logger.info(f"Pair {pair} is currently locked.")
             return False
 
+        # get_free_open_trades is checked before create_trade is called
+        # but it is still used here to prevent opening too many trades within one iteration
         if not self.get_free_open_trades():
             logger.debug(f"Can't open a new trade for {pair}: max number of trades is reached.")
             return False
 
-        stake_amount = self.get_trade_stake_amount(pair)
-        if not stake_amount:
-            logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
-            return False
-
         # running get_signal on historical data fetched
         (buy, sell) = self.strategy.get_signal(
             pair, self.strategy.ticker_interval,
             self.dataprovider.ohlcv(pair, self.strategy.ticker_interval))
 
         if buy and not sell:
+            stake_amount = self.get_trade_stake_amount(pair)
+            if not stake_amount:
+                logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
+                return False
+
             logger.info(f"Buy signal found: about create a new trade with stake_amount: "
                         f"{stake_amount} ...")
 

From e30faf8c8c371de43b0e1c35b5eae89489e01ab9 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 21 Mar 2020 20:04:05 +0100
Subject: [PATCH 037/151] Remove partial candle documentation

It wasn't working 100% correctly - see #2993
---
 docs/exchanges.md | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/docs/exchanges.md b/docs/exchanges.md
index 66a0e96da..c600077ce 100644
--- a/docs/exchanges.md
+++ b/docs/exchanges.md
@@ -74,23 +74,13 @@ Should you experience constant errors with Nonce (like `InvalidNonce`), it is be
 $ pip3 install web3
 ```
 
-### Send incomplete candles to the strategy
+### Getting latest price / Incomplete candles
 
 Most exchanges return current incomplete candle via their OHLCV/klines API interface.
 By default, Freqtrade assumes that incomplete candle is fetched from the exchange and removes the last candle assuming it's the incomplete candle.
 
 Whether your exchange returns incomplete candles or not can be checked using [the helper script](developer.md#Incomplete-candles) from the Contributor documentation.
 
-If the exchange does return incomplete candles and you would like to have incomplete candles in your strategy, you can set the following parameter in the configuration file.
+Due to the danger of repainting, Freqtrade does not allow you to use this incomplete candle.
 
-``` json
-{
-
-    "exchange": {
-        "_ft_has_params": {"ohlcv_partial_candle": false}
-    }
-}
-```
-
-!!! Warning "Danger of repainting"
-    Changing this parameter makes the strategy responsible to avoid repainting and handle this accordingly. Doing this is therefore not recommended, and should only be performed by experienced users who are fully aware of the impact this setting has.
+However, usually, this requirement is based on the need for the latest price - which can be aquired using the [data provider](strategy-customization.md#possible-options-for-dataprovider) from within the strategy.

From 2c434e9b116fdfb71c2da4f57db4e25683ce7d39 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 22 Mar 2020 11:16:09 +0100
Subject: [PATCH 038/151] Add close_proit_abs column

---
 freqtrade/freqtradebot.py |  1 +
 freqtrade/persistence.py  | 11 ++++++++---
 tests/test_persistence.py | 20 ++++++++++++++++++++
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index db632693a..570f8bea8 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -954,6 +954,7 @@ class FreqtradeBot:
 
             trade.close_rate = None
             trade.close_profit = None
+            trade.close_profit_abs = None
             trade.close_date = None
             trade.is_open = True
             trade.open_order_id = None
diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py
index ac084d12e..0d668596c 100644
--- a/freqtrade/persistence.py
+++ b/freqtrade/persistence.py
@@ -86,7 +86,7 @@ def check_migrate(engine) -> None:
         logger.debug(f'trying {table_back_name}')
 
     # Check for latest column
-    if not has_column(cols, 'open_trade_price'):
+    if not has_column(cols, 'close_profit_abs'):
         logger.info(f'Running database migration - backup available as {table_back_name}')
 
         fee_open = get_column_def(cols, 'fee_open', 'fee')
@@ -106,6 +106,9 @@ def check_migrate(engine) -> None:
         ticker_interval = get_column_def(cols, 'ticker_interval', 'null')
         open_trade_price = get_column_def(cols, 'open_trade_price',
                                           f'amount * open_rate * (1 + {fee_open})')
+        close_profit_abs = get_column_def(
+            cols, 'close_profit_abs',
+            f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
 
         # Schema migration necessary
         engine.execute(f"alter table trades rename to {table_back_name}")
@@ -123,7 +126,7 @@ def check_migrate(engine) -> None:
                 stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
                 stoploss_order_id, stoploss_last_update,
                 max_rate, min_rate, sell_reason, strategy,
-                ticker_interval, open_trade_price
+                ticker_interval, open_trade_price, close_profit_abs
                 )
             select id, lower(exchange),
                 case
@@ -143,7 +146,7 @@ def check_migrate(engine) -> None:
                 {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
                 {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
                 {strategy} strategy, {ticker_interval} ticker_interval,
-                {open_trade_price} open_trade_price
+                {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
                 from {table_back_name}
              """)
 
@@ -190,6 +193,7 @@ class Trade(_DECL_BASE):
     close_rate = Column(Float)
     close_rate_requested = Column(Float)
     close_profit = Column(Float)
+    close_profit_abs = Column(Float)
     stake_amount = Column(Float, nullable=False)
     amount = Column(Float)
     open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
@@ -334,6 +338,7 @@ class Trade(_DECL_BASE):
         """
         self.close_rate = Decimal(rate)
         self.close_profit = self.calc_profit_ratio()
+        self.close_profit_abs = self.calc_profit()
         self.close_date = datetime.utcnow()
         self.is_open = False
         self.open_order_id = None
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 6bd7971a7..991922cba 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -476,12 +476,22 @@ def test_migrate_old(mocker, default_conf, fee):
                                      stake=default_conf.get("stake_amount"),
                                      amount=amount
                                      )
+    insert_table_old2 = """INSERT INTO trades (exchange, pair, is_open, fee,
+                          open_rate, close_rate, stake_amount, amount, open_date)
+                          VALUES ('BITTREX', 'BTC_ETC', 0, {fee},
+                          0.00258580, 0.00268580, {stake}, {amount},
+                          '2017-11-28 12:44:24.000000')
+                          """.format(fee=fee.return_value,
+                                     stake=default_conf.get("stake_amount"),
+                                     amount=amount
+                                     )
     engine = create_engine('sqlite://')
     mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine)
 
     # Create table using the old format
     engine.execute(create_table_old)
     engine.execute(insert_table_old)
+    engine.execute(insert_table_old2)
     # Run init to test migration
     init(default_conf['db_url'], default_conf['dry_run'])
 
@@ -500,6 +510,15 @@ def test_migrate_old(mocker, default_conf, fee):
     assert trade.stop_loss == 0.0
     assert trade.initial_stop_loss == 0.0
     assert trade.open_trade_price == trade._calc_open_trade_price()
+    assert trade.close_profit_abs is None
+
+    trade = Trade.query.filter(Trade.id == 2).first()
+    assert trade.close_rate is not None
+    assert trade.is_open == 0
+    assert trade.open_rate_requested is None
+    assert trade.close_rate_requested is None
+    assert trade.close_rate is not None
+    assert pytest.approx(trade.close_profit_abs) == trade.calc_profit()
 
 
 def test_migrate_new(mocker, default_conf, fee, caplog):
@@ -583,6 +602,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert log_has("trying trades_bak2", caplog)
     assert log_has("Running database migration - backup available as trades_bak2", caplog)
     assert trade.open_trade_price == trade._calc_open_trade_price()
+    assert trade.close_profit_abs is None
 
 
 def test_migrate_mid_state(mocker, default_conf, fee, caplog):

From efd94c84debab78c00bc141b6108f43b366c2e07 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 22 Mar 2020 11:22:49 +0100
Subject: [PATCH 039/151] Add example notebook to gitignore again

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 9ac2c9d5d..f206fce66 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,7 +6,6 @@ user_data/*
 !user_data/strategy/sample_strategy.py
 !user_data/notebooks
 user_data/notebooks/*
-!user_data/notebooks/*example.ipynb
 freqtrade-plot.html
 freqtrade-profit-plot.html
 

From f14c496ce9ffca5186d4479a611c37c76c0a35ce Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 22 Mar 2020 11:28:18 +0100
Subject: [PATCH 040/151] Remove calc_close_profit from RPC

This is now possible - but only for closed trades, so certain occurances
need to remain.
---
 freqtrade/rpc/rpc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 9014c1874..a0f50b070 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -197,7 +197,7 @@ class RPC:
                 Trade.close_date >= profitday,
                 Trade.close_date < (profitday + timedelta(days=1))
             ]).order_by(Trade.close_date).all()
-            curdayprofit = sum(trade.calc_profit() for trade in trades)
+            curdayprofit = sum(trade.close_profit_abs for trade in trades)
             profit_days[profitday] = {
                 'amount': f'{curdayprofit:.8f}',
                 'trades': len(trades)
@@ -246,8 +246,8 @@ class RPC:
                 durations.append((trade.close_date - trade.open_date).total_seconds())
 
             if not trade.is_open:
-                profit_ratio = trade.calc_profit_ratio()
-                profit_closed_coin.append(trade.calc_profit())
+                profit_ratio = trade.close_profit
+                profit_closed_coin.append(trade.close_profit_abs)
                 profit_closed_ratio.append(profit_ratio)
             else:
                 # Get current rate

From acd402187a179e1f040fc42fe1dba22b2519a2e9 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 22 Mar 2020 19:39:36 +0100
Subject: [PATCH 041/151] Update max_open_trades documentation

---
 docs/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index b0f4c7554..78c643868 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -40,7 +40,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 
 |  Parameter | Description |
 |------------|-------------|
-| `max_open_trades` | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades). [More information below](#configuring-amount-per-trade).<br> **Datatype:** Positive integer or -1.
+| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the lenght of your whitelist another limitation which can apply. If -1 then it is ignored (i.e. potentially unlimited open trades). [More information below](#configuring-amount-per-trade).<br> **Datatype:** Positive integer or -1.
 | `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** String
 | `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Positive float or `"unlimited"`.
 | `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade). <br>*Defaults to `0.99` 99%).*<br> **Datatype:** Positive float between `0.1` and `1.0`.

From 45aaa8c09d2a704e49b56ef47b360425f305b7b5 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 22 Mar 2020 20:09:01 +0100
Subject: [PATCH 042/151] Parse and show relevant configuration section

---
 freqtrade/configuration/load_config.py | 28 ++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py
index 19179c6c3..0d2dc0955 100644
--- a/freqtrade/configuration/load_config.py
+++ b/freqtrade/configuration/load_config.py
@@ -1,13 +1,15 @@
 """
 This module contain functions to load the configuration file
 """
-import rapidjson
 import logging
+import re
 import sys
+from pathlib import Path
 from typing import Any, Dict
 
-from freqtrade.exceptions import OperationalException
+import rapidjson
 
+from freqtrade.exceptions import OperationalException
 
 logger = logging.getLogger(__name__)
 
@@ -15,6 +17,22 @@ logger = logging.getLogger(__name__)
 CONFIG_PARSE_MODE = rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS
 
 
+def log_config_error_range(path: str, errmsg: str) -> str:
+    """
+    Parses configuration file and prints range around error
+    """
+    if path != '-':
+        offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg)
+        if offsetlist:
+            offset = int(offsetlist[0])
+            text = Path(path).read_text()
+            # Fetch an offset of 80 characters around the error line
+            subtext = text[offset-min(80, offset):offset+80]
+            segments = subtext.split('\n')
+            # Remove first and last lines, to avoid odd truncations
+            return '\n'.join(segments[1:-1])
+
+
 def load_config_file(path: str) -> Dict[str, Any]:
     """
     Loads a config file from the given path
@@ -29,5 +47,11 @@ def load_config_file(path: str) -> Dict[str, Any]:
         raise OperationalException(
             f'Config file "{path}" not found!'
             ' Please create a config file or check whether it exists.')
+    except rapidjson.JSONDecodeError as e:
+        err_range = log_config_error_range(path, str(e))
+        raise OperationalException(
+            f'{e}\n'
+            f'Please verify the following segment of your configuration:\n{err_range}'
+        )
 
     return config

From 6c55b40fe04f4cea4b92841212b150d69f5d200a Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 22 Mar 2020 20:15:33 +0100
Subject: [PATCH 043/151] Add test verifying config printing

---
 tests/test_configuration.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index 1e9d6440d..ec6753feb 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -66,6 +66,17 @@ def test_load_config_file(default_conf, mocker, caplog) -> None:
     assert validated_conf.items() >= default_conf.items()
 
 
+def test_load_config_file_error(default_conf, mocker, caplog) -> None:
+    del default_conf['user_data_dir']
+    filedata = json.dumps(default_conf).replace(
+        '"stake_amount": 0.001,', '"stake_amount": .001,')
+    mocker.patch('freqtrade.configuration.load_config.open', mocker.mock_open(read_data=filedata))
+    mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata))
+
+    with pytest.raises(OperationalException, match=f".*Please verify the following segment.*"):
+        load_config_file('somefile')
+
+
 def test__args_to_config(caplog):
 
     arg_list = ['trade', '--strategy-path', 'TestTest']

From 8f7e113d798bb8b973efd6a66b59d1abf18722f4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 23 Mar 2020 07:54:27 +0100
Subject: [PATCH 044/151] Add additional test

---
 freqtrade/configuration/load_config.py |  7 +++++--
 tests/test_configuration.py            | 15 ++++++++++++++-
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py
index 0d2dc0955..55da28913 100644
--- a/freqtrade/configuration/load_config.py
+++ b/freqtrade/configuration/load_config.py
@@ -29,8 +29,11 @@ def log_config_error_range(path: str, errmsg: str) -> str:
             # Fetch an offset of 80 characters around the error line
             subtext = text[offset-min(80, offset):offset+80]
             segments = subtext.split('\n')
-            # Remove first and last lines, to avoid odd truncations
-            return '\n'.join(segments[1:-1])
+            if len(segments) > 3:
+                # Remove first and last lines, to avoid odd truncations
+                return '\n'.join(segments[1:-1])
+            else:
+                return subtext
 
 
 def load_config_file(path: str) -> Dict[str, Any]:
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index ec6753feb..a1936aee0 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -18,7 +18,7 @@ from freqtrade.configuration.config_validation import validate_config_schema
 from freqtrade.configuration.deprecated_settings import (
     check_conflicting_settings, process_deprecated_setting,
     process_temporary_deprecated_settings)
-from freqtrade.configuration.load_config import load_config_file
+from freqtrade.configuration.load_config import load_config_file, log_config_error_range
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
 from freqtrade.exceptions import OperationalException
 from freqtrade.loggers import _set_loggers, setup_logging
@@ -77,6 +77,19 @@ def test_load_config_file_error(default_conf, mocker, caplog) -> None:
         load_config_file('somefile')
 
 
+def test_load_config_file_error_range(default_conf, mocker, caplog) -> None:
+    del default_conf['user_data_dir']
+    filedata = json.dumps(default_conf).replace(
+        '"stake_amount": 0.001,', '"stake_amount": .001,')
+    mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata))
+
+    x = log_config_error_range('somefile', 'Parse error at offset 64: Invalid value.')
+    assert isinstance(x, str)
+    assert (x == '{"max_open_trades": 1, "stake_currency": "BTC", '
+            '"stake_amount": .001, "fiat_display_currency": "USD", '
+            '"ticker_interval": "5m", "dry_run": true, ')
+
+
 def test__args_to_config(caplog):
 
     arg_list = ['trade', '--strategy-path', 'TestTest']

From d581b7e2d79493ddccb6cbd2cbe41b6baa028b38 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 23 Mar 2020 07:57:30 +0100
Subject: [PATCH 045/151] Add fallback if no error could be determined

---
 freqtrade/configuration/load_config.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py
index 55da28913..a24ee3d0a 100644
--- a/freqtrade/configuration/load_config.py
+++ b/freqtrade/configuration/load_config.py
@@ -34,6 +34,7 @@ def log_config_error_range(path: str, errmsg: str) -> str:
                 return '\n'.join(segments[1:-1])
             else:
                 return subtext
+    return ''
 
 
 def load_config_file(path: str) -> Dict[str, Any]:
@@ -55,6 +56,7 @@ def load_config_file(path: str) -> Dict[str, Any]:
         raise OperationalException(
             f'{e}\n'
             f'Please verify the following segment of your configuration:\n{err_range}'
+            if err_range else 'Please verify your configuration file for syntax errors.'
         )
 
     return config

From f4a69ba5a753c38f9b18ec81e596d38c86ce66de Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 23 Mar 2020 09:04:50 +0000
Subject: [PATCH 046/151] Bump numpy from 1.18.1 to 1.18.2

Bumps [numpy](https://github.com/numpy/numpy) from 1.18.1 to 1.18.2.
- [Release notes](https://github.com/numpy/numpy/releases)
- [Changelog](https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt)
- [Commits](https://github.com/numpy/numpy/compare/v1.18.1...v1.18.2)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 12d7336aa..e69d919e0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 # Load common requirements
 -r requirements-common.txt
 
-numpy==1.18.1
+numpy==1.18.2
 pandas==1.0.2

From de91e169bc6f3316f5d3a5ad52add147ec71eba6 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 23 Mar 2020 09:05:12 +0000
Subject: [PATCH 047/151] Bump tabulate from 0.8.6 to 0.8.7

Bumps [tabulate](https://github.com/astanin/python-tabulate) from 0.8.6 to 0.8.7.
- [Release notes](https://github.com/astanin/python-tabulate/releases)
- [Changelog](https://github.com/astanin/python-tabulate/blob/master/CHANGELOG)
- [Commits](https://github.com/astanin/python-tabulate/compare/v0.8.6...v0.8.7)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 7c2ad4283..c1e23dfa9 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -10,7 +10,7 @@ urllib3==1.25.8
 wrapt==1.12.1
 jsonschema==3.2.0
 TA-Lib==0.4.17
-tabulate==0.8.6
+tabulate==0.8.7
 pycoingecko==1.2.0
 jinja2==2.11.1
 

From 98dc4b4ca3893452e80db2f58dd9d72baaeb5f26 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 23 Mar 2020 09:06:41 +0000
Subject: [PATCH 048/151] Bump ccxt from 1.24.31 to 1.24.83

Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.24.31 to 1.24.83.
- [Release notes](https://github.com/ccxt/ccxt/releases)
- [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst)
- [Commits](https://github.com/ccxt/ccxt/compare/1.24.31...1.24.83)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 7c2ad4283..6274297c2 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.24.31
+ccxt==1.24.83
 SQLAlchemy==1.3.15
 python-telegram-bot==12.4.2
 arrow==0.15.5

From cb1bc5d5abb1b125be1282c7538f9e169d17d3b8 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 23 Mar 2020 09:15:43 +0000
Subject: [PATCH 049/151] Bump pandas from 1.0.2 to 1.0.3

Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.0.2 to 1.0.3.
- [Release notes](https://github.com/pandas-dev/pandas/releases)
- [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md)
- [Commits](https://github.com/pandas-dev/pandas/compare/v1.0.2...v1.0.3)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index e69d919e0..b1a4b4403 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@
 -r requirements-common.txt
 
 numpy==1.18.2
-pandas==1.0.2
+pandas==1.0.3

From 0f53e646fd7d3e707f470244d785539211a17793 Mon Sep 17 00:00:00 2001
From: orehunt <basso.bassista@gmail.com>
Date: Tue, 24 Mar 2020 13:54:46 +0100
Subject: [PATCH 050/151] check that the strategy dataframe matches the one
 given by the bot

---
 freqtrade/strategy/interface.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index 696d2b2d2..530cd0af4 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -241,8 +241,18 @@ class IStrategy(ABC):
 
         return dataframe
 
-    def get_signal(self, pair: str, interval: str,
-                   dataframe: DataFrame) -> Tuple[bool, bool]:
+    @staticmethod
+    def preserve_df(d: DataFrame) -> Tuple[int, float, datetime]:
+        """ keep some data for dataframes """
+        return len(d), d["close"].iloc[-1], d["date"].iloc[-1]
+
+    @staticmethod
+    def assert_df(d: DataFrame, df_len: int, df_close: float, df_date: datetime):
+        """ make sure data is unmodified """
+        if df_len != len(d) or df_close != d["close"].iloc[-1] or df_date != d["date"].iloc[-1]:
+            raise Exception("Dataframe returned from strategy does not match original")
+
+    def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]:
         """
         Calculates current signal based several technical analysis indicators
         :param pair: pair in format ANT/BTC
@@ -254,8 +264,11 @@ class IStrategy(ABC):
             logger.warning('Empty candle (OHLCV) data for pair %s', pair)
             return False, False
 
+        latest_date = dataframe['date'].max()
         try:
+            df_len, df_close, df_date = self.preserve_df(dataframe)
             dataframe = self._analyze_ticker_internal(dataframe, {'pair': pair})
+            self.assert_df(dataframe, df_len, df_close, df_date)
         except ValueError as error:
             logger.warning(
                 'Unable to analyze candle (OHLCV) data for pair %s: %s',
@@ -275,7 +288,7 @@ class IStrategy(ABC):
             logger.warning('Empty dataframe for pair %s', pair)
             return False, False
 
-        latest = dataframe.iloc[-1]
+        latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1]
 
         # Check if dataframe is out of date
         signal_date = arrow.get(latest['date'])

From be41981ef08f65d2d377d7a225456fbdc6221b83 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 20:10:15 +0100
Subject: [PATCH 051/151] Test warnings with filter always on

---
 tests/test_configuration.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index 1e9d6440d..79387ba7a 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -73,6 +73,7 @@ def test__args_to_config(caplog):
     configuration = Configuration(args)
     config = {}
     with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
         # No warnings ...
         configuration._args_to_config(config, argname="strategy_path", logstring="DeadBeef")
         assert len(w) == 0
@@ -82,6 +83,7 @@ def test__args_to_config(caplog):
     configuration = Configuration(args)
     config = {}
     with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
         # Deprecation warnings!
         configuration._args_to_config(config, argname="strategy_path", logstring="DeadBeef",
                                       deprecated_msg="Going away soon!")

From d9a5e1cd48955041a6e41e1e6138be75b15c46f1 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 25 Mar 2020 08:30:18 +0100
Subject: [PATCH 052/151] Update docs/exchanges.md

Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
---
 docs/exchanges.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/exchanges.md b/docs/exchanges.md
index c600077ce..06db26f89 100644
--- a/docs/exchanges.md
+++ b/docs/exchanges.md
@@ -83,4 +83,4 @@ Whether your exchange returns incomplete candles or not can be checked using [th
 
 Due to the danger of repainting, Freqtrade does not allow you to use this incomplete candle.
 
-However, usually, this requirement is based on the need for the latest price - which can be aquired using the [data provider](strategy-customization.md#possible-options-for-dataprovider) from within the strategy.
+However, if it is based on the need for the latest price for your strategy - then this requirement can be acquired using the [data provider](strategy-customization.md#possible-options-for-dataprovider) from within the strategy.

From dfac7448d1260b81653e600fda27dc57d41af0ab Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 25 Mar 2020 08:33:22 +0100
Subject: [PATCH 053/151] fix typo

Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
---
 docs/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index 78c643868..9ecb0a13e 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -40,7 +40,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 
 |  Parameter | Description |
 |------------|-------------|
-| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the lenght of your whitelist another limitation which can apply. If -1 then it is ignored (i.e. potentially unlimited open trades). [More information below](#configuring-amount-per-trade).<br> **Datatype:** Positive integer or -1.
+| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the length of your whitelist another limitation which can apply. If -1 then it is ignored (i.e. potentially unlimited open trades). [More information below](#configuring-amount-per-trade).<br> **Datatype:** Positive integer or -1.
 | `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** String
 | `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Positive float or `"unlimited"`.
 | `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade). <br>*Defaults to `0.99` 99%).*<br> **Datatype:** Positive float between `0.1` and `1.0`.

From 6f687c97ce1743e43dc8b4a1bcb166cf7be4a523 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 25 Mar 2020 09:24:37 +0100
Subject: [PATCH 054/151] Update docs/configuration.md

Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
---
 docs/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index 9ecb0a13e..d8a9653c3 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -40,7 +40,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 
 |  Parameter | Description |
 |------------|-------------|
-| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the length of your whitelist another limitation which can apply. If -1 then it is ignored (i.e. potentially unlimited open trades). [More information below](#configuring-amount-per-trade).<br> **Datatype:** Positive integer or -1.
+| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the length of your pairlist is another limitation which can apply. If -1 then it is ignored (i.e. potentially unlimited open trades, limited by the pairlist). [More information below](#configuring-amount-per-trade).<br> **Datatype:** Positive integer or -1.
 | `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** String
 | `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade). [Strategy Override](#parameters-in-the-strategy). <br> **Datatype:** Positive float or `"unlimited"`.
 | `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade). <br>*Defaults to `0.99` 99%).*<br> **Datatype:** Positive float between `0.1` and `1.0`.

From 91b058cf11d2ed585c0a67e9fa4dd89d280907f9 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 16:16:10 +0100
Subject: [PATCH 055/151] Fix typo in tests

---
 tests/test_freqtradebot.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index e37270bd3..abc9babba 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -2129,7 +2129,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
     assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
-    # Verify that tradehas been updated
+    # Verify that trade has been updated
     assert trades[0].amount == (limit_buy_order_old_partial['amount'] -
                                 limit_buy_order_old_partial['remaining']) - 0.0001
     assert trades[0].open_order_id is None
@@ -2168,7 +2168,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade,
     assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
-    # Verify that tradehas been updated
+    # Verify that trade has been updated
 
     assert trades[0].amount == (limit_buy_order_old_partial['amount'] -
                                 limit_buy_order_old_partial['remaining'])

From 1817e6fbdfecbbca308e2fbaf8bdc9b57756182e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 16:16:19 +0100
Subject: [PATCH 056/151] Combine real_amount updating into one method

---
 freqtrade/freqtradebot.py | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 570f8bea8..b30f55ccd 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -916,17 +916,7 @@ class FreqtradeBot:
         # we need to fall back to the values from order if corder does not contain these keys.
         trade.amount = order['amount'] - corder.get('remaining', order['remaining'])
         trade.stake_amount = trade.amount * trade.open_rate
-        # verify if fees were taken from amount to avoid problems during selling
-        try:
-            new_amount = self.get_real_amount(trade, corder if 'fee' in corder else order,
-                                              trade.amount)
-            if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
-                trade.amount = new_amount
-                # Fee was applied, so set to 0
-                trade.fee_open = 0
-                trade.recalc_open_trade_price()
-        except DependencyException as e:
-            logger.warning("Could not update trade amount: %s", e)
+        self.update_trade_state(trade, corder if 'fee' in corder else order, trade.amount)
 
         trade.open_order_id = None
         logger.info('Partial buy order timeout for %s.', trade)
@@ -1122,9 +1112,11 @@ class FreqtradeBot:
 # Common update trade state methods
 #
 
-    def update_trade_state(self, trade: Trade, action_order: dict = None) -> None:
+    def update_trade_state(self, trade: Trade, action_order: dict = None,
+                           order_amount: float = None) -> None:
         """
         Checks trades with open orders and updates the amount if necessary
+        Handles closing both buy and sell orders.
         """
         # Get order details for actual price per unit
         if trade.open_order_id:
@@ -1137,7 +1129,7 @@ class FreqtradeBot:
                 return
             # Try update amount (binance-fix)
             try:
-                new_amount = self.get_real_amount(trade, order)
+                new_amount = self.get_real_amount(trade, order, order_amount)
                 if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
                     order['amount'] = new_amount
                     # Fee was applied, so set to 0

From 9c351007f5a1fa4b27f0c123f6a8c736c9843f55 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 17:12:24 +0100
Subject: [PATCH 057/151] Provide reason for cancelled sell order

---
 freqtrade/freqtradebot.py      | 12 +++++++-----
 freqtrade/rpc/telegram.py      |  3 ++-
 tests/rpc/test_rpc_telegram.py |  6 ++++--
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index b30f55ccd..af0de108a 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -926,10 +926,10 @@ class FreqtradeBot:
         })
         return False
 
-    def handle_timedout_limit_sell(self, trade: Trade, order: Dict) -> bool:
+    def handle_timedout_limit_sell(self, trade: Trade, order: Dict) -> str:
         """
         Sell timeout - cancel order and update trade
-        :return: True if order was fully cancelled
+        :return: Reason for cancel
         """
         # if trade is not partially completed, just cancel the trade
         if order['remaining'] == order['amount']:
@@ -943,16 +943,17 @@ class FreqtradeBot:
                 logger.info('Sell order %s for %s.', reason, trade)
 
             trade.close_rate = None
+            trade.close_rate_requested = None
             trade.close_profit = None
             trade.close_profit_abs = None
             trade.close_date = None
             trade.is_open = True
             trade.open_order_id = None
 
-            return True
+            return reason
 
         # TODO: figure out how to handle partially complete sell orders
-        return False
+        return 'partially filled - keeping order open'
 
     def _safe_sell_amount(self, pair: str, amount: float) -> float:
         """
@@ -1071,7 +1072,7 @@ class FreqtradeBot:
         # Send the message
         self.rpc.send_msg(msg)
 
-    def _notify_sell_cancel(self, trade: Trade, order_type: str) -> None:
+    def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
         """
         Sends rpc notification when a sell cancel occured.
         """
@@ -1098,6 +1099,7 @@ class FreqtradeBot:
             'close_date': trade.close_date,
             'stake_currency': self.config['stake_currency'],
             'fiat_currency': self.config.get('fiat_display_currency', None),
+            'reason': reason,
         }
 
         if 'fiat_display_currency' in self.config:
diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index ad01700ab..a21f7556c 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -172,7 +172,8 @@ class Telegram(RPC):
                             ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg)
 
         elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
-            message = "*{exchange}:* Cancelling Open Sell Order for {pair}".format(**msg)
+            message = ("*{exchange}:* Cancelling Open Sell Order "
+                       "for {pair}. Reason: {reason}").format(**msg)
 
         elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION:
             message = '*Status:* `{status}`'.format(**msg)
diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py
index d769016c4..bbc961763 100644
--- a/tests/rpc/test_rpc_telegram.py
+++ b/tests/rpc/test_rpc_telegram.py
@@ -1316,18 +1316,20 @@ def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None:
         'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
         'exchange': 'Binance',
         'pair': 'KEY/ETH',
+        'reason': 'Cancelled on exchange'
     })
     assert msg_mock.call_args[0][0] \
-        == ('*Binance:* Cancelling Open Sell Order for KEY/ETH')
+        == ('*Binance:* Cancelling Open Sell Order for KEY/ETH. Reason: Cancelled on exchange')
 
     msg_mock.reset_mock()
     telegram.send_msg({
         'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
         'exchange': 'Binance',
         'pair': 'KEY/ETH',
+        'reason': 'timeout'
     })
     assert msg_mock.call_args[0][0] \
-        == ('*Binance:* Cancelling Open Sell Order for KEY/ETH')
+        == ('*Binance:* Cancelling Open Sell Order for KEY/ETH. Reason: timeout')
     # Reset singleton function to avoid random breaks
     telegram._fiat_converter.convert_amount = old_convamount
 

From 270ac2e8c113f39a55a14c019e19c578f8b059ab Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 17:15:47 +0100
Subject: [PATCH 058/151] Add check_order_cancelled_empty method to exchange

---
 freqtrade/exchange/exchange.py  |  8 ++++++++
 tests/exchange/test_exchange.py | 12 ++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index f4c94a1ca..073e28659 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -902,6 +902,14 @@ class Exchange:
             self._async_get_trade_history(pair=pair, since=since,
                                           until=until, from_id=from_id))
 
+    def check_order_canceled_empty(self, order: Dict) -> bool:
+        """
+        Verify if an order has been cancelled without being partially filled
+        :param order: Order dict as returned from get_order()
+        :return: True if order has been cancelled without being filled, False otherwise.
+        """
+        return order['status'] in ('closed', 'canceled') and order.get('filled') == 0.0
+
     @retrier
     def cancel_order(self, order_id: str, pair: str) -> None:
         if self._config['dry_run']:
diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py
index 8d8930f66..7f03eb547 100644
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@@ -1705,6 +1705,18 @@ def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
     assert exchange.cancel_order(order_id='123', pair='TKN/BTC') is None
 
 
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+@pytest.mark.parametrize("order,result", [
+    ({'status': 'closed', 'filled': 10}, False),
+    ({'status': 'closed', 'filled': 0.0}, True),
+    ({'status': 'canceled', 'filled': 0.0}, True),
+    ({'status': 'canceled', 'filled': 10.0}, False),
+    ])
+def test_check_order_canceled_empty(mocker, default_conf, exchange_name, order, result):
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    assert exchange.check_order_canceled_empty(order) == result
+
+
 # Ensure that if not dry_run, we should call API
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_cancel_order(default_conf, mocker, exchange_name):

From 7c47c6e3bd52e30d7d81462d0188c5c31baa18a0 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 17:16:35 +0100
Subject: [PATCH 059/151] check for timeouts before exiting positions

---
 freqtrade/freqtradebot.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index af0de108a..14cda9192 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -143,6 +143,10 @@ class FreqtradeBot:
         self.dataprovider.refresh(self._create_pair_whitelist(self.active_pair_whitelist),
                                   self.strategy.informative_pairs())
 
+        with self._sell_lock:
+            # Check and handle any timed out open orders
+            self.check_handle_timedout()
+
         # Protect from collisions with forcesell.
         # Without this, freqtrade my try to recreate stoploss_on_exchange orders
         # while selling is in process, since telegram messages arrive in an different thread.
@@ -154,8 +158,6 @@ class FreqtradeBot:
         if self.get_free_open_trades():
             self.enter_positions()
 
-        # Check and handle any timed out open orders
-        self.check_handle_timedout()
         Trade.session.flush()
 
     def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]:

From 700cedc57312b02e18a2aa98b5f71ab253953c80 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 17:17:40 +0100
Subject: [PATCH 060/151] Unify handling of open orders to update_trade_state

---
 freqtrade/freqtradebot.py | 42 ++++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 14cda9192..e27569317 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -602,7 +602,6 @@ class FreqtradeBot:
         trades_closed = 0
         for trade in trades:
             try:
-                self.update_trade_state(trade)
 
                 if (self.strategy.order_types.get('stoploss_on_exchange') and
                         self.handle_stoploss_on_exchange(trade)):
@@ -862,30 +861,32 @@ class FreqtradeBot:
                     continue
                 order = self.exchange.get_order(trade.open_order_id, trade.pair)
             except (RequestException, DependencyException, InvalidOrderException):
-                logger.info(
-                    'Cannot query order for %s due to %s',
-                    trade,
-                    traceback.format_exc())
+                logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
                 continue
 
+            trade_state_update = self.update_trade_state(trade, order)
+
             # Check if trade is still actually open
-            if float(order.get('remaining', 0.0)) == 0.0:
-                self.wallets.update()
-                continue
+            # TODO: this seems questionable at best
+            # if float(order.get('remaining', 0.0)) == 0.0:
+            #     self.wallets.update()
+            #     continue
 
-            if ((order['side'] == 'buy' and order['status'] == 'canceled')
-                    or (self._check_timed_out('buy', order))):
+            if (order['side'] == 'buy' and (
+                    trade_state_update
+                    or self._check_timed_out('buy', order))):
                 self.handle_timedout_limit_buy(trade, order)
                 self.wallets.update()
                 order_type = self.strategy.order_types['buy']
                 self._notify_buy_cancel(trade, order_type)
 
-            elif ((order['side'] == 'sell' and order['status'] == 'canceled')
-                  or (self._check_timed_out('sell', order))):
-                self.handle_timedout_limit_sell(trade, order)
+            elif (order['side'] == 'sell' and (
+                    trade_state_update
+                    or self._check_timed_out('sell', order))):
+                reason = self.handle_timedout_limit_sell(trade, order)
                 self.wallets.update()
                 order_type = self.strategy.order_types['sell']
-                self._notify_sell_cancel(trade, order_type)
+                self._notify_sell_cancel(trade, order_type, reason)
 
     def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool:
         """
@@ -934,8 +935,8 @@ class FreqtradeBot:
         :return: Reason for cancel
         """
         # if trade is not partially completed, just cancel the trade
-        if order['remaining'] == order['amount']:
-            if order["status"] != "canceled":
+        if order['remaining'] == order['amount'] or order['filled'] == 0.0:
+            if not self.exchange.check_order_canceled_empty(order):
                 reason = "cancelled due to timeout"
                 # if trade is not partially completed, just delete the trade
                 self.exchange.cancel_order(trade.open_order_id, trade.pair)
@@ -1117,7 +1118,7 @@ class FreqtradeBot:
 #
 
     def update_trade_state(self, trade: Trade, action_order: dict = None,
-                           order_amount: float = None) -> None:
+                           order_amount: float = None) -> bool:
         """
         Checks trades with open orders and updates the amount if necessary
         Handles closing both buy and sell orders.
@@ -1139,16 +1140,21 @@ class FreqtradeBot:
                     # Fee was applied, so set to 0
                     trade.fee_open = 0
                     trade.recalc_open_trade_price()
-
             except DependencyException as exception:
                 logger.warning("Could not update trade amount: %s", exception)
 
+            if self.exchange.check_order_canceled_empty(order):
+                # Trade has been cancelled on exchange
+                # Handling of this will happen in check_handle_timeout.
+                return True
             trade.update(order)
 
             # Updating wallets when order is closed
             if not trade.is_open:
                 self.wallets.update()
 
+        return False
+
     def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float:
         """
         Get real amount for the trade

From f3103be15c22a09648b58bffbf88611faf9dc4d1 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 17:20:16 +0100
Subject: [PATCH 061/151] Fix test

---
 freqtrade/freqtradebot.py  |  2 +-
 tests/test_freqtradebot.py | 13 ++++++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index e27569317..bec08fbc2 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -935,7 +935,7 @@ class FreqtradeBot:
         :return: Reason for cancel
         """
         # if trade is not partially completed, just cancel the trade
-        if order['remaining'] == order['amount'] or order['filled'] == 0.0:
+        if order['remaining'] == order['amount'] or order.get('filled') == 0.0:
             if not self.exchange.check_order_canceled_empty(order):
                 reason = "cancelled due to timeout"
                 # if trade is not partially completed, just delete the trade
diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index abc9babba..fd6136799 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -1592,13 +1592,13 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog)
     mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order)
 
     trade = MagicMock()
-    trade.open_order_id = '123'
+    trade.open_order_id = None
     trade.open_fee = 0.001
     trades = [trade]
 
     # Test raise of DependencyException exception
     mocker.patch(
-        'freqtrade.freqtradebot.FreqtradeBot.update_trade_state',
+        'freqtrade.freqtradebot.FreqtradeBot.handle_trade',
         side_effect=DependencyException()
     )
     n = freqtrade.exit_positions(trades)
@@ -1970,7 +1970,7 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o
     rpc_mock = patch_RPCManager(mocker)
     cancel_order_mock = MagicMock()
     patch_exchange(mocker)
-    limit_buy_order_old.update({"status": "canceled"})
+    limit_buy_order_old.update({"status": "canceled", 'filled': 0.0})
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
@@ -2049,7 +2049,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old,
     """ Handle sell order cancelled on exchange"""
     rpc_mock = patch_RPCManager(mocker)
     cancel_order_mock = MagicMock()
-    limit_sell_order_old.update({"status": "canceled"})
+    limit_sell_order_old.update({"status": "canceled", 'filled': 0.0})
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
@@ -2276,7 +2276,7 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None:
     assert freqtrade.handle_timedout_limit_sell(trade, order)
     assert cancel_order_mock.call_count == 1
     order['amount'] = 2
-    assert not freqtrade.handle_timedout_limit_sell(trade, order)
+    assert freqtrade.handle_timedout_limit_sell(trade, order) == 'partially filled - keeping order open'
     # Assert cancel_order was not called (callcount remains unchanged)
     assert cancel_order_mock.call_count == 1
 
@@ -2544,8 +2544,11 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f
 
     # Create some test data
     freqtrade.enter_positions()
+    freqtrade.check_handle_timedout()
     trade = Trade.query.first()
     trades = [trade]
+    assert trade.stoploss_order_id is None
+
     freqtrade.exit_positions(trades)
     assert trade
     assert trade.stoploss_order_id == '123'

From 19e5dbddc6670d9d759bc140bdbd4298ec9c13e4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 19:53:35 +0100
Subject: [PATCH 062/151] Add filled to all orders

---
 tests/conftest.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/conftest.py b/tests/conftest.py
index 64d0cd5ee..4b971c1bc 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -712,6 +712,7 @@ def limit_buy_order():
         'datetime': arrow.utcnow().isoformat(),
         'price': 0.00001099,
         'amount': 90.99181073,
+        'filled': 90.99181073,
         'remaining': 0.0,
         'status': 'closed'
     }
@@ -727,6 +728,7 @@ def market_buy_order():
         'datetime': arrow.utcnow().isoformat(),
         'price': 0.00004099,
         'amount': 91.99181073,
+        'filled': 91.99181073,
         'remaining': 0.0,
         'status': 'closed'
     }
@@ -742,6 +744,7 @@ def market_sell_order():
         'datetime': arrow.utcnow().isoformat(),
         'price': 0.00004173,
         'amount': 91.99181073,
+        'filled': 91.99181073,
         'remaining': 0.0,
         'status': 'closed'
     }
@@ -757,6 +760,7 @@ def limit_buy_order_old():
         'datetime': str(arrow.utcnow().shift(minutes=-601).datetime),
         'price': 0.00001099,
         'amount': 90.99181073,
+        'filled': 0.0,
         'remaining': 90.99181073,
         'status': 'open'
     }
@@ -772,6 +776,7 @@ def limit_sell_order_old():
         'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
         'price': 0.00001099,
         'amount': 90.99181073,
+        'filled': 0.0,
         'remaining': 90.99181073,
         'status': 'open'
     }
@@ -787,6 +792,7 @@ def limit_buy_order_old_partial():
         'datetime': arrow.utcnow().shift(minutes=-601).isoformat(),
         'price': 0.00001099,
         'amount': 90.99181073,
+        'filled': 23.0,
         'remaining': 67.99181073,
         'status': 'open'
     }
@@ -810,6 +816,7 @@ def limit_sell_order():
         'datetime': arrow.utcnow().isoformat(),
         'price': 0.00001173,
         'amount': 90.99181073,
+        'filled': 90.99181073,
         'remaining': 0.0,
         'status': 'closed'
     }

From f04f606b707ee39d03224e3c015610c12c5a252d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 19:53:50 +0100
Subject: [PATCH 063/151] Updateing order amount should use filled - not amount
 if possible

---
 freqtrade/persistence.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py
index 0d668596c..97a6b6084 100644
--- a/freqtrade/persistence.py
+++ b/freqtrade/persistence.py
@@ -315,7 +315,7 @@ class Trade(_DECL_BASE):
         if order_type in ('market', 'limit') and order['side'] == 'buy':
             # Update open rate and actual amount
             self.open_rate = Decimal(order['price'])
-            self.amount = Decimal(order['amount'])
+            self.amount = Decimal(order.get('filled', order['amount']))
             self.recalc_open_trade_price()
             logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self)
             self.open_order_id = None

From 1e2fadbc022c93e4e4fb3b2679af3e86938d1182 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 19:54:13 +0100
Subject: [PATCH 064/151] Fix failing test

---
 tests/test_freqtradebot.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index fd6136799..4a633e0d8 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -2499,6 +2499,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke
     assert trade
     trades = [trade]
 
+    freqtrade.check_handle_timedout()
     freqtrade.exit_positions(trades)
 
     # Increase the price and sell it

From 3c1b155e9ff0f837ffbd950545f93c4f8a9ffe1f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 24 Mar 2020 20:05:25 +0100
Subject: [PATCH 065/151] Remove filled if amount is modified to suit fee
 structure

---
 freqtrade/freqtradebot.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index bec08fbc2..d0b37fefb 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -1122,6 +1122,7 @@ class FreqtradeBot:
         """
         Checks trades with open orders and updates the amount if necessary
         Handles closing both buy and sell orders.
+        :return: True if order has been cancelled without being filled partially, False otherwise
         """
         # Get order details for actual price per unit
         if trade.open_order_id:
@@ -1131,12 +1132,13 @@ class FreqtradeBot:
                 order = action_order or self.exchange.get_order(trade.open_order_id, trade.pair)
             except InvalidOrderException as exception:
                 logger.warning('Unable to fetch order %s: %s', trade.open_order_id, exception)
-                return
+                return False
             # Try update amount (binance-fix)
             try:
                 new_amount = self.get_real_amount(trade, order, order_amount)
                 if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
                     order['amount'] = new_amount
+                    del order['filled']
                     # Fee was applied, so set to 0
                     trade.fee_open = 0
                     trade.recalc_open_trade_price()

From 95011919d3c2857b6fb34cb61d83d00c8d791a0f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 25 Mar 2020 10:45:05 +0100
Subject: [PATCH 066/151] Remove questionable handling of orders

---
 freqtrade/freqtradebot.py  | 6 ------
 tests/test_freqtradebot.py | 3 ++-
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index d0b37fefb..a0d142dee 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -866,12 +866,6 @@ class FreqtradeBot:
 
             trade_state_update = self.update_trade_state(trade, order)
 
-            # Check if trade is still actually open
-            # TODO: this seems questionable at best
-            # if float(order.get('remaining', 0.0)) == 0.0:
-            #     self.wallets.update()
-            #     continue
-
             if (order['side'] == 'buy' and (
                     trade_state_update
                     or self._check_timed_out('buy', order))):
diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index 4a633e0d8..9fb545b15 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -2276,7 +2276,8 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None:
     assert freqtrade.handle_timedout_limit_sell(trade, order)
     assert cancel_order_mock.call_count == 1
     order['amount'] = 2
-    assert freqtrade.handle_timedout_limit_sell(trade, order) == 'partially filled - keeping order open'
+    assert (freqtrade.handle_timedout_limit_sell(trade, order)
+            == 'partially filled - keeping order open')
     # Assert cancel_order was not called (callcount remains unchanged)
     assert cancel_order_mock.call_count == 1
 

From 3ef568029f48a55dc22b64d4ea904da75d50b2d3 Mon Sep 17 00:00:00 2001
From: orehunt <basso.bassista@gmail.com>
Date: Thu, 26 Mar 2020 07:05:30 +0100
Subject: [PATCH 067/151] different exception messages

---
 freqtrade/strategy/interface.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index 530cd0af4..4f833be23 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -16,6 +16,7 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.persistence import Trade
 from freqtrade.wallets import Wallets
+from freqtrade.exceptions import DependencyException
 
 
 logger = logging.getLogger(__name__)
@@ -242,15 +243,23 @@ class IStrategy(ABC):
         return dataframe
 
     @staticmethod
-    def preserve_df(d: DataFrame) -> Tuple[int, float, datetime]:
+    def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]:
         """ keep some data for dataframes """
-        return len(d), d["close"].iloc[-1], d["date"].iloc[-1]
+        return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1]
 
     @staticmethod
-    def assert_df(d: DataFrame, df_len: int, df_close: float, df_date: datetime):
+    def assert_df(dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime):
         """ make sure data is unmodified """
-        if df_len != len(d) or df_close != d["close"].iloc[-1] or df_date != d["date"].iloc[-1]:
-            raise Exception("Dataframe returned from strategy does not match original")
+        message = ""
+        if df_len != len(dataframe):
+            message = "length"
+        elif df_close != dataframe["close"].iloc[-1]:
+            message = "last close price"
+        elif df_date != dataframe["date"].iloc[-1]:
+            message = "last date"
+        if message:
+            raise DependencyException("Dataframe returned from strategy has mismatching "
+                                      f"{message}.")
 
     def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]:
         """

From 78aa65825550726142ca154b9b0c4427fac283c1 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 29 Mar 2020 11:27:40 +0200
Subject: [PATCH 068/151] Remove unnecessary test (it's a copy of the remaining
 test)

---
 tests/strategy/test_interface.py | 38 ++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py
index 949dda4a0..4e8d8f708 100644
--- a/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@@ -17,33 +17,36 @@ from tests.conftest import get_patched_exchange, log_has
 _STRATEGY = DefaultStrategy(config={})
 
 
-def test_returns_latest_buy_signal(mocker, default_conf, ohlcv_history):
-    mocker.patch.object(
-        _STRATEGY, '_analyze_ticker_internal',
-        return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}])
-    )
-    assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (True, False)
+def test_returns_latest_signal(mocker, default_conf, ohlcv_history):
+    ohlcv_history.loc[1, 'date'] = arrow.utcnow()
+    # Take a copy to correctly modify the call
+    mocked_history = ohlcv_history.copy()
+    mocked_history['sell'] = 0
+    mocked_history['buy'] = 0
+    mocked_history.loc[1, 'sell'] = 1
 
     mocker.patch.object(
         _STRATEGY, '_analyze_ticker_internal',
-        return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}])
-    )
-    assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (False, True)
-
-
-def test_returns_latest_sell_signal(mocker, default_conf, ohlcv_history):
-    mocker.patch.object(
-        _STRATEGY, '_analyze_ticker_internal',
-        return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}])
+        return_value=mocked_history
     )
 
     assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (False, True)
+    mocked_history.loc[1, 'sell'] = 0
+    mocked_history.loc[1, 'buy'] = 1
 
     mocker.patch.object(
         _STRATEGY, '_analyze_ticker_internal',
-        return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}])
+        return_value=mocked_history
     )
     assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (True, False)
+    mocked_history.loc[1, 'sell'] = 0
+    mocked_history.loc[1, 'buy'] = 0
+
+    mocker.patch.object(
+        _STRATEGY, '_analyze_ticker_internal',
+        return_value=mocked_history
+    )
+    assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (False, False)
 
 
 def test_get_signal_empty(default_conf, mocker, caplog):
@@ -74,6 +77,8 @@ def test_get_signal_empty_dataframe(default_conf, mocker, caplog, ohlcv_history)
         _STRATEGY, '_analyze_ticker_internal',
         return_value=DataFrame([])
     )
+    mocker.patch.object(_STRATEGY, 'assert_df')
+
     assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
                                                   ohlcv_history)
     assert log_has('Empty dataframe for pair xyz', caplog)
@@ -89,6 +94,7 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history):
         _STRATEGY, '_analyze_ticker_internal',
         return_value=DataFrame(ticks)
     )
+    mocker.patch.object(_STRATEGY, 'assert_df')
     assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
                                                   ohlcv_history)
     assert log_has('Outdated history for pair xyz. Last tick is 16 minutes old', caplog)

From 0887a0212c73816e3ae68c7ca7f7a4cb09a3f192 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 29 Mar 2020 11:29:31 +0200
Subject: [PATCH 069/151] Adjust tests to pass validation

---
 tests/strategy/test_interface.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py
index 4e8d8f708..be8750c3c 100644
--- a/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@@ -85,14 +85,19 @@ def test_get_signal_empty_dataframe(default_conf, mocker, caplog, ohlcv_history)
 
 
 def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history):
-    caplog.set_level(logging.INFO)
     # default_conf defines a 5m interval. we check interval * 2 + 5m
     # this is necessary as the last candle is removed (partial candles) by default
-    oldtime = arrow.utcnow().shift(minutes=-16)
-    ticks = DataFrame([{'buy': 1, 'date': oldtime}])
+    ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16)
+    # Take a copy to correctly modify the call
+    mocked_history = ohlcv_history.copy()
+    mocked_history['sell'] = 0
+    mocked_history['buy'] = 0
+    mocked_history.loc[1, 'buy'] = 1
+
+    caplog.set_level(logging.INFO)
     mocker.patch.object(
         _STRATEGY, '_analyze_ticker_internal',
-        return_value=DataFrame(ticks)
+        return_value=mocked_history
     )
     mocker.patch.object(_STRATEGY, 'assert_df')
     assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],

From cd2e738e351ae2e84d144009291c5346848f029b Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 29 Mar 2020 11:40:13 +0200
Subject: [PATCH 070/151] Add test for assert error

---
 freqtrade/strategy/interface.py  | 11 ++++++-----
 tests/strategy/test_interface.py | 21 +++++++++++++++++++++
 2 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index 4f833be23..89a38bf54 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -279,11 +279,12 @@ class IStrategy(ABC):
             dataframe = self._analyze_ticker_internal(dataframe, {'pair': pair})
             self.assert_df(dataframe, df_len, df_close, df_date)
         except ValueError as error:
-            logger.warning(
-                'Unable to analyze candle (OHLCV) data for pair %s: %s',
-                pair,
-                str(error)
-            )
+            logger.warning('Unable to analyze candle (OHLCV) data for pair %s: %s',
+                           pair, str(error))
+            return False, False
+        except DependencyException as error:
+            logger.warning("Unable to analyze candle (OHLCV) data for pair %s: %s",
+                           pair, str(error))
             return False, False
         except Exception as error:
             logger.exception(
diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py
index be8750c3c..1f496a01b 100644
--- a/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock
 import arrow
 from pandas import DataFrame
 
+from freqtrade.exceptions import DependencyException
 from freqtrade.configuration import TimeRange
 from freqtrade.data.history import load_data
 from freqtrade.persistence import Trade
@@ -105,6 +106,26 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history):
     assert log_has('Outdated history for pair xyz. Last tick is 16 minutes old', caplog)
 
 
+def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history):
+    # default_conf defines a 5m interval. we check interval * 2 + 5m
+    # this is necessary as the last candle is removed (partial candles) by default
+    ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16)
+    # Take a copy to correctly modify the call
+    mocked_history = ohlcv_history.copy()
+    mocked_history['sell'] = 0
+    mocked_history['buy'] = 0
+    mocked_history.loc[1, 'buy'] = 1
+
+    caplog.set_level(logging.INFO)
+    mocker.patch.object(
+        _STRATEGY, 'assert_df',
+        side_effect=DependencyException('Dataframe returned...')
+    )
+    assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
+                                                  ohlcv_history)
+    assert log_has('Unable to analyze candle (OHLCV) data for pair xyz: Dataframe returned...', caplog)
+
+
 def test_get_signal_handles_exceptions(mocker, default_conf):
     exchange = get_patched_exchange(mocker, default_conf)
     mocker.patch.object(

From 83cc121b706efe6e364885f24778255e9c0eb60a Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 29 Mar 2020 11:44:36 +0200
Subject: [PATCH 071/151] Add tsts for assert_df (ensuring it raises when it
 should)

---
 tests/strategy/test_interface.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py
index 1f496a01b..8bc399f42 100644
--- a/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@@ -4,16 +4,18 @@ import logging
 from unittest.mock import MagicMock
 
 import arrow
+import pytest
 from pandas import DataFrame
 
-from freqtrade.exceptions import DependencyException
 from freqtrade.configuration import TimeRange
 from freqtrade.data.history import load_data
+from freqtrade.exceptions import DependencyException
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import StrategyResolver
-from .strats.default_strategy import DefaultStrategy
 from tests.conftest import get_patched_exchange, log_has
 
+from .strats.default_strategy import DefaultStrategy
+
 # Avoid to reinit the same object again and again
 _STRATEGY = DefaultStrategy(config={})
 
@@ -123,7 +125,27 @@ def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history):
     )
     assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
                                                   ohlcv_history)
-    assert log_has('Unable to analyze candle (OHLCV) data for pair xyz: Dataframe returned...', caplog)
+    assert log_has('Unable to analyze candle (OHLCV) data for pair xyz: Dataframe returned...',
+                   caplog)
+
+
+def test_assert_df(default_conf, mocker, ohlcv_history):
+    # Ensure it's running when passed correctly
+    _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
+                        ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date'])
+
+    with pytest.raises(DependencyException, match=r"Dataframe returned from strategy.*length\."):
+        _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history) + 1,
+                            ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date'])
+
+    with pytest.raises(DependencyException,
+                       match=r"Dataframe returned from strategy.*last close price\."):
+        _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
+                            ohlcv_history.loc[1, 'close'] + 0.01, ohlcv_history.loc[1, 'date'])
+    with pytest.raises(DependencyException,
+                       match=r"Dataframe returned from strategy.*last date\."):
+        _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
+                            ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date'])
 
 
 def test_get_signal_handles_exceptions(mocker, default_conf):

From a5d00ce7176a78058cb9af3d091a51902fa5b764 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 30 Mar 2020 07:56:17 +0200
Subject: [PATCH 072/151] Remove defaultstrategy occurance from docs

---
 docs/bot-usage.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/bot-usage.md b/docs/bot-usage.md
index 78e137676..60cacfb94 100644
--- a/docs/bot-usage.md
+++ b/docs/bot-usage.md
@@ -145,9 +145,9 @@ It is recommended to use version control to keep track of changes to your strate
 
 This parameter will allow you to load your custom strategy class.
 Per default without `--strategy` or `-s` the bot will load the
-`DefaultStrategy` included with the bot (`freqtrade/strategy/default_strategy.py`).
+`SampleStrategy` installed by the `create-userdir` subcommand (usually `user_data/strategy/sample_strategy.py`).
 
-The bot will search your strategy file within `user_data/strategies` and `freqtrade/strategy`.
+The bot will search your strategy file within `user_data/strategies`.
 
 To load a strategy, simply pass the class name (e.g.: `CustomStrategy`) in this parameter.
 

From f1b92e2569f8db26fb092ad3e579d94542d28f01 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 30 Mar 2020 08:11:38 +0200
Subject: [PATCH 073/151] Improve wording of documentation

---
 docs/bot-usage.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/bot-usage.md b/docs/bot-usage.md
index 60cacfb94..b1649374a 100644
--- a/docs/bot-usage.md
+++ b/docs/bot-usage.md
@@ -144,10 +144,10 @@ It is recommended to use version control to keep track of changes to your strate
 ### How to use **--strategy**?
 
 This parameter will allow you to load your custom strategy class.
-Per default without `--strategy` or `-s` the bot will load the
-`SampleStrategy` installed by the `create-userdir` subcommand (usually `user_data/strategy/sample_strategy.py`).
+To test the bot installation, you can use the `SampleStrategy` installed by the `create-userdir` subcommand (usually `user_data/strategy/sample_strategy.py`).
 
 The bot will search your strategy file within `user_data/strategies`.
+To use other directories, please read the next section about `--strategy-path`.
 
 To load a strategy, simply pass the class name (e.g.: `CustomStrategy`) in this parameter.
 

From 7e1719cfc71b0e4a26cf3922b6b279a78c4bd326 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 30 Mar 2020 08:56:50 +0000
Subject: [PATCH 074/151] Bump prompt-toolkit from 3.0.4 to 3.0.5

Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.4 to 3.0.5.
- [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases)
- [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG)
- [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.4...3.0.5)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index e490c8927..c3ae57c72 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -30,4 +30,4 @@ flask==1.1.1
 colorama==0.4.3
 # Building config files interactively
 questionary==1.5.1
-prompt-toolkit==3.0.4
+prompt-toolkit==3.0.5

From 7e60e0549afd5a20c8d10178baa33ab89ae72d02 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 30 Mar 2020 08:57:16 +0000
Subject: [PATCH 075/151] Bump flake8-tidy-imports from 4.0.0 to 4.1.0

Bumps [flake8-tidy-imports](https://github.com/adamchainz/flake8-tidy-imports) from 4.0.0 to 4.1.0.
- [Release notes](https://github.com/adamchainz/flake8-tidy-imports/releases)
- [Changelog](https://github.com/adamchainz/flake8-tidy-imports/blob/master/HISTORY.rst)
- [Commits](https://github.com/adamchainz/flake8-tidy-imports/compare/4.0.0...4.1.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-dev.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index a4d83eb4f..01f189c56 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -6,7 +6,7 @@
 coveralls==1.11.1
 flake8==3.7.9
 flake8-type-annotations==0.1.0
-flake8-tidy-imports==4.0.0
+flake8-tidy-imports==4.1.0
 mypy==0.770
 pytest==5.4.1
 pytest-asyncio==0.10.0

From 2de10d4c5665e6183f2d9d72f035f20288beb1af Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 30 Mar 2020 08:58:21 +0000
Subject: [PATCH 076/151] Bump ccxt from 1.24.83 to 1.25.38

Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.24.83 to 1.25.38.
- [Release notes](https://github.com/ccxt/ccxt/releases)
- [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst)
- [Commits](https://github.com/ccxt/ccxt/compare/1.24.83...1.25.38)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index e490c8927..54a738e06 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.24.83
+ccxt==1.25.38
 SQLAlchemy==1.3.15
 python-telegram-bot==12.4.2
 arrow==0.15.5

From d8d6fe3574ae73f8867233fc7f6daeb9cb03a3ec Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 30 Mar 2020 08:58:54 +0000
Subject: [PATCH 077/151] Bump python-telegram-bot from 12.4.2 to 12.5

Bumps [python-telegram-bot](https://github.com/python-telegram-bot/python-telegram-bot) from 12.4.2 to 12.5.
- [Release notes](https://github.com/python-telegram-bot/python-telegram-bot/releases)
- [Changelog](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/CHANGES.rst)
- [Commits](https://github.com/python-telegram-bot/python-telegram-bot/compare/v12.4.2...v12.5)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index e490c8927..57617697f 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -2,7 +2,7 @@
 # mainly used for Raspberry pi installs
 ccxt==1.24.83
 SQLAlchemy==1.3.15
-python-telegram-bot==12.4.2
+python-telegram-bot==12.5
 arrow==0.15.5
 cachetools==4.0.0
 requests==2.23.0

From 54d20cb81c90f567924e4a41a6aabeb19a46b288 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 30 Mar 2020 20:08:07 +0200
Subject: [PATCH 078/151] Plot percent correctly

---
 freqtrade/plot/plotting.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py
index fc8f25612..fac3aa2a3 100644
--- a/freqtrade/plot/plotting.py
+++ b/freqtrade/plot/plotting.py
@@ -136,8 +136,8 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame) -> m
                 df_comb.loc[lowdate, 'cum_profit'],
             ],
             mode='markers',
-            name=f"Max drawdown {max_drawdown:.2f}%",
-            text=f"Max drawdown {max_drawdown:.2f}%",
+            name=f"Max drawdown {max_drawdown * 100:.2f}%",
+            text=f"Max drawdown {max_drawdown * 100:.2f}%",
             marker=dict(
                 symbol='square-open',
                 size=9,

From 45fb4d25abf06484c8a0020102fd9786826eaa8b Mon Sep 17 00:00:00 2001
From: orehunt <basso.bassista@gmail.com>
Date: Tue, 31 Mar 2020 18:47:53 +0200
Subject: [PATCH 079/151] use equality instead of index for row lookups

---
 freqtrade/plot/plotting.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py
index fc8f25612..b311c591a 100644
--- a/freqtrade/plot/plotting.py
+++ b/freqtrade/plot/plotting.py
@@ -132,8 +132,8 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame) -> m
         drawdown = go.Scatter(
             x=[highdate, lowdate],
             y=[
-                df_comb.loc[highdate, 'cum_profit'],
-                df_comb.loc[lowdate, 'cum_profit'],
+                df_comb.loc[df_comb.index == highdate, 'cum_profit'],
+                df_comb.loc[df_comb.index == lowdate, 'cum_profit'],
             ],
             mode='markers',
             name=f"Max drawdown {max_drawdown:.2f}%",

From cf6e6488c72c58e80567bdc90a27de0550c9154b Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 2 Apr 2020 17:29:18 +0200
Subject: [PATCH 080/151] Fix filename handling with --strategy-list

---
 freqtrade/optimize/optimize_reports.py  | 7 ++++---
 tests/optimize/test_optimize_reports.py | 9 +++++++--
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py
index 251da9159..646afb5df 100644
--- a/freqtrade/optimize/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports.py
@@ -24,13 +24,14 @@ def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame
                    for index, t in results.iterrows()]
 
         if records:
+            filename = recordfilename
             if len(all_results) > 1:
                 # Inject strategy to filename
-                recordfilename = Path.joinpath(
+                filename = Path.joinpath(
                     recordfilename.parent,
                     f'{recordfilename.stem}-{strategy}').with_suffix(recordfilename.suffix)
-            logger.info(f'Dumping backtest results to {recordfilename}')
-            file_dump_json(recordfilename, records)
+            logger.info(f'Dumping backtest results to {filename}')
+            file_dump_json(filename, records)
 
 
 def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_trades: int,
diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py
index f19668459..3bbc396c8 100644
--- a/tests/optimize/test_optimize_reports.py
+++ b/tests/optimize/test_optimize_reports.py
@@ -160,10 +160,15 @@ def test_backtest_record(default_conf, fee, mocker):
     # reset test to test with strategy name
     names = []
     records = []
-    results['Strat'] = pd.DataFrame()
+    results['Strat'] = results['DefStrat']
+    results['Strat2'] = results['DefStrat']
     store_backtest_result(Path("backtest-result.json"), results)
     # Assert file_dump_json was only called once
-    assert names == [Path('backtest-result-DefStrat.json')]
+    assert names == [
+        Path('backtest-result-DefStrat.json'),
+        Path('backtest-result-Strat.json'),
+        Path('backtest-result-Strat2.json'),
+    ]
     records = records[0]
     # Ensure records are of correct type
     assert len(records) == 4

From 3fcd531eacbda7ca89e64ff68e0185efc8bd621d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 4 Nov 2019 20:19:43 +0100
Subject: [PATCH 081/151] Copy dataframe in interfac.py (reduces memory
 consumption)

---
 freqtrade/strategy/interface.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index 89a38bf54..6307e664e 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -468,7 +468,7 @@ class IStrategy(ABC):
         Creates a dataframe and populates indicators for given candle (OHLCV) data
         Used by optimize operations only, not during dry / live runs.
         """
-        return {pair: self.advise_indicators(pair_data, {'pair': pair})
+        return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair})
                 for pair, pair_data in data.items()}
 
     def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

From de471862635f7344924c47ae91d2b00bd7a3c4e3 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 25 Jan 2020 11:42:31 +0100
Subject: [PATCH 082/151] Use .loc for assignments

---
 freqtrade/optimize/backtesting.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 1725a7d13..f29f599a6 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -149,8 +149,8 @@ class Backtesting:
 
             # To avoid using data from future, we use buy/sell signals shifted
             # from the previous candle
-            df_analyzed.loc[:, 'buy'] = df_analyzed['buy'].shift(1)
-            df_analyzed.loc[:, 'sell'] = df_analyzed['sell'].shift(1)
+            df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1)
+            df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1)
 
             df_analyzed.drop(df_analyzed.head(1).index, inplace=True)
 

From c465552df4b37be165157a1ad6c7e9e22e92f4b2 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 2 Apr 2020 20:17:54 +0200
Subject: [PATCH 083/151] Update comment to mention .copy() usage

---
 freqtrade/strategy/interface.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index 6307e664e..c6f711b74 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -467,6 +467,9 @@ class IStrategy(ABC):
         """
         Creates a dataframe and populates indicators for given candle (OHLCV) data
         Used by optimize operations only, not during dry / live runs.
+        Using .copy() to get a fresh copy of the dataframe for every strategy run.
+        Has positive effects on memory usage for whatever reason - also when
+        using only one strategy.
         """
         return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair})
                 for pair, pair_data in data.items()}

From d4dde011405365fced9a93b2c916e31a66233344 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 2 Apr 2020 20:23:20 +0200
Subject: [PATCH 084/151] Add test

---
 tests/strategy/test_interface.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py
index 8bc399f42..1c31aeb6a 100644
--- a/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@@ -168,6 +168,19 @@ def test_ohlcvdata_to_dataframe(default_conf, testdatadir) -> None:
     assert len(processed['UNITTEST/BTC']) == 102  # partial candle was removed
 
 
+def test_ohlcvdata_to_dataframe_copy(mocker, default_conf, testdatadir) -> None:
+    default_conf.update({'strategy': 'DefaultStrategy'})
+    strategy = StrategyResolver.load_strategy(default_conf)
+    aimock = mocker.patch('freqtrade.strategy.interface.IStrategy.advise_indicators')
+    timerange = TimeRange.parse_timerange('1510694220-1510700340')
+    data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange,
+                     fill_up_missing=True)
+    strategy.ohlcvdata_to_dataframe(data)
+    assert aimock.call_count == 1
+    # Ensure that a copy of the dataframe is passed to advice_indicators
+    assert aimock.call_args_list[0][0][0] is not data
+
+
 def test_min_roi_reached(default_conf, fee) -> None:
 
     # Use list to confirm sequence does not matter

From a99c53f1ec4f89cd5fa2fe6c57f958f9fa4fd1cf Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 5 Apr 2020 14:29:03 +0200
Subject: [PATCH 085/151] Add test showing that high is before low

---
 tests/data/test_btanalysis.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 463e5ae36..9f23ecf8f 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -191,3 +191,18 @@ def test_calculate_max_drawdown(testdatadir):
     assert low == Timestamp('2018-01-30 04:45:00', tz='UTC')
     with pytest.raises(ValueError, match='Trade dataframe empty.'):
         drawdown, h, low = calculate_max_drawdown(DataFrame())
+
+
+def test_calculate_max_drawdown2():
+    values = [0.011580, 0.010048, 0.011340, 0.012161, 0.010416, 0.010009, 0.020024,
+              -0.024662, -0.022350, 0.020496, -0.029859, -0.030511, 0.010041, 0.010872,
+              -0.025782, 0.010400, 0.012374, 0.012467, 0.114741, 0.010303, 0.010088,
+              -0.033961, 0.010680, 0.010886, -0.029274, 0.011178, 0.010693, 0.010711]
+
+    dates = [Arrow(2020, 1, 1).shift(days=i) for i in range(len(values))]
+    df = DataFrame(zip(values, dates), columns=['profit', 'open_time'])
+    drawdown, h, low = calculate_max_drawdown(df, date_col='open_time', value_col='profit')
+    assert isinstance(drawdown, float)
+    # High must be before low
+    assert h < low
+    assert drawdown == 0.091755

From e204170eb65055d487ab68e23150c658162d0007 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 5 Apr 2020 14:29:40 +0200
Subject: [PATCH 086/151] Fix max_drawdown bug finding low before high!

---
 freqtrade/data/btanalysis.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py
index 23a9f720c..1ff737b90 100644
--- a/freqtrade/data/btanalysis.py
+++ b/freqtrade/data/btanalysis.py
@@ -219,7 +219,7 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time'
     max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
     max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
 
-    high_date = profit_results.loc[max_drawdown_df['high_value'].idxmax(), date_col]
-    low_date = profit_results.loc[max_drawdown_df['drawdown'].idxmin(), date_col]
-
+    idxmin = max_drawdown_df['drawdown'].idxmin()
+    high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
+    low_date = profit_results.loc[idxmin, date_col]
     return abs(min(max_drawdown_df['drawdown'])), high_date, low_date

From 4e907e2304da16c5ed28f93aa239f244e4c2a79d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 5 Apr 2020 14:35:53 +0200
Subject: [PATCH 087/151] Use timeframe_to_prev_date to move trade-date to
 candle

---
 freqtrade/plot/plotting.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py
index b311c591a..5da067069 100644
--- a/freqtrade/plot/plotting.py
+++ b/freqtrade/plot/plotting.py
@@ -10,6 +10,7 @@ from freqtrade.data.btanalysis import (calculate_max_drawdown,
                                        create_cum_profit,
                                        extract_trades_of_period, load_trades)
 from freqtrade.data.converter import trim_dataframe
+from freqtrade.exchange import timeframe_to_prev_date
 from freqtrade.data.history import load_data
 from freqtrade.misc import pair_to_filename
 from freqtrade.resolvers import StrategyResolver
@@ -122,7 +123,8 @@ def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> make_sub
     return fig
 
 
-def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame) -> make_subplots:
+def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame,
+                     timeframe: str) -> make_subplots:
     """
     Add scatter points indicating max drawdown
     """
@@ -132,8 +134,8 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame) -> m
         drawdown = go.Scatter(
             x=[highdate, lowdate],
             y=[
-                df_comb.loc[df_comb.index == highdate, 'cum_profit'],
-                df_comb.loc[df_comb.index == lowdate, 'cum_profit'],
+                df_comb.loc[timeframe_to_prev_date(timeframe, highdate), 'cum_profit'],
+                df_comb.loc[timeframe_to_prev_date(timeframe, lowdate), 'cum_profit'],
             ],
             mode='markers',
             name=f"Max drawdown {max_drawdown:.2f}%",
@@ -405,7 +407,7 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
 
     fig.add_trace(avgclose, 1, 1)
     fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
-    fig = add_max_drawdown(fig, 2, trades, df_comb)
+    fig = add_max_drawdown(fig, 2, trades, df_comb, timeframe)
 
     for pair in pairs:
         profit_col = f'cum_profit_{pair}'

From 41d5c40f1099a0f35c9906a6554edb7ff1de9015 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 5 Apr 2020 14:43:01 +0200
Subject: [PATCH 088/151] Correctly test drawdown plot

---
 freqtrade/data/btanalysis.py  | 2 ++
 tests/data/test_btanalysis.py | 4 ++++
 tests/test_plotting.py        | 4 ++--
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py
index 1ff737b90..4505ea52a 100644
--- a/freqtrade/data/btanalysis.py
+++ b/freqtrade/data/btanalysis.py
@@ -220,6 +220,8 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time'
     max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
 
     idxmin = max_drawdown_df['drawdown'].idxmin()
+    if idxmin == 0:
+        raise ValueError("No losing trade, therefore no drawdown.")
     high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
     low_date = profit_results.loc[idxmin, date_col]
     return abs(min(max_drawdown_df['drawdown'])), high_date, low_date
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 9f23ecf8f..4eec20976 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -206,3 +206,7 @@ def test_calculate_max_drawdown2():
     # High must be before low
     assert h < low
     assert drawdown == 0.091755
+
+    df = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_time'])
+    with pytest.raises(ValueError, match='No losing trade, therefore no drawdown.'):
+        calculate_max_drawdown(df, date_col='open_time', value_col='profit')
diff --git a/tests/test_plotting.py b/tests/test_plotting.py
index a5c965429..0258b94d1 100644
--- a/tests/test_plotting.py
+++ b/tests/test_plotting.py
@@ -266,7 +266,7 @@ def test_generate_profit_graph(testdatadir):
     filename = testdatadir / "backtest-result_test.json"
     trades = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
-    pairs = ["TRX/BTC", "ADA/BTC"]
+    pairs = ["TRX/BTC", "XLM/BTC"]
     trades = trades[trades['close_time'] < pd.Timestamp('2018-01-12', tz='UTC')]
 
     data = history.load_data(datadir=testdatadir,
@@ -292,7 +292,7 @@ def test_generate_profit_graph(testdatadir):
 
     profit = find_trace_in_fig_data(figure.data, "Profit")
     assert isinstance(profit, go.Scatter)
-    profit = find_trace_in_fig_data(figure.data, "Max drawdown 0.00%")
+    profit = find_trace_in_fig_data(figure.data, "Max drawdown 10.45%")
     assert isinstance(profit, go.Scatter)
 
     for pair in pairs:

From 0a14d5ec467bdab197d1d100446f7c4dd5fdb698 Mon Sep 17 00:00:00 2001
From: Ork Blutt <orkblutt@msn.com>
Date: Sun, 5 Apr 2020 16:14:02 +0200
Subject: [PATCH 089/151] trades history RPC

---
 freqtrade/rpc/api_server.py | 16 ++++++++++++-
 freqtrade/rpc/rpc.py        | 47 +++++++++++++++++++++++++++++++++++++
 scripts/rest_client.py      |  7 ++++++
 3 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py
index 8f4cc4787..7e0cdd71d 100644
--- a/freqtrade/rpc/api_server.py
+++ b/freqtrade/rpc/api_server.py
@@ -173,7 +173,8 @@ class ApiServer(RPC):
                               view_func=self._show_config, methods=['GET'])
         self.app.add_url_rule(f'{BASE_URI}/ping', 'ping',
                               view_func=self._ping, methods=['GET'])
-
+        self.app.add_url_rule(f'{BASE_URI}/trades', 'trades',
+                              view_func=self._trades, methods=['GET'])
         # Combined actions and infos
         self.app.add_url_rule(f'{BASE_URI}/blacklist', 'blacklist', view_func=self._blacklist,
                               methods=['GET', 'POST'])
@@ -357,6 +358,19 @@ class ApiServer(RPC):
         results = self._rpc_balance(self._config['stake_currency'],
                                     self._config.get('fiat_display_currency', ''))
         return self.rest_dump(results)
+    
+    @require_login
+    @rpc_catch_errors
+    def _trades(self):
+        """
+        Handler for /trades.
+
+        Returns the X last trades in json format
+        """
+        last_trades_number = request.args.get('last_trades_number', 0)
+        last_trades_number = int(last_trades_number)
+        results = self._rpc_trade_history(last_trades_number)
+        return self.rest_dump(results)
 
     @require_login
     @rpc_catch_errors
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index a0f50b070..b0c045e4a 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -226,6 +226,53 @@ class RPC:
             for key, value in profit_days.items()
         ]
 
+    def _rpc_trade_history(
+            self, last_trades_number: int) -> List[List[Any]]:
+        """ Returns the X last trades """
+        if last_trades_number > 0:
+            trades = Trade.get_trades().order_by(Trade.id.desc()).limit(last_trades_number)
+        else:
+            trades = Trade.get_trades().order_by(Trade.id.desc()).all()
+
+        output = []
+
+        for trade in trades:
+            output.append({
+                'id': trade.id,
+                'pair': trade.pair,
+                'exchange': trade.exchange,
+                'is_open': trade.is_open if trade.is_open is not None else 0,
+                'open_rate': trade.open_rate,
+                'close_rate': trade.close_rate,
+                'fee_open': trade.fee_open,
+                'fee_close': trade.fee_close,
+                'open_rate_requested': trade.open_rate_requested,
+                'open_trade_price': trade.open_trade_price,
+                'close_rate_requested': trade.close_rate_requested,
+                'close_profit': trade.close_profit,
+                'close_profit_abs': trade.close_profit_abs,
+                'stake_amount': trade.stake_amount,
+                'amount': trade.amount,
+                'open_date': trade.open_date,
+                'close_date': trade.close_date,
+                'open_order_id': trade.open_order_id,
+                'stop_loss': trade.stop_loss,
+                'stop_loss_pct': trade.stop_loss_pct,
+                'initial_stop_loss': trade.initial_stop_loss,
+                'initial_stop_loss_pct': trade.initial_stop_loss_pct,
+                'stoploss_order_id': trade.stoploss_order_id,
+                'stoploss_last_update': trade.stoploss_last_update,
+                'max_rate': trade.max_rate,
+                'min_rate': trade.min_rate,
+                'sell_reason': trade.sell_reason,
+                'strategy': trade.strategy,
+                'ticker_interval': trade.ticker_interval,
+            })
+
+        return {
+            "trades" : output
+        }
+
     def _rpc_trade_statistics(
             self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]:
         """ Returns cumulative profit statistics """
diff --git a/scripts/rest_client.py b/scripts/rest_client.py
index ccb33604f..5cbdd8e07 100755
--- a/scripts/rest_client.py
+++ b/scripts/rest_client.py
@@ -156,6 +156,13 @@ class FtRestClient():
         """
         return self._get("show_config")
 
+    def history(self, number=None):
+        """Return the amount of open trades.
+
+        :return: json object
+        """
+        return self._get("trades", params={"last_trades_number": number} if number else 0)
+
     def whitelist(self):
         """Show the current whitelist.
 

From 15c45b984d20d3079875d4b1480e9bac0067b7cd Mon Sep 17 00:00:00 2001
From: Ork Blutt <orkblutt@msn.com>
Date: Sun, 5 Apr 2020 16:47:46 +0200
Subject: [PATCH 090/151] removing whitespace

---
 freqtrade/rpc/api_server.py | 2 +-
 freqtrade/rpc/rpc.py        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py
index 7e0cdd71d..d2fb5bfad 100644
--- a/freqtrade/rpc/api_server.py
+++ b/freqtrade/rpc/api_server.py
@@ -358,7 +358,7 @@ class ApiServer(RPC):
         results = self._rpc_balance(self._config['stake_currency'],
                                     self._config.get('fiat_display_currency', ''))
         return self.rest_dump(results)
-    
+
     @require_login
     @rpc_catch_errors
     def _trades(self):
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index b0c045e4a..a3506816b 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -270,7 +270,7 @@ class RPC:
             })
 
         return {
-            "trades" : output
+            "trades": output
         }
 
     def _rpc_trade_statistics(

From 8555c5b2110b29a65d20843fa1a66bf79a3a8219 Mon Sep 17 00:00:00 2001
From: Ork Blutt <orkblutt@msn.com>
Date: Sun, 5 Apr 2020 17:03:51 +0200
Subject: [PATCH 091/151] fix return value

---
 freqtrade/rpc/rpc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index a3506816b..01e593e82 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -227,7 +227,7 @@ class RPC:
         ]
 
     def _rpc_trade_history(
-            self, last_trades_number: int) -> List[List[Any]]:
+            self, last_trades_number: int) -> Dict[str, List[Dict[str, Any]]]:
         """ Returns the X last trades """
         if last_trades_number > 0:
             trades = Trade.get_trades().order_by(Trade.id.desc()).limit(last_trades_number)

From 6256025c73ae59ce9f9f3c9e52216bd157db0a22 Mon Sep 17 00:00:00 2001
From: Ork Blutt <orkblutt@msn.com>
Date: Mon, 6 Apr 2020 11:00:31 +0200
Subject: [PATCH 092/151] various adjustement from PR discussion

---
 freqtrade/persistence.py    | 15 +++++++++++++-
 freqtrade/rpc/api_server.py |  5 ++---
 freqtrade/rpc/rpc.py        | 41 ++++++-------------------------------
 scripts/rest_client.py      |  7 ++++---
 4 files changed, 26 insertions(+), 42 deletions(-)

diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py
index 0d668596c..b0ef4bd8f 100644
--- a/freqtrade/persistence.py
+++ b/freqtrade/persistence.py
@@ -188,7 +188,7 @@ class Trade(_DECL_BASE):
     fee_close = Column(Float, nullable=False, default=0.0)
     open_rate = Column(Float)
     open_rate_requested = Column(Float)
-    # open_trade_price - calcuated via _calc_open_trade_price
+    # open_trade_price - calculated via _calc_open_trade_price
     open_trade_price = Column(Float)
     close_rate = Column(Float)
     close_rate_requested = Column(Float)
@@ -233,6 +233,9 @@ class Trade(_DECL_BASE):
         return {
             'trade_id': self.id,
             'pair': self.pair,
+            'is_open': self.is_open,
+            'fee_open': self.fee_open,
+            'fee_close': self.fee_close,
             'open_date_hum': arrow.get(self.open_date).humanize(),
             'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"),
             'close_date_hum': (arrow.get(self.close_date).humanize()
@@ -240,14 +243,24 @@ class Trade(_DECL_BASE):
             'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S")
                            if self.close_date else None),
             'open_rate': self.open_rate,
+            'open_rate_requested': self.open_rate_requested,
+            'open_trade_price': self.open_trade_price,
             'close_rate': self.close_rate,
+            'close_rate_requested': self.close_rate_requested,
             'amount': round(self.amount, 8),
             'stake_amount': round(self.stake_amount, 8),
+            'close_profit': self.close_profit,
+            'sell_reason': self.sell_reason,
             'stop_loss': self.stop_loss,
             'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None,
             'initial_stop_loss': self.initial_stop_loss,
             'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100
                                       if self.initial_stop_loss_pct else None),
+            'min_rate': self.min_rate,
+            'max_rate': self.max_rate,
+            'strategy': self.strategy,
+            'ticker_interval': self.ticker_interval,
+            'open_order_id': self.open_order_id,
         }
 
     def adjust_min_max_rates(self, current_price: float) -> None:
diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py
index d2fb5bfad..0335bb151 100644
--- a/freqtrade/rpc/api_server.py
+++ b/freqtrade/rpc/api_server.py
@@ -367,9 +367,8 @@ class ApiServer(RPC):
 
         Returns the X last trades in json format
         """
-        last_trades_number = request.args.get('last_trades_number', 0)
-        last_trades_number = int(last_trades_number)
-        results = self._rpc_trade_history(last_trades_number)
+        limit = int(request.args.get('limit', 0))
+        results = self._rpc_trade_history(limit)
         return self.rest_dump(results)
 
     @require_login
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 01e593e82..b78856265 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -227,50 +227,21 @@ class RPC:
         ]
 
     def _rpc_trade_history(
-            self, last_trades_number: int) -> Dict[str, List[Dict[str, Any]]]:
+            self, limit: int) -> Dict[str, List[Dict[str, Any]]]:
         """ Returns the X last trades """
-        if last_trades_number > 0:
-            trades = Trade.get_trades().order_by(Trade.id.desc()).limit(last_trades_number)
+        if limit > 0:
+            trades = Trade.get_trades().order_by(Trade.id.desc()).limit(limit)
         else:
             trades = Trade.get_trades().order_by(Trade.id.desc()).all()
 
         output = []
 
         for trade in trades:
-            output.append({
-                'id': trade.id,
-                'pair': trade.pair,
-                'exchange': trade.exchange,
-                'is_open': trade.is_open if trade.is_open is not None else 0,
-                'open_rate': trade.open_rate,
-                'close_rate': trade.close_rate,
-                'fee_open': trade.fee_open,
-                'fee_close': trade.fee_close,
-                'open_rate_requested': trade.open_rate_requested,
-                'open_trade_price': trade.open_trade_price,
-                'close_rate_requested': trade.close_rate_requested,
-                'close_profit': trade.close_profit,
-                'close_profit_abs': trade.close_profit_abs,
-                'stake_amount': trade.stake_amount,
-                'amount': trade.amount,
-                'open_date': trade.open_date,
-                'close_date': trade.close_date,
-                'open_order_id': trade.open_order_id,
-                'stop_loss': trade.stop_loss,
-                'stop_loss_pct': trade.stop_loss_pct,
-                'initial_stop_loss': trade.initial_stop_loss,
-                'initial_stop_loss_pct': trade.initial_stop_loss_pct,
-                'stoploss_order_id': trade.stoploss_order_id,
-                'stoploss_last_update': trade.stoploss_last_update,
-                'max_rate': trade.max_rate,
-                'min_rate': trade.min_rate,
-                'sell_reason': trade.sell_reason,
-                'strategy': trade.strategy,
-                'ticker_interval': trade.ticker_interval,
-            })
+            output.append(trade.to_json())
 
         return {
-            "trades": output
+            "trades": output,
+            "trades_count": len(output)
         }
 
     def _rpc_trade_statistics(
diff --git a/scripts/rest_client.py b/scripts/rest_client.py
index 5cbdd8e07..116c00063 100755
--- a/scripts/rest_client.py
+++ b/scripts/rest_client.py
@@ -156,12 +156,13 @@ class FtRestClient():
         """
         return self._get("show_config")
 
-    def history(self, number=None):
-        """Return the amount of open trades.
+    def history(self, limit=None):
+        """Return trades history.
 
+        :param limit: Limits trades to the X last trades . No limit to get all the trades.
         :return: json object
         """
-        return self._get("trades", params={"last_trades_number": number} if number else 0)
+        return self._get("trades", params={"limit": limit} if limit else 0)
 
     def whitelist(self):
         """Show the current whitelist.

From a1e81a51efcc8497e1cdae8cb0303d894b6f2a17 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 6 Apr 2020 09:09:54 +0000
Subject: [PATCH 093/151] Bump ccxt from 1.25.38 to 1.25.81

Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.25.38 to 1.25.81.
- [Release notes](https://github.com/ccxt/ccxt/releases)
- [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst)
- [Commits](https://github.com/ccxt/ccxt/compare/1.25.38...1.25.81)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 3c9802990..7e873aa83 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.25.38
+ccxt==1.25.81
 SQLAlchemy==1.3.15
 python-telegram-bot==12.5
 arrow==0.15.5

From 5de7ee3bdb1f11d3ae082face12a960c804136db Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 6 Apr 2020 09:10:07 +0000
Subject: [PATCH 094/151] Bump plotly from 4.5.4 to 4.6.0

Bumps [plotly](https://github.com/plotly/plotly.py) from 4.5.4 to 4.6.0.
- [Release notes](https://github.com/plotly/plotly.py/releases)
- [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md)
- [Commits](https://github.com/plotly/plotly.py/compare/v4.5.4...v4.6.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-plot.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-plot.txt b/requirements-plot.txt
index 7a5b21e2d..3db48a201 100644
--- a/requirements-plot.txt
+++ b/requirements-plot.txt
@@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==4.5.4
+plotly==4.6.0
 

From 144d252e19bff28837f3f22fb29b3d42024e99eb Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 6 Apr 2020 09:10:30 +0000
Subject: [PATCH 095/151] Bump flask from 1.1.1 to 1.1.2

Bumps [flask](https://github.com/pallets/flask) from 1.1.1 to 1.1.2.
- [Release notes](https://github.com/pallets/flask/releases)
- [Changelog](https://github.com/pallets/flask/blob/master/CHANGES.rst)
- [Commits](https://github.com/pallets/flask/compare/1.1.1...1.1.2)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 3c9802990..310383680 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -24,7 +24,7 @@ python-rapidjson==0.9.1
 sdnotify==0.3.2
 
 # Api server
-flask==1.1.1
+flask==1.1.2
 
 # Support for colorized terminal output
 colorama==0.4.3

From 0b30cb7f8f42fd0e1fe430c3ec5299fda03dec09 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 6 Apr 2020 09:11:33 +0000
Subject: [PATCH 096/151] Bump pytest-mock from 2.0.0 to 3.0.0

Bumps [pytest-mock](https://github.com/pytest-dev/pytest-mock) from 2.0.0 to 3.0.0.
- [Release notes](https://github.com/pytest-dev/pytest-mock/releases)
- [Changelog](https://github.com/pytest-dev/pytest-mock/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest-mock/compare/v2.0.0...v3.0.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-dev.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 01f189c56..814003cbf 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -11,7 +11,7 @@ mypy==0.770
 pytest==5.4.1
 pytest-asyncio==0.10.0
 pytest-cov==2.8.1
-pytest-mock==2.0.0
+pytest-mock==3.0.0
 pytest-random-order==1.0.4
 
 # Convert jupyter notebooks to markdown documents

From be76e3c55482e9ae55bc820ca8b8e53dec247b75 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 6 Apr 2020 09:23:21 +0000
Subject: [PATCH 097/151] Bump python-telegram-bot from 12.5 to 12.5.1

Bumps [python-telegram-bot](https://github.com/python-telegram-bot/python-telegram-bot) from 12.5 to 12.5.1.
- [Release notes](https://github.com/python-telegram-bot/python-telegram-bot/releases)
- [Changelog](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/CHANGES.rst)
- [Commits](https://github.com/python-telegram-bot/python-telegram-bot/compare/v12.5...v12.5.1)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 7e873aa83..7f2b39ff2 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -2,7 +2,7 @@
 # mainly used for Raspberry pi installs
 ccxt==1.25.81
 SQLAlchemy==1.3.15
-python-telegram-bot==12.5
+python-telegram-bot==12.5.1
 arrow==0.15.5
 cachetools==4.0.0
 requests==2.23.0

From 815660c0700726d7c7cb9a8b82bf9e76186f016f Mon Sep 17 00:00:00 2001
From: Ork Blutt <orkblutt@msn.com>
Date: Mon, 6 Apr 2020 11:32:00 +0200
Subject: [PATCH 098/151] fix tests

---
 tests/rpc/test_rpc.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py
index 47ffb771b..875a234b2 100644
--- a/tests/rpc/test_rpc.py
+++ b/tests/rpc/test_rpc.py
@@ -49,6 +49,18 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'base_currency': 'BTC',
         'open_date': ANY,
         'open_date_hum': ANY,
+        'is_open': ANY,
+        'fee_open': ANY,
+        'fee_close': ANY,
+        'open_rate_requested': ANY,
+        'open_trade_price': ANY,
+        'close_rate_requested': ANY,
+        'sell_reason': ANY,
+        'min_rate': ANY,
+        'max_rate': ANY,
+        'strategy': ANY,
+        'ticker_interval': ANY,
+        'open_order_id': ANY,
         'close_date': None,
         'close_date_hum': None,
         'open_rate': 1.098e-05,
@@ -76,6 +88,18 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'base_currency': 'BTC',
         'open_date': ANY,
         'open_date_hum': ANY,
+        'is_open': ANY,
+        'fee_open': ANY,
+        'fee_close': ANY,
+        'open_rate_requested': ANY,
+        'open_trade_price': ANY,
+        'close_rate_requested': ANY,
+        'sell_reason': ANY,
+        'min_rate': ANY,
+        'max_rate': ANY,
+        'strategy': ANY,
+        'ticker_interval': ANY,
+        'open_order_id': ANY,
         'close_date': None,
         'close_date_hum': None,
         'open_rate': 1.098e-05,

From d5609d49972f2d8ddc873b945090bba2635db00f Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Mon, 6 Apr 2020 13:12:32 +0200
Subject: [PATCH 099/151] Changed back to progressbar2 for better handling of
 logger. Coloring still needs some work (bug + what colors to use)

---
 freqtrade/optimize/hyperopt.py | 107 ++++++++++++++++++---------------
 requirements-hyperopt.txt      |   2 +-
 setup.py                       |   1 +
 3 files changed, 59 insertions(+), 51 deletions(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index a5921703f..1adaf54e1 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -22,7 +22,7 @@ from colorama import init as colorama_init
 from joblib import (Parallel, cpu_count, delayed, dump, load,
                     wrap_non_picklable_objects)
 from pandas import DataFrame, json_normalize, isna
-from tqdm import tqdm
+import progressbar
 import tabulate
 from os import path
 import io
@@ -44,7 +44,8 @@ with warnings.catch_warnings():
     from skopt import Optimizer
     from skopt.space import Dimension
 
-
+progressbar.streams.wrap_stderr()
+progressbar.streams.wrap_stdout()
 logger = logging.getLogger(__name__)
 
 
@@ -682,55 +683,67 @@ class Hyperopt:
                 logger.info(f'Effective number of parallel workers used: {jobs}')
 
                 # Define progressbar
-                self.progress_bar = tqdm(
-                    total=self.total_epochs, ncols=108, unit=' Epoch',
-                    bar_format='Epoch {n_fmt}/{total_fmt} ({percentage:3.0f}%)|{bar}|'
-                    ' [{elapsed}<{remaining} {rate_fmt}{postfix}]'
-                )
+                if self.print_colorized:
+                    widgets = [
+                        ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
+                        ' (', progressbar.Percentage(), ')] ',
+                        progressbar.Bar(marker=progressbar.AnimatedMarker(
+                            fill='█',
+                            fill_wrap='\x1b[32m{}\x1b[39m',
+                            marker_wrap='\x1b[31m{}\x1b[39m',
+                        )),
+                        ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
+                    ]
+                else:
+                    widgets = [
+                        ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), '] ',
+                        progressbar.Bar(marker='█'),
+                        ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
+                    ]
+                with progressbar.ProgressBar(
+                         maxval=self.total_epochs, redirect_stdout=True, redirect_stderr=True,
+                         widgets=widgets
+                     ) as pbar:
+                    EVALS = ceil(self.total_epochs / jobs)
+                    for i in range(EVALS):
+                        # Correct the number of epochs to be processed for the last
+                        # iteration (should not exceed self.total_epochs in total)
+                        n_rest = (i + 1) * jobs - self.total_epochs
+                        current_jobs = jobs - n_rest if n_rest > 0 else jobs
 
-                EVALS = ceil(self.total_epochs / jobs)
-                for i in range(EVALS):
-                    # Correct the number of epochs to be processed for the last
-                    # iteration (should not exceed self.total_epochs in total)
-                    n_rest = (i + 1) * jobs - self.total_epochs
-                    current_jobs = jobs - n_rest if n_rest > 0 else jobs
+                        asked = self.opt.ask(n_points=current_jobs)
+                        f_val = self.run_optimizer_parallel(parallel, asked, i)
+                        self.opt.tell(asked, [v['loss'] for v in f_val])
+                        self.fix_optimizer_models_list()
 
-                    asked = self.opt.ask(n_points=current_jobs)
-                    f_val = self.run_optimizer_parallel(parallel, asked, i)
-                    self.opt.tell(asked, [v['loss'] for v in f_val])
-                    self.fix_optimizer_models_list()
+                        # Calculate progressbar outputs
+                        for j, val in enumerate(f_val):
+                            # Use human-friendly indexes here (starting from 1)
+                            current = i * jobs + j + 1
+                            val['current_epoch'] = current
+                            val['is_initial_point'] = current <= INITIAL_POINTS
 
-                    # Calculate progressbar outputs
-                    for j, val in enumerate(f_val):
-                        # Use human-friendly indexes here (starting from 1)
-                        current = i * jobs + j + 1
-                        val['current_epoch'] = current
-                        val['is_initial_point'] = current <= INITIAL_POINTS
-                        logger.debug(f"Optimizer epoch evaluated: {val}")
+                            logger.debug(f"Optimizer epoch evaluated: {val}")
 
-                        is_best = self.is_best_loss(val, self.current_best_loss)
-                        # This value is assigned here and not in the optimization method
-                        # to keep proper order in the list of results. That's because
-                        # evaluations can take different time. Here they are aligned in the
-                        # order they will be shown to the user.
-                        val['is_best'] = is_best
-                        output = self.get_results(val)
-                        if output:
-                            self.progress_bar.write(output)
-                        self.progress_bar.ncols = 108
-                        self.progress_bar.update(1)
+                            is_best = self.is_best_loss(val, self.current_best_loss)
+                            # This value is assigned here and not in the optimization method
+                            # to keep proper order in the list of results. That's because
+                            # evaluations can take different time. Here they are aligned in the
+                            # order they will be shown to the user.
+                            val['is_best'] = is_best
+                            self.print_results(val)
 
-                        if is_best:
-                            self.current_best_loss = val['loss']
-                        self.trials.append(val)
-                        # Save results after each best epoch and every 100 epochs
-                        if is_best or current % 100 == 0:
-                            self.save_trials()
-                self.progress_bar.ncols = 108
-                self.progress_bar.close()
+                            if is_best:
+                                self.current_best_loss = val['loss']
+                            self.trials.append(val)
+
+                            # Save results after each best epoch and every 100 epochs
+                            if is_best or current % 100 == 0:
+                                self.save_trials()
+
+                            pbar.update(current)
 
         except KeyboardInterrupt:
-            self.progress_bar.close()
             print('User interrupted..')
 
         self.save_trials(final=True)
@@ -743,9 +756,3 @@ class Hyperopt:
             # This is printed when Ctrl+C is pressed quickly, before first epochs have
             # a chance to be evaluated.
             print("No epochs evaluated yet, no best result.")
-
-    def __getstate__(self):
-        state = self.__dict__.copy()
-        del state['trials']
-        del state['progress_bar']
-        return state
diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt
index 7469674cd..6df1eb157 100644
--- a/requirements-hyperopt.txt
+++ b/requirements-hyperopt.txt
@@ -7,4 +7,4 @@ scikit-learn==0.22.2.post1
 scikit-optimize==0.7.4
 filelock==3.0.12
 joblib==0.14.1
-tqdm==4.43.0
+progressbar2==3.50.1
diff --git a/setup.py b/setup.py
index 7890f862e..94c48a6a7 100644
--- a/setup.py
+++ b/setup.py
@@ -24,6 +24,7 @@ hyperopt = [
     'scikit-optimize',
     'filelock',
     'joblib',
+    'progressbar2',
     ]
 
 develop = [

From c1f9595086b3a3b2f7871ac0818c878a9c11b876 Mon Sep 17 00:00:00 2001
From: Ork Blutt <orkblutt@msn.com>
Date: Mon, 6 Apr 2020 15:49:24 +0200
Subject: [PATCH 100/151] fix broken tests

---
 tests/rpc/test_rpc_apiserver.py | 33 +++++++++++++++++++++++++++++++--
 tests/test_persistence.py       | 30 ++++++++++++++++++++++++++++--
 2 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py
index e0abd886d..79073825d 100644
--- a/tests/rpc/test_rpc_apiserver.py
+++ b/tests/rpc/test_rpc_apiserver.py
@@ -444,7 +444,22 @@ def test_api_status(botclient, mocker, ticker, fee, markets):
                         'stake_amount': 0.001,
                         'stop_loss': 0.0,
                         'stop_loss_pct': None,
-                        'trade_id': 1}]
+                        'trade_id': 1,
+                        'close_rate_requested': None,
+                        'current_profit': -0.41,
+                        'current_rate': 1.099e-05,
+                        'fee_close': 0.0025,
+                        'fee_open': 0.0025,
+                        'open_date': ANY,
+                        'is_open': True,
+                        'max_rate': 0.0,
+                        'min_rate': None,
+                        'open_order_id': ANY,
+                        'open_rate_requested': 1.098e-05,
+                        'open_trade_price': 0.0010025,
+                        'sell_reason': None,
+                        'strategy': 'DefaultStrategy',
+                        'ticker_interval': 5}]
 
 
 def test_api_version(botclient):
@@ -533,7 +548,21 @@ def test_api_forcebuy(botclient, mocker, fee):
                        'stake_amount': 1,
                        'stop_loss': None,
                        'stop_loss_pct': None,
-                       'trade_id': None}
+                       'trade_id': None,
+                       'close_profit': None,
+                       'close_rate_requested': None,
+                       'fee_close': 0.0025,
+                       'fee_open': 0.0025,
+                       'is_open': False,
+                       'max_rate': None,
+                       'min_rate': None,
+                       'open_order_id': '123456',
+                       'open_rate_requested': None,
+                       'open_trade_price': 0.2460546025,
+                       'sell_reason': None,
+                       'strategy': None,
+                       'ticker_interval': None
+                       }
 
 
 def test_api_forcesell(botclient, mocker, ticker, fee, markets):
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 991922cba..c6de10e3d 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -777,18 +777,31 @@ def test_to_json(default_conf, fee):
 
     assert result == {'trade_id': None,
                       'pair': 'ETH/BTC',
+                      'is_open': None,
                       'open_date_hum': '2 hours ago',
                       'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"),
+                      'open_order_id': 'dry_run_buy_12345',
                       'close_date_hum': None,
                       'close_date': None,
                       'open_rate': 0.123,
+                      'open_rate_requested': None,
+                      'open_trade_price': 15.1668225,
+                      'fee_close': 0.0025,
+                      'fee_open': 0.0025,
                       'close_rate': None,
+                      'close_rate_requested': None,
                       'amount': 123.0,
                       'stake_amount': 0.001,
+                      'close_profit': None,
+                      'sell_reason': None,
                       'stop_loss': None,
                       'stop_loss_pct': None,
                       'initial_stop_loss': None,
-                      'initial_stop_loss_pct': None}
+                      'initial_stop_loss_pct': None,
+                      'min_rate': None,
+                      'max_rate': None,
+                      'strategy': None,
+                      'ticker_interval': None}
 
     # Simulate dry_run entries
     trade = Trade(
@@ -819,7 +832,20 @@ def test_to_json(default_conf, fee):
                       'stop_loss': None,
                       'stop_loss_pct': None,
                       'initial_stop_loss': None,
-                      'initial_stop_loss_pct': None}
+                      'initial_stop_loss_pct': None,
+                      'close_profit': None,
+                      'close_rate_requested': None,
+                      'fee_close': 0.0025,
+                      'fee_open': 0.0025,
+                      'is_open': None,
+                      'max_rate': None,
+                      'min_rate': None,
+                      'open_order_id': None,
+                      'open_rate_requested': None,
+                      'open_trade_price': 12.33075,
+                      'sell_reason': None,
+                      'strategy': None,
+                      'ticker_interval': None}
 
 
 def test_stoploss_reinitialization(default_conf, fee):

From 20abb379aaeaea97f63b193d5e1a2caddb892533 Mon Sep 17 00:00:00 2001
From: orehunt <basso.bassista@gmail.com>
Date: Mon, 6 Apr 2020 15:49:59 +0200
Subject: [PATCH 101/151] trim trades to the available ohlcv data before
 plotting profits

---
 freqtrade/data/btanalysis.py | 13 ++++++++++---
 freqtrade/plot/plotting.py   |  3 +++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py
index 23a9f720c..780980ad6 100644
--- a/freqtrade/data/btanalysis.py
+++ b/freqtrade/data/btanalysis.py
@@ -151,13 +151,20 @@ def load_trades(source: str, db_url: str, exportfilename: Path,
         return load_backtest_data(exportfilename)
 
 
-def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame:
+def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
+                             date_index=False) -> pd.DataFrame:
     """
     Compare trades and backtested pair DataFrames to get trades performed on backtested period
     :return: the DataFrame of a trades of period
     """
-    trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) &
-                        (trades['close_time'] <= dataframe.iloc[-1]['date'])]
+    if date_index:
+        trades_start = dataframe.index[0]
+        trades_stop = dataframe.index[-1]
+    else:
+        trades_start = dataframe.iloc[0]['date']
+        trades_stop = dataframe.iloc[-1]['date']
+    trades = trades.loc[(trades['open_time'] >= trades_start) &
+                        (trades['close_time'] <= trades_stop)]
     return trades
 
 
diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py
index 5da067069..6dcb71cc4 100644
--- a/freqtrade/plot/plotting.py
+++ b/freqtrade/plot/plotting.py
@@ -385,6 +385,9 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
     # Combine close-values for all pairs, rename columns to "pair"
     df_comb = combine_dataframes_with_mean(data, "close")
 
+    # Trim trades to available OHLCV data
+    trades = extract_trades_of_period(df_comb, trades, date_index=True)
+
     # Add combined cumulative profit
     df_comb = create_cum_profit(df_comb, trades, 'cum_profit', timeframe)
 

From 2444fb9cd6afe8fe9cd086872d7e6d0b29e95ab7 Mon Sep 17 00:00:00 2001
From: Ork Blutt <orkblutt@msn.com>
Date: Mon, 6 Apr 2020 15:56:57 +0200
Subject: [PATCH 102/151] fix broken tests: remove duplicated value

---
 tests/rpc/test_rpc_apiserver.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py
index 79073825d..c96f68f29 100644
--- a/tests/rpc/test_rpc_apiserver.py
+++ b/tests/rpc/test_rpc_apiserver.py
@@ -446,7 +446,6 @@ def test_api_status(botclient, mocker, ticker, fee, markets):
                         'stop_loss_pct': None,
                         'trade_id': 1,
                         'close_rate_requested': None,
-                        'current_profit': -0.41,
                         'current_rate': 1.099e-05,
                         'fee_close': 0.0025,
                         'fee_open': 0.0025,

From 200111fef6b4b8ff90818a4e76d6caa207fb4034 Mon Sep 17 00:00:00 2001
From: Ork Blutt <orkblutt@msn.com>
Date: Mon, 6 Apr 2020 16:07:43 +0200
Subject: [PATCH 103/151] fix method return value

---
 freqtrade/rpc/rpc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index b78856265..35386760d 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -227,7 +227,7 @@ class RPC:
         ]
 
     def _rpc_trade_history(
-            self, limit: int) -> Dict[str, List[Dict[str, Any]]]:
+            self, limit: int) -> Dict:
         """ Returns the X last trades """
         if limit > 0:
             trades = Trade.get_trades().order_by(Trade.id.desc()).limit(limit)

From c95906cfcf89210b7045fc9f95ace14aa6c2cc31 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Tue, 7 Apr 2020 10:42:15 +0200
Subject: [PATCH 104/151] Update hyperopt.py

---
 freqtrade/optimize/hyperopt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 1adaf54e1..a839d2111 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -649,7 +649,7 @@ class Hyperopt:
         self.hyperopt_table_header = -1
         data, timerange = self.backtesting.load_bt_data()
 
-        preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
+        preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
         # Trim startup period from analyzed dataframe
         for pair, df in preprocessed.items():
             preprocessed[pair] = trim_dataframe(df, timerange)

From 132f5f73f5bd7c083891802704e7bb208b12012b Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Tue, 7 Apr 2020 10:44:18 +0200
Subject: [PATCH 105/151] Update hyperopt.py

---
 freqtrade/optimize/hyperopt.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index a839d2111..bf32997c1 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -650,6 +650,7 @@ class Hyperopt:
         data, timerange = self.backtesting.load_bt_data()
 
         preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
+
         # Trim startup period from analyzed dataframe
         for pair, df in preprocessed.items():
             preprocessed[pair] = trim_dataframe(df, timerange)

From bdc85ec89b23a99abebe8722da799d81c42a208c Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 7 Apr 2020 19:42:16 +0200
Subject: [PATCH 106/151] Move create_mock_tests to conftest and add test for
 test_trade-history

---
 tests/conftest.py             | 46 +++++++++++++++++++++++++++++++++
 tests/data/test_btanalysis.py |  2 +-
 tests/rpc/test_rpc.py         | 28 +++++++++++++++++++-
 tests/test_persistence.py     | 48 +----------------------------------
 4 files changed, 75 insertions(+), 49 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 64d0cd5ee..da1fbd6d4 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -166,6 +166,52 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     freqtrade.exchange.refresh_latest_ohlcv = lambda p: None
 
 
+def create_mock_trades(fee):
+    """
+    Create some fake trades ...
+    """
+    # Simulate dry_run entries
+    trade = Trade(
+        pair='ETH/BTC',
+        stake_amount=0.001,
+        amount=123.0,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.123,
+        exchange='bittrex',
+        open_order_id='dry_run_buy_12345'
+    )
+    Trade.session.add(trade)
+
+    trade = Trade(
+        pair='ETC/BTC',
+        stake_amount=0.001,
+        amount=123.0,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.123,
+        close_rate=0.128,
+        close_profit=0.005,
+        exchange='bittrex',
+        is_open=False,
+        open_order_id='dry_run_sell_12345'
+    )
+    Trade.session.add(trade)
+
+    # Simulate prod entry
+    trade = Trade(
+        pair='ETC/BTC',
+        stake_amount=0.001,
+        amount=123.0,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.123,
+        exchange='bittrex',
+        open_order_id='prod_buy_12345'
+    )
+    Trade.session.add(trade)
+
+
 @pytest.fixture(autouse=True)
 def patch_coingekko(mocker) -> None:
     """
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 463e5ae36..0edad8e78 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -15,7 +15,7 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
                                        load_backtest_data, load_trades,
                                        load_trades_from_db)
 from freqtrade.data.history import load_data, load_pair_history
-from tests.test_persistence import create_mock_trades
+from tests.conftest import create_mock_trades
 
 
 def test_load_backtest_data(testdatadir):
diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py
index 875a234b2..d2af4bd87 100644
--- a/tests/rpc/test_rpc.py
+++ b/tests/rpc/test_rpc.py
@@ -13,7 +13,7 @@ from freqtrade.persistence import Trade
 from freqtrade.rpc import RPC, RPCException
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
 from freqtrade.state import State
-from tests.conftest import get_patched_freqtradebot, patch_get_signal
+from tests.conftest import get_patched_freqtradebot, patch_get_signal, create_mock_trades
 
 
 # Functions for recurrent object patching
@@ -211,6 +211,32 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee,
         rpc._rpc_daily_profit(0, stake_currency, fiat_display_currency)
 
 
+def test_rpc_trade_history(mocker, default_conf, markets, fee):
+    mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        markets=PropertyMock(return_value=markets)
+    )
+
+    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
+    create_mock_trades(fee)
+    rpc = RPC(freqtradebot)
+    rpc._fiat_converter = CryptoToFiatConverter()
+    trades = rpc._rpc_trade_history(2)
+    assert len(trades['trades']) == 2
+    assert trades['trades_count'] == 2
+    assert isinstance(trades['trades'][0], dict)
+    assert isinstance(trades['trades'][1], dict)
+
+    trades = rpc._rpc_trade_history(0)
+    assert len(trades['trades']) == 3
+    assert trades['trades_count'] == 3
+    # The first trade is for ETH ... sorting is descending
+    assert trades['trades'][-1]['pair'] == 'ETH/BTC'
+    assert trades['trades'][0]['pair'] == 'ETC/BTC'
+    assert trades['trades'][1]['pair'] == 'ETC/BTC'
+
+
 def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee,
                               limit_buy_order, limit_sell_order, mocker) -> None:
     mocker.patch.multiple(
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index c6de10e3d..ceac24356 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -9,53 +9,7 @@ from sqlalchemy import create_engine
 from freqtrade import constants
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence import Trade, clean_dry_run_db, init
-from tests.conftest import log_has
-
-
-def create_mock_trades(fee):
-    """
-    Create some fake trades ...
-    """
-    # Simulate dry_run entries
-    trade = Trade(
-        pair='ETH/BTC',
-        stake_amount=0.001,
-        amount=123.0,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.123,
-        exchange='bittrex',
-        open_order_id='dry_run_buy_12345'
-    )
-    Trade.session.add(trade)
-
-    trade = Trade(
-        pair='ETC/BTC',
-        stake_amount=0.001,
-        amount=123.0,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.123,
-        close_rate=0.128,
-        close_profit=0.005,
-        exchange='bittrex',
-        is_open=False,
-        open_order_id='dry_run_sell_12345'
-    )
-    Trade.session.add(trade)
-
-    # Simulate prod entry
-    trade = Trade(
-        pair='ETC/BTC',
-        stake_amount=0.001,
-        amount=123.0,
-        fee_open=fee.return_value,
-        fee_close=fee.return_value,
-        open_rate=0.123,
-        exchange='bittrex',
-        open_order_id='prod_buy_12345'
-    )
-    Trade.session.add(trade)
+from tests.conftest import log_has, create_mock_trades
 
 
 def test_init_create_session(default_conf):

From 296c616ce7c2595ec65ba11d4c0ad0d7e670721f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 7 Apr 2020 19:50:13 +0200
Subject: [PATCH 107/151] Add test for api-trades call

---
 tests/rpc/test_rpc_apiserver.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py
index c96f68f29..6548790cb 100644
--- a/tests/rpc/test_rpc_apiserver.py
+++ b/tests/rpc/test_rpc_apiserver.py
@@ -13,7 +13,7 @@ from freqtrade.__init__ import __version__
 from freqtrade.persistence import Trade
 from freqtrade.rpc.api_server import BASE_URI, ApiServer
 from freqtrade.state import State
-from tests.conftest import get_patched_freqtradebot, log_has, patch_get_signal
+from tests.conftest import get_patched_freqtradebot, log_has, patch_get_signal, create_mock_trades
 
 _TEST_USER = "FreqTrader"
 _TEST_PASS = "SuperSecurePassword1!"
@@ -302,6 +302,30 @@ def test_api_daily(botclient, mocker, ticker, fee, markets):
     assert rc.json[0][0] == str(datetime.utcnow().date())
 
 
+def test_api_trades(botclient, mocker, ticker, fee, markets):
+    ftbot, client = botclient
+    patch_get_signal(ftbot, (True, False))
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        markets=PropertyMock(return_value=markets)
+    )
+    rc = client_get(client, f"{BASE_URI}/trades")
+    assert_response(rc)
+    assert len(rc.json) == 2
+    assert rc.json['trades_count'] == 0
+
+    create_mock_trades(fee)
+
+    rc = client_get(client, f"{BASE_URI}/trades")
+    assert_response(rc)
+    assert len(rc.json['trades']) == 3
+    assert rc.json['trades_count'] == 3
+    rc = client_get(client, f"{BASE_URI}/trades?limit=2")
+    assert_response(rc)
+    assert len(rc.json['trades']) == 2
+    assert rc.json['trades_count'] == 2
+
+
 def test_api_edge_disabled(botclient, mocker, ticker, fee, markets):
     ftbot, client = botclient
     patch_get_signal(ftbot, (True, False))

From 492c2799dc78c38ece251c06f4377fdf03113b61 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 7 Apr 2020 19:52:34 +0200
Subject: [PATCH 108/151] Rename rest-client script history to trades

---
 scripts/rest_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/rest_client.py b/scripts/rest_client.py
index 116c00063..b26c32479 100755
--- a/scripts/rest_client.py
+++ b/scripts/rest_client.py
@@ -156,10 +156,10 @@ class FtRestClient():
         """
         return self._get("show_config")
 
-    def history(self, limit=None):
+    def trades(self, limit=None):
         """Return trades history.
 
-        :param limit: Limits trades to the X last trades . No limit to get all the trades.
+        :param limit: Limits trades to the X last trades. No limit to get all the trades.
         :return: json object
         """
         return self._get("trades", params={"limit": limit} if limit else 0)

From 02192f28cd62a1fe4b0bf95545f718079122b715 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 8 Apr 2020 07:56:21 +0200
Subject: [PATCH 109/151] Small stylistic updates

---
 freqtrade/rpc/rpc.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 35386760d..8645e466e 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -226,18 +226,14 @@ class RPC:
             for key, value in profit_days.items()
         ]
 
-    def _rpc_trade_history(
-            self, limit: int) -> Dict:
+    def _rpc_trade_history(self, limit: int) -> Dict:
         """ Returns the X last trades """
         if limit > 0:
             trades = Trade.get_trades().order_by(Trade.id.desc()).limit(limit)
         else:
             trades = Trade.get_trades().order_by(Trade.id.desc()).all()
 
-        output = []
-
-        for trade in trades:
-            output.append(trade.to_json())
+        output = [trade.to_json() for trade in trades]
 
         return {
             "trades": output,

From 5cff72a42e1ab58d841d42cae43cf59f6fbcf932 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 9 Apr 2020 09:22:38 +0200
Subject: [PATCH 110/151] Improve logging to ensure which branch is used for
 buy order cancels

---
 freqtrade/freqtradebot.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 570f8bea8..16e4d4d2d 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -904,6 +904,7 @@ class FreqtradeBot:
             logger.info('Buy order %s for %s.', reason, trade)
 
         if corder.get('remaining', order['remaining']) == order['amount']:
+            logger.info('Buy order removed from database %s', trade)
             # if trade is not partially completed, just delete the trade
             Trade.session.delete(trade)
             Trade.session.flush()

From 4707484a4cf8982bb9aa1548b5ba754356981d0d Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Thu, 9 Apr 2020 11:42:13 +0200
Subject: [PATCH 111/151] Fix issue with colring enabled + styling

---
 freqtrade/optimize/hyperopt.py | 27 +++++++++------------------
 1 file changed, 9 insertions(+), 18 deletions(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index fac613549..f34967af2 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -7,7 +7,6 @@ This module contains the hyperopt logic
 import locale
 import logging
 import random
-import sys
 import warnings
 from math import ceil
 from collections import OrderedDict
@@ -18,7 +17,6 @@ from typing import Any, Dict, List, Optional
 
 import rapidjson
 from colorama import Fore, Style
-from colorama import init as colorama_init
 from joblib import (Parallel, cpu_count, delayed, dump, load,
                     wrap_non_picklable_objects)
 from pandas import DataFrame, json_normalize, isna
@@ -268,17 +266,10 @@ class Hyperopt:
         Log results if it is better than any previous evaluation
         """
         is_best = results['is_best']
-        if not self.print_all:
-            # Print '\n' after each 100th epoch to separate dots from the log messages.
-            # Otherwise output is messy on a terminal.
-            print('.', end='' if results['current_epoch'] % 100 != 0 else None)  # type: ignore
-            sys.stdout.flush()
 
         if self.print_all or is_best:
-            if not self.print_all:
-                # Separate the results explanation string from dots
-                print("\n")
-            print(self.get_result_table(
+            print(
+                self.get_result_table(
                     self.config, results, self.total_epochs,
                     self.print_all, self.print_colorized,
                     self.hyperopt_table_header
@@ -675,9 +666,6 @@ class Hyperopt:
         self.dimensions: List[Dimension] = self.hyperopt_space()
         self.opt = self.get_optimizer(self.dimensions, config_jobs)
 
-        if self.print_colorized:
-            colorama_init(autoreset=True)
-
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
                 jobs = parallel._effective_n_jobs()
@@ -690,15 +678,18 @@ class Hyperopt:
                         ' (', progressbar.Percentage(), ')] ',
                         progressbar.Bar(marker=progressbar.AnimatedMarker(
                             fill='█',
-                            fill_wrap='\x1b[32m{}\x1b[39m',
-                            marker_wrap='\x1b[31m{}\x1b[39m',
+                            fill_wrap=Fore.GREEN + '{}' + Fore.RESET,
+                            marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL,
                         )),
                         ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
                     ]
                 else:
                     widgets = [
-                        ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), '] ',
-                        progressbar.Bar(marker='█'),
+                        ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
+                        ' (', progressbar.Percentage(), ')] ',
+                        progressbar.Bar(marker=progressbar.AnimatedMarker(
+                            fill='█',
+                        )),
                         ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
                     ]
                 with progressbar.ProgressBar(

From 346e09fed1045b73a9fa5aa25fa8226660fb7356 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 9 Apr 2020 19:32:10 +0200
Subject: [PATCH 112/151] Add test verifying that cancel_order with empty
 remaining is causing the bug

---
 tests/test_freqtradebot.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index e37270bd3..2536d57db 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -2204,14 +2204,11 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke
                       caplog)
 
 
-def test_handle_timedout_limit_buy(mocker, default_conf, limit_buy_order) -> None:
+def test_handle_timedout_limit_buy(mocker, caplog, default_conf, limit_buy_order) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     cancel_order_mock = MagicMock(return_value=limit_buy_order)
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        cancel_order=cancel_order_mock
-    )
+    mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order_mock)
 
     freqtrade = FreqtradeBot(default_conf)
 
@@ -2227,9 +2224,14 @@ def test_handle_timedout_limit_buy(mocker, default_conf, limit_buy_order) -> Non
     assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
     assert cancel_order_mock.call_count == 1
 
+    mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException)
+    assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
+    assert log_has_re(r"Could not cancel buy order", caplog)
+
 
 @pytest.mark.parametrize('cancelorder', [
     {},
+    {'remaining': None},
     'String Return value',
     123
 ])

From cbf5bf6735fa1b0aabfce9ba58fe370dd2d600c4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 9 Apr 2020 19:34:48 +0200
Subject: [PATCH 113/151] Add safe_value_fallback function

---
 freqtrade/misc.py  | 15 +++++++++++++++
 tests/test_misc.py | 15 ++++++++++++++-
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 1f52b75ec..ea7bab843 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -134,6 +134,21 @@ def round_dict(d, n):
     return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()}
 
 
+def safe_value_fallback(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None):
+    """
+    Search a value in dict1, return this if it's not None.
+    Fall back to dict2 - return key2 from dict2 if it's not None.
+    Else falls back to None.
+
+    """
+    if key1 in dict1 and dict1[key1] is not None:
+        return dict1[key1]
+    else:
+        if key2 in dict2 and dict2[key2] is not None:
+            return dict2[key2]
+    return default_value
+
+
 def plural(num: float, singular: str, plural: str = None) -> str:
     return singular if (num == 1 or num == -1) else plural or singular + 's'
 
diff --git a/tests/test_misc.py b/tests/test_misc.py
index c1e23926b..1bb5b40af 100644
--- a/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -9,7 +9,7 @@ import pytest
 from freqtrade.data.converter import ohlcv_to_dataframe
 from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json,
                             file_load_json, format_ms_time, pair_to_filename,
-                            plural, shorten_date)
+                            plural, safe_value_fallback, shorten_date)
 
 
 def test_shorten_date() -> None:
@@ -93,6 +93,19 @@ def test_format_ms_time() -> None:
     assert format_ms_time(date_in_epoch_ms) == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S')
 
 
+def test_safe_value_fallback():
+    dict1 = {'keya': None, 'keyb': 2, 'keyc': 5}
+    dict2 = {'keya': 20, 'keyb': None, 'keyc': 6}
+    assert safe_value_fallback(dict1, dict2, 'keya', 'keya') == 20
+    assert safe_value_fallback(dict2, dict1, 'keya', 'keya') == 20
+
+    assert safe_value_fallback(dict1, dict2, 'keyb', 'keyb') == 2
+    assert safe_value_fallback(dict2, dict1, 'keyb', 'keyb') == 2
+
+    assert safe_value_fallback(dict1, dict2, 'keyc', 'keyc') == 5
+    assert safe_value_fallback(dict2, dict1, 'keyc', 'keyc') == 6
+
+
 def test_plural() -> None:
     assert plural(0, "page") == "pages"
     assert plural(0.0, "page") == "pages"

From f39706cabd78c35aaf668bf290ba4c79ae744bcb Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 9 Apr 2020 19:35:27 +0200
Subject: [PATCH 114/151] Fix #3130 - when corder['remaining'] contains
 none-type

---
 freqtrade/freqtradebot.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 16e4d4d2d..b5c2ac4ee 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -20,6 +20,7 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
 from freqtrade.exceptions import DependencyException, InvalidOrderException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
+from freqtrade.misc import safe_value_fallback
 from freqtrade.pairlist.pairlistmanager import PairListManager
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -892,18 +893,23 @@ class FreqtradeBot:
         """
         if order['status'] != 'canceled':
             reason = "cancelled due to timeout"
-            corder = self.exchange.cancel_order(trade.open_order_id, trade.pair)
-            # Some exchanges don't return a dict here.
-            if not isinstance(corder, dict):
+            try:
+                corder = self.exchange.cancel_order(trade.open_order_id, trade.pair)
+                # Some exchanges don't return a dict here.
+                if not isinstance(corder, dict):
+                    corder = {}
+                logger.info('Buy order %s for %s.', reason, trade)
+            except InvalidOrderException:
                 corder = {}
-            logger.info('Buy order %s for %s.', reason, trade)
+                logger.exception(
+                    f"Could not cancel buy order {trade.open_order_id} for pair {trade.pair}")
         else:
             # Order was cancelled already, so we can reuse the existing dict
             corder = order
             reason = "cancelled on exchange"
             logger.info('Buy order %s for %s.', reason, trade)
 
-        if corder.get('remaining', order['remaining']) == order['amount']:
+        if safe_value_fallback(corder, order, 'remaining', 'remaining') == order['amount']:
             logger.info('Buy order removed from database %s', trade)
             # if trade is not partially completed, just delete the trade
             Trade.session.delete(trade)
@@ -915,7 +921,8 @@ class FreqtradeBot:
         # cancel_order may not contain the full order dict, so we need to fallback
         # to the order dict aquired before cancelling.
         # we need to fall back to the values from order if corder does not contain these keys.
-        trade.amount = order['amount'] - corder.get('remaining', order['remaining'])
+        trade.amount = order['amount'] - safe_value_fallback(corder, order,
+                                                             'remaining', 'remaining')
         trade.stake_amount = trade.amount * trade.open_rate
         # verify if fees were taken from amount to avoid problems during selling
         try:

From c03f637f5b629b55e74c4b8e0405bc8f7767bd7e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 9 Apr 2020 19:40:04 +0200
Subject: [PATCH 115/151] Improve safe_value_fallback test

---
 tests/test_misc.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/test_misc.py b/tests/test_misc.py
index 1bb5b40af..832bcb3a9 100644
--- a/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -94,8 +94,8 @@ def test_format_ms_time() -> None:
 
 
 def test_safe_value_fallback():
-    dict1 = {'keya': None, 'keyb': 2, 'keyc': 5}
-    dict2 = {'keya': 20, 'keyb': None, 'keyc': 6}
+    dict1 = {'keya': None, 'keyb': 2, 'keyc': 5, 'keyd': None}
+    dict2 = {'keya': 20, 'keyb': None, 'keyc': 6, 'keyd': None}
     assert safe_value_fallback(dict1, dict2, 'keya', 'keya') == 20
     assert safe_value_fallback(dict2, dict1, 'keya', 'keya') == 20
 
@@ -105,6 +105,14 @@ def test_safe_value_fallback():
     assert safe_value_fallback(dict1, dict2, 'keyc', 'keyc') == 5
     assert safe_value_fallback(dict2, dict1, 'keyc', 'keyc') == 6
 
+    assert safe_value_fallback(dict1, dict2, 'keyd', 'keyd') is None
+    assert safe_value_fallback(dict2, dict1, 'keyd', 'keyd') is None
+    assert safe_value_fallback(dict2, dict1, 'keyd', 'keyd', 1234) == 1234
+
+    assert safe_value_fallback(dict1, dict2, 'keyNo', 'keyNo') is None
+    assert safe_value_fallback(dict2, dict1, 'keyNo', 'keyNo') is None
+    assert safe_value_fallback(dict2, dict1, 'keyNo', 'keyNo', 1234) == 1234
+
 
 def test_plural() -> None:
     assert plural(0, "page") == "pages"

From d9e54ab7a4b386a1faa89dbb995fceb1f1c90a92 Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Sat, 11 Apr 2020 17:42:19 +0200
Subject: [PATCH 116/151] Update freqtrade/optimize/hyperopt.py

nice find

Co-Authored-By: Matthias <xmatthias@outlook.com>
---
 freqtrade/optimize/hyperopt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index f34967af2..4b9139681 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -677,7 +677,7 @@ class Hyperopt:
                         ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
                         ' (', progressbar.Percentage(), ')] ',
                         progressbar.Bar(marker=progressbar.AnimatedMarker(
-                            fill='█',
+                            fill='\N{FULL BLOCK}',
                             fill_wrap=Fore.GREEN + '{}' + Fore.RESET,
                             marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL,
                         )),

From 2c1c1c7f166871d1ba7ce95803426c50f85ff4df Mon Sep 17 00:00:00 2001
From: Fredrik81 <devnull81@gmail.com>
Date: Sat, 11 Apr 2020 17:42:32 +0200
Subject: [PATCH 117/151] Update freqtrade/optimize/hyperopt.py

nice find

Co-Authored-By: Matthias <xmatthias@outlook.com>
---
 freqtrade/optimize/hyperopt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 4b9139681..affd6f3b3 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -688,7 +688,7 @@ class Hyperopt:
                         ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
                         ' (', progressbar.Percentage(), ')] ',
                         progressbar.Bar(marker=progressbar.AnimatedMarker(
-                            fill='█',
+                            fill='\N{FULL BLOCK}',
                         )),
                         ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
                     ]

From 952d2f7513a7b28d6ee0fad8142169aeb6ebe166 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 12 Apr 2020 09:55:21 +0200
Subject: [PATCH 118/151] have version-detection fall back to freqtrade_commit

this allows freqtrade --version to work in docker too.

sample command:
`docker-compose run --rm freqtrade -version`
---
 freqtrade/__init__.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py
index ad432a20b..4da56c3aa 100644
--- a/freqtrade/__init__.py
+++ b/freqtrade/__init__.py
@@ -24,4 +24,11 @@ if __version__ == 'develop':
         #     stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"')
     except Exception:
         # git not available, ignore
-        pass
+        try:
+            # Try Fallback to freqtrade_commit file (created by CI whild building docker image)
+            from pathlib import Path
+            versionfile = Path('./freqtrade_commit')
+            if versionfile.is_file():
+                __version__ = f"docker-{versionfile.read_text()[:8]}"
+        except Exception:
+            pass

From 4ee0cbb5751c2bdb67724d379c6f2e3b1d12b339 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 12 Apr 2020 10:40:02 +0200
Subject: [PATCH 119/151] Reset index to correctly gather index

---
 freqtrade/data/btanalysis.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py
index 4505ea52a..a96e43dff 100644
--- a/freqtrade/data/btanalysis.py
+++ b/freqtrade/data/btanalysis.py
@@ -213,7 +213,7 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time'
     """
     if len(trades) == 0:
         raise ValueError("Trade dataframe empty.")
-    profit_results = trades.sort_values(date_col)
+    profit_results = trades.sort_values(date_col).reset_index()
     max_drawdown_df = pd.DataFrame()
     max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
     max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()

From eac4dbcd287871aeff837b875d730118d2165081 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 13 Apr 2020 08:30:31 +0000
Subject: [PATCH 120/151] Bump sqlalchemy from 1.3.15 to 1.3.16

Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.3.15 to 1.3.16.
- [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases)
- [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES)
- [Commits](https://github.com/sqlalchemy/sqlalchemy/commits)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index eb6ed8464..4a5d66c09 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,7 +1,7 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
 ccxt==1.25.81
-SQLAlchemy==1.3.15
+SQLAlchemy==1.3.16
 python-telegram-bot==12.5.1
 arrow==0.15.5
 cachetools==4.0.0

From 350b4d5e7deaca04b36805503616f8f8001cad17 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 13 Apr 2020 08:31:00 +0000
Subject: [PATCH 121/151] Bump coveralls from 1.11.1 to 2.0.0

Bumps [coveralls](https://github.com/coveralls-clients/coveralls-python) from 1.11.1 to 2.0.0.
- [Release notes](https://github.com/coveralls-clients/coveralls-python/releases)
- [Changelog](https://github.com/coveralls-clients/coveralls-python/blob/master/CHANGELOG.md)
- [Commits](https://github.com/coveralls-clients/coveralls-python/compare/1.11.1...2.0.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-dev.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 814003cbf..ae240d7c8 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -3,7 +3,7 @@
 -r requirements-plot.txt
 -r requirements-hyperopt.txt
 
-coveralls==1.11.1
+coveralls==2.0.0
 flake8==3.7.9
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==4.1.0

From cfcce0e657feb91f69198f426d9f91f2a46ea290 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 13 Apr 2020 08:32:29 +0000
Subject: [PATCH 122/151] Bump mkdocs-material from 4.6.3 to 5.1.0

Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 4.6.3 to 5.1.0.
- [Release notes](https://github.com/squidfunk/mkdocs-material/releases)
- [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG)
- [Commits](https://github.com/squidfunk/mkdocs-material/compare/4.6.3...5.1.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 docs/requirements-docs.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt
index 48ade026e..138b6e862 100644
--- a/docs/requirements-docs.txt
+++ b/docs/requirements-docs.txt
@@ -1,2 +1,2 @@
-mkdocs-material==4.6.3
+mkdocs-material==5.1.0
 mdx_truly_sane_lists==1.2

From fb0d76b94a73c4764adff2fc89dbf98bfc2e5589 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 13 Apr 2020 11:47:12 +0000
Subject: [PATCH 123/151] Bump cachetools from 4.0.0 to 4.1.0

Bumps [cachetools](https://github.com/tkem/cachetools) from 4.0.0 to 4.1.0.
- [Release notes](https://github.com/tkem/cachetools/releases)
- [Changelog](https://github.com/tkem/cachetools/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/tkem/cachetools/compare/v4.0.0...v4.1.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 4a5d66c09..e22bebf28 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -4,7 +4,7 @@ ccxt==1.25.81
 SQLAlchemy==1.3.16
 python-telegram-bot==12.5.1
 arrow==0.15.5
-cachetools==4.0.0
+cachetools==4.1.0
 requests==2.23.0
 urllib3==1.25.8
 wrapt==1.12.1

From a166fc887f27e1670ea29f38155059206e85e8cf Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 13 Apr 2020 11:47:14 +0000
Subject: [PATCH 124/151] Bump python-telegram-bot from 12.5.1 to 12.6.1

Bumps [python-telegram-bot](https://github.com/python-telegram-bot/python-telegram-bot) from 12.5.1 to 12.6.1.
- [Release notes](https://github.com/python-telegram-bot/python-telegram-bot/releases)
- [Changelog](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/CHANGES.rst)
- [Commits](https://github.com/python-telegram-bot/python-telegram-bot/compare/v12.5.1...v12.6.1)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index 4a5d66c09..e1572e215 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -2,7 +2,7 @@
 # mainly used for Raspberry pi installs
 ccxt==1.25.81
 SQLAlchemy==1.3.16
-python-telegram-bot==12.5.1
+python-telegram-bot==12.6.1
 arrow==0.15.5
 cachetools==4.0.0
 requests==2.23.0

From a1d2124e450db4993c9d0bd8c81db0aab8f59fc2 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Mon, 13 Apr 2020 12:29:38 +0000
Subject: [PATCH 125/151] Bump ccxt from 1.25.81 to 1.26.12

Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.25.81 to 1.26.12.
- [Release notes](https://github.com/ccxt/ccxt/releases)
- [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst)
- [Commits](https://github.com/ccxt/ccxt/compare/1.25.81...1.26.12)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index ca8918caf..a6a0fb78e 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.25.81
+ccxt==1.26.12
 SQLAlchemy==1.3.16
 python-telegram-bot==12.6.1
 arrow==0.15.5

From ddf37ef059655c93ec9109382815e683ae834dd2 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Apr 2020 08:02:42 +0200
Subject: [PATCH 126/151] Add test to demonstrate that the dataframe is not
 changed

---
 freqtrade/data/btanalysis.py  | 2 +-
 tests/data/test_btanalysis.py | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py
index a96e43dff..0c2fbff1e 100644
--- a/freqtrade/data/btanalysis.py
+++ b/freqtrade/data/btanalysis.py
@@ -213,7 +213,7 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time'
     """
     if len(trades) == 0:
         raise ValueError("Trade dataframe empty.")
-    profit_results = trades.sort_values(date_col).reset_index()
+    profit_results = trades.sort_values(date_col).reset_index(drop=True)
     max_drawdown_df = pd.DataFrame()
     max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
     max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 4eec20976..7b894cccc 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -201,7 +201,13 @@ def test_calculate_max_drawdown2():
 
     dates = [Arrow(2020, 1, 1).shift(days=i) for i in range(len(values))]
     df = DataFrame(zip(values, dates), columns=['profit', 'open_time'])
+    # sort by profit and reset index
+    df = df.sort_values('profit').reset_index(drop=True)
+    df1 = df.copy()
     drawdown, h, low = calculate_max_drawdown(df, date_col='open_time', value_col='profit')
+    # Ensure df has not been altered.
+    assert df.equals(df1)
+
     assert isinstance(drawdown, float)
     # High must be before low
     assert h < low

From 55a052bcf6853e0b1a6e2e21b3d1ff61d730db40 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Apr 2020 08:05:46 +0200
Subject: [PATCH 127/151] fix typo in comment

Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
---
 freqtrade/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py
index 4da56c3aa..e96e7f530 100644
--- a/freqtrade/__init__.py
+++ b/freqtrade/__init__.py
@@ -25,7 +25,7 @@ if __version__ == 'develop':
     except Exception:
         # git not available, ignore
         try:
-            # Try Fallback to freqtrade_commit file (created by CI whild building docker image)
+            # Try Fallback to freqtrade_commit file (created by CI while building docker image)
             from pathlib import Path
             versionfile = Path('./freqtrade_commit')
             if versionfile.is_file():

From cfe1e4876a00196cd40b938360042d08468b8ca2 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Apr 2020 19:20:47 +0200
Subject: [PATCH 128/151] Improve testcase for cancel_order_empty

---
 freqtrade/exchange/exchange.py  | 2 +-
 tests/exchange/test_exchange.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index 073e28659..ad4824a10 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -908,7 +908,7 @@ class Exchange:
         :param order: Order dict as returned from get_order()
         :return: True if order has been cancelled without being filled, False otherwise.
         """
-        return order['status'] in ('closed', 'canceled') and order.get('filled') == 0.0
+        return order.get('status') in ('closed', 'canceled') and order.get('filled') == 0.0
 
     @retrier
     def cancel_order(self, order_id: str, pair: str) -> None:
diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py
index 7f03eb547..cc3f95735 100644
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@@ -1711,6 +1711,8 @@ def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
     ({'status': 'closed', 'filled': 0.0}, True),
     ({'status': 'canceled', 'filled': 0.0}, True),
     ({'status': 'canceled', 'filled': 10.0}, False),
+    ({'status': 'unknown', 'filled': 10.0}, False),
+    ({'result': 'testest123'}, False),
     ])
 def test_check_order_canceled_empty(mocker, default_conf, exchange_name, order, result):
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)

From 7c15375f5da6fb66efc00f356486eb4a5736534d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Apr 2020 20:20:36 +0200
Subject: [PATCH 129/151] Add log_on_refresh - using TTL caching to avoid
 spamming logs

---
 freqtrade/pairlist/IPairList.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py
index 35844a99e..e57bc4e88 100644
--- a/freqtrade/pairlist/IPairList.py
+++ b/freqtrade/pairlist/IPairList.py
@@ -9,6 +9,8 @@ from abc import ABC, abstractmethod, abstractproperty
 from copy import deepcopy
 from typing import Any, Dict, List
 
+from cachetools import TTLCache, cached
+
 from freqtrade.exchange import market_is_active
 
 logger = logging.getLogger(__name__)
@@ -31,6 +33,9 @@ class IPairList(ABC):
         self._config = config
         self._pairlistconfig = pairlistconfig
         self._pairlist_pos = pairlist_pos
+        self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
+        self._last_refresh = 0
+        self._log_cache = TTLCache(maxsize=1024, ttl=self.refresh_period)
 
     @property
     def name(self) -> str:
@@ -40,6 +45,24 @@ class IPairList(ABC):
         """
         return self.__class__.__name__
 
+    def log_on_refresh(self, logmethod, message: str) -> None:
+        """
+        Logs message - not more often than "refresh_period" to avoid log spamming
+        Logs the log-message as debug as well to simplify debugging.
+        :param logmethod: Function that'll be called. Most likely `logger.info`.
+        :param message: String containing the message to be sent to the function.
+        :return: None.
+        """
+
+        @cached(cache=self._log_cache)
+        def _log_on_refresh(logmethod, message: str):
+            logmethod(message)
+
+        # Log as debug first
+        logger.debug(message)
+        # Call hidden function.
+        _log_on_refresh(logmethod, message)
+
     @abstractproperty
     def needstickers(self) -> bool:
         """

From 5d876ca0a359f129a79a3f486216261734fd9003 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Apr 2020 20:21:10 +0200
Subject: [PATCH 130/151] Use log-spamprevention methods

---
 freqtrade/pairlist/PriceFilter.py  | 4 ++--
 freqtrade/pairlist/SpreadFilter.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py
index dc02ae251..7ea1e6003 100644
--- a/freqtrade/pairlist/PriceFilter.py
+++ b/freqtrade/pairlist/PriceFilter.py
@@ -43,8 +43,8 @@ class PriceFilter(IPairList):
         compare = ticker['last'] + 1 / pow(10, precision)
         changeperc = (compare - ticker['last']) / ticker['last']
         if changeperc > self._low_price_ratio:
-            logger.info(f"Removed {ticker['symbol']} from whitelist, "
-                        f"because 1 unit is {changeperc * 100:.3f}%")
+            self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
+                                             f"because 1 unit is {changeperc * 100:.3f}%")
             return False
         return True
 
diff --git a/freqtrade/pairlist/SpreadFilter.py b/freqtrade/pairlist/SpreadFilter.py
index 9361837cc..49731ef11 100644
--- a/freqtrade/pairlist/SpreadFilter.py
+++ b/freqtrade/pairlist/SpreadFilter.py
@@ -49,9 +49,9 @@ class SpreadFilter(IPairList):
             if 'bid' in ticker and 'ask' in ticker:
                 spread = 1 - ticker['bid'] / ticker['ask']
                 if not ticker or spread > self._max_spread_ratio:
-                    logger.info(f"Removed {ticker['symbol']} from whitelist, "
-                                f"because spread {spread * 100:.3f}% >"
-                                f"{self._max_spread_ratio * 100}%")
+                    self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
+                                                     f"because spread {spread * 100:.3f}% >"
+                                                     f"{self._max_spread_ratio * 100}%")
                     pairlist.remove(p)
             else:
                 pairlist.remove(p)

From 13ee7a55c4756035d647f00a25767407e2cafe40 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Apr 2020 20:21:30 +0200
Subject: [PATCH 131/151] Fix #3166

Always call _gen_pair_whitelist if volumepairlist is not the first in
the list.
---
 freqtrade/pairlist/VolumePairList.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py
index 9ce2adc9e..65f43245c 100644
--- a/freqtrade/pairlist/VolumePairList.py
+++ b/freqtrade/pairlist/VolumePairList.py
@@ -39,7 +39,6 @@ class VolumePairList(IPairList):
         if not self._validate_keys(self._sort_key):
             raise OperationalException(
                 f'key {self._sort_key} not in {SORT_VALUES}')
-        self._last_refresh = 0
 
     @property
     def needstickers(self) -> bool:
@@ -68,16 +67,18 @@ class VolumePairList(IPairList):
         :return: new whitelist
         """
         # Generate dynamic whitelist
-        if self._last_refresh + self.refresh_period < datetime.now().timestamp():
+        # Must always run if this pairlist is not the first in the list.
+        if (self._pairlist_pos != 0 or
+                (self._last_refresh + self.refresh_period < datetime.now().timestamp())):
+
             self._last_refresh = int(datetime.now().timestamp())
-            return self._gen_pair_whitelist(pairlist,
-                                            tickers,
-                                            self._config['stake_currency'],
-                                            self._sort_key,
-                                            self._min_value
-                                            )
+            pairs = self._gen_pair_whitelist(pairlist, tickers,
+                                             self._config['stake_currency'],
+                                             self._sort_key, self._min_value)
         else:
-            return pairlist
+            pairs = pairlist
+        self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
+        return pairs
 
     def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict,
                             base_currency: str, key: str, min_val: int) -> List[str]:
@@ -88,7 +89,6 @@ class VolumePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()).
         :return: List of pairs
         """
-
         if self._pairlist_pos == 0:
             # If VolumePairList is the first in the list, use fresh pairlist
             # Check if pair quote currency equals to the stake currency.
@@ -109,6 +109,5 @@ class VolumePairList(IPairList):
         pairs = self._verify_blacklist(pairs, aswarning=False)
         # Limit to X number of pairs
         pairs = pairs[:self._number_pairs]
-        logger.info(f"Searching {self._number_pairs} pairs: {pairs}")
 
         return pairs

From ceca0a659cef27ed872bcb504af6ec74c99f1511 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Apr 2020 20:25:58 +0200
Subject: [PATCH 132/151] Simplify cached stuff to only what's needed

---
 freqtrade/pairlist/IPairList.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py
index e57bc4e88..e089e546c 100644
--- a/freqtrade/pairlist/IPairList.py
+++ b/freqtrade/pairlist/IPairList.py
@@ -55,13 +55,13 @@ class IPairList(ABC):
         """
 
         @cached(cache=self._log_cache)
-        def _log_on_refresh(logmethod, message: str):
+        def _log_on_refresh(message: str):
             logmethod(message)
 
         # Log as debug first
         logger.debug(message)
         # Call hidden function.
-        _log_on_refresh(logmethod, message)
+        _log_on_refresh(message)
 
     @abstractproperty
     def needstickers(self) -> bool:

From 1b2bf2c9b69680d2ed09e3edfeffc1d00451932a Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Apr 2020 20:39:54 +0200
Subject: [PATCH 133/151] Add test for cached log method

---
 tests/pairlist/test_pairlist.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py
index 1ce1151b7..6275bdafc 100644
--- a/tests/pairlist/test_pairlist.py
+++ b/tests/pairlist/test_pairlist.py
@@ -46,6 +46,28 @@ def static_pl_conf(whitelist_conf):
     return whitelist_conf
 
 
+def test_log_on_refresh(mocker, static_pl_conf, markets, tickers):
+    mocker.patch.multiple('freqtrade.exchange.Exchange',
+                          markets=PropertyMock(return_value=markets),
+                          exchange_has=MagicMock(return_value=True),
+                          get_tickers=tickers
+                          )
+    freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
+    logmock = MagicMock()
+    # Assign starting whitelist
+    pl = freqtrade.pairlists._pairlists[0]
+    pl.log_on_refresh(logmock, 'Hello world')
+    assert logmock.call_count == 1
+    pl.log_on_refresh(logmock, 'Hello world')
+    assert logmock.call_count == 1
+    assert pl._log_cache.currsize == 1
+    assert ('Hello world',) in pl._log_cache._Cache__data
+
+    pl.log_on_refresh(logmock, 'Hello world2')
+    assert logmock.call_count == 2
+    assert pl._log_cache.currsize == 2
+
+
 def test_load_pairlist_noexist(mocker, markets, default_conf):
     bot = get_patched_freqtradebot(mocker, default_conf)
     mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))

From 2b7376f6f334d78ede1d98fb1fee5ca3af4a9b34 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Apr 2020 20:45:30 +0200
Subject: [PATCH 134/151] Implement log-filtering for all pairlists

---
 freqtrade/pairlist/PrecisionFilter.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py
index f16458ca5..2a2ba46b7 100644
--- a/freqtrade/pairlist/PrecisionFilter.py
+++ b/freqtrade/pairlist/PrecisionFilter.py
@@ -39,8 +39,9 @@ class PrecisionFilter(IPairList):
         stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99)
         logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}")
         if sp <= stop_gap_price:
-            logger.info(f"Removed {ticker['symbol']} from whitelist, "
-                        f"because stop price {sp} would be <= stop limit {stop_gap_price}")
+            self.log_on_refresh(logger.info,
+                                f"Removed {ticker['symbol']} from whitelist, "
+                                f"because stop price {sp} would be <= stop limit {stop_gap_price}")
             return False
         return True
 

From ac008a4758e16ef7da0523bc32a37e25ea85e32c Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Apr 2020 06:58:54 +0200
Subject: [PATCH 135/151] Remove obsolete comment in tests

---
 tests/optimize/test_optimize_reports.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py
index 3bbc396c8..e0782146a 100644
--- a/tests/optimize/test_optimize_reports.py
+++ b/tests/optimize/test_optimize_reports.py
@@ -163,7 +163,6 @@ def test_backtest_record(default_conf, fee, mocker):
     results['Strat'] = results['DefStrat']
     results['Strat2'] = results['DefStrat']
     store_backtest_result(Path("backtest-result.json"), results)
-    # Assert file_dump_json was only called once
     assert names == [
         Path('backtest-result-DefStrat.json'),
         Path('backtest-result-Strat.json'),

From 36e714a7b258576df0d920fef50fba700037ffad Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Apr 2020 07:19:27 +0200
Subject: [PATCH 136/151] Add price_get_one_pip filter

---
 freqtrade/exchange/exchange.py    | 11 +++++++++++
 freqtrade/pairlist/PriceFilter.py |  5 ++---
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index f4c94a1ca..7dd910fab 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -452,6 +452,17 @@ class Exchange:
                 price = ceil(big_price) / pow(10, symbol_prec)
         return price
 
+    def price_get_one_pip(self, pair: str, price: float) -> float:
+        """
+        Get's the "1 pip" value for this pair.
+        Used in PriceFilter to calculate the 1pip movements.
+        """
+        precision = self.markets[pair]['precision']['price']
+        if self.precisionMode == TICK_SIZE:
+            return price % precision
+        else:
+            return 1 / pow(10, precision)
+
     def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
                       rate: float, params: Dict = {}) -> Dict[str, Any]:
         order_id = f'dry_run_{side}_{randint(0, 10**6)}'
diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py
index dc02ae251..00c9d44c9 100644
--- a/freqtrade/pairlist/PriceFilter.py
+++ b/freqtrade/pairlist/PriceFilter.py
@@ -38,9 +38,8 @@ class PriceFilter(IPairList):
         :param precision: Precision
         :return: True if the pair can stay, false if it should be removed
         """
-        precision = self._exchange.markets[ticker['symbol']]['precision']['price']
-
-        compare = ticker['last'] + 1 / pow(10, precision)
+        compare = ticker['last'] + self._exchange.price_get_one_pip(ticker['symbol'],
+                                                                    ticker['last'])
         changeperc = (compare - ticker['last']) / ticker['last']
         if changeperc > self._low_price_ratio:
             logger.info(f"Removed {ticker['symbol']} from whitelist, "

From 33b6c7de5b01fdb31026da307b9546c6cbca3529 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Apr 2020 07:53:31 +0200
Subject: [PATCH 137/151] Add tests for price_one_pip

---
 freqtrade/exchange/exchange.py  |  2 +-
 tests/exchange/test_exchange.py | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index 7dd910fab..b3255f1bd 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -459,7 +459,7 @@ class Exchange:
         """
         precision = self.markets[pair]['precision']['price']
         if self.precisionMode == TICK_SIZE:
-            return price % precision
+            return precision
         else:
             return 1 / pow(10, precision)
 
diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py
index 8d8930f66..c0aedc7f9 100644
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@@ -253,6 +253,32 @@ def test_price_to_precision(default_conf, mocker, price, precision_mode, precisi
     assert pytest.approx(exchange.price_to_precision(pair, price)) == expected
 
 
+@pytest.mark.parametrize("price,precision_mode,precision,expected", [
+    (2.34559, 2, 4, 0.0001),
+    (2.34559, 2, 5, 0.00001),
+    (2.34559, 2, 3, 0.001),
+    (2.9999, 2, 3, 0.001),
+    (200.0511, 2, 3, 0.001),
+    # Tests for Tick_size
+    (2.34559, 4, 0.0001, 0.0001),
+    (2.34559, 4, 0.00001, 0.00001),
+    (2.34559, 4, 0.0025, 0.0025),
+    (2.9909, 4, 0.0025, 0.0025),
+    (234.43, 4, 0.5, 0.5),
+    (234.43, 4, 0.0025, 0.0025),
+    (234.43, 4, 0.00013, 0.00013),
+
+])
+def test_price_get_one_pip(default_conf, mocker, price, precision_mode, precision, expected):
+    markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'price': precision}}})
+    exchange = get_patched_exchange(mocker, default_conf, id="binance")
+    mocker.patch('freqtrade.exchange.Exchange.markets', markets)
+    mocker.patch('freqtrade.exchange.Exchange.precisionMode',
+                 PropertyMock(return_value=precision_mode))
+    pair = 'ETH/BTC'
+    assert pytest.approx(exchange.price_get_one_pip(pair, price)) == expected
+
+
 def test_set_sandbox(default_conf, mocker):
     """
     Test working scenario

From 99f3e9ed777c33ad537e799893d4675da8aa8aec Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Apr 2020 07:55:15 +0200
Subject: [PATCH 138/151] Remove wrong comment

---
 freqtrade/pairlist/PriceFilter.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py
index 00c9d44c9..f0ba6cd7f 100644
--- a/freqtrade/pairlist/PriceFilter.py
+++ b/freqtrade/pairlist/PriceFilter.py
@@ -35,7 +35,6 @@ class PriceFilter(IPairList):
         """
         Check if if one price-step (pip) is > than a certain barrier.
         :param ticker: ticker dict as returned from ccxt.load_markets()
-        :param precision: Precision
         :return: True if the pair can stay, false if it should be removed
         """
         compare = ticker['last'] + self._exchange.price_get_one_pip(ticker['symbol'],

From 8314759228f019488df8e204b64ddc06dc0e0c0f Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Wed, 15 Apr 2020 12:14:35 +0000
Subject: [PATCH 139/151] Bump ccxt from 1.26.12 to 1.26.32

Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.26.12 to 1.26.32.
- [Release notes](https://github.com/ccxt/ccxt/releases)
- [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst)
- [Commits](https://github.com/ccxt/ccxt/compare/1.26.12...1.26.32)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index a6a0fb78e..206e104ae 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -1,6 +1,6 @@
 # requirements without requirements installable via conda
 # mainly used for Raspberry pi installs
-ccxt==1.26.12
+ccxt==1.26.32
 SQLAlchemy==1.3.16
 python-telegram-bot==12.6.1
 arrow==0.15.5

From 16a810a0f61e59bb29f35fdd2333427b63f6c1a2 Mon Sep 17 00:00:00 2001
From: "dependabot-preview[bot]"
 <27856297+dependabot-preview[bot]@users.noreply.github.com>
Date: Wed, 15 Apr 2020 12:14:53 +0000
Subject: [PATCH 140/151] Bump jinja2 from 2.11.1 to 2.11.2

Bumps [jinja2](https://github.com/pallets/jinja) from 2.11.1 to 2.11.2.
- [Release notes](https://github.com/pallets/jinja/releases)
- [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst)
- [Commits](https://github.com/pallets/jinja/compare/2.11.1...2.11.2)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
---
 requirements-common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-common.txt b/requirements-common.txt
index a6a0fb78e..e98540e18 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -12,7 +12,7 @@ jsonschema==3.2.0
 TA-Lib==0.4.17
 tabulate==0.8.7
 pycoingecko==1.2.0
-jinja2==2.11.1
+jinja2==2.11.2
 
 # find first, C search in arrays
 py_find_1st==1.1.4

From d36e2cf6ab85be1e0993f8ef69f67e117385dbf2 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 16 Apr 2020 07:06:47 +0200
Subject: [PATCH 141/151] Fix random test failure in hyperopt

---
 freqtrade/optimize/hyperopt.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index affd6f3b3..68e7032d9 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -665,7 +665,6 @@ class Hyperopt:
 
         self.dimensions: List[Dimension] = self.hyperopt_space()
         self.opt = self.get_optimizer(self.dimensions, config_jobs)
-
         try:
             with Parallel(n_jobs=config_jobs) as parallel:
                 jobs = parallel._effective_n_jobs()
@@ -693,7 +692,7 @@ class Hyperopt:
                         ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
                     ]
                 with progressbar.ProgressBar(
-                         maxval=self.total_epochs, redirect_stdout=True, redirect_stderr=True,
+                         maxval=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
                          widgets=widgets
                      ) as pbar:
                     EVALS = ceil(self.total_epochs / jobs)

From 1f70fcfa2da744b70f71190f575928c0c42ea6ca Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 16 Apr 2020 20:19:34 +0200
Subject: [PATCH 142/151] Update logmessage

---
 freqtrade/freqtradebot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index b5c2ac4ee..8afe7d6f9 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -910,7 +910,7 @@ class FreqtradeBot:
             logger.info('Buy order %s for %s.', reason, trade)
 
         if safe_value_fallback(corder, order, 'remaining', 'remaining') == order['amount']:
-            logger.info('Buy order removed from database %s', trade)
+            logger.info('Buy order fully cancelled. Removing %s from database.', trade)
             # if trade is not partially completed, just delete the trade
             Trade.session.delete(trade)
             Trade.session.flush()

From fc684b00917d2d135704e52a2974e0c5a272fb26 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 17 Apr 2020 06:59:52 +0200
Subject: [PATCH 143/151] Ensure deleting filled is not raising an error if
 filled does not exist

---
 freqtrade/freqtradebot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 213444f72..8865dd20b 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -1140,7 +1140,7 @@ class FreqtradeBot:
                 new_amount = self.get_real_amount(trade, order, order_amount)
                 if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC):
                     order['amount'] = new_amount
-                    del order['filled']
+                    order.pop('filled', None)
                     # Fee was applied, so set to 0
                     trade.fee_open = 0
                     trade.recalc_open_trade_price()

From 800891a4759b3f2e96f3593be9471f089c7e2e92 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 17 Apr 2020 07:18:46 +0200
Subject: [PATCH 144/151] Add tests for cancel_order_with_result

---
 freqtrade/exchange/exchange.py  | 30 ++++++++++++++++++++++++++++++
 tests/exchange/test_exchange.py | 14 ++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index f2f8cd69f..412a60386 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -937,6 +937,36 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    def is_cancel_order_result_suitable(self, corder) -> bool:
+        if not isinstance(corder, dict):
+            return False
+
+        required = ('fee', 'status', 'amount')
+        return all(k in corder for k in required)
+
+    def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict:
+        """
+        Cancel order returning a result.
+        Creates a fake result if cancel order returns a non-usable result
+        and get_order does not work (certain exchanges don't return cancelled orders)
+        :param order_id: Orderid to cancel
+        :param pair: Pair corresponding to order_id
+        :param amount: Amount to use for fake response
+        :return: Result from either cancel_order if usable, or fetch_order
+        """
+        if self._config['dry_run']:
+            return {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
+        corder = self.cancel_order(order_id, pair)
+        if self.is_cancel_order_result_suitable(corder):
+            return corder
+        try:
+            order = self.get_order(order_id, pair)
+        except InvalidOrderException:
+            logger.warning(f"Could not fetch cancelled order {order_id}.")
+            order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
+
+        return order
+
     @retrier
     def get_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py
index b74ea1a6b..1f0df5aa5 100644
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@@ -1745,6 +1745,20 @@ def test_check_order_canceled_empty(mocker, default_conf, exchange_name, order,
     assert exchange.check_order_canceled_empty(order) == result
 
 
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+@pytest.mark.parametrize("order,result", [
+    ({'status': 'closed', 'amount': 10, 'fee': {}}, True),
+    ({'status': 'closed', 'amount': 0.0, 'fee': {}}, True),
+    ({'status': 'canceled', 'amount': 0.0, 'fee': {}}, True),
+    ({'status': 'canceled', 'amount': 10.0}, False),
+    ({'amount': 10.0, 'fee': {}}, False),
+    ({'result': 'testest123'}, False),
+    ('hello_world', False),
+])
+def test_is_cancel_order_result_suitable(mocker, default_conf, exchange_name, order, result):
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    assert exchange.is_cancel_order_result_suitable(order) == result
+
 # Ensure that if not dry_run, we should call API
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_cancel_order(default_conf, mocker, exchange_name):

From 5e3e0e819f20869b4d3e9a04abe4cb3b040bede9 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 17 Apr 2020 17:53:18 +0200
Subject: [PATCH 145/151] Add tests for cancel_order_with_result

---
 freqtrade/exchange/exchange.py  | 11 ++++++-----
 tests/exchange/test_exchange.py | 34 +++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index 412a60386..d3a520eaa 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -954,11 +954,12 @@ class Exchange:
         :param amount: Amount to use for fake response
         :return: Result from either cancel_order if usable, or fetch_order
         """
-        if self._config['dry_run']:
-            return {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
-        corder = self.cancel_order(order_id, pair)
-        if self.is_cancel_order_result_suitable(corder):
-            return corder
+        try:
+            corder = self.cancel_order(order_id, pair)
+            if self.is_cancel_order_result_suitable(corder):
+                return corder
+        except InvalidOrderException:
+            logger.warning(f"Could not cancel order {order_id}.")
         try:
             order = self.get_order(order_id, pair)
         except InvalidOrderException:
diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py
index 1f0df5aa5..e3ab76b92 100644
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@@ -1759,6 +1759,40 @@ def test_is_cancel_order_result_suitable(mocker, default_conf, exchange_name, or
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
     assert exchange.is_cancel_order_result_suitable(order) == result
 
+
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+@pytest.mark.parametrize("corder,call_corder,call_forder", [
+    ({'status': 'closed', 'amount': 10, 'fee': {}}, 1, 0),
+    ({'amount': 10, 'fee': {}}, 1, 1),
+])
+def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder,
+                                  call_corder, call_forder):
+    default_conf['dry_run'] = False
+    api_mock = MagicMock()
+    api_mock.cancel_order = MagicMock(return_value=corder)
+    api_mock.fetch_order = MagicMock(return_value={})
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+    res = exchange.cancel_order_with_result('1234', 'ETH/BTC', 1234)
+    assert isinstance(res, dict)
+    assert api_mock.cancel_order.call_count == call_corder
+    assert api_mock.fetch_order.call_count == call_forder
+
+
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+def test_cancel_order_with_result_error(default_conf, mocker, exchange_name, caplog):
+    default_conf['dry_run'] = False
+    api_mock = MagicMock()
+    api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order"))
+    api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order"))
+    exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name)
+
+    res = exchange.cancel_order_with_result('1234', 'ETH/BTC', 1541)
+    assert isinstance(res, dict)
+    assert log_has("Could not cancel order 1234.", caplog)
+    assert log_has("Could not fetch cancelled order 1234.", caplog)
+    assert res['amount'] == 1541
+
+
 # Ensure that if not dry_run, we should call API
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_cancel_order(default_conf, mocker, exchange_name):

From 1069cb3616001e44bdc17e68e36d46bb93544d4a Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 17 Apr 2020 17:53:56 +0200
Subject: [PATCH 146/151] Use cancel_order_with_result when cancelling orders
 after timeout

---
 freqtrade/freqtradebot.py  | 12 +++++-------
 tests/test_freqtradebot.py | 13 ++++++-------
 2 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 8865dd20b..f4aeba3fe 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -891,11 +891,8 @@ class FreqtradeBot:
         if order['status'] != 'canceled':
             reason = "cancelled due to timeout"
             try:
-                corder = self.exchange.cancel_order(trade.open_order_id, trade.pair)
-                # Some exchanges don't return a dict here.
-                if not isinstance(corder, dict):
-                    corder = {}
-                logger.info('Buy order %s for %s.', reason, trade)
+                corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
+                                                                trade.amount)
             except InvalidOrderException:
                 corder = {}
                 logger.exception(
@@ -904,7 +901,8 @@ class FreqtradeBot:
             # Order was cancelled already, so we can reuse the existing dict
             corder = order
             reason = "cancelled on exchange"
-            logger.info('Buy order %s for %s.', reason, trade)
+
+        logger.info('Buy order %s for %s.', reason, trade)
 
         if safe_value_fallback(corder, order, 'remaining', 'remaining') == order['amount']:
             logger.info('Buy order fully cancelled. Removing %s from database.', trade)
@@ -921,7 +919,7 @@ class FreqtradeBot:
         trade.amount = order['amount'] - safe_value_fallback(corder, order,
                                                              'remaining', 'remaining')
         trade.stake_amount = trade.amount * trade.open_rate
-        self.update_trade_state(trade, corder if 'fee' in corder else order, trade.amount)
+        self.update_trade_state(trade, corder, trade.amount)
 
         trade.open_order_id = None
         logger.info('Partial buy order timeout for %s.', trade)
diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index 1102dd344..c3e78c9be 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -1948,7 +1948,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, op
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_order=MagicMock(return_value=limit_buy_order_old),
-        cancel_order=cancel_order_mock,
+        cancel_order_with_result=cancel_order_mock,
         get_fee=fee
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -2055,7 +2055,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old,
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_order=MagicMock(return_value=limit_sell_order_old),
-        cancel_order=cancel_order_mock
+        cancel_order_with_result=cancel_order_mock
     )
     freqtrade = FreqtradeBot(default_conf)
 
@@ -2082,7 +2082,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_order=MagicMock(return_value=limit_buy_order_old_partial),
-        cancel_order=cancel_order_mock
+        cancel_order_with_result=cancel_order_mock
     )
     freqtrade = FreqtradeBot(default_conf)
 
@@ -2109,7 +2109,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_order=MagicMock(return_value=limit_buy_order_old_partial),
-        cancel_order=cancel_order_mock,
+        cancel_order_with_result=cancel_order_mock,
         get_trades_for_order=MagicMock(return_value=trades_for_order),
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -2146,7 +2146,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade,
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_order=MagicMock(return_value=limit_buy_order_old_partial),
-        cancel_order=cancel_order_mock,
+        cancel_order_with_result=cancel_order_mock,
         get_trades_for_order=MagicMock(return_value=trades_for_order),
     )
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount',
@@ -2208,7 +2208,7 @@ def test_handle_timedout_limit_buy(mocker, caplog, default_conf, limit_buy_order
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     cancel_order_mock = MagicMock(return_value=limit_buy_order)
-    mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order_mock)
+    mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock)
 
     freqtrade = FreqtradeBot(default_conf)
 
@@ -2226,7 +2226,6 @@ def test_handle_timedout_limit_buy(mocker, caplog, default_conf, limit_buy_order
 
     mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException)
     assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order)
-    assert log_has_re(r"Could not cancel buy order", caplog)
 
 
 @pytest.mark.parametrize('cancelorder', [

From 55af8bf26f33cca56cf9b0ccda4f088bb2c7ef83 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 17 Apr 2020 19:49:43 +0200
Subject: [PATCH 147/151] document to install hyperopt dependencies

---
 docs/hyperopt.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/docs/hyperopt.md b/docs/hyperopt.md
index c5055a3a8..825cc0267 100644
--- a/docs/hyperopt.md
+++ b/docs/hyperopt.md
@@ -16,6 +16,24 @@ To learn how to get data for the pairs and exchange you're interested in, head o
 !!! Bug
     Hyperopt can crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133)
 
+## Install hyperopt dependencies
+
+Since Hyperopt dependencies are not needed to run a bot, they are not installed by default.
+
+### Docker
+
+The docker-image includes hyperopt dependencies, no further action needed
+
+!!! Note
+    Since Hyperopt is a resource intensive process, running it on a Raspberry Pi is not recommended, therefore the Raspberry image does not include Hyperopt dependencies.
+
+### Simple installation script (setup.sh) / Manual installation
+
+```bash
+source .env/bin/activate
+pip install -r requirements-hyperopt.txt
+```
+
 ## Prepare Hyperopting
 
 Before we start digging into Hyperopt, we recommend you to take a look at

From 0273539f0614d58578f354bbc55448152d5036c8 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 17 Apr 2020 19:55:53 +0200
Subject: [PATCH 148/151] Remove exceptionhandler, this exception is handled in
 cancel_with_response

---
 freqtrade/freqtradebot.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index f4aeba3fe..65ba46987 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -890,13 +890,8 @@ class FreqtradeBot:
         """
         if order['status'] != 'canceled':
             reason = "cancelled due to timeout"
-            try:
-                corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
-                                                                trade.amount)
-            except InvalidOrderException:
-                corder = {}
-                logger.exception(
-                    f"Could not cancel buy order {trade.open_order_id} for pair {trade.pair}")
+            corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
+                                                            trade.amount)
         else:
             # Order was cancelled already, so we can reuse the existing dict
             corder = order

From 506781f4102064651423a923c2342e931c08f3d3 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 17 Apr 2020 20:48:27 +0200
Subject: [PATCH 149/151] Reword hyperopt install docs

---
 docs/hyperopt.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/hyperopt.md b/docs/hyperopt.md
index 825cc0267..ad812a5ad 100644
--- a/docs/hyperopt.md
+++ b/docs/hyperopt.md
@@ -18,16 +18,16 @@ To learn how to get data for the pairs and exchange you're interested in, head o
 
 ## Install hyperopt dependencies
 
-Since Hyperopt dependencies are not needed to run a bot, they are not installed by default.
+Since Hyperopt dependencies are not needed to run the bot itself, are heavy, can not be easily built on some platforms (like Raspberry PI), they are not installed by default. Before you run Hyperopt, you need to install the corresponding dependencies, as described in this section below.
+
+!!! Note
+    Since Hyperopt is a resource intensive process, running it on a Raspberry Pi is not recommended nor supported.
 
 ### Docker
 
-The docker-image includes hyperopt dependencies, no further action needed
+The docker-image includes hyperopt dependencies, no further action needed.
 
-!!! Note
-    Since Hyperopt is a resource intensive process, running it on a Raspberry Pi is not recommended, therefore the Raspberry image does not include Hyperopt dependencies.
-
-### Simple installation script (setup.sh) / Manual installation
+### Easy installation script (setup.sh) / Manual installation
 
 ```bash
 source .env/bin/activate

From 2f60d9cad4a6cc73542c6ff4f00d982241993f89 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Fri, 17 Apr 2020 23:23:22 +0300
Subject: [PATCH 150/151] minor: fix typo in the docs

---
 docs/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index d8a9653c3..338299781 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -34,7 +34,7 @@ The prevelance for all Options is as follows:
 
 - CLI arguments override any other option
 - Configuration files are used in sequence (last file wins), and override Strategy configurations.
-- Strategy configurations are only used if they are not set via configuration or via command line arguments. These options are market with [Strategy Override](#parameters-in-the-strategy) in the below table.
+- Strategy configurations are only used if they are not set via configuration or via command line arguments. These options are marked with [Strategy Override](#parameters-in-the-strategy) in the below table.
 
 Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways.
 

From c775d6512670916f206d39d02ca2ec8dc52ecfd8 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 18 Apr 2020 06:55:25 +0200
Subject: [PATCH 151/151] Update typehint for cancel_order

---
 freqtrade/exchange/exchange.py  | 4 ++--
 tests/exchange/test_exchange.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index d3a520eaa..1a0565959 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -922,9 +922,9 @@ class Exchange:
         return order.get('status') in ('closed', 'canceled') and order.get('filled') == 0.0
 
     @retrier
-    def cancel_order(self, order_id: str, pair: str) -> None:
+    def cancel_order(self, order_id: str, pair: str) -> Dict:
         if self._config['dry_run']:
-            return
+            return {}
 
         try:
             return self._api.cancel_order(order_id, pair)
diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py
index e3ab76b92..3c92612a0 100644
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@@ -1728,7 +1728,7 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange
 def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
     default_conf['dry_run'] = True
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
-    assert exchange.cancel_order(order_id='123', pair='TKN/BTC') is None
+    assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {}
 
 
 @pytest.mark.parametrize("exchange_name", EXCHANGES)