From 44d67389d2b88133c8ca745c4574b6c9fb4c3e10 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 4 Feb 2020 02:02:57 +0100 Subject: [PATCH 01/97] initial push of sortino, work not done, still need own tests --- freqtrade/optimize/hyperopt_loss_sortino.py | 49 +++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 freqtrade/optimize/hyperopt_loss_sortino.py diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss_sortino.py new file mode 100644 index 000000000..e84ecb402 --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sortino.py @@ -0,0 +1,49 @@ +""" +SortinoHyperOptLoss + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +from datetime import datetime + +from pandas import DataFrame +import numpy as np + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SortinoHyperOptLoss(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sharpe Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sharpe Ratio calculation. + """ + total_profit = results["profit_percent"] + days_period = (max_date - min_date).days + + # adding slippage of 0.1% per trade + total_profit = total_profit - 0.0005 + expected_returns_mean = total_profit.sum() / days_period + + results['downside_returns'] = 0 + results.loc[total_profit < 0, 'downside_returns'] = results['profit_percent'] + down_stdev = np.std(results['downside_returns']) + + if np.std(total_profit) != 0.0: + sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365) + else: + # Define high (negative) sharpe ratio to be clear that this is NOT optimal. + sortino_ratio = -20. + + # print(expected_returns_mean, down_stdev, sortino_ratio) + return -sortino_ratio From deb0b7ad675b16cdbbde41d289fbe45840689c9b Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 01:18:15 +0100 Subject: [PATCH 02/97] Added both SortinoHyperOptLoss and SortinoHyperOptLossDaily --- freqtrade/optimize/hyperopt_loss_sortino.py | 6 +- .../optimize/hyperopt_loss_sortino_daily.py | 64 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 freqtrade/optimize/hyperopt_loss_sortino_daily.py diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss_sortino.py index e84ecb402..83f644a43 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino.py +++ b/freqtrade/optimize/hyperopt_loss_sortino.py @@ -16,7 +16,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): """ Defines the loss function for hyperopt. - This implementation uses the Sharpe Ratio calculation. + This implementation uses the Sortino Ratio calculation. """ @staticmethod @@ -26,7 +26,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): """ Objective function, returns smaller number for more optimal results. - Uses Sharpe Ratio calculation. + Uses Sortino Ratio calculation. """ total_profit = results["profit_percent"] days_period = (max_date - min_date).days @@ -42,7 +42,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): if np.std(total_profit) != 0.0: sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365) else: - # Define high (negative) sharpe ratio to be clear that this is NOT optimal. + # Define high (negative) sortino ratio to be clear that this is NOT optimal. sortino_ratio = -20. # print(expected_returns_mean, down_stdev, sortino_ratio) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py new file mode 100644 index 000000000..b59baabcb --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -0,0 +1,64 @@ +""" +SortinoHyperOptLossDaily + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +import math +from datetime import datetime + +from pandas import DataFrame, date_range + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SortinoHyperOptLossDaily(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sortino Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sortino Ratio calculation. + """ + resample_freq = '1D' + slippage_per_trade_ratio = 0.0005 + days_in_year = 365 + annual_risk_free_rate = 0.0 + risk_free_rate = annual_risk_free_rate / days_in_year + + # apply slippage per trade to profit_percent + results.loc[:, 'profit_percent_after_slippage'] = \ + results['profit_percent'] - slippage_per_trade_ratio + + # create the index within the min_date and end max_date + t_index = date_range(start=min_date, end=max_date, freq=resample_freq) + + sum_daily = ( + results.resample(resample_freq, on='close_time').agg( + {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0) + ) + + total_profit = sum_daily["profit_percent_after_slippage"] - risk_free_rate + expected_returns_mean = total_profit.mean() + + results['downside_returns'] = 0 + results.loc[total_profit < 0, 'downside_returns'] = results['profit_percent'] + down_stdev = results['downside_returns'].std() + + if (down_stdev != 0.): + sortino_ratio = expected_returns_mean / down_stdev * math.sqrt(days_in_year) + else: + # Define high (negative) sortino ratio to be clear that this is NOT optimal. + sortino_ratio = -20. + + # print(t_index, sum_daily, total_profit) + # print(risk_free_rate, expected_returns_mean, down_stdev, sortino_ratio) + return -sortino_ratio From b56a1f060318c22fa414aa0060eeb9c7c0754ac9 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Tue, 4 Feb 2020 02:02:57 +0100 Subject: [PATCH 03/97] initial push of sortino, work not done, still need own tests --- freqtrade/optimize/hyperopt_loss_sortino.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss_sortino.py index 83f644a43..e84ecb402 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino.py +++ b/freqtrade/optimize/hyperopt_loss_sortino.py @@ -16,7 +16,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): """ Defines the loss function for hyperopt. - This implementation uses the Sortino Ratio calculation. + This implementation uses the Sharpe Ratio calculation. """ @staticmethod @@ -26,7 +26,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): """ Objective function, returns smaller number for more optimal results. - Uses Sortino Ratio calculation. + Uses Sharpe Ratio calculation. """ total_profit = results["profit_percent"] days_period = (max_date - min_date).days @@ -42,7 +42,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): if np.std(total_profit) != 0.0: sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365) else: - # Define high (negative) sortino ratio to be clear that this is NOT optimal. + # Define high (negative) sharpe ratio to be clear that this is NOT optimal. sortino_ratio = -20. # print(expected_returns_mean, down_stdev, sortino_ratio) From 728ab0ff2115dac6affd80c5b6b5f6bfb4e85a71 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 01:18:15 +0100 Subject: [PATCH 04/97] Added both SortinoHyperOptLoss and SortinoHyperOptLossDaily --- freqtrade/optimize/hyperopt_loss_sortino.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss_sortino.py index e84ecb402..83f644a43 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino.py +++ b/freqtrade/optimize/hyperopt_loss_sortino.py @@ -16,7 +16,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): """ Defines the loss function for hyperopt. - This implementation uses the Sharpe Ratio calculation. + This implementation uses the Sortino Ratio calculation. """ @staticmethod @@ -26,7 +26,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): """ Objective function, returns smaller number for more optimal results. - Uses Sharpe Ratio calculation. + Uses Sortino Ratio calculation. """ total_profit = results["profit_percent"] days_period = (max_date - min_date).days @@ -42,7 +42,7 @@ class SortinoHyperOptLoss(IHyperOptLoss): if np.std(total_profit) != 0.0: sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365) else: - # Define high (negative) sharpe ratio to be clear that this is NOT optimal. + # Define high (negative) sortino ratio to be clear that this is NOT optimal. sortino_ratio = -20. # print(expected_returns_mean, down_stdev, sortino_ratio) From 9bcc5d2eedb07417a0a468b3b6fa6514bc3fe2fd Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 01:24:03 +0100 Subject: [PATCH 05/97] fixed downside_returns to read from profit_percent_after_slippage --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index b59baabcb..0739379f6 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -50,7 +50,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): expected_returns_mean = total_profit.mean() results['downside_returns'] = 0 - results.loc[total_profit < 0, 'downside_returns'] = results['profit_percent'] + results.loc[total_profit < 0, 'downside_returns'] = results['profit_percent_after_slippage'] down_stdev = results['downside_returns'].std() if (down_stdev != 0.): From be34dc463b31e4698638d72dc2c295f2c3ec91f5 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 01:30:03 +0100 Subject: [PATCH 06/97] fixed bad commit From 951a19fb00f7e7203a430a254b583eca654cfd08 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 01:50:51 +0100 Subject: [PATCH 07/97] added tests for both sortino methods --- tests/optimize/test_hyperopt.py | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index b3356bd6d..5cdbe7b5a 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -360,6 +360,42 @@ def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results assert under > correct +def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: results_over = hyperopt_results.copy() results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 From a46b7bcd6d7705bb4d2c6adc4c85ddbf8d2ad18e Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 02:12:24 +0100 Subject: [PATCH 08/97] more fixes... --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 8 +++++--- tests/optimize/test_hyperopt.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index 0739379f6..31482b981 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -47,11 +47,13 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): ) total_profit = sum_daily["profit_percent_after_slippage"] - risk_free_rate + total_downside = sum_daily['downside_returns'] - risk_free_rate expected_returns_mean = total_profit.mean() - results['downside_returns'] = 0 - results.loc[total_profit < 0, 'downside_returns'] = results['profit_percent_after_slippage'] - down_stdev = results['downside_returns'].std() + sum_daily['downside_returns'] = 0 + sum_daily.loc[total_profit < 0, + 'downside_returns'] = sum_daily['profit_percent_after_slippage'] + down_stdev = total_downside.std() if (down_stdev != 0.): sortino_ratio = expected_returns_mean / down_stdev * math.sqrt(days_in_year) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 5cdbe7b5a..a2a98b0fb 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -380,9 +380,9 @@ def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_over['profit_percent_after_slippage'] = hyperopt_results['profit_percent_after_slippage'] * 2 results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + results_under['profit_percent_after_slippage'] = hyperopt_results['profit_percent_after_slippage'] / 2 default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) hl = HyperOptLossResolver.load_hyperoptloss(default_conf) From e8b9d88eb68cdda26b666dd42b8be39dccfc9940 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 02:14:12 +0100 Subject: [PATCH 09/97] moved line for total_downside --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index 31482b981..891e2bf1c 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -47,12 +47,12 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): ) total_profit = sum_daily["profit_percent_after_slippage"] - risk_free_rate - total_downside = sum_daily['downside_returns'] - risk_free_rate expected_returns_mean = total_profit.mean() sum_daily['downside_returns'] = 0 sum_daily.loc[total_profit < 0, 'downside_returns'] = sum_daily['profit_percent_after_slippage'] + total_downside = sum_daily['downside_returns'] - risk_free_rate down_stdev = total_downside.std() if (down_stdev != 0.): From 6b279f297c1eb0e8f75957f7562fc2b6784677fe Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Fri, 7 Feb 2020 02:22:05 +0100 Subject: [PATCH 10/97] fixed test --- tests/optimize/test_hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index a2a98b0fb..5cdbe7b5a 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -380,9 +380,9 @@ def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: results_over = hyperopt_results.copy() - results_over['profit_percent_after_slippage'] = hyperopt_results['profit_percent_after_slippage'] * 2 + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 results_under = hyperopt_results.copy() - results_under['profit_percent_after_slippage'] = hyperopt_results['profit_percent_after_slippage'] / 2 + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) hl = HyperOptLossResolver.load_hyperoptloss(default_conf) From 9ec9a7b124a05c4f25905e8b391d06b2ec17cc26 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sun, 9 Feb 2020 21:20:15 +0300 Subject: [PATCH 11/97] Fix t_index to be normalized --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index 891e2bf1c..c02f88434 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -39,7 +39,8 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): results['profit_percent'] - slippage_per_trade_ratio # create the index within the min_date and end max_date - t_index = date_range(start=min_date, end=max_date, freq=resample_freq) + t_index = date_range(start=min_date, end=max_date, freq=resample_freq, + normalize=True) sum_daily = ( results.resample(resample_freq, on='close_time').agg( From b2328cdf4fdf45c1e2f3842ec08f855fa04f1552 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Thu, 13 Feb 2020 07:07:35 +0300 Subject: [PATCH 12/97] Do not subtract risk_free_ratio twice --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index c02f88434..d869a4e4e 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -53,7 +53,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): sum_daily['downside_returns'] = 0 sum_daily.loc[total_profit < 0, 'downside_returns'] = sum_daily['profit_percent_after_slippage'] - total_downside = sum_daily['downside_returns'] - risk_free_rate + total_downside = sum_daily['downside_returns'] down_stdev = total_downside.std() if (down_stdev != 0.): From 161dd1a3e60e70928d7bab25f1a8913a079420f9 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 16 Feb 2020 03:55:16 +0300 Subject: [PATCH 13/97] Rename risk_free_return to minumum_accepted_return --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index d869a4e4e..72f70e3f9 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -31,8 +31,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): resample_freq = '1D' slippage_per_trade_ratio = 0.0005 days_in_year = 365 - annual_risk_free_rate = 0.0 - risk_free_rate = annual_risk_free_rate / days_in_year + minimum_acceptable_return = 0.0 # apply slippage per trade to profit_percent results.loc[:, 'profit_percent_after_slippage'] = \ @@ -47,7 +46,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0) ) - total_profit = sum_daily["profit_percent_after_slippage"] - risk_free_rate + total_profit = sum_daily["profit_percent_after_slippage"] - minimum_acceptable_return expected_returns_mean = total_profit.mean() sum_daily['downside_returns'] = 0 From 1e84b2770cfa2e0e5e8177615dfee178ac136c1f Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 16 Feb 2020 04:10:53 +0300 Subject: [PATCH 14/97] Fix values of downside_returns --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index 72f70e3f9..26eb54c25 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -50,8 +50,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): expected_returns_mean = total_profit.mean() sum_daily['downside_returns'] = 0 - sum_daily.loc[total_profit < 0, - 'downside_returns'] = sum_daily['profit_percent_after_slippage'] + sum_daily.loc[total_profit < 0, 'downside_returns'] = total_profit total_downside = sum_daily['downside_returns'] down_stdev = total_downside.std() From fbe5cc44da260d50db28dedf5acd12aad06913ce Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 16 Feb 2020 13:43:23 +0300 Subject: [PATCH 15/97] Use statistics.pstdev --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index 26eb54c25..09042cb48 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -5,6 +5,7 @@ This module defines the alternative HyperOptLoss class which can be used for Hyperoptimization. """ import math +import statistics from datetime import datetime from pandas import DataFrame, date_range @@ -52,7 +53,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): sum_daily['downside_returns'] = 0 sum_daily.loc[total_profit < 0, 'downside_returns'] = total_profit total_downside = sum_daily['downside_returns'] - down_stdev = total_downside.std() + down_stdev = statistics.pstdev(total_downside, 0) if (down_stdev != 0.): sortino_ratio = expected_returns_mean / down_stdev * math.sqrt(days_in_year) From 42dfda92316b4678107eac65359d56d5e8b6ee0a Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 16 Feb 2020 13:46:07 +0300 Subject: [PATCH 16/97] Adjust docstring --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index 09042cb48..22cd93984 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -28,6 +28,9 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): Objective function, returns smaller number for more optimal results. Uses Sortino Ratio calculation. + + Sortino Ratio calculated as described in + http://www.redrockcapital.com/Sortino__A__Sharper__Ratio_Red_Rock_Capital.pdf """ resample_freq = '1D' slippage_per_trade_ratio = 0.0005 From 674898bd32eede682dad5a688ee1428b89a8805c Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 16 Feb 2020 15:26:40 +0300 Subject: [PATCH 17/97] Fix usage of vars in the commented out line --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index 22cd93984..0f81ffca5 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -65,5 +65,5 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): sortino_ratio = -20. # print(t_index, sum_daily, total_profit) - # print(risk_free_rate, expected_returns_mean, down_stdev, sortino_ratio) + # print(minimum_acceptable_return, expected_returns_mean, down_stdev, sortino_ratio) return -sortino_ratio From 2058b492eb2b668616f06d0d001e01ffca17e38f Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Tue, 18 Feb 2020 22:46:53 +0100 Subject: [PATCH 18/97] Added function to print hyperopt-list as table using tabulate --- docs/utils.md | 1 + freqtrade/commands/arguments.py | 3 +- freqtrade/commands/cli_options.py | 6 +++ freqtrade/commands/hyperopt_commands.py | 36 ++++++++++-------- freqtrade/configuration/configuration.py | 3 ++ freqtrade/optimize/hyperopt.py | 47 +++++++++++++++++++++++- tests/commands/test_commands.py | 16 ++++++++ 7 files changed, 95 insertions(+), 17 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index abb7fd0db..6ca0b7920 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -440,6 +440,7 @@ optional arguments: --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. --print-json Print best result detailization in JSON format. + --print-table Print results in table format. --no-details Do not print best epoch details. Common arguments: diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index fe6f49039..0f6478a60 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -66,7 +66,8 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit", "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit", - "print_colorized", "print_json", "hyperopt_list_no_details"] + "print_colorized", "print_json", "print_table", + "hyperopt_list_no_details"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", "print_json", "hyperopt_show_no_header"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 1776955b1..9efc1151a 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -220,6 +220,12 @@ AVAILABLE_CLI_OPTIONS = { action='store_true', default=False, ), + "print_table": Arg( + '--print-table', + help='Print results in table format.', + action='store_true', + default=False, + ), "hyperopt_jobs": Arg( '-j', '--job-workers', help='The number of concurrently running jobs for hyperoptimization ' diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 8c1c80d98..88cd79468 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -21,6 +21,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: print_colorized = config.get('print_colorized', False) print_json = config.get('print_json', False) + print_table = config.get('print_table', False) no_details = config.get('hyperopt_list_no_details', False) no_header = False @@ -52,14 +53,20 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if print_colorized: colorama_init(autoreset=True) - try: - # Human-friendly indexes used here (starting from 1) - for val in trials[epoch_start:epoch_stop]: - Hyperopt.print_results_explanation(val, total_epochs, - not filteroptions['only_best'], print_colorized) - - except KeyboardInterrupt: - print('User interrupted..') + if print_table: + try: + Hyperopt.print_result_table(config, trials, total_epochs, + not filteroptions['only_best'], print_colorized) + except KeyboardInterrupt: + print('User interrupted..') + else: + try: + # Human-friendly indexes used here (starting from 1) + for val in trials[epoch_start:epoch_stop]: + Hyperopt.print_results_explanation(val, total_epochs, + not filteroptions['only_best'], print_colorized) + except KeyboardInterrupt: + print('User interrupted..') if trials and not no_details: sorted_trials = sorted(trials, key=itemgetter('loss')) @@ -75,6 +82,12 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + print_json = config.get('print_json', False) + no_header = config.get('hyperopt_show_no_header', False) + trials_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + n = config.get('hyperopt_show_index', -1) + filteroptions = { 'only_best': config.get('hyperopt_list_best', False), 'only_profitable': config.get('hyperopt_list_profitable', False), @@ -87,10 +100,6 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None) } - no_header = config.get('hyperopt_show_no_header', False) - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') # Previous evaluations trials = Hyperopt.load_previous_results(trials_file) @@ -99,7 +108,6 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: trials = _hyperopt_filter_trials(trials, filteroptions) trials_epochs = len(trials) - n = config.get('hyperopt_show_index', -1) if n > trials_epochs: raise OperationalException( f"The index of the epoch to show should be less than {trials_epochs + 1}.") @@ -111,8 +119,6 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: if n > 0: n -= 1 - print_json = config.get('print_json', False) - if trials: val = trials[n] Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index c2613ba99..0adfe03e7 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -286,6 +286,9 @@ class Configuration: self._args_to_config(config, argname='print_json', logstring='Parameter --print-json detected ...') + self._args_to_config(config, argname='print_table', + logstring='Parameter --print-table detected: {}') + self._args_to_config(config, argname='hyperopt_jobs', logstring='Parameter -j/--job-workers detected: {}') diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ff6e7f3bc..72b3516c7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -20,7 +20,8 @@ from colorama import Fore, Style from colorama import init as colorama_init from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) -from pandas import DataFrame +from pandas import DataFrame, json_normalize, isna +from tabulate import tabulate from freqtrade.data.history import get_timerange, trim_dataframe from freqtrade.exceptions import OperationalException @@ -295,6 +296,50 @@ class Hyperopt: f"{results['results_explanation']} " + f"Objective: {results['loss']:.5f}") + @staticmethod + def print_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, + print_colorized: bool) -> None: + """ + Log result table + """ + if not results: + return + + trials = json_normalize(results, max_level=1) + trials = trials[['is_best', 'current_epoch', + 'results_metrics.trade_count', 'results_metrics.avg_profit', + 'results_metrics.total_profit', 'results_metrics.profit', + 'results_metrics.duration', 'loss']] + trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', + 'Profit', 'Avg duration', 'Objective'] + + trials['Best'] = trials['Best'].apply(lambda x: '*' if x else '') + trials['Objective'] = trials['Objective'].astype(str) + + if print_colorized: + for i in range(len(trials)): + if trials.loc[i]['Total profit'] > 0: + trials.at[i, 'Best'] = Fore.GREEN + trials.loc[i]['Best'] + trials.at[i, 'Objective'] = "{}{}".format(trials.loc[i]['Objective'], + Fore.RESET) + if '*' in trials.loc[i]['Best'] and highlight_best: + trials.at[i, 'Best'] = Style.BRIGHT + trials.loc[i]['Best'] + trials.at[i, 'Objective'] = "{}{}".format(trials.loc[i]['Objective'], + Style.RESET_ALL) + + trials['Epoch'] = trials['Epoch'].apply( + lambda x: "{}/{}".format(x, total_epochs)) + trials['Avg profit'] = trials['Avg profit'].apply( + lambda x: '{:,.2f}%'.format(x) if not isna(x) else x) + trials['Profit'] = trials['Profit'].apply( + lambda x: '{:,.2f}%'.format(x) if not isna(x) else x) + trials['Total profit'] = trials['Total profit'].apply( + lambda x: '{: 11.8f} '.format(x) + config['stake_currency'] if not isna(x) else x) + trials['Avg duration'] = trials['Avg duration'].apply( + lambda x: '{:,.1f}m'.format(x) if not isna(x) else x) + + print(tabulate(trials.to_dict(orient='list'), headers='keys', tablefmt='psql')) + def has_space(self, space: str) -> bool: """ Tell if the space value is contained in the configuration diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index ee1db5db5..9ecd990ec 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -893,6 +893,22 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): assert all(x not in captured.out for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12" " 9/12", " 10/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--print-table", + "--max-trades", "20" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) def test_hyperopt_show(mocker, capsys, hyperopt_results): From 585545405dea4d34bc1ce6d8296ceb38861a7b85 Mon Sep 17 00:00:00 2001 From: Fredrik Rydin Date: Wed, 19 Feb 2020 00:51:44 +0100 Subject: [PATCH 19/97] Changed tests --- tests/commands/test_commands.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 9ecd990ec..9c233019c 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -895,20 +895,20 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): " 9/12", " 10/12", " 11/12", " 12/12"]) args = [ "hyperopt-list", - "--profitable", "--no-details", "--print-table", - "--max-trades", "20" + "--min-trades", "100", + "--print-json" ] pargs = get_args(args) pargs['config'] = None start_hyperopt_list(pargs) captured = capsys.readouterr() assert all(x in captured.out - for x in [" 2/12", " 10/12"]) + for x in [" 3/12", " 7/12", " 9/12", " 11/12"]) assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12", " 12/12"]) + for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 6/12", " 8/12", " 10/12" + " 12/12"]) def test_hyperopt_show(mocker, capsys, hyperopt_results): From df26c357d293f69e57cffc4de60284776d183026 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Wed, 19 Feb 2020 01:31:25 +0100 Subject: [PATCH 20/97] doc updates --- docs/bot-usage.md | 12 +++++++----- docs/hyperopt.md | 14 ++++++++------ freqtrade/commands/cli_options.py | 3 ++- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 56e6008a1..e0d96a8c2 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -270,7 +270,7 @@ Check the corresponding [Data Downloading](data-download.md) section for more de ## Hyperopt commands To optimize your strategy, you can use hyperopt parameter hyperoptimization -to find optimal parameter values for your stategy. +to find optimal parameter values for your strategy. ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] @@ -318,7 +318,7 @@ optional arguments: --print-all Print all results, not only the best ones. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. - --print-json Print best result detailization in JSON format. + --print-json Print best results in JSON format. -j JOBS, --job-workers JOBS The number of concurrently running jobs for hyperoptimization (hyperopt worker processes). If -1 @@ -336,9 +336,11 @@ optional arguments: class (IHyperOptLoss). Different functions can generate completely different results, since the target for optimization is different. Built-in - Hyperopt-loss-functions are: DefaultHyperOptLoss, - OnlyProfitHyperOptLoss, SharpeHyperOptLoss, - SharpeHyperOptLossDaily (default: `DefaultHyperOptLoss`). + Hyperopt-loss-functions are: + DefaultHyperOptLoss, OnlyProfitHyperOptLoss, + SharpeHyperOptLoss, SharpeHyperOptLossDaily, + SortinoHyperOptLoss, SortinoHyperOptLossDaily. + (default: `DefaultHyperOptLoss`). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 3e10f66da..3fb7ce7ba 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -81,11 +81,11 @@ There are two places you need to change in your hyperopt file to add a new buy h There you have two different types of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. -2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". +2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower Bollinger band". Hyperoptimization will, for each eval round, pick one trigger and possibly multiple guards. The constructed strategy will be something like -"*buy exactly when close price touches lower bollinger band, BUT only if +"*buy exactly when close price touches lower Bollinger band, BUT only if ADX > 10*". If you have updated the buy strategy, i.e. changed the contents of @@ -172,7 +172,7 @@ So let's write the buy strategy using these values: Hyperopting will now call this `populate_buy_trend` as many times you ask it (`epochs`) with different value combinations. It will then use the given historical data and make buys based on the buy signals generated with the above function and based on the results -it will end with telling you which paramter combination produced the best profits. +it will end with telling you which parameter combination produced the best profits. The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. When you want to test an indicator that isn't used by the bot currently, remember to @@ -191,8 +191,10 @@ Currently, the following loss functions are builtin: * `DefaultHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) * `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) -* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on the trade returns) -* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on daily trade returns) +* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on trade returns relative to **upside** standard deviation) +* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on **daily** trade returns relative to **upside** standard deviation) +* `SortinoHyperOptLoss` (optimizes Sortino Ratio calculated on trade returns relative to **downside** standard deviation) +* `SortinoHyperOptLossDaily` (optimizes Sortino Ratio calculated on **daily** trade returns relative to **downside** standard deviation) Creation of a custom loss function is covered in the [Advanced Hyperopt](advanced-hyperopt.md) part of the documentation. @@ -272,7 +274,7 @@ In some situations, you may need to run Hyperopt (and Backtesting) with the By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one open trade is allowed for every traded pair. The total number of trades open for all pairs is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to -some potential trades to be hidden (or masked) by previosly open trades. +some potential trades to be hidden (or masked) by previously open trades. The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times, while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades` diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 6d8d13129..5faff1a97 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -256,7 +256,8 @@ AVAILABLE_CLI_OPTIONS = { help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). ' 'Different functions can generate completely different results, ' 'since the target for optimization is different. Built-in Hyperopt-loss-functions are: ' - 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily.' + 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, ' + 'SortinoHyperOptLoss, SortinoHyperOptLossDaily.' '(default: `%(default)s`).', metavar='NAME', default=constants.DEFAULT_HYPEROPT_LOSS, From 41b4fa3b7f93018a5c01042b8aa8734108e57407 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Wed, 19 Feb 2020 02:59:51 +0100 Subject: [PATCH 21/97] fixed two more typos --- docs/hyperopt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index b471f849e..1f9907b83 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -31,9 +31,9 @@ This will create a new hyperopt file from a template, which will be located unde Depending on the space you want to optimize, only some of the below are required: * fill `buy_strategy_generator` - for buy signal optimization -* fill `indicator_space` - for buy signal optimzation +* fill `indicator_space` - for buy signal optimization * fill `sell_strategy_generator` - for sell signal optimization -* fill `sell_indicator_space` - for sell signal optimzation +* fill `sell_indicator_space` - for sell signal optimization !!! Note `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. From 09a1c9eed6b629fe9f4502cbcf805b8e78b8da13 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Wed, 19 Feb 2020 22:25:34 +0100 Subject: [PATCH 22/97] fixed docs description of hyperopts --- docs/hyperopt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 1f9907b83..9bc5888ce 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -191,8 +191,8 @@ Currently, the following loss functions are builtin: * `DefaultHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) * `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) -* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on trade returns relative to **upside** standard deviation) -* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on **daily** trade returns relative to **upside** standard deviation) +* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on trade returns relative to standard deviation) +* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on **daily** trade returns relative to standard deviation) * `SortinoHyperOptLoss` (optimizes Sortino Ratio calculated on trade returns relative to **downside** standard deviation) * `SortinoHyperOptLossDaily` (optimizes Sortino Ratio calculated on **daily** trade returns relative to **downside** standard deviation) From bca5f804a87d2b2e1556711a1f96d35161d279a2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 20 Feb 2020 08:17:24 +0300 Subject: [PATCH 23/97] Move divider log message --- freqtrade/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 64cc97026..509ba018e 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -109,14 +109,14 @@ class Worker: """ start = time.time() result = func(*args, **kwargs) + logger.debug("========================================") end = time.time() duration = max(min_secs - (end - start), 0.0) - logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) + logger.debug(f"Throttling {func.__name__} for {duration:.2f} seconds") time.sleep(duration) return result def _process(self) -> None: - logger.debug("========================================") try: self.freqtrade.process() except TemporaryError as error: From 56a06cbd331daa07216b77094d6b98ce0a651439 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 20 Feb 2020 08:19:22 +0300 Subject: [PATCH 24/97] Update strings to f-strings --- freqtrade/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 509ba018e..b3b3b712a 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -26,7 +26,7 @@ class Worker: """ Init all variables and objects the bot needs to work """ - logger.info('Starting worker %s', __version__) + logger.info(f"Starting worker {__version__}") self._args = args self._config = config @@ -77,7 +77,7 @@ class Worker: if state != old_state: self.freqtrade.notify_status(f'{state.name.lower()}') - logger.info('Changing state to: %s', state.name) + logger.info(f"Changing state to: {state.name}") if state == State.RUNNING: self.freqtrade.startup() From 78ee36a8c6e8d17289059be945324c8166d514dd Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 20 Feb 2020 15:18:26 +0300 Subject: [PATCH 25/97] Use _throttle() in stopped state instead of sleep() --- freqtrade/worker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index b3b3b712a..c397beaab 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -87,7 +87,7 @@ class Worker: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: STOPPED.") - time.sleep(throttle_secs) + self._throttle(func=self._process_stopped, min_secs=throttle_secs) elif state == State.RUNNING: # Ping systemd watchdog before throttling @@ -95,7 +95,7 @@ class Worker: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: RUNNING.") - self._throttle(func=self._process, min_secs=throttle_secs) + self._throttle(func=self._process_running, min_secs=throttle_secs) return state @@ -116,7 +116,11 @@ class Worker: time.sleep(duration) return result - def _process(self) -> None: + def _process_stopped(self) -> None: + # Maybe do here something in the future... + pass + + def _process_running(self) -> None: try: self.freqtrade.process() except TemporaryError as error: From e7b12704dec4c8265e5adf51dfb3d7f36367adae Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Thu, 20 Feb 2020 19:12:55 +0100 Subject: [PATCH 26/97] Added test for details --- tests/commands/test_commands.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 9c233019c..dfaa936ed 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -778,6 +778,19 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): assert all(x not in captured.out for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params", "Sell hyperspace params", "ROI table", "Stoploss"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) args = [ "hyperopt-list", "--no-details", From 09226fd5d5e378e9f7ee11382528ec6c06ac7571 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Thu, 20 Feb 2020 19:18:42 +0100 Subject: [PATCH 27/97] PEP8 correction --- tests/commands/test_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index dfaa936ed..ceae6f372 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -787,7 +787,8 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): start_hyperopt_list(pargs) captured = capsys.readouterr() assert all(x in captured.out - for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params", "Sell hyperspace params", "ROI table", "Stoploss"]) + for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params", + "Sell hyperspace params", "ROI table", "Stoploss"]) assert all(x not in captured.out for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", " 11/12", " 12/12"]) From 04aa74e5add9425f73e2485a98a319c32b3ca2ad Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 03:37:38 +0300 Subject: [PATCH 28/97] Better throttling --- freqtrade/worker.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index c397beaab..40bfb54d8 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -108,12 +108,13 @@ class Worker: :return: Any """ start = time.time() - result = func(*args, **kwargs) logger.debug("========================================") - end = time.time() - duration = max(min_secs - (end - start), 0.0) - logger.debug(f"Throttling {func.__name__} for {duration:.2f} seconds") - time.sleep(duration) + result = func(*args, **kwargs) + time_passed = time.time() - start + sleep_duration = max(min_secs - time_passed, 0.0) + logger.debug(f"Throttling with '{func.__name__}()': sleep for {sleep_duration:.2f} s, " + f"last iteration took {time_passed:.2f} s.") + time.sleep(sleep_duration) return result def _process_stopped(self) -> None: From e0800b7c29fb0fe3b53afe2fa6732da7f37e06c1 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 03:52:14 +0300 Subject: [PATCH 29/97] Make throttle start time an worker object attribute --- freqtrade/worker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 40bfb54d8..dc8f9109f 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -32,6 +32,8 @@ class Worker: self._config = config self._init(False) + self.last_throttle_start_time: float = None + # Tell systemd that we completed initialization phase if self._sd_notify: logger.debug("sd_notify: READY=1") @@ -107,10 +109,10 @@ class Worker: :param min_secs: minimum execution time in seconds :return: Any """ - start = time.time() + self.last_throttle_start_time = time.time() logger.debug("========================================") result = func(*args, **kwargs) - time_passed = time.time() - start + time_passed = time.time() - self.last_throttle_start_time sleep_duration = max(min_secs - time_passed, 0.0) logger.debug(f"Throttling with '{func.__name__}()': sleep for {sleep_duration:.2f} s, " f"last iteration took {time_passed:.2f} s.") From 881f602f91b6afa5607f44eed8ad06165951ebe6 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 04:00:23 +0300 Subject: [PATCH 30/97] Adjust methods params --- freqtrade/worker.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index dc8f9109f..088526d85 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -32,7 +32,7 @@ class Worker: self._config = config self._init(False) - self.last_throttle_start_time: float = None + self.last_throttle_start_time: Optional[float] = None # Tell systemd that we completed initialization phase if self._sd_notify: @@ -65,15 +65,13 @@ class Worker: if state == State.RELOAD_CONF: self._reconfigure() - def _worker(self, old_state: Optional[State], throttle_secs: Optional[float] = None) -> State: + def _worker(self, old_state: Optional[State]) -> State: """ Trading routine that must be run at each loop :param old_state: the previous service state from the previous call :return: current service state """ state = self.freqtrade.state - if throttle_secs is None: - throttle_secs = self._throttle_secs # Log state transition if state != old_state: @@ -89,7 +87,7 @@ class Worker: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: STOPPED.") - self._throttle(func=self._process_stopped, min_secs=throttle_secs) + self._throttle(func=self._process_stopped, throttle_secs=self._throttle_secs) elif state == State.RUNNING: # Ping systemd watchdog before throttling @@ -97,23 +95,23 @@ class Worker: logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.") self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: RUNNING.") - self._throttle(func=self._process_running, min_secs=throttle_secs) + self._throttle(func=self._process_running, throttle_secs=self._throttle_secs) return state - def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: + def _throttle(self, func: Callable[..., Any], throttle_secs: float, *args, **kwargs) -> Any: """ Throttles the given callable that it takes at least `min_secs` to finish execution. :param func: Any callable - :param min_secs: minimum execution time in seconds - :return: Any + :param throttle_secs: throttling interation execution time limit in seconds + :return: Any (result of execution of func) """ self.last_throttle_start_time = time.time() logger.debug("========================================") result = func(*args, **kwargs) time_passed = time.time() - self.last_throttle_start_time - sleep_duration = max(min_secs - time_passed, 0.0) + sleep_duration = max(throttle_secs - time_passed, 0.0) logger.debug(f"Throttling with '{func.__name__}()': sleep for {sleep_duration:.2f} s, " f"last iteration took {time_passed:.2f} s.") time.sleep(sleep_duration) From 269a669af82b586e69d061ed5866b8c56af173d0 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 05:07:31 +0300 Subject: [PATCH 31/97] Move heartbeat to worker --- freqtrade/freqtradebot.py | 10 ---------- freqtrade/worker.py | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 127586437..00d5c369a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -6,7 +6,6 @@ import logging import traceback from datetime import datetime from math import isclose -from os import getpid from threading import Lock from typing import Any, Dict, List, Optional, Tuple @@ -52,10 +51,6 @@ class FreqtradeBot: # Init objects self.config = config - self._heartbeat_msg = 0 - - self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) - self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) # Check config consistency here since strategies can set certain options @@ -159,11 +154,6 @@ class FreqtradeBot: self.check_handle_timedout() Trade.session.flush() - if (self.heartbeat_interval - and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)): - logger.info(f"Bot heartbeat. PID={getpid()}") - self._heartbeat_msg = arrow.utcnow().timestamp - def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]: """ Refresh whitelist from pairlist or edge and extend it with trades. diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 088526d85..adce7ddda 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -4,8 +4,10 @@ Main Freqtrade worker class. import logging import time import traceback +from os import getpid from typing import Any, Callable, Dict, Optional +import arrow import sdnotify from freqtrade import __version__, constants @@ -33,6 +35,7 @@ class Worker: self._init(False) self.last_throttle_start_time: Optional[float] = None + self._heartbeat_msg = 0 # Tell systemd that we completed initialization phase if self._sd_notify: @@ -50,10 +53,10 @@ class Worker: # Init the instance of the bot self.freqtrade = FreqtradeBot(self._config) - self._throttle_secs = self._config.get('internals', {}).get( - 'process_throttle_secs', - constants.PROCESS_THROTTLE_SECS - ) + internals_config = self._config.get('internals', {}) + self._throttle_secs = internals_config.get('process_throttle_secs', + constants.PROCESS_THROTTLE_SECS) + self._heartbeat_interval = internals_config.get('heartbeat_interval', 60) self._sd_notify = sdnotify.SystemdNotifier() if \ self._config.get('internals', {}).get('sd_notify', False) else None @@ -97,6 +100,11 @@ class Worker: self._throttle(func=self._process_running, throttle_secs=self._throttle_secs) + if (self._heartbeat_interval + and (arrow.utcnow().timestamp - self._heartbeat_msg > self._heartbeat_interval)): + logger.info(f"Bot heartbeat. PID={getpid()}") + self._heartbeat_msg = arrow.utcnow().timestamp + return state def _throttle(self, func: Callable[..., Any], throttle_secs: float, *args, **kwargs) -> Any: From d2e20d86bb8808ac11427b08b9d2c3b1d27723bd Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 05:31:21 +0300 Subject: [PATCH 32/97] Align heartbeat to throttling logging --- freqtrade/worker.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index adce7ddda..523b9038f 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -7,7 +7,6 @@ import traceback from os import getpid from typing import Any, Callable, Dict, Optional -import arrow import sdnotify from freqtrade import __version__, constants @@ -34,8 +33,8 @@ class Worker: self._config = config self._init(False) - self.last_throttle_start_time: Optional[float] = None - self._heartbeat_msg = 0 + self.last_throttle_start_time: float = 0 + self._heartbeat_msg: float = 0 # Tell systemd that we completed initialization phase if self._sd_notify: @@ -100,10 +99,11 @@ class Worker: self._throttle(func=self._process_running, throttle_secs=self._throttle_secs) - if (self._heartbeat_interval - and (arrow.utcnow().timestamp - self._heartbeat_msg > self._heartbeat_interval)): - logger.info(f"Bot heartbeat. PID={getpid()}") - self._heartbeat_msg = arrow.utcnow().timestamp + if self._heartbeat_interval: + now = time.time() + if (now - self._heartbeat_msg) > self._heartbeat_interval: + logger.info(f"Bot heartbeat. PID={getpid()}") + self._heartbeat_msg = now return state From d9ecf3e4bfbdf2969c772ffb06e88ddecba4cf56 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 21 Feb 2020 12:26:32 +0300 Subject: [PATCH 33/97] Add version and state to heartbeat message --- freqtrade/worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 523b9038f..f4b9f275b 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -102,7 +102,8 @@ class Worker: if self._heartbeat_interval: now = time.time() if (now - self._heartbeat_msg) > self._heartbeat_interval: - logger.info(f"Bot heartbeat. PID={getpid()}") + logger.info(f"Bot heartbeat. PID={getpid()}, " + f"version='{__version__}', state='{state.name}'") self._heartbeat_msg = now return state From f5b4a6d3d72fea870d912e6a1b28c47ccd3247dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 11:03:25 +0100 Subject: [PATCH 34/97] Remove fetch_ticker caching --- freqtrade/exchange/exchange.py | 35 +++++++++++---------------------- freqtrade/freqtradebot.py | 4 ++-- tests/exchange/test_exchange.py | 15 +++----------- tests/rpc/test_rpc.py | 6 ------ 4 files changed, 16 insertions(+), 44 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b3b347016..e45238b07 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -66,8 +66,6 @@ class Exchange: self._config.update(config) - self._cached_ticker: Dict[str, Any] = {} - # Holds last candle refreshed time of each pair self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} # Timestamp of last markets refresh @@ -591,28 +589,17 @@ class Exchange: raise OperationalException(e) from e @retrier - def fetch_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: - if refresh or pair not in self._cached_ticker.keys(): - try: - if pair not in self._api.markets or not self._api.markets[pair].get('active'): - raise DependencyException(f"Pair {pair} not available") - data = self._api.fetch_ticker(pair) - try: - self._cached_ticker[pair] = { - 'bid': float(data['bid']), - 'ask': float(data['ask']), - } - except KeyError: - logger.debug("Could not cache ticker data for %s", pair) - return data - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - else: - logger.info("returning cached ticker-data for %s", pair) - return self._cached_ticker[pair] + def fetch_ticker(self, pair: str) -> dict: + try: + if pair not in self._api.markets or not self._api.markets[pair].get('active'): + raise DependencyException(f"Pair {pair} not available") + data = self._api.fetch_ticker(pair) + return data + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 127586437..032e3e8f7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -253,7 +253,7 @@ class FreqtradeBot: else: if not tick: logger.info('Using Last Ask / Last Price') - ticker = self.exchange.fetch_ticker(pair, refresh) + ticker = self.exchange.fetch_ticker(pair) else: ticker = tick if ticker['ask'] < ticker['last']: @@ -631,7 +631,7 @@ class FreqtradeBot: rate = order_book['bids'][0][0] else: - rate = self.exchange.fetch_ticker(pair, refresh)['bid'] + rate = self.exchange.fetch_ticker(pair)['bid'] return rate def handle_trade(self, trade: Trade) -> bool: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8b2e439c3..3830d0acb 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1121,25 +1121,16 @@ def test_fetch_ticker(default_conf, mocker, exchange_name): assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 - assert 'ETH/BTC' in exchange._cached_ticker - assert exchange._cached_ticker['ETH/BTC']['bid'] == 0.5 - assert exchange._cached_ticker['ETH/BTC']['ask'] == 1 - - # Test caching - api_mock.fetch_ticker = MagicMock() - exchange.fetch_ticker(pair='ETH/BTC', refresh=False) - assert api_mock.fetch_ticker.call_count == 0 - ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, "fetch_ticker", "fetch_ticker", - pair='ETH/BTC', refresh=True) + pair='ETH/BTC') api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.fetch_ticker(pair='ETH/BTC', refresh=True) + exchange.fetch_ticker(pair='ETH/BTC') with pytest.raises(DependencyException, match=r'Pair XRP/ETH not available'): - exchange.fetch_ticker(pair='XRP/ETH', refresh=True) + exchange.fetch_ticker(pair='XRP/ETH') @pytest.mark.parametrize("exchange_name", EXCHANGES) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index a35bfa0d6..2d1fa5b2d 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -67,8 +67,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) assert isnan(results[0]['current_rate']) @@ -136,8 +134,6 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] assert 'ETH/BTC' in result[0][1] @@ -262,8 +258,6 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Test non-available pair mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' From 97e6e5e9766c87bb728fe396f4f5119616eeda1a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 11:12:33 +0100 Subject: [PATCH 35/97] Implement caching in the correct place --- freqtrade/freqtradebot.py | 32 ++++++++++++++++++++++++++------ tests/rpc/test_rpc.py | 6 +++--- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 032e3e8f7..d9126370b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -11,6 +11,7 @@ from threading import Lock from typing import Any, Dict, List, Optional, Tuple import arrow +from cachetools import TTLCache from requests.exceptions import RequestException from freqtrade import __version__, constants, persistence @@ -54,6 +55,9 @@ class FreqtradeBot: self._heartbeat_msg = 0 + self._sell_rate_cache = TTLCache(maxsize=100, ttl=5) + self._buy_rate_cache = TTLCache(maxsize=100, ttl=5) + self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) @@ -234,11 +238,19 @@ class FreqtradeBot: return trades_created - def get_buy_rate(self, pair: str, refresh: bool, tick: Dict = None) -> float: + def get_buy_rate(self, pair: str, refresh: bool) -> float: """ Calculates bid target between current ask price and last price + :param pair: Pair to get rate for + :param refresh: allow cached data :return: float: Price """ + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + return rate + config_bid_strategy = self.config.get('bid_strategy', {}) if 'use_order_book' in config_bid_strategy and\ config_bid_strategy.get('use_order_book', False): @@ -251,11 +263,8 @@ class FreqtradeBot: logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) used_rate = order_book_rate else: - if not tick: - logger.info('Using Last Ask / Last Price') - ticker = self.exchange.fetch_ticker(pair) - else: - ticker = tick + logger.info('Using Last Ask / Last Price') + ticker = self.exchange.fetch_ticker(pair) if ticker['ask'] < ticker['last']: ticker_rate = ticker['ask'] else: @@ -263,6 +272,8 @@ class FreqtradeBot: ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) used_rate = ticker_rate + self._buy_rate_cache[pair] = used_rate + return used_rate def get_trade_stake_amount(self, pair: str) -> float: @@ -621,8 +632,16 @@ class FreqtradeBot: The orderbook portion is only used for rpc messaging, which would otherwise fail for BitMex (has no bid/ask in fetch_ticker) or remain static in any other case since it's not updating. + :param pair: Pair to get rate for + :param refresh: allow cached data :return: Bid rate """ + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + return rate + config_ask_strategy = self.config.get('ask_strategy', {}) if config_ask_strategy.get('use_order_book', False): logger.debug('Using order book to get sell rate') @@ -632,6 +651,7 @@ class FreqtradeBot: else: rate = self.exchange.fetch_ticker(pair)['bid'] + self._sell_rate_cache[pair] = rate return rate def handle_trade(self, trade: Trade) -> bool: diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 2d1fa5b2d..40b2d6627 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -65,7 +65,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_order': '(limit buy rem=0.00000000)' } == results[0] - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) @@ -132,7 +132,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert 'ETH/BTC' in result[0][1] assert '-0.59% (-0.09)' == result[0][3] - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] @@ -256,7 +256,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 From 77ef3240cd17b932f499a19e7820e39818cf1aa9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 11:16:20 +0100 Subject: [PATCH 36/97] Implement log messages --- freqtrade/freqtradebot.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d9126370b..4b54b79e1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -246,9 +246,10 @@ class FreqtradeBot: :return: float: Price """ if not refresh: - rate = self._sell_rate_cache.get(pair) + rate = self._buy_rate_cache.get(pair) # Check if cache has been invalidated if rate: + logger.info(f"Using cached buy rate for {pair}.") return rate config_bid_strategy = self.config.get('bid_strategy', {}) @@ -577,7 +578,7 @@ class FreqtradeBot: """ Sends rpc notification when a buy cancel occured. """ - current_rate = self.get_buy_rate(trade.pair, True) + current_rate = self.get_buy_rate(trade.pair, False) msg = { 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, @@ -640,6 +641,7 @@ class FreqtradeBot: rate = self._sell_rate_cache.get(pair) # Check if cache has been invalidated if rate: + logger.info(f"Using cached sell rate for {pair}.") return rate config_ask_strategy = self.config.get('ask_strategy', {}) @@ -1078,7 +1080,7 @@ class FreqtradeBot: """ profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - current_rate = self.get_sell_rate(trade.pair, True) + current_rate = self.get_sell_rate(trade.pair, False) profit_percent = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_percent > 0 else "loss" From 2fe7b683cb3a11b4c629b7a796b30ff5838a5682 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 11:23:13 +0100 Subject: [PATCH 37/97] Add tests for cached rates --- tests/test_freqtradebot.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5ed4d296c..c7a70be8c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -915,13 +915,21 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: (5, 10, 1.0, 5), # last bigger than ask (5, 10, 0.5, 5), # last bigger than ask ]) -def test_get_buy_rate(mocker, default_conf, ask, last, last_ab, expected) -> None: +def test_get_buy_rate(mocker, default_conf, caplog, ask, last, last_ab, expected) -> None: default_conf['bid_strategy']['ask_last_balance'] = last_ab freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={'ask': ask, 'last': last})) assert freqtrade.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + assert freqtrade.get_buy_rate('ETH/BTC', False) == expected + assert log_has("Using cached buy rate for ETH/BTC.", caplog) + # Running a 2nd time with Refresh on! + caplog.clear() + assert freqtrade.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: @@ -3614,7 +3622,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order assert freqtrade.handle_trade(trade) is True -def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: +def test_get_sell_rate(default_conf, mocker, caplog, ticker, order_book_l2) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -3626,8 +3634,15 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: # Test regular mode ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) assert rate == 0.00001098 + # Use caching + rate = ft.get_sell_rate(pair, False) + assert rate == 0.00001098 + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + caplog.clear() # Test orderbook mode default_conf['ask_strategy']['use_order_book'] = True @@ -3635,8 +3650,12 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: default_conf['ask_strategy']['order_book_max'] = 2 ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) assert rate == 0.043936 + rate = ft.get_sell_rate(pair, False) + assert rate == 0.043936 + assert log_has("Using cached sell rate for ETH/BTC.", caplog) def test_startup_state(default_conf, mocker): From 7ecc56fa44d56d3ceaf70cc81b205cbcaff90795 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 13:10:41 +0100 Subject: [PATCH 38/97] Load ohlcv data as float --- freqtrade/data/history/jsondatahandler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 17b9fd7d7..7219d8c01 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -69,7 +69,7 @@ class JsonDataHandler(IDataHandler): filename = self._pair_data_filename(self._datadir, pair, timeframe) if not filename.exists(): return DataFrame(columns=self._columns) - pairdata = read_json(filename, orient='values') + pairdata = read_json(filename, orient='values', dtype='float64') pairdata.columns = self._columns pairdata['date'] = to_datetime(pairdata['date'], unit='ms', From 3186add87b82837018e8e9e46a0fb5c5d71a2b23 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Feb 2020 14:46:54 +0100 Subject: [PATCH 39/97] Use explicit column list for float parsing --- freqtrade/data/history/jsondatahandler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 7219d8c01..606018f34 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -69,7 +69,9 @@ class JsonDataHandler(IDataHandler): filename = self._pair_data_filename(self._datadir, pair, timeframe) if not filename.exists(): return DataFrame(columns=self._columns) - pairdata = read_json(filename, orient='values', dtype='float64') + pairdata = read_json(filename, orient='values', + dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) pairdata.columns = self._columns pairdata['date'] = to_datetime(pairdata['date'], unit='ms', From c651e0ac8278c854dc5901189d598b247e8d64ea Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 22 Feb 2020 19:46:40 +0300 Subject: [PATCH 40/97] Fix #2948 --- freqtrade/data/history/jsondatahandler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 606018f34..ee653d937 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -69,10 +69,11 @@ class JsonDataHandler(IDataHandler): filename = self._pair_data_filename(self._datadir, pair, timeframe) if not filename.exists(): return DataFrame(columns=self._columns) - pairdata = read_json(filename, orient='values', - dtype={'open': 'float', 'high': 'float', - 'low': 'float', 'close': 'float', 'volume': 'float'}) + pairdata = read_json(filename, orient='values') pairdata.columns = self._columns + pairdata = pairdata.astype(copy=False, + dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) pairdata['date'] = to_datetime(pairdata['date'], unit='ms', utc=True, From e2e6b940a3252985a7167ab49625e03131af0090 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 22 Feb 2020 19:54:19 +0300 Subject: [PATCH 41/97] copy=False does not make the changes inline anyway, so not needed --- freqtrade/data/history/jsondatahandler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index ee653d937..2b738a94a 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -71,8 +71,7 @@ class JsonDataHandler(IDataHandler): return DataFrame(columns=self._columns) pairdata = read_json(filename, orient='values') pairdata.columns = self._columns - pairdata = pairdata.astype(copy=False, - dtype={'open': 'float', 'high': 'float', + pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float', 'volume': 'float'}) pairdata['date'] = to_datetime(pairdata['date'], unit='ms', From ca8e52dc2cd61a80822cb158e1221d3d24ba62e7 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 23 Feb 2020 00:21:19 +0300 Subject: [PATCH 42/97] Show heartbeat message earlier after changing the state --- freqtrade/worker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index f4b9f275b..e17f61f2f 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -83,6 +83,10 @@ class Worker: if state == State.RUNNING: self.freqtrade.startup() + # Reset heartbeat timestamp to log the heartbeat message at + # first throttling iteration when the state changes + self._heartbeat_msg = 0 + if state == State.STOPPED: # Ping systemd watchdog before sleeping in the stopped state if self._sd_notify: From 259dc75a3083b76980bf12d6575764eac96dd202 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:10:46 +0100 Subject: [PATCH 43/97] some order and added weighted BB indicator to list --- .../templates/subtemplates/indicators_full.j2 | 103 +++++++++++------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 879a2daa0..87b385dd0 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -2,12 +2,17 @@ # Momentum Indicators # ------------------------------------ -# RSI -dataframe['rsi'] = ta.RSI(dataframe) - # ADX dataframe['adx'] = ta.ADX(dataframe) +# # Plus Directional Indicator / Movement +# dataframe['plus_dm'] = ta.PLUS_DM(dataframe) +# dataframe['plus_di'] = ta.PLUS_DI(dataframe) + +# # Minus Directional Indicator / Movement +# dataframe['minus_dm'] = ta.MINUS_DM(dataframe) +# dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) # dataframe['aroonup'] = aroon['aroonup'] @@ -20,6 +25,31 @@ dataframe['adx'] = ta.ADX(dataframe) # # Commodity Channel Index: values Oversold:<-100, Overbought:>100 # dataframe['cci'] = ta.CCI(dataframe) +# RSI +dataframe['rsi'] = ta.RSI(dataframe) + +# # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) +# rsi = 0.1 * (dataframe['rsi'] - 50) +# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) + +# # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) +# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + +# # Stochastic Slow +# stoch = ta.STOCH(dataframe) +# dataframe['slowd'] = stoch['slowd'] +# dataframe['slowk'] = stoch['slowk'] + +# Stochastic Fast +stoch_fast = ta.STOCHF(dataframe) +dataframe['fastd'] = stoch_fast['fastd'] +dataframe['fastk'] = stoch_fast['fastk'] + +# # Stochastic RSI +# stoch_rsi = ta.STOCHRSI(dataframe) +# dataframe['fastd_rsi'] = stoch_rsi['fastd'] +# dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -29,60 +59,57 @@ dataframe['macdhist'] = macd['macdhist'] # MFI dataframe['mfi'] = ta.MFI(dataframe) -# # Minus Directional Indicator / Movement -# dataframe['minus_dm'] = ta.MINUS_DM(dataframe) -# dataframe['minus_di'] = ta.MINUS_DI(dataframe) - -# # Plus Directional Indicator / Movement -# dataframe['plus_dm'] = ta.PLUS_DM(dataframe) -# dataframe['plus_di'] = ta.PLUS_DI(dataframe) -# dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # # ROC # dataframe['roc'] = ta.ROC(dataframe) -# # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) -# rsi = 0.1 * (dataframe['rsi'] - 50) -# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) - -# # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) -# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - -# # Stoch -# stoch = ta.STOCH(dataframe) -# dataframe['slowd'] = stoch['slowd'] -# dataframe['slowk'] = stoch['slowk'] - -# Stoch fast -stoch_fast = ta.STOCHF(dataframe) -dataframe['fastd'] = stoch_fast['fastd'] -dataframe['fastk'] = stoch_fast['fastk'] - -# # Stoch RSI -# stoch_rsi = ta.STOCHRSI(dataframe) -# dataframe['fastd_rsi'] = stoch_rsi['fastd'] -# dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Overlap Studies # ------------------------------------ -# Bollinger bands +# Bollinger Bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] +dataframe["bb_percent"] = ( + (dataframe["close"] - dataframe["bb_lowerband"]) / + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) +) +dataframe["bb_width"] = ( + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] +) + +# Bollinger Bands - Weighted (EMA based instead of SMA) +# weighted_bollinger = qtpylib.weighted_bollinger_bands( +# qtpylib.typical_price(dataframe), window=20, stds=2 +# ) +# dataframe["wbb_upperband"] = weighted_bollinger["upper"] +# dataframe["wbb_lowerband"] = weighted_bollinger["lower"] +# dataframe["wbb_middleband"] = weighted_bollinger["mid"] +# dataframe["wbb_percent"] = ( +# (dataframe["close"] - dataframe["wbb_lowerband"]) / +# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) +# ) +# dataframe["wbb_width"] = ( +# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / dataframe["wbb_middleband"] +# ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) +# dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average -# dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) +# dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) +# dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) +# dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) +# dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) +# dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) +# dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) -# SAR Parabol +# Parabolic SAR dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average @@ -142,7 +169,7 @@ dataframe['htleadsine'] = hilbert['leadsine'] # # Chart type # # ------------------------------------ -# # Heikinashi stategy +# # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] From b49b9b515ed14d1c501e9350fe727f0f16a7722d Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:37:15 +0100 Subject: [PATCH 44/97] final touches --- freqtrade/templates/subtemplates/indicators_full.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 87b385dd0..cd106451e 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -19,7 +19,7 @@ dataframe['adx'] = ta.ADX(dataframe) # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) -# # Awesome oscillator +# # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) # # Commodity Channel Index: values Oversold:<-100, Overbought:>100 From 2957756275367bcd35734989cfe61558356a4a09 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:39:01 +0100 Subject: [PATCH 45/97] final touches plus --- freqtrade/templates/subtemplates/indicators_full.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index cd106451e..903aebb73 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -22,7 +22,7 @@ dataframe['adx'] = ta.ADX(dataframe) # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) -# # Commodity Channel Index: values Oversold:<-100, Overbought:>100 +# # Commodity Channel Index: values Oversold:-100, Overbought:100 # dataframe['cci'] = ta.CCI(dataframe) # RSI From 5ac624446587f95af22b322b53caddd87692f021 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:50:26 +0100 Subject: [PATCH 46/97] added keltner channel and uo --- .../templates/subtemplates/indicators_full.j2 | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 903aebb73..af472faef 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -22,7 +22,23 @@ dataframe['adx'] = ta.ADX(dataframe) # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) -# # Commodity Channel Index: values Oversold:-100, Overbought:100 +# # Keltner Channel +# keltner = qtpylib.keltner_channel(dataframe) +# dataframe["kc_upperband"] = keltner["upper"] +# dataframe["kc_lowerband"] = keltner["lower"] +# dataframe["kc_middleband"] = keltner["mid"] +# dataframe["kc_percent"] = ( +# (dataframe["close"] - dataframe["kc_lowerband"]) / +# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) +# ) +# dataframe["kc_width"] = ( +# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] +# ) + +# # Ultimate Oscillator +# dataframe['ao'] = ta.ULTOSC(dataframe) + +# # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) # RSI From d2181bdd9492d16be409d82cd2fa5f0782c15838 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 23 Feb 2020 01:45:15 +0300 Subject: [PATCH 47/97] Adjust tests --- tests/test_freqtradebot.py | 28 ++--------------- tests/test_worker.py | 62 ++++++++++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5ed4d296c..20db46fac 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -782,7 +782,7 @@ def test_process_exchange_failures(default_conf, ticker, mocker) -> None: worker = Worker(args=None, config=default_conf) patch_get_signal(worker.freqtrade) - worker._process() + worker._process_running() assert sleep_mock.has_calls() @@ -799,7 +799,7 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: assert worker.freqtrade.state == State.RUNNING - worker._process() + worker._process_running() assert worker.freqtrade.state == State.STOPPED assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] @@ -3665,30 +3665,6 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): assert reinit_mock.call_count == 0 -def test_process_i_am_alive(default_conf, mocker, caplog): - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - - ftbot = get_patched_freqtradebot(mocker, default_conf) - message = r"Bot heartbeat\. PID=.*" - ftbot.process() - assert log_has_re(message, caplog) - assert ftbot._heartbeat_msg != 0 - - caplog.clear() - # Message is not shown before interval is up - ftbot.process() - assert not log_has_re(message, caplog) - - caplog.clear() - # Set clock - 70 seconds - ftbot._heartbeat_msg -= 70 - - ftbot.process() - assert log_has_re(message, caplog) - - @pytest.mark.usefixtures("init_persistence") def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, caplog): default_conf['dry_run'] = True diff --git a/tests/test_worker.py b/tests/test_worker.py index 2fb42d47e..7b446ac6a 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock from freqtrade.data.dataprovider import DataProvider from freqtrade.state import State from freqtrade.worker import Worker -from tests.conftest import get_patched_worker, log_has +from tests.conftest import get_patched_worker, log_has, log_has_re def test_worker_state(mocker, default_conf, markets) -> None: @@ -38,15 +38,13 @@ def test_worker_running(mocker, default_conf, caplog) -> None: def test_worker_stopped(mocker, default_conf, caplog) -> None: mock_throttle = MagicMock() mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) - mock_sleep = mocker.patch('time.sleep', return_value=None) worker = get_patched_worker(mocker, default_conf) worker.freqtrade.state = State.STOPPED state = worker._worker(old_state=State.RUNNING) assert state is State.STOPPED assert log_has('Changing state to: STOPPED', caplog) - assert mock_throttle.call_count == 0 - assert mock_sleep.call_count == 1 + assert mock_throttle.call_count == 1 def test_throttle(mocker, default_conf, caplog) -> None: @@ -57,14 +55,14 @@ def test_throttle(mocker, default_conf, caplog) -> None: worker = get_patched_worker(mocker, default_conf) start = time.time() - result = worker._throttle(throttled_func, min_secs=0.1) + result = worker._throttle(throttled_func, throttle_secs=0.1) end = time.time() assert result == 42 assert end - start > 0.1 - assert log_has('Throttling throttled_func for 0.10 seconds', caplog) + assert log_has_re(r"Throttling with 'throttled_func\(\)': sleep for 0\.10 s.*", caplog) - result = worker._throttle(throttled_func, min_secs=-1) + result = worker._throttle(throttled_func, throttle_secs=-1) assert result == 42 @@ -74,8 +72,54 @@ def test_throttle_with_assets(mocker, default_conf) -> None: worker = get_patched_worker(mocker, default_conf) - result = worker._throttle(throttled_func, min_secs=0.1, nb_assets=666) + result = worker._throttle(throttled_func, throttle_secs=0.1, nb_assets=666) assert result == 666 - result = worker._throttle(throttled_func, min_secs=0.1) + result = worker._throttle(throttled_func, throttle_secs=0.1) assert result == -1 + + +def test_worker_heartbeat_running(default_conf, mocker, caplog): + message = r"Bot heartbeat\. PID=.*state='RUNNING'" + + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + worker = get_patched_worker(mocker, default_conf) + + worker.freqtrade.state = State.RUNNING + worker._worker(old_state=State.STOPPED) + assert log_has_re(message, caplog) + + caplog.clear() + # Message is not shown before interval is up + worker._worker(old_state=State.RUNNING) + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + worker._heartbeat_msg -= 70 + worker._worker(old_state=State.RUNNING) + assert log_has_re(message, caplog) + + +def test_worker_heartbeat_stopped(default_conf, mocker, caplog): + message = r"Bot heartbeat\. PID=.*state='STOPPED'" + + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + worker = get_patched_worker(mocker, default_conf) + + worker.freqtrade.state = State.STOPPED + worker._worker(old_state=State.RUNNING) + assert log_has_re(message, caplog) + + caplog.clear() + # Message is not shown before interval is up + worker._worker(old_state=State.STOPPED) + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + worker._heartbeat_msg -= 70 + worker._worker(old_state=State.STOPPED) + assert log_has_re(message, caplog) From e04c2dda2cfd3c18690be04dd3b22f98731b0b04 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sat, 22 Feb 2020 23:58:31 +0100 Subject: [PATCH 48/97] fixed typo --- freqtrade/templates/subtemplates/indicators_full.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index af472faef..60a358bec 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -36,7 +36,7 @@ dataframe['adx'] = ta.ADX(dataframe) # ) # # Ultimate Oscillator -# dataframe['ao'] = ta.ULTOSC(dataframe) +# dataframe['uo'] = ta.ULTOSC(dataframe) # # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) From f25d6224ddeab8a3889daa69c1fac8eb375d169b Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sun, 23 Feb 2020 16:22:19 +0100 Subject: [PATCH 49/97] modified sample_strategy --- freqtrade/templates/sample_strategy.py | 136 ++++++++++++++++--------- 1 file changed, 90 insertions(+), 46 deletions(-) diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index 92f6aefba..8a4b27c72 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -124,11 +124,16 @@ class SampleStrategy(IStrategy): # Momentum Indicators # ------------------------------------ - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - # ADX - dataframe['adx'] = ta.ADX(dataframe) + # dataframe['adx'] = ta.ADX(dataframe) + + # # Plus Directional Indicator / Movement + # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + # dataframe['plus_di'] = ta.PLUS_DI(dataframe) + + # # Minus Directional Indicator / Movement + # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + # dataframe['minus_di'] = ta.MINUS_DI(dataframe) # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) @@ -136,12 +141,53 @@ class SampleStrategy(IStrategy): # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) - # # Awesome oscillator + # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - # # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + # # Keltner Channel + # keltner = qtpylib.keltner_channel(dataframe) + # dataframe["kc_upperband"] = keltner["upper"] + # dataframe["kc_lowerband"] = keltner["lower"] + # dataframe["kc_middleband"] = keltner["mid"] + # dataframe["kc_percent"] = ( + # (dataframe["close"] - dataframe["kc_lowerband"]) / + # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) + # ) + # dataframe["kc_width"] = ( + # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] + # ) + + # # Ultimate Oscillator + # dataframe['uo'] = ta.ULTOSC(dataframe) + + # # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) + # rsi = 0.1 * (dataframe['rsi'] - 50) + # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) + + # # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) + # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # # Stochastic Slow + # stoch = ta.STOCH(dataframe) + # dataframe['slowd'] = stoch['slowd'] + # dataframe['slowk'] = stoch['slowk'] + + # Stochastic Fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # # Stochastic RSI + # stoch_rsi = ta.STOCHRSI(dataframe) + # dataframe['fastd_rsi'] = stoch_rsi['fastd'] + # dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -151,71 +197,69 @@ class SampleStrategy(IStrategy): # MFI dataframe['mfi'] = ta.MFI(dataframe) - # # Minus Directional Indicator / Movement - # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - # dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # # Plus Directional Indicator / Movement - # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - # dataframe['plus_di'] = ta.PLUS_DI(dataframe) - # dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # # ROC # dataframe['roc'] = ta.ROC(dataframe) - # # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - # rsi = 0.1 * (dataframe['rsi'] - 50) - # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) - - # # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - - # # Stoch - # stoch = ta.STOCH(dataframe) - # dataframe['slowd'] = stoch['slowd'] - # dataframe['slowk'] = stoch['slowk'] - - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - - # # Stoch RSI - # stoch_rsi = ta.STOCHRSI(dataframe) - # dataframe['fastd_rsi'] = stoch_rsi['fastd'] - # dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Overlap Studies # ------------------------------------ - # Bollinger bands + # Bollinger Bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] + dataframe["bb_percent"] = ( + (dataframe["close"] - dataframe["bb_lowerband"]) / + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) + ) + dataframe["bb_width"] = ( + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] + ) + + # Bollinger Bands - Weighted (EMA based instead of SMA) + # weighted_bollinger = qtpylib.weighted_bollinger_bands( + # qtpylib.typical_price(dataframe), window=20, stds=2 + # ) + # dataframe["wbb_upperband"] = weighted_bollinger["upper"] + # dataframe["wbb_lowerband"] = weighted_bollinger["lower"] + # dataframe["wbb_middleband"] = weighted_bollinger["mid"] + # dataframe["wbb_percent"] = ( + # (dataframe["close"] - dataframe["wbb_lowerband"]) / + # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) + # ) + # dataframe["wbb_width"] = ( + # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / + # dataframe["wbb_middleband"] + # ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + # dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average - # dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) + # dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) + # dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) + # dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) + # dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) + # dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) - # SAR Parabol - dataframe['sar'] = ta.SAR(dataframe) + # Parabolic SAR + # dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) # Cycle Indicator # ------------------------------------ # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] + # hilbert = ta.HT_SINE(dataframe) + # dataframe['htsine'] = hilbert['sine'] + # dataframe['htleadsine'] = hilbert['leadsine'] # Pattern Recognition - Bullish candlestick patterns # ------------------------------------ @@ -264,7 +308,7 @@ class SampleStrategy(IStrategy): # # Chart type # # ------------------------------------ - # # Heikinashi stategy + # # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] From 0eeafcd157c17d6f7de92ab66c6f267c454bcec2 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Sun, 23 Feb 2020 16:56:55 +0100 Subject: [PATCH 50/97] matched commenting on previous sample_strategy.py --- freqtrade/templates/sample_strategy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index 8a4b27c72..17372e1e0 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -125,7 +125,7 @@ class SampleStrategy(IStrategy): # ------------------------------------ # ADX - # dataframe['adx'] = ta.ADX(dataframe) + dataframe['adx'] = ta.ADX(dataframe) # # Plus Directional Indicator / Movement # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) @@ -249,17 +249,17 @@ class SampleStrategy(IStrategy): # dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) # Parabolic SAR - # dataframe['sar'] = ta.SAR(dataframe) + dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average - # dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) # Cycle Indicator # ------------------------------------ # Hilbert Transform Indicator - SineWave - # hilbert = ta.HT_SINE(dataframe) - # dataframe['htsine'] = hilbert['sine'] - # dataframe['htleadsine'] = hilbert['leadsine'] + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] # Pattern Recognition - Bullish candlestick patterns # ------------------------------------ From e545ef563c0b5aaafc618bc3501c1d1e43f68c07 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sun, 23 Feb 2020 22:50:58 +0300 Subject: [PATCH 51/97] Wording adjusted in helpstring --- freqtrade/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index e17f61f2f..4c28ecaeb 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -69,7 +69,7 @@ class Worker: def _worker(self, old_state: Optional[State]) -> State: """ - Trading routine that must be run at each loop + The main routine that runs each throttling iteration and handles the states. :param old_state: the previous service state from the previous call :return: current service state """ From 7eb62ed32e7acc7004821a9213c7752fe31f1f9c Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Mon, 24 Feb 2020 00:33:01 +0100 Subject: [PATCH 52/97] Remove old print option for hyperopt-list and made table as default --- docs/utils.md | 1 - freqtrade/commands/arguments.py | 3 +-- freqtrade/commands/cli_options.py | 6 ------ freqtrade/commands/hyperopt_commands.py | 23 +++++------------------ freqtrade/configuration/configuration.py | 3 --- tests/commands/test_commands.py | 16 ---------------- 6 files changed, 6 insertions(+), 46 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 6ca0b7920..abb7fd0db 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -440,7 +440,6 @@ optional arguments: --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. --print-json Print best result detailization in JSON format. - --print-table Print results in table format. --no-details Do not print best epoch details. Common arguments: diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 0f6478a60..fe6f49039 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -66,8 +66,7 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit", "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit", - "print_colorized", "print_json", "print_table", - "hyperopt_list_no_details"] + "print_colorized", "print_json", "hyperopt_list_no_details"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", "print_json", "hyperopt_show_no_header"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 9efc1151a..1776955b1 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -220,12 +220,6 @@ AVAILABLE_CLI_OPTIONS = { action='store_true', default=False, ), - "print_table": Arg( - '--print-table', - help='Print results in table format.', - action='store_true', - default=False, - ), "hyperopt_jobs": Arg( '-j', '--job-workers', help='The number of concurrently running jobs for hyperoptimization ' diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 88cd79468..ccaa59e54 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -21,7 +21,6 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: print_colorized = config.get('print_colorized', False) print_json = config.get('print_json', False) - print_table = config.get('print_table', False) no_details = config.get('hyperopt_list_no_details', False) no_header = False @@ -47,26 +46,14 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: trials = _hyperopt_filter_trials(trials, filteroptions) - # TODO: fetch the interval for epochs to print from the cli option - epoch_start, epoch_stop = 0, None - if print_colorized: colorama_init(autoreset=True) - if print_table: - try: - Hyperopt.print_result_table(config, trials, total_epochs, - not filteroptions['only_best'], print_colorized) - except KeyboardInterrupt: - print('User interrupted..') - else: - try: - # Human-friendly indexes used here (starting from 1) - for val in trials[epoch_start:epoch_stop]: - Hyperopt.print_results_explanation(val, total_epochs, - not filteroptions['only_best'], print_colorized) - except KeyboardInterrupt: - print('User interrupted..') + try: + Hyperopt.print_result_table(config, trials, total_epochs, + not filteroptions['only_best'], print_colorized) + except KeyboardInterrupt: + print('User interrupted..') if trials and not no_details: sorted_trials = sorted(trials, key=itemgetter('loss')) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 0adfe03e7..c2613ba99 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -286,9 +286,6 @@ class Configuration: self._args_to_config(config, argname='print_json', logstring='Parameter --print-json detected ...') - self._args_to_config(config, argname='print_table', - logstring='Parameter --print-table detected: {}') - self._args_to_config(config, argname='hyperopt_jobs', logstring='Parameter -j/--job-workers detected: {}') diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index ceae6f372..995b504c5 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -907,22 +907,6 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): assert all(x not in captured.out for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12" " 9/12", " 10/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--print-table", - "--min-trades", "100", - "--print-json" - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 3/12", " 7/12", " 9/12", " 11/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 6/12", " 8/12", " 10/12" - " 12/12"]) def test_hyperopt_show(mocker, capsys, hyperopt_results): From 353f722dc54dcd6eab646a844596383eccc5d326 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 08:04:07 +0000 Subject: [PATCH 53/97] Bump requests from 2.22.0 to 2.23.0 Bumps [requests](https://github.com/psf/requests) from 2.22.0 to 2.23.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.22.0...v2.23.0) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 2be51ba73..61809c698 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -5,7 +5,7 @@ SQLAlchemy==1.3.13 python-telegram-bot==12.4.2 arrow==0.15.5 cachetools==4.0.0 -requests==2.22.0 +requests==2.23.0 urllib3==1.25.8 wrapt==1.12.0 jsonschema==3.2.0 From 4054dec7a029594503222de5af6677669956d1cd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 08:04:32 +0000 Subject: [PATCH 54/97] Bump plotly from 4.5.0 to 4.5.1 Bumps [plotly](https://github.com/plotly/plotly.py) from 4.5.0 to 4.5.1. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v4.5.0...v4.5.1) Signed-off-by: dependabot-preview[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 26467d90b..5e62a5e95 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.5.0 +plotly==4.5.1 From ff69b511e311bb73d36d0d2661f7e4100a46d283 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 08:04:44 +0000 Subject: [PATCH 55/97] Bump scikit-optimize from 0.7.2 to 0.7.4 Bumps [scikit-optimize](https://github.com/scikit-optimize/scikit-optimize) from 0.7.2 to 0.7.4. - [Release notes](https://github.com/scikit-optimize/scikit-optimize/releases) - [Changelog](https://github.com/scikit-optimize/scikit-optimize/blob/master/CHANGELOG.md) - [Commits](https://github.com/scikit-optimize/scikit-optimize/compare/v0.7.2...v0.7.4) Signed-off-by: dependabot-preview[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index e97e7f6be..2984229c1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -4,6 +4,6 @@ # Required for hyperopt scipy==1.4.1 scikit-learn==0.22.1 -scikit-optimize==0.7.2 +scikit-optimize==0.7.4 filelock==3.0.12 joblib==0.14.1 From d63aaf3bfd7eb511387625b76435ea873e51e444 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 08:05:15 +0000 Subject: [PATCH 56/97] Bump ccxt from 1.22.61 to 1.22.95 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.22.61 to 1.22.95. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.22.61...1.22.95) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 2be51ba73..f792f5348 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.22.61 +ccxt==1.22.95 SQLAlchemy==1.3.13 python-telegram-bot==12.4.2 arrow==0.15.5 From 23bf135b8aeb704c64e775dba8a12b166a692a41 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Mon, 24 Feb 2020 11:01:14 +0100 Subject: [PATCH 57/97] Alignment of table content, changed coloring, changed 'Best' column to show if it's initial_point or best --- freqtrade/optimize/hyperopt.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 72b3516c7..7ff5c3500 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -306,15 +306,18 @@ class Hyperopt: return trials = json_normalize(results, max_level=1) - trials = trials[['is_best', 'current_epoch', - 'results_metrics.trade_count', 'results_metrics.avg_profit', - 'results_metrics.total_profit', 'results_metrics.profit', - 'results_metrics.duration', 'loss']] + trials['Best'] = '' + trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', + 'results_metrics.avg_profit', 'results_metrics.total_profit', + 'results_metrics.profit', 'results_metrics.duration', + 'loss', 'is_initial_point', 'is_best']] trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', - 'Profit', 'Avg duration', 'Objective'] + 'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best'] - trials['Best'] = trials['Best'].apply(lambda x: '*' if x else '') + trials.loc[trials['is_initial_point'], 'Best'] = '*' + trials.loc[trials['is_best'], 'Best'] = 'Best' trials['Objective'] = trials['Objective'].astype(str) + trials = trials.drop(columns=['is_initial_point', 'is_best']) if print_colorized: for i in range(len(trials)): @@ -322,10 +325,10 @@ class Hyperopt: trials.at[i, 'Best'] = Fore.GREEN + trials.loc[i]['Best'] trials.at[i, 'Objective'] = "{}{}".format(trials.loc[i]['Objective'], Fore.RESET) - if '*' in trials.loc[i]['Best'] and highlight_best: - trials.at[i, 'Best'] = Style.BRIGHT + trials.loc[i]['Best'] - trials.at[i, 'Objective'] = "{}{}".format(trials.loc[i]['Objective'], - Style.RESET_ALL) + if 'Best' in trials.loc[i]['Best'] and highlight_best: + trials.at[i, 'Best'] = Style.BRIGHT + trials.loc[i]['Best'] + trials.at[i, 'Objective'] = "{}{}".format(trials.loc[i]['Objective'], + Style.RESET_ALL) trials['Epoch'] = trials['Epoch'].apply( lambda x: "{}/{}".format(x, total_epochs)) @@ -338,7 +341,8 @@ class Hyperopt: trials['Avg duration'] = trials['Avg duration'].apply( lambda x: '{:,.1f}m'.format(x) if not isna(x) else x) - print(tabulate(trials.to_dict(orient='list'), headers='keys', tablefmt='psql')) + print(tabulate(trials.to_dict(orient='list'), headers='keys', tablefmt='psql', + stralign="right")) def has_space(self, space: str) -> bool: """ From 23b47b66eccb408d56ba0abd6e6e5ab71b8ae3f5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Feb 2020 20:11:25 +0100 Subject: [PATCH 58/97] Update install-script documentation and reorder installation steps --- docs/installation.md | 8 ++++---- mkdocs.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 054cafe9b..0feaf509d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -65,11 +65,11 @@ usage: ** --install ** -With this option, the script will install everything you need to run the bot: +With this option, the script will install the bot and most dependencies: +You will need to have git and python3.6+ installed beforehand for this to work. * Mandatory software as: `ta-lib` -* Setup your virtualenv -* Configure your `config.json` file +* Setup your virtualenv under `.env/` This option is a combination of installation tasks, `--reset` and `--config`. @@ -83,7 +83,7 @@ This option will hard reset your branch (only if you are on either `master` or ` ** --config ** -Use this option to configure the `config.json` configuration file. The script will interactively ask you questions to setup your bot and create your `config.json`. +DEPRECATED - use `freqtrade new-config -c config.json` instead. ------ diff --git a/mkdocs.yml b/mkdocs.yml index d53687c64..4e7e6ff75 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,8 +1,8 @@ site_name: Freqtrade nav: - - About: index.md - - Installation: installation.md + - Home: index.md - Installation Docker: docker.md + - Installation: installation.md - Configuration: configuration.md - Strategy Customization: strategy-customization.md - Stoploss: stoploss.md From 2f349e0504d014e01f428e8fc0b6e8f8a619aeef Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Feb 2020 20:21:25 +0100 Subject: [PATCH 59/97] Improve install documentation by streamlining the process --- docs/configuration.md | 14 +++++++++++--- docs/installation.md | 12 +++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0b9519688..234ff49ba 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -626,6 +626,11 @@ In production mode, the bot will engage your money. Be careful, since a wrong strategy can lose all your money. Be aware of what you are doing when you run it in production mode. +### Setup your exchange account + +You will need to create API Keys (usually you get `key` and `secret`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. +API Keys are usually only required for real / production trading, but are not required for paper-trading / dry-run. + ### To switch your bot in production mode **Edit your `config.json` file.** @@ -647,11 +652,14 @@ you run it in production mode. } ``` -!!! Note - If you have an exchange API key yet, [see our tutorial](installation.md#setup-your-exchange-account). - You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange. +### Setup your exchange account + +You will need to create API Keys (usually you get `key` and `secret`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. +API Keys are usually only required for real / production trading, but are not required for paper-trading / dry-run. + + ### Using proxy with Freqtrade To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. diff --git a/docs/installation.md b/docs/installation.md index 0feaf509d..5a15be234 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,6 +2,8 @@ This page explains how to prepare your environment for running the bot. +Please consider using the prebuilt [docker images](docker.md) to get started quickly while trying out freqtrade. + ## Prerequisite ### Requirements @@ -14,15 +16,7 @@ Click each one for install guide: * [virtualenv](https://virtualenv.pypa.io/en/stable/installation/) (Recommended) * [TA-Lib](https://mrjbq7.github.io/ta-lib/install.html) (install instructions below) -### API keys - -Before running your bot in production you will need to setup few -external API. In production mode, the bot will require valid Exchange API -credentials. We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot) (optional but recommended). - -### Setup your exchange account - -You will need to create API Keys (Usually you get `key` and `secret`) from the Exchange website and insert this into the appropriate fields in the configuration or when asked by the installation script. + We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot), which is optional but recommended. ## Quick start From 6581ba56cab28f025c7444bd57e2e09beefb7ded Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Feb 2020 20:41:45 +0100 Subject: [PATCH 60/97] Use markets.quote to validate --- freqtrade/pairlist/IPairList.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 1ad4da523..7d489ece7 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -99,7 +99,8 @@ class IPairList(ABC): logger.warning(f"Pair {pair} is not compatible with exchange " f"{self._exchange.name}. Removing it from whitelist..") continue - if not pair.endswith(self._config['stake_currency']): + + if markets[pair]['quote'] != self._config['stake_currency']: logger.warning(f"Pair {pair} is not compatible with your stake currency " f"{self._config['stake_currency']}. Removing it from whitelist..") continue From 3e4f663418236f84123eeb186fa0f642f84de89e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Feb 2020 21:22:58 +0100 Subject: [PATCH 61/97] Move pairlist validation to exchange (we need to use .quote) from markets --- freqtrade/configuration/config_validation.py | 12 ------ freqtrade/exchange/exchange.py | 8 +++- tests/exchange/test_exchange.py | 42 ++++++++++++++++++-- tests/test_configuration.py | 7 ---- 4 files changed, 45 insertions(+), 24 deletions(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 5183ad0b4..5ba7ff294 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -150,15 +150,3 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: if (pl.get('method') == 'StaticPairList' and not conf.get('exchange', {}).get('pair_whitelist')): raise OperationalException("StaticPairList requires pair_whitelist to be set.") - - if pl.get('method') == 'StaticPairList': - stake = conf['stake_currency'] - invalid_pairs = [] - for pair in conf['exchange'].get('pair_whitelist'): - if not pair.endswith(f'/{stake}'): - invalid_pairs.append(pair) - - if invalid_pairs: - raise OperationalException( - f"Stake-currency '{stake}' not compatible with pair-whitelist. " - f"Please remove the following pairs: {invalid_pairs}") diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b3b347016..8023417a3 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -300,7 +300,7 @@ class Exchange: if not self.markets: logger.warning('Unable to validate pairs (assuming they are correct).') return - + invalid_pairs = [] for pair in pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs # TODO: add a support for having coins in BTC/USDT format @@ -322,6 +322,12 @@ class Exchange: logger.warning(f"Pair {pair} is restricted for some users on this exchange." f"Please check if you are impacted by this restriction " f"on the exchange and eventually remove {pair} from your whitelist.") + if not self.markets[pair].get('quote') == self._config['stake_currency']: + invalid_pairs.append(pair) + if invalid_pairs: + raise OperationalException( + f"Stake-currency '{self._config['stake_currency']}' not compatible with " + f"pair-whitelist. Please remove the following pairs: {invalid_pairs}") def get_valid_pair_combination(self, curr_1: str, curr_2: str) -> str: """ diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8b2e439c3..d1c105591 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -406,7 +406,10 @@ def test_get_quote_currencies(default_conf, mocker): def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs directly api_mock = MagicMock() type(api_mock).markets = PropertyMock(return_value={ - 'ETH/BTC': {}, 'LTC/BTC': {}, 'XRP/BTC': {}, 'NEO/BTC': {} + 'ETH/BTC': {'quote': 'BTC'}, + 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, + 'NEO/BTC': {'quote': 'BTC'}, }) id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock @@ -454,9 +457,9 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): def test_validate_pairs_restricted(default_conf, mocker, caplog): api_mock = MagicMock() type(api_mock).markets = PropertyMock(return_value={ - 'ETH/BTC': {}, 'LTC/BTC': {}, - 'XRP/BTC': {'info': {'IsRestricted': True}}, - 'NEO/BTC': {'info': 'TestString'}, # info can also be a string ... + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC', 'info': {'IsRestricted': True}}, + 'NEO/BTC': {'quote': 'BTC', 'info': 'TestString'}, # info can also be a string ... }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') @@ -469,6 +472,37 @@ def test_validate_pairs_restricted(default_conf, mocker, caplog): f"on the exchange and eventually remove XRP/BTC from your whitelist.", caplog) +def test_validate_pairs_stakecompatibility(default_conf, mocker, caplog): + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'BTC'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + Exchange(default_conf) + + +def test_validate_pairs_stakecompatibility_fail(default_conf, mocker, caplog): + default_conf['exchange']['pair_whitelist'].append('HELLO-WORLD') + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'USDT'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + with pytest.raises(OperationalException, match=r"Stake-currency 'BTC' not compatible with.*"): + Exchange(default_conf) + @pytest.mark.parametrize("timeframe", [ ('5m'), ("1m"), ("15m"), ("1h") ]) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index d810305db..828db4d83 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -810,13 +810,6 @@ def test_validate_whitelist(default_conf): validate_config_consistency(conf) - conf = deepcopy(default_conf) - conf['stake_currency'] = 'USDT' - with pytest.raises(OperationalException, - match=r"Stake-currency 'USDT' not compatible with pair-whitelist.*"): - validate_config_consistency(conf) - - def test_load_config_test_comments() -> None: """ Load config with comments From 61037ab7b8f3f2467555c1a2a767580ea25f43a6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Feb 2020 21:50:27 +0100 Subject: [PATCH 62/97] Implement get_pair_base_curr and get_pair_quote_curr --- freqtrade/exchange/exchange.py | 12 ++++++++++++ freqtrade/freqtradebot.py | 6 ++++-- tests/commands/test_commands.py | 31 ++++++++++++++++--------------- tests/conftest.py | 29 ++++++++++++++++++++++++++++- tests/exchange/test_exchange.py | 15 ++++++++------- tests/test_freqtradebot.py | 2 ++ 6 files changed, 70 insertions(+), 25 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 8023417a3..0e0d2dabe 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -228,6 +228,18 @@ class Exchange: markets = self.markets return sorted(set([x['quote'] for _, x in markets.items()])) + def get_pair_quote_currency(self, pair: str) -> str: + """ + Return a pair's quote currency + """ + return self.markets[pair].get('quote') + + def get_pair_base_currency(self, pair: str) -> str: + """ + Return a pair's quote currency + """ + return self.markets[pair].get('base') + def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: if pair_interval in self._klines: return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 00d5c369a..38583b5ad 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1125,12 +1125,13 @@ class FreqtradeBot: if trade.fee_open == 0 or order['status'] == 'open': return order_amount + trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) # use fee from order-dict if possible if ('fee' in order and order['fee'] is not None and (order['fee'].keys() >= {'currency', 'cost'})): if (order['fee']['currency'] is not None and order['fee']['cost'] is not None and - trade.pair.startswith(order['fee']['currency'])): + trade_base_currency == order['fee']['currency']): new_amount = order_amount - order['fee']['cost'] logger.info("Applying fee on amount for %s (from %s to %s) from Order", trade, order['amount'], new_amount) @@ -1145,6 +1146,7 @@ class FreqtradeBot: return order_amount amount = 0 fee_abs = 0 + trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) for exectrade in trades: amount += exectrade['amount'] if ("fee" in exectrade and exectrade['fee'] is not None and @@ -1152,7 +1154,7 @@ class FreqtradeBot: # only applies if fee is in quote currency! if (exectrade['fee']['currency'] is not None and exectrade['fee']['cost'] is not None and - trade.pair.startswith(exectrade['fee']['currency'])): + trade_base_currency == exectrade['fee']['currency']): fee_abs += exectrade['fee']['cost'] if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index a9fe0f637..fd8df4b56 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -217,8 +217,9 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 9 active markets: " - "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XLTCUSDT, XRP/BTC.\n" + assert ("Exchange Bittrex has 10 active markets: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, NEO/BTC, " + "TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) patch_exchange(mocker, api_mock=api_mock, id="binance") @@ -231,7 +232,7 @@ def test_list_markets(mocker, markets, capsys): pargs['config'] = None start_list_markets(pargs, False) captured = capsys.readouterr() - assert re.match("\nExchange Binance has 9 active markets:\n", + assert re.match("\nExchange Binance has 10 active markets:\n", captured.out) patch_exchange(mocker, api_mock=api_mock, id="bittrex") @@ -243,8 +244,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 11 markets: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, LTC/USDT, NEO/BTC, " + assert ("Exchange Bittrex has 12 markets: " + "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, LTC/USDT, NEO/BTC, " "TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) @@ -256,8 +257,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 8 active pairs: " - "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XRP/BTC.\n" + assert ("Exchange Bittrex has 9 active pairs: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, NEO/BTC, TKN/BTC, XRP/BTC.\n" in captured.out) # Test list-pairs subcommand with --all: all pairs @@ -268,8 +269,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 10 pairs: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, LTC/USDT, NEO/BTC, " + assert ("Exchange Bittrex has 11 pairs: " + "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, LTC/USDT, NEO/BTC, " "TKN/BTC, XRP/BTC.\n" in captured.out) @@ -282,8 +283,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 5 active markets with ETH, LTC as base currencies: " - "ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, XLTCUSDT.\n" + assert ("Exchange Bittrex has 6 active markets with ETH, LTC as base currencies: " + "ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, base=LTC @@ -295,8 +296,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 3 active markets with LTC as base currency: " - "LTC/BTC, LTC/USD, XLTCUSDT.\n" + assert ("Exchange Bittrex has 4 active markets with LTC as base currency: " + "LTC/BTC, LTC/ETH, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, quote=USDT, USD @@ -384,7 +385,7 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 9 active markets:\n" + assert ("Exchange Bittrex has 10 active markets:\n" in captured.out) # Test tabular output, no markets found @@ -407,7 +408,7 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/USD","NEO/BTC",' + assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/ETH","LTC/USD","NEO/BTC",' '"TKN/BTC","XLTCUSDT","XRP/BTC"]' in captured.out) diff --git a/tests/conftest.py b/tests/conftest.py index acb730330..000f62868 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -575,7 +575,34 @@ def get_markets(): } }, 'info': {}, - } + }, + 'LTC/ETH': { + 'id': 'LTCETH', + 'symbol': 'LTC/ETH', + 'base': 'LTC', + 'quote': 'ETH', + 'active': True, + 'precision': { + 'base': 8, + 'quote': 8, + 'amount': 3, + 'price': 5 + }, + 'limits': { + 'amount': { + 'min': 0.001, + 'max': 10000000.0 + }, + 'price': { + 'min': 1e-05, + 'max': 1000.0 + }, + 'cost': { + 'min': 0.01, + 'max': None + } + }, + }, } diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index d1c105591..98edd0dbb 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -400,7 +400,7 @@ def test_validate_stake_currency_error(default_conf, mocker, caplog): def test_get_quote_currencies(default_conf, mocker): ex = get_patched_exchange(mocker, default_conf) - assert set(ex.get_quote_currencies()) == set(['USD', 'BTC', 'USDT']) + assert set(ex.get_quote_currencies()) == set(['USD', 'ETH', 'BTC', 'USDT']) def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs directly @@ -1862,6 +1862,7 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): # 'ETH/BTC': 'active': True # 'ETH/USDT': 'active': True # 'LTC/BTC': 'active': False + # 'LTC/ETH': 'active': True # 'LTC/USD': 'active': True # 'LTC/USDT': 'active': True # 'NEO/BTC': 'active': False @@ -1870,26 +1871,26 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): # 'XRP/BTC': 'active': False # all markets ([], [], False, False, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', + ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # active markets ([], [], False, True, - ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # all pairs ([], [], True, False, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', + ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), # active pairs ([], [], True, True, - ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), # all markets, base=ETH, LTC (['ETH', 'LTC'], [], False, False, - ['ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), + ['ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), # all markets, base=LTC (['LTC'], [], False, False, - ['LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), + ['LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), # all markets, quote=USDT ([], ['USDT'], False, False, ['ETH/USDT', 'LTC/USDT', 'XLTCUSDT']), diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 20db46fac..9a7816d35 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2192,6 +2192,7 @@ def test_handle_timedout_limit_buy(mocker, default_conf, limit_buy_order) -> Non Trade.session = MagicMock() trade = MagicMock() + trade.pair = 'LTC/ETH' limit_buy_order['remaining'] = limit_buy_order['amount'] assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order) assert cancel_order_mock.call_count == 1 @@ -2215,6 +2216,7 @@ def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_ Trade.session = MagicMock() trade = MagicMock() + trade.pair = 'LTC/ETH' limit_buy_order['remaining'] = limit_buy_order['amount'] assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order) assert cancel_order_mock.call_count == 1 From cd7efde6c0618c3a40d44432dece4592510e5c87 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Mon, 24 Feb 2020 22:06:21 +0100 Subject: [PATCH 63/97] Fixed coloring so it's only targeting the values not the table borders --- freqtrade/optimize/hyperopt.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7ff5c3500..9c18d6803 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -313,22 +313,12 @@ class Hyperopt: 'loss', 'is_initial_point', 'is_best']] trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best'] - + trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '*' trials.loc[trials['is_best'], 'Best'] = 'Best' trials['Objective'] = trials['Objective'].astype(str) - trials = trials.drop(columns=['is_initial_point', 'is_best']) - - if print_colorized: - for i in range(len(trials)): - if trials.loc[i]['Total profit'] > 0: - trials.at[i, 'Best'] = Fore.GREEN + trials.loc[i]['Best'] - trials.at[i, 'Objective'] = "{}{}".format(trials.loc[i]['Objective'], - Fore.RESET) - if 'Best' in trials.loc[i]['Best'] and highlight_best: - trials.at[i, 'Best'] = Style.BRIGHT + trials.loc[i]['Best'] - trials.at[i, 'Objective'] = "{}{}".format(trials.loc[i]['Objective'], - Style.RESET_ALL) + trials.loc[trials['Total profit'] > 0, 'is_profit'] = True + trials['Trades'] = trials['Trades'].astype(str) trials['Epoch'] = trials['Epoch'].apply( lambda x: "{}/{}".format(x, total_epochs)) @@ -340,9 +330,21 @@ class Hyperopt: lambda x: '{: 11.8f} '.format(x) + config['stake_currency'] if not isna(x) else x) trials['Avg duration'] = trials['Avg duration'].apply( lambda x: '{:,.1f}m'.format(x) if not isna(x) else x) + if print_colorized: + for i in range(len(trials)): + if trials.loc[i]['is_profit']: + for z in range(len(trials.loc[i])-3): + trials.iat[i, z] = "{}{}{}".format(Fore.GREEN, + str(trials.loc[i][z]), Fore.RESET) + if trials.loc[i]['is_best'] and highlight_best: + for z in range(len(trials.loc[i])-3): + trials.iat[i, z] = "{}{}{}".format(Style.BRIGHT, + str(trials.loc[i][z]), Style.RESET_ALL) + + trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) print(tabulate(trials.to_dict(orient='list'), headers='keys', tablefmt='psql', - stralign="right")) + stralign="right")) def has_space(self, space: str) -> bool: """ From e9448dc5e248f702b80ddd9be62faefd6e1526d1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Feb 2020 07:01:23 +0100 Subject: [PATCH 64/97] Add tsts for quote and base currency --- tests/exchange/test_exchange.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 98edd0dbb..def4e6ab6 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -403,6 +403,28 @@ def test_get_quote_currencies(default_conf, mocker): assert set(ex.get_quote_currencies()) == set(['USD', 'ETH', 'BTC', 'USDT']) +@pytest.mark.parametrize('pair,expected', [ + ('XRP/BTC', 'BTC'), + ('LTC/USD', 'USD'), + ('ETH/USDT', 'USDT'), + ('XLTCUSDT', 'USDT'), +]) +def test_get_pair_quote_currency(default_conf, mocker, pair, expected): + ex = get_patched_exchange(mocker, default_conf) + assert ex.get_pair_quote_currency(pair) == expected + + +@pytest.mark.parametrize('pair,expected', [ + ('XRP/BTC', 'XRP'), + ('LTC/USD', 'LTC'), + ('ETH/USDT', 'ETH'), + ('XLTCUSDT', 'LTC'), +]) +def test_get_pair_base_currency(default_conf, mocker, pair, expected): + ex = get_patched_exchange(mocker, default_conf) + assert ex.get_pair_base_currency(pair) == expected + + def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs directly api_mock = MagicMock() type(api_mock).markets = PropertyMock(return_value={ From e8eaa8920ea397a9403ece95d27c3a647ed428d8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Feb 2020 07:01:31 +0100 Subject: [PATCH 65/97] Use get_base_currency instead of splitting by / --- freqtrade/freqtradebot.py | 3 +-- freqtrade/wallets.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 38583b5ad..424a6a220 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -938,8 +938,7 @@ class FreqtradeBot: """ # Update wallets to ensure amounts tied up in a stoploss is now free! self.wallets.update() - - wallet_amount = self.wallets.get_free(pair.split('/')[0]) + wallet_amount = self.wallets.get_free(self.exchange.get_pair_base_currency(pair)) logger.debug(f"{pair} - Wallet: {wallet_amount} - Trade-amount: {amount}") if wallet_amount >= amount: return amount diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index dd5e34fe6..b913155bc 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -74,7 +74,7 @@ class Wallets: ) for trade in open_trades: - curr = trade.pair.split('/')[0] + curr = self._exchange.get_pair_base_currency(trade.pair) _wallets[curr] = Wallet( curr, trade.amount, From d34515a5de5b08e21cfc2e6854d4037758fc1f11 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Feb 2020 07:03:11 +0100 Subject: [PATCH 66/97] Remove constraint to have pairs in base/quote format --- freqtrade/constants.py | 2 -- tests/exchange/test_exchange.py | 1 + tests/test_configuration.py | 8 +------- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 105cd6b53..1504d1f1c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -251,7 +251,6 @@ CONF_SCHEMA = { 'type': 'array', 'items': { 'type': 'string', - 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' }, 'uniqueItems': True }, @@ -259,7 +258,6 @@ CONF_SCHEMA = { 'type': 'array', 'items': { 'type': 'string', - 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' }, 'uniqueItems': True }, diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index def4e6ab6..ca2bedb01 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -525,6 +525,7 @@ def test_validate_pairs_stakecompatibility_fail(default_conf, mocker, caplog): with pytest.raises(OperationalException, match=r"Stake-currency 'BTC' not compatible with.*"): Exchange(default_conf) + @pytest.mark.parametrize("timeframe", [ ('5m'), ("1m"), ("15m"), ("1h") ]) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 828db4d83..a58e88ea0 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -34,13 +34,6 @@ def all_conf(): return conf -def test_load_config_invalid_pair(default_conf) -> None: - default_conf['exchange']['pair_whitelist'].append('ETH-BTC') - - with pytest.raises(ValidationError, match=r'.*does not match.*'): - validate_config_schema(default_conf) - - def test_load_config_missing_attributes(default_conf) -> None: conf = deepcopy(default_conf) conf.pop('exchange') @@ -810,6 +803,7 @@ def test_validate_whitelist(default_conf): validate_config_consistency(conf) + def test_load_config_test_comments() -> None: """ Load config with comments From 31ac4598ba966b60be897ef22884442fa63eeb6f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Feb 2020 07:16:37 +0100 Subject: [PATCH 67/97] Fix last occurances of pair splitting --- freqtrade/exchange/exchange.py | 4 ++-- freqtrade/pairlist/VolumePairList.py | 4 ++-- freqtrade/rpc/rpc.py | 4 ++-- tests/rpc/test_rpc.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 0e0d2dabe..6964986b0 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -232,13 +232,13 @@ class Exchange: """ Return a pair's quote currency """ - return self.markets[pair].get('quote') + return self.markets.get(pair, {}).get('quote') def get_pair_base_currency(self, pair: str) -> str: """ Return a pair's quote currency """ - return self.markets[pair].get('base') + return self.markets.get(pair, {}).get('base') def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: if pair_interval in self._klines: diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index e50dafb63..e1cdb4a43 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -91,9 +91,9 @@ class VolumePairList(IPairList): if self._pairlist_pos == 0: # If VolumePairList is the first in the list, use fresh pairlist - # check length so that we make sure that '/' is actually in the string + # check base currency equals to stake currency. filtered_tickers = [v for k, v in tickers.items() - if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency + if (self._exchange.get_pair_quote_currency(k) == base_currency and v[key] is not None)] else: # If other pairlist is in front, use the incomming pairlist. diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 3411318bb..d680d36b1 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -462,7 +462,7 @@ class RPC: # Check pair is in stake currency stake_currency = self._freqtrade.config.get('stake_currency') - if not pair.endswith(stake_currency): + if not self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency: raise RPCException( f'Wrong pair selected. Please pairs with stake {stake_currency} pairs only') # check if valid pair @@ -517,7 +517,7 @@ class RPC: if add: stake_currency = self._freqtrade.config.get('stake_currency') for pair in add: - if (pair.endswith(stake_currency) + if (self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency and pair not in self._freqtrade.pairlists.blacklist): self._freqtrade.pairlists.blacklist.append(pair) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index a35bfa0d6..3c99e6b92 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -687,7 +687,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None # Test buy pair not with stakes with pytest.raises(RPCException, match=r'Wrong pair selected. Please pairs with stake.*'): - rpc._rpc_forcebuy('XRP/ETH', 0.0001) + rpc._rpc_forcebuy('LTC/ETH', 0.0001) pair = 'XRP/BTC' # Test not buying From cfc22577bed042947d40cfa15676c1456aa6f7d4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Feb 2020 16:52:01 +0100 Subject: [PATCH 68/97] Add timeframe_to_minutes to ROI documentation --- docs/strategy-customization.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 07833da34..0dfeb3978 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -249,6 +249,22 @@ minimal_roi = { While technically not completely disabled, this would sell once the trade reaches 10000% Profit. +To use times based on candles, the following snippet can be handy. +This will allow you to change the ticket_interval, and ROI will be set as candles (e.g. after 3 candles ...) + +``` python +from freqtrade.exchange import timeframe_to_minutes + +class AwesomeStrategy(IStrategy): + + ticker_interval = '1d' + ticker_interval_mins = timeframe_to_minutes(ticker_interval) + minimal_roi = { + (ticker_interval_mins * 3): 0.02, # After 3 candles + (ticker_interval_mins * 6): 0.01, # After 6 candles + } +``` + ### Stoploss Setting a stoploss is highly recommended to protect your capital from strong moves against you. From 7d7318a3ea5af47dada77e6595142e8c4bba8237 Mon Sep 17 00:00:00 2001 From: gaugau3000 Date: Tue, 25 Feb 2020 19:41:20 +0000 Subject: [PATCH 69/97] fix_wrong_order_type --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0b9519688..6f139bebe 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -340,7 +340,7 @@ This is most of the time the default time in force. It means the order will rema on exchange till it is canceled by user. It can be fully or partially fulfilled. If partially fulfilled, the remaining will stay on the exchange till cancelled. -**FOK (Full Or Kill):** +**FOK (Fill Or Kill):** It means if the order is not executed immediately AND fully then it is canceled by the exchange. From 76c449c0c232032d128a854764356cebfe504fcb Mon Sep 17 00:00:00 2001 From: gaugau3000 Date: Tue, 25 Feb 2020 19:45:23 +0000 Subject: [PATCH 70/97] volume_pair_list_extra_doc_infos --- docs/configuration.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 6f139bebe..6e8813211 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -532,6 +532,8 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis `refresh_period` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). +`VolumePairList` is based on the volume of the last 24 hours. + ```json "pairlists": [{ "method": "VolumePairList", From ce2e039e5fb9eba15864a1a3df45be17120127d7 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 26 Feb 2020 01:58:32 +0300 Subject: [PATCH 71/97] Update docs/configuration.md --- docs/configuration.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6e8813211..5d1820bdb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -532,7 +532,11 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis `refresh_period` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). -`VolumePairList` is based on the volume of the last 24 hours. +`VolumePairList` is based on the ticker data, as reported by the ccxt library: + +* The `bidVolume` is the volume (amount) of current best bid in the orderbook. +* The `askVolume` is the volume (amount) of current best ask in the orderbook. +* The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours. ```json "pairlists": [{ From df49b98c2578a9d45c0a052f643d00e6fbba2bdd Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 06:40:13 +0100 Subject: [PATCH 72/97] Implement wording-changes Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> --- docs/strategy-customization.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 0dfeb3978..acd985287 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -249,8 +249,8 @@ minimal_roi = { While technically not completely disabled, this would sell once the trade reaches 10000% Profit. -To use times based on candles, the following snippet can be handy. -This will allow you to change the ticket_interval, and ROI will be set as candles (e.g. after 3 candles ...) +To use times based on candle duration (ticker_interval or timeframe), the following snippet can be handy. +This will allow you to change the ticket_interval for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...) ``` python from freqtrade.exchange import timeframe_to_minutes From af4469f073501150c4aaa520fcd614ea18177fad Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 06:43:15 +0100 Subject: [PATCH 73/97] Convert to str to avoid errors --- docs/strategy-customization.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index acd985287..fd40a971e 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -260,8 +260,8 @@ class AwesomeStrategy(IStrategy): ticker_interval = '1d' ticker_interval_mins = timeframe_to_minutes(ticker_interval) minimal_roi = { - (ticker_interval_mins * 3): 0.02, # After 3 candles - (ticker_interval_mins * 6): 0.01, # After 6 candles + str(ticker_interval_mins * 3)): 0.02, # After 3 candles + str(ticker_interval_mins * 6)): 0.01, # After 6 candles } ``` From 1e869b86f279f71c42d7f2bb6a34a2eb4ed60b47 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 06:54:04 +0100 Subject: [PATCH 74/97] Update checkout aciton to v2 https://github.com/actions/checkout/issues/23 suggests that it's fixed in v2. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc8906af5..dc3d324a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: python-version: [3.7, 3.8] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 @@ -118,7 +118,7 @@ jobs: python-version: [3.7] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 @@ -175,7 +175,7 @@ jobs: docs_check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Documentation syntax run: | @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-18.04 if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 From 1021ffa1c3051d17482a00e79eef5a2b211cd103 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 07:00:08 +0100 Subject: [PATCH 75/97] Apply suggestions from code review Add suggested changes to comments Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> --- freqtrade/pairlist/VolumePairList.py | 2 +- freqtrade/rpc/rpc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index e1cdb4a43..d067d5e8a 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -91,7 +91,7 @@ class VolumePairList(IPairList): if self._pairlist_pos == 0: # If VolumePairList is the first in the list, use fresh pairlist - # check base currency equals to stake currency. + # Check if pair quote currency equals to the stake currency. filtered_tickers = [v for k, v in tickers.items() if (self._exchange.get_pair_quote_currency(k) == base_currency and v[key] is not None)] diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index d680d36b1..1e4eaa3e0 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -460,7 +460,7 @@ class RPC: if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') - # Check pair is in stake currency + # Check if pair quote currency equals to the stake currency. stake_currency = self._freqtrade.config.get('stake_currency') if not self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency: raise RPCException( From 4e218be51db55e16bccc5513be244b98467ccb9f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 07:08:09 +0100 Subject: [PATCH 76/97] Don't use markets[pair]['quote'] --- freqtrade/exchange/exchange.py | 2 +- freqtrade/freqtradebot.py | 4 ++-- freqtrade/pairlist/IPairList.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6964986b0..cc0ecc6cd 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -334,7 +334,7 @@ class Exchange: logger.warning(f"Pair {pair} is restricted for some users on this exchange." f"Please check if you are impacted by this restriction " f"on the exchange and eventually remove {pair} from your whitelist.") - if not self.markets[pair].get('quote') == self._config['stake_currency']: + if not self.get_pair_quote_currency(pair) == self._config['stake_currency']: invalid_pairs.append(pair) if invalid_pairs: raise OperationalException( diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 424a6a220..c3b642095 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -938,7 +938,8 @@ class FreqtradeBot: """ # Update wallets to ensure amounts tied up in a stoploss is now free! self.wallets.update() - wallet_amount = self.wallets.get_free(self.exchange.get_pair_base_currency(pair)) + trade_base_currency = self.exchange.get_pair_base_currency(pair) + wallet_amount = self.wallets.get_free(trade_base_currency) logger.debug(f"{pair} - Wallet: {wallet_amount} - Trade-amount: {amount}") if wallet_amount >= amount: return amount @@ -1145,7 +1146,6 @@ class FreqtradeBot: return order_amount amount = 0 fee_abs = 0 - trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) for exectrade in trades: amount += exectrade['amount'] if ("fee" in exectrade and exectrade['fee'] is not None and diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 7d489ece7..d45a329dd 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -100,7 +100,7 @@ class IPairList(ABC): f"{self._exchange.name}. Removing it from whitelist..") continue - if markets[pair]['quote'] != self._config['stake_currency']: + if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']: logger.warning(f"Pair {pair} is not compatible with your stake currency " f"{self._config['stake_currency']}. Removing it from whitelist..") continue From f38accb77b05accfbf328e62c5b417ed7613d527 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 07:09:54 +0100 Subject: [PATCH 77/97] Return empty string if no quote / base currency can be found --- freqtrade/exchange/exchange.py | 4 ++-- tests/exchange/test_exchange.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index cc0ecc6cd..21627679f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -232,13 +232,13 @@ class Exchange: """ Return a pair's quote currency """ - return self.markets.get(pair, {}).get('quote') + return self.markets.get(pair, {}).get('quote', '') def get_pair_base_currency(self, pair: str) -> str: """ Return a pair's quote currency """ - return self.markets.get(pair, {}).get('base') + return self.markets.get(pair, {}).get('base', '') def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: if pair_interval in self._klines: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ca2bedb01..3a653edb6 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -408,6 +408,7 @@ def test_get_quote_currencies(default_conf, mocker): ('LTC/USD', 'USD'), ('ETH/USDT', 'USDT'), ('XLTCUSDT', 'USDT'), + ('XRP/NOCURRENCY', ''), ]) def test_get_pair_quote_currency(default_conf, mocker, pair, expected): ex = get_patched_exchange(mocker, default_conf) @@ -419,6 +420,7 @@ def test_get_pair_quote_currency(default_conf, mocker, pair, expected): ('LTC/USD', 'LTC'), ('ETH/USDT', 'ETH'), ('XLTCUSDT', 'LTC'), + ('XRP/NOCURRENCY', ''), ]) def test_get_pair_base_currency(default_conf, mocker, pair, expected): ex = get_patched_exchange(mocker, default_conf) From a29653b510722251debc426981e3afc1d9929a4d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 08:59:27 +0100 Subject: [PATCH 78/97] Wording changes to install docs Co-Authored-By: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> --- docs/configuration.md | 4 ++-- docs/installation.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 234ff49ba..3844f2812 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -656,8 +656,8 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d ### Setup your exchange account -You will need to create API Keys (usually you get `key` and `secret`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. -API Keys are usually only required for real / production trading, but are not required for paper-trading / dry-run. +You will need to create API Keys (usually you get `key` and `secret`, some exchanges supply it with `password`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. +API Keys are usually only required for live trade (trading for real money, bot running in the "production mode", executing real orders on the exchange) and are not required for the bot running in the dry-run (trade simulation) mode. When you setup the bot in the dry-run mode, you may fill these fields with empty values. ### Using proxy with Freqtrade diff --git a/docs/installation.md b/docs/installation.md index 5a15be234..88e2ef6eb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,7 +2,7 @@ This page explains how to prepare your environment for running the bot. -Please consider using the prebuilt [docker images](docker.md) to get started quickly while trying out freqtrade. +Please consider using the prebuilt [docker images](docker.md) to get started quickly while trying out freqtrade evaluating how it operates. ## Prerequisite From 8ae0f99a960fa4e7bdb615fa19c29fac887cb9e8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Feb 2020 09:05:48 +0100 Subject: [PATCH 79/97] Remove duplicate section --- docs/configuration.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3844f2812..e0dc43f5d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -628,8 +628,8 @@ you run it in production mode. ### Setup your exchange account -You will need to create API Keys (usually you get `key` and `secret`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. -API Keys are usually only required for real / production trading, but are not required for paper-trading / dry-run. +You will need to create API Keys (usually you get `key` and `secret`, some exchanges require an additional `password`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the `freqtrade new-config` command. +API Keys are usually only required for live trading (trading for real money, bot running in "production mode", executing real orders on the exchange) and are not required for the bot running in dry-run (trade simulation) mode. When you setup the bot in dry-run mode, you may fill these fields with empty values. ### To switch your bot in production mode @@ -654,12 +654,6 @@ API Keys are usually only required for real / production trading, but are not re You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange. -### Setup your exchange account - -You will need to create API Keys (usually you get `key` and `secret`, some exchanges supply it with `password`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the installation script. -API Keys are usually only required for live trade (trading for real money, bot running in the "production mode", executing real orders on the exchange) and are not required for the bot running in the dry-run (trade simulation) mode. When you setup the bot in the dry-run mode, you may fill these fields with empty values. - - ### Using proxy with Freqtrade To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. From e5ec97495ddbf331fe43a0ed55a2f938d343d943 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Feb 2020 07:01:00 +0100 Subject: [PATCH 80/97] Logging should be initialized first --- freqtrade/configuration/configuration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 21b3e3bd3..035b091b3 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -96,6 +96,8 @@ class Configuration: # Keep a copy of the original configuration file config['original_config'] = deepcopy(config) + self._process_logging_options(config) + self._process_runmode(config) self._process_common_options(config) @@ -146,8 +148,6 @@ class Configuration: def _process_common_options(self, config: Dict[str, Any]) -> None: - self._process_logging_options(config) - # Set strategy if not specified in config and or if it's non default if self.args.get("strategy") or not config.get('strategy'): config.update({'strategy': self.args.get("strategy")}) @@ -379,7 +379,7 @@ class Configuration: if not self.runmode: # Handle real mode, infer dry/live from config self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE - logger.info(f"Runmode set to {self.runmode}.") + logger.info(f"Runmode set to {self.runmode.value}.") config.update({'runmode': self.runmode}) From bbb438bd4093d38e8e41e05fd151ec1fa2799411 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 27 Feb 2020 06:18:09 +0000 Subject: [PATCH 81/97] Bump python from 3.8.1-slim-buster to 3.8.2-slim-buster Bumps python from 3.8.1-slim-buster to 3.8.2-slim-buster. Signed-off-by: dependabot-preview[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 923285f39..d986f20ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.1-slim-buster +FROM python:3.8.2-slim-buster RUN apt-get update \ && apt-get -y install curl build-essential libssl-dev \ From 55d471190a3afc52984edab5035b460abd429434 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Thu, 27 Feb 2020 13:28:28 +0100 Subject: [PATCH 82/97] Changed table style of backtesting and alignment of headers --- freqtrade/optimize/backtesting.py | 47 +++++++++++++++---------- freqtrade/optimize/optimize_reports.py | 8 ++--- tests/optimize/test_optimize_reports.py | 43 +++++++++++----------- 3 files changed, 53 insertions(+), 45 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c18aefc76..94441ce24 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -423,28 +423,37 @@ class Backtesting: strategy if len(self.strategylist) > 1 else None) print(f"Result for strategy {strategy}") - print(' BACKTESTING REPORT '.center(133, '=')) - print(generate_text_table(data, - stake_currency=self.config['stake_currency'], - max_open_trades=self.config['max_open_trades'], - results=results)) + table = generate_text_table(data, stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results) + if isinstance(table, str): + print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) - print(' SELL REASON STATS '.center(133, '=')) - print(generate_text_table_sell_reason(data, - stake_currency=self.config['stake_currency'], - max_open_trades=self.config['max_open_trades'], - results=results)) + table = generate_text_table_sell_reason(data, + stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results) + if isinstance(table, str): + print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '=')) + print(table) - print(' LEFT OPEN TRADES REPORT '.center(133, '=')) - print(generate_text_table(data, - stake_currency=self.config['stake_currency'], - max_open_trades=self.config['max_open_trades'], - results=results.loc[results.open_at_end], skip_nan=True)) + table = generate_text_table(data, + stake_currency=self.config['stake_currency'], + max_open_trades=self.config['max_open_trades'], + results=results.loc[results.open_at_end], skip_nan=True) + if isinstance(table, str): + print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + if isinstance(table, str): + print('=' * len(table.splitlines()[0])) print() if len(all_results) > 1: # Print Strategy summary table - print(' STRATEGY SUMMARY '.center(133, '=')) - print(generate_text_table_strategy(self.config['stake_currency'], - self.config['max_open_trades'], - all_results=all_results)) + table = generate_text_table_strategy(self.config['stake_currency'], + self.config['max_open_trades'], + all_results=all_results) + print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '=')) + print(table) + print('=' * len(table.splitlines()[0])) print('\nFor more details, please look at the detail tables above') diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index b00adbd48..39bde50a8 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -66,7 +66,7 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra ]) # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore def generate_text_table_sell_reason( @@ -112,7 +112,7 @@ def generate_text_table_sell_reason( profit_percent_tot, ] ) - return tabulate(tabular_data, headers=headers, tablefmt="pipe") + return tabulate(tabular_data, headers=headers, tablefmt="orgtbl", stralign="right") def generate_text_table_strategy(stake_currency: str, max_open_trades: str, @@ -146,7 +146,7 @@ def generate_text_table_strategy(stake_currency: str, max_open_trades: str, ]) # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore def generate_edge_table(results: dict) -> str: @@ -172,4 +172,4 @@ def generate_edge_table(results: dict) -> str: # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 57e928cca..285ecaa02 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -22,14 +22,14 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' - ' Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' - '|:--------|-------:|---------------:|---------------:|-----------------:|' - '---------------:|:---------------|-------:|--------:|---------:|\n' + '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' + ' Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|---------+--------+----------------+----------------+------------------+' + '----------------+----------------+--------+---------+----------|\n' '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 |' - ' 15.00 | 0:20:00 | 2 | 0 | 0 |\n' - '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 |' - ' 15.00 | 0:20:00 | 2 | 0 | 0 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |\n' + '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |' ) assert generate_text_table(data={'ETH/BTC': {}}, stake_currency='BTC', max_open_trades=2, @@ -52,13 +52,13 @@ def test_generate_text_table_sell_reason(default_conf, mocker): ) result_str = ( - '| Sell Reason | Sells | Wins | Draws | Losses |' + '| Sell Reason | Sells | Wins | Draws | Losses |' ' Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |\n' - '|:--------------|--------:|-------:|--------:|---------:|' - '---------------:|---------------:|-----------------:|---------------:|\n' - '| roi | 2 | 2 | 0 | 0 |' + '|---------------+---------+--------+---------+----------+' + '----------------+----------------+------------------+----------------|\n' + '| roi | 2 | 2 | 0 | 0 |' ' 15 | 30 | 0.6 | 15 |\n' - '| stop_loss | 1 | 0 | 0 | 1 |' + '| stop_loss | 1 | 0 | 0 | 1 |' ' -10 | -10 | -0.2 | -5 |' ) assert generate_text_table_sell_reason( @@ -95,14 +95,14 @@ def test_generate_text_table_strategy(default_conf, mocker): ) result_str = ( - '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' - ' Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' - '|:--------------|-------:|---------------:|---------------:|------' - '-----------:|---------------:|:---------------|-------:|--------:|---------:|\n' - '| TestStrategy1 | 3 | 20.00 | 60.00 | ' - ' 1.10000000 | 30.00 | 0:17:00 | 3 | 0 | 0 |\n' - '| TestStrategy2 | 3 | 30.00 | 90.00 | ' - ' 1.30000000 | 45.00 | 0:20:00 | 3 | 0 | 0 |' + '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' + ' Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|---------------+--------+----------------+----------------+------------------+' + '----------------+----------------+--------+---------+----------|\n' + '| TestStrategy1 | 3 | 20.00 | 60.00 | 1.10000000 |' + ' 30.00 | 0:17:00 | 3 | 0 | 0 |\n' + '| TestStrategy2 | 3 | 30.00 | 90.00 | 1.30000000 |' + ' 45.00 | 0:20:00 | 3 | 0 | 0 |' ) assert generate_text_table_strategy('BTC', 2, all_results=results) == result_str @@ -111,8 +111,7 @@ def test_generate_edge_table(edge_conf, mocker): results = {} results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) - - assert generate_edge_table(results).count(':|') == 7 + assert generate_edge_table(results).count('+') == 7 assert generate_edge_table(results).count('| ETH/BTC |') == 1 assert generate_edge_table(results).count( '| Risk Reward Ratio | Required Risk Reward | Expectancy |') == 1 From e5a9c81412b4a6846872fde59489203184244288 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Feb 2020 14:04:12 +0100 Subject: [PATCH 83/97] Add 0 entry to enhanced ROI example --- docs/strategy-customization.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index fd40a971e..4aacd3af6 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -257,11 +257,12 @@ from freqtrade.exchange import timeframe_to_minutes class AwesomeStrategy(IStrategy): - ticker_interval = '1d' + ticker_interval = "1d" ticker_interval_mins = timeframe_to_minutes(ticker_interval) minimal_roi = { - str(ticker_interval_mins * 3)): 0.02, # After 3 candles - str(ticker_interval_mins * 6)): 0.01, # After 6 candles + "0": 0.05, # 5% for the first 3 candles + str(ticker_interval_mins * 3)): 0.02, # 2% after 3 candles + str(ticker_interval_mins * 6)): 0.01, # 1% After 6 candles } ``` From 15e59654d9121bbbe8a1a20f4d1e0529906cc18d Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Thu, 27 Feb 2020 16:10:45 +0100 Subject: [PATCH 84/97] Minor change to standardize table style. This PR will target commands. --- freqtrade/commands/list_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 49674b81a..327901dc0 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -58,7 +58,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: else yellow + "DUPLICATE NAME" + reset) } for s in objs] - print(tabulate(objss_to_print, headers='keys', tablefmt='pipe')) + print(tabulate(objss_to_print, headers='keys', tablefmt='psql', stralign='right')) def start_list_strategies(args: Dict[str, Any]) -> None: @@ -192,7 +192,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: else: # print data as a table, with the human-readable summary print(f"{summary_str}:") - print(tabulate(tabular_data, headers='keys', tablefmt='pipe')) + print(tabulate(tabular_data, headers='keys', tablefmt='psql', stralign='right')) elif not (args.get('print_one_column', False) or args.get('list_pairs_print_json', False) or args.get('print_csv', False)): From 5a02026f82decd18789f78440e4a71b51e24ff89 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Feb 2020 19:35:58 +0100 Subject: [PATCH 85/97] Add test validating behaviour --- tests/test_configuration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index a58e88ea0..1e9d6440d 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -319,6 +319,7 @@ def test_load_dry_run(default_conf, mocker, config_value, expected, arglist) -> validated_conf = configuration.load_config() assert validated_conf['dry_run'] is expected + assert validated_conf['runmode'] == (RunMode.DRY_RUN if expected else RunMode.LIVE) def test_load_custom_strategy(default_conf, mocker) -> None: From a55964a6221c1806ba195bf7be1dfe4167fd15f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Feb 2020 19:36:54 +0100 Subject: [PATCH 86/97] we Must parse --dry-run before setting run-mode --- freqtrade/configuration/configuration.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 035b091b3..6a0441957 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -167,10 +167,6 @@ class Configuration: if 'sd_notify' in self.args and self.args["sd_notify"]: config['internals'].update({'sd_notify': True}) - self._args_to_config(config, argname='dry_run', - logstring='Parameter --dry-run detected, ' - 'overriding dry_run to: {} ...') - def _process_datadir_options(self, config: Dict[str, Any]) -> None: """ Extract information for sys.argv and load directory configurations @@ -376,6 +372,10 @@ class Configuration: def _process_runmode(self, config: Dict[str, Any]) -> None: + self._args_to_config(config, argname='dry_run', + logstring='Parameter --dry-run detected, ' + 'overriding dry_run to: {} ...') + if not self.runmode: # Handle real mode, infer dry/live from config self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE From bee8e92f0287aec18b77e936a5a5d1771aa869a0 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Fri, 28 Feb 2020 23:50:25 +0300 Subject: [PATCH 87/97] Final changes, use sqrt i.o. statistics.pstdev --- freqtrade/optimize/hyperopt_loss_sortino_daily.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index 0f81ffca5..16dc26142 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -5,7 +5,6 @@ This module defines the alternative HyperOptLoss class which can be used for Hyperoptimization. """ import math -import statistics from datetime import datetime from pandas import DataFrame, date_range @@ -56,7 +55,9 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): sum_daily['downside_returns'] = 0 sum_daily.loc[total_profit < 0, 'downside_returns'] = total_profit total_downside = sum_daily['downside_returns'] - down_stdev = statistics.pstdev(total_downside, 0) + # Here total_downside contains min(0, P - MAR) values, + # where P = sum_daily["profit_percent_after_slippage"] + down_stdev = math.sqrt((total_downside**2).sum() / len(total_downside)) if (down_stdev != 0.): sortino_ratio = expected_returns_mean / down_stdev * math.sqrt(days_in_year) From 5277d71913566088747c6499740bcdc68ac51f27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 14:56:04 +0100 Subject: [PATCH 88/97] Add test for empty stake-currency --- tests/exchange/test_exchange.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 179566bb0..6bec53d49 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -511,6 +511,22 @@ def test_validate_pairs_stakecompatibility(default_conf, mocker, caplog): Exchange(default_conf) +def test_validate_pairs_stakecompatibility_downloaddata(default_conf, mocker, caplog): + api_mock = MagicMock() + default_conf['stake_currency'] = '' + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'BTC'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + Exchange(default_conf) + + def test_validate_pairs_stakecompatibility_fail(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'].append('HELLO-WORLD') api_mock = MagicMock() From 60579485e52dd36d1bc20344766baf9850510b57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 14:56:36 +0100 Subject: [PATCH 89/97] fix empty stake currency problem --- freqtrade/exchange/exchange.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 799ee2c37..f6b722c9a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -332,7 +332,8 @@ class Exchange: logger.warning(f"Pair {pair} is restricted for some users on this exchange." f"Please check if you are impacted by this restriction " f"on the exchange and eventually remove {pair} from your whitelist.") - if not self.get_pair_quote_currency(pair) == self._config['stake_currency']: + if (self._config['stake_currency'] and + not self.get_pair_quote_currency(pair) == self._config['stake_currency']): invalid_pairs.append(pair) if invalid_pairs: raise OperationalException( From 9336d8ee02d63e56513fe0a76c5bbef00aab9390 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 15:44:45 +0100 Subject: [PATCH 90/97] Try fix random testfailure --- tests/commands/test_commands.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 5f9bc0aa2..1877aaa43 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -447,11 +447,9 @@ def test_create_datadir_failed(caplog): def test_create_datadir(caplog, mocker): - # Ensure that caplog is empty before starting ... - # Should prevent random failures. - caplog.clear() - # Added assert here to analyze random test-failures ... - assert len(caplog.record_tuples) == 0 + + # Capture caplog length here trying to avoid random test failure + len_caplog_before = len(caplog.record_tuples) cud = mocker.patch("freqtrade.commands.deploy_commands.create_userdata_dir", MagicMock()) csf = mocker.patch("freqtrade.commands.deploy_commands.copy_sample_files", MagicMock()) @@ -464,7 +462,7 @@ def test_create_datadir(caplog, mocker): assert cud.call_count == 1 assert csf.call_count == 1 - assert len(caplog.record_tuples) == 0 + assert len(caplog.record_tuples) == len_caplog_before def test_start_new_strategy(mocker, caplog): From f25adf3b12096fab33dbcb098a8d679067334a17 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 15:48:36 +0100 Subject: [PATCH 91/97] improve and correct release documentation --- docs/developer.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index b128ffd2b..ef9232a59 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -234,7 +234,7 @@ git checkout -b new_release Determine if crucial bugfixes have been made between this commit and the current state, and eventually cherry-pick these. -* Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7-1` should we need to do a second release that month. +* Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi. * Commit this part * push that branch to the remote and create a PR against the master branch @@ -268,11 +268,6 @@ Once the PR against master is merged (best right after merging): * Use "master" as reference (this step comes after the above PR is merged). * Use the above changelog as release comment (as codeblock) -### After-release - -* Update version in develop by postfixing that with `-dev` (`2019.6 -> 2019.6-dev`). -* Create a PR against develop to update that branch. - ## Releases ### pypi From 848054d140243a18e2dfa8eb70077e6be15a4f94 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 15:53:54 +0100 Subject: [PATCH 92/97] Fix jupyter notebook example - generate_candlestick_graph() needs a filtered pairlist, not a list containing all pairs --- docs/strategy_analysis_example.md | 6 ++++-- freqtrade/templates/strategy_analysis_example.ipynb | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 53b35ca09..d26d684ce 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -121,7 +121,6 @@ from freqtrade.data.btanalysis import analyze_trade_parallelism # Analyze the above parallel_trades = analyze_trade_parallelism(trades, '5m') - parallel_trades.plot() ``` @@ -134,11 +133,14 @@ Freqtrade offers interactive plotting capabilities based on plotly. from freqtrade.plot.plotting import generate_candlestick_graph # Limit graph period to keep plotly quick and reactive +# Filter trades to one pair +trades_red = trades.loc[trades['pair'] == pair] + data_red = data['2019-06-01':'2019-06-10'] # Generate candlestick graph graph = generate_candlestick_graph(pair=pair, data=data_red, - trades=trades, + trades=trades_red, indicators1=['sma20', 'ema50', 'ema55'], indicators2=['rsi', 'macd', 'macdsignal', 'macdhist'] ) diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 399235cfe..dffa308ce 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -190,7 +190,6 @@ "# Analyze the above\n", "parallel_trades = analyze_trade_parallelism(trades, '5m')\n", "\n", - "\n", "parallel_trades.plot()" ] }, @@ -212,11 +211,14 @@ "from freqtrade.plot.plotting import generate_candlestick_graph\n", "# Limit graph period to keep plotly quick and reactive\n", "\n", + "# Filter trades to one pair\n", + "trades_red = trades.loc[trades['pair'] == pair]\n", + "\n", "data_red = data['2019-06-01':'2019-06-10']\n", "# Generate candlestick graph\n", "graph = generate_candlestick_graph(pair=pair,\n", " data=data_red,\n", - " trades=trades,\n", + " trades=trades_red,\n", " indicators1=['sma20', 'ema50', 'ema55'],\n", " indicators2=['rsi', 'macd', 'macdsignal', 'macdhist']\n", " )\n", From 4c39f360844a5db0352914e190a7a933489adb4d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 16:36:33 +0100 Subject: [PATCH 93/97] Add note about InvalidNonce to documentation --- docs/exchanges.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/exchanges.md b/docs/exchanges.md index f615bc61a..70dae0aa5 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -62,6 +62,11 @@ res = [ f"{x['MarketCurrency']}/{x['BaseCurrency']}" for x in ct.publicGetMarket print(res) ``` +## All exchanges + +Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys. + + ## Random notes for other exchanges * The Ocean (exchange id: `theocean`) exchange uses Web3 functionality and requires `web3` python package to be installed: From d7373be55344f80883203fa493f766d542911882 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 16:58:22 +0100 Subject: [PATCH 94/97] Add official support for Kraken --- README.md | 3 ++- docs/configuration.md | 5 ++++- freqtrade/exchange/exchange.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 59799da84..88070d45e 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ hesitate to read the source code and understand the mechanism of this bot. ## Exchange marketplaces supported - [X] [Bittrex](https://bittrex.com/) -- [X] [Binance](https://www.binance.com/) ([*Note for binance users](#a-note-on-binance)) +- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#blacklists)) +- [X] [Kraken](https://kraken.com/) - [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Documentation diff --git a/docs/configuration.md b/docs/configuration.md index b05dab7c9..95af2c049 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -371,15 +371,18 @@ The possible values are: `gtc` (default), `fok` or `ioc`. Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency exchange markets and trading APIs. The complete up-to-date list can be found in the [CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested -with only Bittrex and Binance. +with only Bittrex, Binance and Kraken. The bot was tested with the following exchanges: - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" +- [Kraken](https://kraken.com/): "kraken" Feel free to test other exchanges and submit your PR to improve the bot. +Some exchanges require special configuration, which can be found on the [Exchange-specific Notes](exchanges.md) documentation page. + #### Sample exchange configuration A exchange configuration for "binance" would look as follows: diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 799ee2c37..4f03bde80 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1023,7 +1023,7 @@ def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = Non def is_exchange_officially_supported(exchange_name: str) -> bool: - return exchange_name in ['bittrex', 'binance'] + return exchange_name in ['bittrex', 'binance', 'kraken'] def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: From 18d724f7a18994ba4bf50b605218f9ee76e5750a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 19:22:36 +0100 Subject: [PATCH 95/97] Adjust wording of supported exchanges --- docs/configuration.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 95af2c049..b3f032bc6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -370,10 +370,9 @@ The possible values are: `gtc` (default), `fok` or `ioc`. Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency exchange markets and trading APIs. The complete up-to-date list can be found in the -[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested -with only Bittrex, Binance and Kraken. - -The bot was tested with the following exchanges: +[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). + However, the bot was tested by the development team with only Bittrex, Binance and Kraken, + so the these are the only officially supported exhanges: - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" From 60f04cff4de955361b310295b702150e256b8789 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Feb 2020 20:41:03 +0100 Subject: [PATCH 96/97] Simplify expression --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index f6b722c9a..d1397a282 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -333,7 +333,7 @@ class Exchange: f"Please check if you are impacted by this restriction " f"on the exchange and eventually remove {pair} from your whitelist.") if (self._config['stake_currency'] and - not self.get_pair_quote_currency(pair) == self._config['stake_currency']): + self.get_pair_quote_currency(pair) != self._config['stake_currency']): invalid_pairs.append(pair) if invalid_pairs: raise OperationalException( From 77b7f95efb786c71e325cc662dea0bb6bc61d291 Mon Sep 17 00:00:00 2001 From: Yazeed Al Oyoun Date: Mon, 2 Mar 2020 00:14:01 +0100 Subject: [PATCH 97/97] simple code styling fixes --- freqtrade/commands/hyperopt_commands.py | 56 ++++++++++++------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index ccaa59e54..e7f89a375 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -97,10 +97,10 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: if n > trials_epochs: raise OperationalException( - f"The index of the epoch to show should be less than {trials_epochs + 1}.") + f"The index of the epoch to show should be less than {trials_epochs + 1}.") if n < -trials_epochs: raise OperationalException( - f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") + f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") # Translate epoch index from human-readable format to pythonic if n > 0: @@ -122,52 +122,52 @@ def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: trials = [x for x in trials if x['results_metrics']['profit'] > 0] if filteroptions['filter_min_trades'] > 0: trials = [ - x for x in trials - if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] - ] + x for x in trials + if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] + ] if filteroptions['filter_max_trades'] > 0: trials = [ - x for x in trials - if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] - ] + x for x in trials + if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] + ] if filteroptions['filter_min_avg_time'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] - ] + x for x in trials + if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] + ] if filteroptions['filter_max_avg_time'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] - ] + x for x in trials + if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] + ] if filteroptions['filter_min_avg_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['avg_profit'] - > filteroptions['filter_min_avg_profit'] - ] + x for x in trials + if x['results_metrics']['avg_profit'] + > filteroptions['filter_min_avg_profit'] + ] if filteroptions['filter_max_avg_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['avg_profit'] - < filteroptions['filter_max_avg_profit'] - ] + x for x in trials + if x['results_metrics']['avg_profit'] + < filteroptions['filter_max_avg_profit'] + ] if filteroptions['filter_min_total_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] - ] + x for x in trials + if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] + ] if filteroptions['filter_max_total_profit'] is not None: trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] trials = [ - x for x in trials - if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] - ] + x for x in trials + if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] + ] logger.info(f"{len(trials)} " + ("best " if filteroptions['only_best'] else "") +