From 9c180e587bb96bc9d70dc257ca4390ac513aa5cc Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 30 Oct 2019 04:04:28 +0300 Subject: [PATCH 01/51] Log to stderr --- freqtrade/loggers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index 90b8905e5..e00f4fc11 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -33,8 +33,8 @@ def setup_logging(config: Dict[str, Any]) -> None: # Log level verbosity = config['verbosity'] - # Log to stdout, not stderr - log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stdout)] + # Log to stderr + log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)] if config.get('logfile'): log_handlers.append(RotatingFileHandler(config['logfile'], From f20f5cebbe24f53660c0161b8556249d4ce0ba19 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 11:09:41 +0100 Subject: [PATCH 02/51] Move performance-calculation to persistence --- freqtrade/persistence.py | 23 ++++++++++++++++++++--- freqtrade/rpc/rpc.py | 13 +------------ tests/test_persistence.py | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 1850aafd9..6bd4b0a30 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -8,17 +8,15 @@ from typing import Any, Dict, List, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, inspect) + create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker -from sqlalchemy import func from sqlalchemy.pool import StaticPool from freqtrade import OperationalException - logger = logging.getLogger(__name__) @@ -404,6 +402,25 @@ class Trade(_DECL_BASE): .scalar() return total_open_stake_amount or 0 + @staticmethod + def get_overall_performance() -> Dict: + pair_rates = Trade.session.query( + Trade.pair, + func.sum(Trade.close_profit).label('profit_sum'), + func.count(Trade.pair).label('count') + ).filter(Trade.is_open.is_(False))\ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum')) \ + .all() + return [ + { + 'pair': pair, + 'profit': round(rate * 100, 2), + 'count': count + } + for pair, rate, count in pair_rates + ] + @staticmethod def get_open_trades() -> List[Any]: """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f994ac006..c50a7937e 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -442,18 +442,7 @@ class RPC: Handler for performance. Shows a performance statistic from finished trades """ - - pair_rates = Trade.session.query(Trade.pair, - sql.func.sum(Trade.close_profit).label('profit_sum'), - sql.func.count(Trade.pair).label('count')) \ - .filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')) \ - .all() - return [ - {'pair': pair, 'profit': round(rate * 100, 2), 'count': count} - for pair, rate, count in pair_rates - ] + return Trade.get_overall_performance() def _rpc_count(self) -> Dict[str, float]: """ Returns the number of trades running """ diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 6bd223a9b..8cf5f1756 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -35,6 +35,8 @@ def create_mock_trades(fee): fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.123, + close_rate=0.128, + close_profit=0.005, exchange='bittrex', is_open=False, open_order_id='dry_run_sell_12345' @@ -835,3 +837,25 @@ def test_stoploss_reinitialization(default_conf, fee): assert trade_adj.stop_loss_pct == -0.04 assert trade_adj.initial_stop_loss == 0.96 assert trade_adj.initial_stop_loss_pct == -0.04 + + +@pytest.mark.usefixtures("init_persistence") +def test_total_open_trades_stakes(fee): + + res = Trade.total_open_trades_stakes() + assert res == 0 + create_mock_trades(fee) + res = Trade.total_open_trades_stakes() + assert res == 0.002 + + +@pytest.mark.usefixtures("init_persistence") +def test_get_overall_performance(fee): + + create_mock_trades(fee) + res = Trade.get_overall_performance() + + assert len(res) == 1 + assert 'pair' in res[0] + assert 'profit' in res[0] + assert 'count' in res[0] From ab117527c9a89d68056023982ff0129c8fe71605 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 11:15:33 +0100 Subject: [PATCH 03/51] Refactor get_best_pair to persistence --- freqtrade/persistence.py | 9 +++++++++ freqtrade/rpc/rpc.py | 6 +----- tests/test_persistence.py | 13 +++++++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 6bd4b0a30..fe0b64bcc 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -421,6 +421,15 @@ class Trade(_DECL_BASE): for pair, rate, count in pair_rates ] + @staticmethod + def get_best_pair(): + best_pair = Trade.session.query( + Trade.pair, func.sum(Trade.close_profit).label('profit_sum') + ).filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum')).first() + return best_pair + @staticmethod def get_open_trades() -> List[Any]: """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c50a7937e..dc25c3743 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -225,11 +225,7 @@ class RPC: ) profit_all_perc.append(profit_percent) - best_pair = Trade.session.query( - Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum') - ).filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(sql.text('profit_sum DESC')).first() + best_pair = Trade.get_best_pair() if not best_pair: raise RPCException('no closed trade') diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 8cf5f1756..4aa69423e 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -859,3 +859,16 @@ def test_get_overall_performance(fee): assert 'pair' in res[0] assert 'profit' in res[0] assert 'count' in res[0] + + +@pytest.mark.usefixtures("init_persistence") +def test_get_best_pair(fee): + + res = Trade.get_best_pair() + assert res is None + + create_mock_trades(fee) + res = Trade.get_best_pair() + assert len(res) == 2 + assert res[0] == 'ETC/BTC' + assert res[1] == 0.005 From 01efebc42f27ae0250955d84bc99868ed0877a2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 13:32:07 +0100 Subject: [PATCH 04/51] Extract query to it's own function --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a8fc6bc7e..ef9a85154 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -768,7 +768,7 @@ class FreqtradeBot: buy_timeout_threshold = arrow.utcnow().shift(minutes=-buy_timeout).datetime sell_timeout_threshold = arrow.utcnow().shift(minutes=-sell_timeout).datetime - for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): + for trade in Trade.get_open_order_trades(): try: # FIXME: Somehow the query above returns results # where the open_order_id is in fact None. diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index fe0b64bcc..e527cde16 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -391,6 +391,13 @@ class Trade(_DECL_BASE): profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + @staticmethod + def get_open_order_trades(): + """ + Returns all open trades + """ + return Trade.query.filter(Trade.open_order_id.isnot(None)).all() + @staticmethod def total_open_trades_stakes() -> float: """ @@ -403,7 +410,10 @@ class Trade(_DECL_BASE): return total_open_stake_amount or 0 @staticmethod - def get_overall_performance() -> Dict: + def get_overall_performance() -> List[Dict]: + """ + Returns List of dicts containing all Trades, including profit and trade count + """ pair_rates = Trade.session.query( Trade.pair, func.sum(Trade.close_profit).label('profit_sum'), @@ -423,6 +433,9 @@ class Trade(_DECL_BASE): @staticmethod def get_best_pair(): + """ + Get best pair with closed trade. + """ best_pair = Trade.session.query( Trade.pair, func.sum(Trade.close_profit).label('profit_sum') ).filter(Trade.is_open.is_(False)) \ From 26a5800a7f9afbd8236e2d89d6c26c29556f1268 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 15:01:10 +0100 Subject: [PATCH 05/51] Extract get_trades function --- freqtrade/data/btanalysis.py | 2 +- freqtrade/persistence.py | 31 +++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0f5d395ff..462547d9e 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -106,7 +106,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: t.stop_loss, t.initial_stop_loss, t.strategy, t.ticker_interval ) - for t in Trade.query.all()], + for t in Trade.get_trades().all()], columns=columns) return trades diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index e527cde16..808e42c9a 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -11,6 +11,7 @@ from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import Query from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool @@ -391,12 +392,33 @@ class Trade(_DECL_BASE): profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + @staticmethod + def get_trades(trade_filter=None) -> Query: + """ + Helper function to query Trades using filter. + :param trade_filter: Filter to apply to trades + :return: Query object + """ + if trade_filter is not None: + if not isinstance(trade_filter, list): + trade_filter = [trade_filter] + return Trade.query.filter(*trade_filter) + else: + return Trade.query + + @staticmethod + def get_open_trades() -> List[Any]: + """ + Query trades from persistence layer + """ + return Trade.get_trades(Trade.is_open.is_(True)).all() + @staticmethod def get_open_order_trades(): """ Returns all open trades """ - return Trade.query.filter(Trade.open_order_id.isnot(None)).all() + return Trade.get_trades(Trade.open_order_id.isnot(None)).all() @staticmethod def total_open_trades_stakes() -> float: @@ -443,13 +465,6 @@ class Trade(_DECL_BASE): .order_by(desc('profit_sum')).first() return best_pair - @staticmethod - def get_open_trades() -> List[Any]: - """ - Query trades from persistence layer - """ - return Trade.query.filter(Trade.is_open.is_(True)).all() - @staticmethod def stoploss_reinitialization(desired_stoploss): """ From b37c5e4878c1900f8cb34d68438a59cf2c5c987d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 15:09:01 +0100 Subject: [PATCH 06/51] use get_trades in rpc modules --- freqtrade/persistence.py | 9 ++++++--- freqtrade/rpc/rpc.py | 25 ++++++++++--------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 808e42c9a..a15db87c7 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -395,9 +395,12 @@ class Trade(_DECL_BASE): @staticmethod def get_trades(trade_filter=None) -> Query: """ - Helper function to query Trades using filter. - :param trade_filter: Filter to apply to trades - :return: Query object + Helper function to query Trades using filters. + :param trade_filter: Optional filter to apply to trades + Can be either a Filter object, or a List of filters + e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])` + e.g. `(trade_filter=Trade.id == trade_id)` + :return: unsorted query object """ if trade_filter is not None: if not isinstance(trade_filter, list): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dc25c3743..8eecb04f9 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -9,7 +9,6 @@ from enum import Enum from typing import Dict, Any, List, Optional import arrow -import sqlalchemy as sql from numpy import mean, NAN from pandas import DataFrame @@ -154,12 +153,11 @@ class RPC: for day in range(0, timescale): profitday = today - timedelta(days=day) - trades = Trade.query \ - .filter(Trade.is_open.is_(False)) \ - .filter(Trade.close_date >= profitday)\ - .filter(Trade.close_date < (profitday + timedelta(days=1)))\ - .order_by(Trade.close_date)\ - .all() + trades = Trade.get_trades(trade_filter=[ + Trade.is_open.is_(False), + Trade.close_date >= profitday, + Trade.close_date < (profitday + timedelta(days=1)) + ]).order_by(Trade.close_date).all() curdayprofit = sum(trade.calc_profit() for trade in trades) profit_days[profitday] = { 'amount': f'{curdayprofit:.8f}', @@ -192,7 +190,7 @@ class RPC: def _rpc_trade_statistics( self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: """ Returns cumulative profit statistics """ - trades = Trade.query.order_by(Trade.id).all() + trades = Trade.get_trades().order_by(Trade.id).all() profit_all_coin = [] profit_all_perc = [] @@ -385,11 +383,8 @@ class RPC: return {'result': 'Created sell orders for all open trades.'} # Query for trade - trade = Trade.query.filter( - sql.and_( - Trade.id == trade_id, - Trade.is_open.is_(True) - ) + trade = Trade.get_trades( + trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ] ).first() if not trade: logger.warning('forcesell: Invalid argument received') @@ -419,7 +414,7 @@ class RPC: # check if valid pair # check if pair already has an open pair - trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first() + trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first() if trade: raise RPCException(f'position for {pair} already open - id: {trade.id}') @@ -428,7 +423,7 @@ class RPC: # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price): - trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first() + trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first() return trade else: return None From c2076d86a4f32901784f905f251d3b94bd1cc1a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 14:26:03 +0100 Subject: [PATCH 07/51] Use scoped_session as intended --- freqtrade/persistence.py | 8 +++++--- tests/test_persistence.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index a15db87c7..27a283378 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -51,9 +51,11 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: raise OperationalException(f"Given value for db_url: '{db_url}' " f"is no valid database URL! (See {_SQL_DOCS_URL})") - session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) - Trade.session = session() - Trade.query = session.query_property() + # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope + # Scoped sessions proxy requests to the appropriate thread-local session. + # We should use the scoped_session object - not a seperately initialized version + Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) + Trade.query = Trade.session.query_property() _DECL_BASE.metadata.create_all(engine) check_migrate(engine) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 4aa69423e..231a1d2e2 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -61,7 +61,7 @@ def test_init_create_session(default_conf): # Check if init create a session init(default_conf['db_url'], default_conf['dry_run']) assert hasattr(Trade, 'session') - assert 'Session' in type(Trade.session).__name__ + assert 'scoped_session' in type(Trade.session).__name__ def test_init_custom_db_url(default_conf, mocker): From 5ed777114837cd549b17193df6a37b3e64c41792 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2019 15:39:36 +0100 Subject: [PATCH 08/51] Update documentation to include get_trades fixes #1753 --- docs/strategy-customization.md | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index cef362ffd..71d6f3e8f 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -405,6 +405,52 @@ if self.wallets: - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above +### Additional data (Trades) + +A history of Trades can be retrieved in the strategy by querying the database. + +At the top of the file, import Trade. + +```python +from freqtrade.persistence import Trade +``` + +The following example queries for the current pair and trades from today, however other filters easily be added. + +``` python +if self.config['runmode'] in ('live', 'dry_run'): + trades = Trade.get_trades([Trade.pair == metadata['pair'], + Trade.open_date > datetime.utcnow() - timedelta(days=1), + Trade.is_open == False, + ]).order_by(Trade.close_date).all() + # Summarize profit for this pair. + curdayprofit = sum(trade.close_profit for trade in trades) +``` + +Get amount of stake_currency currently invested in Trades: + +``` python +if self.config['runmode'] in ('live', 'dry_run'): + total_stakes = Trade.total_open_trades_stakes() +``` + +Retrieve performance per pair. +Returns a List of dicts per pair. + +``` python +if self.config['runmode'] in ('live', 'dry_run'): + performance = Trade.get_overall_performance() +``` + +Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5%. + +``` json +{'pair': "ETH/BTC", 'profit': 1.5, 'count': 5} +``` + +!!! Warning + Trade history is not available during backtesting or hyperopt. + ### Print created dataframe To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`. From b7b1e66c6ee3c6dc4342aef73dc5c4e7c8056701 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 09:59:54 +0100 Subject: [PATCH 09/51] Convert to % as part of RPC to allow users to use unrounded ratio --- docs/strategy-customization.md | 6 +++--- freqtrade/persistence.py | 4 ++-- freqtrade/rpc/rpc.py | 7 +++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 71d6f3e8f..b3b6e3548 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -415,7 +415,7 @@ At the top of the file, import Trade. from freqtrade.persistence import Trade ``` -The following example queries for the current pair and trades from today, however other filters easily be added. +The following example queries for the current pair and trades from today, however other filters can easily be added. ``` python if self.config['runmode'] in ('live', 'dry_run'): @@ -442,10 +442,10 @@ if self.config['runmode'] in ('live', 'dry_run'): performance = Trade.get_overall_performance() ``` -Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5%. +Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5% (ratio of 0.015). ``` json -{'pair': "ETH/BTC", 'profit': 1.5, 'count': 5} +{'pair': "ETH/BTC", 'profit': 0.015, 'count': 5} ``` !!! Warning diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 27a283378..735c740c3 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -437,7 +437,7 @@ class Trade(_DECL_BASE): return total_open_stake_amount or 0 @staticmethod - def get_overall_performance() -> List[Dict]: + def get_overall_performance() -> List[Dict[str, Any]]: """ Returns List of dicts containing all Trades, including profit and trade count """ @@ -452,7 +452,7 @@ class Trade(_DECL_BASE): return [ { 'pair': pair, - 'profit': round(rate * 100, 2), + 'profit': rate, 'count': count } for pair, rate, count in pair_rates diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8eecb04f9..4aed48f74 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -428,12 +428,15 @@ class RPC: else: return None - def _rpc_performance(self) -> List[Dict]: + def _rpc_performance(self) -> List[Dict[str, Any]]: """ Handler for performance. Shows a performance statistic from finished trades """ - return Trade.get_overall_performance() + pair_rates = Trade.get_overall_performance() + # Round and convert to % + [x.update({'profit': round(x['profit'] * 100, 2)}) for x in pair_rates] + return pair_rates def _rpc_count(self) -> Dict[str, float]: """ Returns the number of trades running """ From 7a96d3c9ae8992d2962640e668be1d6bebbb5a72 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 13:27:04 +0100 Subject: [PATCH 10/51] Update raspbian install documentation Fix "box" titles ... they need to be in quotes! --- docs/configuration.md | 2 +- docs/data-download.md | 2 +- docs/docker.md | 2 +- docs/installation.md | 25 +++++++++++-------------- docs/plotting.md | 2 +- docs/rest-api.md | 8 ++++---- docs/strategy-customization.md | 8 ++++---- docs/telegram-usage.md | 2 +- 8 files changed, 24 insertions(+), 27 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1ad13c87a..6ba0447b5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -254,7 +254,7 @@ Configuration: !!! Note If `stoploss_on_exchange` is enabled and the stoploss is cancelled manually on the exchange, then the bot will create a new order. -!!! Warning stoploss_on_exchange failures +!!! Warning "Warning: stoploss_on_exchange failures" If stoploss on exchange creation fails for some reason, then an "emergency sell" is initiated. By default, this will sell the asset using a market order. The order-type for the emergency-sell can be changed by setting the `emergencysell` value in the `order_types` dictionary - however this is not advised. ### Understand order_time_in_force diff --git a/docs/data-download.md b/docs/data-download.md index bf4792ea3..f105e7a56 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -8,7 +8,7 @@ If no additional parameter is specified, freqtrade will download data for `"1m"` Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. -!!! Tip Updating existing data +!!! Tip "Tip: Updating existing data" If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. diff --git a/docs/docker.md b/docs/docker.md index 923dec1e2..8a254b749 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -26,7 +26,7 @@ To update the image, simply run the above commands again and restart your runnin Should you require additional libraries, please [build the image yourself](#build-your-own-docker-image). -!!! Note Docker image update frequency +!!! Note "Docker image update frequency" The official docker images with tags `master`, `develop` and `latest` are automatically rebuild once a week to keep the base image uptodate. In addition to that, every merge to `develop` will trigger a rebuild for `develop` and `latest`. diff --git a/docs/installation.md b/docs/installation.md index 02870a1c1..50f24d3e8 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -97,27 +97,24 @@ sudo apt-get install build-essential git #### Raspberry Pi / Raspbian -Before installing FreqTrade on a Raspberry Pi running the official Raspbian Image, make sure you have at least Python 3.6 installed. The default image only provides Python 3.5. Probably the easiest way to get a recent version of python is [miniconda](https://repo.continuum.io/miniconda/). +The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/) from at least September 2019. +This image comes with python3.7 preinstalled, making it easy to get freqtrade up and running. -The following assumes that miniconda3 is installed and available in your environment. Since the last miniconda3 installation file uses python 3.4, we will update to python 3.6 on this installation. -It's recommended to use (mini)conda for this as installation/compilation of `numpy` and `pandas` takes a long time. - -Additional package to install on your Raspbian, `libffi-dev` required by cryptography (from python-telegram-bot). +Tested using a Raspberry Pi 3 with the Raspbian Buster lite image, all updates applied. ``` bash -conda config --add channels rpi -conda install python=3.6 -conda create -n freqtrade python=3.6 -conda activate freqtrade -conda install pandas numpy +sudo apt-get install python3-venv libatlas-base-dev +git clone https://github.com/freqtrade/freqtrade.git +cd freqtrade -sudo apt install libffi-dev -python3 -m pip install -r requirements-common.txt -python3 -m pip install -e . +bash setup.sh -i ``` +!!! Note "Installation duration" + Depending on your internet speed and the Raspberry Pi version, installation can take multiple hours to complete. + !!! Note - This does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. + The above does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerful machine. ### Common diff --git a/docs/plotting.md b/docs/plotting.md index 89404f8b1..ca56bdaba 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -79,7 +79,7 @@ The `-p/--pairs` argument can be used to specify pairs you would like to plot. Specify custom indicators. Use `--indicators1` for the main plot and `--indicators2` for the subplot below (if values are in a different range than prices). -!!! tip +!!! Tip You will almost certainly want to specify a custom strategy! This can be done by adding `-s Classname` / `--strategy ClassName` to the command. ``` bash diff --git a/docs/rest-api.md b/docs/rest-api.md index efcacc8a1..4d5bc5730 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -16,11 +16,11 @@ Sample configuration: }, ``` -!!! Danger Security warning - By default, the configuration listens on localhost only (so it's not reachable from other systems). We strongly recommend to not expose this API to the internet and choose a strong, unique password, since others will potentially be able to control your bot. +!!! Danger "Security warning" + By default, the configuration listens on localhost only (so it's not reachable from other systems). We strongly recommend to not expose this API to the internet and choose a strong, unique password, since others will potentially be able to control your bot. -!!! Danger Password selection - Please make sure to select a very strong, unique password to protect your bot from unauthorized access. +!!! Danger "Password selection" + Please make sure to select a very strong, unique password to protect your bot from unauthorized access. You can then access the API by going to `http://127.0.0.1:8080/api/v1/version` to check if the API is running correctly. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index cef362ffd..8836f4f88 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -51,13 +51,13 @@ freqtrade --strategy AwesomeStrategy **For the following section we will use the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/sample_strategy.py) file as reference.** -!!! Note Strategies and Backtesting +!!! Note "Strategies and Backtesting" To avoid problems and unexpected differences between Backtesting and dry/live modes, please be aware that during backtesting the full time-interval is passed to the `populate_*()` methods at once. It is therefore best to use vectorized operations (across the whole dataframe, not loops) and avoid index referencing (`df.iloc[-1]`), but instead use `df.shift()` to get to the previous candle. -!!! Warning Using future data +!!! Warning "Warning: Using future data" Since backtesting passes the full time interval to the `populate_*()` methods, the strategy author needs to take care to avoid having the strategy utilize data from the future. Some common patterns for this are listed in the [Common Mistakes](#common-mistakes-when-developing-strategies) section of this document. @@ -330,12 +330,12 @@ if self.dp: ticker_interval=inf_timeframe) ``` -!!! Warning Warning about backtesting +!!! Warning "Warning about backtesting" Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` for the backtesting runmode) provides the full time-range in one go, so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). -!!! Warning Warning in hyperopt +!!! Warning "Warning in hyperopt" This option cannot currently be used during hyperopt. #### Orderbook diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index e06d4fdfc..424b55faf 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -93,7 +93,7 @@ Once all positions are sold, run `/stop` to completely stop the bot. `/reload_conf` resets "max_open_trades" to the value set in the configuration and resets this command. -!!! warning +!!! Warning The stop-buy signal is ONLY active while the bot is running, and is not persisted anyway, so restarting the bot will cause this to reset. ### /status From 78fe5a46c1c8d18836740eddc77f478fc3dadad2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 13:27:36 +0100 Subject: [PATCH 11/51] Update travis to verify for correct title usage --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 405228ab8..eb171521d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,8 @@ jobs: - script: # Test Documentation boxes - # !!! : is not allowed! - - grep -Er '^!{3}\s\S+:' docs/*; test $? -ne 0 + # !!! "title" - Title needs to be quoted! + - grep -Er '^!{3}\s\S+:|^!{3}\s\S+\s[^"]' docs/*; test $? -ne 0 name: doc syntax - script: mypy freqtrade scripts name: mypy From dac88c6aedf8582ea608df7471720cc83e015727 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 13:35:55 +0100 Subject: [PATCH 12/51] extract Find parallel trades per interval --- freqtrade/data/btanalysis.py | 32 ++++++++++++++++++++++-------- tests/optimize/test_backtesting.py | 6 +++--- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0f5d395ff..9dbd69e3e 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -52,16 +52,17 @@ def load_backtest_data(filename) -> pd.DataFrame: return df -def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int) -> pd.DataFrame: +def parallel_trade_analysis(results: pd.DataFrame, timeframe: str) -> pd.DataFrame: """ Find overlapping trades by expanding each trade once per period it was open - and then counting overlaps + and then counting overlaps. :param results: Results Dataframe - can be loaded - :param freq: Frequency used for the backtest - :param max_open_trades: parameter max_open_trades used during backtest run - :return: dataframe with open-counts per time-period in freq + :param timeframe: Timeframe used for backtest + :return: dataframe with open-counts per time-period in timeframe """ - dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, freq=freq)) + from freqtrade.exchange import timeframe_to_minutes + timeframe_min = timeframe_to_minutes(timeframe) + dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, freq=f"{timeframe_min}min")) for row in results[['open_time', 'close_time']].iterrows()] deltas = [len(x) for x in dates] dates = pd.Series(pd.concat(dates).values, name='date') @@ -69,8 +70,23 @@ def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int df2 = pd.concat([dates, df2], axis=1) df2 = df2.set_index('date') - df_final = df2.resample(freq)[['pair']].count() - return df_final[df_final['pair'] > max_open_trades] + df_final = df2.resample(f"{timeframe_min}min")[['pair']].count() + df_final = df_final.rename({'pair': 'open_trades'}, axis=1) + return df_final + + +def evaluate_result_multi(results: pd.DataFrame, timeframe: str, + max_open_trades: int) -> pd.DataFrame: + """ + Find overlapping trades by expanding each trade once per period it was open + and then counting overlaps + :param results: Results Dataframe - can be loaded + :param timeframe: Frequency used for the backtest + :param max_open_trades: parameter max_open_trades used during backtest run + :return: dataframe with open-counts per time-period in freq + """ + df_final = parallel_trade_analysis(results, timeframe) + return df_final[df_final['open_trades'] > max_open_trades] def load_trades_from_db(db_url: str) -> pd.DataFrame: diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index ba87848ec..5912c5489 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -714,9 +714,9 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) results = backtesting.backtest(backtest_conf) # Make sure we have parallel trades - assert len(evaluate_result_multi(results, '5min', 2)) > 0 + assert len(evaluate_result_multi(results, '5m', 2)) > 0 # make sure we don't have trades with more than configured max_open_trades - assert len(evaluate_result_multi(results, '5min', 3)) == 0 + assert len(evaluate_result_multi(results, '5m', 3)) == 0 backtest_conf = { 'stake_amount': default_conf['stake_amount'], @@ -727,7 +727,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) 'end_date': max_date, } results = backtesting.backtest(backtest_conf) - assert len(evaluate_result_multi(results, '5min', 1)) == 0 + assert len(evaluate_result_multi(results, '5m', 1)) == 0 def test_backtest_record(default_conf, fee, mocker): From dd408aa5d63df6e02884616910da9b9fc7eb1378 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 14:07:23 +0100 Subject: [PATCH 13/51] Add analyze_trade_parallelism analysis function --- freqtrade/data/btanalysis.py | 7 ++++--- tests/data/test_btanalysis.py | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 9dbd69e3e..b9625e745 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -52,7 +52,7 @@ def load_backtest_data(filename) -> pd.DataFrame: return df -def parallel_trade_analysis(results: pd.DataFrame, timeframe: str) -> pd.DataFrame: +def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataFrame: """ Find overlapping trades by expanding each trade once per period it was open and then counting overlaps. @@ -62,7 +62,8 @@ def parallel_trade_analysis(results: pd.DataFrame, timeframe: str) -> pd.DataFra """ from freqtrade.exchange import timeframe_to_minutes timeframe_min = timeframe_to_minutes(timeframe) - dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, freq=f"{timeframe_min}min")) + dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, + freq=f"{timeframe_min}min")) for row in results[['open_time', 'close_time']].iterrows()] deltas = [len(x) for x in dates] dates = pd.Series(pd.concat(dates).values, name='date') @@ -85,7 +86,7 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str, :param max_open_trades: parameter max_open_trades used during backtest run :return: dataframe with open-counts per time-period in freq """ - df_final = parallel_trade_analysis(results, timeframe) + df_final = analyze_trade_parallelism(results, timeframe) return df_final[df_final['open_trades'] > max_open_trades] diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 78781cffd..b49344bbd 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -10,7 +10,7 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, create_cum_profit, extract_trades_of_period, load_backtest_data, load_trades, - load_trades_from_db) + load_trades_from_db, analyze_trade_parallelism) from freqtrade.data.history import load_data, load_pair_history from tests.test_persistence import create_mock_trades @@ -32,7 +32,7 @@ def test_load_backtest_data(testdatadir): @pytest.mark.usefixtures("init_persistence") -def test_load_trades_db(default_conf, fee, mocker): +def test_load_trades_from_db(default_conf, fee, mocker): create_mock_trades(fee) # remove init so it does not init again @@ -84,6 +84,17 @@ def test_extract_trades_of_period(testdatadir): assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime +def test_analyze_trade_parallelism(default_conf, mocker, testdatadir): + filename = testdatadir / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + + res = analyze_trade_parallelism(bt_data, "5m") + assert isinstance(res, DataFrame) + assert 'open_trades' in res.columns + assert res['open_trades'].max() == 3 + assert res['open_trades'].min() == 0 + + def test_load_trades(default_conf, mocker): db_mock = mocker.patch("freqtrade.data.btanalysis.load_trades_from_db", MagicMock()) bt_mock = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock()) From 6928c685a8ee8894b5494974f90ae079dcae9fc1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 14:12:41 +0100 Subject: [PATCH 14/51] Add documentation sample for parallel_trade_analysis --- docs/strategy_analysis_example.md | 16 ++++++++++ .../notebooks/strategy_analysis_example.ipynb | 29 +++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 49800bbb3..55f1bd908 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -107,6 +107,22 @@ trades = load_trades_from_db("sqlite:///tradesv3.sqlite") trades.groupby("pair")["sell_reason"].value_counts() ``` +## Analyze the loaded trades for trade parallelism +This can be useful to find the best `max_open_trades` parameter, when used with backtesting in conjunction with `--disable-max-market-positions`. + +`parallel_trade_analysis()` returns a timeseries dataframe with an "open_trades" column, specifying the number of open trades for each candle. + + +```python +from freqtrade.data.btanalysis import parallel_trade_analysis + +# Analyze the above +parallel_trades = parallel_trade_analysis(trades, '5m') + + +parallel_trades.plot() +``` + ## Plot results Freqtrade offers interactive plotting capabilities based on plotly. diff --git a/user_data/notebooks/strategy_analysis_example.ipynb b/user_data/notebooks/strategy_analysis_example.ipynb index b9576e0bb..edb05a7ca 100644 --- a/user_data/notebooks/strategy_analysis_example.ipynb +++ b/user_data/notebooks/strategy_analysis_example.ipynb @@ -68,9 +68,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, + "metadata": {}, "outputs": [], "source": [ "# Load strategy using values set above\n", @@ -169,6 +167,31 @@ "trades.groupby(\"pair\")[\"sell_reason\"].value_counts()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analyze the loaded trades for trade parallelism\n", + "This can be useful to find the best `max_open_trades` parameter, when used with backtesting in conjunction with `--disable-max-market-positions`.\n", + "\n", + "`parallel_trade_analysis()` returns a timeseries dataframe with an \"open_trades\" column, specifying the number of open trades for each candle." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from freqtrade.data.btanalysis import parallel_trade_analysis\n", + "\n", + "# Analyze the above\n", + "parallel_trades = parallel_trade_analysis(trades, '5m')\n", + "\n", + "\n", + "parallel_trades.plot()" + ] + }, { "cell_type": "markdown", "metadata": {}, From bba8e614094605c254bed875f0a846026305a437 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 19:30:35 +0100 Subject: [PATCH 15/51] Rename function in samples --- docs/strategy_analysis_example.md | 6 +++--- user_data/notebooks/strategy_analysis_example.ipynb | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 55f1bd908..aa4578ca7 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -110,14 +110,14 @@ trades.groupby("pair")["sell_reason"].value_counts() ## Analyze the loaded trades for trade parallelism This can be useful to find the best `max_open_trades` parameter, when used with backtesting in conjunction with `--disable-max-market-positions`. -`parallel_trade_analysis()` returns a timeseries dataframe with an "open_trades" column, specifying the number of open trades for each candle. +`analyze_trade_parallelism()` returns a timeseries dataframe with an "open_trades" column, specifying the number of open trades for each candle. ```python -from freqtrade.data.btanalysis import parallel_trade_analysis +from freqtrade.data.btanalysis import analyze_trade_parallelism # Analyze the above -parallel_trades = parallel_trade_analysis(trades, '5m') +parallel_trades = analyze_trade_parallelism(trades, '5m') parallel_trades.plot() diff --git a/user_data/notebooks/strategy_analysis_example.ipynb b/user_data/notebooks/strategy_analysis_example.ipynb index edb05a7ca..03dc83b4e 100644 --- a/user_data/notebooks/strategy_analysis_example.ipynb +++ b/user_data/notebooks/strategy_analysis_example.ipynb @@ -174,7 +174,7 @@ "## Analyze the loaded trades for trade parallelism\n", "This can be useful to find the best `max_open_trades` parameter, when used with backtesting in conjunction with `--disable-max-market-positions`.\n", "\n", - "`parallel_trade_analysis()` returns a timeseries dataframe with an \"open_trades\" column, specifying the number of open trades for each candle." + "`analyze_trade_parallelism()` returns a timeseries dataframe with an \"open_trades\" column, specifying the number of open trades for each candle." ] }, { @@ -183,10 +183,10 @@ "metadata": {}, "outputs": [], "source": [ - "from freqtrade.data.btanalysis import parallel_trade_analysis\n", + "from freqtrade.data.btanalysis import analyze_trade_parallelism\n", "\n", "# Analyze the above\n", - "parallel_trades = parallel_trade_analysis(trades, '5m')\n", + "parallel_trades = analyze_trade_parallelism(trades, '5m')\n", "\n", "\n", "parallel_trades.plot()" From 9e988783de6218a441f3a1bbd52dea59a9b61d6e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Oct 2019 20:06:49 +0100 Subject: [PATCH 16/51] Allow configuration of stoploss on exchange limit fixes #1717 --- docs/configuration.md | 8 +++++++- freqtrade/freqtradebot.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1ad13c87a..217e9f37b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -215,6 +215,11 @@ If this is configured, the following 4 values (`buy`, `sell`, `stoploss` and `emergencysell` is an optional value, which defaults to `market` and is used when creating stoploss on exchange orders fails. The below is the default which is used if this is not configured in either strategy or configuration file. +Since `stoploss_on_exchange` uses limit orders, the exchange needs 2 prices, the stoploss_price and the Limit price. +`stoploss` defines the stop-price - and limit should be slightly below this. This defaults to 0.99 / 1%. +Calculation example: we bought the asset at 100$. +Stop-price is 95$, then limit would be `95 * 0.99 = 94.05$` - so the stoploss will happen between 95$ and 94.05$. + Syntax for Strategy: ```python @@ -224,7 +229,8 @@ order_types = { "emergencysell": "market", "stoploss": "market", "stoploss_on_exchange": False, - "stoploss_on_exchange_interval": 60 + "stoploss_on_exchange_interval": 60, + "stoploss_on_exchange_limit_ratio": 0.99, } ``` diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a8fc6bc7e..4a494696d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -634,8 +634,8 @@ class FreqtradeBot: Force-sells the pair (using EmergencySell reason) in case of Problems creating the order. :return: True if the order succeeded, and False in case of problems. """ - # Limit price threshold: As limit price should always be below price - LIMIT_PRICE_PCT = 0.99 + # Limit price threshold: As limit price should always be below stop-price + LIMIT_PRICE_PCT = self.strategy.order_types.get('stoploss_on_exchange_limit_ratio', 0.99) try: stoploss_order = self.exchange.stoploss_limit(pair=trade.pair, amount=trade.amount, From 365a408df53b2a646bfe1f622544dac63c376119 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Oct 2019 06:43:42 +0100 Subject: [PATCH 17/51] Update release-documentation to fit new release style --- docs/developer.md | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 391493b09..67c0912a7 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -204,14 +204,15 @@ This part of the documentation is aimed at maintainers, and shows how to create ### Create release branch -``` bash -# make sure you're in develop branch -git checkout develop +First, pick a commit that's about one week old (to not include latest additions to releases). +``` bash # create new branch -git checkout -b new_release +git checkout -b new_release ``` +Determine if crucial bugfixes have been made between this commit and the current state, and eventually cherry-pick these. + * Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7-1` should we need to do a second release that month. * Commit this part * push that branch to the remote and create a PR against the master branch @@ -219,23 +220,18 @@ git checkout -b new_release ### Create changelog from git commits !!! Note - Make sure that both master and develop are up-todate!. + Make sure that the master branch is uptodate! ``` bash # Needs to be done before merging / pulling that branch. -git log --oneline --no-decorate --no-merges master..develop +git log --oneline --no-decorate --no-merges master..new_release ``` ### Create github release / tag Once the PR against master is merged (best right after merging): -* Use the button "Draft a new release" in the Github UI (subsection releases) +* Use the button "Draft a new release" in the Github UI (subsection releases). * Use the version-number specified as tag. * Use "master" as reference (this step comes after the above PR is merged). -* Use the above changelog as release comment (as codeblock) - -### After-release - -* Update version in develop by postfixing that with `-dev` (`2019.6 -> 2019.6-dev`). -* Create a PR against develop to update that branch. +* Use the above changelog as release comment (as codeblock). From 5dcf28cafb0e29d06a4830a5c31cecdec377f110 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Oct 2019 06:51:36 +0100 Subject: [PATCH 18/51] Reduce frequency of "startup-period" message --- freqtrade/data/history.py | 3 ++- tests/data/test_history.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 412b086c0..b1e4313ca 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -148,7 +148,6 @@ def load_pair_history(pair: str, timerange_startup = deepcopy(timerange) if startup_candles > 0 and timerange_startup: - logger.info('Using indicator startup period: %s ...', startup_candles) timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles) # The user forced the refresh of pairs @@ -204,6 +203,8 @@ def load_data(datadir: Path, exchange and refresh_pairs are then not needed here nor in load_pair_history. """ result: Dict[str, DataFrame] = {} + if startup_candles > 0 and timerange: + logger.info(f'Using indicator startup period: {startup_candles} ...') for pair in pairs: hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, diff --git a/tests/data/test_history.py b/tests/data/test_history.py index d9627a0e4..89120b4f5 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -103,9 +103,7 @@ def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> datadir=testdatadir, timerange=timerange, startup_candles=20, ) - assert log_has( - 'Using indicator startup period: 20 ...', caplog - ) + assert ltfmock.call_count == 1 assert ltfmock.call_args_list[0][1]['timerange'] != timerange # startts is 20 minutes earlier @@ -354,8 +352,12 @@ def test_load_partial_missing(testdatadir, caplog) -> None: start = arrow.get('2018-01-01T00:00:00') end = arrow.get('2018-01-11T00:00:00') tickerdata = history.load_data(testdatadir, '5m', ['UNITTEST/BTC'], + startup_candles=20, timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) + assert log_has( + 'Using indicator startup period: 20 ...', caplog + ) # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(tickerdata['UNITTEST/BTC']) From dc5f1b28785c366de20ffe07cb1296067e1f5405 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Oct 2019 07:08:02 +0100 Subject: [PATCH 19/51] Extract integration tests into sepearte file --- tests/test_freqtradebot.py | 108 ---------------------------------- tests/test_integration.py | 117 +++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 108 deletions(-) create mode 100644 tests/test_integration.py diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 30f9ba0a4..8538c7f41 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2526,114 +2526,6 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, assert rpc_mock.call_count == 2 -def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, - ticker, fee, - limit_buy_order, - markets, mocker) -> None: - """ - Tests workflow of selling stoploss_on_exchange. - Sells - * first trade as stoploss - * 2nd trade is kept - * 3rd trade is sold via sell-signal - """ - default_conf['max_open_trades'] = 3 - default_conf['exchange']['name'] = 'binance' - patch_RPCManager(mocker) - patch_exchange(mocker) - - stoploss_limit = { - 'id': 123, - 'info': {} - } - stoploss_order_open = { - "id": "123", - "timestamp": 1542707426845, - "datetime": "2018-11-20T09:50:26.845Z", - "lastTradeTimestamp": None, - "symbol": "BTC/USDT", - "type": "stop_loss_limit", - "side": "sell", - "price": 1.08801, - "amount": 90.99181074, - "cost": 0.0, - "average": 0.0, - "filled": 0.0, - "remaining": 0.0, - "status": "open", - "fee": None, - "trades": None - } - stoploss_order_closed = stoploss_order_open.copy() - stoploss_order_closed['status'] = 'closed' - # Sell first trade based on stoploss, keep 2nd and 3rd trade open - stoploss_order_mock = MagicMock( - side_effect=[stoploss_order_closed, stoploss_order_open, stoploss_order_open]) - # Sell 3rd trade (not called for the first trade) - should_sell_mock = MagicMock(side_effect=[ - SellCheckTuple(sell_flag=False, sell_type=SellType.NONE), - SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL)] - ) - cancel_order_mock = MagicMock() - mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_ticker=ticker, - get_fee=fee, - markets=PropertyMock(return_value=markets), - symbol_amount_prec=lambda s, x, y: y, - symbol_price_prec=lambda s, x, y: y, - get_order=stoploss_order_mock, - cancel_order=cancel_order_mock, - ) - - wallets_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.freqtradebot.FreqtradeBot', - create_stoploss_order=MagicMock(return_value=True), - update_trade_state=MagicMock(), - _notify_sell=MagicMock(), - ) - mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) - mocker.patch("freqtrade.wallets.Wallets.update", wallets_mock) - - freqtrade = FreqtradeBot(default_conf) - freqtrade.strategy.order_types['stoploss_on_exchange'] = True - # Switch ordertype to market to close trade immediately - freqtrade.strategy.order_types['sell'] = 'market' - patch_get_signal(freqtrade) - - # Create some test data - freqtrade.create_trades() - wallets_mock.reset_mock() - Trade.session = MagicMock() - - trades = Trade.query.all() - # Make sure stoploss-order is open and trade is bought (since we mock update_trade_state) - for trade in trades: - trade.stoploss_order_id = 3 - trade.open_order_id = None - - freqtrade.process_maybe_execute_sells(trades) - assert should_sell_mock.call_count == 2 - - # Only order for 3rd trade needs to be cancelled - assert cancel_order_mock.call_count == 1 - # Wallets should only be called once per sell cycle - assert wallets_mock.call_count == 1 - - trade = trades[0] - assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value - assert not trade.is_open - - trade = trades[1] - assert not trade.sell_reason - assert trade.is_open - - trade = trades[2] - assert trade.sell_reason == SellType.SELL_SIGNAL.value - assert not trade.is_open - def test_execute_sell_market_order(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 000000000..857e0a2e3 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,117 @@ + +from unittest.mock import MagicMock, PropertyMock + + +from freqtrade.freqtradebot import FreqtradeBot +from freqtrade.persistence import Trade +from freqtrade.strategy.interface import SellCheckTuple, SellType +from tests.conftest import (patch_exchange, + patch_get_signal) + + +def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, + ticker, fee, + limit_buy_order, + markets, mocker) -> None: + """ + Tests workflow of selling stoploss_on_exchange. + Sells + * first trade as stoploss + * 2nd trade is kept + * 3rd trade is sold via sell-signal + """ + default_conf['max_open_trades'] = 3 + default_conf['exchange']['name'] = 'binance' + patch_exchange(mocker) + + stoploss_limit = { + 'id': 123, + 'info': {} + } + stoploss_order_open = { + "id": "123", + "timestamp": 1542707426845, + "datetime": "2018-11-20T09:50:26.845Z", + "lastTradeTimestamp": None, + "symbol": "BTC/USDT", + "type": "stop_loss_limit", + "side": "sell", + "price": 1.08801, + "amount": 90.99181074, + "cost": 0.0, + "average": 0.0, + "filled": 0.0, + "remaining": 0.0, + "status": "open", + "fee": None, + "trades": None + } + stoploss_order_closed = stoploss_order_open.copy() + stoploss_order_closed['status'] = 'closed' + # Sell first trade based on stoploss, keep 2nd and 3rd trade open + stoploss_order_mock = MagicMock( + side_effect=[stoploss_order_closed, stoploss_order_open, stoploss_order_open]) + # Sell 3rd trade (not called for the first trade) + should_sell_mock = MagicMock(side_effect=[ + SellCheckTuple(sell_flag=False, sell_type=SellType.NONE), + SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL)] + ) + cancel_order_mock = MagicMock() + mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + get_fee=fee, + markets=PropertyMock(return_value=markets), + symbol_amount_prec=lambda s, x, y: y, + symbol_price_prec=lambda s, x, y: y, + get_order=stoploss_order_mock, + cancel_order=cancel_order_mock, + ) + + wallets_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.freqtradebot.FreqtradeBot', + create_stoploss_order=MagicMock(return_value=True), + update_trade_state=MagicMock(), + _notify_sell=MagicMock(), + ) + mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) + mocker.patch("freqtrade.wallets.Wallets.update", wallets_mock) + + freqtrade = FreqtradeBot(default_conf) + freqtrade.strategy.order_types['stoploss_on_exchange'] = True + # Switch ordertype to market to close trade immediately + freqtrade.strategy.order_types['sell'] = 'market' + patch_get_signal(freqtrade) + + # Create some test data + freqtrade.create_trades() + wallets_mock.reset_mock() + Trade.session = MagicMock() + + trades = Trade.query.all() + # Make sure stoploss-order is open and trade is bought (since we mock update_trade_state) + for trade in trades: + trade.stoploss_order_id = 3 + trade.open_order_id = None + + freqtrade.process_maybe_execute_sells(trades) + assert should_sell_mock.call_count == 2 + + # Only order for 3rd trade needs to be cancelled + assert cancel_order_mock.call_count == 1 + # Wallets should only be called once per sell cycle + assert wallets_mock.call_count == 1 + + trade = trades[0] + assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value + assert not trade.is_open + + trade = trades[1] + assert not trade.sell_reason + assert trade.is_open + + trade = trades[2] + assert trade.sell_reason == SellType.SELL_SIGNAL.value + assert not trade.is_open From ce6b869f84d8d0dc0ef5d0b5c28b4bec792c36d1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Oct 2019 07:11:57 +0100 Subject: [PATCH 20/51] Cleanup test --- tests/test_integration.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 857e0a2e3..c3a3ada07 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,10 +2,9 @@ from unittest.mock import MagicMock, PropertyMock -from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellCheckTuple, SellType -from tests.conftest import (patch_exchange, +from tests.conftest import (patch_exchange, get_patched_freqtradebot, patch_get_signal) @@ -22,7 +21,6 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, """ default_conf['max_open_trades'] = 3 default_conf['exchange']['name'] = 'binance' - patch_exchange(mocker) stoploss_limit = { 'id': 123, @@ -62,14 +60,12 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), symbol_amount_prec=lambda s, x, y: y, symbol_price_prec=lambda s, x, y: y, get_order=stoploss_order_mock, cancel_order=cancel_order_mock, ) - wallets_mock = MagicMock() mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), @@ -77,9 +73,9 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, _notify_sell=MagicMock(), ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) - mocker.patch("freqtrade.wallets.Wallets.update", wallets_mock) + wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update", MagicMock()) - freqtrade = FreqtradeBot(default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True # Switch ordertype to market to close trade immediately freqtrade.strategy.order_types['sell'] = 'market' From 734a9d5d87717bf4ed1be8c6d8b718aeaa2b81da Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Oct 2019 07:15:45 +0100 Subject: [PATCH 21/51] Seperate tests related to worker from test_freqtradebot --- tests/test_freqtradebot.py | 75 ----------------------------------- tests/test_integration.py | 6 +-- tests/test_worker.py | 81 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 79 deletions(-) create mode 100644 tests/test_worker.py diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 8538c7f41..1f761b55f 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -14,7 +14,6 @@ import requests from freqtrade import (DependencyException, InvalidOrderException, OperationalException, TemporaryError, constants) from freqtrade.constants import MATH_CLOSE_PREC -from freqtrade.data.dataprovider import DataProvider from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType @@ -49,16 +48,6 @@ def test_freqtradebot_state(mocker, default_conf, markets) -> None: assert freqtrade.state is State.STOPPED -def test_worker_state(mocker, default_conf, markets) -> None: - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - worker = get_patched_worker(mocker, default_conf) - assert worker.state is State.RUNNING - - default_conf.pop('initial_state') - worker = Worker(args=None, config=default_conf) - assert worker.state is State.STOPPED - - def test_cleanup(mocker, default_conf, caplog) -> None: mock_cleanup = MagicMock() mocker.patch('freqtrade.persistence.cleanup', mock_cleanup) @@ -68,69 +57,6 @@ def test_cleanup(mocker, default_conf, caplog) -> None: assert mock_cleanup.call_count == 1 -def test_worker_running(mocker, default_conf, caplog) -> None: - mock_throttle = MagicMock() - mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) - mocker.patch('freqtrade.persistence.Trade.stoploss_reinitialization', MagicMock()) - - worker = get_patched_worker(mocker, default_conf) - - state = worker._worker(old_state=None) - assert state is State.RUNNING - assert log_has('Changing state to: RUNNING', caplog) - assert mock_throttle.call_count == 1 - # Check strategy is loaded, and received a dataprovider object - assert worker.freqtrade.strategy - assert worker.freqtrade.strategy.dp - assert isinstance(worker.freqtrade.strategy.dp, DataProvider) - - -def test_worker_stopped(mocker, default_conf, caplog) -> None: - mock_throttle = MagicMock() - mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) - mock_sleep = mocker.patch('time.sleep', return_value=None) - - worker = get_patched_worker(mocker, default_conf) - worker.state = State.STOPPED - state = worker._worker(old_state=State.RUNNING) - assert state is State.STOPPED - assert log_has('Changing state to: STOPPED', caplog) - assert mock_throttle.call_count == 0 - assert mock_sleep.call_count == 1 - - -def test_throttle(mocker, default_conf, caplog) -> None: - def throttled_func(): - return 42 - - caplog.set_level(logging.DEBUG) - worker = get_patched_worker(mocker, default_conf) - - start = time.time() - result = worker._throttle(throttled_func, min_secs=0.1) - end = time.time() - - assert result == 42 - assert end - start > 0.1 - assert log_has('Throttling throttled_func for 0.10 seconds', caplog) - - result = worker._throttle(throttled_func, min_secs=-1) - assert result == 42 - - -def test_throttle_with_assets(mocker, default_conf) -> None: - def throttled_func(nb_assets=-1): - return nb_assets - - worker = get_patched_worker(mocker, default_conf) - - result = worker._throttle(throttled_func, min_secs=0.1, nb_assets=666) - assert result == 666 - - result = worker._throttle(throttled_func, min_secs=0.1) - assert result == -1 - - def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2526,7 +2452,6 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, assert rpc_mock.call_count == 2 - def test_execute_sell_market_order(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) diff --git a/tests/test_integration.py b/tests/test_integration.py index c3a3ada07..37ee24d6c 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,11 +1,9 @@ -from unittest.mock import MagicMock, PropertyMock - +from unittest.mock import MagicMock from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellCheckTuple, SellType -from tests.conftest import (patch_exchange, get_patched_freqtradebot, - patch_get_signal) +from tests.conftest import get_patched_freqtradebot, patch_get_signal def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, diff --git a/tests/test_worker.py b/tests/test_worker.py new file mode 100644 index 000000000..72e215210 --- /dev/null +++ b/tests/test_worker.py @@ -0,0 +1,81 @@ +import logging +import time +from unittest.mock import MagicMock, PropertyMock + +from freqtrade.data.dataprovider import DataProvider +from freqtrade.state import State +from freqtrade.worker import Worker +from tests.conftest import get_patched_worker, log_has + + +def test_worker_state(mocker, default_conf, markets) -> None: + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) + worker = get_patched_worker(mocker, default_conf) + assert worker.state is State.RUNNING + + default_conf.pop('initial_state') + worker = Worker(args=None, config=default_conf) + assert worker.state is State.STOPPED + + +def test_worker_running(mocker, default_conf, caplog) -> None: + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + mocker.patch('freqtrade.persistence.Trade.stoploss_reinitialization', MagicMock()) + + worker = get_patched_worker(mocker, default_conf) + + state = worker._worker(old_state=None) + assert state is State.RUNNING + assert log_has('Changing state to: RUNNING', caplog) + assert mock_throttle.call_count == 1 + # Check strategy is loaded, and received a dataprovider object + assert worker.freqtrade.strategy + assert worker.freqtrade.strategy.dp + assert isinstance(worker.freqtrade.strategy.dp, DataProvider) + + +def test_worker_stopped(mocker, default_conf, caplog) -> None: + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + mock_sleep = mocker.patch('time.sleep', return_value=None) + + worker = get_patched_worker(mocker, default_conf) + worker.state = State.STOPPED + state = worker._worker(old_state=State.RUNNING) + assert state is State.STOPPED + assert log_has('Changing state to: STOPPED', caplog) + assert mock_throttle.call_count == 0 + assert mock_sleep.call_count == 1 + + +def test_throttle(mocker, default_conf, caplog) -> None: + def throttled_func(): + return 42 + + caplog.set_level(logging.DEBUG) + worker = get_patched_worker(mocker, default_conf) + + start = time.time() + result = worker._throttle(throttled_func, min_secs=0.1) + end = time.time() + + assert result == 42 + assert end - start > 0.1 + assert log_has('Throttling throttled_func for 0.10 seconds', caplog) + + result = worker._throttle(throttled_func, min_secs=-1) + assert result == 42 + + +def test_throttle_with_assets(mocker, default_conf) -> None: + def throttled_func(nb_assets=-1): + return nb_assets + + worker = get_patched_worker(mocker, default_conf) + + result = worker._throttle(throttled_func, min_secs=0.1, nb_assets=666) + assert result == 666 + + result = worker._throttle(throttled_func, min_secs=0.1) + assert result == -1 From 7be378aaa919befbdbc6fa27dd942638f4a306bb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Oct 2019 07:26:48 +0100 Subject: [PATCH 22/51] Remove markets mock where it's not needed --- tests/test_freqtradebot.py | 157 ++++++++++++------------------------- tests/test_integration.py | 6 +- 2 files changed, 51 insertions(+), 112 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 1f761b55f..0ed8e8a77 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -150,18 +150,13 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None: freqtrade._get_trade_stake_amount('ETH/BTC') -def test_get_trade_stake_amount_unlimited_amount(default_conf, - ticker, - limit_buy_order, - fee, - markets, - mocker) -> None: +def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, + limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_wallet(mocker, free=default_conf['stake_amount']) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee @@ -222,7 +217,7 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: assert freqtrade._get_trade_stake_amount('LTC/BTC') == (999.9 * 0.5 * 0.01) / 0.21 -def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, edge_conf) -> None: +def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -243,7 +238,6 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) ############################################# @@ -263,7 +257,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, assert trade.sell_reason == SellType.STOP_LOSS.value -def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, +def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -284,7 +278,6 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), ) ############################################# @@ -303,7 +296,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, def test_total_open_trades_stakes(mocker, default_conf, ticker, - limit_buy_order, fee, markets) -> None: + limit_buy_order, fee) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf['stake_amount'] = 0.0000098751 @@ -313,7 +306,6 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -448,7 +440,7 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: assert result == min(8, 2 * 2) / 0.9 -def test_create_trades(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: +def test_create_trades(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -456,7 +448,6 @@ def test_create_trades(default_conf, ticker, limit_buy_order, fee, markets, mock get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) # Save state of current whitelist @@ -482,7 +473,7 @@ def test_create_trades(default_conf, ticker, limit_buy_order, fee, markets, mock def test_create_trades_no_stake_amount(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5) @@ -491,7 +482,6 @@ def test_create_trades_no_stake_amount(default_conf, ticker, limit_buy_order, get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -501,7 +491,7 @@ def test_create_trades_no_stake_amount(default_conf, ticker, limit_buy_order, def test_create_trades_minimal_amount(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -510,7 +500,6 @@ def test_create_trades_minimal_amount(default_conf, ticker, limit_buy_order, get_ticker=ticker, buy=buy_mock, get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['stake_amount'] = 0.0005 freqtrade = FreqtradeBot(default_conf) @@ -522,7 +511,7 @@ def test_create_trades_minimal_amount(default_conf, ticker, limit_buy_order, def test_create_trades_too_small_stake_amount(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -531,7 +520,6 @@ def test_create_trades_too_small_stake_amount(default_conf, ticker, limit_buy_or get_ticker=ticker, buy=buy_mock, get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['stake_amount'] = 0.000000005 @@ -551,7 +539,6 @@ def test_create_trades_limit_reached(default_conf, ticker, limit_buy_order, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_balance=MagicMock(return_value=default_conf['stake_amount']), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['max_open_trades'] = 0 default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT @@ -564,7 +551,7 @@ def test_create_trades_limit_reached(default_conf, ticker, limit_buy_order, def test_create_trades_no_pairs_let(default_conf, ticker, limit_buy_order, fee, - markets, mocker, caplog) -> None: + mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -572,7 +559,6 @@ def test_create_trades_no_pairs_let(default_conf, ticker, limit_buy_order, fee, get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['exchange']['pair_whitelist'] = ["ETH/BTC"] @@ -586,7 +572,7 @@ def test_create_trades_no_pairs_let(default_conf, ticker, limit_buy_order, fee, def test_create_trades_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, - markets, mocker, caplog) -> None: + mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -594,7 +580,6 @@ def test_create_trades_no_pairs_in_whitelist(default_conf, ticker, limit_buy_ord get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['exchange']['pair_whitelist'] = [] freqtrade = FreqtradeBot(default_conf) @@ -625,7 +610,7 @@ def test_create_trades_no_signal(default_conf, fee, mocker) -> None: @pytest.mark.parametrize("max_open", range(0, 5)) def test_create_trades_multiple_trades(default_conf, ticker, - fee, markets, mocker, max_open) -> None: + fee, mocker, max_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf['max_open_trades'] = max_open @@ -634,7 +619,6 @@ def test_create_trades_multiple_trades(default_conf, ticker, get_ticker=ticker, buy=MagicMock(return_value={'id': "12355555"}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -645,7 +629,7 @@ def test_create_trades_multiple_trades(default_conf, ticker, assert len(trades) == max_open -def test_create_trades_preopen(default_conf, ticker, fee, markets, mocker) -> None: +def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf['max_open_trades'] = 4 @@ -654,7 +638,6 @@ def test_create_trades_preopen(default_conf, ticker, fee, markets, mocker) -> No get_ticker=ticker, buy=MagicMock(return_value={'id': "12355555"}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -673,13 +656,12 @@ def test_create_trades_preopen(default_conf, ticker, fee, markets, mocker) -> No def test_process_trade_creation(default_conf, ticker, limit_buy_order, - markets, fee, mocker, caplog) -> None: + fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, - markets=PropertyMock(return_value=markets), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, @@ -708,13 +690,12 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, ) -def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> None: +def test_process_exchange_failures(default_conf, ticker, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, - markets=PropertyMock(return_value=markets), buy=MagicMock(side_effect=TemporaryError) ) sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) @@ -726,13 +707,12 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non assert sleep_mock.has_calls() -def test_process_operational_exception(default_conf, ticker, markets, mocker) -> None: +def test_process_operational_exception(default_conf, ticker, mocker) -> None: msg_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, - markets=PropertyMock(return_value=markets), buy=MagicMock(side_effect=OperationalException) ) worker = Worker(args=None, config=default_conf) @@ -745,14 +725,12 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] -def test_process_trade_handling( - default_conf, ticker, limit_buy_order, markets, fee, mocker) -> None: +def test_process_trade_handling(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, - markets=PropertyMock(return_value=markets), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, @@ -772,15 +750,14 @@ def test_process_trade_handling( assert len(trades) == 1 -def test_process_trade_no_whitelist_pair( - default_conf, ticker, limit_buy_order, markets, fee, mocker) -> None: +def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order, + fee, mocker) -> None: """ Test process with trade not in pair list """ patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, - markets=PropertyMock(return_value=markets), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, @@ -817,7 +794,7 @@ def test_process_trade_no_whitelist_pair( assert len(freqtrade.active_pair_whitelist) == len(set(freqtrade.active_pair_whitelist)) -def test_process_informative_pairs_added(default_conf, ticker, markets, mocker) -> None: +def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -828,7 +805,6 @@ def test_process_informative_pairs_added(default_conf, ticker, markets, mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, - markets=PropertyMock(return_value=markets), buy=MagicMock(side_effect=TemporaryError), refresh_latest_ohlcv=refresh_mock, ) @@ -874,7 +850,7 @@ def test_balance_bigger_last_ask(mocker, default_conf) -> None: assert freqtrade.get_target_bid('ETH/BTC') == 5 -def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> None: +def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: patch_RPCManager(mocker) patch_exchange(mocker) freqtrade = FreqtradeBot(default_conf) @@ -896,7 +872,6 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non }), buy=buy_mm, get_fee=fee, - markets=PropertyMock(return_value=markets) ) pair = 'ETH/BTC' @@ -993,7 +968,7 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, - markets, limit_buy_order, limit_sell_order) -> None: + limit_buy_order, limit_sell_order) -> None: stoploss_limit = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) patch_exchange(mocker) @@ -1007,7 +982,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), stoploss_limit=stoploss_limit ) freqtrade = FreqtradeBot(default_conf) @@ -1094,7 +1068,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, - markets, limit_buy_order, limit_sell_order) -> None: + limit_buy_order, limit_sell_order) -> None: # Sixth case: stoploss order was cancelled but couldn't create new one patch_RPCManager(mocker) patch_exchange(mocker) @@ -1108,7 +1082,6 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), get_order=MagicMock(return_value={'status': 'canceled'}), stoploss_limit=MagicMock(side_effect=DependencyException()), ) @@ -1129,7 +1102,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, - markets, limit_buy_order, limit_sell_order): + limit_buy_order, limit_sell_order): rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) sell_mock = MagicMock(return_value={'id': limit_sell_order['id']}) @@ -1143,7 +1116,6 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=sell_mock, get_fee=fee, - markets=PropertyMock(return_value=markets), get_order=MagicMock(return_value={'status': 'canceled'}), stoploss_limit=MagicMock(side_effect=InvalidOrderException()), ) @@ -1266,8 +1238,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, caplog, - markets, limit_buy_order, - limit_sell_order) -> None: + limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set stoploss_limit = MagicMock(return_value={'id': 13434334}) patch_exchange(mocker) @@ -1282,7 +1253,6 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), stoploss_limit=stoploss_limit ) @@ -1335,7 +1305,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, - markets, limit_buy_order, limit_sell_order) -> None: + limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set stoploss_limit = MagicMock(return_value={'id': 13434334}) @@ -1353,7 +1323,6 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), stoploss_limit=stoploss_limit ) @@ -1654,8 +1623,7 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde assert not trade.is_open -def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, - fee, markets, mocker) -> None: +def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -1668,7 +1636,6 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -1695,8 +1662,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, assert trade.close_date is not None -def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: +def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -1704,7 +1670,6 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) @@ -1810,7 +1775,7 @@ def test_handle_trade_use_sell_signal( def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, - fee, markets, mocker) -> None: + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -1818,7 +1783,6 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2148,14 +2112,13 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None: assert cancel_order_mock.call_count == 1 -def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: +def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2195,14 +2158,13 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc } == last_msg -def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: +def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2244,15 +2206,13 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fee, - ticker_sell_down, - markets, mocker) -> None: + ticker_sell_down, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2300,8 +2260,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe } == last_msg -def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, - markets, caplog) -> None: +def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException()) sellmock = MagicMock() @@ -2310,7 +2269,6 @@ def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), sell=sellmock ) @@ -2330,9 +2288,8 @@ def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, assert log_has('Could not cancel stoploss order abcd', caplog) -def test_execute_sell_with_stoploss_on_exchange(default_conf, - ticker, fee, ticker_sell_up, - markets, mocker) -> None: +def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticker_sell_up, + mocker) -> None: default_conf['exchange']['name'] = 'binance' rpc_mock = patch_RPCManager(mocker) @@ -2349,7 +2306,6 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), symbol_amount_prec=lambda s, x, y: y, symbol_price_prec=lambda s, x, y: y, stoploss_limit=stoploss_limit, @@ -2384,10 +2340,8 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, assert rpc_mock.call_count == 2 -def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, - ticker, fee, - limit_buy_order, - markets, mocker) -> None: +def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, fee, + limit_buy_order, mocker) -> None: default_conf['exchange']['name'] = 'binance' rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) @@ -2395,7 +2349,6 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), symbol_amount_prec=lambda s, x, y: y, symbol_price_prec=lambda s, x, y: y, ) @@ -2453,14 +2406,13 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, def test_execute_sell_market_order(default_conf, ticker, fee, - ticker_sell_up, markets, mocker) -> None: + ticker_sell_up, mocker) -> None: rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2506,7 +2458,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, - fee, markets, mocker) -> None: + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2518,7 +2470,6 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['ask_strategy'] = { 'use_sell_signal': True, @@ -2538,7 +2489,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, - fee, markets, mocker) -> None: + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2550,7 +2501,6 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['ask_strategy'] = { 'use_sell_signal': True, @@ -2568,7 +2518,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: +def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2580,7 +2530,6 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['ask_strategy'] = { 'use_sell_signal': True, @@ -2598,7 +2547,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market assert freqtrade.handle_trade(trade) is False -def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, markets, mocker) -> None: +def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2610,7 +2559,6 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['ask_strategy'] = { 'use_sell_signal': True, @@ -2932,7 +2880,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, - fee, markets, mocker) -> None: + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2944,7 +2892,6 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['ask_strategy'] = { 'ignore_roi_if_buy_signal': False @@ -3239,7 +3186,7 @@ def test_get_real_amount_open_trade(default_conf, mocker): assert freqtrade.get_real_amount(trade, order) == amount -def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, markets, mocker, +def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker, order_book_l2): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 @@ -3251,7 +3198,6 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) # Save state of current whitelist @@ -3275,7 +3221,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order, - fee, markets, mocker, order_book_l2): + fee, mocker, order_book_l2): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True # delta is 100 which is impossible to reach. hence check_depth_of_market will return false default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 @@ -3287,7 +3233,6 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) # Save state of current whitelist freqtrade = FreqtradeBot(default_conf) @@ -3298,7 +3243,7 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o assert trade is None -def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) -> None: +def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: """ test if function get_target_bid will return the order book price instead of the ask rate @@ -3307,7 +3252,6 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) ticker_mock = MagicMock(return_value={'ask': 0.045, 'last': 0.046}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), get_order_book=order_book_l2, get_ticker=ticker_mock, @@ -3323,7 +3267,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) assert ticker_mock.call_count == 0 -def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) -> None: +def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: """ test if function get_target_bid will return the ask rate (since its value is lower) instead of the order book rate (even if enabled) @@ -3332,7 +3276,6 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) ticker_mock = MagicMock(return_value={'ask': 0.042, 'last': 0.046}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), get_order_book=order_book_l2, get_ticker=ticker_mock, @@ -3349,14 +3292,13 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) assert ticker_mock.call_count == 0 -def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) -> None: +def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: """ test check depth of market """ patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), get_order_book=order_book_l2 ) default_conf['telegram']['enabled'] = False @@ -3371,7 +3313,7 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order, - fee, markets, mocker, order_book_l2) -> None: + fee, mocker, order_book_l2) -> None: """ test order book ask strategy """ @@ -3393,7 +3335,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) diff --git a/tests/test_integration.py b/tests/test_integration.py index 37ee24d6c..c83b2de6d 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -6,10 +6,8 @@ from freqtrade.strategy.interface import SellCheckTuple, SellType from tests.conftest import get_patched_freqtradebot, patch_get_signal -def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, - ticker, fee, - limit_buy_order, - markets, mocker) -> None: +def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, + limit_buy_order, mocker) -> None: """ Tests workflow of selling stoploss_on_exchange. Sells From b6616d7a13e727adf6a43f61d321fdaff88e4cfa Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Oct 2019 10:04:28 +0100 Subject: [PATCH 23/51] Add test helping debug #1985 --- tests/test_integration.py | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index c83b2de6d..228ed8468 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellCheckTuple, SellType from tests.conftest import get_patched_freqtradebot, patch_get_signal +from freqtrade.rpc.rpc import RPC def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, @@ -107,3 +108,52 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, trade = trades[2] assert trade.sell_reason == SellType.SELL_SIGNAL.value assert not trade.is_open + + +def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, mocker) -> None: + """ + Tests workflow + """ + default_conf['max_open_trades'] = 5 + default_conf['forcebuy_enable'] = True + default_conf['stake_amount'] = 'unlimited' + default_conf['exchange']['name'] = 'binance' + default_conf['telegram']['enabled'] = True + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock( + side_effect=[1000, 800, 600, 400, 200] + )) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + get_fee=fee, + symbol_amount_prec=lambda s, x, y: y, + symbol_price_prec=lambda s, x, y: y, + ) + + mocker.patch.multiple( + 'freqtrade.freqtradebot.FreqtradeBot', + create_stoploss_order=MagicMock(return_value=True), + update_trade_state=MagicMock(), + _notify_sell=MagicMock(), + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + rpc = RPC(freqtrade) + freqtrade.strategy.order_types['stoploss_on_exchange'] = True + # Switch ordertype to market to close trade immediately + freqtrade.strategy.order_types['sell'] = 'market' + patch_get_signal(freqtrade) + + # Create 4 trades + freqtrade.create_trades() + + trades = Trade.query.all() + assert len(trades) == 4 + rpc._rpc_forcebuy('TKN/BTC', None) + + trades = Trade.query.all() + assert len(trades) == 5 + + for trade in trades: + assert trade.stake_amount == 200 From 9a42afe0bec5442b4f0805ba190211829efde22d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Oct 2019 10:39:24 +0100 Subject: [PATCH 24/51] Move exchange-constants and retriers to exchange.common --- freqtrade/exchange/__init__.py | 3 +- freqtrade/exchange/common.py | 124 ++++++++++++++++++++++++++++++++ freqtrade/exchange/exchange.py | 123 +------------------------------ tests/exchange/test_exchange.py | 8 +-- 4 files changed, 132 insertions(+), 126 deletions(-) create mode 100644 freqtrade/exchange/common.py diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 0948692f1..c107f7abc 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,4 +1,5 @@ -from freqtrade.exchange.exchange import Exchange, MAP_EXCHANGE_CHILDCLASS # noqa: F401 +from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS # noqa: F401 +from freqtrade.exchange.exchange import Exchange # noqa: F401 from freqtrade.exchange.exchange import (get_exchange_bad_reason, # noqa: F401 is_exchange_bad, is_exchange_known_ccxt, diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py new file mode 100644 index 000000000..ed30b95c7 --- /dev/null +++ b/freqtrade/exchange/common.py @@ -0,0 +1,124 @@ +import logging + +from freqtrade import DependencyException, TemporaryError + +logger = logging.getLogger(__name__) + + +API_RETRY_COUNT = 4 +BAD_EXCHANGES = { + "bitmex": "Various reasons.", + "bitstamp": "Does not provide history. " + "Details in https://github.com/freqtrade/freqtrade/issues/1983", + "hitbtc": "This API cannot be used with Freqtrade. " + "Use `hitbtc2` exchange id to access this exchange.", + **dict.fromkeys([ + 'adara', + 'anxpro', + 'bigone', + 'coinbase', + 'coinexchange', + 'coinmarketcap', + 'lykke', + 'xbtce', + ], "Does not provide timeframes. ccxt fetchOHLCV: False"), + **dict.fromkeys([ + 'bcex', + 'bit2c', + 'bitbay', + 'bitflyer', + 'bitforex', + 'bithumb', + 'bitso', + 'bitstamp1', + 'bl3p', + 'braziliex', + 'btcbox', + 'btcchina', + 'btctradeim', + 'btctradeua', + 'bxinth', + 'chilebit', + 'coincheck', + 'coinegg', + 'coinfalcon', + 'coinfloor', + 'coingi', + 'coinmate', + 'coinone', + 'coinspot', + 'coolcoin', + 'crypton', + 'deribit', + 'exmo', + 'exx', + 'flowbtc', + 'foxbit', + 'fybse', + # 'hitbtc', + 'ice3x', + 'independentreserve', + 'indodax', + 'itbit', + 'lakebtc', + 'latoken', + 'liquid', + 'livecoin', + 'luno', + 'mixcoins', + 'negociecoins', + 'nova', + 'paymium', + 'southxchange', + 'stronghold', + 'surbitcoin', + 'therock', + 'tidex', + 'vaultoro', + 'vbtc', + 'virwox', + 'yobit', + 'zaif', + ], "Does not provide timeframes. ccxt fetchOHLCV: emulated"), +} + +MAP_EXCHANGE_CHILDCLASS = { + 'binanceus': 'binance', + 'binanceje': 'binance', +} + + +def retrier_async(f): + async def wrapper(*args, **kwargs): + count = kwargs.pop('count', API_RETRY_COUNT) + try: + return await f(*args, **kwargs) + except (TemporaryError, DependencyException) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + count -= 1 + kwargs.update({'count': count}) + logger.warning('retrying %s() still for %s times', f.__name__, count) + return await wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper + + +def retrier(f): + def wrapper(*args, **kwargs): + count = kwargs.pop('count', API_RETRY_COUNT) + try: + return f(*args, **kwargs) + except (TemporaryError, DependencyException) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + count -= 1 + kwargs.update({'count': count}) + logger.warning('retrying %s() still for %s times', f.__name__, count) + return wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 023e16cc5..430a2ff54 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -14,137 +14,18 @@ from typing import Any, Dict, List, Optional, Tuple import arrow import ccxt import ccxt.async_support as ccxt_async -from ccxt.base.decimal_to_precision import ROUND_UP, ROUND_DOWN +from ccxt.base.decimal_to_precision import ROUND_DOWN, ROUND_UP from pandas import DataFrame from freqtrade import (DependencyException, InvalidOrderException, OperationalException, TemporaryError, constants) from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async from freqtrade.misc import deep_merge_dicts - logger = logging.getLogger(__name__) -API_RETRY_COUNT = 4 -BAD_EXCHANGES = { - "bitmex": "Various reasons.", - "bitstamp": "Does not provide history. " - "Details in https://github.com/freqtrade/freqtrade/issues/1983", - "hitbtc": "This API cannot be used with Freqtrade. " - "Use `hitbtc2` exchange id to access this exchange.", - **dict.fromkeys([ - 'adara', - 'anxpro', - 'bigone', - 'coinbase', - 'coinexchange', - 'coinmarketcap', - 'lykke', - 'xbtce', - ], "Does not provide timeframes. ccxt fetchOHLCV: False"), - **dict.fromkeys([ - 'bcex', - 'bit2c', - 'bitbay', - 'bitflyer', - 'bitforex', - 'bithumb', - 'bitso', - 'bitstamp1', - 'bl3p', - 'braziliex', - 'btcbox', - 'btcchina', - 'btctradeim', - 'btctradeua', - 'bxinth', - 'chilebit', - 'coincheck', - 'coinegg', - 'coinfalcon', - 'coinfloor', - 'coingi', - 'coinmate', - 'coinone', - 'coinspot', - 'coolcoin', - 'crypton', - 'deribit', - 'exmo', - 'exx', - 'flowbtc', - 'foxbit', - 'fybse', - # 'hitbtc', - 'ice3x', - 'independentreserve', - 'indodax', - 'itbit', - 'lakebtc', - 'latoken', - 'liquid', - 'livecoin', - 'luno', - 'mixcoins', - 'negociecoins', - 'nova', - 'paymium', - 'southxchange', - 'stronghold', - 'surbitcoin', - 'therock', - 'tidex', - 'vaultoro', - 'vbtc', - 'virwox', - 'yobit', - 'zaif', - ], "Does not provide timeframes. ccxt fetchOHLCV: emulated"), - } - -MAP_EXCHANGE_CHILDCLASS = { - 'binanceus': 'binance', - 'binanceje': 'binance', -} - - -def retrier_async(f): - async def wrapper(*args, **kwargs): - count = kwargs.pop('count', API_RETRY_COUNT) - try: - return await f(*args, **kwargs) - except (TemporaryError, DependencyException) as ex: - logger.warning('%s() returned exception: "%s"', f.__name__, ex) - if count > 0: - count -= 1 - kwargs.update({'count': count}) - logger.warning('retrying %s() still for %s times', f.__name__, count) - return await wrapper(*args, **kwargs) - else: - logger.warning('Giving up retrying: %s()', f.__name__) - raise ex - return wrapper - - -def retrier(f): - def wrapper(*args, **kwargs): - count = kwargs.pop('count', API_RETRY_COUNT) - try: - return f(*args, **kwargs) - except (TemporaryError, DependencyException) as ex: - logger.warning('%s() returned exception: "%s"', f.__name__, ex) - if count > 0: - count -= 1 - kwargs.update({'count': count}) - logger.warning('retrying %s() still for %s times', f.__name__, count) - return wrapper(*args, **kwargs) - else: - logger.warning('Giving up retrying: %s()', f.__name__) - raise ex - return wrapper - - class Exchange: _config: Dict = {} diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 4eb0df1a3..8a4121d80 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -14,13 +14,13 @@ from pandas import DataFrame from freqtrade import (DependencyException, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Binance, Exchange, Kraken -from freqtrade.exchange.exchange import (API_RETRY_COUNT, timeframe_to_minutes, +from freqtrade.exchange.common import API_RETRY_COUNT +from freqtrade.exchange.exchange import (market_is_active, symbol_is_pair, + timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, - timeframe_to_seconds, - symbol_is_pair, - market_is_active) + timeframe_to_seconds) from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_patched_exchange, log_has, log_has_re From a80e49bd8174d7a335c1393f0880e5e36543e1c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Oct 2019 12:49:41 +0100 Subject: [PATCH 25/51] Change level of rpi header --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 50f24d3e8..9180beb40 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -95,7 +95,7 @@ sudo apt-get update sudo apt-get install build-essential git ``` -#### Raspberry Pi / Raspbian +### Raspberry Pi / Raspbian The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/) from at least September 2019. This image comes with python3.7 preinstalled, making it easy to get freqtrade up and running. From 241d94756466c1a65b966ba8d3e8db81d5812e2e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Nov 2019 15:39:25 +0100 Subject: [PATCH 26/51] Add new runmodes --- freqtrade/configuration/check_exchange.py | 3 ++- freqtrade/configuration/config_validation.py | 3 ++- freqtrade/state.py | 4 +++- freqtrade/utils.py | 6 +++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 5e811fb81..5d963db47 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -21,7 +21,8 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: and thus is not known for the Freqtrade at all. """ - if config['runmode'] in [RunMode.PLOT] and not config.get('exchange', {}).get('name'): + if (config['runmode'] in [RunMode.PLOT, RunMode.UTIL_NO_EXCHANGE] + and not config.get('exchange', {}).get('name')): # Skip checking exchange in plot mode, since it requires no exchange return True logger.info("Checking exchange...") diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 93d93263f..8083264e2 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -118,7 +118,8 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: """ Dynamic whitelist does not require pair_whitelist to be set - however StaticWhitelist does. """ - if conf.get('runmode', RunMode.OTHER) in [RunMode.OTHER, RunMode.PLOT]: + if conf.get('runmode', RunMode.OTHER) in [RunMode.OTHER, RunMode.PLOT, + RunMode.UTIL_NO_EXCHANGE, RunMode.UTIL_EXCHANGE]: return if (conf.get('pairlist', {}).get('method', 'StaticPairList') == 'StaticPairList' diff --git a/freqtrade/state.py b/freqtrade/state.py index d4a2adba0..ed8c75def 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -25,5 +25,7 @@ class RunMode(Enum): BACKTEST = "backtest" EDGE = "edge" HYPEROPT = "hyperopt" + UTIL_EXCHANGE = "util_exchange" + UTIL_NO_EXCHANGE = "util_no_exchange" PLOT = "plot" - OTHER = "other" # Used for plotting scripts and test + OTHER = "other" diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 25e883c76..630de0f5a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -74,7 +74,7 @@ def start_download_data(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) """ - config = setup_utils_configuration(args, RunMode.OTHER) + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) timerange = TimeRange() if 'days' in config: @@ -123,7 +123,7 @@ def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print ticker intervals (timeframes) available on Exchange """ - config = setup_utils_configuration(args, RunMode.OTHER) + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) # Do not use ticker_interval set in the config config['ticker_interval'] = None @@ -144,7 +144,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: :param pairs_only: if True print only pairs, otherwise print all instruments (markets) :return: None """ - config = setup_utils_configuration(args, RunMode.OTHER) + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) # Init exchange exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange From 691cec7956e17256af27506216954b2cd6f043cf Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Nov 2019 16:42:57 +0100 Subject: [PATCH 27/51] Be more selective which startup-messages are shown --- freqtrade/configuration/configuration.py | 46 +++++++++++++----------- freqtrade/state.py | 5 +++ 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index be1c7ab4e..4637e3e5d 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -17,7 +17,7 @@ from freqtrade.configuration.directory_operations import (create_datadir, from freqtrade.configuration.load_config import load_config_file from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts, json_load -from freqtrade.state import RunMode +from freqtrade.state import RunMode, TRADING_MODES, NON_UTIL_MODES logger = logging.getLogger(__name__) @@ -98,14 +98,16 @@ class Configuration: # Keep a copy of the original configuration file config['original_config'] = deepcopy(config) + self._process_runmode(config) + self._process_common_options(config) + self._process_trading_options(config) + self._process_optimize_options(config) self._process_plot_options(config) - self._process_runmode(config) - # Check if the exchange set by the user is supported check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) @@ -130,6 +132,22 @@ class Configuration: setup_logging(config) + def _process_trading_options(self, config: Dict[str, Any]) -> None: + if config['runmode'] not in TRADING_MODES: + return + + if config.get('dry_run', False): + logger.info('Dry run is enabled') + if config.get('db_url') in [None, constants.DEFAULT_DB_PROD_URL]: + # Default to in-memory db for dry_run if not specified + config['db_url'] = constants.DEFAULT_DB_DRYRUN_URL + else: + if not config.get('db_url', None): + config['db_url'] = constants.DEFAULT_DB_PROD_URL + logger.info('Dry run is disabled') + + logger.info(f'Using DB: "{config["db_url"]}"') + def _process_common_options(self, config: Dict[str, Any]) -> None: self._process_logging_options(config) @@ -146,25 +164,9 @@ class Configuration: config.update({'db_url': self.args["db_url"]}) logger.info('Parameter --db-url detected ...') - if config.get('dry_run', False): - logger.info('Dry run is enabled') - if config.get('db_url') in [None, constants.DEFAULT_DB_PROD_URL]: - # Default to in-memory db for dry_run if not specified - config['db_url'] = constants.DEFAULT_DB_DRYRUN_URL - else: - if not config.get('db_url', None): - config['db_url'] = constants.DEFAULT_DB_PROD_URL - logger.info('Dry run is disabled') - - logger.info(f'Using DB: "{config["db_url"]}"') - if config.get('forcebuy_enable', False): logger.warning('`forcebuy` RPC message enabled.') - # Setting max_open_trades to infinite if -1 - if config.get('max_open_trades') == -1: - config['max_open_trades'] = float('inf') - # Support for sd_notify if 'sd_notify' in self.args and self.args["sd_notify"]: config['internals'].update({'sd_notify': True}) @@ -212,6 +214,10 @@ class Configuration: self._args_to_config(config, argname='position_stacking', logstring='Parameter --enable-position-stacking detected ...') + # Setting max_open_trades to infinite if -1 + if config.get('max_open_trades') == -1: + config['max_open_trades'] = float('inf') + if 'use_max_market_positions' in self.args and not self.args["use_max_market_positions"]: config.update({'use_max_market_positions': False}) logger.info('Parameter --disable-max-market-positions detected ...') @@ -220,7 +226,7 @@ class Configuration: config.update({'max_open_trades': self.args["max_open_trades"]}) logger.info('Parameter --max_open_trades detected, ' 'overriding max_open_trades to: %s ...', config.get('max_open_trades')) - else: + elif config['runmode'] in NON_UTIL_MODES: logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) self._args_to_config(config, argname='stake_amount', diff --git a/freqtrade/state.py b/freqtrade/state.py index ed8c75def..415f6f5f2 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -29,3 +29,8 @@ class RunMode(Enum): UTIL_NO_EXCHANGE = "util_no_exchange" PLOT = "plot" OTHER = "other" + + +TRADING_MODES = [RunMode.LIVE, RunMode.DRY_RUN] +OPTIMIZE_MODES = [RunMode.BACKTEST, RunMode.EDGE, RunMode.HYPEROPT] +NON_UTIL_MODES = TRADING_MODES + OPTIMIZE_MODES From 2124661cee43affc660cb08f07af26d68004c35d Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sat, 2 Nov 2019 02:22:58 +0300 Subject: [PATCH 28/51] Update faq with examples of grepping the log --- docs/faq.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index dd92d310e..7652ec278 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -55,6 +55,44 @@ If you have restricted pairs in your whitelist, you'll get a warning message in If you're an "International" Customer on the Bittrex exchange, then this warning will probably not impact you. If you're a US customer, the bot will fail to create orders for these pairs, and you should remove them from your Whitelist. +### How do I search the bot logs for something? + +By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility subcommands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout. + +* In unix shells, this normally can be done as simple as: +```shell +freqtrade --some-options 2>&1 >/dev/null | grep 'something' +``` +(note, `2>&1` and `>/dev/null` should be written in this order) + +* Bash interpreter also supports so called process substitution syntax, you can grep the log for a string with it as: +```shell +$ freqtrade --some-options 2> >(grep 'something') >/dev/null +``` +or +```shell +$ freqtrade --some-options 2> >(grep -v 'something' 1>&2) +``` + +* You can also write the copy of Freqtrade log messages to a file with the `--logfile` option: +```shell +$ freqtrade --logfile /path/to/mylogfile.log --some-options +``` +and then grep it as: +```shell +$ cat /path/to/mylogfile.log | grep 'something' +``` +or even on the fly, as the bot works and the logfile grows: +```shell +$ tail -f /path/to/mylogfile.log | grep 'something' +``` +from a separate terminal window. + +On Windows, the `--logfilename` option is also supported by Freqtrade and you can use the `findstr` command to search the log for the string of interest: +``` +> type \path\to\mylogfile.log | findstr "something" +``` + ## Hyperopt module ### How many epoch do I need to get a good Hyperopt result? From e9af6b393f5ff7aa34e2e0d63879ced22741343e Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sat, 2 Nov 2019 02:32:57 +0300 Subject: [PATCH 29/51] Fix typo --- docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index 7652ec278..7fdd54958 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -61,7 +61,7 @@ By default, the bot writes its log into stderr stream. This is implemented this * In unix shells, this normally can be done as simple as: ```shell -freqtrade --some-options 2>&1 >/dev/null | grep 'something' +$ freqtrade --some-options 2>&1 >/dev/null | grep 'something' ``` (note, `2>&1` and `>/dev/null` should be written in this order) From 861f10dca65b64a76a3fd83459161b95e2a737ef Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 11:10:33 +0100 Subject: [PATCH 30/51] Allow populate-indicators to come from strategy --- freqtrade/optimize/hyperopt_interface.py | 12 +----------- freqtrade/resolvers/hyperopt_resolver.py | 3 +++ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 4208b29d3..142f305df 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -5,10 +5,9 @@ This module defines the interface to apply for hyperopts import logging import math -from abc import ABC, abstractmethod +from abc import ABC from typing import Dict, Any, Callable, List -from pandas import DataFrame from skopt.space import Dimension, Integer, Real from freqtrade import OperationalException @@ -42,15 +41,6 @@ class IHyperOpt(ABC): # Assign ticker_interval to be used in hyperopt IHyperOpt.ticker_interval = str(config['ticker_interval']) - @staticmethod - @abstractmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Populate indicators that will be used in the Buy and Sell strategy. - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe(). - :return: A Dataframe with all mandatory indicators for the strategies. - """ - @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index db51c3ca5..1ad53fe33 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -34,6 +34,9 @@ class HyperOptResolver(IResolver): self.hyperopt = self._load_hyperopt(hyperopt_name, config, extra_dir=config.get('hyperopt_path')) + if not hasattr(self.hyperopt, 'populate_indicators'): + logger.warning("Hyperopt class does not provide populate_indicators() method. " + "Using populate_indicators from the strategy.") if not hasattr(self.hyperopt, 'populate_buy_trend'): logger.warning("Hyperopt class does not provide populate_buy_trend() method. " "Using populate_buy_trend from the strategy.") From 97d0f93d3ce0d4ab3decaa75f7b0432fc320efd4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 11:11:13 +0100 Subject: [PATCH 31/51] Align samples (hyperopt and strategy) to work together --- user_data/hyperopts/sample_hyperopt.py | 33 ++----------------------- user_data/strategies/sample_strategy.py | 15 +++++------ 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index fabfdb23e..2721ab405 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -2,12 +2,11 @@ from functools import reduce from typing import Any, Callable, Dict, List -from datetime import datetime -import numpy as np +import numpy as np # noqa import talib.abstract as ta from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer, Real +from skopt.space import Categorical, Dimension, Integer, Real # noqa import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.optimize.hyperopt_interface import IHyperOpt @@ -34,34 +33,6 @@ class SampleHyperOpts(IHyperOpt): Sample implementation of these methods can be found in https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py """ - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Add several indicators needed for buy and sell strategies defined below. - """ - # ADX - dataframe['adx'] = ta.ADX(dataframe) - # MACD - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - # MFI - dataframe['mfi'] = ta.MFI(dataframe) - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - # Stochastic Fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - # Minus-DI - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_upperband'] = bollinger['upper'] - # SAR - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: diff --git a/user_data/strategies/sample_strategy.py b/user_data/strategies/sample_strategy.py index c2fd681d2..36dea65c9 100644 --- a/user_data/strategies/sample_strategy.py +++ b/user_data/strategies/sample_strategy.py @@ -107,16 +107,16 @@ class SampleStrategy(IStrategy): # RSI dataframe['rsi'] = ta.RSI(dataframe) - """ + # ADX dataframe['adx'] = ta.ADX(dataframe) - + """ # Awesome oscillator dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) # Commodity Channel Index: values Oversold:<-100, Overbought:>100 dataframe['cci'] = ta.CCI(dataframe) - + """ # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -126,6 +126,7 @@ class SampleStrategy(IStrategy): # MFI dataframe['mfi'] = ta.MFI(dataframe) + """ # Minus Directional Indicator / Movement dataframe['minus_dm'] = ta.MINUS_DM(dataframe) dataframe['minus_di'] = ta.MINUS_DI(dataframe) @@ -149,12 +150,13 @@ class SampleStrategy(IStrategy): stoch = ta.STOCH(dataframe) dataframe['slowd'] = stoch['slowd'] dataframe['slowk'] = stoch['slowk'] - + """ # Stoch fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] dataframe['fastk'] = stoch_fast['fastk'] + """ # Stoch RSI stoch_rsi = ta.STOCHRSI(dataframe) dataframe['fastd_rsi'] = stoch_rsi['fastd'] @@ -178,12 +180,11 @@ class SampleStrategy(IStrategy): dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabol - dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) """ + # SAR Parabol + dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) From 12e86ee4bd48300cf5057ebecc11f7b8b257089c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 11:12:08 +0100 Subject: [PATCH 32/51] Make travis test-hyperopt the sample strategy --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eb171521d..1cc22dfbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ jobs: name: backtest - script: - cp config.json.example config.json - - freqtrade --datadir tests/testdata hyperopt -e 5 + - freqtrade --datadir tests/testdata --strategy SampleStrategy hyperopt --customhyperopt SampleHyperOpts -e 5 name: hyperopt - script: flake8 name: flake8 From 3287cdd47ad5fa042f7f82f85c66a47666aa1530 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 13:01:36 +0100 Subject: [PATCH 33/51] Improve documentation regarding loading methods from hyperopt --- docs/hyperopt.md | 23 +++++++++++++++---- .../hyperopts/sample_hyperopt_advanced.py | 13 +++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 99331707f..3c42a0428 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -23,17 +23,23 @@ Configuring hyperopt is similar to writing your own strategy, and many tasks wil Depending on the space you want to optimize, only some of the below are required: -* fill `populate_indicators` - probably a copy from your strategy * fill `buy_strategy_generator` - for buy signal optimization * fill `indicator_space` - for buy signal optimzation * fill `sell_strategy_generator` - for sell signal optimization * fill `sell_indicator_space` - for sell signal optimzation -Optional, but recommended: +!!! Note + `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. +Optional - can also be loaded from a strategy: + +* copy `populate_indicators` from your strategy - otherwise default-strategy will be used * copy `populate_buy_trend` from your strategy - otherwise default-strategy will be used * copy `populate_sell_trend` from your strategy - otherwise default-strategy will be used +!!! Note + Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. + Rarely you may also need to override: * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default) @@ -156,7 +162,7 @@ that minimizes the value of the [loss function](#loss-functions). The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. When you want to test an indicator that isn't used by the bot currently, remember to -add it to the `populate_indicators()` method in `hyperopt.py`. +add it to the `populate_indicators()` method in your custom hyperopt file. ## Loss-functions @@ -270,6 +276,14 @@ For example, to use one month of data, pass the following parameter to the hyper freqtrade hyperopt --timerange 20180401-20180501 ``` +### Running Hyperopt using methods from a strategy + +Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided. + +```bash +freqtrade --strategy SampleStrategy hyperopt --customhyperopt SampleHyperopt +``` + ### Running Hyperopt with Smaller Search Space Use the `--spaces` argument to limit the search space used by hyperopt. @@ -341,8 +355,7 @@ So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that t (dataframe['rsi'] < 29.0) ``` -Translating your whole hyperopt result as the new buy-signal -would then look like: +Translating your whole hyperopt result as the new buy-signal would then look like: ```python def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: diff --git a/user_data/hyperopts/sample_hyperopt_advanced.py b/user_data/hyperopts/sample_hyperopt_advanced.py index 00062a58d..6986854ee 100644 --- a/user_data/hyperopts/sample_hyperopt_advanced.py +++ b/user_data/hyperopts/sample_hyperopt_advanced.py @@ -37,6 +37,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): """ @staticmethod def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + This method can also be loaded from the strategy, if it doesn't exist in the hyperopt class. + """ dataframe['adx'] = ta.ADX(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -229,8 +232,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file + Based on TA indicators. + Can be a copy of from the strategy, or will be loaded from the strategy. + must align to populate_indicators used (either from this File, or from the strategy) Only used when --spaces does not include buy """ dataframe.loc[ @@ -246,8 +250,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file + Based on TA indicators. + Can be a copy of from the strategy, or will be loaded from the strategy. + must align to populate_indicators used (either from this File, or from the strategy) Only used when --spaces does not include sell """ dataframe.loc[ From 80ad37ad93761b8ab69df1a2f10a3a14b4e6ad85 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Nov 2019 14:17:15 +0100 Subject: [PATCH 34/51] Updated plot_indicators test --- tests/optimize/test_hyperopt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 675bbd62e..36902dcfc 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -149,6 +149,7 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) hyperopt = DefaultHyperOpt + delattr(hyperopt, 'populate_indicators') delattr(hyperopt, 'populate_buy_trend') delattr(hyperopt, 'populate_sell_trend') mocker.patch( @@ -156,8 +157,11 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: MagicMock(return_value=hyperopt(default_conf)) ) x = HyperOptResolver(default_conf, ).hyperopt + assert not hasattr(x, 'populate_indicators') assert not hasattr(x, 'populate_buy_trend') assert not hasattr(x, 'populate_sell_trend') + assert log_has("Hyperopt class does not provide populate_indicators() method. " + "Using populate_indicators from the strategy.", caplog) assert log_has("Hyperopt class does not provide populate_sell_trend() method. " "Using populate_sell_trend from the strategy.", caplog) assert log_has("Hyperopt class does not provide populate_buy_trend() method. " From 6550e1fa992bef966fcffe1703ede0a20851da04 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Nov 2019 09:55:38 +0100 Subject: [PATCH 35/51] Change docstring in sampleHyperopt --- user_data/hyperopts/sample_hyperopt_advanced.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/user_data/hyperopts/sample_hyperopt_advanced.py b/user_data/hyperopts/sample_hyperopt_advanced.py index 6986854ee..c5d28878c 100644 --- a/user_data/hyperopts/sample_hyperopt_advanced.py +++ b/user_data/hyperopts/sample_hyperopt_advanced.py @@ -233,8 +233,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators. - Can be a copy of from the strategy, or will be loaded from the strategy. - must align to populate_indicators used (either from this File, or from the strategy) + Can be a copy of the corresponding method from the strategy, + or will be loaded from the strategy. + Must align to populate_indicators used (either from this File, or from the strategy) Only used when --spaces does not include buy """ dataframe.loc[ @@ -251,8 +252,9 @@ class AdvancedSampleHyperOpts(IHyperOpt): def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators. - Can be a copy of from the strategy, or will be loaded from the strategy. - must align to populate_indicators used (either from this File, or from the strategy) + Can be a copy of the corresponding method from the strategy, + or will be loaded from the strategy. + Must align to populate_indicators used (either from this File, or from the strategy) Only used when --spaces does not include sell """ dataframe.loc[ From 3eca80217c741f226a2dafa2c7db69cac2561de9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Nov 2019 10:18:46 +0100 Subject: [PATCH 36/51] Don't check exchange for Utils commands --- freqtrade/configuration/check_exchange.py | 2 +- tests/test_configuration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 5d963db47..9e7ce70a9 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -21,7 +21,7 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: and thus is not known for the Freqtrade at all. """ - if (config['runmode'] in [RunMode.PLOT, RunMode.UTIL_NO_EXCHANGE] + if (config['runmode'] in [RunMode.PLOT, RunMode.UTIL_NO_EXCHANGE, RunMode.OTHER] and not config.get('exchange', {}).get('name')): # Skip checking exchange in plot mode, since it requires no exchange return True diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 545dd5df4..6cb6ca758 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -545,7 +545,7 @@ def test_check_exchange(default_conf, caplog) -> None: # Test no exchange... default_conf.get('exchange').update({'name': ''}) - default_conf['runmode'] = RunMode.OTHER + default_conf['runmode'] = RunMode.UTIL_EXCHANGE with pytest.raises(OperationalException, match=r'This command requires a configured exchange.*'): check_exchange(default_conf) From 1e44f93c31a5adbc0617f3767574b2a06b9ec56d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Nov 2019 10:38:21 +0100 Subject: [PATCH 37/51] Fix pandas access warning --- freqtrade/optimize/backtesting.py | 3 ++- tests/optimize/test_backtest_detail.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fe31912bc..076857e62 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -245,7 +245,8 @@ class Backtesting: ticker: Dict = {} # Create ticker dict for pair, pair_data in processed.items(): - pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run + pair_data.loc[:, 'buy'] = 0 # cleanup from previous run + pair_data.loc[:, 'sell'] = 0 # cleanup from previous run ticker_data = self.strategy.advise_sell( self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 345e423cd..54f4c8796 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -3,7 +3,6 @@ import logging from unittest.mock import MagicMock import pytest -from pandas import DataFrame from freqtrade.data.history import get_timeframe from freqtrade.optimize.backtesting import Backtesting @@ -313,7 +312,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: pair = "UNITTEST/BTC" # Dummy data as we mock the analyze functions - data_processed = {pair: DataFrame()} + data_processed = {pair: frame.copy()} min_date, max_date = get_timeframe({pair: frame}) results = backtesting.backtest( { From 871019c8b9f8e50f98f8f45128690672e44b681d Mon Sep 17 00:00:00 2001 From: Gautier Pialat Date: Tue, 5 Nov 2019 12:08:57 +0100 Subject: [PATCH 38/51] docker doc update about restart policy --- docs/docker.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/docker.md b/docs/docker.md index 8a254b749..7b9fce85f 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -170,6 +170,9 @@ docker run -d \ !!! Note All available bot command line parameters can be added to the end of the `docker run` command. +!!! Note + you can define a [restart policy](https://docs.docker.com/config/containers/start-containers-automatically/) in docker. It can be useful in some cases to use the `--restart unless-stopped` flag (crash of freqtrade or reboot of your system). + ### Monitor your Docker instance You can use the following commands to monitor and manage your container: From f6a66cd3de7a1e1dafc6e85788c37dce5435eaed Mon Sep 17 00:00:00 2001 From: Gautier Pialat Date: Tue, 5 Nov 2019 12:14:39 +0100 Subject: [PATCH 39/51] Fix typo --- docs/docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docker.md b/docs/docker.md index 7b9fce85f..a06d82bdc 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -171,7 +171,7 @@ docker run -d \ All available bot command line parameters can be added to the end of the `docker run` command. !!! Note - you can define a [restart policy](https://docs.docker.com/config/containers/start-containers-automatically/) in docker. It can be useful in some cases to use the `--restart unless-stopped` flag (crash of freqtrade or reboot of your system). + You can define a [restart policy](https://docs.docker.com/config/containers/start-containers-automatically/) in docker. It can be useful in some cases to use the `--restart unless-stopped` flag (crash of freqtrade or reboot of your system). ### Monitor your Docker instance From eb0b0350e031bdd1dca7fc9d7d8ef9eaa41fdfe7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Nov 2019 12:39:19 +0100 Subject: [PATCH 40/51] Introduce remove_credentials to remove code duplication --- freqtrade/configuration/__init__.py | 1 + freqtrade/configuration/check_exchange.py | 13 +++++++++++++ freqtrade/optimize/backtesting.py | 10 +++------- freqtrade/optimize/edge_cli.py | 17 +++++++---------- freqtrade/utils.py | 6 ++---- tests/test_configuration.py | 22 +++++++++++++++++----- tests/test_utils.py | 2 +- 7 files changed, 44 insertions(+), 27 deletions(-) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index ac59421a7..63c38d8c5 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,4 +1,5 @@ from freqtrade.configuration.arguments import Arguments # noqa: F401 +from freqtrade.configuration.check_exchange import check_exchange, remove_credentials # noqa: F401 from freqtrade.configuration.timerange import TimeRange # noqa: F401 from freqtrade.configuration.configuration import Configuration # noqa: F401 from freqtrade.configuration.config_validation import validate_config_consistency # noqa: F401 diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 9e7ce70a9..c739de692 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -10,6 +10,19 @@ from freqtrade.state import RunMode logger = logging.getLogger(__name__) +def remove_credentials(config: Dict[str, Any]): + """ + Removes exchange keys from the configuration and specifies dry-run + Used for backtesting / hyperopt / edge and utils. + Modifies the input dict! + """ + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['exchange']['password'] = '' + config['exchange']['uid'] = '' + config['dry_run'] = True + + def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: """ Check if the exchange name in the config file is supported by Freqtrade diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 076857e62..ee3a135d2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -10,9 +10,10 @@ from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional from pandas import DataFrame +from tabulate import tabulate from freqtrade import OperationalException -from freqtrade.configuration import TimeRange +from freqtrade.configuration import TimeRange, remove_credentials from freqtrade.data import history from freqtrade.data.dataprovider import DataProvider from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds @@ -21,7 +22,6 @@ from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode from freqtrade.strategy.interface import IStrategy, SellType -from tabulate import tabulate logger = logging.getLogger(__name__) @@ -57,11 +57,7 @@ class Backtesting: self.config = config # Reset keys for backtesting - self.config['exchange']['key'] = '' - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' - self.config['dry_run'] = True + remove_credentials(self.config) self.strategylist: List[IStrategy] = [] self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 1ba6bcc65..5a4543884 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -4,12 +4,13 @@ This module contains the edge backtesting interface """ import logging -from typing import Dict, Any -from tabulate import tabulate -from freqtrade import constants -from freqtrade.edge import Edge +from typing import Any, Dict -from freqtrade.configuration import TimeRange +from tabulate import tabulate + +from freqtrade import constants +from freqtrade.configuration import TimeRange, remove_credentials +from freqtrade.edge import Edge from freqtrade.exchange import Exchange from freqtrade.resolvers import StrategyResolver @@ -29,12 +30,8 @@ class EdgeCli: self.config = config # Reset keys for edge - self.config['exchange']['key'] = '' - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' + remove_credentials(self.config) self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT - self.config['dry_run'] = True self.exchange = Exchange(self.config) self.strategy = StrategyResolver(self.config).strategy diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 630de0f5a..03cf7dabf 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -10,7 +10,7 @@ import rapidjson from tabulate import tabulate from freqtrade import OperationalException -from freqtrade.configuration import Configuration, TimeRange +from freqtrade.configuration import Configuration, TimeRange, remove_credentials from freqtrade.configuration.directory_operations import create_userdata_dir from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, @@ -33,10 +33,8 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str configuration = Configuration(args, method) config = configuration.get_config() - config['exchange']['dry_run'] = True # Ensure we do not use Exchange credentials - config['exchange']['key'] = '' - config['exchange']['secret'] = '' + remove_credentials(config) return config diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 6cb6ca758..d27cd92f4 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -10,13 +10,13 @@ import pytest from jsonschema import Draft4Validator, ValidationError, validate from freqtrade import OperationalException, constants -from freqtrade.configuration import (Arguments, Configuration, +from freqtrade.configuration import (Arguments, Configuration, check_exchange, + remove_credentials, validate_config_consistency) -from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.config_validation import validate_config_schema -from freqtrade.configuration.deprecated_settings import (check_conflicting_settings, - process_deprecated_setting, - process_temporary_deprecated_settings) +from freqtrade.configuration.deprecated_settings import ( + check_conflicting_settings, process_deprecated_setting, + process_temporary_deprecated_settings) from freqtrade.configuration.directory_operations import (create_datadir, create_userdata_dir) from freqtrade.configuration.load_config import load_config_file @@ -551,6 +551,18 @@ def test_check_exchange(default_conf, caplog) -> None: check_exchange(default_conf) +def test_remove_credentials(default_conf, caplog) -> None: + conf = deepcopy(default_conf) + conf['dry_run'] = False + remove_credentials(conf) + + assert conf['dry_run'] is True + assert conf['exchange']['key'] == '' + assert conf['exchange']['secret'] == '' + assert conf['exchange']['password'] == '' + assert conf['exchange']['uid'] == '' + + def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7d6b82809..474a842d0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,7 +19,7 @@ def test_setup_utils_configuration(): config = setup_utils_configuration(get_args(args), RunMode.OTHER) assert "exchange" in config - assert config['exchange']['dry_run'] is True + assert config['dry_run'] is True assert config['exchange']['key'] == '' assert config['exchange']['secret'] == '' From c8638ce82fae82e9558a292e6c9dc2a9dcbb7d70 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Nov 2019 21:03:06 +0100 Subject: [PATCH 41/51] Fix bug where bids_to_ask_delta causes doublebuys The continue must happen irrespective of the outcome of this - otherwise the below BUY will happen anyway. --- freqtrade/freqtradebot.py | 3 +-- tests/test_freqtradebot.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7206e555f..7e9706803 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -319,8 +319,7 @@ class FreqtradeBot: (bidstrat_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): if self._check_depth_of_market_buy(_pair, bidstrat_check_depth_of_market): buycount += self.execute_buy(_pair, stake_amount) - else: - continue + continue buycount += self.execute_buy(_pair, stake_amount) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 0ed8e8a77..f3baff7ce 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3213,6 +3213,8 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, assert trade.open_date is not None assert trade.exchange == 'bittrex' + assert len(Trade.query.all()) == 1 + # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) From b8a6c55b105592bde1095ca90c9cfc283e5ce5cc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2019 07:21:09 +0000 Subject: [PATCH 42/51] Bump arrow from 0.15.2 to 0.15.4 Bumps [arrow](https://github.com/crsmithdev/arrow) from 0.15.2 to 0.15.4. - [Release notes](https://github.com/crsmithdev/arrow/releases) - [Changelog](https://github.com/crsmithdev/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/crsmithdev/arrow/compare/0.15.2...0.15.4) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 64a43ee62..08838c3d4 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -3,7 +3,7 @@ ccxt==1.18.1346 SQLAlchemy==1.3.10 python-telegram-bot==12.2.0 -arrow==0.15.2 +arrow==0.15.4 cachetools==3.1.1 requests==2.22.0 urllib3==1.25.6 From bc78316aa52d49f0df194e9dc11813b796368db5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2019 07:21:39 +0000 Subject: [PATCH 43/51] Bump flake8 from 3.7.8 to 3.7.9 Bumps [flake8](https://gitlab.com/pycqa/flake8) from 3.7.8 to 3.7.9. - [Release notes](https://gitlab.com/pycqa/flake8/tags) - [Commits](https://gitlab.com/pycqa/flake8/compare/3.7.8...3.7.9) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 589ca7c54..f346439af 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ -r requirements-hyperopt.txt coveralls==1.8.2 -flake8==3.7.8 +flake8==3.7.9 flake8-type-annotations==0.1.0 flake8-tidy-imports==3.0.0 mypy==0.740 From 28f0c00281c3e3fedcef268f1475e329574be461 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2019 07:21:55 +0000 Subject: [PATCH 44/51] Bump pandas from 0.25.2 to 0.25.3 Bumps [pandas](https://github.com/pandas-dev/pandas) from 0.25.2 to 0.25.3. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v0.25.2...v0.25.3) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8d9b4953f..331e3dc67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ -r requirements-common.txt numpy==1.17.3 -pandas==0.25.2 +pandas==0.25.3 From 60109aaa1f83923ed424ab1ad1b088bfe13cec21 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2019 08:37:38 +0000 Subject: [PATCH 45/51] Bump ccxt from 1.18.1346 to 1.19.14 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.18.1346 to 1.19.14. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.18.1346...1.19.14) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 08838c3d4..c11179fbb 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1346 +ccxt==1.19.14 SQLAlchemy==1.3.10 python-telegram-bot==12.2.0 arrow==0.15.4 From ca77dbe8dab200367c90654e411a7510d224a9cb Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Nov 2019 19:33:15 +0100 Subject: [PATCH 46/51] Fix UnicodeError in hyperopt output --- freqtrade/optimize/hyperopt.py | 18 ++++++++++-------- tests/optimize/test_hyperopt.py | 6 ++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c576ea6f8..6ea2f5133 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,9 +4,9 @@ This module contains the hyperopt logic """ +import locale import logging import sys - from collections import OrderedDict from operator import itemgetter from pathlib import Path @@ -14,10 +14,10 @@ from pprint import pprint from typing import Any, Dict, List, Optional import rapidjson - -from colorama import init as colorama_init from colorama import Fore, Style -from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects, cpu_count +from colorama import init as colorama_init +from joblib import (Parallel, cpu_count, delayed, dump, load, + wrap_non_picklable_objects) from pandas import DataFrame from skopt import Optimizer from skopt.space import Dimension @@ -28,8 +28,8 @@ from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F4 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F4 -from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver, HyperOptLossResolver - +from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, + HyperOptResolver) logger = logging.getLogger(__name__) @@ -216,7 +216,7 @@ class Hyperopt: if print_all: print(log_str) else: - print('\n' + log_str) + print(f'\n{log_str}') else: print('.', end='') sys.stdout.flush() @@ -335,7 +335,9 @@ class Hyperopt: return (f'{trades:6d} trades. Avg profit {avg_profit: 5.2f}%. ' f'Total profit {total_profit: 11.8f} {stake_cur} ' - f'({profit: 7.2f}Σ%). Avg duration {duration:5.1f} mins.') + f'({profit: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). ' + f'Avg duration {duration:5.1f} mins.' + ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8') def get_optimizer(self, dimensions, cpu_count) -> Optimizer: return Optimizer( diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 36902dcfc..23d8a887c 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,4 +1,5 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 +import locale from datetime import datetime from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -565,8 +566,9 @@ def test_generate_optimizer(mocker, default_conf) -> None: } response_expected = { 'loss': 1.9840569076926293, - 'results_explanation': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' - '( 2.31Σ%). Avg duration 100.0 mins.', + 'results_explanation': (' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' + '( 2.31\N{GREEK CAPITAL LETTER SIGMA}%). Avg duration 100.0 mins.' + ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8'), 'params': optimizer_param, 'total_profit': 0.00023300 } From da57396d071131102c366d9821e13566cf594998 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Nov 2019 06:55:07 +0100 Subject: [PATCH 47/51] Fix UTC handling of timestamp() conversation in fetch_my_trades --- freqtrade/data/btanalysis.py | 6 +++--- freqtrade/data/history.py | 7 +++---- freqtrade/exchange/exchange.py | 19 ++++++++++++++++++- tests/exchange/test_exchange.py | 17 +++++++++++++++-- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 388deb4b3..2f7a234ce 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -7,7 +7,7 @@ from typing import Dict import numpy as np import pandas as pd -import pytz +from datetime import timezone from freqtrade import persistence from freqtrade.misc import json_load @@ -106,8 +106,8 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: "stop_loss", "initial_stop_loss", "strategy", "ticker_interval"] trades = pd.DataFrame([(t.pair, - t.open_date.replace(tzinfo=pytz.UTC), - t.close_date.replace(tzinfo=pytz.UTC) if t.close_date else None, + t.open_date.replace(tzinfo=timezone.utc), + t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None, t.calc_profit(), t.calc_profit_percent(), t.open_rate, t.close_rate, t.amount, (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index b1e4313ca..3dd40d2b4 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -9,12 +9,11 @@ Includes: import logging import operator from copy import deepcopy -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import arrow -import pytz from pandas import DataFrame from freqtrade import OperationalException, misc @@ -56,10 +55,10 @@ def trim_dataframe(df: DataFrame, timerange: TimeRange) -> DataFrame: Trim dataframe based on given timerange """ if timerange.starttype == 'date': - start = datetime.fromtimestamp(timerange.startts, tz=pytz.utc) + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) df = df.loc[df['date'] >= start, :] if timerange.stoptype == 'date': - stop = datetime.fromtimestamp(timerange.stopts, tz=pytz.utc) + stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) df = df.loc[df['date'] <= stop, :] return df diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 430a2ff54..a198e8cdb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -875,6 +875,22 @@ class Exchange: @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: + """ + Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. + The "since" argument passed in is coming from the database and is in UTC, + as timezone-native datetime object. + From the python documentation: + > Naive datetime instances are assumed to represent local time + Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the + transformation from local timezone to UTC. + This works for timezones UTC+ since then the result will contain trades from a few hours + instead of from the last 5 seconds, however fails for UTC- timezones, + since we're then asking for trades with a "since" argument in the future. + + :param order_id order_id: Order-id as given when creating the order + :param pair: Pair the order is for + :param since: datetime object of the order creation time. Assumes object is in UTC. + """ if self._config['dry_run']: return [] if not self.exchange_has('fetchMyTrades'): @@ -882,7 +898,8 @@ class Exchange: try: # Allow 5s offset to catch slight time offsets (discovered in #1185) # since needs to be int in milliseconds - my_trades = self._api.fetch_my_trades(pair, int((since.timestamp() - 5) * 1000)) + my_trades = self._api.fetch_my_trades( + pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) matched_trades = [trade for trade in my_trades if trade['order'] == order_id] return matched_trades diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8a4121d80..1bb643e24 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1586,8 +1586,20 @@ def test_name(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_get_trades_for_order(default_conf, mocker, exchange_name): + """ + Crucial part in this test is the "since" calculation. + The "since" argument passed in is coming from the database and is in UTC, + as timezone-native datetime object. + From the python documentation: + > Naive datetime instances are assumed to represent local time + Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the + transformation from local timezone to UTC. + This works for timezones UTC+ since then the result will contain trades from a few hours + instead of from the last 5 seconds, however fails for UTC- timezones, + since we're then asking for trades with a "since" argument in the future. + """ order_id = 'ABCD-ABCD' - since = datetime(2018, 5, 5, tzinfo=timezone.utc) + since = datetime(2018, 5, 5, 0, 0, 0) default_conf["dry_run"] = False mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True) api_mock = MagicMock() @@ -1623,7 +1635,8 @@ def test_get_trades_for_order(default_conf, mocker, exchange_name): assert api_mock.fetch_my_trades.call_args[0][0] == 'LTC/BTC' # Same test twice, hardcoded number and doing the same calculation assert api_mock.fetch_my_trades.call_args[0][1] == 1525478395000 - assert api_mock.fetch_my_trades.call_args[0][1] == int(since.timestamp() - 5) * 1000 + assert api_mock.fetch_my_trades.call_args[0][1] == int(since.replace( + tzinfo=timezone.utc).timestamp() - 5) * 1000 ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, 'get_trades_for_order', 'fetch_my_trades', From dd47bd04cd905c8822f5c27a285db8d37d72847f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Nov 2019 01:32:08 -0500 Subject: [PATCH 48/51] Move description to correct place --- tests/exchange/test_exchange.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 1bb643e24..925a53c95 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1586,18 +1586,7 @@ def test_name(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_get_trades_for_order(default_conf, mocker, exchange_name): - """ - Crucial part in this test is the "since" calculation. - The "since" argument passed in is coming from the database and is in UTC, - as timezone-native datetime object. - From the python documentation: - > Naive datetime instances are assumed to represent local time - Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the - transformation from local timezone to UTC. - This works for timezones UTC+ since then the result will contain trades from a few hours - instead of from the last 5 seconds, however fails for UTC- timezones, - since we're then asking for trades with a "since" argument in the future. - """ + order_id = 'ABCD-ABCD' since = datetime(2018, 5, 5, 0, 0, 0) default_conf["dry_run"] = False From b0150d548a741153791fd1baf9e40394f75a38a1 Mon Sep 17 00:00:00 2001 From: Gautier Pialat Date: Fri, 8 Nov 2019 09:37:54 +0100 Subject: [PATCH 49/51] remove not use statement --- docs/installation.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 9180beb40..6edb28481 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -34,7 +34,6 @@ Freqtrade provides a Linux/MacOS script to install all dependencies and help you ```bash git clone git@github.com:freqtrade/freqtrade.git cd freqtrade -git checkout develop ./setup.sh --install ``` From 076ef0407b2057b6c8ad42c63e2ae9760d5ceb73 Mon Sep 17 00:00:00 2001 From: Gautier Pialat Date: Fri, 8 Nov 2019 09:39:06 +0100 Subject: [PATCH 50/51] git branch note explanation --- docs/installation.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/installation.md b/docs/installation.md index 6edb28481..d373c95d0 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -37,6 +37,7 @@ cd freqtrade ./setup.sh --install ``` + When cloning the repository the default working branch is name `develop`. This branch contains the last features (can be considered as relatively stable thanks to automated tests). The `master` branch contains the code of the last release (done once per month with a one week old snapshot of the `develop` branch to prevent packaging bugs so potentially more stable). !!! Note Windows installation is explained [here](#windows). From bc5c91f6815b20ae64f6b8e8a5e29abac1ab51f2 Mon Sep 17 00:00:00 2001 From: Gautier Pialat Date: Fri, 8 Nov 2019 10:29:00 +0100 Subject: [PATCH 51/51] add missing note block --- docs/installation.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index d373c95d0..ce35572c4 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -37,7 +37,9 @@ cd freqtrade ./setup.sh --install ``` +!!! Note "Version considerations" When cloning the repository the default working branch is name `develop`. This branch contains the last features (can be considered as relatively stable thanks to automated tests). The `master` branch contains the code of the last release (done once per month with a one week old snapshot of the `develop` branch to prevent packaging bugs so potentially more stable). + !!! Note Windows installation is explained [here](#windows).