From 0a1e15988f9149670a8c7df2f77ad2a04a20e046 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Fri, 18 Jun 2021 09:48:59 +0200 Subject: [PATCH 01/16] Fix errors during ubuntu install Encountering the python header error on a fresh ubuntu install: ``` utils_find_1st/find_1st.cpp:3:10: fatal error: Python.h: No such file or directory #include "Python.h" ^~~~~~~~~~ compilation terminated. ``` solved by installing python3.7-dev. Also need to ensure python3.7-venv for fresh install. --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index c19965a18..25994fdc0 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -60,7 +60,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces sudo apt-get update # install packages - sudo apt install -y python3-pip python3-venv python3-pandas git + sudo apt install -y python3-pip python3.7-venv python3.7-dev python3-pandas git ``` === "RaspberryPi/Raspbian" From de17bf4312e4a444d97fbb5879ace599be7aa178 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 20 Jun 2021 15:41:01 -0600 Subject: [PATCH 02/16] Added TODO list --- TODO | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 TODO diff --git a/TODO b/TODO new file mode 100644 index 000000000..2bd99ae2b --- /dev/null +++ b/TODO @@ -0,0 +1,32 @@ +Files to edit + freqtrade/freqtradebot.py + freqtrade/wallets.py + freqtrade/data/btanalysis.py + configuration + freqtrade/commands/deploy_commands.py + freqtrade/commands/arguments.py + freqtrade/strategy + freqtrade/constants.py + +Tests + tests/test_persistence.pys + tests/test_freqtradebot.py + +later + freqtrade/commands/build_config_commands.py + freqtrade/commands/cli_options.py + freqtrade/commands/list_commands.py + freqtrade/commands/hyperopt_commands.py + config_binance.json.example + config_kraken.json.example + freqtrade/enums/selltype.py + +Did not look at these files + freqtrade/plot/plotting.py + freqtrade/plugins + freqtrade/resolvers/strategy_resolver.py + freqtrade/rpc + +Already Edited + freqtrade/persistence/migrations.py + freqtrade/persistence/models.py \ No newline at end of file From 17f8936f420b83901d8b78c0f7a5a7099fa07637 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:00:59 +0000 Subject: [PATCH 03/16] Bump scipy from 1.6.3 to 1.7.0 Bumps [scipy](https://github.com/scipy/scipy) from 1.6.3 to 1.7.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.6.3...v1.7.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 5e7e9d9d2..83e23e3ec 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.6.3 +scipy==1.7.0 scikit-learn==0.24.2 scikit-optimize==0.8.1 filelock==3.0.12 From fc7b372ce43e58aa46a5ab0cff6a71a278a69196 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:01:06 +0000 Subject: [PATCH 04/16] Bump ccxt from 1.51.40 to 1.51.77 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.51.40 to 1.51.77. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.51.40...1.51.77) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d0b0256e..dab9861e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.20.3 pandas==1.2.4 -ccxt==1.51.40 +ccxt==1.51.77 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From eab6399490445bacc8a17fc97defa2a764fd0f8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:01:17 +0000 Subject: [PATCH 05/16] Bump prompt-toolkit from 3.0.18 to 3.0.19 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.18 to 3.0.19. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.18...3.0.19) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d0b0256e..7dee5cb01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,4 +40,4 @@ aiofiles==0.7.0 colorama==0.4.4 # Building config files interactively questionary==1.9.0 -prompt-toolkit==3.0.18 +prompt-toolkit==3.0.19 From a6628fc65f9e0b7d0be0dbec154a47d7eadc4264 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:01:21 +0000 Subject: [PATCH 06/16] Bump types-requests from 0.1.11 to 0.1.13 Bumps [types-requests](https://github.com/python/typeshed) from 0.1.11 to 0.1.13. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 924b35e1a..1867c543d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -21,5 +21,5 @@ nbconvert==6.0.7 # mypy types types-cachetools==0.1.7 types-filelock==0.1.3 -types-requests==0.1.11 +types-requests==0.1.13 types-tabulate==0.1.0 From bb0ee837bc5f03a59dcc09956cab8f91e9530633 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 03:01:22 +0000 Subject: [PATCH 07/16] Bump pycoingecko from 2.1.0 to 2.2.0 Bumps [pycoingecko](https://github.com/man-c/pycoingecko) from 2.1.0 to 2.2.0. - [Release notes](https://github.com/man-c/pycoingecko/releases) - [Changelog](https://github.com/man-c/pycoingecko/blob/master/CHANGELOG.md) - [Commits](https://github.com/man-c/pycoingecko/commits) --- updated-dependencies: - dependency-name: pycoingecko dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d0b0256e..b3419e437 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ jsonschema==3.2.0 TA-Lib==0.4.20 technical==1.3.0 tabulate==0.8.9 -pycoingecko==2.1.0 +pycoingecko==2.2.0 jinja2==3.0.1 tables==3.6.1 blosc==1.10.4 From fdc04e27a4e6a71314dad58b0001bc947aa3b305 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 04:25:59 +0000 Subject: [PATCH 08/16] Bump types-tabulate from 0.1.0 to 0.1.1 Bumps [types-tabulate](https://github.com/python/typeshed) from 0.1.0 to 0.1.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-tabulate dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1867c543d..927e1c813 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,4 +22,4 @@ nbconvert==6.0.7 types-cachetools==0.1.7 types-filelock==0.1.3 types-requests==0.1.13 -types-tabulate==0.1.0 +types-tabulate==0.1.1 From 2d05a8bea1d656f2d2f7c2261b6bd99537c8686b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 04:35:56 +0000 Subject: [PATCH 09/16] Bump types-cachetools from 0.1.7 to 0.1.8 Bumps [types-cachetools](https://github.com/python/typeshed) from 0.1.7 to 0.1.8. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1867c543d..bdac6d0f4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,7 +19,7 @@ isort==5.8.0 nbconvert==6.0.7 # mypy types -types-cachetools==0.1.7 +types-cachetools==0.1.8 types-filelock==0.1.3 types-requests==0.1.13 types-tabulate==0.1.0 From 8c1484ed5e95ca0cef7af498e5870e886b85f6ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 06:06:08 +0000 Subject: [PATCH 10/16] Bump types-filelock from 0.1.3 to 0.1.4 Bumps [types-filelock](https://github.com/python/typeshed) from 0.1.3 to 0.1.4. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-filelock dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 328830454..30044058b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,6 +20,6 @@ nbconvert==6.0.7 # mypy types types-cachetools==0.1.8 -types-filelock==0.1.3 +types-filelock==0.1.4 types-requests==0.1.13 types-tabulate==0.1.1 From a9bd0700ede902b68183939beeeee8f000056fb4 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Mon, 21 Jun 2021 21:26:31 -0600 Subject: [PATCH 11/16] Adding templates for leverage/short tests --- TODO | 27 ++++++ freqtrade/persistence/migrations.py | 13 +-- freqtrade/persistence/models.py | 23 ++++-- tests/conftest.py | 123 ++++++++++++++++++++++++---- tests/conftest_trades.py | 4 +- tests/test_persistence.py | 8 +- 6 files changed, 168 insertions(+), 30 deletions(-) diff --git a/TODO b/TODO index c3f571cb9..c89e7ddf0 100644 --- a/TODO +++ b/TODO @@ -12,6 +12,33 @@ Files to edit Tests tests/test_persistence.pys + init with + lev & bor + lev + bor + neither lev nor bor + adjust_stop_loss + short + leverage + is_opening_trade + short + long + shortBuy + longSell + is_closing_trade + short + long + shortBuy + longSell + update, close, update fee + possible to test? + calc_profit + * * create a few shorts, a few leveraged longs test correct ratio + calc_profit_ratio + * create a few shorts, a few leveraged longs test correct ratio + get_open_trades + * create a short, check if exists + tests/test_freqtradebot.py later diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 298b18775..c4e6368c5 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -47,7 +47,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col min_rate = get_column_def(cols, 'min_rate', 'null') sell_reason = get_column_def(cols, 'sell_reason', 'null') strategy = get_column_def(cols, 'strategy', 'null') - + leverage = get_column_def(cols, 'leverage', '0.0') borrowed = get_column_def(cols, 'borrowed', '0.0') borrowed_currency = get_column_def(cols, 'borrowed_currency', 'null') @@ -66,7 +66,8 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col close_profit_abs = get_column_def( cols, 'close_profit_abs', f"(amount * close_rate * (1 - {fee_close})) - {open_trade_value}") - close_order_status = get_column_def(cols, 'close_order_status', 'null') + # TODO-mg: update to exit order status + sell_order_status = get_column_def(cols, 'sell_order_status', 'null') amount_requested = get_column_def(cols, 'amount_requested', 'amount') # Schema migration necessary @@ -88,7 +89,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col stake_amount, amount, amount_requested, open_date, close_date, open_order_id, stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, stoploss_order_id, stoploss_last_update, - max_rate, min_rate, sell_reason, close_order_status, strategy, + max_rate, min_rate, sell_reason, sell_order_status, strategy, timeframe, open_trade_value, close_profit_abs, leverage, borrowed, borrowed_currency, collateral_currency, interest_rate, liquidation_price, is_short ) @@ -111,7 +112,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col {initial_stop_loss_pct} initial_stop_loss_pct, {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, - {close_order_status} close_order_status, + {sell_order_status} sell_order_status, {strategy} strategy, {timeframe} timeframe, {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs, {leverage} leverage, {borrowed} borrowed, {borrowed_currency} borrowed_currency, @@ -120,7 +121,9 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col from {table_back_name} """)) -#TODO: Does leverage go in here? +# TODO: Does leverage go in here? + + def migrate_open_orders_to_trades(engine): with engine.begin() as connection: connection.execute(text(""" diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index a3801d24a..5e9862463 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -258,7 +258,7 @@ class LocalTrade(): # Lowest price reached min_rate: float = 0.0 sell_reason: str = '' - close_order_status: str = '' + sell_order_status: str = '' strategy: str = '' timeframe: Optional[int] = None @@ -348,7 +348,7 @@ class LocalTrade(): 'profit_abs': self.close_profit_abs, 'sell_reason': self.sell_reason, - 'close_order_status': self.close_order_status, + 'sell_order_status': self.sell_order_status, 'stop_loss_abs': self.stop_loss, 'stop_loss_ratio': self.stop_loss_pct if self.stop_loss_pct else None, 'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None, @@ -502,7 +502,7 @@ class LocalTrade(): self.close_profit = self.calc_profit_ratio() self.close_profit_abs = self.calc_profit() self.is_open = False - self.close_order_status = 'closed' + self.sell_order_status = 'closed' self.open_order_id = None if show_msg: logger.info( @@ -576,8 +576,10 @@ class LocalTrade(): close_trade = Decimal(self.amount) * Decimal(rate or self.close_rate) # type: ignore fees = close_trade * Decimal(fee or self.fee_close) - #TODO: Interest rate could be hourly instead of daily - interest = ((Decimal(self.interest_rate) * Decimal(self.borrowed)) * Decimal((datetime.utcnow() - self.open_date).days)) or 0 # Interest/day * num of days + # TODO: This interest rate is bad, doesn't get fractions of days + + interest = ((Decimal(self.interest_rate) * Decimal(self.borrowed)) * + Decimal((datetime.utcnow() - self.open_date).days)) or 0 # Interest/day * num of days if (self.is_short): return float(close_trade + fees + interest) else: @@ -622,7 +624,10 @@ class LocalTrade(): if self.is_short: profit_ratio = (close_trade_value / self.open_trade_value) - 1 else: - profit_ratio = (self.open_trade_value / close_trade_value) - 1 + if close_trade_value == 0: + profit_ratio = 0 + else: + profit_ratio = (self.open_trade_value / close_trade_value) - 1 return float(f"{profit_ratio:.8f}") def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]: @@ -672,7 +677,7 @@ class LocalTrade(): sel_trades = [trade for trade in sel_trades if trade.close_date and trade.close_date > close_date] - return sel_trades + return sel_trades @staticmethod def close_bt_trade(trade): @@ -768,8 +773,8 @@ class Trade(_DECL_BASE, LocalTrade): max_rate = Column(Float, nullable=True, default=0.0) # Lowest price reached min_rate = Column(Float, nullable=True) - sell_reason = Column(String(100), nullable=True) #TODO: Change to close_reason - close_order_status = Column(String(100), nullable=True) + sell_reason = Column(String(100), nullable=True) # TODO: Change to close_reason + sell_order_status = Column(String(100), nullable=True) strategy = Column(String(100), nullable=True) timeframe = Column(Integer, nullable=True) diff --git a/tests/conftest.py b/tests/conftest.py index c6a0dfcfd..864cfd243 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -221,6 +221,8 @@ def create_mock_trades(fee, use_db: bool = True): trade = mock_trade_6(fee) add_trade(trade) + # TODO-mg: Add margin trades + if use_db: Trade.query.session.flush() @@ -250,6 +252,7 @@ def patch_coingekko(mocker) -> None: @pytest.fixture(scope='function') def init_persistence(default_conf): init_db(default_conf['db_url'], default_conf['dry_run']) + # TODO-mg: margin with leverage and/or borrowed? @pytest.fixture(scope="function") @@ -812,7 +815,7 @@ def shitcoinmarkets(markets): "future": False, "active": True }, - }) + }) return shitmarkets @@ -914,18 +917,17 @@ def limit_sell_order_old(): @pytest.fixture def limit_buy_order_old_partial(): - return { - 'id': 'mocked_limit_buy_old_partial', - 'type': 'limit', - 'side': 'buy', - 'symbol': 'ETH/BTC', - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), - 'price': 0.00001099, - 'amount': 90.99181073, - 'filled': 23.0, - 'remaining': 67.99181073, - 'status': 'open' - } + return {'id': 'mocked_limit_buy_old_partial', + 'type': 'limit', + 'side': 'buy', + 'symbol': 'ETH/BTC', + 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'price': 0.00001099, + 'amount': 90.99181073, + 'filled': 23.0, + 'remaining': 67.99181073, + 'status': 'open' + } @pytest.fixture @@ -1728,13 +1730,14 @@ def rpc_balance(): 'total': 0.1, 'free': 0.01, 'used': 0.0 - }, + }, 'EUR': { 'total': 10.0, 'free': 10.0, 'used': 0.0 }, } + # TODO-mg: Add shorts and leverage? @pytest.fixture @@ -2049,3 +2052,95 @@ def saved_hyperopt_results(): ].total_seconds() return hyperopt_res + + +# * Margin Tests + +@pytest.fixture +def leveraged_fee(): + return + + +@pytest.fixture +def short_fee(): + return + + +@pytest.fixture +def ticker_short(): + return + + +@pytest.fixture +def ticker_exit_short_up(): + return + + +@pytest.fixture +def ticker_exit_short_down(): + return + + +@pytest.fixture +def leveraged_markets(): + return + + +@pytest.fixture(scope='function') +def limit_short_order_open(): + return + + +@pytest.fixture(scope='function') +def limit_short_order(limit_short_order_open): + return + + +@pytest.fixture(scope='function') +def market_short_order(): + return + + +@pytest.fixture +def market_short_exit_order(): + return + + +@pytest.fixture +def limit_short_order_old(): + return + + +@pytest.fixture +def limit_exit_short_order_old(): + return + + +@pytest.fixture +def limit_short_order_old_partial(): + return + + +@pytest.fixture +def limit_short_order_old_partial_canceled(limit_short_order_old_partial): + return + + +@pytest.fixture(scope='function') +def limit_short_order_canceled_empty(request): + return + + +@pytest.fixture +def limit_exit_short_order_open(): + return + + +@pytest.fixture +def limit_exit_short_order(limit_sell_order_open): + return + + +@pytest.fixture +def short_order_fee(): + return diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index b92b51144..de856a98d 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta, timezone from freqtrade.persistence.models import Order, Trade -MOCK_TRADE_COUNT = 6 +MOCK_TRADE_COUNT = 6 # TODO-mg: Increase for short and leverage def mock_order_1(): @@ -303,3 +303,5 @@ def mock_trade_6(fee): o = Order.parse_from_ccxt_object(mock_order_6_sell(), 'LTC/BTC', 'sell') trade.orders.append(o) return trade + +# TODO-mg: Mock orders for leveraged and short trades diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 1576aaa5a..c926ed2f8 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -129,6 +129,9 @@ def test_update_with_binance(limit_buy_order, limit_sell_order, fee, caplog): r"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=.*\).", caplog) + # TODO-mg: create a short order + # TODO-mg: create a leveraged long order + @pytest.mark.usefixtures("init_persistence") def test_update_market_order(market_buy_order, market_sell_order, fee, caplog): @@ -167,6 +170,9 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog): r"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=.*\).", caplog) + # TODO-mg: market short + # TODO-mg: market leveraged long + @pytest.mark.usefixtures("init_persistence") def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): @@ -1303,7 +1309,7 @@ def test_Trade_object_idem(): 'get_open_trades_without_assigned_fees', 'get_open_order_trades', 'get_trades', - ) + ) # Parent (LocalTrade) should have the same attributes for item in trade: From 85632eec05ddb6c62d16510ad1fecbdb453e502b Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 22 Jun 2021 02:22:16 -0600 Subject: [PATCH 12/16] A lot of the pytests pass now, 1562 pass, 3 fail --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence/models.py | 34 ++++++++++++++++++++++----------- tests/test_persistence.py | 2 +- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e8a321e94..1dc7de179 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -268,7 +268,7 @@ class FreqtradeBot(LoggingMixin): # Updating open orders in dry-run does not make sense and will fail. return - trades: List[Trade] = Trade.get_sold_trades_without_assigned_fees() + trades: List[Trade] = Trade.get_closed_trades_without_assigned_fees() for trade in trades: if not trade.is_open and not trade.fee_updated('sell'): diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 5e9862463..59d52bee0 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -287,6 +287,8 @@ class LocalTrade(): for key in kwargs: setattr(self, key, kwargs[key]) + if not self.is_short: + self.is_short = False self.recalc_open_trade_value() def __repr__(self): @@ -474,12 +476,12 @@ class LocalTrade(): self.recalc_open_trade_value() if self.is_open: payment = "SELL" if self.is_short else "BUY" - logger.info(f'{order_type.upper()}_{payment} order has been fulfilled for {self}.') + logger.info(f'{order_type.upper()}_{payment} has been fulfilled for {self}.') self.open_order_id = None elif order_type in ('market', 'limit') and self.is_closing_trade(order['side']): if self.is_open: payment = "BUY" if self.is_short else "SELL" - logger.info(f'{order_type.upper()}_{payment} order has been fulfilled for {self}.') + logger.info(f'{order_type.upper()}_{payment} has been fulfilled for {self}.') self.close(safe_value_fallback(order, 'average', 'price')) # TODO: Double check this elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'): self.stoploss_order_id = None @@ -576,10 +578,18 @@ class LocalTrade(): close_trade = Decimal(self.amount) * Decimal(rate or self.close_rate) # type: ignore fees = close_trade * Decimal(fee or self.fee_close) - # TODO: This interest rate is bad, doesn't get fractions of days - interest = ((Decimal(self.interest_rate) * Decimal(self.borrowed)) * - Decimal((datetime.utcnow() - self.open_date).days)) or 0 # Interest/day * num of days + # TODO: Need to set other conditions because sometimes self.open_date is not defined, but why would it ever not be set + try: + open = self.open_date.replace(tzinfo=None) + now = datetime.now() + + # breakpoint() + interest = ((Decimal(self.interest_rate or 0) * Decimal(self.borrowed or 0)) * + Decimal((now - open).total_seconds())/86400) or 0 # Interest/day * (seconds in trade)/(seconds per day) + except: + interest = 0 + if (self.is_short): return float(close_trade + fees + interest) else: @@ -619,15 +629,17 @@ class LocalTrade(): rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - if self.open_trade_value == 0.0: - return 0.0 if self.is_short: - profit_ratio = (close_trade_value / self.open_trade_value) - 1 - else: - if close_trade_value == 0: - profit_ratio = 0 + if close_trade_value == 0.0: + return 0.0 else: profit_ratio = (self.open_trade_value / close_trade_value) - 1 + + else: + if self.open_trade_value == 0.0: + return 0.0 + else: + profit_ratio = (close_trade_value / self.open_trade_value) - 1 return float(f"{profit_ratio:.8f}") def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]: diff --git a/tests/test_persistence.py b/tests/test_persistence.py index c926ed2f8..602446373 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1305,7 +1305,7 @@ def test_Trade_object_idem(): 'get_best_pair', 'get_overall_performance', 'total_open_trades_stakes', - 'get_sold_trades_without_assigned_fees', + 'get_closed_trades_without_assigned_fees', 'get_open_trades_without_assigned_fees', 'get_open_order_trades', 'get_trades', From 759f5636312c13c0765ac093443afcc185c7ab4d Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 22 Jun 2021 02:55:53 -0600 Subject: [PATCH 13/16] All previous pytests pass --- freqtrade/persistence/models.py | 8 ++++---- tests/rpc/test_rpc.py | 17 +++++++++++++++++ tests/test_persistence.py | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 59d52bee0..3b6b7aa3f 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -132,7 +132,7 @@ class Order(_DECL_BASE): order_filled_date = Column(DateTime, nullable=True) order_update_date = Column(DateTime, nullable=True) - leverage = Column(Float, nullable=True, default=0.0) + leverage = Column(Float, nullable=True, default=1.0) def __repr__(self): return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, ' @@ -263,7 +263,7 @@ class LocalTrade(): timeframe: Optional[int] = None # Margin trading properties - leverage: Optional[float] = 0.0 + leverage: Optional[float] = 1.0 borrowed: float = 0.0 borrowed_currency: str = None collateral_currency: str = None @@ -373,7 +373,7 @@ class LocalTrade(): 'collateral_currency': self.collateral_currency, 'interest_rate': self.interest_rate, 'liquidation_price': self.liquidation_price, - 'leverage': self.leverage, + 'is_short': self.is_short, 'open_order_id': self.open_order_id, } @@ -791,7 +791,7 @@ class Trade(_DECL_BASE, LocalTrade): timeframe = Column(Integer, nullable=True) # Margin trading properties - leverage = Column(Float, nullable=True, default=0.0) + leverage = Column(Float, nullable=True, default=1.0) borrowed = Column(Float, nullable=False, default=0.0) borrowed_currency = Column(Float, nullable=True) collateral_currency = Column(String(25), nullable=True) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 8a41099f4..1e7cfb102 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -107,6 +107,14 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'stoploss_entry_dist_ratio': -0.10448878, 'open_order': None, 'exchange': 'binance', + + 'leverage': 1.0, + 'borrowed': 0.0, + 'borrowed_currency': None, + 'collateral_currency': None, + 'interest_rate': 0.0, + 'liquidation_price': None, + 'is_short': False, } mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', @@ -173,6 +181,15 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'stoploss_entry_dist_ratio': -0.10448878, 'open_order': None, 'exchange': 'binance', + + 'leverage': 1.0, + 'borrowed': 0.0, + 'borrowed_currency': None, + 'collateral_currency': None, + 'interest_rate': 0.0, + 'liquidation_price': None, + 'is_short': False, + } diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 602446373..30798e60c 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -665,11 +665,13 @@ def test_migrate_new(mocker, default_conf, fee, caplog): order_date DATETIME, order_filled_date DATETIME, order_update_date DATETIME, + leverage FLOAT, PRIMARY KEY (id), CONSTRAINT _order_pair_order_id UNIQUE (ft_pair, order_id), FOREIGN KEY(ft_trade_id) REFERENCES trades (id) ) """)) + # TODO-mg: Had to add field leverage to this table, check that this is correct connection.execute(text(""" insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, @@ -918,6 +920,14 @@ def test_to_json(default_conf, fee): 'strategy': None, 'timeframe': None, 'exchange': 'binance', + + 'leverage': None, + 'borrowed': None, + 'borrowed_currency': None, + 'collateral_currency': None, + 'interest_rate': None, + 'liquidation_price': None, + 'is_short': None, } # Simulate dry_run entries @@ -983,6 +993,14 @@ def test_to_json(default_conf, fee): 'strategy': None, 'timeframe': None, 'exchange': 'binance', + + 'leverage': None, + 'borrowed': None, + 'borrowed_currency': None, + 'collateral_currency': None, + 'interest_rate': None, + 'liquidation_price': None, + 'is_short': None, } From 10e94350e9c9052af7ba869eb9812a1ca72a03e4 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Tue, 22 Jun 2021 14:59:43 +0200 Subject: [PATCH 14/16] Update installation.md --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 25994fdc0..5c6ac001f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -60,7 +60,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces sudo apt-get update # install packages - sudo apt install -y python3-pip python3.7-venv python3.7-dev python3-pandas git + sudo apt install -y python3-pip python3-venv python3-dev python3-pandas git ``` === "RaspberryPi/Raspbian" From c6a818eb17e2f5669802fa8e5ced9778eb42e445 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 22 Jun 2021 21:09:52 -0600 Subject: [PATCH 15/16] Created interest function --- freqtrade/persistence/models.py | 78 ++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 3b6b7aa3f..168cfa6d7 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -563,6 +563,42 @@ class LocalTrade(): """ self.open_trade_value = self._calc_open_trade_value() + def calculate_interest(self) -> Decimal: + # TODO-mg: Need to set other conditions because sometimes self.open_date is not defined, but why would it ever not be set + if not self.interest_rate or not (self.borrowed): + return Decimal(0.0) + + try: + open_date = self.open_date.replace(tzinfo=None) + now = datetime.now() + secPerDay = 86400 + days = Decimal((now - open_date).total_seconds()/secPerDay) or 0.0 + hours = days/24 + except: + raise OperationalException("Time isn't calculated properly") + + rate = Decimal(self.interest_rate) + borrowed = Decimal(self.borrowed) + + if self.exchange == 'binance': + # Rate is per day but accrued hourly or something + # binance: https://www.binance.com/en-AU/support/faq/360030157812 + return borrowed * (rate/24) * max(hours, 1.0) # TODO-mg: Is hours rounded? + elif self.exchange == 'kraken': + # https://support.kraken.com/hc/en-us/articles/206161568-What-are-the-fees-for-margin-trading- + opening_fee = borrowed * rate + roll_over_fee = borrowed * rate * max(0, (hours-4)/4) + return opening_fee + roll_over_fee + elif self.exchange == 'binance_usdm_futures': + # ! TODO-mg: This is incorrect, I didn't look it up + return borrowed * (rate/24) * max(hours, 1.0) + elif self.exchange == 'binance_coinm_futures': + # ! TODO-mg: This is incorrect, I didn't look it up + return borrowed * (rate/24) * max(hours, 1.0) + else: + # TODO-mg: make sure this breaks and can't be squelched + raise OperationalException("Leverage not available on this exchange") + def calc_close_trade_value(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: """ @@ -578,17 +614,7 @@ class LocalTrade(): close_trade = Decimal(self.amount) * Decimal(rate or self.close_rate) # type: ignore fees = close_trade * Decimal(fee or self.fee_close) - - # TODO: Need to set other conditions because sometimes self.open_date is not defined, but why would it ever not be set - try: - open = self.open_date.replace(tzinfo=None) - now = datetime.now() - - # breakpoint() - interest = ((Decimal(self.interest_rate or 0) * Decimal(self.borrowed or 0)) * - Decimal((now - open).total_seconds())/86400) or 0 # Interest/day * (seconds in trade)/(seconds per day) - except: - interest = 0 + interest = self.calculate_interest() if (self.is_short): return float(close_trade + fees + interest) @@ -657,7 +683,7 @@ class LocalTrade(): else: return None - @staticmethod + @ staticmethod def get_trades_proxy(*, pair: str = None, is_open: bool = None, open_date: datetime = None, close_date: datetime = None, ) -> List['LocalTrade']: @@ -691,27 +717,27 @@ class LocalTrade(): return sel_trades - @staticmethod + @ staticmethod def close_bt_trade(trade): LocalTrade.trades_open.remove(trade) LocalTrade.trades.append(trade) LocalTrade.total_profit += trade.close_profit_abs - @staticmethod + @ staticmethod def add_bt_trade(trade): if trade.is_open: LocalTrade.trades_open.append(trade) else: LocalTrade.trades.append(trade) - @staticmethod + @ staticmethod def get_open_trades() -> List[Any]: """ Query trades from persistence layer """ return Trade.get_trades_proxy(is_open=True) - @staticmethod + @ staticmethod def stoploss_reinitialization(desired_stoploss): """ Adjust initial Stoploss to desired stoploss for all open trades. @@ -812,11 +838,11 @@ class Trade(_DECL_BASE, LocalTrade): Trade.query.session.delete(self) Trade.commit() - @staticmethod + @ staticmethod def commit(): Trade.query.session.commit() - @staticmethod + @ staticmethod def get_trades_proxy(*, pair: str = None, is_open: bool = None, open_date: datetime = None, close_date: datetime = None, ) -> List['LocalTrade']: @@ -846,7 +872,7 @@ class Trade(_DECL_BASE, LocalTrade): close_date=close_date ) - @staticmethod + @ staticmethod def get_trades(trade_filter=None) -> Query: """ Helper function to query Trades using filters. @@ -866,7 +892,7 @@ class Trade(_DECL_BASE, LocalTrade): else: return Trade.query - @staticmethod + @ staticmethod def get_open_order_trades(): """ Returns all open trades @@ -874,7 +900,7 @@ class Trade(_DECL_BASE, LocalTrade): """ return Trade.get_trades(Trade.open_order_id.isnot(None)).all() - @staticmethod + @ staticmethod def get_open_trades_without_assigned_fees(): """ Returns all open trades which don't have open fees set correctly @@ -885,7 +911,7 @@ class Trade(_DECL_BASE, LocalTrade): Trade.is_open.is_(True), ]).all() - @staticmethod + @ staticmethod def get_closed_trades_without_assigned_fees(): """ Returns all closed trades which don't have fees set correctly @@ -896,7 +922,7 @@ class Trade(_DECL_BASE, LocalTrade): Trade.is_open.is_(False), ]).all() - @staticmethod + @ staticmethod def total_open_trades_stakes() -> float: """ Calculates total invested amount in open trades @@ -910,7 +936,7 @@ class Trade(_DECL_BASE, LocalTrade): t.stake_amount for t in LocalTrade.get_trades_proxy(is_open=True)) return total_open_stake_amount or 0 - @staticmethod + @ staticmethod def get_overall_performance() -> List[Dict[str, Any]]: """ Returns List of dicts containing all Trades, including profit and trade count @@ -935,7 +961,7 @@ class Trade(_DECL_BASE, LocalTrade): for pair, profit, profit_abs, count in pair_rates ] - @staticmethod + @ staticmethod def get_best_pair(): """ Get best pair with closed trade. @@ -973,7 +999,7 @@ class PairLock(_DECL_BASE): return (f'PairLock(id={self.id}, pair={self.pair}, lock_time={lock_time}, ' f'lock_end_time={lock_end_time})') - @staticmethod + @ staticmethod def query_pair_locks(pair: Optional[str], now: datetime) -> Query: """ Get all currently active locks for this pair From e00a93a92b2836bd4162464950e633c8df085226 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 22 Jun 2021 22:26:10 -0600 Subject: [PATCH 16/16] Started some pytests for short and leverage --- tests/conftest.py | 11 +++- tests/conftest_trades.py | 131 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e43ed5156..deabb2ac7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,7 +24,7 @@ from freqtrade.persistence import LocalTrade, Trade, init_db from freqtrade.resolvers import ExchangeResolver from freqtrade.worker import Worker from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4, - mock_trade_5, mock_trade_6) + mock_trade_5, mock_trade_6, short_trade, leverage_trade) logging.getLogger('').setLevel(logging.INFO) @@ -221,7 +221,12 @@ def create_mock_trades(fee, use_db: bool = True): trade = mock_trade_6(fee) add_trade(trade) - # TODO-mg: Add margin trades + # TODO: margin trades + # trade = short_trade(fee) + # add_trade(trade) + + # trade = leverage_trade(fee) + # add_trade(trade) if use_db: Trade.query.session.flush() @@ -252,7 +257,7 @@ def patch_coingekko(mocker) -> None: @pytest.fixture(scope='function') def init_persistence(default_conf): init_db(default_conf['db_url'], default_conf['dry_run']) - # TODO-mg: margin with leverage and/or borrowed? + # TODO-mg: trade with leverage and/or borrowed? @pytest.fixture(scope="function") diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index de856a98d..2aa1d6b4c 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -304,4 +304,133 @@ def mock_trade_6(fee): trade.orders.append(o) return trade -# TODO-mg: Mock orders for leveraged and short trades + +#! TODO Currently the following short_trade test and leverage_trade test will fail + + +def short_order(): + return { + 'id': '1235', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'sell', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + 'leverage': 5.0, + 'isShort': True + } + + +def exit_short_order(): + return { + 'id': '12366', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.128, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + 'leverage': 5.0, + 'isShort': True + } + + +def short_trade(fee): + """ + Closed trade... + """ + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, # TODO-mg: In BTC? + amount_requested=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + close_rate=0.128, + close_profit=0.005, # TODO-mg: Would this be -0.005 or -0.025 + close_profit_abs=0.000584127, + exchange='binance', + is_open=False, + open_order_id='dry_run_exit_short_12345', + strategy='DefaultStrategy', + timeframe=5, + sell_reason='sell_signal', # TODO-mg: Update to exit/close reason + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=2), + # borrowed= + isShort=True + ) + o = Order.parse_from_ccxt_object(short_order(), 'ETC/BTC', 'sell') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(exit_short_order(), 'ETC/BTC', 'sell') + trade.orders.append(o) + return trade + + +def leverage_order(): + return { + 'id': '1235', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + 'leverage': 5.0 + } + + +def leverage_order_sell(): + return { + 'id': '12366', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'sell', + 'type': 'limit', + 'price': 0.128, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + 'leverage': 5.0, + 'isShort': True + } + + +def leverage_trade(fee): + """ + Closed trade... + """ + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=615.0, + amount_requested=615.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + close_rate=0.128, + close_profit=0.005, # TODO-mg: Would this be -0.005 or -0.025 + close_profit_abs=0.000584127, + exchange='binance', + is_open=False, + open_order_id='dry_run_leverage_sell_12345', + strategy='DefaultStrategy', + timeframe=5, + sell_reason='sell_signal', # TODO-mg: Update to exit/close reason + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=2), + # borrowed= + ) + o = Order.parse_from_ccxt_object(leverage_order(), 'ETC/BTC', 'sell') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(leverage_order_sell(), 'ETC/BTC', 'sell') + trade.orders.append(o) + return trade