Update migrations to populate Orders table for open orders
This commit is contained in:
Matthias
parent
396e781bf4
commit
73182bb2dd
@ -5,6 +5,7 @@ from sqlalchemy import inspect
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_table_names_for_table(inspector, tabletype):
|
||||
return [t for t in inspector.get_table_names() if t.startswith(tabletype)]
|
||||
|
||||
@ -17,7 +18,110 @@ def get_column_def(columns: List, column: str, default: str) -> str:
|
||||
return default if not has_column(columns, column) else column
|
||||
|
||||
|
||||
def check_migrate(engine, decl_base) -> None:
|
||||
def get_backup_name(tabs, backup_prefix: str):
|
||||
table_back_name = backup_prefix
|
||||
for i, table_back_name in enumerate(tabs):
|
||||
table_back_name = f'{backup_prefix}{i}'
|
||||
logger.debug(f'trying {table_back_name}')
|
||||
|
||||
return table_back_name
|
||||
|
||||
|
||||
def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List):
|
||||
fee_open = get_column_def(cols, 'fee_open', 'fee')
|
||||
fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
|
||||
fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
|
||||
fee_close = get_column_def(cols, 'fee_close', 'fee')
|
||||
fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
|
||||
fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
|
||||
open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
|
||||
close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
|
||||
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
|
||||
stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
|
||||
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
|
||||
initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
|
||||
stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
|
||||
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
|
||||
max_rate = get_column_def(cols, 'max_rate', '0.0')
|
||||
min_rate = get_column_def(cols, 'min_rate', 'null')
|
||||
sell_reason = get_column_def(cols, 'sell_reason', 'null')
|
||||
strategy = get_column_def(cols, 'strategy', 'null')
|
||||
# If ticker-interval existed use that, else null.
|
||||
if has_column(cols, 'ticker_interval'):
|
||||
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
|
||||
else:
|
||||
timeframe = get_column_def(cols, 'timeframe', 'null')
|
||||
|
||||
open_trade_price = get_column_def(cols, 'open_trade_price',
|
||||
f'amount * open_rate * (1 + {fee_open})')
|
||||
close_profit_abs = get_column_def(
|
||||
cols, 'close_profit_abs',
|
||||
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
|
||||
sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
|
||||
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
|
||||
|
||||
# Schema migration necessary
|
||||
engine.execute(f"alter table trades rename to {table_back_name}")
|
||||
# drop indexes on backup table
|
||||
for index in inspector.get_indexes(table_back_name):
|
||||
engine.execute(f"drop index {index['name']}")
|
||||
# let SQLAlchemy create the schema as required
|
||||
decl_base.metadata.create_all(engine)
|
||||
|
||||
# Copy data back - following the correct schema
|
||||
engine.execute(f"""insert into trades
|
||||
(id, exchange, pair, is_open,
|
||||
fee_open, fee_open_cost, fee_open_currency,
|
||||
fee_close, fee_close_cost, fee_open_currency, open_rate,
|
||||
open_rate_requested, close_rate, close_rate_requested, close_profit,
|
||||
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
|
||||
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
|
||||
stoploss_order_id, stoploss_last_update,
|
||||
max_rate, min_rate, sell_reason, sell_order_status, strategy,
|
||||
timeframe, open_trade_price, close_profit_abs
|
||||
)
|
||||
select id, lower(exchange),
|
||||
case
|
||||
when instr(pair, '_') != 0 then
|
||||
substr(pair, instr(pair, '_') + 1) || '/' ||
|
||||
substr(pair, 1, instr(pair, '_') - 1)
|
||||
else pair
|
||||
end
|
||||
pair,
|
||||
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
|
||||
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
|
||||
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
|
||||
open_rate, {open_rate_requested} open_rate_requested, close_rate,
|
||||
{close_rate_requested} close_rate_requested, close_profit,
|
||||
stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
|
||||
{stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
|
||||
{initial_stop_loss} initial_stop_loss,
|
||||
{initial_stop_loss_pct} initial_stop_loss_pct,
|
||||
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
|
||||
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
|
||||
{sell_order_status} sell_order_status,
|
||||
{strategy} strategy, {timeframe} timeframe,
|
||||
{open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
|
||||
from {table_back_name}
|
||||
""")
|
||||
|
||||
|
||||
def migrate_open_orders_to_trades(engine):
|
||||
engine.execute("""
|
||||
insert into orders (trade_id, order_id, ft_order_side)
|
||||
select id, open_order_id,
|
||||
case when close_rate_requested is null then 'buy'
|
||||
else 'sell' end ft_order_side
|
||||
from trades
|
||||
where open_order_id is not null
|
||||
union all
|
||||
select id, stoploss_order_id, 'stoploss'
|
||||
from trades
|
||||
where stoploss_order_id is not null
|
||||
""")
|
||||
|
||||
|
||||
def check_migrate(engine, decl_base, previous_tables) -> None:
|
||||
"""
|
||||
Checks if migration is necessary and migrates if necessary
|
||||
"""
|
||||
@ -25,92 +129,21 @@ def check_migrate(engine, decl_base) -> None:
|
||||
|
||||
cols = inspector.get_columns('trades')
|
||||
tabs = get_table_names_for_table(inspector, 'trades')
|
||||
table_back_name = 'trades_bak'
|
||||
for i, table_back_name in enumerate(tabs):
|
||||
table_back_name = f'trades_bak{i}'
|
||||
logger.debug(f'trying {table_back_name}')
|
||||
table_back_name = get_backup_name(tabs, 'trades_bak')
|
||||
|
||||
# Check for latest column
|
||||
if not has_column(cols, 'amount_requested'):
|
||||
logger.info(f'Running database migration - backup available as {table_back_name}')
|
||||
|
||||
fee_open = get_column_def(cols, 'fee_open', 'fee')
|
||||
fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
|
||||
fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
|
||||
fee_close = get_column_def(cols, 'fee_close', 'fee')
|
||||
fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
|
||||
fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
|
||||
open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
|
||||
close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
|
||||
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
|
||||
stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
|
||||
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
|
||||
initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
|
||||
stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
|
||||
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
|
||||
max_rate = get_column_def(cols, 'max_rate', '0.0')
|
||||
min_rate = get_column_def(cols, 'min_rate', 'null')
|
||||
sell_reason = get_column_def(cols, 'sell_reason', 'null')
|
||||
strategy = get_column_def(cols, 'strategy', 'null')
|
||||
# If ticker-interval existed use that, else null.
|
||||
if has_column(cols, 'ticker_interval'):
|
||||
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
|
||||
else:
|
||||
timeframe = get_column_def(cols, 'timeframe', 'null')
|
||||
|
||||
open_trade_price = get_column_def(cols, 'open_trade_price',
|
||||
f'amount * open_rate * (1 + {fee_open})')
|
||||
close_profit_abs = get_column_def(
|
||||
cols, 'close_profit_abs',
|
||||
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
|
||||
sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
|
||||
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
|
||||
|
||||
# Schema migration necessary
|
||||
engine.execute(f"alter table trades rename to {table_back_name}")
|
||||
# drop indexes on backup table
|
||||
for index in inspector.get_indexes(table_back_name):
|
||||
engine.execute(f"drop index {index['name']}")
|
||||
# let SQLAlchemy create the schema as required
|
||||
decl_base.metadata.create_all(engine)
|
||||
|
||||
# Copy data back - following the correct schema
|
||||
engine.execute(f"""insert into trades
|
||||
(id, exchange, pair, is_open,
|
||||
fee_open, fee_open_cost, fee_open_currency,
|
||||
fee_close, fee_close_cost, fee_open_currency, open_rate,
|
||||
open_rate_requested, close_rate, close_rate_requested, close_profit,
|
||||
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
|
||||
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
|
||||
stoploss_order_id, stoploss_last_update,
|
||||
max_rate, min_rate, sell_reason, sell_order_status, strategy,
|
||||
timeframe, open_trade_price, close_profit_abs
|
||||
)
|
||||
select id, lower(exchange),
|
||||
case
|
||||
when instr(pair, '_') != 0 then
|
||||
substr(pair, instr(pair, '_') + 1) || '/' ||
|
||||
substr(pair, 1, instr(pair, '_') - 1)
|
||||
else pair
|
||||
end
|
||||
pair,
|
||||
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
|
||||
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
|
||||
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
|
||||
open_rate, {open_rate_requested} open_rate_requested, close_rate,
|
||||
{close_rate_requested} close_rate_requested, close_profit,
|
||||
stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
|
||||
{stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
|
||||
{initial_stop_loss} initial_stop_loss,
|
||||
{initial_stop_loss_pct} initial_stop_loss_pct,
|
||||
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
|
||||
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
|
||||
{sell_order_status} sell_order_status,
|
||||
{strategy} strategy, {timeframe} timeframe,
|
||||
{open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
|
||||
from {table_back_name}
|
||||
""")
|
||||
|
||||
logger.info(f'Running database migration for trades - backup: {table_back_name}')
|
||||
migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
|
||||
# Reread columns - the above recreated the table!
|
||||
inspector = inspect(engine)
|
||||
cols = inspector.get_columns('trades')
|
||||
|
||||
if 'orders' not in previous_tables:
|
||||
logger.info('Moving open orders to Orders table.')
|
||||
migrate_open_orders_to_trades(engine)
|
||||
else:
|
||||
logger.info(f'Running database migration for orders - backup: {table_back_name}')
|
||||
pass
|
||||
# Empty for now - as there is only one iteration of the orders table so far.
|
||||
# table_back_name = get_backup_name(tabs, 'orders_bak')
|
||||
|
||||
@ -7,8 +7,8 @@ from decimal import Decimal
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import arrow
|
||||
from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, ForeignKey,
|
||||
create_engine, desc, func)
|
||||
from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer,
|
||||
String, create_engine, desc, func, inspect)
|
||||
from sqlalchemy.exc import NoSuchModuleError
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import Query, relationship
|
||||
@ -61,9 +61,9 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
|
||||
# Copy session attributes to order object too
|
||||
Order.session = Trade.session
|
||||
Order.query = Order.session.query_property()
|
||||
|
||||
previous_tables = inspect(engine).get_table_names()
|
||||
_DECL_BASE.metadata.create_all(engine)
|
||||
check_migrate(engine, decl_base=_DECL_BASE)
|
||||
check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
|
||||
|
||||
# Clean dry_run DB if the db is not in-memory
|
||||
if clean_open_orders and db_url != 'sqlite://':
|
||||
@ -117,7 +117,7 @@ class Order(_DECL_BASE):
|
||||
filled = Column(Float, nullable=True)
|
||||
remaining = Column(Float, nullable=True)
|
||||
cost = Column(Float, nullable=True)
|
||||
order_date = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
order_date = Column(DateTime, nullable=True, default=datetime.utcnow)
|
||||
order_filled_date = Column(DateTime, nullable=True)
|
||||
order_update_date = Column(DateTime, nullable=True)
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@ from sqlalchemy import create_engine
|
||||
|
||||
from freqtrade import constants
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.persistence import Trade, clean_dry_run_db, init
|
||||
from freqtrade.persistence import Trade, Order, clean_dry_run_db, init
|
||||
from tests.conftest import log_has, create_mock_trades
|
||||
|
||||
|
||||
@ -421,9 +421,9 @@ def test_migrate_old(mocker, default_conf, fee):
|
||||
PRIMARY KEY (id),
|
||||
CHECK (is_open IN (0, 1))
|
||||
);"""
|
||||
insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
|
||||
insert_table_old = """INSERT INTO trades (exchange, pair, is_open, open_order_id, fee,
|
||||
open_rate, stake_amount, amount, open_date)
|
||||
VALUES ('BITTREX', 'BTC_ETC', 1, {fee},
|
||||
VALUES ('BITTREX', 'BTC_ETC', 1, '123123', {fee},
|
||||
0.00258580, {stake}, {amount},
|
||||
'2017-11-28 12:44:24.000000')
|
||||
""".format(fee=fee.return_value,
|
||||
@ -481,6 +481,12 @@ def test_migrate_old(mocker, default_conf, fee):
|
||||
assert pytest.approx(trade.close_profit_abs) == trade.calc_profit()
|
||||
assert trade.sell_order_status is None
|
||||
|
||||
# Should've created one order
|
||||
assert len(Order.query.all()) == 1
|
||||
order = Order.query.first()
|
||||
assert order.order_id == '123123'
|
||||
assert order.ft_order_side == 'buy'
|
||||
|
||||
|
||||
def test_migrate_new(mocker, default_conf, fee, caplog):
|
||||
"""
|
||||
@ -509,16 +515,19 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
|
||||
sell_reason VARCHAR,
|
||||
strategy VARCHAR,
|
||||
ticker_interval INTEGER,
|
||||
stoploss_order_id VARCHAR,
|
||||
PRIMARY KEY (id),
|
||||
CHECK (is_open IN (0, 1))
|
||||
);"""
|
||||
insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
|
||||
open_rate, stake_amount, amount, open_date,
|
||||
stop_loss, initial_stop_loss, max_rate, ticker_interval)
|
||||
stop_loss, initial_stop_loss, max_rate, ticker_interval,
|
||||
open_order_id, stoploss_order_id)
|
||||
VALUES ('binance', 'ETC/BTC', 1, {fee},
|
||||
0.00258580, {stake}, {amount},
|
||||
'2019-11-28 12:44:24.000000',
|
||||
0.0, 0.0, 0.0, '5m')
|
||||
0.0, 0.0, 0.0, '5m',
|
||||
'buy_order', 'stop_order_id222')
|
||||
""".format(fee=fee.return_value,
|
||||
stake=default_conf.get("stake_amount"),
|
||||
amount=amount
|
||||
@ -558,14 +567,23 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
|
||||
assert trade.sell_reason is None
|
||||
assert trade.strategy is None
|
||||
assert trade.timeframe == '5m'
|
||||
assert trade.stoploss_order_id is None
|
||||
assert trade.stoploss_order_id == 'stop_order_id222'
|
||||
assert trade.stoploss_last_update is None
|
||||
assert log_has("trying trades_bak1", caplog)
|
||||
assert log_has("trying trades_bak2", caplog)
|
||||
assert log_has("Running database migration - backup available as trades_bak2", caplog)
|
||||
assert log_has("Running database migration for trades - backup: trades_bak2", caplog)
|
||||
assert trade.open_trade_price == trade._calc_open_trade_price()
|
||||
assert trade.close_profit_abs is None
|
||||
|
||||
assert log_has("Moving open orders to Orders table.", caplog)
|
||||
orders = Order.query.all()
|
||||
assert len(orders) == 2
|
||||
assert orders[0].order_id == 'buy_order'
|
||||
assert orders[0].ft_order_side == 'buy'
|
||||
|
||||
assert orders[1].order_id == 'stop_order_id222'
|
||||
assert orders[1].ft_order_side == 'stoploss'
|
||||
|
||||
|
||||
def test_migrate_mid_state(mocker, default_conf, fee, caplog):
|
||||
"""
|
||||
@ -626,7 +644,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
|
||||
assert trade.initial_stop_loss == 0.0
|
||||
assert trade.open_trade_price == trade._calc_open_trade_price()
|
||||
assert log_has("trying trades_bak0", caplog)
|
||||
assert log_has("Running database migration - backup available as trades_bak0", caplog)
|
||||
assert log_has("Running database migration for trades - backup: trades_bak0", caplog)
|
||||
|
||||
|
||||
def test_adjust_stop_loss(fee):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user