From ada4847987c35c72393409574e28d30b3974ad56 Mon Sep 17 00:00:00 2001 From: Sebastien Moreau Date: Mon, 13 Nov 2017 07:30:32 -0500 Subject: [PATCH] Adds migrations management --- README.md | 6 +- freqtrade/migrate.py | 43 ++++++++++++ freqtrade/persistence.py | 2 +- migrations/20171111135341_v2-to-v3.py | 99 +++++++++++++++++++++++++++ requirements.txt | 1 + setup.py | 16 ++++- 6 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 freqtrade/migrate.py create mode 100644 migrations/20171111135341_v2-to-v3.py diff --git a/README.md b/README.md index 0eea256ae..62baa6f40 100644 --- a/README.md +++ b/README.md @@ -115,14 +115,14 @@ filesystem): ``` $ cd ~/.freq -$ touch tradesv3.sqlite +$ touch trades.sqlite $ docker run -d \ --name freqtrade \ -v ~/.freq/config.json:/freqtrade/config.json \ - -v ~/.freq/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ + -v ~/.freq/trades.sqlite:/freqtrade/trades.sqlite \ freqtrade ``` -If you are using `dry_run=True` it's not necessary to mount `tradesv3.sqlite`. +If you are using `dry_run=True` it's not necessary to mount `trades.sqlite`. You can then use the following commands to monitor and manage your container: diff --git a/freqtrade/migrate.py b/freqtrade/migrate.py new file mode 100644 index 000000000..c24cb58de --- /dev/null +++ b/freqtrade/migrate.py @@ -0,0 +1,43 @@ +""" +This script manages the migrations +""" +from pathlib import Path +import os +import caribou + +DB_NAME = 'trades' +DB_PATH = '{}.sqlite'.format(DB_NAME) + +PARENT_DIR = os.path.abspath('.') +MIGRATIONS_PATH = PARENT_DIR + '/migrations/' +INITIAL_MIGRATION = '20171111135341' + +def run(): + """ + Run Migrations + """ + # Check if old version of db is present + db_path_v2 = PARENT_DIR + '/{}v2.sqlite'.format(DB_NAME) + db_path_v3 = PARENT_DIR + '/{}v3.sqlite'.format(DB_NAME) + + # If we use old db file format v2 or v3, rename it to new db filename + if Path(db_path_v2).is_file(): + os.rename(db_path_v2, DB_PATH) + # Executes initial migration (catching up) + caribou.upgrade(DB_PATH, MIGRATIONS_PATH, INITIAL_MIGRATION) + elif Path(db_path_v3).is_file(): + os.rename(db_path_v3, DB_PATH) + else: + # Rename initial migration so it is not executed for next releases + files = [i for i in os.listdir(MIGRATIONS_PATH) if \ + os.path.isfile(os.path.join(MIGRATIONS_PATH, i)) and \ + INITIAL_MIGRATION in i and \ + '.{}'.format(INITIAL_MIGRATION) not in i] + if files: + os.rename('{}{}'.format(MIGRATIONS_PATH, files[0]), \ + '{}.{}'.format(MIGRATIONS_PATH, files[0])) + + # If we have new db file (so from previous renaming or for the future) + if Path(DB_PATH).is_file(): + # Execute migrations + caribou.upgrade(DB_PATH, MIGRATIONS_PATH) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index e48538754..2f059b9f0 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -34,7 +34,7 @@ def init(config: dict, engine: Optional[Engine] = None) -> None: poolclass=StaticPool, echo=False) else: - engine = create_engine('sqlite:///tradesv3.sqlite') + engine = create_engine('sqlite:///trades.sqlite') session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = session() diff --git a/migrations/20171111135341_v2-to-v3.py b/migrations/20171111135341_v2-to-v3.py new file mode 100644 index 000000000..4829b7bef --- /dev/null +++ b/migrations/20171111135341_v2-to-v3.py @@ -0,0 +1,99 @@ +""" +a caribou migration + +name: v2-to-v3 +version: 20171111135341 +""" + +def upgrade(connection): + # Rename old table + sql = """ + ALTER TABLE trades RENAME TO trades_orig; + """ + connection.execute(sql) + + # Create new table + sql = """ + CREATE TABLE trades ( + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + fee FLOAT NOT NULL DEFAULT 0.0, + is_open BOOLEAN NOT NULL, + open_rate FLOAT, + close_rate FLOAT, + close_profit FLOAT, + stake_amount FLOAT NOT NULL, + amount FLOAT, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) + ); + """ + connection.execute(sql) + + # Copy data from old table to the new one + sql = """ + INSERT INTO trades(id, exchange, pair, is_open, open_rate, close_rate, + close_profit, stake_amount, amount, open_date, close_date, open_order_id) + SELECT id, exchange, pair, is_open, open_rate, close_rate, close_profit, + btc_amount, amount, open_date, close_date, open_order_id + FROM trades_orig; + """ + connection.execute(sql) + + # Remove old table + sql = """ + DROP TABLE trades_orig; + """ + connection.execute(sql) + + connection.commit() + + +def downgrade(connection): + sql = """ + ALTER TABLE trades RENAME TO trades_orig; + """ + connection.execute(sql) + + # Create new table + sql = """ + CREATE TABLE trades ( + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + is_open BOOLEAN NOT NULL, + open_rate FLOAT NOT NULL, + close_rate FLOAT, + close_profit FLOAT, + btc_amount FLOAT NOT NULL, + amount FLOAT NOT NULL, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) + ); + """ + connection.execute(sql) + + # Copy data from old table to the new one + sql = """ + INSERT INTO trades(id, exchange, pair, is_open, open_rate, close_rate, + close_profit, btc_amount, amount, open_date, close_date, open_order_id) + SELECT id, exchange, pair, is_open, open_rate, close_rate, close_profit, + stake_amount, amount, open_date, close_date, open_order_id + FROM trades_orig; + """ + connection.execute(sql) + + # Remove old table + sql = """ + DROP TABLE trades_orig; + """ + connection.execute(sql) + + connection.commit() diff --git a/requirements.txt b/requirements.txt index 1ed05807b..925eb90e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.1 +caribou==0.2.1 # Required for plotting data #matplotlib==2.1.0 diff --git a/setup.py b/setup.py index 2c4a3f4c9..cd62b311e 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,20 @@ from sys import version_info from setuptools import setup +from setuptools.command.develop import develop +from setuptools.command.install import install if version_info.major == 3 and version_info.minor < 6 or \ version_info.major < 3: print('Your Python interpreter must be 3.6 or greater!') exit(1) -from freqtrade import __version__ +from freqtrade import __version__, migrate + +class PostDevelopCommand(develop): + """Post-installation for development mode.""" + def run(self): + migrate.run() + develop.run(self) setup(name='freqtrade', @@ -35,6 +43,7 @@ setup(name='freqtrade', 'TA-Lib', 'tabulate', 'cachetools', + 'caribou', ], dependency_links=[ "git+https://github.com/ericsomdahl/python-bittrex.git@0.2.0#egg=python-bittrex" @@ -46,4 +55,7 @@ setup(name='freqtrade', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 'Topic :: Office/Business :: Financial :: Investment', 'Intended Audience :: Science/Research', - ]) + ], + cmdclass={ + 'develop': PostDevelopCommand, + })