Adds migrations management

README.md

@@ -115,14 +115,14 @@ filesystem):
 
 ```
 $ cd ~/.freq
-$ touch tradesv3.sqlite
+$ touch trades.sqlite
 $ docker run -d \
   --name freqtrade \
   -v ~/.freq/config.json:/freqtrade/config.json \
-  -v ~/.freq/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \
+  -v ~/.freq/trades.sqlite:/freqtrade/trades.sqlite \
   freqtrade
 ```
-If you are using `dry_run=True` it's not necessary to mount `tradesv3.sqlite`.
+If you are using `dry_run=True` it's not necessary to mount `trades.sqlite`.
 
 You can then use the following commands to monitor and manage your container:
 

freqtrade/migrate.py

@@ -0,0 +1,43 @@
+"""
+This script manages the migrations
+"""
+from pathlib import Path
+import os
+import caribou
+
+DB_NAME = 'trades'
+DB_PATH = '{}.sqlite'.format(DB_NAME)
+
+PARENT_DIR = os.path.abspath('.')
+MIGRATIONS_PATH = PARENT_DIR + '/migrations/'
+INITIAL_MIGRATION = '20171111135341'
+
+def run():
+    """
+    Run Migrations
+    """
+    # Check if old version of db is present
+    db_path_v2 = PARENT_DIR + '/{}v2.sqlite'.format(DB_NAME)
+    db_path_v3 = PARENT_DIR + '/{}v3.sqlite'.format(DB_NAME)
+
+    # If we use old db file format v2 or v3, rename it to new db filename
+    if Path(db_path_v2).is_file():
+        os.rename(db_path_v2, DB_PATH)
+        # Executes initial migration (catching up)
+        caribou.upgrade(DB_PATH, MIGRATIONS_PATH, INITIAL_MIGRATION)
+    elif Path(db_path_v3).is_file():
+        os.rename(db_path_v3, DB_PATH)
+    else:
+        # Rename initial migration so it is not executed for next releases
+        files = [i for i in os.listdir(MIGRATIONS_PATH) if \
+            os.path.isfile(os.path.join(MIGRATIONS_PATH, i)) and \
+            INITIAL_MIGRATION in i and \
+            '.{}'.format(INITIAL_MIGRATION) not in i]
+        if files:
+            os.rename('{}{}'.format(MIGRATIONS_PATH, files[0]), \
+                '{}.{}'.format(MIGRATIONS_PATH, files[0]))
+
+    # If we have new db file (so from previous renaming or for the future)
+    if Path(DB_PATH).is_file():
+        # Execute migrations
+        caribou.upgrade(DB_PATH, MIGRATIONS_PATH)

freqtrade/persistence.py

@@ -34,7 +34,7 @@ def init(config: dict, engine: Optional[Engine] = None) -> None:
                                    poolclass=StaticPool,
                                    echo=False)
         else:
-            engine = create_engine('sqlite:///tradesv3.sqlite')
+            engine = create_engine('sqlite:///trades.sqlite')
 
     session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
     Trade.session = session()

migrations/20171111135341_v2-to-v3.py

@@ -0,0 +1,99 @@
+"""
+a caribou migration
+
+name: v2-to-v3
+version: 20171111135341
+"""
+
+def upgrade(connection):
+    # Rename old table
+    sql = """
+        ALTER TABLE trades RENAME TO trades_orig;
+        """
+    connection.execute(sql)
+
+    # Create new table
+    sql = """
+        CREATE TABLE trades (
+            id INTEGER NOT NULL,
+            exchange VARCHAR NOT NULL,
+            pair VARCHAR NOT NULL,
+            fee FLOAT NOT NULL DEFAULT 0.0,
+            is_open BOOLEAN NOT NULL,
+            open_rate FLOAT,
+            close_rate FLOAT,
+            close_profit FLOAT,
+            stake_amount FLOAT NOT NULL,
+            amount FLOAT,
+            open_date DATETIME NOT NULL,
+            close_date DATETIME,
+            open_order_id VARCHAR,
+            PRIMARY KEY (id),
+            CHECK (is_open IN (0, 1))
+        );
+        """
+    connection.execute(sql)
+
+    # Copy data from old table to the new one
+    sql = """
+        INSERT INTO trades(id, exchange, pair, is_open, open_rate, close_rate,
+        close_profit, stake_amount, amount, open_date, close_date, open_order_id)
+        SELECT id, exchange, pair, is_open, open_rate, close_rate, close_profit,
+        btc_amount, amount, open_date, close_date, open_order_id
+        FROM trades_orig;
+        """
+    connection.execute(sql)
+
+    # Remove old table
+    sql = """
+        DROP TABLE trades_orig;
+        """
+    connection.execute(sql)
+
+    connection.commit()
+
+
+def downgrade(connection):
+    sql = """
+        ALTER TABLE trades RENAME TO trades_orig;
+        """
+    connection.execute(sql)
+
+    # Create new table
+    sql = """
+        CREATE TABLE trades (
+            id INTEGER NOT NULL,
+            exchange VARCHAR NOT NULL,
+            pair VARCHAR NOT NULL,
+            is_open BOOLEAN NOT NULL,
+            open_rate FLOAT NOT NULL,
+            close_rate FLOAT,
+            close_profit FLOAT,
+            btc_amount FLOAT NOT NULL,
+            amount FLOAT NOT NULL,
+            open_date DATETIME NOT NULL,
+            close_date DATETIME,
+            open_order_id VARCHAR,
+            PRIMARY KEY (id),
+            CHECK (is_open IN (0, 1))
+        );
+        """
+    connection.execute(sql)
+
+    # Copy data from old table to the new one
+    sql = """
+        INSERT INTO trades(id, exchange, pair, is_open, open_rate, close_rate,
+        close_profit, btc_amount, amount, open_date, close_date, open_order_id)
+        SELECT id, exchange, pair, is_open, open_rate, close_rate, close_profit,
+        stake_amount, amount, open_date, close_date, open_order_id
+        FROM trades_orig;
+        """
+    connection.execute(sql)
+
+    # Remove old table
+    sql = """
+        DROP TABLE trades_orig;
+        """
+    connection.execute(sql)
+
+    connection.commit()

requirements.txt

@@ -19,6 +19,7 @@ hyperopt==0.1
 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325
 networkx==1.11
 tabulate==0.8.1
+caribou==0.2.1
 
 # Required for plotting data
 #matplotlib==2.1.0

setup.py

@@ -1,12 +1,20 @@
 from sys import version_info
 from setuptools import setup
+from setuptools.command.develop import develop
+from setuptools.command.install import install
 
 if version_info.major == 3 and version_info.minor < 6 or \
         version_info.major < 3:
     print('Your Python interpreter must be 3.6 or greater!')
     exit(1)
 
-from freqtrade import __version__
+from freqtrade import __version__, migrate
+
+class PostDevelopCommand(develop):
+    """Post-installation for development mode."""
+    def run(self):
+        migrate.run()
+        develop.run(self)
 
 
 setup(name='freqtrade',
@@ -35,6 +43,7 @@ setup(name='freqtrade',
           'TA-Lib',
           'tabulate',
           'cachetools',
+          'caribou',
       ],
       dependency_links=[
           "git+https://github.com/ericsomdahl/python-bittrex.git@0.2.0#egg=python-bittrex"
@@ -46,4 +55,7 @@ setup(name='freqtrade',
           'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
           'Topic :: Office/Business :: Financial :: Investment',
           'Intended Audience :: Science/Research',
-      ])
+      ],
+      cmdclass={
+          'develop': PostDevelopCommand,
+      })