Change history data filename and format

.gitignore

@@ -1,5 +1,5 @@
 # Freqtrade rules
-freqtrade/tests/testdata/*.json
+#freqtrade/tests/testdata/*.json
 hyperopt_conf.py
 config.json
 *.sqlite

Vagrantfile

@@ -0,0 +1,74 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "bento/ubuntu-16.04"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # NOTE: This will enable public access to the opened port
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine and only allow access
+  # via 127.0.0.1 to disable public access
+  # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  config.vm.synced_folder ".", "/vagrant/freqtrade"
+  config.vm.synced_folder ".env", "/vagrant/freqtrade/.env", disabled: true
+  config.vm.synced_folder ".hyperopt", "/vagrant/freqtrade/.hyperopt", disabled: true
+
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+     # Customize the amount of memory on the VM:
+     vb.memory = "4096"
+     vb.cpus = "10"
+   end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  # config.vm.provision "shell", inline: <<-SHELL
+  #   apt-get update
+  #   apt-get install -y apache2
+  # SHELL
+end

freqtrade/analyze.py

@@ -32,7 +32,7 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame:
         .rename(columns=columns)
     if 'BV' in frame:
         frame.drop('BV', 1, inplace=True)
-    frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True)
+    frame['date'] = to_datetime(frame['date'], unit='ms')
     frame.sort_values('date', inplace=True)
     return frame
 

freqtrade/exchange/__init__.py

@@ -187,4 +187,7 @@ def get_name() -> str:
 
 
 def get_fee() -> float:
+    if not _API.markets:
+        _API.load_markets()
+
     return _API.calculate_fee('ETH/BTC', '', '', 1, 1)['rate']

freqtrade/misc.py

@@ -271,9 +271,8 @@ def hyperopt_options(parser: argparse.ArgumentParser) -> None:
         '-i', '--ticker-interval',
         help='specify ticker interval in minutes (default: 5)',
         dest='ticker_interval',
-        default=5,
-        type=int,
-        metavar='INT',
+        default='5m',
+        type=str,
     )
     parser.add_argument(
         '--timerange',
@@ -336,7 +335,7 @@ CONF_SCHEMA = {
     'type': 'object',
     'properties': {
         'max_open_trades': {'type': 'integer', 'minimum': 1},
-        'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]},
+        'ticker_interval': {'type': 'string', 'enum': ['1m', '5m', '30m', '1h', '12h']},
         'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']},
         'stake_amount': {'type': 'number', 'minimum': 0.0005},
         'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF',

freqtrade/optimize/__init__.py

@@ -57,7 +57,7 @@ def load_tickerdata_file(datadir, pair, ticker_interval, timerange=None):
     return pairdata
 
 
-def load_data(datadir: str, ticker_interval: int, pairs: Optional[List[str]] = None,
+def load_data(datadir: str, ticker_interval: str, pairs: Optional[List[str]] = None,
               refresh_pairs: Optional[bool] = False, timerange=None) -> Dict[str, List]:
     """
     Loads ticker history data for the given parameters
@@ -102,7 +102,7 @@ def make_testdata_path(datadir: str) -> str:
                                                    '..', 'tests', 'testdata'))
 
 
-def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool:
+def download_pairs(datadir, pairs: List[str], ticker_interval: str) -> bool:
     """For each pairs passed in parameters, download the ticker intervals"""
     for pair in pairs:
         try:
@@ -122,7 +122,7 @@ def file_dump_json(filename, data):
 
 
 # FIX: 20180110, suggest rename interval to tick_interval
-def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool:
+def download_backtesting_testdata(datadir: str, pair: str, interval: str = '5m') -> bool:
     """
     Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
     Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
@@ -131,7 +131,7 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) ->
     """
 
     path = make_testdata_path(datadir)
-    logger.info('Download the pair: "{pair}", Interval: {interval} min'.format(
+    logger.info('Downloading the pair: "{pair}", Interval: {interval}'.format(
         pair=pair,
         interval=interval,
     ))
@@ -152,14 +152,21 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) ->
         logger.debug("Current Start: None")
         logger.debug("Current End: None")
 
-    new_data = get_ticker_history(pair=pair, tick_interval=int(interval))
-    for row in new_data:
-        if row not in data:
-            data.append(row)
-    logger.debug("New Start: {}".format(data[1]['T']))
-    logger.debug("New End: {}".format(data[-1:][0]['T']))
-    data = sorted(data, key=lambda data: data['T'])
+    new_data = get_ticker_history(pair=pair, tick_interval=interval)
+    data_json = []
+    for candlestick in new_data:
+        data_json.append({
+            'T': candlestick[0],
+            'O': candlestick[1],
+            'H': candlestick[2],
+            'L': candlestick[3],
+            'C': candlestick[4],
+            'V': candlestick[5],
+        })
+    logger.debug("New Start: {}".format(data_json[1]['T']))
+    logger.debug("New End: {}".format(data_json[-1]['T']))
+    data = sorted(data, key=lambda data_json: data_json['T'])
 
-    misc.file_dump_json(filename, data)
+    misc.file_dump_json(filename, data_json)
 
     return True

freqtrade/tests/optimize/test_optimize.py

@@ -51,7 +51,7 @@ def test_load_data_30min_ticker(default_conf, ticker_history, mocker, caplog):
 
     exchange._API = ccxt.binance({'key': '', 'secret': ''})
 
-    file = 'freqtrade/tests/testdata/UNITTEST_BTC-30.json'
+    file = 'freqtrade/tests/testdata/UNITTEST_BTC-30m.json'
     _backup_file(file, copy_file=True)
     optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval=30)
     assert os.path.isfile(file) is True

freqtrade/tests/testdata/BTC-XEL-1.json

@@ -1 +0,0 @@
-[{"O": 8.794e-05, "H": 8.948e-05, "L": 8.794e-05, "C": 8.88e-05, "V": 991.09056638, "T": "2017-11-26T08:50:00", "BV": 0.0877869}, {"O": 8.88e-05, "H": 8.942e-05, "L": 8.88e-05, "C": 8.893e-05, "V": 658.77935965, "T": "2017-11-26T08:55:00", "BV": 0.05874751}, {"O": 8.891e-05, "H": 8.893e-05, "L": 8.875e-05, "C": 8.877e-05, "V": 7920.73570705, "T": "2017-11-26T09:00:00", "BV": 0.7039405}]

freqtrade/tests/testdata/BTC_XEL-1.json

@@ -1 +0,0 @@
-[{"O": 8.794e-05, "H": 8.948e-05, "L": 8.794e-05, "C": 8.88e-05, "V": 991.09056638, "T": "2017-11-26T08:50:00", "BV": 0.0877869}, {"O": 8.88e-05, "H": 8.942e-05, "L": 8.88e-05, "C": 8.893e-05, "V": 658.77935965, "T": "2017-11-26T08:55:00", "BV": 0.05874751}, {"O": 8.891e-05, "H": 8.893e-05, "L": 8.875e-05, "C": 8.877e-05, "V": 7920.73570705, "T": "2017-11-26T09:00:00", "BV": 0.7039405}]

freqtrade/tests/testdata/MEME_BTC-1.json

@@ -1 +0,0 @@
-[{"O": 8.794e-05, "H": 8.948e-05, "L": 8.794e-05, "C": 8.88e-05, "V": 991.09056638, "T": "2017-11-26T08:50:00", "BV": 0.0877869}, {"O": 8.88e-05, "H": 8.942e-05, "L": 8.88e-05, "C": 8.893e-05, "V": 658.77935965, "T": "2017-11-26T08:55:00", "BV": 0.05874751}, {"O": 8.891e-05, "H": 8.893e-05, "L": 8.875e-05, "C": 8.877e-05, "V": 7920.73570705, "T": "2017-11-26T09:00:00", "BV": 0.7039405}]

freqtrade/tests/testdata/STORJ_BTC-5.json

@@ -1 +0,0 @@
-[{"O": 8.794e-05, "H": 8.948e-05, "L": 8.794e-05, "C": 8.88e-05, "V": 991.09056638, "T": "2017-11-26T08:50:00", "BV": 0.0877869}, {"O": 8.88e-05, "H": 8.942e-05, "L": 8.88e-05, "C": 8.893e-05, "V": 658.77935965, "T": "2017-11-26T08:55:00", "BV": 0.05874751}, {"O": 8.891e-05, "H": 8.893e-05, "L": 8.875e-05, "C": 8.877e-05, "V": 7920.73570705, "T": "2017-11-26T09:00:00", "BV": 0.7039405}]

freqtrade/tests/testdata/UNITTEST_BTC-8.json

@@ -1,3 +0,0 @@
-[
-  {"O": 0.00162008, "H": 0.00162008, "L": 0.00162008, "C": 0.00162008, "V": 108.14853839, "T": "2017-11-04T23:02:00", "BV": 0.17520927}
-]

freqtrade/tests/testdata/UNITTEST_BTC-8m.json

@@ -0,0 +1 @@
+[{"O": 0.00162008, "H": 0.00162008, "L": 0.00162008, "C": 0.00162008, "V": 108.14853839, "T": "2017-11-04T23:02:00.000000"}]

freqtrade/tests/testdata/download_backtest_data.py

@@ -4,6 +4,7 @@
 import sys
 import json
 import ccxt
+import datetime
 
 from freqtrade import exchange
 from freqtrade import misc
@@ -17,7 +18,7 @@ parser.add_argument(
 )
 args = parser.parse_args(sys.argv[1:])
 
-TICKER_INTERVALS = [1, 5]  # ticker interval in minutes (currently implemented: 1 and 5)
+TICKER_INTERVALS = ['1m', '5m']  # ticker interval in minutes (currently implemented: 1 and 5)
 PAIRS = []
 
 if args.pair:
@@ -36,4 +37,18 @@ for pair in PAIRS:
         data = exchange.get_ticker_history(pair, tick_interval)
         pair_print = pair.replace('/', '_')
         filename = '{}-{}.json'.format(pair_print, tick_interval)
-        misc.file_dump_json(filename, data)
+
+        data_json = []
+        for candlestick in data:
+            # Timestamp in unix milliseconds formatted
+            data_json.append({
+                'T': datetime.datetime.fromtimestamp(candlestick[0]/1000.0).strftime('%Y-%m-%dT%H:%M:%S.%f'),
+                'O': candlestick[1],
+                'H': candlestick[2],
+                'L': candlestick[3],
+                'C': candlestick[4],
+                'V': candlestick[5],
+            })
+        data_json = sorted(data_json, key=lambda d: d['T'])
+
+        misc.file_dump_json(filename, data_json)

freqtrade/tests/testdata/pairs.json

@@ -1,26 +1,8 @@
 [
     "ADA/BTC",
-    "BAT/BTC",
     "DASH/BTC",
-    "ETC/BTC",
     "ETH/BTC",
-    "GBYTE/BTC",
-    "LSK/BTC",
-    "LTC/BTC",
-    "NEO/BTC",
-    "NXT/BTC",
     "POWR/BTC",
-    "STORJ/BTC",
-    "QTUM/BTC",
-    "WAVES/BTC",
-    "VTC/BTC",
-    "XLM/BTC",
-    "XMR/BTC",
-    "XVG/BTC",
-    "XRP/BTC",
-    "ZEC/BTC",
-    "BTC/USDT",
-    "LTC/USDT",
-    "ETH/USDT"
+    "LTC/BTC"
 ]