diff --git a/Dockerfile b/Dockerfile index 309763d2a..5d1b44f8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,11 @@ -FROM python:3.6.6-slim-stretch +FROM python:3.7.0-slim-stretch # Install TA-lib RUN apt-get update && apt-get -y install curl build-essential && apt-get clean RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \ tar xzvf - && \ cd ta-lib && \ + sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ ./configure && make && make install && \ cd .. && rm -rf ta-lib ENV LD_LIBRARY_PATH /usr/local/lib diff --git a/config.json.example b/config.json.example index 8bd3942e6..7a0bb6b9b 100644 --- a/config.json.example +++ b/config.json.example @@ -11,7 +11,18 @@ "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0 + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 }, "exchange": { "name": "bittrex", diff --git a/config_full.json.example b/config_full.json.example index cc3b3d630..7083bada6 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -20,7 +20,18 @@ "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0 + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 }, "exchange": { "name": "bittrex", @@ -41,7 +52,8 @@ ], "pair_blacklist": [ "DOGE/BTC" - ] + ], + "outdated_offset": 5 }, "experimental": { "use_sell_signal": false, diff --git a/docs/configuration.md b/docs/configuration.md index 1d5fc0922..4674f4b40 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -26,12 +26,19 @@ The table below will list all configuration parameters. | `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. -| `trailing_stoploss` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). -| `trailing_stoploss_positve` | 0 | No | Changes stop-loss once profit has been reached. -| `trailing_stoploss_positve_offset` | 0 | No | Offset on when to apply `trailing_stoploss_positive`. Percentage value which should be positive. +| `trailing_stop` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). +| `trailing_stop_positve` | 0 | No | Changes stop-loss once profit has been reached. +| `trailing_stop_positve_offset` | 0 | No | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. | `unfilledtimeout.buy` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. | `unfilledtimeout.sell` | 10 | Yes | How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. +| `bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids. +| `bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. +| `bid_strategy.check_depth_of_market.enabled` | false | No | Does not buy if the % difference of buy orders and sell orders is met in Order Book. +| `bid_strategy.check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. +| `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. +| `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. +| `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. diff --git a/docs/installation.md b/docs/installation.md index 7a7719fc0..0fecfcf78 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,7 +8,6 @@ To understand how to set up the bot please read the [Bot Configuration](https:// * [Table of Contents](#table-of-contents) * [Easy Installation - Linux Script](#easy-installation---linux-script) -* [Manual installation](#manual-installation) * [Automatic Installation - Docker](#automatic-installation---docker) * [Custom Linux MacOS Installation](#custom-installation) - [Requirements](#requirements) @@ -56,34 +55,6 @@ Reset parameter will hard reset your branch (only if you are on `master` or `dev Config parameter is a `config.json` configurator. This script will ask you questions to setup your bot and create your `config.json`. -## Manual installation - Linux/MacOS - -The following steps are made for Linux/MacOS environment - -### 1. Clone the repo - -```bash -git clone git@github.com:freqtrade/freqtrade.git -git checkout develop -cd freqtrade -``` - -### 2. Create the config file - -Switch `"dry_run": true,` - -```bash -cp config.json.example config.json -vi config.json -``` - -### 3. Build your docker image and run it - -```bash -docker build -t freqtrade . -docker run --rm -v /etc/localtime:/etc/localtime:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` - ------ ## Automatic Installation - Docker @@ -196,7 +167,7 @@ docker run -d \ freqtrade --db-url sqlite:///tradesv3.sqlite ``` -NOTE: db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. +*Note*: db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` ### 6. Monitor your Docker instance @@ -211,14 +182,15 @@ docker stop freqtrade docker start freqtrade ``` -You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. +For more information on how to operate Docker, please refer to the [official Docker documentation](https://docs.docker.com/). + +*Note*: You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. ### 7. Backtest with docker The following assumes that the above steps (1-4) have been completed successfully. Also, backtest-data should be available at `~/.freqtrade/user_data/`. - ``` bash docker run -d \ --name freqtrade \ @@ -238,12 +210,13 @@ Head over to the [Backtesting Documentation](https://github.com/freqtrade/freqtr ## Custom Installation We've included/collected install instructions for Ubuntu 16.04, MacOS, and Windows. These are guidelines and your success may vary with other distros. +OS Specific steps are listed first, the [common](#common) section below is necessary for all systems. ### Requirements Click each one for install guide: -* [Python 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/), note the bot was not tested on Python >= 3.7.x +* [Python >= 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/) * [pip](https://pip.pypa.io/en/stable/installing/) * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * [virtualenv](https://virtualenv.pypa.io/en/stable/installation/) (Recommended) @@ -251,7 +224,7 @@ Click each one for install guide: ### Linux - Ubuntu 16.04 -#### 1. Install Python 3.6, Git, and wget +#### Install Python 3.6, Git, and wget ```bash sudo add-apt-repository ppa:jonathonf/python-3.6 @@ -259,7 +232,34 @@ sudo apt-get update sudo apt-get install python3.6 python3.6-venv python3.6-dev build-essential autoconf libtool pkg-config make wget git ``` -#### 2. Install TA-Lib +#### Raspberry Pi / Raspbian + +Before installing FreqTrade on a Raspberry Pi running the official Raspbian Image, make sure you have at least Python 3.6 installed. The default image only provides Python 3.5. Probably the easiest way to get a recent version of python is [miniconda](https://repo.continuum.io/miniconda/). + +The following assumes that miniconda3 is installed and available in your environment, and is installed. +It's recommended to use (mini)conda for this as installation/compilation of `scipy` and `pandas` takes a long time. + +``` bash +conda config --add channels rpi +conda install python=3.6 +conda create -n freqtrade python=3.6 +conda install scipy pandas + +pip install -r requirements.txt +pip install -e . +``` + +### MacOS + +#### Install Python 3.6, git, wget and ta-lib + +```bash +brew install python3 git wget +``` + +### common + +#### 1. Install TA-Lib Official webpage: https://mrjbq7.github.io/ta-lib/install.html @@ -267,6 +267,7 @@ Official webpage: https://mrjbq7.github.io/ta-lib/install.html wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar xvzf ta-lib-0.4.0-src.tar.gz cd ta-lib +sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h ./configure --prefix=/usr make make install @@ -274,15 +275,60 @@ cd .. rm -rf ./ta-lib* ``` +*Note*: An already downloaded version of ta-lib is included in the repository, as the sourceforge.net source seems to have problems frequently. + +#### 2. Setup your Python virtual environment (virtualenv) + +*Note*: This step is optional but strongly recommended to keep your system organized + +```bash +python3 -m venv .env +source .env/bin/activate +``` + #### 3. Install FreqTrade Clone the git repository: ```bash git clone https://github.com/freqtrade/freqtrade.git + ``` -#### 4. Configure `freqtrade` as a `systemd` service +Optionally checkout the stable/master branch: + +```bash +git checkout master +``` + +#### 4. Initialize the configuration + +```bash +cd freqtrade +cp config.json.example config.json +``` + +> *To edit the config please refer to [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md).* + +#### 5. Install python dependencies + +``` bash +pip3 install --upgrade pip +pip3 install -r requirements.txt +pip3 install -e . +``` + +#### 6. Run the Bot + +If this is the first time you run the bot, ensure you are running it in Dry-run `"dry_run": true,` otherwise it will start to buy and sell coins. + +```bash +python3.6 ./freqtrade/main.py -c config.json +``` + +*Note*: If you run the bot on a server, you should consider using [Docker](#automatic-installation---docker) a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. + +#### 7. [Optional] Configure `freqtrade` as a `systemd` service From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. @@ -298,57 +344,6 @@ For this to be persistent (run when user is logged out) you'll need to enable `l sudo loginctl enable-linger "$USER" ``` -### MacOS - -#### 1. Install Python 3.6, git, wget and ta-lib - -```bash -brew install python3 git wget ta-lib -``` - -#### 2. Install FreqTrade - -Clone the git repository: - -```bash -git clone https://github.com/freqtrade/freqtrade.git -``` - -Optionally checkout the develop branch: - -```bash -git checkout develop -``` - -### Setup Config and virtual env - -#### 1. Initialize the configuration - -```bash -cd freqtrade -cp config.json.example config.json -``` - -> *To edit the config please refer to [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md).* - -#### 2. Setup your Python virtual environment (virtualenv) - -```bash -python3.6 -m venv .env -source .env/bin/activate -pip3.6 install --upgrade pip -pip3.6 install -r requirements.txt -pip3.6 install -e . -``` - -#### 3. Run the Bot - -If this is the first time you run the bot, ensure you are running it in Dry-run `"dry_run": true,` otherwise it will start to buy and sell coins. - -```bash -python3.6 ./freqtrade/main.py -c config.json -``` - ------ ## Windows @@ -368,7 +363,7 @@ git clone https://github.com/freqtrade/freqtrade.git copy paste `config.json` to ``\path\freqtrade-develop\freqtrade` -#### install ta-lib +#### Install ta-lib Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). @@ -389,5 +384,17 @@ REM >pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl > Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222) +#### Error during installation under Windows + +``` bash +error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools +``` + +Unfortunately, many packages requiring compilation don't provide a pre-build wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use. + +The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or docker first. + +--- + Now you have an environment ready, the next step is [Bot Configuration](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md)... diff --git a/docs/sandbox-testing.md b/docs/sandbox-testing.md index 572fbccef..7f3457d15 100644 --- a/docs/sandbox-testing.md +++ b/docs/sandbox-testing.md @@ -1,4 +1,5 @@ # Sandbox API testing + Where an exchange provides a sandbox for risk-free integration, or end-to-end, testing CCXT provides access to these. This document is a *light overview of configuring Freqtrade and GDAX sandbox. @@ -11,8 +12,11 @@ https://public.sandbox.gdax.com https://api-public.sandbox.gdax.com --- + # Configure a Sandbox account on Gdax + Aim of this document section + - An sanbox account - create 2FA (needed to create an API) - Add test 50BTC to account @@ -30,122 +34,108 @@ After registration and Email confimation you wil be redirected into your sanbox > https://public.sandbox.pro.coinbase.com/ ## Enable 2Fa (a prerequisite to creating sandbox API Keys) + From within sand box site select your profile, top right. >Or as a direct link: https://public.sandbox.pro.coinbase.com/profile -From the menu panel to the left of the screen select +From the menu panel to the left of the screen select + > Security: "*View or Update*" -In the new site select "enable authenticator" as typical google Authenticator. -- open Google Authenticator on your phone -- scan barcode -- enter your generated 2fa +In the new site select "enable authenticator" as typical google Authenticator. + +- open Google Authenticator on your phone +- scan barcode +- enter your generated 2fa + +## Enable API Access -## Enable API Access From within sandbox select profile>api>create api-keys >or as a direct link: https://public.sandbox.pro.coinbase.com/profile/api -Click on "create one" and ensure **view** and **trade** are "checked" and sumbit your 2Fa +Click on "create one" and ensure **view** and **trade** are "checked" and sumbit your 2FA + - **Copy and paste the Passphase** into a notepade this will be needed later - **Copy and paste the API Secret** popup into a notepad this will needed later - **Copy and paste the API Key** into a notepad this will needed later ## Add 50 BTC test funds -To add funds, use the web interface deposit and withdraw buttons. +To add funds, use the web interface deposit and withdraw buttons. To begin select 'Wallets' from the top menu. > Or as a direct link: https://public.sandbox.pro.coinbase.com/wallets - Deposits (bottom left of screen) -- - Deposit Funds Bitcoin -- - - Coinbase BTC Wallet -- - - - Max (50 BTC) +- - Deposit Funds Bitcoin +- - - Coinbase BTC Wallet +- - - - Max (50 BTC) - - - - - Deposit *This process may be repeated for other currencies, ETH as example* + --- + # Configure Freqtrade to use Gax Sandbox The aim of this document section - - Enable sandbox URLs in Freqtrade - - Configure API - - - secret - - - key - - - passphrase + +- Enable sandbox URLs in Freqtrade +- Configure API +- - secret +- - key +- - passphrase ## Sandbox URLs -Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade. -These include `['test']` and `['api']`. + +Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade. +These include `['test']` and `['api']`. + - `[Test]` if available will point to an Exchanges sandbox. - `[Api]` normally used, and resolves to live API target on the exchange To make use of sandbox / test add "sandbox": true, to your config.json -``` + +```json "exchange": { "name": "gdax", "sandbox": true, "key": "5wowfxemogxeowo;heiohgmd", "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==", "password": "1bkjfkhfhfu6sr", + "outdated_offset": 5 "pair_whitelist": [ "BTC/USD" ``` + Also insert your + - api-key (noted earlier) - api-secret (noted earlier) - password (the passphrase - noted earlier) --- -## You should now be ready to test your sandbox! + +## You should now be ready to test your sandbox + Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox. -** Typically the BTC/USD has the most activity in sandbox to test against. +** Typically the BTC/USD has the most activity in sandbox to test against. ## GDAX - Old Candles problem -It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks -To disable this check, edit: ->strategy/interface.py -Look for the following section: -``` - # Check if dataframe is out of date - signal_date = arrow.get(latest['date']) - interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): - logger.warning( - 'Outdated history for pair %s. Last tick is %s minutes old', - pair, - (arrow.utcnow() - signal_date).seconds // 60 - ) - return False, False -``` +It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks. -You could Hash out the entire check as follows: +To disable this check, add / change the `"outdated_offset"` parameter in the exchange section of your configuration to adjust for this delay. +Example based on the above configuration: + +```json + "exchange": { + "name": "gdax", + "sandbox": true, + "key": "5wowfxemogxeowo;heiohgmd", + "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==", + "password": "1bkjfkhfhfu6sr", + "outdated_offset": 30 + "pair_whitelist": [ + "BTC/USD" ``` - # # Check if dataframe is out of date - # signal_date = arrow.get(latest['date']) - # interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - # if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): - # logger.warning( - # 'Outdated history for pair %s. Last tick is %s minutes old', - # pair, - # (arrow.utcnow() - signal_date).seconds // 60 - # ) - # return False, False - ``` - - Or inrease the timeout to offer a level of protection/alignment of this test to freqtrade in live. - - As example, to allow an additional 30 minutes. "(interval_minutes * 2 + 5 + 30)" - ``` - # Check if dataframe is out of date - signal_date = arrow.get(latest['date']) - interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5 + 30))): - logger.warning( - 'Outdated history for pair %s. Last tick is %s minutes old', - pair, - (arrow.utcnow() - signal_date).seconds // 60 - ) - return False, False -``` \ No newline at end of file diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index ac00264f0..46825f548 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.17.1' +__version__ = '0.17.2' class DependencyException(BaseException): diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 92b090432..eadfa6eba 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -36,7 +36,7 @@ SUPPORTED_FIAT = [ "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", - "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" + "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" ] # Required json-schema for user specified config @@ -45,7 +45,7 @@ CONF_SCHEMA = { 'properties': { 'max_open_trades': {'type': 'integer', 'minimum': 0}, 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT', 'EUR', 'USD']}, + 'stake_currency': {'type': 'string', 'enum': ['BTC', 'XBT', 'ETH', 'USDT', 'EUR', 'USD']}, 'stake_amount': { "type": ["number", "string"], "minimum": 0.0005, @@ -79,18 +79,35 @@ CONF_SCHEMA = { 'type': 'number', 'minimum': 0, 'maximum': 1, - 'exclusiveMaximum': False + 'exclusiveMaximum': False, + 'use_order_book': {'type': 'boolean'}, + 'order_book_top': {'type': 'number', 'maximum': 20, 'minimum': 1}, + 'check_depth_of_market': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, + } + }, }, }, 'required': ['ask_last_balance'] }, + 'ask_strategy': { + 'type': 'object', + 'properties': { + 'use_order_book': {'type': 'boolean'}, + 'order_book_min': {'type': 'number', 'minimum': 1}, + 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50} + } + }, 'exchange': {'$ref': '#/definitions/exchange'}, 'experimental': { 'type': 'object', 'properties': { 'use_sell_signal': {'type': 'boolean'}, 'sell_profit_only': {'type': 'boolean'}, - "ignore_roi_if_buy_signal_true": {'type': 'boolean'} + 'ignore_roi_if_buy_signal_true': {'type': 'boolean'} } }, 'telegram': { @@ -146,7 +163,8 @@ CONF_SCHEMA = { 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' }, 'uniqueItems': True - } + }, + 'outdated_offset': {'type': 'integer', 'minimum': 1} }, 'required': ['name', 'key', 'secret', 'pair_whitelist'] } diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index a6ec70636..e7f94c499 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -116,7 +116,7 @@ class Exchange(object): api.urls['api'] = api.urls['test'] logger.info("Enabled Sandbox API on %s", name) else: - logger.warning(self, "No Sandbox URL in CCXT, exiting. " + logger.warning(name, "No Sandbox URL in CCXT, exiting. " "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') @@ -322,7 +322,7 @@ class Exchange(object): return data except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( - f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') + f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') except ccxt.BaseError as e: raise OperationalException(e) else: @@ -409,6 +409,37 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) + @retrier + def get_order_book(self, pair: str, limit: int = 100) -> dict: + """ + get order book level 2 from exchange + + Notes: + 20180619: bittrex doesnt support limits -.- + 20180619: binance support limits but only on specific range + """ + try: + if self._api.name == 'Binance': + limit_range = [5, 10, 20, 50, 100, 500, 1000] + # get next-higher step in the limit_range list + limit = min(list(filter(lambda x: limit <= x, limit_range))) + # above script works like loop below (but with slightly better performance): + # for limitx in limit_range: + # if limit <= limitx: + # limit = limitx + # break + + return self._api.fetch_l2_order_book(pair, limit) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching order book.' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order book due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: if self._conf['dry_run']: @@ -462,12 +493,3 @@ class Exchange(object): f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') except ccxt.BaseError as e: raise OperationalException(e) - - def get_amount_lots(self, pair: str, amount: float) -> float: - """ - get buyable amount rounding, .. - """ - # validate that markets are loaded before trying to get fee - if not self._api.markets: - self._api.load_markets() - return self._api.amount_to_lots(pair, amount) diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 46f04328c..8f4b03daf 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -2,6 +2,7 @@ Functions to analyze ticker data with indicators and produce buy and sell signals """ import logging +import pandas as pd from pandas import DataFrame, to_datetime logger = logging.getLogger(__name__) @@ -31,3 +32,27 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: }) frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle return frame + + +def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: + """ + Gets order book list, returns dataframe with below format per suggested by creslin + ------------------------------------------------------------------- + b_sum b_size bids asks a_size a_sum + ------------------------------------------------------------------- + """ + cols = ['bids', 'b_size'] + + bids_frame = DataFrame(bids, columns=cols) + # add cumulative sum column + bids_frame['b_sum'] = bids_frame['b_size'].cumsum() + cols2 = ['asks', 'a_size'] + asks_frame = DataFrame(asks, columns=cols2) + # add cumulative sum column + asks_frame['a_sum'] = asks_frame['a_size'].cumsum() + + frame = pd.concat([bids_frame['b_sum'], bids_frame['b_size'], bids_frame['bids'], + asks_frame['asks'], asks_frame['a_size'], asks_frame['a_sum']], axis=1, + keys=['b_sum', 'b_size', 'bids', 'asks', 'a_size', 'a_sum']) + # logger.info('order book %s', frame ) + return frame diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 706435017..3d3737372 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -21,6 +21,7 @@ from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.exchange.exchange_helpers import order_book_to_dataframe logger = logging.getLogger(__name__) @@ -94,6 +95,8 @@ class FreqtradeBot(object): 'status': f'{state.name.lower()}' }) logger.info('Changing state to: %s', state.name) + if state == State.RUNNING: + self._startup_messages() if state == State.STOPPED: time.sleep(1) @@ -110,6 +113,38 @@ class FreqtradeBot(object): nb_assets=nb_assets) return state + def _startup_messages(self) -> None: + if self.config.get('dry_run', False): + self.rpc.send_msg({ + 'type': RPCMessageType.WARNING_NOTIFICATION, + 'status': 'Dry run is enabled. All trades are simulated.' + }) + stake_currency = self.config['stake_currency'] + stake_amount = self.config['stake_amount'] + minimal_roi = self.config['minimal_roi'] + ticker_interval = self.config['ticker_interval'] + exchange_name = self.config['exchange']['name'] + strategy_name = self.config.get('strategy', '') + self.rpc.send_msg({ + 'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'status': f'*Exchange:* `{exchange_name}`\n' + f'*Stake per trade:* `{stake_amount} {stake_currency}`\n' + f'*Minimum ROI:* `{minimal_roi}`\n' + f'*Ticker Interval:* `{ticker_interval}`\n' + f'*Strategy:* `{strategy_name}`' + }) + if self.config.get('dynamic_whitelist', False): + top_pairs = 'top ' + str(self.config.get('dynamic_whitelist', 20)) + specific_pairs = '' + else: + top_pairs = 'whitelisted' + specific_pairs = '\n' + ', '.join(self.config['exchange'].get('pair_whitelist', '')) + self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': f'Searching for {top_pairs} {stake_currency} pairs to buy and sell...' + f'{specific_pairs}' + }) + def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: """ Throttles the given callable that it @@ -233,16 +268,40 @@ class FreqtradeBot(object): return final_list - def get_target_bid(self, ticker: Dict[str, float]) -> float: + def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price :param ticker: Ticker to use for getting Ask and Last Price :return: float: Price """ if ticker['ask'] < ticker['last']: - return ticker['ask'] - balance = self.config['bid_strategy']['ask_last_balance'] - return ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + ticker_rate = ticker['ask'] + else: + balance = self.config['bid_strategy']['ask_last_balance'] + ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + + used_rate = ticker_rate + config_bid_strategy = self.config.get('bid_strategy', {}) + if 'use_order_book' in config_bid_strategy and\ + config_bid_strategy.get('use_order_book', False): + logger.info('Getting price from order book') + order_book_top = config_bid_strategy.get('order_book_top', 1) + order_book = self.exchange.get_order_book(pair, order_book_top) + logger.debug('order_book %s', order_book) + # top 1 = index 0 + order_book_rate = order_book['bids'][order_book_top - 1][0] + # if ticker has lower rate, then use ticker ( usefull if down trending ) + logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) + if ticker_rate < order_book_rate: + logger.info('...using ticker rate instead %0.8f', ticker_rate) + used_rate = ticker_rate + else: + used_rate = order_book_rate + else: + logger.info('Using Last Ask / Last Price') + used_rate = ticker_rate + + return used_rate def _get_trade_stake_amount(self) -> Optional[float]: """ @@ -334,9 +393,34 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) if buy and not sell: + bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ + get('check_depth_of_market', {}) + if (bidstrat_check_depth_of_market.get('enabled', False)) and\ + (bidstrat_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): + if self._check_depth_of_market_buy(_pair, bidstrat_check_depth_of_market): + return self.execute_buy(_pair, stake_amount) + else: + return False return self.execute_buy(_pair, stake_amount) return False + def _check_depth_of_market_buy(self, pair: str, conf: Dict) -> bool: + """ + Checks depth of market before executing a buy + """ + conf_bids_to_ask_delta = conf.get('bids_to_ask_delta', 0) + logger.info('checking depth of market for %s', pair) + order_book = self.exchange.get_order_book(pair, 1000) + order_book_data_frame = order_book_to_dataframe(order_book['bids'], order_book['asks']) + order_book_bids = order_book_data_frame['b_size'].sum() + order_book_asks = order_book_data_frame['a_size'].sum() + bids_ask_delta = order_book_bids / order_book_asks + logger.info('bids: %s, asks: %s, delta: %s', order_book_bids, + order_book_asks, bids_ask_delta) + if bids_ask_delta >= conf_bids_to_ask_delta: + return True + return False + def execute_buy(self, pair: str, stake_amount: float) -> bool: """ Executes a limit buy for the given pair @@ -349,7 +433,7 @@ class FreqtradeBot(object): fiat_currency = self.config.get('fiat_display_currency', None) # Calculate amount - buy_limit = self.get_target_bid(self.exchange.get_ticker(pair)) + buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: @@ -492,7 +576,7 @@ class FreqtradeBot(object): raise ValueError(f'attempt to handle closed trade: {trade}') logger.debug('Handling %s ...', trade) - current_rate = self.exchange.get_ticker(trade.pair)['bid'] + sell_rate = self.exchange.get_ticker(trade.pair)['bid'] (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) @@ -501,13 +585,43 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) - should_sell = self.strategy.should_sell(trade, current_rate, datetime.utcnow(), buy, sell) - if should_sell.sell_flag: - self.execute_sell(trade, current_rate, should_sell.sell_type) - return True + config_ask_strategy = self.config.get('ask_strategy', {}) + if config_ask_strategy.get('use_order_book', False): + logger.info('Using order book for selling...') + # logger.debug('Order book %s',orderBook) + order_book_min = config_ask_strategy.get('order_book_min', 1) + order_book_max = config_ask_strategy.get('order_book_max', 1) + + order_book = self.exchange.get_order_book(trade.pair, order_book_max) + + for i in range(order_book_min, order_book_max + 1): + order_book_rate = order_book['asks'][i - 1][0] + + # if orderbook has higher rate (high profit), + # use orderbook, otherwise just use bids rate + logger.info(' order book asks top %s: %0.8f', i, order_book_rate) + if sell_rate < order_book_rate: + sell_rate = order_book_rate + + if self.check_sell(trade, sell_rate, buy, sell): + return True + break + else: + logger.info('checking sell') + if self.check_sell(trade, sell_rate, buy, sell): + return True + logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False + def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if should_sell.sell_flag: + self.execute_sell(trade, sell_rate, should_sell.sell_type) + logger.info('excuted sell') + return True + return False + def check_handle_timedout(self) -> None: """ Check if any orders are timed out and cancel if neccessary diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 8d5350fe5..0478dbda6 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,7 +1,13 @@ # pragma pylint: disable=missing-docstring import gzip -import json +try: + import ujson as json + _UJSON = True +except ImportError: + # see mypy/issues/1153 + import json # type: ignore + _UJSON = False import logging import os from typing import Optional, List, Dict, Tuple, Any @@ -14,6 +20,14 @@ from freqtrade.arguments import TimeRange logger = logging.getLogger(__name__) +def json_load(data): + """Try to load data with ujson""" + if _UJSON: + return json.load(data, precise_float=True) + else: + return json.load(data) + + def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: return tickerlist @@ -163,7 +177,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json.load(file) + data = json_load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 8fb01d074..c26d74015 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -79,10 +79,12 @@ def check_migrate(engine) -> None: table_back_name = 'trades_bak' for i, table_back_name in enumerate(tabs): table_back_name = f'trades_bak{i}' - logger.info(f'trying {table_back_name}') + logger.debug(f'trying {table_back_name}') # Check for latest column - if not has_column(cols, 'max_rate'): + if not has_column(cols, 'ticker_interval'): + logger.info(f'Running database migration - backup available as {table_back_name}') + fee_open = get_column_def(cols, 'fee_open', 'fee') fee_close = get_column_def(cols, 'fee_close', 'fee') open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') @@ -157,8 +159,8 @@ class Trade(_DECL_BASE): id = Column(Integer, primary_key=True) exchange = Column(String, nullable=False) - pair = Column(String, nullable=False) - is_open = Column(Boolean, nullable=False, default=True) + pair = Column(String, nullable=False, index=True) + is_open = Column(Boolean, nullable=False, default=True, index=True) fee_open = Column(Float, nullable=False, default=0.0) fee_close = Column(Float, nullable=False, default=0.0) open_rate = Column(Float) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f58fbae9a..dc3d9bd65 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -13,6 +13,7 @@ import sqlalchemy as sql from numpy import mean, nan_to_num from pandas import DataFrame +from freqtrade import TemporaryError from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.misc import shorten_date from freqtrade.persistence import Trade @@ -24,6 +25,8 @@ logger = logging.getLogger(__name__) class RPCMessageType(Enum): STATUS_NOTIFICATION = 'status' + WARNING_NOTIFICATION = 'warning' + CUSTOM_NOTIFICATION = 'custom' BUY_NOTIFICATION = 'buy' SELL_NOTIFICATION = 'sell' @@ -271,10 +274,13 @@ class RPC(object): if coin == 'BTC': rate = 1.0 else: - if coin == 'USDT': - rate = 1.0 / self._freqtrade.exchange.get_ticker('BTC/USDT', False)['bid'] - else: - rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid'] + try: + if coin == 'USDT': + rate = 1.0 / self._freqtrade.exchange.get_ticker('BTC/USDT', False)['bid'] + else: + rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid'] + except TemporaryError: + continue est_btc: float = rate * balance['total'] total = total + est_btc output.append({ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 3b5ce3f74..64708ef74 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -154,6 +154,12 @@ class Telegram(RPC): elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: message = '*Status:* `{status}`'.format(**msg) + elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION: + message = '*Warning:* `{status}`'.format(**msg) + + elif msg['type'] == RPCMessageType.CUSTOM_NOTIFICATION: + message = '{status}'.format(**msg) + else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index 283426dfa..38a110bd7 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -1,4 +1,5 @@ import logging +import sys from copy import deepcopy from freqtrade.strategy.interface import IStrategy @@ -12,8 +13,18 @@ def import_strategy(strategy: IStrategy, config: dict) -> IStrategy: Imports given Strategy instance to global scope of freqtrade.strategy and returns an instance of it """ + # Copy all attributes from base class and class - attr = deepcopy({**strategy.__class__.__dict__, **strategy.__dict__}) + + comb = {**strategy.__class__.__dict__, **strategy.__dict__} + + # Delete '_abc_impl' from dict as deepcopy fails on 3.7 with + # `TypeError: can't pickle _abc_data objects`` + # This will only apply to python 3.7 + if sys.version_info.major == 3 and sys.version_info.minor == 7 and '_abc_impl' in comb: + del comb['_abc_impl'] + + attr = deepcopy(comb) # Adjust module name attr['__module__'] = 'freqtrade.strategy' diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6d865f4bc..c86b85e62 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -182,7 +182,8 @@ class IStrategy(ABC): # Check if dataframe is out of date signal_date = arrow.get(latest['date']) interval_minutes = constants.TICKER_INTERVAL_MINUTES[interval] - if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + 5))): + offset = self.config.get('exchange', {}).get('outdated_offset', 5) + if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', pair, diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 35aee8d20..aee47580c 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -44,7 +44,8 @@ class StrategyResolver(object): # Check if we need to override configuration if 'minimal_roi' in config: self.strategy.minimal_roi = config['minimal_roi'] - logger.info("Override strategy 'minimal_roi' with value in config file.") + logger.info("Override strategy 'minimal_roi' with value in config file: %s.", + config['minimal_roi']) else: config['minimal_roi'] = self.strategy.minimal_roi diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index d18016e16..af9062cab 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -102,7 +102,18 @@ def default_conf(): "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0 + "ask_last_balance": 0.0, + "use_order_book": False, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": False, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy": { + "use_order_book": False, + "order_book_min": 1, + "order_book_max": 1 }, "exchange": { "name": "bittrex", @@ -403,6 +414,39 @@ def limit_sell_order(): } +@pytest.fixture +def order_book_l2(): + return MagicMock(return_value={ + 'bids': [ + [0.043936, 10.442], + [0.043935, 31.865], + [0.043933, 11.212], + [0.043928, 0.088], + [0.043925, 10.0], + [0.043921, 10.0], + [0.04392, 37.64], + [0.043899, 0.066], + [0.043885, 0.676], + [0.04387, 22.758] + ], + 'asks': [ + [0.043949, 0.346], + [0.04395, 0.608], + [0.043951, 3.948], + [0.043954, 0.288], + [0.043958, 9.277], + [0.043995, 1.566], + [0.044, 0.588], + [0.044002, 0.992], + [0.044003, 0.095], + [0.04402, 37.64] + ], + 'timestamp': None, + 'datetime': None, + 'nonce': 288004540 + }) + + @pytest.fixture def ticker_history(): return [ diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6918e9da1..701eeb776 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -515,6 +515,35 @@ def test_get_ticker(default_conf, mocker): exchange.get_ticker(pair='ETH/BTC', refresh=True) +def test_get_order_book(default_conf, mocker, order_book_l2): + default_conf['exchange']['name'] = 'binance' + api_mock = MagicMock() + + api_mock.fetch_l2_order_book = order_book_l2 + exchange = get_patched_exchange(mocker, default_conf, api_mock) + order_book = exchange.get_order_book(pair='ETH/BTC', limit=10) + assert 'bids' in order_book + assert 'asks' in order_book + assert len(order_book['bids']) == 10 + assert len(order_book['asks']) == 10 + + +def test_get_order_book_exception(default_conf, mocker): + api_mock = MagicMock() + with pytest.raises(OperationalException): + api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NotSupported) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_order_book(pair='ETH/BTC', limit=50) + with pytest.raises(TemporaryError): + api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_order_book(pair='ETH/BTC', limit=50) + with pytest.raises(OperationalException): + api_mock.fetch_l2_order_book = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.get_order_book(pair='ETH/BTC', limit=50) + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: @@ -823,15 +852,3 @@ def test_get_fee(default_conf, mocker): ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'get_fee', 'calculate_fee') - - -def test_get_amount_lots(default_conf, mocker): - api_mock = MagicMock() - api_mock.amount_to_lots = MagicMock(return_value=1.0) - api_mock.markets = None - marketmock = MagicMock() - api_mock.load_markets = marketmock - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - assert exchange.get_amount_lots('LTC/BTC', 1.54) == 1 - assert marketmock.call_count == 1 diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 70b7dcfd9..c17ab6b2f 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock, ANY import pytest +from freqtrade import TemporaryError from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade @@ -285,11 +286,12 @@ def test_rpc_balance_handle(default_conf, mocker): 'used': 2.0, }, 'ETH': { - 'free': 0.0, - 'total': 0.0, - 'used': 0.0, + 'free': 1.0, + 'total': 5.0, + 'used': 4.0, } } + # ETH will be skipped due to mocked Error below mocker.patch.multiple( 'freqtrade.fiat_convert.Market', @@ -301,7 +303,8 @@ def test_rpc_balance_handle(default_conf, mocker): mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - get_balances=MagicMock(return_value=mock_balance) + get_balances=MagicMock(return_value=mock_balance), + get_ticker=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx')) ) freqtradebot = FreqtradeBot(default_conf) @@ -320,6 +323,7 @@ def test_rpc_balance_handle(default_conf, mocker): 'pending': 2.0, 'est_btc': 12.0, }] + assert result['total'] == 12.0 def test_rpc_start(mocker, default_conf) -> None: diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 4b2fe4cf5..4d2b9cda2 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1095,6 +1095,38 @@ def test_send_msg_status_notification(default_conf, mocker) -> None: assert msg_mock.call_args[0][0] == '*Status:* `running`' +def test_warning_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.WARNING_NOTIFICATION, + 'status': 'message' + }) + assert msg_mock.call_args[0][0] == '*Warning:* `message`' + + +def test_custom_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'status': '*Custom:* `Hello World`' + }) + assert msg_mock.call_args[0][0] == '*Custom:* `Hello World`' + + def test_send_msg_unknown_type(default_conf, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 5afffd87f..29a6dc1de 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -8,6 +8,7 @@ from pandas import DataFrame from freqtrade.arguments import TimeRange from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.persistence import Trade from freqtrade.tests.conftest import get_patched_exchange, log_has from freqtrade.strategy.default_strategy import DefaultStrategy @@ -107,6 +108,29 @@ def test_tickerdata_to_dataframe(default_conf) -> None: assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed +def test_min_roi_reached(default_conf, fee) -> None: + strategy = DefaultStrategy(default_conf) + strategy.minimal_roi = {0: 0.1, 20: 0.05, 55: 0.01} + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + open_date=arrow.utcnow().shift(hours=-1).datetime, + fee_open=fee.return_value, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + ) + + assert not strategy.min_roi_reached(trade, 0.01, arrow.utcnow().shift(minutes=-55).datetime) + assert strategy.min_roi_reached(trade, 0.12, arrow.utcnow().shift(minutes=-55).datetime) + + assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-39).datetime) + assert strategy.min_roi_reached(trade, 0.06, arrow.utcnow().shift(minutes=-39).datetime) + + assert not strategy.min_roi_reached(trade, -0.01, arrow.utcnow().shift(minutes=-1).datetime) + assert strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-1).datetime) + + def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 8a6e7e617..abc531689 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -130,7 +130,7 @@ def test_strategy_override_minimal_roi(caplog): assert resolver.strategy.minimal_roi[0] == 0.5 assert ('freqtrade.strategy.resolver', logging.INFO, - "Override strategy 'minimal_roi' with value in config file." + "Override strategy 'minimal_roi' with value in config file: {'0': 0.5}." ) in caplog.record_tuples diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 89adae6ab..49c9c4431 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -159,6 +159,15 @@ def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: assert whitelist == [] +def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + + with pytest.raises(OperationalException): + freqtrade._gen_pair_whitelist(base_currency='BTC') + + @pytest.mark.skip(reason="Test not implemented") def test_refresh_whitelist() -> None: pass @@ -664,21 +673,21 @@ def test_balance_fully_ask_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 20 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 20 def test_balance_fully_last_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 20, 'last': 10}) == 10 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 20, 'last': 10}) == 10 def test_balance_bigger_last_ask(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 1.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_target_bid({'ask': 5, 'last': 10}) == 5 + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 5, 'last': 10}) == 5 def test_process_maybe_execute_buy(mocker, default_conf) -> None: @@ -1876,3 +1885,194 @@ def test_get_real_amount_open_trade(default_conf, mocker): freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) assert freqtrade.get_real_amount(trade, order) == amount + + +def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, markets, mocker, + order_book_l2): + default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 + patch_RPCManager(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets + ) + + # Save state of current whitelist + whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade is not None + assert trade.stake_amount == 0.001 + assert trade.is_open + assert trade.open_date is not None + assert trade.exchange == 'bittrex' + + # Simulate fulfilled LIMIT_BUY order for trade + trade.update(limit_buy_order) + + assert trade.open_rate == 0.00001099 + assert whitelist == default_conf['exchange']['pair_whitelist'] + + +def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_order, + fee, markets, mocker, order_book_l2): + default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True + # delta is 100 which is impossible to reach. hence check_depth_of_market will return false + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 + patch_RPCManager(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets + ) + # Save state of current whitelist + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade is None + + +def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2, markets) -> None: + """ + test if function get_target_bid will return the order book price + instead of the ask rate + """ + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) + default_conf['exchange']['name'] = 'binance' + default_conf['bid_strategy']['use_order_book'] = True + default_conf['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['ask_last_balance'] = 0 + default_conf['telegram']['enabled'] = False + + freqtrade = FreqtradeBot(default_conf) + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.045, 'last': 0.046}) == 0.043935 + + +def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) -> None: + """ + test if function get_target_bid will return the ask rate (since its value is lower) + instead of the order book rate (even if enabled) + """ + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) + default_conf['exchange']['name'] = 'binance' + default_conf['bid_strategy']['use_order_book'] = True + default_conf['bid_strategy']['order_book_top'] = 2 + default_conf['bid_strategy']['ask_last_balance'] = 0 + default_conf['telegram']['enabled'] = False + + freqtrade = FreqtradeBot(default_conf) + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.042, 'last': 0.046}) == 0.042 + + +def test_order_book_bid_strategy3(default_conf, mocker, order_book_l2, markets) -> None: + """ + test if function get_target_bid will return ask rate instead + of the order book rate + """ + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) + default_conf['exchange']['name'] = 'binance' + default_conf['bid_strategy']['use_order_book'] = True + default_conf['bid_strategy']['order_book_top'] = 1 + default_conf['bid_strategy']['ask_last_balance'] = 0 + default_conf['telegram']['enabled'] = False + + freqtrade = FreqtradeBot(default_conf) + + assert freqtrade.get_target_bid('ETH/BTC', {'ask': 0.03, 'last': 0.029}) == 0.03 + + +def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2, markets) -> None: + """ + test check depth of market + """ + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_markets=markets, + get_order_book=order_book_l2 + ) + default_conf['telegram']['enabled'] = False + default_conf['exchange']['name'] = 'binance' + default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True + # delta is 100 which is impossible to reach. hence function will return false + default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 100 + freqtrade = FreqtradeBot(default_conf) + + conf = default_conf['bid_strategy']['check_depth_of_market'] + assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False + + +def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order, + fee, markets, mocker, order_book_l2) -> None: + """ + test order book ask strategy + """ + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) + default_conf['exchange']['name'] = 'binance' + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + default_conf['telegram']['enabled'] = False + patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + validate_pairs=MagicMock(), + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + sell=MagicMock(return_value={'id': limit_sell_order['id']}), + get_fee=fee, + get_markets=markets + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + time.sleep(0.01) # Race condition fix + trade.update(limit_buy_order) + assert trade.is_open is True + + patch_get_signal(freqtrade, value=(False, True)) + assert freqtrade.handle_trade(trade) is True + + +def test_startup_messages(default_conf, mocker): + default_conf['dynamic_whitelist'] = 20 + freqtrade = get_patched_freqtradebot(mocker, default_conf) + assert freqtrade.state is State.RUNNING diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 26932136a..7584537e2 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103 from unittest.mock import MagicMock +import logging import pytest from sqlalchemy import create_engine @@ -403,7 +404,9 @@ def test_migrate_new(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat) """ + caplog.set_level(logging.DEBUG) amount = 103.223 + # Always create all columns apart from the last! create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( id INTEGER NOT NULL, exchange VARCHAR NOT NULL, @@ -418,14 +421,21 @@ def test_migrate_new(mocker, default_conf, fee, caplog): open_date DATETIME NOT NULL, close_date DATETIME, open_order_id VARCHAR, + stop_loss FLOAT, + initial_stop_loss FLOAT, + max_rate FLOAT, + sell_reason VARCHAR, + strategy VARCHAR, PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, - open_rate, stake_amount, amount, open_date) + open_rate, stake_amount, amount, open_date, + stop_loss, initial_stop_loss, max_rate) VALUES ('binance', 'ETC/BTC', 1, {fee}, 0.00258580, {stake}, {amount}, - '2019-11-28 12:44:24.000000') + '2019-11-28 12:44:24.000000', + 0.0, 0.0, 0.0) """.format(fee=fee.return_value, stake=default_conf.get("stake_amount"), amount=amount @@ -463,12 +473,15 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.ticker_interval is None assert log_has("trying trades_bak1", caplog.record_tuples) assert log_has("trying trades_bak2", caplog.record_tuples) + assert log_has("Running database migration - backup available as trades_bak2", + caplog.record_tuples) def test_migrate_mid_state(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat) """ + caplog.set_level(logging.DEBUG) amount = 103.223 create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( id INTEGER NOT NULL, @@ -522,6 +535,8 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 assert log_has("trying trades_bak0", caplog.record_tuples) + assert log_has("Running database migration - backup available as trades_bak0", + caplog.record_tuples) def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee): diff --git a/freqtrade/tests/test_talib.py b/freqtrade/tests/test_talib.py new file mode 100644 index 000000000..093c3023c --- /dev/null +++ b/freqtrade/tests/test_talib.py @@ -0,0 +1,16 @@ + + +import talib.abstract as ta +import pandas as pd + + +def test_talib_bollingerbands_near_zero_values(): + inputs = pd.DataFrame([ + {'close': 0.00000010}, + {'close': 0.00000011}, + {'close': 0.00000012}, + {'close': 0.00000013}, + {'close': 0.00000014} + ]) + bollinger = ta.BBANDS(inputs, matype=0, timeperiod=2) + assert (bollinger['upperband'][3] != bollinger['middleband'][3]) diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 21e69cbba..18e7b8bbb 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,6 +1,6 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && ./configure && make && sudo make install && cd .. + cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." cd ta-lib && sudo make install && cd .. diff --git a/requirements.txt b/requirements.txt index 5ff5d3694..5cb061855 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,18 @@ -ccxt==1.17.60 -SQLAlchemy==1.2.10 -python-telegram-bot==10.1.0 +ccxt==1.17.216 +SQLAlchemy==1.2.11 +python-telegram-bot==11.0.0 arrow==0.12.1 cachetools==2.1.0 requests==2.19.1 urllib3==1.22 wrapt==1.10.11 -pandas==0.23.3 +pandas==0.23.4 scikit-learn==0.19.2 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.15.0 +numpy==1.15.1 TA-Lib==0.4.17 -pytest==3.7.0 +pytest==3.7.4 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 @@ -22,4 +22,4 @@ coinmarketcap==5.0.3 scikit-optimize==0.5.2 # Required for plotting data -#plotly==3.0.0 +#plotly==3.1.1 diff --git a/setup.sh b/setup.sh index a825ca41f..bd58edbee 100755 --- a/setup.sh +++ b/setup.sh @@ -1,13 +1,31 @@ #!/usr/bin/env bash #encoding=utf8 +# Check which python version is installed +function check_installed_python() { + which python3.7 + if [ $? -eq 0 ]; then + echo "using Python 3.7" + PYTHON=python3.7 + return + fi + + which python3.6 + if [ $? -eq 0 ]; then + echo "using Python 3.6" + PYTHON=python3.6 + return + fi + +} + function updateenv () { echo "-------------------------" echo "Update your virtual env" echo "-------------------------" source .env/bin/activate echo "pip3 install in-progress. Please wait..." - pip3.6 install --quiet --upgrade pip + pip3 install --quiet --upgrade pip pip3 install --quiet -r requirements.txt --upgrade pip3 install --quiet -r requirements.txt pip3 install --quiet -e . @@ -79,7 +97,7 @@ function reset () { fi echo - python3.6 -m venv .env + ${PYTHON} -m venv .env updateenv } @@ -183,7 +201,7 @@ function install () { install_debian else echo "This script does not support your OS." - echo "If you have Python3.6, pip, virtualenv, ta-lib you can continue." + echo "If you have Python3.6 or Python3.7, pip, virtualenv, ta-lib you can continue." echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell." sleep 10 fi @@ -193,7 +211,7 @@ function install () { echo "-------------------------" echo "Run the bot" echo "-------------------------" - echo "You can now use the bot by executing 'source .env/bin/activate; python3.6 freqtrade/main.py'." + echo "You can now use the bot by executing 'source .env/bin/activate; python freqtrade/main.py'." } function plot () { @@ -214,6 +232,9 @@ function help () { echo " -p,--plot Install dependencies for Plotting scripts." } +# Verify if 3.6 or 3.7 is installed +check_installed_python + case $* in --install|-i) install