Merge branch 'develop' into rename_master_branch

.devcontainer/Dockerfile

@@ -0,0 +1,18 @@
+FROM freqtradeorg/freqtrade:develop
+
+# Install dependencies
+COPY requirements-dev.txt /freqtrade/
+RUN apt-get update \
+    && apt-get -y install git sudo vim \
+    && apt-get clean \
+    && pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \
+    && useradd -u 1000 -U -m ftuser \
+    && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \
+    && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \
+    && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \
+    && chown ftuser: -R /home/ftuser/
+
+USER ftuser
+
+# Empty the ENTRYPOINT to allow all commands
+ENTRYPOINT []

.devcontainer/devcontainer.json

@@ -0,0 +1,44 @@
+{
+    "name": "freqtrade Develop",
+
+    "dockerComposeFile": [
+        "docker-compose.yml"
+    ],
+
+    "service": "ft_vscode",
+
+    "workspaceFolder": "/freqtrade/",
+
+    "settings": {
+        "terminal.integrated.shell.linux": "/bin/bash",
+        "editor.insertSpaces": true,
+        "files.trimTrailingWhitespace": true,
+        "[markdown]": {
+            "files.trimTrailingWhitespace": false,
+        },
+        "python.pythonPath": "/usr/local/bin/python",
+    },
+
+    // Add the IDs of extensions you want installed when the container is created.
+    "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "davidanson.vscode-markdownlint",
+        "ms-azuretools.vscode-docker",
+    ],
+
+    // Use 'forwardPorts' to make a list of ports inside the container available locally.
+    // "forwardPorts": [],
+
+    // Uncomment the next line if you want start specific services in your Docker Compose config.
+    // "runServices": [],
+
+    // Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+    // "shutdownAction": "none",
+
+    // Uncomment the next line to run commands after the container is created - for example installing curl.
+    // "postCreateCommand": "sudo apt-get update && apt-get install -y git",
+
+    // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
+    "remoteUser": "ftuser"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,24 @@
+---
+version: '3'
+services:
+  ft_vscode:
+    build:
+      context: ..
+      dockerfile: ".devcontainer/Dockerfile"
+    volumes:
+      # Allow git usage within container
+      - "/home/${USER}/.ssh:/home/ftuser/.ssh:ro"
+      - "/home/${USER}/.gitconfig:/home/ftuser/.gitconfig:ro"
+      - ..:/freqtrade:cached
+      # Persist bash-history
+      - freqtrade-vscode-server:/home/ftuser/.vscode-server
+      - freqtrade-bashhistory:/home/ftuser/commandhistory
+    # Expose API port
+    ports:
+      - "127.0.0.1:8080:8080"
+    command: /bin/sh -c "while sleep 1000; do :; done"
+
+
+volumes:
+  freqtrade-vscode-server:
+  freqtrade-bashhistory:

.dockerignore

@@ -13,3 +13,4 @@ CONTRIBUTING.md
 MANIFEST.in
 README.md
 freqtrade.service
+user_data

Dockerfile

@@ -22,7 +22,8 @@ RUN pip install numpy --no-cache-dir \
 
 # Install and execute
 COPY . /freqtrade/
-RUN pip install -e . --no-cache-dir
+RUN pip install -e . --no-cache-dir \
+  && mkdir /freqtrade/user_data/
 ENTRYPOINT ["freqtrade"]
 # Default to trade mode
 CMD [ "trade" ]

build_helpers/publish_docker.sh

@@ -2,6 +2,7 @@
 
 # Replace / with _ to create a valid tag
 TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
+TAG_PLOT=${TAG}_plot
 echo "Running for ${TAG}"
 
 # Add commit and commit_message to docker container
@@ -16,6 +17,7 @@ else
     docker pull ${IMAGE_NAME}:${TAG}
     docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} .
 fi
+docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot .
 
 if [ $? -ne 0 ]; then
     echo "failed building image"
@@ -32,6 +34,7 @@ fi
 
 # Tag image for upload
 docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG
+docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT
 if [ $? -ne 0 ]; then
     echo "failed tagging image"
     return 1

docker-compose.develop.yml

@@ -1,20 +0,0 @@
----
-version: '3'
-services:
-  freqtrade_develop:
-    build:
-      context: .
-      dockerfile: "./Dockerfile.develop"
-    volumes:
-      - ".:/freqtrade"
-    entrypoint:
-      - "freqtrade"
-
-  freqtrade_bash:
-    build:
-      context: .
-      dockerfile: "./Dockerfile.develop"
-    volumes:
-      - ".:/freqtrade"
-    entrypoint:
-      - "/bin/bash"

docker-compose.yml

@@ -4,6 +4,8 @@ services:
   freqtrade:
     image: freqtradeorg/freqtrade:stable
     # image: freqtradeorg/freqtrade:develop
+    # Use plotting image
+    # image: freqtradeorg/freqtrade:develop_plot
     # Build step - only needed when additional dependencies are needed
     # build:
     #   context: .

docker/Dockerfile.develop

@@ -2,6 +2,7 @@ FROM freqtradeorg/freqtrade:develop
 
 # Install dependencies
 COPY requirements-dev.txt /freqtrade/
+
 RUN pip install numpy --no-cache-dir \
   && pip install -r requirements-dev.txt --no-cache-dir
 

docker/Dockerfile.jupyter

@@ -0,0 +1,7 @@
+FROM freqtradeorg/freqtrade:develop_plot
+
+
+RUN pip install jupyterlab --no-cache-dir
+
+# Empty the ENTRYPOINT to allow all commands
+ENTRYPOINT []

docker/Dockerfile.plot

@@ -0,0 +1,10 @@
+ARG sourceimage=develop
+FROM freqtradeorg/freqtrade:${sourceimage}
+
+# Install dependencies
+COPY requirements-plot.txt /freqtrade/
+
+RUN pip install -r requirements-plot.txt --no-cache-dir
+
+# Empty the ENTRYPOINT to allow all commands
+ENTRYPOINT []

docker/docker-compose-jupyter.yml

@@ -0,0 +1,16 @@
+---
+version: '3'
+services:
+  ft_jupyterlab:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile.jupyter
+    restart: unless-stopped
+    container_name: freqtrade
+    ports:
+      - "127.0.0.1:8888:8888"
+    volumes:
+      - "./user_data:/freqtrade/user_data"
+    # Default command used when running `docker compose up`
+    command: >
+      jupyter lab --port=8888 --ip 0.0.0.0 --allow-root

docs/data-analysis.md

@@ -1,12 +1,22 @@
 # Analyzing bot data with Jupyter notebooks  
 
-You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/`.  
+You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/` after initializing the user directory with `freqtrade create-userdir --userdir user_data`.  
 
-## Pro tips  
+## Quick start with docker
+
+Freqtrade provides a docker-compose file which starts up a jupyter lab server.
+You can run this server using the following command: `docker-compose -f docker/docker-compose-jupyter.yml up`
+
+This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`.
+Please use the link that's printed in the console after startup for simplified login.
+
+For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analayis-using-docker-compose) section.
+
+### Pro tips
 
 * See [jupyter.org](https://jupyter.org/documentation) for usage instructions.
 * Don't forget to start a Jupyter notebook server from within your conda or venv environment or use [nb_conda_kernels](https://github.com/Anaconda-Platform/nb_conda_kernels)*
-* Copy the example notebook before use so your changes don't get clobbered with the next freqtrade update.
+* Copy the example notebook before use so your changes don't get overwritten with the next freqtrade update.
 
 ### Using virtual environment with system-wide Jupyter installation
 
@@ -28,10 +38,8 @@ ipython kernel install --user --name=freqtrade
 !!! Note
     This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get jupyter notebooks up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community).
 
-    
+!!! Warning
-## Fine print  
+    Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually.
-
-Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually.
 
 ## Recommended workflow  
 

docs/data-download.md

@@ -22,7 +22,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [-p PAIRS [PAIRS ...]] [--pairs-file FILE]
                                [--days INT] [--timerange TIMERANGE]
                                [--dl-trades] [--exchange EXCHANGE]
-                               [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
+                               [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
                                [--erase]
                                [--data-format-ohlcv {json,jsongz,hdf5}]
                                [--data-format-trades {json,jsongz,hdf5}]
@@ -41,7 +41,7 @@ optional arguments:
                         as --timeframes/-t.
   --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
                         config is provided.
-  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
+  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
                         Specify which tickers to download. Space-separated
                         list. Default: `1m 5m`.
   --erase               Clean all existing data for the selected
@@ -71,6 +71,11 @@ Common arguments:
 
 ```
 
+!!! Note "Startup period"
+    `download-data` is a strategy-independent command. The idea is to download a big chunk of data once, and then iteratively increase the amount of data stored.
+
+    For that reason, `download-data` does not care about the "startup-period" defined in a strategy. It's up to the user to download additional days if the backtest should start at a specific point in time (while respecting startup period).
+
 ### Data format
 
 Freqtrade currently supports 3 data-formats for both OHLCV and trades data:
@@ -104,7 +109,7 @@ usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                               [-p PAIRS [PAIRS ...]] --format-from
                               {json,jsongz,hdf5} --format-to
                               {json,jsongz,hdf5} [--erase]
-                              [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
+                              [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -117,7 +122,7 @@ optional arguments:
                         Destination format for data conversion.
   --erase               Clean all existing data for the selected
                         exchange/pairs/timeframes.
-  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
+  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
                         Specify which tickers to download. Space-separated
                         list. Default: `1m 5m`.
 

docs/developer.md

@@ -21,11 +21,24 @@ This will spin up a local server (usually on port 8000) so you can see if everyt
 
 ## Developer setup
 
-To configure a development environment, best use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
+To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
-Alternatively (if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
+Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
 
 This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`.
 
+### Devcontainer setup
+
+The fastest and easiest way to get started is to use [VSCode](https://code.visualstudio.com/) with the Remote container extension.
+This gives developers the ability to start the bot with all required dependencies *without* needing to install any freqtrade specific dependencies on your local machine.
+
+#### Devcontainer dependencies
+
+* [VSCode](https://code.visualstudio.com/)
+* [docker](https://docs.docker.com/install/)
+* [Remote container extension documentation](https://code.visualstudio.com/docs/remote)
+
+For more information about the [Remote container extension](https://code.visualstudio.com/docs/remote), best consult the documentation.
+
 ### Tests
 
 New code should be covered by basic unittests. Depending on the complexity of the feature, Reviewers may request more in-depth unittests.
@@ -50,51 +63,6 @@ def test_method_to_test(caplog):
 
 ```
 
-### Local docker usage
-
-The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine.
-
-#### Install
-
-* [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
-* [docker](https://docs.docker.com/install/)
-* [docker-compose](https://docs.docker.com/compose/install/)
-
-#### Starting the bot
-
-##### Use the develop dockerfile
-
-``` bash
-rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml
-```
-
-#### Docker Compose
-
-##### Starting
-
-``` bash
-docker-compose up
-```
-
-![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png)
-
-##### Rebuilding
-
-``` bash
-docker-compose build
-```
-
-##### Executing (effectively SSH into the container)
-
-The `exec` command requires that the container already be running, if you want to start it
-that can be effected by `docker-compose up` or `docker-compose run freqtrade_develop`
-
-``` bash
-docker-compose exec freqtrade_develop /bin/bash
-```
-
-![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png)
-
 ## ErrorHandling
 
 Freqtrade Exceptions all inherit from `FreqtradeException`.
@@ -264,6 +232,7 @@ This documents some decisions taken for the CI Pipeline.
 
 * CI runs on all OS variants, Linux (ubuntu), macOS and Windows.
 * Docker images are build for the branches `stable` and `develop`.
+* Docker images containing Plot dependencies are also available as `stable_plot` and `develop_plot`.
 * Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:stable_pi` and `develop_pi`.
 * Docker images contain a file, `/freqtrade/freqtrade_commit` containing the commit this image is based of.
 * Full docker image rebuilds are run once a week via schedule.

docs/docker.md

@@ -70,16 +70,16 @@ cp -n config.json.example config.json
 
 Best start by pulling the official docker image from dockerhub as explained [here](#download-the-official-docker-image) to speed up building.
 
-To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image.
+To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image.
 
 ```bash
-docker build -t freqtrade -f Dockerfile.technical .
+docker build -t freqtrade -f docker/Dockerfile.technical .
 ```
 
-If you are developing using Docker, use `Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies:
+If you are developing using Docker, use `docker/Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies:
 
 ```bash
-docker build -f Dockerfile.develop -t freqtrade-dev .
+docker build -f docker/Dockerfile.develop -t freqtrade-dev .
 ```
 
 !!! Warning "Include your config file manually"

docs/docker_quickstart.md

@@ -148,7 +148,7 @@ Head over to the [Backtesting Documentation](backtesting.md) to learn more.
 ### Additional dependencies with docker-compose
 
 If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host.
-For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) for an example).
+For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) for an example).
 
 You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions.
 
@@ -160,3 +160,32 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the
 ```
 
 You can then run `docker-compose build` to build the docker image, and run it using the commands described above.
+
+## Plotting with docker-compose
+
+Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file.
+You can then use these commands as follows:
+
+``` bash
+docker-compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p BTC/ETH --timerange=20180801-20180805
+```
+
+The output will be stored in the `user_data/plot` directory, and can be opened with any modern browser.
+
+## Data analayis using docker compose
+
+Freqtrade provides a docker-compose file which starts up a jupyter lab server.
+You can run this server using the following command:
+
+``` bash
+docker-compose --rm -f docker/docker-compose-jupyter.yml up
+```
+
+This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`.
+Please use the link that's printed in the console after startup for simplified login.
+
+Since part of this image is built on your machine, it is recommended to rebuild the image from time to time to keep freqtrade (and dependencies) uptodate.
+
+``` bash
+docker-compose -f docker/docker-compose-jupyter.yml build --no-cache
+```

docs/requirements-docs.txt

@@ -1,2 +1,2 @@
-mkdocs-material==5.5.12
+mkdocs-material==5.5.13
 mdx_truly_sane_lists==1.2

freqtrade/commands/cli_options.py

@@ -375,7 +375,7 @@ AVAILABLE_CLI_OPTIONS = {
         help='Specify which tickers to download. Space-separated list. '
         'Default: `1m 5m`.',
         choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h',
-                 '6h', '8h', '12h', '1d', '3d', '1w'],
+                 '6h', '8h', '12h', '1d', '3d', '1w', '2w', '1M', '1y'],
         default=['1m', '5m'],
         nargs='+',
     ),

requirements-dev.txt

@@ -15,4 +15,4 @@ pytest-mock==3.3.1
 pytest-random-order==1.0.4
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==6.0.2
+nbconvert==6.0.4

requirements.txt

@@ -1,7 +1,7 @@
 numpy==1.19.2
 pandas==1.1.2
 
-ccxt==1.34.25
+ccxt==1.34.40
 SQLAlchemy==1.3.19
 python-telegram-bot==12.8
 arrow==0.16.0