adelorenzo
/
stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into rename_master_branch

This commit is contained in:
Matthias 2020-09-21 19:31:25 +02:00
parent 637fe35549 87f6c7bbec
commit d639290f7d
22 changed files with 206 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.devcontainer/Dockerfile Normal file
View File

@ -0,0 +1,18 @@
FROM freqtradeorg/freqtrade:develop
# Install dependencies
COPY requirements-dev.txt /freqtrade/
RUN apt-get update \
&& apt-get -y install git sudo vim \
&& apt-get clean \
&& pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \
&& useradd -u 1000 -U -m ftuser \
&& mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \
&& echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \
&& echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \
&& chown ftuser: -R /home/ftuser/
USER ftuser
# Empty the ENTRYPOINT to allow all commands
ENTRYPOINT []

44
.devcontainer/devcontainer.json Normal file
View File

@ -0,0 +1,44 @@
{
"name": "freqtrade Develop",
"dockerComposeFile": [
"docker-compose.yml"
],
"service": "ft_vscode",
"workspaceFolder": "/freqtrade/",
"settings": {
"terminal.integrated.shell.linux": "/bin/bash",
"editor.insertSpaces": true,
"files.trimTrailingWhitespace": true,
"[markdown]": {
"files.trimTrailingWhitespace": false,
},
"python.pythonPath": "/usr/local/bin/python",
},
// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"ms-python.python",
"ms-python.vscode-pylance",
"davidanson.vscode-markdownlint",
"ms-azuretools.vscode-docker",
],
// Use 'forwardPorts' to make a list of ports inside the container available locally.
// "forwardPorts": [],
// Uncomment the next line if you want start specific services in your Docker Compose config.
// "runServices": [],
// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
// "shutdownAction": "none",
// Uncomment the next line to run commands after the container is created - for example installing curl.
// "postCreateCommand": "sudo apt-get update && apt-get install -y git",
// Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "ftuser"
}

24
.devcontainer/docker-compose.yml Normal file
View File

@ -0,0 +1,24 @@
---
version: '3'
services:
ft_vscode:
build:
context: ..
dockerfile: ".devcontainer/Dockerfile"
volumes:
# Allow git usage within container
- "/home/${USER}/.ssh:/home/ftuser/.ssh:ro"
- "/home/${USER}/.gitconfig:/home/ftuser/.gitconfig:ro"
- ..:/freqtrade:cached
# Persist bash-history
- freqtrade-vscode-server:/home/ftuser/.vscode-server
- freqtrade-bashhistory:/home/ftuser/commandhistory
# Expose API port
ports:
- "127.0.0.1:8080:8080"
command: /bin/sh -c "while sleep 1000; do :; done"
volumes:
freqtrade-vscode-server:
freqtrade-bashhistory:

1
.dockerignore
View File

@ -13,3 +13,4 @@ CONTRIBUTING.md
MANIFEST.in
README.md
freqtrade.service
user_data

3
Dockerfile
View File

@ -22,7 +22,8 @@ RUN pip install numpy --no-cache-dir \
# Install and execute
COPY . /freqtrade/
RUN pip install -e . --no-cache-dir
RUN pip install -e . --no-cache-dir \
&& mkdir /freqtrade/user_data/
ENTRYPOINT ["freqtrade"]
# Default to trade mode
CMD [ "trade" ]

3
build_helpers/publish_docker.sh
View File

@ -2,6 +2,7 @@
# Replace / with _ to create a valid tag
TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
TAG_PLOT=${TAG}_plot
echo "Running for ${TAG}"
# Add commit and commit_message to docker container
@ -16,6 +17,7 @@ else
docker pull ${IMAGE_NAME}:${TAG}
docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} .
fi
docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot .
if [ $? -ne 0 ]; then
echo "failed building image"
@ -32,6 +34,7 @@ fi
# Tag image for upload
docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG
docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT
if [ $? -ne 0 ]; then
echo "failed tagging image"
return 1

20
docker-compose.develop.yml
View File

@ -1,20 +0,0 @@
---
version: '3'
services:
freqtrade_develop:
build:
context: .
dockerfile: "./Dockerfile.develop"
volumes:
- ".:/freqtrade"
entrypoint:
- "freqtrade"
freqtrade_bash:
build:
context: .
dockerfile: "./Dockerfile.develop"
volumes:
- ".:/freqtrade"
entrypoint:
- "/bin/bash"

2
docker-compose.yml
View File

@ -4,6 +4,8 @@ services:
freqtrade:
image: freqtradeorg/freqtrade:stable
# image: freqtradeorg/freqtrade:develop
# Use plotting image
# image: freqtradeorg/freqtrade:develop_plot
# Build step - only needed when additional dependencies are needed
# build:
# context: .

1
Dockerfile.develop → docker/Dockerfile.develop
View File

@ -2,6 +2,7 @@ FROM freqtradeorg/freqtrade:develop
# Install dependencies
COPY requirements-dev.txt /freqtrade/
RUN pip install numpy --no-cache-dir \
&& pip install -r requirements-dev.txt --no-cache-dir

7
docker/Dockerfile.jupyter Normal file
View File

@ -0,0 +1,7 @@
FROM freqtradeorg/freqtrade:develop_plot
RUN pip install jupyterlab --no-cache-dir
# Empty the ENTRYPOINT to allow all commands
ENTRYPOINT []

10
docker/Dockerfile.plot Normal file
View File

@ -0,0 +1,10 @@
ARG sourceimage=develop
FROM freqtradeorg/freqtrade:${sourceimage}
# Install dependencies
COPY requirements-plot.txt /freqtrade/
RUN pip install -r requirements-plot.txt --no-cache-dir
# Empty the ENTRYPOINT to allow all commands
ENTRYPOINT []

0
Dockerfile.technical → docker/Dockerfile.technical
View File

16
docker/docker-compose-jupyter.yml Normal file
View File

@ -0,0 +1,16 @@
---
version: '3'
services:
ft_jupyterlab:
build:
context: ..
dockerfile: docker/Dockerfile.jupyter
restart: unless-stopped
container_name: freqtrade
ports:
- "127.0.0.1:8888:8888"
volumes:
- "./user_data:/freqtrade/user_data"
# Default command used when running `docker compose up`
command: >
jupyter lab --port=8888 --ip 0.0.0.0 --allow-root

22
docs/data-analysis.md
View File

@ -1,12 +1,22 @@
# Analyzing bot data with Jupyter notebooks
You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/`.
You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/` after initializing the user directory with `freqtrade create-userdir --userdir user_data`.
## Pro tips
## Quick start with docker
Freqtrade provides a docker-compose file which starts up a jupyter lab server.
You can run this server using the following command: `docker-compose -f docker/docker-compose-jupyter.yml up`
This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`.
Please use the link that's printed in the console after startup for simplified login.
For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analayis-using-docker-compose) section.
### Pro tips
* See [jupyter.org](https://jupyter.org/documentation) for usage instructions.
* Don't forget to start a Jupyter notebook server from within your conda or venv environment or use [nb_conda_kernels](https://github.com/Anaconda-Platform/nb_conda_kernels)*
* Copy the example notebook before use so your changes don't get clobbered with the next freqtrade update.
* Copy the example notebook before use so your changes don't get overwritten with the next freqtrade update.
### Using virtual environment with system-wide Jupyter installation
@ -28,10 +38,8 @@ ipython kernel install --user --name=freqtrade
!!! Note
This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get jupyter notebooks up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community).
## Fine print
Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually.
!!! Warning
Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually.
## Recommended workflow

13
docs/data-download.md
View File

@ -22,7 +22,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[-p PAIRS [PAIRS ...]] [--pairs-file FILE]
[--days INT] [--timerange TIMERANGE]
[--dl-trades] [--exchange EXCHANGE]
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
[--erase]
[--data-format-ohlcv {json,jsongz,hdf5}]
[--data-format-trades {json,jsongz,hdf5}]
@ -41,7 +41,7 @@ optional arguments:
as --timeframes/-t.
--exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no
config is provided.
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
Specify which tickers to download. Space-separated
list. Default: `1m 5m`.
--erase Clean all existing data for the selected
@ -71,6 +71,11 @@ Common arguments:
```
!!! Note "Startup period"
`download-data` is a strategy-independent command. The idea is to download a big chunk of data once, and then iteratively increase the amount of data stored.
For that reason, `download-data` does not care about the "startup-period" defined in a strategy. It's up to the user to download additional days if the backtest should start at a specific point in time (while respecting startup period).
### Data format
Freqtrade currently supports 3 data-formats for both OHLCV and trades data:
@ -104,7 +109,7 @@ usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[-p PAIRS [PAIRS ...]] --format-from
{json,jsongz,hdf5} --format-to
{json,jsongz,hdf5} [--erase]
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
optional arguments:
-h, --help show this help message and exit
@ -117,7 +122,7 @@ optional arguments:
Destination format for data conversion.
--erase Clean all existing data for the selected
exchange/pairs/timeframes.
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
Specify which tickers to download. Space-separated
list. Default: `1m 5m`.

63
docs/developer.md
View File

@ -21,11 +21,24 @@ This will spin up a local server (usually on port 8000) so you can see if everyt
## Developer setup
To configure a development environment, best use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
Alternatively (if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`.
### Devcontainer setup
The fastest and easiest way to get started is to use [VSCode](https://code.visualstudio.com/) with the Remote container extension.
This gives developers the ability to start the bot with all required dependencies *without* needing to install any freqtrade specific dependencies on your local machine.
#### Devcontainer dependencies
* [VSCode](https://code.visualstudio.com/)
* [docker](https://docs.docker.com/install/)
* [Remote container extension documentation](https://code.visualstudio.com/docs/remote)
For more information about the [Remote container extension](https://code.visualstudio.com/docs/remote), best consult the documentation.
### Tests
New code should be covered by basic unittests. Depending on the complexity of the feature, Reviewers may request more in-depth unittests.
@ -50,51 +63,6 @@ def test_method_to_test(caplog):
```
### Local docker usage
The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine.
#### Install
* [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
* [docker](https://docs.docker.com/install/)
* [docker-compose](https://docs.docker.com/compose/install/)
#### Starting the bot
##### Use the develop dockerfile
``` bash
rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml
```
#### Docker Compose
##### Starting
``` bash
docker-compose up
```
![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png)
##### Rebuilding
``` bash
docker-compose build
```
##### Executing (effectively SSH into the container)
The `exec` command requires that the container already be running, if you want to start it
that can be effected by `docker-compose up` or `docker-compose run freqtrade_develop`
``` bash
docker-compose exec freqtrade_develop /bin/bash
```
![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png)
## ErrorHandling
Freqtrade Exceptions all inherit from `FreqtradeException`.
@ -264,6 +232,7 @@ This documents some decisions taken for the CI Pipeline.
* CI runs on all OS variants, Linux (ubuntu), macOS and Windows.
* Docker images are build for the branches `stable` and `develop`.
* Docker images containing Plot dependencies are also available as `stable_plot` and `develop_plot`.
* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:stable_pi` and `develop_pi`.
* Docker images contain a file, `/freqtrade/freqtrade_commit` containing the commit this image is based of.
* Full docker image rebuilds are run once a week via schedule.

8
docs/docker.md
View File

@ -70,16 +70,16 @@ cp -n config.json.example config.json
Best start by pulling the official docker image from dockerhub as explained [here](#download-the-official-docker-image) to speed up building.
To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image.
To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image.
```bash
docker build -t freqtrade -f Dockerfile.technical .
docker build -t freqtrade -f docker/Dockerfile.technical .
```
If you are developing using Docker, use `Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies:
If you are developing using Docker, use `docker/Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies:
```bash
docker build -f Dockerfile.develop -t freqtrade-dev .
docker build -f docker/Dockerfile.develop -t freqtrade-dev .
```
!!! Warning "Include your config file manually"

31
docs/docker_quickstart.md
View File

@ -148,7 +148,7 @@ Head over to the [Backtesting Documentation](backtesting.md) to learn more.
### Additional dependencies with docker-compose
If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host.
For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) for an example).
For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) for an example).
You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions.
@ -160,3 +160,32 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the
```
You can then run `docker-compose build` to build the docker image, and run it using the commands described above.
## Plotting with docker-compose
Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file.
You can then use these commands as follows:
``` bash
docker-compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p BTC/ETH --timerange=20180801-20180805
```
The output will be stored in the `user_data/plot` directory, and can be opened with any modern browser.
## Data analayis using docker compose
Freqtrade provides a docker-compose file which starts up a jupyter lab server.
You can run this server using the following command:
``` bash
docker-compose --rm -f docker/docker-compose-jupyter.yml up
```
This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`.
Please use the link that's printed in the console after startup for simplified login.
Since part of this image is built on your machine, it is recommended to rebuild the image from time to time to keep freqtrade (and dependencies) uptodate.
``` bash
docker-compose -f docker/docker-compose-jupyter.yml build --no-cache
```

2
docs/requirements-docs.txt
View File

@ -1,2 +1,2 @@
mkdocs-material==5.5.12
mkdocs-material==5.5.13
mdx_truly_sane_lists==1.2

2
freqtrade/commands/cli_options.py
View File

@ -375,7 +375,7 @@ AVAILABLE_CLI_OPTIONS = {
help='Specify which tickers to download. Space-separated list. '
'Default: `1m 5m`.',
choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h',
'6h', '8h', '12h', '1d', '3d', '1w'],
'6h', '8h', '12h', '1d', '3d', '1w', '2w', '1M', '1y'],
default=['1m', '5m'],
nargs='+',
),

2
requirements-dev.txt
View File

@ -15,4 +15,4 @@ pytest-mock==3.3.1
pytest-random-order==1.0.4
# Convert jupyter notebooks to markdown documents
nbconvert==6.0.2
nbconvert==6.0.4

2
requirements.txt
View File

@ -1,7 +1,7 @@
numpy==1.19.2
pandas==1.1.2
ccxt==1.34.25
ccxt==1.34.40
SQLAlchemy==1.3.19
python-telegram-bot==12.8
arrow==0.16.0