From 184b5ca3fc5ad3d102be4846db52a5b1a469b1b2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Aug 2018 19:59:56 +0200 Subject: [PATCH 1/8] cleanup root dir and create build_helpers --- .../install_ta-lib.sh | 0 build_helpers/publish_docker.sh | 53 ++++++++++++++++++ .../ta-lib-0.4.0-src.tar.gz | Bin 3 files changed, 53 insertions(+) rename install_ta-lib.sh => build_helpers/install_ta-lib.sh (100%) create mode 100755 build_helpers/publish_docker.sh rename ta-lib-0.4.0-src.tar.gz => build_helpers/ta-lib-0.4.0-src.tar.gz (100%) diff --git a/install_ta-lib.sh b/build_helpers/install_ta-lib.sh similarity index 100% rename from install_ta-lib.sh rename to build_helpers/install_ta-lib.sh diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh new file mode 100755 index 000000000..a398a8719 --- /dev/null +++ b/build_helpers/publish_docker.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Tag with travis build +TAG=$TRAVIS_BUILD_NUMBER +# - export TAG=`if [ "$TRAVIS_BRANCH" == "develop" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` +# Replace / with _ to create a valid tag +TAG=$(echo "${TRAVIS_BRANCH}" | sed -e "s/\//_/") + + +# Pull last build to avoid rebuilding the whole image +docker pull ${REPO}:${TAG} + +docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . +if [ $? -ne 0 ]; then + echo "failed building image" + return 1 +fi + +# Run backtest +docker run --rm -it -v $(pwd)/config.json.example:/freqtrade/config.json:ro freqtrade:${TAG} --datadir freqtrade/tests/testdata backtesting + +if [ $? -ne 0 ]; then + echo "failed running backtest" + return 1 +fi + +# Tag image for upload +docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG +if [ $? -ne 0 ]; then + echo "failed tagging image" + return 1 +fi + +# Tag as latest for develop builds +if [ "${TRAVIS_BRANCH}" == "develop" ]; then + docker tag freqtrade:$TAG ${IMAGE_NAME}:latest +fi + +# Login +echo "$DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin + +if [ $? -ne 0 ]; then + echo "failed login" + return 1 +fi + +# Show all available images +docker images + +docker push ${IMAGE_NAME} +if [ $? -ne 0 ]; then + echo "failed pushing repo" + return 1 +fi diff --git a/ta-lib-0.4.0-src.tar.gz b/build_helpers/ta-lib-0.4.0-src.tar.gz similarity index 100% rename from ta-lib-0.4.0-src.tar.gz rename to build_helpers/ta-lib-0.4.0-src.tar.gz From 98738c482a7e52ecc9d7d55929e21b54ff2558c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Aug 2018 20:01:57 +0200 Subject: [PATCH 2/8] modify install-ta-lib script to support running in docker --- build_helpers/install_ta-lib.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index d8ae2eeaa..4d4f37c17 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -1,7 +1,13 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. + cd ta-lib \ + && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ + && ./configure \ + && make \ + && which sudo && sudo make install || make install \ + && cd .. else echo "TA-lib already installed, skipping download and build." cd ta-lib && sudo make install && cd .. + fi From 907761f9941e73f931e3b8e97e8d1366539f90ca Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Aug 2018 20:03:20 +0200 Subject: [PATCH 3/8] Install ta-lib in Docker with script works for travis, works for Docker --- Dockerfile | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2506665ab..24cce0049 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,20 @@ FROM python:3.7.0-slim-stretch -# Install TA-lib -RUN apt-get update && apt-get -y install curl build-essential && apt-get clean -RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \ - tar xzvf - && \ - cd ta-lib && \ - sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ - ./configure && make && make install && \ - cd .. && rm -rf ta-lib -ENV LD_LIBRARY_PATH /usr/local/lib +RUN apt-get update \ + && apt-get -y install curl build-essential \ + && apt-get clean \ + && pip install --upgrade pip # Prepare environment RUN mkdir /freqtrade WORKDIR /freqtrade +# Install TA-lib +COPY build_helpers/* /tmp/ +RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* + +ENV LD_LIBRARY_PATH /usr/local/lib + # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy --no-cache-dir \ From af7283017b331f6c5f396b7757f7c84674796a3e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Aug 2018 20:04:25 +0200 Subject: [PATCH 4/8] modify travis to build and push docker * name steps * only build for master / develop and this branch (for now) --- .travis.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 981eedcf8..f1192e80c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,15 @@ sudo: true os: - linux +dist: trusty language: python python: - 3.6 +services: + - docker +env: + global: + - IMAGE_NAME=freqtradeorg/freqtrade addons: apt: packages: @@ -11,24 +17,38 @@ addons: - libdw-dev - binutils-dev install: -- ./install_ta-lib.sh +- ./build_helpers/install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy - pip install -r requirements.txt - pip install -e . jobs: include: - - script: + - stage: tests + script: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls + name: pytest - script: - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting + name: backtest - script: - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5 + name: hyperopt - script: flake8 freqtrade + name: flake8 - script: mypy freqtrade + name: mypy + + - stage: docker + if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron)) + script: + - build_helpers/publish_docker.sh + name: "Build and test and push docker image" + + notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= From 0535660db722342d11e3a7a2da4d3aca6880025c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 09:44:21 +0200 Subject: [PATCH 5/8] build technical image --- Dockerfile.technical | 6 ++++++ build_helpers/publish_docker.sh | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 Dockerfile.technical diff --git a/Dockerfile.technical b/Dockerfile.technical new file mode 100644 index 000000000..5339eb232 --- /dev/null +++ b/Dockerfile.technical @@ -0,0 +1,6 @@ +FROM freqtradeorg/freqtrade:develop + +RUN apt-get update \ + && apt-get -y install git \ + && apt-get clean \ + && pip install git+https://github.com/berlinguyinca/technical diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index a398a8719..95aae0f4f 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -1,10 +1,8 @@ #!/bin/sh -# Tag with travis build -TAG=$TRAVIS_BUILD_NUMBER # - export TAG=`if [ "$TRAVIS_BRANCH" == "develop" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` # Replace / with _ to create a valid tag TAG=$(echo "${TRAVIS_BRANCH}" | sed -e "s/\//_/") - +TAG_TECH="${TAG}_technical" # Pull last build to avoid rebuilding the whole image docker pull ${REPO}:${TAG} @@ -23,6 +21,10 @@ if [ $? -ne 0 ]; then return 1 fi +# build technical image +sed -i Dockerfile.technical -e "s/FROM freqtradeorg\/freqtrade:develop/FROM freqtradeorg\/freqtrade:${TAG}/" +docker build --cache-from freqtrade:${TAG} -t ${IMAGE_NAME}:${TAG_TECH} -f Dockerfile.technical . + # Tag image for upload docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG if [ $? -ne 0 ]; then @@ -31,7 +33,7 @@ if [ $? -ne 0 ]; then fi # Tag as latest for develop builds -if [ "${TRAVIS_BRANCH}" == "develop" ]; then +if [ "${TRAVIS_BRANCH}" = "develop" ]; then docker tag freqtrade:$TAG ${IMAGE_NAME}:latest fi From 7301d76cff46c23a0e9b7c7aee04d83ca124bb27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Oct 2018 13:20:51 +0200 Subject: [PATCH 6/8] Remove autobuild for technical as it's not versioned as it's not versioned and installed from github, we cannot guarantee which version is in the image. --- build_helpers/publish_docker.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 95aae0f4f..5e0809fcf 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -21,10 +21,6 @@ if [ $? -ne 0 ]; then return 1 fi -# build technical image -sed -i Dockerfile.technical -e "s/FROM freqtradeorg\/freqtrade:develop/FROM freqtradeorg\/freqtrade:${TAG}/" -docker build --cache-from freqtrade:${TAG} -t ${IMAGE_NAME}:${TAG_TECH} -f Dockerfile.technical . - # Tag image for upload docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG if [ $? -ne 0 ]; then From 39efda19f4df040576ce55f33e60699a5e1dbe73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Oct 2018 13:33:23 +0200 Subject: [PATCH 7/8] Add freqtradeorg/freqtrade docker images to the documentation --- docs/installation.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 1000600e6..15cfb1467 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -109,7 +109,25 @@ Dry-Run touch tradesv3.dryrun.sqlite ``` -### 2. Build the Docker image +### 2. Download or build the docker image + +Either use the prebuilt image from docker hub - or build the image yourself if you would like more control on which version is used. + +Branches / tags available can be checked out on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). + +#### 2.1. Download the docker image + +Pull the image from docker hub and (optionally) change the name of the image + +```bash +docker pull freqtradeorg/freqtrade:develop +# Optionally tag the repository so the run-commands remain shorter +docker tag freqtradeorg/freqtrade:develop freqtrade +``` + +To update the image, simply run the above commands again and restart your running container. + +#### 2.2. Build the Docker image ```bash cd freqtrade From 530d521d78bc9305face56e2fb8308c6b58f328c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Oct 2018 14:00:01 +0200 Subject: [PATCH 8/8] Rebuild complete image on "cron" events --- build_helpers/publish_docker.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 5e0809fcf..c2b40ba81 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -2,12 +2,18 @@ # - export TAG=`if [ "$TRAVIS_BRANCH" == "develop" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` # Replace / with _ to create a valid tag TAG=$(echo "${TRAVIS_BRANCH}" | sed -e "s/\//_/") -TAG_TECH="${TAG}_technical" -# Pull last build to avoid rebuilding the whole image -docker pull ${REPO}:${TAG} -docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . +if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then + echo "event ${TRAVIS_EVENT_TYPE}: full rebuild - skipping cache" + docker build -t freqtrade:${TAG} . +else + echo "event ${TRAVIS_EVENT_TYPE}: building with cache" + # Pull last build to avoid rebuilding the whole image + docker pull ${REPO}:${TAG} + docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . +fi + if [ $? -ne 0 ]; then echo "failed building image" return 1 @@ -29,7 +35,7 @@ if [ $? -ne 0 ]; then fi # Tag as latest for develop builds -if [ "${TRAVIS_BRANCH}" = "develop" ]; then +if [ "${TRAVIS_BRANCH}" = "develop" ]; then docker tag freqtrade:$TAG ${IMAGE_NAME}:latest fi