From a7c0e86bfdf20b38bbfbda38bd0d73ddb794dddd Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 24 May 2020 14:51:14 +0200
Subject: [PATCH 1/8] Test rpi using buildx and dockerfile.armhf

Uses piwheels for pi image
---
 .github/workflows/ci.yml           | 35 +++++++++++++++--------
 Dockerfile.armhf                   | 29 +++++++++++++++++++
 build_helpers/publish_docker.sh    |  8 ------
 build_helpers/publish_docker_pi.sh | 45 ++++++++++++++++++++++++++++++
 4 files changed, 98 insertions(+), 19 deletions(-)
 create mode 100644 Dockerfile.armhf
 create mode 100755 build_helpers/publish_docker_pi.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 42668e46f..8ab842eeb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,6 +6,7 @@ on:
       - master
       - develop
       - github_actions_tests
+      - rpi_crosscompile
     tags:
     release:
       types: [published]
@@ -226,25 +227,37 @@ jobs:
         user: __token__
         password: ${{ secrets.pypi_password }}
 
+    - name: Dockerhub login
+      env:
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      run: |
+        echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+
     - name: Build and test and push docker image
       env:
         IMAGE_NAME: freqtradeorg/freqtrade
-        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
         BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
       run: |
         build_helpers/publish_docker.sh
 
-    - name: Build raspberry image for ${{ steps.extract_branch.outputs.branch }}_pi
-      uses: elgohr/Publish-Docker-Github-Action@2.7
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: crazy-max/ghaction-docker-buildx@v1
       with:
-        name: freqtradeorg/freqtrade:${{ steps.extract_branch.outputs.branch }}_pi
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-        dockerfile: Dockerfile.pi
-        # cache: true
-        cache: ${{ github.event_name != 'schedule' }}
-        tag_names: true
+        buildx-version: latest
+        qemu-version: latest
+
+    - name: Available platforms
+      run: echo ${{ steps.buildx.outputs.platforms }}
+
+    - name: Build Raspberry docker image
+      env:
+        IMAGE_NAME: freqtradeorg/freqtrade
+        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
+      run: |
+        build_helpers/publish_docker_pi.sh
+
 
     - name: Slack Notification
       uses: homoluctus/slatify@v1.8.0
diff --git a/Dockerfile.armhf b/Dockerfile.armhf
new file mode 100644
index 000000000..d6e2aa3a1
--- /dev/null
+++ b/Dockerfile.armhf
@@ -0,0 +1,29 @@
+FROM --platform=linux/arm/v7 python:3.7.7-slim-buster
+
+RUN apt-get update \
+  && apt-get -y install curl build-essential libssl-dev libatlas3-base libgfortran5 \
+  && apt-get clean \
+  && pip install --upgrade pip \
+  && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf
+
+# Prepare environment
+RUN mkdir /freqtrade
+WORKDIR /freqtrade
+
+# Install TA-lib
+COPY build_helpers/* /tmp/
+RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
+
+ENV LD_LIBRARY_PATH /usr/local/lib
+
+# Install dependencies
+COPY requirements.txt requirements-common.txt /freqtrade/
+RUN pip install numpy --no-cache-dir \
+  && pip install -r requirements.txt --no-cache-dir
+
+# Install and execute
+COPY . /freqtrade/
+RUN pip install -e . --no-cache-dir
+ENTRYPOINT ["freqtrade"]
+# Default to trade mode
+CMD [ "trade" ]
diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh
index 013644563..03a95161b 100755
--- a/build_helpers/publish_docker.sh
+++ b/build_helpers/publish_docker.sh
@@ -42,14 +42,6 @@ if [ "${TAG}" = "develop" ]; then
     docker tag freqtrade:$TAG ${IMAGE_NAME}:latest
 fi
 
-# Login
-docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
-
-if [ $? -ne 0 ]; then
-    echo "failed login"
-    return 1
-fi
-
 # Show all available images
 docker images
 
diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh
new file mode 100755
index 000000000..744f6e196
--- /dev/null
+++ b/build_helpers/publish_docker_pi.sh
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# The below assumes a correctly setup docker buildx environment
+
+# Replace / with _ to create a valid tag
+TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
+PI_PLATFORM="linux/arm/v7"
+echo "Running for ${TAG}"
+
+# Add commit and commit_message to docker container
+echo "${GITHUB_SHA}" > freqtrade_commit
+
+if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
+    echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache"
+    docker buildx build --platform linux/arm/v7 -t ${IMAGE_NAME}:${TAG} --push .
+else
+    echo "event ${GITHUB_EVENT_NAME}: building with cache"
+    # Pull last build to avoid rebuilding the whole image
+    docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG}
+    docker buildx build --cache-from ${IMAGE_NAME}:${TAG} --platform ${PI_PLATFORM} -t ${IMAGE_NAME}:${TAG} --push .
+fi
+
+if [ $? -ne 0 ]; then
+    echo "failed building image"
+    return 1
+fi
+
+# Tag as latest for develop builds
+if [ "${TAG}" = "develop" ]; then
+    docker tag ${IMAGE_NAME}:$TAG ${IMAGE_NAME}:latest
+fi
+
+if [ $? -ne 0 ]; then
+    echo "failed login"
+    return 1
+fi
+
+# Show all available images
+docker images
+
+docker push ${IMAGE_NAME}
+if [ $? -ne 0 ]; then
+    echo "failed pushing to repo"
+    return 1
+fi

From 4fbd2deb376a632462cc77c290caf035c19688cd Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 25 May 2020 15:04:12 +0200
Subject: [PATCH 2/8] Use correct image

---
 build_helpers/publish_docker_pi.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh
index 744f6e196..35b9ff5f7 100755
--- a/build_helpers/publish_docker_pi.sh
+++ b/build_helpers/publish_docker_pi.sh
@@ -12,12 +12,12 @@ echo "${GITHUB_SHA}" > freqtrade_commit
 
 if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
     echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache"
-    docker buildx build --platform linux/arm/v7 -t ${IMAGE_NAME}:${TAG} --push .
+    docker buildx build -f Dockerfile.armhf --platform ${PI_PLATFORM} -t ${IMAGE_NAME}:${TAG} --push .
 else
     echo "event ${GITHUB_EVENT_NAME}: building with cache"
     # Pull last build to avoid rebuilding the whole image
     docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG}
-    docker buildx build --cache-from ${IMAGE_NAME}:${TAG} --platform ${PI_PLATFORM} -t ${IMAGE_NAME}:${TAG} --push .
+    docker buildx build --cache-from ${IMAGE_NAME}:${TAG} -f Dockerfile.armhf --platform ${PI_PLATFORM} -t ${IMAGE_NAME}:${TAG} --push .
 fi
 
 if [ $? -ne 0 ]; then

From 571d61de84edb0796c4bab912d5d4352ed1d5921 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 25 May 2020 15:21:14 +0200
Subject: [PATCH 3/8] Cancel previous automatically?

---
 .github/workflows/ci.yml           | 9 +++++++++
 build_helpers/publish_docker_pi.sh | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8ab842eeb..faa71a4d1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -190,6 +190,15 @@ jobs:
         channel: '#notifications'
         url: ${{ secrets.SLACK_WEBHOOK }}
 
+  cleanup-prior-runs:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Cleanup previous runs on this branch
+      uses: rokroskar/workflow-run-cleanup-action@v0.2.2
+      if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.repository == 'freqtrade/freqtrade'"
+      env:
+        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
   deploy:
     needs: [ build, build_windows, docs_check ]
     runs-on: ubuntu-18.04
diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh
index 35b9ff5f7..f6424c335 100755
--- a/build_helpers/publish_docker_pi.sh
+++ b/build_helpers/publish_docker_pi.sh
@@ -38,7 +38,7 @@ fi
 # Show all available images
 docker images
 
-docker push ${IMAGE_NAME}
+# docker push ${IMAGE_NAME}
 if [ $? -ne 0 ]; then
     echo "failed pushing to repo"
     return 1

From 7f5feab5cfff0afdca3211d78fe2e41af49b5864 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 25 May 2020 21:10:29 +0200
Subject: [PATCH 4/8] Fix notifications, build ...

---
 .github/workflows/ci.yml | 20 +++++++++++++++++---
 Dockerfile.pi            | 41 ----------------------------------------
 2 files changed, 17 insertions(+), 44 deletions(-)
 delete mode 100644 Dockerfile.pi

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index faa71a4d1..86a210b0b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -101,7 +101,7 @@ jobs:
 
     - name: Slack Notification
       uses: homoluctus/slatify@v1.8.0
-      if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
+      if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
       with:
         type: ${{ job.status }}
         job_name: '*Freqtrade CI ${{ matrix.os }}*'
@@ -163,7 +163,7 @@ jobs:
 
     - name: Slack Notification
       uses: homoluctus/slatify@v1.8.0
-      if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
+      if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
       with:
         type: ${{ job.status }}
         job_name: '*Freqtrade CI windows*'
@@ -199,6 +199,20 @@ jobs:
       env:
         GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 
+  # Notify on slack only once - when CI completes (and after deploy) in case it's successfull
+  notify-complete:
+    needs: [ build, build_windows, docs_check ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Slack Notification
+      uses: homoluctus/slatify@v1.8.0
+      if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
+      with:
+        type: ${{ job.status }}
+        job_name: '*Freqtrade CI*'
+        channel: '#notifications'
+        url: ${{ secrets.SLACK_WEBHOOK }}
+
   deploy:
     needs: [ build, build_windows, docs_check ]
     runs-on: ubuntu-18.04
@@ -263,7 +277,7 @@ jobs:
     - name: Build Raspberry docker image
       env:
         IMAGE_NAME: freqtradeorg/freqtrade
-        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
+        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}_pi
       run: |
         build_helpers/publish_docker_pi.sh
 
diff --git a/Dockerfile.pi b/Dockerfile.pi
deleted file mode 100644
index 279f85a04..000000000
--- a/Dockerfile.pi
+++ /dev/null
@@ -1,41 +0,0 @@
-FROM balenalib/raspberrypi3-debian:stretch
-
-RUN [ "cross-build-start" ]
-
-RUN apt-get update \
- && apt-get -y install wget curl build-essential libssl-dev libffi-dev \
- && apt-get clean
-
-# Prepare environment
-RUN mkdir /freqtrade
-WORKDIR /freqtrade
-
-# Install TA-lib
-COPY build_helpers/ta-lib-0.4.0-src.tar.gz /freqtrade/
-RUN tar -xzf /freqtrade/ta-lib-0.4.0-src.tar.gz \
- && cd /freqtrade/ta-lib/ \
- && ./configure \
- && make \
- && make install \
- && rm /freqtrade/ta-lib-0.4.0-src.tar.gz
-
-ENV LD_LIBRARY_PATH /usr/local/lib
-
-# Install berryconda
-RUN wget -q https://github.com/jjhelmus/berryconda/releases/download/v2.0.0/Berryconda3-2.0.0-Linux-armv7l.sh \
- && bash ./Berryconda3-2.0.0-Linux-armv7l.sh -b \
- && rm Berryconda3-2.0.0-Linux-armv7l.sh
-
-# Install dependencies
-COPY requirements-common.txt /freqtrade/
-RUN ~/berryconda3/bin/conda install -y numpy pandas \
- && ~/berryconda3/bin/pip install -r requirements-common.txt --no-cache-dir
-
-# Install and execute
-COPY . /freqtrade/
-RUN ~/berryconda3/bin/pip install -e . --no-cache-dir
-
-RUN [ "cross-build-end" ]
-
-ENTRYPOINT ["/root/berryconda3/bin/python","./freqtrade/main.py"]
-CMD [ "trade" ]

From 500ce50153323dd27537e09f44797532ee767926 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 26 May 2020 06:56:09 +0200
Subject: [PATCH 5/8] Switch to docker experimental ...

---
 .github/workflows/ci.yml           | 8 ++++++++
 build_helpers/publish_docker_pi.sh | 4 ++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 86a210b0b..75844a5ea 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -264,6 +264,14 @@ jobs:
       run: |
         build_helpers/publish_docker.sh
 
+    # We need docker experimental to pull the ARM image.
+    - name: Switch docker to experimental
+      run: |
+          docker version -f '{{.Server.Experimental}}'
+          echo $'{\n    "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+          sudo systemctl restart docker
+          docker version -f '{{.Server.Experimental}}'
+
     - name: Set up Docker Buildx
       id: buildx
       uses: crazy-max/ghaction-docker-buildx@v1
diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh
index f6424c335..52a8164c4 100755
--- a/build_helpers/publish_docker_pi.sh
+++ b/build_helpers/publish_docker_pi.sh
@@ -16,8 +16,8 @@ if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
 else
     echo "event ${GITHUB_EVENT_NAME}: building with cache"
     # Pull last build to avoid rebuilding the whole image
-    docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG}
-    docker buildx build --cache-from ${IMAGE_NAME}:${TAG} -f Dockerfile.armhf --platform ${PI_PLATFORM} -t ${IMAGE_NAME}:${TAG} --push .
+    # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG}
+    docker buildx build --cache-from=type=registry,ref=${IMAGE_NAME}:${TAG} -f Dockerfile.armhf --platform ${PI_PLATFORM} -t ${IMAGE_NAME}:${TAG} --push .
 fi
 
 if [ $? -ne 0 ]; then

From 2f596f739d227ec6c7a88a7be3cb4b37e6500964 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 26 May 2020 17:14:52 +0200
Subject: [PATCH 6/8] Use cache correctly

---
 build_helpers/publish_docker_pi.sh | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh
index 52a8164c4..3a732d8c0 100755
--- a/build_helpers/publish_docker_pi.sh
+++ b/build_helpers/publish_docker_pi.sh
@@ -6,18 +6,28 @@
 TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
 PI_PLATFORM="linux/arm/v7"
 echo "Running for ${TAG}"
+CACHE_TAG=${IMAGE_NAME}:${TAG}_cache
 
 # Add commit and commit_message to docker container
 echo "${GITHUB_SHA}" > freqtrade_commit
 
 if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
     echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache"
-    docker buildx build -f Dockerfile.armhf --platform ${PI_PLATFORM} -t ${IMAGE_NAME}:${TAG} --push .
+    docker buildx build \
+        --cache-to=type=registry,ref=${CACHE_TAG} \
+        -f Dockerfile.armhf \
+        --platform ${PI_PLATFORM} \
+        -t ${IMAGE_NAME}:${TAG} --push .
 else
     echo "event ${GITHUB_EVENT_NAME}: building with cache"
     # Pull last build to avoid rebuilding the whole image
     # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG}
-    docker buildx build --cache-from=type=registry,ref=${IMAGE_NAME}:${TAG} -f Dockerfile.armhf --platform ${PI_PLATFORM} -t ${IMAGE_NAME}:${TAG} --push .
+    docker buildx build \
+        --cache-from=type=registry,ref=${CACHE_TAG} \
+        --cache-to=type=registry,ref=${CACHE_TAG} \
+        -f Dockerfile.armhf \
+        --platform ${PI_PLATFORM} \
+        -t ${IMAGE_NAME}:${TAG} --push .
 fi
 
 if [ $? -ne 0 ]; then

From 448546b0a495f8d403526b4a85845a9f0ab13edc Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 26 May 2020 19:16:44 +0200
Subject: [PATCH 7/8] Some cleanup in script

---
 build_helpers/publish_docker_pi.sh | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh
index 3a732d8c0..20513f045 100755
--- a/build_helpers/publish_docker_pi.sh
+++ b/build_helpers/publish_docker_pi.sh
@@ -34,22 +34,3 @@ if [ $? -ne 0 ]; then
     echo "failed building image"
     return 1
 fi
-
-# Tag as latest for develop builds
-if [ "${TAG}" = "develop" ]; then
-    docker tag ${IMAGE_NAME}:$TAG ${IMAGE_NAME}:latest
-fi
-
-if [ $? -ne 0 ]; then
-    echo "failed login"
-    return 1
-fi
-
-# Show all available images
-docker images
-
-# docker push ${IMAGE_NAME}
-if [ $? -ne 0 ]; then
-    echo "failed pushing to repo"
-    return 1
-fi

From bdf795b8b0d48a141c7705d2a3366f2cbf5b62de Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 26 May 2020 19:36:38 +0200
Subject: [PATCH 8/8] final cleanup - use different cache image for now

---
 .github/workflows/ci.yml           | 1 -
 build_helpers/publish_docker_pi.sh | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 75844a5ea..239576c61 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,7 +6,6 @@ on:
       - master
       - develop
       - github_actions_tests
-      - rpi_crosscompile
     tags:
     release:
       types: [published]
diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh
index 20513f045..060b1deaf 100755
--- a/build_helpers/publish_docker_pi.sh
+++ b/build_helpers/publish_docker_pi.sh
@@ -6,7 +6,7 @@
 TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
 PI_PLATFORM="linux/arm/v7"
 echo "Running for ${TAG}"
-CACHE_TAG=${IMAGE_NAME}:${TAG}_cache
+CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache
 
 # Add commit and commit_message to docker container
 echo "${GITHUB_SHA}" > freqtrade_commit