From cabab44b759934b528fb301023f7d17330131b43 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 May 2021 19:15:59 +0200 Subject: [PATCH 1/2] Combine docker build scripts --- .github/workflows/ci.yml | 14 ++--- .travis.yml | 6 -- ...lish_docker.sh => publish_docker_multi.sh} | 63 +++++++++++++------ build_helpers/publish_docker_pi.sh | 49 --------------- docker/Dockerfile.armhf | 6 +- 5 files changed, 52 insertions(+), 86 deletions(-) rename build_helpers/{publish_docker.sh => publish_docker_multi.sh} (54%) delete mode 100755 build_helpers/publish_docker_pi.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea766d77d..1ec0f22ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,7 @@ on: - master - stable - develop + - test_multiarch tags: release: types: [published] @@ -75,7 +76,7 @@ jobs: COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu run: | # Allow failure for coveralls - coveralls -v || true + coveralls || true - name: Backtesting run: | @@ -392,19 +393,12 @@ jobs: - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} - - name: Build and test and push docker image + - name: Build and test and push docker images env: IMAGE_NAME: freqtradeorg/freqtrade BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} run: | - build_helpers/publish_docker.sh - - - name: Build Raspberry docker image - env: - IMAGE_NAME: freqtradeorg/freqtrade - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} - run: | - build_helpers/publish_docker_pi.sh + build_helpers/publish_docker_multi.sh - name: Slack Notification diff --git a/.travis.yml b/.travis.yml index 03a8df49b..4535c44cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,12 +46,6 @@ jobs: - script: mypy freqtrade scripts name: mypy - # - stage: docker - # if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron)) - # script: - # - build_helpers/publish_docker.sh - # name: "Build and test and push docker image" - notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker_multi.sh similarity index 54% rename from build_helpers/publish_docker.sh rename to build_helpers/publish_docker_multi.sh index da9fc4e34..a6b06ce7d 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker_multi.sh @@ -1,21 +1,48 @@ #!/bin/sh +# The below assumes a correctly setup docker buildx environment + # Replace / with _ to create a valid tag TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") TAG_PLOT=${TAG}_plot +TAG_PI="${TAG}_pi" + +PI_PLATFORM="linux/arm/v7" echo "Running for ${TAG}" +CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache # Add commit and commit_message to docker container echo "${GITHUB_SHA}" > freqtrade_commit if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" + # Build regular image docker build -t freqtrade:${TAG} . + # Build PI image + docker buildx build \ + --cache-to=type=registry,ref=${CACHE_TAG} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} --push . else echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Pull last build to avoid rebuilding the whole image + # Build regular image docker pull ${IMAGE_NAME}:${TAG} docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . + + # Pull last build to avoid rebuilding the whole image + # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} + docker buildx build \ + --cache-from=type=registry,ref=${CACHE_TAG} \ + --cache-to=type=registry,ref=${CACHE_TAG} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} --push . +fi + +if [ $? -ne 0 ]; then + echo "failed building multiarch images" + return 1 fi # Tag image for upload and next build step docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG @@ -24,11 +51,6 @@ docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t fre docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi - # Run backtest docker run --rm -v $(pwd)/config_bittrex.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy @@ -37,24 +59,29 @@ if [ $? -ne 0 ]; then return 1 fi -if [ $? -ne 0 ]; then - echo "failed tagging image" - return 1 -fi - -# Tag as latest for develop builds -if [ "${TAG}" = "develop" ]; then - docker tag freqtrade:$TAG ${IMAGE_NAME}:latest -fi - -# Show all available images docker images docker push ${IMAGE_NAME} docker push ${IMAGE_NAME}:$TAG_PLOT docker push ${IMAGE_NAME}:$TAG +# Create multiarch image +# Make sure that all images contained here are pushed to github first. +# Otherwise installation might fail. + +docker manifest create freqtradeorg/freqtrade:${TAG} ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} +docker manifest push freqtradeorg/freqtrade:${TAG} + +# Tag as latest for develop builds +if [ "${TAG}" = "develop" ]; then + docker manifest create freqtradeorg/freqtrade:latest ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} + docker manifest push freqtradeorg/freqtrade:latest +fi + + +docker images + if [ $? -ne 0 ]; then - echo "failed pushing repo" + echo "failed building image" return 1 fi diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh deleted file mode 100755 index c7024828b..000000000 --- a/build_helpers/publish_docker_pi.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -# The below assumes a correctly setup docker buildx environment - -# Replace / with _ to create a valid tag -TAG_ORIG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") -TAG="${TAG_ORIG}_pi" - -PI_PLATFORM="linux/arm/v7" -echo "Running for ${TAG}" -CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache - -# Add commit and commit_message to docker container -echo "${GITHUB_SHA}" > freqtrade_commit - -if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then - echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" - docker buildx build \ - --cache-to=type=registry,ref=${CACHE_TAG} \ - -f docker/Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG} --push . -else - echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Pull last build to avoid rebuilding the whole image - # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} - docker buildx build \ - --cache-from=type=registry,ref=${CACHE_TAG} \ - --cache-to=type=registry,ref=${CACHE_TAG} \ - -f docker/Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG} --push . -fi - -docker images - -# Create multiarch image -# Make sure that all images contained here are pushed to github first. -# Otherwise installation might fail. - -docker manifest create freqtradeorg/freqtrade:${TAG_ORIG} ${IMAGE_NAME}:${TAG_ORIG} ${IMAGE_NAME}:${TAG} -docker manifest push freqtradeorg/freqtrade:${TAG_ORIG} - -docker images - -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index 8abf0e44b..f9827774e 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM --platform=linux/arm/v7 python:3.7.10-slim-buster as base +FROM python:3.7.10-slim-buster as base # Setup env ENV LANG C.UTF-8 @@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker" # Prepare environment RUN mkdir /freqtrade \ && apt-get update \ - && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev \ && apt-get clean \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ @@ -23,7 +23,7 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps RUN apt-get update \ - && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \ + && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 pkg-config cmake gcc \ && apt-get clean \ && pip install --upgrade pip \ && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf From 6479217cb485e39dcee1142c707ebf8436e9f786 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jun 2021 14:16:32 +0200 Subject: [PATCH 2/2] Don't build for test_multiarch --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ec0f22ae..42959c3b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,6 @@ on: - master - stable - develop - - test_multiarch tags: release: types: [published]