Merge pull request #4697 from freqtrade/docker_user

Docker as user

.dockerignore

@@ -1,9 +1,8 @@
 .git
 .gitignore
 Dockerfile
+Dockerfile.armhf
 .dockerignore
-config.json*
-*.sqlite
 .coveragerc
 .eggs
 .github
@@ -13,4 +12,13 @@ CONTRIBUTING.md
 MANIFEST.in
 README.md
 freqtrade.service
+freqtrade.egg-info
+
+config.json*
+*.sqlite
 user_data
+*.log
+
+.vscode
+.mypy_cache
+.ipynb_checkpoints

Dockerfile

@@ -5,10 +5,19 @@ ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONFAULTHANDLER 1
-ENV PATH=/root/.local/bin:$PATH
+ENV PATH=/home/ftuser/.local/bin:$PATH
+ENV FT_APP_ENV="docker"
 
 # Prepare environment
-RUN mkdir /freqtrade
+RUN mkdir /freqtrade \
+  && apt update \
+  && apt install -y sudo \
+  && apt-get clean \
+  && useradd -u 1000 -G sudo -U -m ftuser \
+  && chown ftuser:ftuser /freqtrade \
+  # Allow sudoers
+  && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
+
 WORKDIR /freqtrade
 
 # Install dependencies
@@ -24,7 +33,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Install dependencies
-COPY requirements.txt requirements-hyperopt.txt /freqtrade/
+COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
+USER ftuser
 RUN  pip install --user --no-cache-dir numpy \
   && pip install --user --no-cache-dir -r requirements-hyperopt.txt
 
@@ -33,13 +43,13 @@ FROM base as runtime-image
 COPY --from=python-deps /usr/local/lib /usr/local/lib
 ENV LD_LIBRARY_PATH /usr/local/lib
 
-COPY --from=python-deps /root/.local /root/.local
-
-
+COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local
 
+USER ftuser
 # Install and execute
-COPY . /freqtrade/
-RUN pip install -e . --no-cache-dir \
+COPY --chown=ftuser:ftuser . /freqtrade/
+
+RUN pip install -e . --user --no-cache-dir \
   && mkdir /freqtrade/user_data/ \
   && freqtrade install-ui
 

Dockerfile.armhf

@@ -5,15 +5,20 @@ ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONFAULTHANDLER 1
-ENV PATH=/root/.local/bin:$PATH
+ENV PATH=/home/ftuser/.local/bin:$PATH
+ENV FT_APP_ENV="docker"
 
 # Prepare environment
-RUN mkdir /freqtrade
-WORKDIR /freqtrade
+RUN mkdir /freqtrade \
+  && apt-get update \
+  && apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \
+  && apt-get clean \
+  && useradd -u 1000 -G sudo -U -m ftuser \
+  && chown ftuser:ftuser /freqtrade \
+  # Allow sudoers
+  && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
 
-RUN  apt-get update \
-  && apt-get -y install libatlas3-base curl sqlite3 \
-  && apt-get clean
+WORKDIR /freqtrade
 
 # Install dependencies
 FROM base as python-deps
@@ -28,7 +33,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Install dependencies
-COPY requirements.txt /freqtrade/
+COPY --chown=ftuser:ftuser requirements.txt /freqtrade/
+USER ftuser
 RUN  pip install --user --no-cache-dir numpy \
   && pip install --user --no-cache-dir -r requirements.txt
 
@@ -37,13 +43,14 @@ FROM base as runtime-image
 COPY --from=python-deps /usr/local/lib /usr/local/lib
 ENV LD_LIBRARY_PATH /usr/local/lib
 
-COPY --from=python-deps /root/.local /root/.local
+COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local
 
+USER ftuser
 # Install and execute
-COPY . /freqtrade/
-RUN apt-get install -y libhdf5-serial-dev \
-  && apt-get clean \
-  && pip install -e . --no-cache-dir \
+COPY --chown=ftuser:ftuser . /freqtrade/
+
+RUN pip install -e . --user --no-cache-dir \
+  && mkdir /freqtrade/user_data/ \
   && freqtrade install-ui
 
 ENTRYPOINT ["freqtrade"]

docker/Dockerfile.custom

@@ -1,7 +1,10 @@
 FROM freqtradeorg/freqtrade:develop
 
-RUN apt-get update \
-    && apt-get -y install git \
-    && apt-get clean \
-    # The below dependency - pyti - serves as an example. Please use whatever you need!
-    && pip install pyti
+# Switch user to root if you must install something from apt
+# Don't forget to switch the user back below!
+# USER root
+
+# The below dependency - pyti - serves as an example. Please use whatever you need!
+RUN pip install --user pyti
+
+# USER ftuser

docker/Dockerfile.develop

@@ -3,8 +3,8 @@ FROM freqtradeorg/freqtrade:develop
 # Install dependencies
 COPY requirements-dev.txt /freqtrade/
 
-RUN pip install numpy --no-cache-dir \
-  && pip install -r requirements-dev.txt --no-cache-dir
+RUN pip install numpy --user --no-cache-dir \
+  && pip install -r requirements-dev.txt --user --no-cache-dir
 
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

docker/Dockerfile.jupyter

@@ -1,7 +1,7 @@
 FROM freqtradeorg/freqtrade:develop_plot
 
 
-RUN pip install jupyterlab --no-cache-dir
+RUN pip install jupyterlab --user --no-cache-dir
 
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

docker/Dockerfile.plot

@@ -4,4 +4,4 @@ FROM freqtradeorg/freqtrade:${sourceimage}
 # Install dependencies
 COPY requirements-plot.txt /freqtrade/
 
-RUN pip install -r requirements-plot.txt --no-cache-dir
+RUN pip install -r requirements-plot.txt --user --no-cache-dir

freqtrade/commands/build_config_commands.py

@@ -5,6 +5,7 @@ from typing import Any, Dict, List
 
 from questionary import Separator, prompt
 
+from freqtrade.configuration.directory_operations import chown_user_directory
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges
@@ -216,6 +217,7 @@ def start_new_config(args: Dict[str, Any]) -> None:
     """
 
     config_path = Path(args['config'][0])
+    chown_user_directory(config_path.parent)
     if config_path.exists():
         overwrite = ask_user_overwrite(config_path)
         if overwrite:

freqtrade/configuration/directory_operations.py

@@ -24,6 +24,21 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Pat
     return folder
 
 
+def chown_user_directory(directory: Path) -> None:
+    """
+    Use Sudo to change permissions of the home-directory if necessary
+    Only applies when running in docker!
+    """
+    import os
+    if os.environ.get('FT_APP_ENV') == 'docker':
+        try:
+            import subprocess
+            subprocess.check_output(
+                ['sudo', 'chown', '-R', 'ftuser:', str(directory.resolve())])
+        except Exception:
+            logger.warning(f"Could not chown {directory}")
+
+
 def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
     """
     Create userdata directory structure.
@@ -37,6 +52,7 @@ def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
     sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "logs",
                 "notebooks", "plot", "strategies", ]
     folder = Path(directory)
+    chown_user_directory(folder)
     if not folder.is_dir():
         if create_dir:
             folder.mkdir(parents=True)

tests/test_directory_operations.py

@@ -1,11 +1,12 @@
 # pragma pylint: disable=missing-docstring, protected-access, invalid-name
+import os
 from pathlib import Path
 from unittest.mock import MagicMock
 
 import pytest
 
-from freqtrade.configuration.directory_operations import (copy_sample_files, create_datadir,
-                                                          create_userdata_dir)
+from freqtrade.configuration.directory_operations import (chown_user_directory, copy_sample_files,
+                                                          create_datadir, create_userdata_dir)
 from freqtrade.exceptions import OperationalException
 from tests.conftest import log_has, log_has_re
 
@@ -31,6 +32,24 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None:
     assert str(x) == str(Path("/tmp/bar"))
 
 
+def test_create_userdata_dir_and_chown(mocker, tmpdir, caplog) -> None:
+    sp_mock = mocker.patch('subprocess.check_output')
+    path = Path(tmpdir / 'bar')
+    assert not path.is_dir()
+
+    x = create_userdata_dir(str(path), create_dir=True)
+    assert sp_mock.call_count == 0
+    assert log_has(f'Created user-data directory: {path}', caplog)
+    assert isinstance(x, Path)
+    assert path.is_dir()
+    assert (path / 'data').is_dir()
+
+    os.environ['FT_APP_ENV'] = 'docker'
+    chown_user_directory(path / 'data')
+    assert sp_mock.call_count == 1
+    del os.environ['FT_APP_ENV']
+
+
 def test_create_userdata_dir_exists(mocker, default_conf, caplog) -> None:
     mocker.patch.object(Path, "is_dir", MagicMock(return_value=True))
     md = mocker.patch.object(Path, 'mkdir', MagicMock())