From 3939106b5ace7808810abd52bf2a8cc5b6f6609b Mon Sep 17 00:00:00 2001 From: richardjozsa Date: Fri, 25 Nov 2022 01:00:53 +0100 Subject: [PATCH] ghf --- Dockerfile.gpu | 58 +++++++++++++++++++++++++++++++++ freqtrade/freqai/data_drawer.py | 25 ++++++++++---- requirements-freqai-rl.txt | 2 +- requirements-freqai.txt | 4 +++ 4 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 Dockerfile.gpu diff --git a/Dockerfile.gpu b/Dockerfile.gpu new file mode 100644 index 000000000..b39332b7c --- /dev/null +++ b/Dockerfile.gpu @@ -0,0 +1,58 @@ +FROM nvidia/cuda:11.8.0-runtime-ubuntu22.04 as base + +# Setup env +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONFAULTHANDLER 1 +ENV PATH=/home/ftuser/.local/bin:$PATH +ENV FT_APP_ENV="docker" + +# Prepare environment +RUN mkdir /freqtrade \ + && apt-get update \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ + && apt-get clean \ + && useradd -u 1000 -G sudo -U -m -s /bin/bash ftuser \ + && chown ftuser:ftuser /freqtrade \ + # Allow sudoers + && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers + +WORKDIR /freqtrade + +# Install dependencies +FROM base as python-deps +RUN apt-get update \ + && apt-get -y install pip build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \ + && apt-get clean \ + && pip install --upgrade pip + +# Install TA-lib +COPY build_helpers/* /tmp/ +RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* +ENV LD_LIBRARY_PATH /usr/local/lib + +# Install dependencies +COPY --chown=ftuser:ftuser requirements.txt requirements-plot.txt requirements-hyperopt.txt requirements-freqai.txt /freqtrade/ +USER ftuser +RUN pip install --user --no-cache-dir numpy \ + && pip install --user --no-cache-dir -r requirements-freqai.txt + +# Copy dependencies to runtime-image +FROM base as runtime-image +COPY --from=python-deps /usr/local/lib /usr/local/lib +ENV LD_LIBRARY_PATH /usr/local/lib + +COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local +RUN sudo apt-get -y install pip +USER ftuser +# Install and execute +COPY --chown=ftuser:ftuser . /freqtrade/ + +RUN pip install -e . --user --no-cache-dir --no-build-isolation \ + && mkdir /freqtrade/user_data/ \ + && freqtrade install-ui + +ENTRYPOINT ["freqtrade"] +# Default to trade mode +CMD [ "trade" ] \ No newline at end of file diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 96b481074..6d38b8c9b 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -1,6 +1,7 @@ import collections import logging import re +import json import shutil import threading from datetime import datetime, timezone @@ -94,14 +95,16 @@ class FreqaiDataDrawer: self.save_lock = threading.Lock() self.pair_dict_lock = threading.Lock() self.metric_tracker_lock = threading.Lock() + self.limit_ram_use = self.freqai_info.get('limit_ram_usage', False) self.old_DBSCAN_eps: Dict[str, float] = {} + self.empty_pair_dict: pair_info = { "model_filename": "", "trained_timestamp": 0, "data_path": "", "extras": {}} - if 'Reinforcement' in self.config['freqaimodel']: - self.model_type = 'stable_baselines' - logger.warning('User passed a ReinforcementLearner model, FreqAI will ' - 'now use stable_baselines3 to save models.') + if 'rl_config' in self.freqai_info: + self.model_type = self.freqai_info['model_save_type'] + logger.warning(f'User passed a ReinforcementLearner model, FreqAI will ' + 'now use {self.model_type} to save models.') else: self.model_type = self.freqai_info.get('model_save_type', 'joblib') @@ -488,6 +491,8 @@ class FreqaiDataDrawer: model.save(save_path / f"{dk.model_filename}_model.h5") elif 'stable_baselines' in self.model_type: model.save(save_path / f"{dk.model_filename}_model.zip") + elif 'sb3_contrib' in self.model_type: + model.save(save_path / f"{dk.model_filename}_model.zip") if dk.svm_model is not None: dump(dk.svm_model, save_path / f"{dk.model_filename}_svm_model.joblib") @@ -565,7 +570,7 @@ class FreqaiDataDrawer: dk.label_list = dk.data["label_list"] # try to access model in memory instead of loading object from disk to save time - if dk.live and coin in self.model_dictionary: + if dk.live and coin in self.model_dictionary and not self.limit_ram_use: model = self.model_dictionary[coin] elif self.model_type == 'joblib': model = load(dk.data_path / f"{dk.model_filename}_model.joblib") @@ -576,10 +581,16 @@ class FreqaiDataDrawer: mod = __import__('stable_baselines3', fromlist=[ self.freqai_info['rl_config']['model_type']]) MODELCLASS = getattr(mod, self.freqai_info['rl_config']['model_type']) - model = MODELCLASS.load(dk.data_path / f"{dk.model_filename}_model") + model = MODELCLASS.load(dk.data_path / f"{dk.model_filename}_model", device="cpu") + + elif self.model_type == 'sb3_contrib': + mod = __import__('sb3_contrib', fromlist=[ + self.freqai_info['rl_config']['model_type']]) + MODELCLASS = getattr(mod, self.freqai_info['rl_config']['model_type']) + model = MODELCLASS.load(dk.data_path / f"{dk.model_filename}_model", device="cpu") if Path(dk.data_path / f"{dk.model_filename}_svm_model.joblib").is_file(): - dk.svm_model = load(dk.data_path / f"{dk.model_filename}_svm_model.joblib") + dk.svm_model = load(path=dk.data_path / f"{dk.model_filename}_svm_model.joblib") if not model: raise OperationalException( diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index b6bd7ef15..e61c032bb 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -2,7 +2,7 @@ -r requirements-freqai.txt # Required for freqai-rl -torch==1.12.1 +torch==1.3.0 stable-baselines3==1.6.1 gym==0.21 sb3-contrib==1.6.1 diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 66730e29f..6a00b77e1 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -9,3 +9,7 @@ catboost==1.1.1; platform_machine != 'aarch64' lightgbm==3.3.3 xgboost==1.7.1 tensorboard==2.11.0 +torch==1.13.0 +stable-baselines3==1.6.2 +gym==0.21 +sb3-contrib==1.6.2 \ No newline at end of file