Merge branch 'develop' into feat/externalsignals

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         additional_dependencies:
           - types-cachetools==5.2.1
           - types-filelock==3.2.7
-          - types-requests==2.28.9
+          - types-requests==2.28.10
           - types-tabulate==0.8.11
           - types-python-dateutil==2.8.19
         # stages: [push]

docs/data-download.md

@@ -25,8 +25,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [--include-inactive-pairs]
                                [--timerange TIMERANGE] [--dl-trades]
                                [--exchange EXCHANGE]
-                               [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
+                               [-t TIMEFRAMES [TIMEFRAMES ...]] [--erase]
-                               [--erase]
                                [--data-format-ohlcv {json,jsongz,hdf5}]
                                [--data-format-trades {json,jsongz,hdf5}]
                                [--trading-mode {spot,margin,futures}]
@@ -37,7 +36,8 @@ optional arguments:
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Limit command to these pairs. Pairs are space-
                         separated.
-  --pairs-file FILE     File containing a list of pairs to download.
+  --pairs-file FILE     File containing a list of pairs. Takes precedence over
+                        --pairs or pairs configured in the configuration.
   --days INT            Download data for given number of days.
   --new-pairs-days INT  Download data of new pairs for given number of days.
                         Default: `None`.
@@ -50,7 +50,7 @@ optional arguments:
                         as --timeframes/-t.
   --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
                         config is provided.
-  -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
+  -t TIMEFRAMES [TIMEFRAMES ...], --timeframes TIMEFRAMES [TIMEFRAMES ...]
                         Specify which tickers to download. Space-separated
                         list. Default: `1m 5m`.
   --erase               Clean all existing data for the selected
@@ -61,7 +61,7 @@ optional arguments:
   --data-format-trades {json,jsongz,hdf5}
                         Storage format for downloaded trades data. (default:
                         `jsongz`).
-  --trading-mode {spot,margin,futures}
+  --trading-mode {spot,margin,futures}, --tradingmode {spot,margin,futures}
                         Select Trading mode
   --prepend             Allow data prepending. (Data-appending is disabled)
 

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.3.7
 mkdocs==1.3.1
-mkdocs-material==8.4.2
+mkdocs-material==8.4.3
 mdx_truly_sane_lists==1.3
 pymdown-extensions==9.5
 jinja2==3.1.2

docs/utils.md

@@ -525,12 +525,14 @@ Requires a configuration with specified `pairlists` attribute.
 Can be used to generate static pairlists to be used during backtesting / hyperopt.
 
 ```
-usage: freqtrade test-pairlist [-h] [-v] [-c PATH]
+usage: freqtrade test-pairlist [-h] [--userdir PATH] [-v] [-c PATH]
                                [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]]
                                [-1] [--print-json] [--exchange EXCHANGE]
 
 optional arguments:
   -h, --help            show this help message and exit
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
   -c PATH, --config PATH
                         Specify configuration file (default:

freqtrade/commands/arguments.py

@@ -53,8 +53,8 @@ ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one
                    "print_csv", "base_currencies", "quote_currencies", "list_pairs_all",
                    "trading_mode"]
 
-ARGS_TEST_PAIRLIST = ["verbosity", "config", "quote_currencies", "print_one_column",
+ARGS_TEST_PAIRLIST = ["user_data_dir", "verbosity", "config", "quote_currencies",
-                      "list_pairs_print_json", "exchange"]
+                      "print_one_column", "list_pairs_print_json", "exchange"]
 
 ARGS_CREATE_USERDIR = ["user_data_dir", "reset"]
 

freqtrade/commands/cli_options.py

@@ -393,7 +393,8 @@ AVAILABLE_CLI_OPTIONS = {
     # Download data
     "pairs_file": Arg(
         '--pairs-file',
-        help='File containing a list of pairs to download.',
+        help='File containing a list of pairs. '
+             'Takes precedence over --pairs or pairs configured in the configuration.',
         metavar='FILE',
     ),
     "days": Arg(

freqtrade/data/dataprovider.py

@@ -206,7 +206,7 @@ class DataProvider:
         """
         _candle_type = CandleType.from_string(
             candle_type) if candle_type != '' else self._config['candle_type_def']
-        saved_pair = (pair, str(timeframe), _candle_type)
+        saved_pair: PairWithTimeframe = (pair, str(timeframe), _candle_type)
         if saved_pair not in self.__cached_pairs_backtesting:
             timerange = TimeRange.parse_timerange(None if self._config.get(
                 'timerange') is None else str(self._config.get('timerange')))

freqtrade/optimize/hyperopt.py

@@ -290,7 +290,7 @@ class Hyperopt:
                 # noinspection PyProtectedMember
                 attr.value = params_dict[attr_name]
 
-    def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
+    def generate_optimizer(self, raw_params: List[Any]) -> Dict[str, Any]:
         """
         Used Optimize function.
         Called once per epoch to optimize whatever is configured.
@@ -410,9 +410,11 @@ class Hyperopt:
             model_queue_size=SKOPT_MODEL_QUEUE_SIZE,
         )
 
-    def run_optimizer_parallel(self, parallel, asked, i) -> List:
+    def run_optimizer_parallel(
+            self, parallel: Parallel, asked: List[List]) -> List[Dict[str, Any]]:
+        """ Start optimizer in a parallel way """
         return parallel(delayed(
-                        wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked)
+                        wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked)
 
     def _set_random_state(self, random_state: Optional[int]) -> int:
         return random_state or random.randint(1, 2**16 - 1)
@@ -491,6 +493,53 @@ class Hyperopt:
         else:
             return self.opt.ask(n_points=n_points), [False for _ in range(n_points)]
 
+    def get_progressbar_widgets(self):
+        if self.print_colorized:
+            widgets = [
+                ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
+                ' (', progressbar.Percentage(), ')] ',
+                progressbar.Bar(marker=progressbar.AnimatedMarker(
+                    fill='\N{FULL BLOCK}',
+                    fill_wrap=Fore.GREEN + '{}' + Fore.RESET,
+                    marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL,
+                )),
+                ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
+            ]
+        else:
+            widgets = [
+                ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
+                ' (', progressbar.Percentage(), ')] ',
+                progressbar.Bar(marker=progressbar.AnimatedMarker(
+                    fill='\N{FULL BLOCK}',
+                )),
+                ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
+            ]
+        return widgets
+
+    def evaluate_result(self, val: Dict[str, Any], current: int, is_random: bool):
+        """
+        Evaluate results returned from generate_optimizer
+        """
+        val['current_epoch'] = current
+        val['is_initial_point'] = current <= INITIAL_POINTS
+
+        logger.debug("Optimizer epoch evaluated: %s", val)
+
+        is_best = HyperoptTools.is_best_loss(val, self.current_best_loss)
+        # This value is assigned here and not in the optimization method
+        # to keep proper order in the list of results. That's because
+        # evaluations can take different time. Here they are aligned in the
+        # order they will be shown to the user.
+        val['is_best'] = is_best
+        val['is_random'] = is_random
+        self.print_results(val)
+
+        if is_best:
+            self.current_best_loss = val['loss']
+            self.current_best_epoch = val
+
+        self._save_result(val)
+
     def start(self) -> None:
         self.random_state = self._set_random_state(self.config.get('hyperopt_random_state'))
         logger.info(f"Using optimizer random state: {self.random_state}")
@@ -526,26 +575,7 @@ class Hyperopt:
                 logger.info(f'Effective number of parallel workers used: {jobs}')
 
                 # Define progressbar
-                if self.print_colorized:
+                widgets = self.get_progressbar_widgets()
-                    widgets = [
-                        ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
-                        ' (', progressbar.Percentage(), ')] ',
-                        progressbar.Bar(marker=progressbar.AnimatedMarker(
-                            fill='\N{FULL BLOCK}',
-                            fill_wrap=Fore.GREEN + '{}' + Fore.RESET,
-                            marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL,
-                        )),
-                        ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
-                    ]
-                else:
-                    widgets = [
-                        ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs),
-                        ' (', progressbar.Percentage(), ')] ',
-                        progressbar.Bar(marker=progressbar.AnimatedMarker(
-                            fill='\N{FULL BLOCK}',
-                        )),
-                        ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
-                    ]
                 with progressbar.ProgressBar(
                     max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
                     widgets=widgets
@@ -558,32 +588,15 @@ class Hyperopt:
                         current_jobs = jobs - n_rest if n_rest > 0 else jobs
 
                         asked, is_random = self.get_asked_points(n_points=current_jobs)
-                        f_val = self.run_optimizer_parallel(parallel, asked, i)
+                        f_val = self.run_optimizer_parallel(parallel, asked)
                         self.opt.tell(asked, [v['loss'] for v in f_val])
 
                         # Calculate progressbar outputs
                         for j, val in enumerate(f_val):
                             # Use human-friendly indexes here (starting from 1)
                             current = i * jobs + j + 1
-                            val['current_epoch'] = current
-                            val['is_initial_point'] = current <= INITIAL_POINTS
 
-                            logger.debug(f"Optimizer epoch evaluated: {val}")
+                            self.evaluate_result(val, current, is_random[j])
-
-                            is_best = HyperoptTools.is_best_loss(val, self.current_best_loss)
-                            # This value is assigned here and not in the optimization method
-                            # to keep proper order in the list of results. That's because
-                            # evaluations can take different time. Here they are aligned in the
-                            # order they will be shown to the user.
-                            val['is_best'] = is_best
-                            val['is_random'] = is_random[j]
-                            self.print_results(val)
-
-                            if is_best:
-                                self.current_best_loss = val['loss']
-                                self.current_best_epoch = val
-
-                            self._save_result(val)
 
                             pbar.update(current)
 

requirements-dev.txt

@@ -25,6 +25,6 @@ nbconvert==7.0.0
 # mypy types
 types-cachetools==5.2.1
 types-filelock==3.2.7
-types-requests==2.28.9
+types-requests==2.28.10
 types-tabulate==0.8.11
 types-python-dateutil==2.8.19

requirements.txt

@@ -1,18 +1,18 @@
-numpy==1.23.2
+numpy==1.23.3
 pandas==1.4.4
 pandas-ta==0.3.14b
 
-ccxt==1.93.3
+ccxt==1.93.35
 # Pin cryptography for now due to rust build errors with piwheels
-cryptography==37.0.4
+cryptography==38.0.1
 aiohttp==3.8.1
-SQLAlchemy==1.4.40
+SQLAlchemy==1.4.41
 python-telegram-bot==13.14
 arrow==1.2.3
 cachetools==4.2.2
 requests==2.28.1
 urllib3==1.26.12
-jsonschema==4.15.0
+jsonschema==4.16.0
 TA-Lib==0.4.24
 technical==1.3.0
 tabulate==0.8.10
@@ -34,10 +34,10 @@ orjson==3.8.0
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.82.0
+fastapi==0.83.0
 uvicorn==0.18.3
 pyjwt==2.4.0
-aiofiles==0.8.0
+aiofiles==22.1.0
 psutil==5.9.2
 
 # Support for colorized terminal output