diff --git a/docs/assets/freqai_algo.png b/docs/assets/freqai_algo.png index 39fb89855..13813e129 100644 Binary files a/docs/assets/freqai_algo.png and b/docs/assets/freqai_algo.png differ diff --git a/docs/assets/freqai_doc_logo.svg b/docs/assets/freqai_doc_logo.svg index 61561a4ae..23b116883 100644 --- a/docs/assets/freqai_doc_logo.svg +++ b/docs/assets/freqai_doc_logo.svg @@ -1,196 +1,122 @@ - - + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" + id="svg1557" + version="1.1" + viewBox="0 0 378.20105 327.63284" + height="327.63284mm" + width="378.20105mm"> + id="linearGradient1933"> + + + + - - - - - - - - - + - + - + - - + + + y1="223.87135" + x1="192.64189" + gradientUnits="userSpaceOnUse" + id="linearGradient1927" + xlink:href="#linearGradient1933" /> + + + + + + + + + + - @@ -199,219 +125,180 @@ image/svg+xml + + id="layer10" /> + id="layer3"> + transform="translate(-26.725716,-4.0064965)"> + style="fill:#000100;fill-opacity:1" + id="g1939"> + style="fill:#000000;fill-opacity:1" + id="g1980"> + d="m 266.32136,50.725653 c -0.17493,-0.245241 -7.68487,8.239682 -26.22382,14.73088 -7.24598,2.533621 -15.77758,4.481015 -25.80626,5.394041 -7.80487,0.710575 -15.62741,0.763515 -24.60618,0.630177 -7.10056,-0.102515 -15.37349,-0.342644 -23.09918,0.04518 -6.98666,0.355519 -14.3529,1.246148 -21.37728,3.473919 -6.97871,2.201021 -13.75622,5.822394 -19.73033,10.773148 -7.23031,5.971496 -13.21751,13.650026 -18.4504,22.28169 -7.71197,11.30811 -13.780321,23.77515 -19.281337,37.15557 1.295375,-2.3837 2.694224,-4.71512 4.192806,-6.9856 5.836914,-8.84339 12.980981,-16.45315 20.947101,-22.55626 10.62125,-7.27796 22.31318,-12.27508 33.83042,-15.689311 12.27636,-3.642327 24.65859,-5.659686 36.2496,-6.893641 0,0 2e-5,-10e-7 2e-5,-10e-7 12.03309,-1.282998 24.15119,-1.797386 34.78058,-2.16223 11.26459,-0.389119 19.86622,-0.525899 29.44169,-1.406911 4.09388,-0.377806 7.84924,-0.855525 11.47336,-1.637412 3.18585,-0.693967 6.04795,-1.536888 8.60531,-2.849228 2.21977,-1.150499 4.08494,-2.575182 5.29519,-4.367908 1.34174,-1.920808 1.87087,-4.302373 1.91674,-6.474343 0.0733,-3.475689 -0.97184,-6.941781 -1.92033,-9.602842 -3.19334,-8.90044 -6.36181,-13.294589 -6.11214,-13.451663 -0.0986,0.06202 3.85329,3.801837 7.67284,12.841713 1.14175,2.686839 2.40195,6.281391 2.50827,10.203303 0.0666,2.503277 -0.46622,5.321701 -2.03856,7.820298 -1.456,2.250702 -3.58735,4.048489 -6.07155,5.390925 -2.81347,1.533986 -5.89422,2.555776 -9.20041,3.349538 -3.76725,0.911916 -7.64484,1.504554 -11.7989,1.977 -9.64024,1.09912 -18.4609,1.414892 -29.61258,1.971722 -10.58225,0.530571 -22.64632,1.198502 -34.47024,2.5876 0,10e-7 -2e-5,2e-6 -2e-5,2e-6 -11.41113,1.342488 -23.52949,3.414879 -35.41817,7.033139 -11.14376,3.39447 -22.38681,8.24563 -32.47462,15.22018 -8.62388,5.31456 -16.51649,12.04079 -23.182072,19.92101 -1.711299,2.02314 -3.326286,4.10426 -4.840621,6.23506 4.087878,-14.26449 8.922214,-27.70649 15.564853,-40.02514 5.43579,-8.92853 11.81856,-16.959506 19.57969,-23.30942 6.46827,-5.275585 13.77515,-9.090373 21.29545,-11.380298 7.54855,-2.286045 15.2578,-3.139455 22.45583,-3.393724 7.96548,-0.286584 16.27109,0.08204 23.35306,0.311442 8.95318,0.286298 16.61396,0.397407 24.27686,-0.12155 9.8236,-0.665207 18.16345,-2.307894 25.31931,-4.527563 18.00102,-5.575038 26.70936,-12.858284 26.95602,-12.512494 z" + style="display:inline;opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" + transform="matrix(1.0227138,0,0,1.0923454,-62.375629,65.947122)" /> + transform="matrix(1.0227138,0,0,1.0923454,-103.51834,34.329414)" /> + d="m 338.69593,78.188464 c -0.10321,0.0166 3.37905,8.851837 3.64691,24.953626 0.0892,4.61727 -0.1108,10.55624 -2.18945,16.22456 -1.2491,3.41284 -3.51339,6.88006 -7.02949,9.22007 -3.18537,2.03978 -6.9108,3.01105 -10.71996,3.31059 -4.32758,0.36681 -8.61596,0.0765 -13.06414,-0.3954 0,0 -1e-5,0 -1e-5,0 -4.85825,-0.50718 -9.81996,-1.2337 -14.88816,-1.8578 -10.52725,-1.92207 -22.25938,-3.65993 -33.51458,-2.56837 5.56514,-2.14789 11.23402,-4.37262 16.96405,-6.67029 14.09947,-5.65373 27.22086,-11.20878 39.76757,-16.84629 10.34679,-4.204445 20.04709,-8.578761 29.2102,-12.254895 3.86403,-1.55087 7.5962,-2.912625 11.31631,-4.062522 4.44997,-1.373998 8.80842,-2.424612 12.87075,-3.255903 25.5702,-5.229185 41.77669,-4.819543 41.7651,-4.910481 0.083,0.651054 -16.04487,2.199237 -41.03765,8.188979 -3.95341,0.947245 -8.17817,2.096907 -12.44827,3.538622 -3.57249,1.204947 -7.16926,2.614455 -10.93178,4.214907 -8.99368,3.826281 -18.60554,8.342603 -29.02526,12.707823 -12.83398,4.83575 -26.25198,9.37248 -40.70852,13.92084 -5.86346,1.84478 -11.6791,3.62297 -17.36766,5.33435 11.65665,-2.73331 23.3668,-2.48912 33.70665,-1.70227 5.13345,0.7007 10.05152,1.49994 14.79046,2.07946 0,0 10e-6,0 10e-6,0 4.36154,0.52566 8.36803,0.89758 12.29878,0.63761 3.41571,-0.24885 6.55597,-0.91553 9.05656,-2.49093 2.75513,-1.64741 4.69418,-4.48749 5.88989,-7.34626 2.04294,-4.90572 2.44367,-10.54405 2.602,-15.03686 0.50756,-15.585165 -1.45279,-24.853973 -0.96031,-24.933166 z" + id="path881" + transform="matrix(1.0227138,0,0,1.0923454,-62.375629,65.947122)" /> + id="layer7"> + transform="matrix(0.99993492,0,0,0.99993492,13.017271,44.180298)"> + style="stroke-width:0.2593" + id="g2294"> + id="path2276" /> + id="path2278" /> + id="path2280" /> + id="path2282" /> + id="path2284" /> + id="path2286" /> + id="path2288" /> + id="path2290" /> + id="path2292" /> + id="path2296" /> + y="146.5" + width="51.759998" + height="21.709999" + ry="2.1700001" + id="rect2298" /> + id="path2302" /> + id="path2304" /> - FreqAI + id="path2306" /> - - - - - - - - - - + id="layer2"> + + + + + + + + + FreqAI + + + + + d="m 409.50476,87.379983 c 0.0817,-0.04926 0.83007,0.887208 2.07545,2.822175 1.04507,1.623744 3.25835,5.167411 4.12211,9.194511 0.24011,1.048051 0.37549,2.363451 -0.10902,3.660331 -0.18859,0.47508 -0.48353,0.9229 -0.88238,1.29561 -0.4008,0.36595 -0.86666,0.63703 -1.35775,0.81687 -1.19769,0.43163 -2.45385,0.47703 -3.68135,0.37305 -1.61056,-0.11875 -3.19793,-0.44421 -4.84759,-0.87259 -4.16825,-1.07786 -8.14201,-2.5878 -12.79809,-4.4764 -9.73087,-3.941923 -20.77511,-9.129422 -33.23314,-14.263637 -5.48498,-2.260172 -11.39743,-4.574201 -17.4774,-6.560008 -5.09213,-1.660645 -10.48843,-3.19017 -16.00487,-4.015333 -2.9914,-0.437506 -6.09153,-0.46961 -8.95818,0.325781 -2.42804,0.652502 -4.7293,1.92692 -6.7488,3.603104 -2.16014,1.788823 -4.01151,4.022183 -5.76455,6.441443 -1.98455,2.749884 -3.76186,5.674924 -5.58974,8.71081 -4.19023,6.97383 -8.63594,14.41792 -14.06138,21.13172 -0.4076,0.45385 -0.82052,0.90435 -1.23872,1.35135 -6.99445,7.4762 -15.14736,13.64043 -23.95364,18.25267 1.08898,-2.79485 1.7844,-5.77129 1.81896,-8.80392 0,0 10e-6,-1e-5 10e-6,-1e-5 0.38,-2.40424 0.21457,-4.88822 -0.38967,-7.23261 -0.63616,-2.48616 -1.77286,-4.85563 -3.1418,-7.05996 -1.53736,-2.46945 -3.39699,-4.74262 -5.3732,-6.86136 -4.50934,-4.83407 -9.76414,-8.980583 -14.8279,-12.716321 -5.31289,-3.914842 -10.63799,-7.453058 -15.64776,-10.989417 -2.22769,-1.571719 -4.36129,-3.115548 -6.42213,-4.778903 -1.64818,-1.324283 -3.23931,-2.705582 -4.62447,-4.342456 -0.53145,-0.625434 -1.03333,-1.314573 -1.44675,-2.091148 -0.34225,-0.627813 -0.60475,-1.357327 -0.69952,-2.169834 0,0 0,-1.4e-5 0,-1.4e-5 -0.0779,-0.750702 0.0462,-1.512763 0.367,-2.209488 0,0 0,-1.2e-5 0,-1.2e-5 0.38316,-0.77978 0.92391,-1.386219 1.50254,-1.835437 0.78178,-0.619545 1.62171,-1.06389 2.4332,-1.407758 1.13734,-0.488832 2.29126,-0.844274 3.40719,-1.132195 3.50309,-0.906013 7.05979,-1.338662 10.22546,-1.650874 23.18125,-2.299439 36.94463,-0.08504 36.9438,-0.452268 8.7e-4,0.388728 -13.90634,-0.336072 -36.71433,2.505031 -3.12101,0.390538 -6.57136,0.885582 -9.85579,1.814406 -1.0434,0.295814 -2.08057,0.634827 -3.04947,1.07809 -0.69296,0.321905 -1.34571,0.671292 -1.86387,1.113717 -0.364,0.320857 -0.67323,0.650293 -0.8072,0.991792 0,0 0,5e-6 0,5e-6 -0.12418,0.268103 -0.16576,0.599529 -0.12769,0.918029 0,0 0,6e-6 0,6e-6 0.0351,0.384659 0.21456,0.805098 0.44854,1.252612 0.2987,0.552334 0.70811,1.082464 1.17179,1.620759 1.22674,1.417525 2.71555,2.653886 4.3349,3.928004 2.01237,1.575728 4.11873,3.052334 6.3518,4.584232 5.01914,3.441387 10.39585,6.923268 15.83064,10.840983 5.16939,3.721829 10.59624,7.938422 15.36962,12.963249 2.09536,2.20557 4.09636,4.62677 5.7996,7.31323 1.52509,2.40018 2.80715,5.06206 3.57603,7.94655 0.72733,2.74549 0.93047,5.67197 0.49493,8.56235 0,0 0,2e-5 0,2e-5 -0.86253,2.79257 -2.45308,5.41852 -4.50792,7.73608 7.7848,-5.12796 14.91437,-11.79762 20.9229,-19.73477 0.35932,-0.47465 0.71367,-0.95257 1.063,-1.43365 5.23266,-6.45055 9.60687,-13.768177 13.82746,-20.73084 1.83968,-3.041164 3.68014,-6.039283 5.77405,-8.922486 1.84136,-2.545239 3.89517,-4.972657 6.36038,-7.016242 2.34517,-1.940232 5.06674,-3.410413 7.99843,-4.196264 3.44628,-0.898759 6.97893,-0.864117 10.27251,-0.332556 5.7724,0.915436 11.31553,2.532415 16.47253,4.277279 6.17081,2.084811 12.11826,4.490476 17.61006,6.839344 12.48717,5.339979 23.40869,10.705759 32.98675,14.843562 4.60517,1.986833 8.42219,3.560413 12.44172,4.725603 1.58792,0.45848 3.05427,0.81419 4.51576,0.96734 1.13127,0.10478 2.14607,0.13666 3.01995,-0.16986 0.3399,-0.11628 0.6453,-0.27004 0.87952,-0.47523 0.23454,-0.19697 0.42225,-0.46007 0.55127,-0.74468 0.39736,-0.81327 0.30177,-1.89803 0.16324,-2.868152 -0.59137,-3.720132 -2.57863,-7.437888 -3.43481,-9.095384 -1.05035,-2.033393 -1.60487,-3.091683 -1.49222,-3.159631 z" + style="display:inline;opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" + transform="matrix(1.0227138,0,0,1.0923454,-180.74242,54.998164)" /> diff --git a/docs/freqai.md b/docs/freqai.md index eb76ab1a4..093637479 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -6,16 +6,16 @@ FreqAI is a module designed to automate a variety of tasks associated with train Among the the features included: -* **Self-adaptive retraining**: automatically retrain models during live deployments to self-adapt to the market in an unsupervised manner. +* **Self-adaptive retraining**: retrain models during live deployments to self-adapt to the market in an unsupervised manner. * **Rapid feature engineering**: create large rich feature sets (10k+ features) based on simple user created strategies. * **High performance**: adaptive retraining occurs on separate thread (or on GPU if available) from inferencing and bot trade operations. Keep newest models and data in memory for rapid inferencing. * **Realistic backtesting**: emulate self-adaptive retraining with backtesting module that automates past retraining. * **Modifiable**: use the generalized and robust architecture for incorporating any machine learning library/method available in Python. Seven examples available. -* **Smart outlier removal**: remove outliers automatically from training and prediction sets using a variety of outlier detection techniques. -* **Crash resilience**: automatic model storage to disk to make reloading from a crash fast and easy (and purge obsolete files automatically for sustained dry/live runs). -* **Automated data normalization**: automatically normalize the data automatically in a smart and statistically safe way. -* **Automatic data download**: automatically compute the data download timerange and downloads data accordingly (in live deployments). -* **Clean the incoming data of NaNs in a safe way before training and prediction. +* **Smart outlier removal**: remove outliers from training and prediction sets using a variety of outlier detection techniques. +* **Crash resilience**: model storage to disk to make reloading from a crash fast and easy (and purge obsolete files for sustained dry/live runs). +* **Automated data normalization**: normalize the data in a smart and statistically safe way. +* **Automatic data download**: compute the data download timerange and update historic data (in live deployments). +* **Clean incoming data** safe NaN handling before training and prediction. * **Dimensionality reduction**: reduce the size of the training data via Principal Component Analysis. * **Deploy bot fleets**: set one bot to train models while a fleet of other bots inference into the models and handle trades. @@ -412,10 +412,75 @@ The FreqAI strategy requires the user to include the following lines of code in dataframe = self.freqai.start(dataframe, metadata, self) return dataframe + + def populate_any_indicators( + self, pair, df, tf, informative=None, set_generalized_indicators=False + ): + """ + Function designed to automatically generate, name and merge features + from user indicated timeframes in the configuration file. User controls the indicators + passed to the training/prediction by prepending indicators with `'%-' + coin ` + (see convention below). I.e. user should not prepend any supporting metrics + (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the + model. + :param pair: pair to be used as informative + :param df: strategy dataframe which will receive merges from informatives + :param tf: timeframe of the dataframe which will modify the feature names + :param informative: the dataframe associated with the informative pair + :param coin: the name of the coin which will modify the feature names. + """ + + coint = pair.split('/')[0] + + with self.freqai.lock: + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) + + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) + + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) + + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) + + return df + + ``` -The user should also include `populate_any_indicators()` from `templates/FreqaiExampleStrategy.py` which builds -the feature set with a proper naming convention for the IFreqaiModel to use later. +Notice how the `populate_any_indicators()` is where the user adds their own features and labels (more information [here](#feature-engineering)). See a full example at `templates/FreqaiExampleStrategy.py`. ### Setting classifier targets diff --git a/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py b/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py index afb439cbf..04eff045f 100644 --- a/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py +++ b/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py @@ -24,11 +24,11 @@ class BaseTensorFlowModel(IFreqaiModel): for storing, saving, loading, and analyzing the data. :param unfiltered_dataframe: Full dataframe for the current training period :param metadata: pair metadata from strategy. - :returns: + :return: :model: Trained model which can be used to inference (self.predict) """ - logger.info("--------------------Starting training " f"{pair} --------------------") + logger.info("-------------------- Starting training " f"{pair} --------------------") # filter the features requested by user in the configuration file and elegantly handle NaNs features_filtered, labels_filtered = dk.filter_features( @@ -38,9 +38,14 @@ class BaseTensorFlowModel(IFreqaiModel): training_filter=True, ) + start_date = unfiltered_dataframe["date"].iloc[0].strftime("%Y-%m-%d") + end_date = unfiltered_dataframe["date"].iloc[-1].strftime("%Y-%m-%d") + logger.info(f"-------------------- Training on data from {start_date} to " + f"{end_date}--------------------") # split data into train/test data. data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered) - + if not self.freqai_info.get('fit_live_predictions', 0) or not self.live: + dk.fit_labels() # normalize all data based on train_dataset only data_dictionary = dk.normalize_data(data_dictionary) @@ -54,17 +59,6 @@ class BaseTensorFlowModel(IFreqaiModel): model = self.fit(data_dictionary) - if pair not in self.dd.historic_predictions: - self.set_initial_historic_predictions( - data_dictionary['train_features'], model, dk, pair) - - if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live: - self.fit_live_predictions(dk) - else: - dk.fit_labels() - - self.dd.save_historic_predictions_to_disk() - logger.info(f"--------------------done training {pair}--------------------") return model