diff --git a/parallel/README.md b/parallel/README.md new file mode 100644 index 000000000..0602116e1 --- /dev/null +++ b/parallel/README.md @@ -0,0 +1,150 @@ +1. install parallel: + +```unzip pp-1.6.4.4.zip``` + +```cd pp-1.6.4.4``` + +```python3.6 setup.py install``` + +2. Move Generator.py into the parent folder, or your "main freqtrade folder." + +```mv Generator.py ../``` + +3. Move the default strategy over the default strategy in freqtrade/strategies folder. + +4. Move backtesting.py over the backtesting.py in freqtrade/optimize folder. + +5. Optinlally install modded bittrex.py + +6. Install dependencies: + +``` +sudo add-apt-repository ppa:jonathonf/python-3.6 + +sudo apt-get update + +sudo apt-get install python3.6 python3.6-venv python3.6-dev build-essential autoconf libtool pkg-config make wget git + +``` + + +7. Install ta-lib: + +``` +wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz + +tar xvzf ta-lib-0.4.0-src.tar.gz + +cd ta-lib + +./configure --prefix=/usr + +make + +make install + +cd .. + +rm -rf ./ta-lib* + +``` + +8. Install freqtrade: + +``` +cd ~/freqtrade && pip3.6 install -r requirements.txt && python3.6 setup.py install && pip3.6 install -e . +``` + +9. Run generator.py: + +```python3.6 generator.py``` + +10. Wait for results. + +11. Implement these results into your default_strategy: + +*** +You will need to read the if statements and populate_buy_signal and populate_sell_signal in this file carefully. + +Once implemented, remove the if statements in the populate_buy_trend. + +*** + +For ease of use, here is an example: + +The if statements that run the random generator are: + +``` + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in str(self.params): + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in str(self.params): + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in str(self.params): + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in str(self.params): + + conditions.append(dataframe['mfi'] < self.valm) + if 'fastd' in str(self.params): + + conditions.append(dataframe['fastd'] < self.valfast) + if 'adx' in str(self.params): + + conditions.append(dataframe['adx'] > self.valadx) + if 'rsi' in str(self.params): + + conditions.append(dataframe['rsi'] < self.valrsi) + if 'over_sar' in str(self.params): + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in str(self.params): + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in str(self.params): + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + if 'closebb' in str(self.params): + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if 'temabb' in str(self.params): + conditions.append(dataframe['tema'] < dataframe['bb_lowerband']) + if 'fastdt' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['fastd'], 10.0)) + if 'ao' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['ao'], 0.0)) + if 'ema3' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['ema3'], dataframe['ema10'])) + if 'macd' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['macd'], dataframe['macdsignal'])) + if 'closesar' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['close'], dataframe['sar'])) + if 'htsine' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['htleadsine'], dataframe['htsine'])) + if 'has' in str(self.params): + conditions.append((qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & (dataframe['ha_low'] == dataframe['ha_open'])) + if 'plusdi' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['plus_di'], dataframe['minus_di'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe +``` + + +So of you get MFI as a option runnning generator.py, and it's option in output is 91, look at the if statements above at mfi, the populate_buy_trend will now look like this: + + +``` + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + dataframe.loc[ + ( + (dataframe['mfi'] < 91) + ), + 'buy'] = 1 + + return dataframe +``` + +It's as simple as that. diff --git a/parallel/backtesting.py b/parallel/backtesting.py new file mode 100644 index 000000000..d45b40796 --- /dev/null +++ b/parallel/backtesting.py @@ -0,0 +1,288 @@ +# pragma pylint: disable=missing-docstring, W0212, too-many-arguments + +""" +This module contains the backtesting logic +""" +import logging +from argparse import Namespace +from typing import Dict, Tuple, Any, List, Optional + +import arrow +from pandas import DataFrame, Series +from tabulate import tabulate + +import freqtrade.optimize as optimize +from freqtrade import exchange +from freqtrade.analyze import Analyze +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.exchange import Bittrex +from freqtrade.misc import file_dump_json +from freqtrade.persistence import Trade + + +logger = logging.getLogger(__name__) + + +class Backtesting(object): + """ + Backtesting class, this class contains all the logic to run a backtest + + To run a backtest: + backtesting = Backtesting(config) + backtesting.start() + """ + def __init__(self, config: Dict[str, Any]) -> None: + self.config = config + self.analyze = None + self.ticker_interval = None + self.tickerdata_to_dataframe = None + self.populate_buy_trend = None + self.populate_sell_trend = None + self._init() + + def _init(self) -> None: + """ + Init objects required for backtesting + :return: None + """ + self.analyze = Analyze(self.config) + self.ticker_interval = self.analyze.strategy.ticker_interval + self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe + self.populate_buy_trend = self.analyze.populate_buy_trend + self.populate_sell_trend = self.analyze.populate_sell_trend + exchange._API = Bittrex({'key': '', 'secret': ''}) + + @staticmethod + def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum timeframe for the given backtest data + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + all_dates = Series([]) + for pair_data in data.values(): + all_dates = all_dates.append(pair_data['date']) + all_dates.sort_values(inplace=True) + return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) + + def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: + """ + Generates and returns a text table for the given backtest data and the results dataframe + :return: pretty printed table with tabulate as str + """ + stake_currency = self.config.get('stake_currency') + + floatfmt = ('.8f', '.8f', '.8f', '.8f', '.8f') + tabular_data = [] + headers = ['total profit ' + stake_currency] + + # Append Total + tabular_data.append([ + 'TOTAL', + results.profit_BTC.sum(), + ]) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) + + def _get_sell_trade_entry( + self, pair: str, buy_row: DataFrame, + partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[Tuple]: + + stake_amount = args['stake_amount'] + max_open_trades = args.get('max_open_trades', 0) + trade = Trade( + open_rate=buy_row.close, + open_date=buy_row.date, + stake_amount=stake_amount, + amount=stake_amount / buy_row.open, + fee=exchange.get_fee() + ) + + # calculate win/lose forwards from buy point + for sell_row in partial_ticker: + if max_open_trades > 0: + # Increase trade_count_lock for every iteration + trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 + + buy_signal = sell_row.buy + if self.analyze.should_sell(trade, sell_row.close, sell_row.date, buy_signal, + sell_row.sell): + return \ + sell_row, \ + ( + pair, + trade.calc_profit_percent(rate=sell_row.close), + trade.calc_profit(rate=sell_row.close), + (sell_row.date - buy_row.date).seconds // 60 + ), \ + sell_row.date + return None + + def backtest(self, args: Dict) -> DataFrame: + """ + Implements backtesting functionality + + NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. + Of course try to not have ugly code. By some accessor are sometime slower than functions. + Avoid, logging on this method + + :param args: a dict containing: + stake_amount: btc amount to use for each trade + processed: a processed dictionary with format {pair, data} + max_open_trades: maximum number of concurrent trades (default: 0, disabled) + realistic: do we try to simulate realistic trades? (default: True) + sell_profit_only: sell if profit only + use_sell_signal: act on sell-signal + :return: DataFrame + """ + headers = ['date', 'buy', 'open', 'close', 'sell'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + record = args.get('record', None) + records = [] + trades = [] + trade_count_lock = {} + for pair, pair_data in processed.items(): + pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run + + ticker_data = self.populate_sell_trend(self.populate_buy_trend(pair_data))[headers] + ticker = [x for x in ticker_data.itertuples()] + + lock_pair_until = None + for index, row in enumerate(ticker): + if row.buy == 0 or row.sell == 1: + continue # skip rows where no buy signal or that would immediately sell off + + if realistic: + if lock_pair_until is not None and row.date <= lock_pair_until: + continue + if max_open_trades > 0: + # Check if max_open_trades has already been reached for the given date + if not trade_count_lock.get(row.date, 0) < max_open_trades: + continue + + trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + + ret = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) + + if ret: + row2, trade_entry, next_date = ret + lock_pair_until = next_date + trades.append(trade_entry) + if record: + # Note, need to be json.dump friendly + # record a tuple of pair, current_profit_percent, + # entry-date, duration + records.append((pair, trade_entry[1], + row.date.strftime('%s'), + row2.date.strftime('%s'), + row.date, trade_entry[3])) + # For now export inside backtest(), maybe change so that backtest() + # returns a tuple like: (dataframe, records, logs, etc) + if record and record.find('trades') >= 0: + logger.info('Dumping backtest results') + file_dump_json('backtest-result.json', records) + labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + return DataFrame.from_records(trades, columns=labels) + + def start(self) -> None: + """ + Run a backtesting end-to-end + :return: None + """ + data = {} + pairs = self.config['exchange']['pair_whitelist'] + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + + if self.config.get('live'): + logger.info('Downloading data for all pairs in whitelist ...') + for pair in pairs: + data[pair] = exchange.get_ticker_history(pair, self.ticker_interval) + else: + logger.info('Using local backtesting data (using whitelist in given config) ...') + + timerange = Arguments.parse_timerange(self.config.get('timerange')) + data = optimize.load_data( + self.config['datadir'], + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + timerange=timerange + ) + + # Ignore max_open_trades in backtesting, except realistic flag was passed + if self.config.get('realistic_simulation', False): + max_open_trades = self.config['max_open_trades'] + else: + logger.info('Ignoring max_open_trades (realistic_simulation not set) ...') + max_open_trades = 0 + + preprocessed = self.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + + # Execute backtest and print results + sell_profit_only = self.config.get('experimental', {}).get('sell_profit_only', False) + use_sell_signal = self.config.get('experimental', {}).get('use_sell_signal', False) + results = self.backtest( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'max_open_trades': max_open_trades, + 'realistic': self.config.get('realistic_simulation', False), + 'sell_profit_only': sell_profit_only, + 'use_sell_signal': use_sell_signal, + 'record': self.config.get('export') + } + ) + print( + '\n==================================== ' + 'BACKTESTING REPORT' + ' ====================================\n' + '%s', + self._generate_text_table( + data, + results + ) + ) + + +def setup_configuration(args: Namespace) -> Dict[str, Any]: + """ + Prepare the configuration for the backtesting + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args) + config = configuration.get_config() + + # Ensure we do not use Exchange credentials + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + + return config + + +def start(args: Namespace) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + # Initialize configuration + config = setup_configuration(args) + logger.info('Starting freqtrade in Backtesting mode') + + # Initialize backtesting object + backtesting = Backtesting(config) + backtesting.start() diff --git a/parallel/bittrex.py b/parallel/bittrex.py new file mode 100644 index 000000000..002aa2ddf --- /dev/null +++ b/parallel/bittrex.py @@ -0,0 +1,241 @@ +import logging +from typing import Dict, List, Optional + +from bittrex.bittrex import API_V1_1, API_V2_0 +from bittrex.bittrex import Bittrex as _Bittrex +from requests.exceptions import ContentDecodingError + +from freqtrade import OperationalException +from freqtrade.exchange.interface import Exchange + +logger = logging.getLogger(__name__) + +_API: _Bittrex = None +_API_V2: _Bittrex = None +_EXCHANGE_CONF: dict = {} + + +class Bittrex(Exchange): + """ + Bittrex API wrapper. + """ + # Base URL and API endpoints + BASE_URL: str = 'https://www.bittrex.com' + PAIR_DETAIL_METHOD: str = BASE_URL + '/Market/Index' + + def __init__(self, config: dict) -> None: + global _API, _API_V2, _EXCHANGE_CONF + + _EXCHANGE_CONF.update(config) + _API = _Bittrex( + api_key=_EXCHANGE_CONF['key'], + api_secret=_EXCHANGE_CONF['secret'], + calls_per_second=1, + api_version=API_V1_1, + ) + _API_V2 = _Bittrex( + api_key=_EXCHANGE_CONF['key'], + api_secret=_EXCHANGE_CONF['secret'], + calls_per_second=1, + api_version=API_V2_0, + ) + self.cached_ticker = {} + + @staticmethod + def _validate_response(response) -> None: + """ + Validates the given bittrex response + and raises a ContentDecodingError if a non-fatal issue happened. + """ + temp_error_messages = [ + 'NO_API_RESPONSE', + 'MIN_TRADE_REQUIREMENT_NOT_MET', + ] + if response['message'] in temp_error_messages: + raise ContentDecodingError(response['message']) + + @property + def fee(self) -> float: + # 0.25 %: See https://bittrex.com/fees + return 0.0025 + + def buy(self, pair: str, rate: float, amount: float) -> str: + data = _API.buy_limit(pair.replace('_', '-'), amount, rate) + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( + message=data['message'], + pair=pair, + rate=rate, + amount=amount)) + return data['result']['uuid'] + + def sell(self, pair: str, rate: float, amount: float) -> str: + data = _API.sell_limit(pair.replace('_', '-'), amount, rate) + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( + message=data['message'], + pair=pair, + rate=rate, + amount=amount)) + return data['result']['uuid'] + + def get_balance(self, currency: str) -> float: + data = _API.get_balance(currency) + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({currency})'.format( + message=data['message'], + currency=currency)) + return float(data['result']['Balance'] or 0.0) + + def get_balances(self): + data = _API.get_balances() + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message}'.format(message=data['message'])) + return data['result'] + + def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: + if refresh or pair not in self.cached_ticker.keys(): + data = _API.get_ticker(pair.replace('_', '-')) + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair})'.format( + message=data['message'], + pair=pair)) + keys = ['Bid', 'Ask', 'Last'] + if not data.get('result') or\ + not all(key in data.get('result', {}) for key in keys) or\ + not all(data.get('result', {})[key] is not None for key in keys): + raise ContentDecodingError('Invalid response from Bittrex params=({pair})'.format( + pair=pair)) + # Update the pair + self.cached_ticker[pair] = { + 'bid': float(data['result']['Bid']), + 'ask': float(data['result']['Ask']), + 'last': float(data['result']['Last']), + } + return self.cached_ticker[pair] + + def get_ticker_history(self, pair: str, tick_interval: int) -> List[Dict]: + if tick_interval == 1: + interval = 'oneMin' + elif tick_interval == 5: + interval = 'fiveMin' + elif tick_interval == 30: + interval = 'thirtyMin' + elif tick_interval == 60: + interval = 'hour' + elif tick_interval == 1440: + interval = 'Day' + else: + raise ValueError('Unknown tick_interval: {}'.format(tick_interval)) + + data = _API_V2.get_candles(pair.replace('_', '-'), interval) + + # These sanity check are necessary because bittrex cannot keep their API stable. + if not data.get('result'): + raise ContentDecodingError('Invalid response from Bittrex params=({pair})'.format( + pair=pair)) + + for prop in ['C', 'V', 'O', 'H', 'L', 'T']: + for tick in data['result']: + if prop not in tick.keys(): + raise ContentDecodingError('Required property {} not present ' + 'in response params=({})'.format(prop, pair)) + + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair})'.format( + message=data['message'], + pair=pair)) + + return data['result'] + + def get_order(self, order_id: str) -> Dict: + data = _API.get_order(order_id) + if not data['success']: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({order_id})'.format( + message=data['message'], + order_id=order_id)) + data = data['result'] + return { + 'id': data['OrderUuid'], + 'type': data['Type'], + 'pair': data['Exchange'].replace('-', '_'), + 'opened': data['Opened'], + 'rate': data['PricePerUnit'], + 'amount': data['Quantity'], + 'remaining': data['QuantityRemaining'], + 'closed': data['Closed'], + } + + def cancel_order(self, order_id: str) -> None: + data = _API.cancel(order_id) + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({order_id})'.format( + message=data['message'], + order_id=order_id)) + + def get_pair_detail_url(self, pair: str) -> str: + return self.PAIR_DETAIL_METHOD + '?MarketName={}'.format(pair.replace('_', '-')) + + def get_markets(self) -> List[str]: + data = _API.get_markets() + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException(data['message']) + return [m['MarketName'].replace('-', '_') for m in data['result']] + + def get_market_summaries(self) -> List[Dict]: + data = _API.get_market_summaries() + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException(data['message']) + return data['result'] + + def get_wallet_health(self) -> List[Dict]: + data = _API_V2.get_wallet_health() + if not data['success']: + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException(data['message']) + return [{ + 'Currency': entry['Health']['Currency'], + 'IsActive': entry['Health']['IsActive'], + 'LastChecked': entry['Health']['LastChecked'], + 'Notice': entry['Currency'].get('Notice'), + } for entry in data['result']] diff --git a/parallel/config.json b/parallel/config.json new file mode 100644 index 000000000..0bcba4b5e --- /dev/null +++ b/parallel/config.json @@ -0,0 +1,56 @@ +{ + "max_open_trades": 3, + "stake_currency": "BTC", + "stake_amount": 0.00075, + "fiat_display_currency": "USD", + "dry_run": false, + "unfilledtimeout": 600, + "ticker_interval": 5, + "bid_strategy": { + "ask_last_balance": 0.0 + }, + "minimal_roi": { + "35": 0.000, + "30": 0.005, + "25": 0.006, + "20": 0.007, + "15": 0.008, + "10": 0.009, + "5": 0.01, + "0": 0.015 + }, + "stoploss": -0.10, + "exchange": { + "name": "bittrex", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "pair_whitelist": [ + "BTC_ETH", + "BTC_LTC", + "BTC_ETC", + "BTC_DASH", + "BTC_ZEC", + "BTC_XLM", + "BTC_NXT", + "BTC_POWR", + "BTC_ADA", + "BTC_XMR" + ], + "pair_blacklist": [ + "BTC_DOGE" + ] + }, + "experimental": { + "use_sell_signal": false, + "sell_profit_only": false + }, + "telegram": { + "enabled": true, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id" + }, + "initial_state": "running", + "internals": { + "process_throttle_secs": 5 + } +} diff --git a/parallel/default_strategy.py b/parallel/default_strategy.py new file mode 100644 index 000000000..fe642225d --- /dev/null +++ b/parallel/default_strategy.py @@ -0,0 +1,342 @@ + +# --- Do not remove these libs --- +from freqtrade.strategy.interface import IStrategy +from typing import Dict, List +from hyperopt import hp +from functools import reduce +from pandas import DataFrame +# -------------------------------- + +# Add your lib to import here +import talib.abstract as ta +import freqtrade.vendor.qtpylib.indicators as qtpylib +import numpy # noqa + +import random + +# Update this variable if you change the class name +class_name = 'DefaultStrategy' + + +# This class is a sample. Feel free to customize it. + + + + +def Select(): + param = [] + random_items = [] + param.append(str('[' + 'uptrend_long_ema' + '[' + 'enabled' + ']')) + param.append(str('[' + 'macd_below_zero' + '][' + 'enabled' + ']')) + param.append(str('[' + 'uptrend_short_ema' '][' + 'enabled'+ ']')) + param.append(str('[' + 'mfi' '][' + 'enabled'+ ']')) + param.append(str('[' + 'fastd' '][' + 'enabled'+ ']')) + param.append(str('[' + 'adx' '][' + 'enabled'+ ']')) + param.append(str('[' + 'rsi' '][' + 'enabled'+ ']')) + param.append(str('[' + 'over_sar' '][' + 'enabled'+ ']')) + param.append(str('[' + 'green_candle' '][' + 'enabled'+ ']')) + param.append(str('[' + 'uptrend_sma' '][' + 'enabled'+ ']')) + param.append(str('[' + 'closebb' '][' + 'enabled'+ ']')) + param.append(str('[' + 'temabb' '][' + 'enabled'+ ']')) + param.append(str('[' + 'fastdt' '][' + 'enabled'+ ']')) + param.append(str('[' + 'ao' '][' + 'enabled'+ ']')) + param.append(str('[' + 'ema3' '][' + 'enabled'+ ']')) + param.append(str('[' + 'macd' '][' + 'enabled'+ ']')) + param.append(str('[' + 'closesar' '][' + 'enabled'+ ']')) + param.append(str('[' + 'htsine' '][' + 'enabled'+ ']')) + param.append(str('[' + 'has' '][' + 'enabled'+ ']')) + param.append(str('[' + 'plusdi' '][' + 'enabled'+ ']')) + howmany = random.randint(1,20) + random_items = random.choices(population=param, k=howmany) + print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') + print('The Parameters Enabled Are As Follows!!!: ' + str(random_items)) + print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') + return random_items + + + + +class DefaultStrategy(IStrategy): + """ + This is a test strategy to inspire you. + More information in https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md + + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your strategy + - Add any lib you need to build your strategy + + You must keep: + - the lib in the section "Do not remove these libs" + - the prototype for the methods: minimal_roi, stoploss, populate_indicators, populate_buy_trend, + populate_sell_trend, hyperopt_space, buy_strategy_generator + """ + + # Minimal ROI designed for the strategy. + # This attribute will be overridden if the config file contains "minimal_roi" + minimal_roi = { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + } + + ticker_interval = 5 + + # Optimal stoploss designed for the strategy + # This attribute will be overridden if the config file contains "stoploss" + stoploss = -0.10 + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + """ + + # Momentum Indicator + # ------------------------------------ + + # ADX + dataframe['adx'] = ta.ADX(dataframe) + + + # Awesome oscillator + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + + # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + dataframe['cci'] = ta.CCI(dataframe) + + # MACD + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + + # MFI + dataframe['mfi'] = ta.MFI(dataframe) + + # Minus Directional Indicator / Movement + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # Plus Directional Indicator / Movement + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # ROC + dataframe['roc'] = ta.ROC(dataframe) + + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + + + # Overlap Studies + # ------------------------------------ + + """ + # Previous Bollinger bands + # Because ta.BBANDS implementation is broken with small numbers, it actually + # returns middle band for all the three bands. Switch to qtpylib.bollinger_bands + # and use middle band instead. + + dataframe['blower'] = ta.BBANDS(dataframe, nbdevup=2, nbdevdn=2)['lowerband'] + """ + + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + + + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + + # SAR Parabol + dataframe['sar'] = ta.SAR(dataframe) + + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + + + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + + # Cycle Indicator + # ------------------------------------ + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + + + # Chart type + # ------------------------------------ + + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + + + return dataframe + + params = Select() + valm = random.randint(1,100) + print('MFI Value :' + str(valm) + ' XXX') + valfast = random.randint(1,100) + print('FASTD Value :' + str(valfast) + ' XXX') + valadx = random.randint(1,100) + print('ADX Value :' + str(valadx) + ' XXX') + valrsi = random.randint(1,100) + print('RSI Value :' + str(valrsi) + ' XXX') + def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in str(self.params): + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in str(self.params): + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in str(self.params): + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in str(self.params): + + conditions.append(dataframe['mfi'] < self.valm) + if 'fastd' in str(self.params): + + conditions.append(dataframe['fastd'] < self.valfast) + if 'adx' in str(self.params): + + conditions.append(dataframe['adx'] > self.valadx) + if 'rsi' in str(self.params): + + conditions.append(dataframe['rsi'] < self.valrsi) + if 'over_sar' in str(self.params): + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in str(self.params): + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in str(self.params): + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + if 'closebb' in str(self.params): + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if 'temabb' in str(self.params): + conditions.append(dataframe['tema'] < dataframe['bb_lowerband']) + if 'fastdt' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['fastd'], 10.0)) + if 'ao' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['ao'], 0.0)) + if 'ema3' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['ema3'], dataframe['ema10'])) + if 'macd' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['macd'], dataframe['macdsignal'])) + if 'closesar' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['close'], dataframe['sar'])) + if 'htsine' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['htleadsine'], dataframe['htsine'])) + if 'has' in str(self.params): + conditions.append((qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & (dataframe['ha_low'] == dataframe['ha_open'])) + if 'plusdi' in str(self.params): + conditions.append(qtpylib.crossed_above(dataframe['plus_di'], dataframe['minus_di'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: + """ + Based on TA indicators, populates the sell signal for the given dataframe + :param dataframe: DataFrame + :return: DataFrame with buy column + """ + dataframe.loc[ + ( + ), + 'sell'] = 1 + return dataframe + diff --git a/parallel/generator.py b/parallel/generator.py new file mode 100644 index 000000000..850ec3f7c --- /dev/null +++ b/parallel/generator.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +import math +import sys +import os +import time +import pp +import math +import re +# tuple of all parallel python servers to connect with +ppservers = () +# ppservers = ("10.0.0.1",) + +# Number of jobs to run +work = 32 +cycles = 100 + +jobs = [] +current = 0 + + +def backtesting(ind): + er1 = str(ind) + ou1 = str(ind * 1024) + import threading + from io import StringIO + from freqtrade.main import main, set_loggers + old_stdout = sys.stdout + old_stderr = sys.stderr + ind1 = sys.stdout = StringIO() + ind2 = sys.stderr = StringIO() + dat = threading.Thread(target=main(['backtesting'])) + dat.start() + dat.join() + er1 = ind2.getvalue() + ou1 = ind1.getvalue() + sys.stdout = old_stdout + sys.stderr = old_stderr + return er1, ou1 + +if len(sys.argv) > 1: + ncpus = int(sys.argv[1]) + # Creates jobserver with ncpus workers + job_server = pp.Server(ncpus, ppservers=ppservers) +else: + # Creates jobserver with automatically detected number of workers + job_server = pp.Server(ppservers=ppservers) + +print("Starting pp with", job_server.get_ncpus(), "workers") + + +start_time = time.time() +index = 1 +indey = 1 +print('Please wait... sending jobs to server') +while cycles > index: + cycles += 1 + while work > indey: + jobs.append(job_server.submit(backtesting, (index,))) + indey += 1 + job_server.wait() + print('Searching this cycle....') + for job in jobs: + try: + res = job() + string = str(res) + params = re.search(r'~~~~(.*)~~~~', string).group(1) + mfi = re.search(r'MFI Value(.*)XXX', string) + fastd = re.search(r'FASTD Value(.*)XXX', string) + adx = re.search(r'ADX Value(.*)XXX', string) + rsi = re.search(r'RSI Value(.*)XXX', string) + tot = re.search(r'TOTAL (.*)\\n', string).group(1) + total = float(tot) + if total and (float(total) > float(current)): + current = total + print('total better profit paremeters: ') + print(format(total, '.8f')) + if params: + print(params) + print('~~~~~~') + print('Only enable the above settings! Not all below!') + print('~~~~~~') + if mfi: + print('~~~MFI~~~') + print(mfi.group(1)) + if fastd: + print('~~~FASTD~~~') + print(fastd.group(1)) + if adx: + print('~~~ADX~~~') + print(adx.group(1)) + if rsi: + print('~~~RSI~~~') + print(rsi.group(1)) + print("Time elapsed: ", time.time() - start_time, "s") + job_server.print_stats() + except: + pass + jobs = [] + indey = 1 +print('DONE') +print("Time elapsed: ", time.time() - start_time, "s") +job_server.print_stats() diff --git a/parallel/pp-1.6.4.4.zip b/parallel/pp-1.6.4.4.zip new file mode 100644 index 000000000..793a49caa Binary files /dev/null and b/parallel/pp-1.6.4.4.zip differ