From 764f9449b428be8b744ebea989ee81b81330aae9 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 5 May 2022 14:37:37 +0200 Subject: [PATCH] fix logger, debug some flake8 appeasements --- freqtrade/constants.py | 837 ++++++++++-------- freqtrade/freqai/data_handler.py | 29 +- freqtrade/freqai/freqai_interface.py | 15 +- freqtrade/optimize/backtesting.py | 8 +- freqtrade/templates/ExamplePredictionModel.py | 5 +- 5 files changed, 479 insertions(+), 415 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index f8a9dc06d..d9664cff8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -8,88 +8,133 @@ from typing import List, Literal, Tuple from freqtrade.enums import CandleType -DEFAULT_CONFIG = 'config.json' -DEFAULT_EXCHANGE = 'bittrex' +DEFAULT_CONFIG = "config.json" +DEFAULT_EXCHANGE = "bittrex" PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec -TIMEOUT_UNITS = ['minutes', 'seconds'] -EXPORT_OPTIONS = ['none', 'trades', 'signals'] -DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' -DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' -UNLIMITED_STAKE_AMOUNT = 'unlimited' +TIMEOUT_UNITS = ["minutes", "seconds"] +EXPORT_OPTIONS = ["none", "trades", "signals"] +DEFAULT_DB_PROD_URL = "sqlite:///tradesv3.sqlite" +DEFAULT_DB_DRYRUN_URL = "sqlite:///tradesv3.dryrun.sqlite" +UNLIMITED_STAKE_AMOUNT = "unlimited" DEFAULT_AMOUNT_RESERVE_PERCENT = 0.05 -REQUIRED_ORDERTIF = ['entry', 'exit'] -REQUIRED_ORDERTYPES = ['entry', 'exit', 'stoploss', 'stoploss_on_exchange'] -PRICING_SIDES = ['ask', 'bid', 'same', 'other'] -ORDERTYPE_POSSIBILITIES = ['limit', 'market'] -ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] -HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', - 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', - 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily', - 'CalmarHyperOptLoss', - 'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss', - 'ProfitDrawDownHyperOptLoss'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', - 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', - 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', - 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] -AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] -AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] -BACKTEST_BREAKDOWNS = ['day', 'week', 'month'] -BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month'] -BACKTEST_CACHE_DEFAULT = 'day' +REQUIRED_ORDERTIF = ["entry", "exit"] +REQUIRED_ORDERTYPES = ["entry", "exit", "stoploss", "stoploss_on_exchange"] +PRICING_SIDES = ["ask", "bid", "same", "other"] +ORDERTYPE_POSSIBILITIES = ["limit", "market"] +ORDERTIF_POSSIBILITIES = ["gtc", "fok", "ioc"] +HYPEROPT_LOSS_BUILTIN = [ + "ShortTradeDurHyperOptLoss", + "OnlyProfitHyperOptLoss", + "SharpeHyperOptLoss", + "SharpeHyperOptLossDaily", + "SortinoHyperOptLoss", + "SortinoHyperOptLossDaily", + "CalmarHyperOptLoss", + "MaxDrawDownHyperOptLoss", + "MaxDrawDownRelativeHyperOptLoss", + "ProfitDrawDownHyperOptLoss", +] +AVAILABLE_PAIRLISTS = [ + "StaticPairList", + "VolumePairList", + "AgeFilter", + "OffsetFilter", + "PerformanceFilter", + "PrecisionFilter", + "PriceFilter", + "RangeStabilityFilter", + "ShuffleFilter", + "SpreadFilter", + "VolatilityFilter", +] +AVAILABLE_PROTECTIONS = ["CooldownPeriod", "LowProfitPairs", "MaxDrawdown", "StoplossGuard"] +AVAILABLE_DATAHANDLERS = ["json", "jsongz", "hdf5"] +BACKTEST_BREAKDOWNS = ["day", "week", "month"] +BACKTEST_CACHE_AGE = ["none", "day", "week", "month"] +BACKTEST_CACHE_DEFAULT = "day" DRY_RUN_WALLET = 1000 -DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' +DATETIME_PRINT_FORMAT = "%Y-%m-%d %H:%M:%S" MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons -DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume'] +DEFAULT_DATAFRAME_COLUMNS = ["date", "open", "high", "low", "close", "volume"] # Don't modify sequence of DEFAULT_TRADES_COLUMNS # it has wide consequences for stored trades files -DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost'] -TRADING_MODES = ['spot', 'margin', 'futures'] -MARGIN_MODES = ['cross', 'isolated', ''] +DEFAULT_TRADES_COLUMNS = ["timestamp", "id", "type", "side", "price", "amount", "cost"] +TRADING_MODES = ["spot", "margin", "futures"] +MARGIN_MODES = ["cross", "isolated", ""] -LAST_BT_RESULT_FN = '.last_result.json' -FTHYPT_FILEVERSION = 'fthypt_fileversion' +LAST_BT_RESULT_FN = ".last_result.json" +FTHYPT_FILEVERSION = "fthypt_fileversion" -USERPATH_HYPEROPTS = 'hyperopts' -USERPATH_STRATEGIES = 'strategies' -USERPATH_NOTEBOOKS = 'notebooks' -USERPATH_FREQAIMODELS = 'freqaimodels' +USERPATH_HYPEROPTS = "hyperopts" +USERPATH_STRATEGIES = "strategies" +USERPATH_NOTEBOOKS = "notebooks" +USERPATH_FREQAIMODELS = "freqaimodels" -TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent'] -WEBHOOK_FORMAT_OPTIONS = ['form', 'json', 'raw'] +TELEGRAM_SETTING_OPTIONS = ["on", "off", "silent"] +WEBHOOK_FORMAT_OPTIONS = ["form", "json", "raw"] -ENV_VAR_PREFIX = 'FREQTRADE__' +ENV_VAR_PREFIX = "FREQTRADE__" -NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired') +NON_OPEN_EXCHANGE_STATES = ("cancelled", "canceled", "closed", "expired") # Define decimals per coin for outputs # Only used for outputs. DECIMAL_PER_COIN_FALLBACK = 3 # Should be low to avoid listing all possible FIAT's DECIMALS_PER_COIN = { - 'BTC': 8, - 'ETH': 5, + "BTC": 8, + "ETH": 5, } -DUST_PER_COIN = { - 'BTC': 0.0001, - 'ETH': 0.01 -} +DUST_PER_COIN = {"BTC": 0.0001, "ETH": 0.01} # Source files with destination directories within user-directory USER_DATA_FILES = { - 'sample_strategy.py': USERPATH_STRATEGIES, - 'sample_hyperopt_loss.py': USERPATH_HYPEROPTS, - 'strategy_analysis_example.ipynb': USERPATH_NOTEBOOKS, + "sample_strategy.py": USERPATH_STRATEGIES, + "sample_hyperopt_loss.py": USERPATH_HYPEROPTS, + "strategy_analysis_example.ipynb": USERPATH_NOTEBOOKS, } SUPPORTED_FIAT = [ - "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", - "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", - "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", - "RUB", "UAH", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", - "USD", "BTC", "ETH", "XRP", "LTC", "BCH" + "AUD", + "BRL", + "CAD", + "CHF", + "CLP", + "CNY", + "CZK", + "DKK", + "EUR", + "GBP", + "HKD", + "HUF", + "IDR", + "ILS", + "INR", + "JPY", + "KRW", + "MXN", + "MYR", + "NOK", + "NZD", + "PHP", + "PKR", + "PLN", + "RUB", + "UAH", + "SEK", + "SGD", + "THB", + "TRY", + "TWD", + "ZAR", + "USD", + "BTC", + "ETH", + "XRP", + "LTC", + "BCH", ] MINIMAL_CONFIG = { @@ -100,380 +145,416 @@ MINIMAL_CONFIG = { "key": "", "secret": "", "pair_whitelist": [], - "ccxt_async_config": { - } - } + "ccxt_async_config": {}, + }, } # Required json-schema for user specified config CONF_SCHEMA = { - 'type': 'object', - 'properties': { - 'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1}, - 'new_pairs_days': {'type': 'integer', 'default': 30}, - 'timeframe': {'type': 'string'}, - 'stake_currency': {'type': 'string'}, - 'stake_amount': { - 'type': ['number', 'string'], - 'minimum': 0.0001, - 'pattern': UNLIMITED_STAKE_AMOUNT + "type": "object", + "properties": { + "max_open_trades": {"type": ["integer", "number"], "minimum": -1}, + "new_pairs_days": {"type": "integer", "default": 30}, + "timeframe": {"type": "string"}, + "stake_currency": {"type": "string"}, + "stake_amount": { + "type": ["number", "string"], + "minimum": 0.0001, + "pattern": UNLIMITED_STAKE_AMOUNT, }, - 'tradable_balance_ratio': { - 'type': 'number', - 'minimum': 0.0, - 'maximum': 1, - 'default': 0.99 + "tradable_balance_ratio": {"type": "number", "minimum": 0.0, "maximum": 1, "default": 0.99}, + "available_capital": { + "type": "number", + "minimum": 0, }, - 'available_capital': { - 'type': 'number', - 'minimum': 0, + "amend_last_stake_amount": {"type": "boolean", "default": False}, + "last_stake_amount_min_ratio": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "default": 0.5, }, - 'amend_last_stake_amount': {'type': 'boolean', 'default': False}, - 'last_stake_amount_min_ratio': { - 'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5 + "fiat_display_currency": {"type": "string", "enum": SUPPORTED_FIAT}, + "dry_run": {"type": "boolean"}, + "dry_run_wallet": {"type": "number", "default": DRY_RUN_WALLET}, + "cancel_open_orders_on_exit": {"type": "boolean", "default": False}, + "process_only_new_candles": {"type": "boolean"}, + "minimal_roi": { + "type": "object", + "patternProperties": {"^[0-9.]+$": {"type": "number"}}, + "minProperties": 1, }, - 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, - 'dry_run': {'type': 'boolean'}, - 'dry_run_wallet': {'type': 'number', 'default': DRY_RUN_WALLET}, - 'cancel_open_orders_on_exit': {'type': 'boolean', 'default': False}, - 'process_only_new_candles': {'type': 'boolean'}, - 'minimal_roi': { - 'type': 'object', - 'patternProperties': { - '^[0-9.]+$': {'type': 'number'} + "amount_reserve_percent": {"type": "number", "minimum": 0.0, "maximum": 0.5}, + "stoploss": {"type": "number", "maximum": 0, "exclusiveMaximum": True, "minimum": -1}, + "trailing_stop": {"type": "boolean"}, + "trailing_stop_positive": {"type": "number", "minimum": 0, "maximum": 1}, + "trailing_stop_positive_offset": {"type": "number", "minimum": 0, "maximum": 1}, + "trailing_only_offset_is_reached": {"type": "boolean"}, + "use_exit_signal": {"type": "boolean"}, + "exit_profit_only": {"type": "boolean"}, + "exit_profit_offset": {"type": "number"}, + "ignore_roi_if_entry_signal": {"type": "boolean"}, + "ignore_buying_expired_candle_after": {"type": "number"}, + "trading_mode": {"type": "string", "enum": TRADING_MODES}, + "margin_mode": {"type": "string", "enum": MARGIN_MODES}, + "liquidation_buffer": {"type": "number", "minimum": 0.0, "maximum": 0.99}, + "backtest_breakdown": { + "type": "array", + "items": {"type": "string", "enum": BACKTEST_BREAKDOWNS}, + }, + "bot_name": {"type": "string"}, + "unfilledtimeout": { + "type": "object", + "properties": { + "entry": {"type": "number", "minimum": 1}, + "exit": {"type": "number", "minimum": 1}, + "exit_timeout_count": {"type": "number", "minimum": 0, "default": 0}, + "unit": {"type": "string", "enum": TIMEOUT_UNITS, "default": "minutes"}, }, - 'minProperties': 1 }, - 'amount_reserve_percent': {'type': 'number', 'minimum': 0.0, 'maximum': 0.5}, - 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True, 'minimum': -1}, - 'trailing_stop': {'type': 'boolean'}, - 'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1}, - 'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1}, - 'trailing_only_offset_is_reached': {'type': 'boolean'}, - 'use_exit_signal': {'type': 'boolean'}, - 'exit_profit_only': {'type': 'boolean'}, - 'exit_profit_offset': {'type': 'number'}, - 'ignore_roi_if_entry_signal': {'type': 'boolean'}, - 'ignore_buying_expired_candle_after': {'type': 'number'}, - 'trading_mode': {'type': 'string', 'enum': TRADING_MODES}, - 'margin_mode': {'type': 'string', 'enum': MARGIN_MODES}, - 'liquidation_buffer': {'type': 'number', 'minimum': 0.0, 'maximum': 0.99}, - 'backtest_breakdown': { - 'type': 'array', - 'items': {'type': 'string', 'enum': BACKTEST_BREAKDOWNS} - }, - 'bot_name': {'type': 'string'}, - 'unfilledtimeout': { - 'type': 'object', - 'properties': { - 'entry': {'type': 'number', 'minimum': 1}, - 'exit': {'type': 'number', 'minimum': 1}, - 'exit_timeout_count': {'type': 'number', 'minimum': 0, 'default': 0}, - 'unit': {'type': 'string', 'enum': TIMEOUT_UNITS, 'default': 'minutes'} - } - }, - 'entry_pricing': { - 'type': 'object', - 'properties': { - 'price_last_balance': { - 'type': 'number', - 'minimum': 0, - 'maximum': 1, - 'exclusiveMaximum': False, + "entry_pricing": { + "type": "object", + "properties": { + "price_last_balance": { + "type": "number", + "minimum": 0, + "maximum": 1, + "exclusiveMaximum": False, }, - 'price_side': {'type': 'string', 'enum': PRICING_SIDES, 'default': 'same'}, - 'use_order_book': {'type': 'boolean'}, - 'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, }, - 'check_depth_of_market': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, - } + "price_side": {"type": "string", "enum": PRICING_SIDES, "default": "same"}, + "use_order_book": {"type": "boolean"}, + "order_book_top": { + "type": "integer", + "minimum": 1, + "maximum": 50, + }, + "check_depth_of_market": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "bids_to_ask_delta": {"type": "number", "minimum": 0}, + }, }, }, - 'required': ['price_side'] + "required": ["price_side"], }, - 'exit_pricing': { - 'type': 'object', - 'properties': { - 'price_side': {'type': 'string', 'enum': PRICING_SIDES, 'default': 'same'}, - 'price_last_balance': { - 'type': 'number', - 'minimum': 0, - 'maximum': 1, - 'exclusiveMaximum': False, + "exit_pricing": { + "type": "object", + "properties": { + "price_side": {"type": "string", "enum": PRICING_SIDES, "default": "same"}, + "price_last_balance": { + "type": "number", + "minimum": 0, + "maximum": 1, + "exclusiveMaximum": False, }, - 'use_order_book': {'type': 'boolean'}, - 'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, }, - }, - 'required': ['price_side'] - }, - 'custom_price_max_distance_ratio': { - 'type': 'number', 'minimum': 0.0 - }, - 'order_types': { - 'type': 'object', - 'properties': { - 'entry': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'exit': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'force_exit': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'force_entry': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'emergency_exit': { - 'type': 'string', - 'enum': ORDERTYPE_POSSIBILITIES, - 'default': 'market'}, - 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'stoploss_on_exchange': {'type': 'boolean'}, - 'stoploss_on_exchange_interval': {'type': 'number'}, - 'stoploss_on_exchange_limit_ratio': {'type': 'number', 'minimum': 0.0, - 'maximum': 1.0} - }, - 'required': ['entry', 'exit', 'stoploss', 'stoploss_on_exchange'] - }, - 'order_time_in_force': { - 'type': 'object', - 'properties': { - 'entry': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES}, - 'exit': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES} - }, - 'required': REQUIRED_ORDERTIF - }, - 'exchange': {'$ref': '#/definitions/exchange'}, - 'edge': {'$ref': '#/definitions/edge'}, - 'experimental': { - 'type': 'object', - 'properties': { - 'block_bad_exchanges': {'type': 'boolean'} - } - }, - 'pairlists': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, + "use_order_book": {"type": "boolean"}, + "order_book_top": { + "type": "integer", + "minimum": 1, + "maximum": 50, }, - 'required': ['method'], - } + }, + "required": ["price_side"], }, - 'protections': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'method': {'type': 'string', 'enum': AVAILABLE_PROTECTIONS}, - 'stop_duration': {'type': 'number', 'minimum': 0.0}, - 'stop_duration_candles': {'type': 'number', 'minimum': 0}, - 'trade_limit': {'type': 'number', 'minimum': 1}, - 'lookback_period': {'type': 'number', 'minimum': 1}, - 'lookback_period_candles': {'type': 'number', 'minimum': 1}, + "custom_price_max_distance_ratio": {"type": "number", "minimum": 0.0}, + "order_types": { + "type": "object", + "properties": { + "entry": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES}, + "exit": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES}, + "force_exit": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES}, + "force_entry": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES}, + "emergency_exit": { + "type": "string", + "enum": ORDERTYPE_POSSIBILITIES, + "default": "market", }, - 'required': ['method'], - } + "stoploss": {"type": "string", "enum": ORDERTYPE_POSSIBILITIES}, + "stoploss_on_exchange": {"type": "boolean"}, + "stoploss_on_exchange_interval": {"type": "number"}, + "stoploss_on_exchange_limit_ratio": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + }, + }, + "required": ["entry", "exit", "stoploss", "stoploss_on_exchange"], }, - 'telegram': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'token': {'type': 'string'}, - 'chat_id': {'type': 'string'}, - 'balance_dust_level': {'type': 'number', 'minimum': 0.0}, - 'notification_settings': { - 'type': 'object', - 'default': {}, - 'properties': { - 'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'entry': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'entry_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'entry_fill': {'type': 'string', - 'enum': TELEGRAM_SETTING_OPTIONS, - 'default': 'off' - }, - 'exit': { - 'type': ['string', 'object'], - 'additionalProperties': { - 'type': 'string', - 'enum': TELEGRAM_SETTING_OPTIONS - } + "order_time_in_force": { + "type": "object", + "properties": { + "entry": {"type": "string", "enum": ORDERTIF_POSSIBILITIES}, + "exit": {"type": "string", "enum": ORDERTIF_POSSIBILITIES}, + }, + "required": REQUIRED_ORDERTIF, + }, + "exchange": {"$ref": "#/definitions/exchange"}, + "edge": {"$ref": "#/definitions/edge"}, + "experimental": { + "type": "object", + "properties": {"block_bad_exchanges": {"type": "boolean"}}, + }, + "pairlists": { + "type": "array", + "items": { + "type": "object", + "properties": { + "method": {"type": "string", "enum": AVAILABLE_PAIRLISTS}, + }, + "required": ["method"], + }, + }, + "protections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "method": {"type": "string", "enum": AVAILABLE_PROTECTIONS}, + "stop_duration": {"type": "number", "minimum": 0.0}, + "stop_duration_candles": {"type": "number", "minimum": 0}, + "trade_limit": {"type": "number", "minimum": 1}, + "lookback_period": {"type": "number", "minimum": 1}, + "lookback_period_candles": {"type": "number", "minimum": 1}, + }, + "required": ["method"], + }, + }, + "telegram": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "token": {"type": "string"}, + "chat_id": {"type": "string"}, + "balance_dust_level": {"type": "number", "minimum": 0.0}, + "notification_settings": { + "type": "object", + "default": {}, + "properties": { + "status": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS}, + "warning": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS}, + "startup": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS}, + "entry": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS}, + "entry_cancel": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS}, + "entry_fill": { + "type": "string", + "enum": TELEGRAM_SETTING_OPTIONS, + "default": "off", }, - 'exit_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'exit_fill': { - 'type': 'string', - 'enum': TELEGRAM_SETTING_OPTIONS, - 'default': 'off' + "exit": { + "type": ["string", "object"], + "additionalProperties": { + "type": "string", + "enum": TELEGRAM_SETTING_OPTIONS, + }, }, - 'protection_trigger': { - 'type': 'string', - 'enum': TELEGRAM_SETTING_OPTIONS, - 'default': 'off' + "exit_cancel": {"type": "string", "enum": TELEGRAM_SETTING_OPTIONS}, + "exit_fill": { + "type": "string", + "enum": TELEGRAM_SETTING_OPTIONS, + "default": "off", }, - 'protection_trigger_global': { - 'type': 'string', - 'enum': TELEGRAM_SETTING_OPTIONS, + "protection_trigger": { + "type": "string", + "enum": TELEGRAM_SETTING_OPTIONS, + "default": "off", }, - } + "protection_trigger_global": { + "type": "string", + "enum": TELEGRAM_SETTING_OPTIONS, + }, + }, }, - 'reload': {'type': 'boolean'}, + "reload": {"type": "boolean"}, }, - 'required': ['enabled', 'token', 'chat_id'], + "required": ["enabled", "token", "chat_id"], }, - 'webhook': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'url': {'type': 'string'}, - 'format': {'type': 'string', 'enum': WEBHOOK_FORMAT_OPTIONS, 'default': 'form'}, - 'retries': {'type': 'integer', 'minimum': 0}, - 'retry_delay': {'type': 'number', 'minimum': 0}, - 'webhookentry': {'type': 'object'}, - 'webhookentrycancel': {'type': 'object'}, - 'webhookentryfill': {'type': 'object'}, - 'webhookexit': {'type': 'object'}, - 'webhookexitcancel': {'type': 'object'}, - 'webhookexitfill': {'type': 'object'}, - 'webhookstatus': {'type': 'object'}, + "webhook": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "url": {"type": "string"}, + "format": {"type": "string", "enum": WEBHOOK_FORMAT_OPTIONS, "default": "form"}, + "retries": {"type": "integer", "minimum": 0}, + "retry_delay": {"type": "number", "minimum": 0}, + "webhookentry": {"type": "object"}, + "webhookentrycancel": {"type": "object"}, + "webhookentryfill": {"type": "object"}, + "webhookexit": {"type": "object"}, + "webhookexitcancel": {"type": "object"}, + "webhookexitfill": {"type": "object"}, + "webhookstatus": {"type": "object"}, }, }, - 'api_server': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'listen_ip_address': {'format': 'ipv4'}, - 'listen_port': { - 'type': 'integer', - 'minimum': 1024, - 'maximum': 65535 - }, - 'username': {'type': 'string'}, - 'password': {'type': 'string'}, - 'jwt_secret_key': {'type': 'string'}, - 'CORS_origins': {'type': 'array', 'items': {'type': 'string'}}, - 'verbosity': {'type': 'string', 'enum': ['error', 'info']}, + "api_server": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "listen_ip_address": {"format": "ipv4"}, + "listen_port": {"type": "integer", "minimum": 1024, "maximum": 65535}, + "username": {"type": "string"}, + "password": {"type": "string"}, + "jwt_secret_key": {"type": "string"}, + "CORS_origins": {"type": "array", "items": {"type": "string"}}, + "verbosity": {"type": "string", "enum": ["error", "info"]}, }, - 'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password'] + "required": ["enabled", "listen_ip_address", "listen_port", "username", "password"], }, - 'db_url': {'type': 'string'}, - 'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'}, - 'disableparamexport': {'type': 'boolean'}, - 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, - 'force_entry_enable': {'type': 'boolean'}, - 'disable_dataframe_checks': {'type': 'boolean'}, - 'internals': { - 'type': 'object', - 'default': {}, - 'properties': { - 'process_throttle_secs': {'type': 'integer'}, - 'interval': {'type': 'integer'}, - 'sd_notify': {'type': 'boolean'}, - } + "db_url": {"type": "string"}, + "export": {"type": "string", "enum": EXPORT_OPTIONS, "default": "trades"}, + "disableparamexport": {"type": "boolean"}, + "initial_state": {"type": "string", "enum": ["running", "stopped"]}, + "force_entry_enable": {"type": "boolean"}, + "disable_dataframe_checks": {"type": "boolean"}, + "internals": { + "type": "object", + "default": {}, + "properties": { + "process_throttle_secs": {"type": "integer"}, + "interval": {"type": "integer"}, + "sd_notify": {"type": "boolean"}, + }, }, - 'dataformat_ohlcv': { - 'type': 'string', - 'enum': AVAILABLE_DATAHANDLERS, - 'default': 'json' + "dataformat_ohlcv": {"type": "string", "enum": AVAILABLE_DATAHANDLERS, "default": "json"}, + "dataformat_trades": { + "type": "string", + "enum": AVAILABLE_DATAHANDLERS, + "default": "jsongz", }, - 'dataformat_trades': { - 'type': 'string', - 'enum': AVAILABLE_DATAHANDLERS, - 'default': 'jsongz' - }, - 'position_adjustment_enable': {'type': 'boolean'}, - 'max_entry_position_adjustment': {'type': ['integer', 'number'], 'minimum': -1}, + "position_adjustment_enable": {"type": "boolean"}, + "max_entry_position_adjustment": {"type": ["integer", "number"], "minimum": -1}, }, - 'definitions': { - 'exchange': { - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'sandbox': {'type': 'boolean', 'default': False}, - 'key': {'type': 'string', 'default': ''}, - 'secret': {'type': 'string', 'default': ''}, - 'password': {'type': 'string', 'default': ''}, - 'uid': {'type': 'string'}, - 'pair_whitelist': { - 'type': 'array', - 'items': { - 'type': 'string', + "definitions": { + "exchange": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "sandbox": {"type": "boolean", "default": False}, + "key": {"type": "string", "default": ""}, + "secret": {"type": "string", "default": ""}, + "password": {"type": "string", "default": ""}, + "uid": {"type": "string"}, + "pair_whitelist": { + "type": "array", + "items": { + "type": "string", }, - 'uniqueItems': True + "uniqueItems": True, }, - 'pair_blacklist': { - 'type': 'array', - 'items': { - 'type': 'string', + "pair_blacklist": { + "type": "array", + "items": { + "type": "string", }, - 'uniqueItems': True + "uniqueItems": True, }, - 'unknown_fee_rate': {'type': 'number'}, - 'outdated_offset': {'type': 'integer', 'minimum': 1}, - 'markets_refresh_interval': {'type': 'integer'}, - 'ccxt_config': {'type': 'object'}, - 'ccxt_async_config': {'type': 'object'} + "unknown_fee_rate": {"type": "number"}, + "outdated_offset": {"type": "integer", "minimum": 1}, + "markets_refresh_interval": {"type": "integer"}, + "ccxt_config": {"type": "object"}, + "ccxt_async_config": {"type": "object"}, }, - 'required': ['name'] + "required": ["name"], }, - 'edge': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'process_throttle_secs': {'type': 'integer', 'minimum': 600}, - 'calculate_since_number_of_days': {'type': 'integer'}, - 'allowed_risk': {'type': 'number'}, - 'stoploss_range_min': {'type': 'number'}, - 'stoploss_range_max': {'type': 'number'}, - 'stoploss_range_step': {'type': 'number'}, - 'minimum_winrate': {'type': 'number'}, - 'minimum_expectancy': {'type': 'number'}, - 'min_trade_number': {'type': 'number'}, - 'max_trade_duration_minute': {'type': 'integer'}, - 'remove_pumps': {'type': 'boolean'} + "edge": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "process_throttle_secs": {"type": "integer", "minimum": 600}, + "calculate_since_number_of_days": {"type": "integer"}, + "allowed_risk": {"type": "number"}, + "stoploss_range_min": {"type": "number"}, + "stoploss_range_max": {"type": "number"}, + "stoploss_range_step": {"type": "number"}, + "minimum_winrate": {"type": "number"}, + "minimum_expectancy": {"type": "number"}, + "min_trade_number": {"type": "number"}, + "max_trade_duration_minute": {"type": "integer"}, + "remove_pumps": {"type": "boolean"}, }, - 'required': ['process_throttle_secs', 'allowed_risk'] - } + "required": ["process_throttle_secs", "allowed_risk"], + }, + "freqai": { + "type": "object", + "properties": { + "timeframes": {"type": "list"}, + "full_timerange": {"type": "str"}, + "train_period": {"type": "integer", "default": 0}, + "backtest_period": {"type": "integer", "default": 7}, + "identifier": {"type": "str", "default": "example"}, + "base_features": {"type": "list"}, + "corr_pairlist": {"type": "list"}, + "training_timerange": {"type": "string", "default": None}, + "feature_parameters": { + "type": "object", + "properties": { + "period": {"type": "integer"}, + "shift": {"type": "integer", "default": 0}, + "DI_threshold": {"type": "integer", "default": 0}, + "weight_factor": {"type": "number", "default": 0}, + "principal_component_analysis": {"type": "boolean", "default": False}, + "remove_outliers": {"type": "boolean", "default": False}, + }, + }, + "data_split_parameters": { + "type": "object", + "properties": { + "test_size": {"type": "number"}, + "random_state": {"type": "integer"}, + }, + }, + "model_training_parameters": { + "type": "object", + "properties": { + "n_estimators": {"type": "integer", "default": 2000}, + "random_state": {"type": "integer", "default": 1}, + "learning_rate": {"type": "number", "default": 0.02}, + "task_type": {"type": "string", "default": "CPU"}, + }, + }, + }, + }, }, } SCHEMA_TRADE_REQUIRED = [ - 'exchange', - 'timeframe', - 'max_open_trades', - 'stake_currency', - 'stake_amount', - 'tradable_balance_ratio', - 'last_stake_amount_min_ratio', - 'dry_run', - 'dry_run_wallet', - 'exit_pricing', - 'entry_pricing', - 'stoploss', - 'minimal_roi', - 'internals', - 'dataformat_ohlcv', - 'dataformat_trades', + "exchange", + "timeframe", + "max_open_trades", + "stake_currency", + "stake_amount", + "tradable_balance_ratio", + "last_stake_amount_min_ratio", + "dry_run", + "dry_run_wallet", + "exit_pricing", + "entry_pricing", + "stoploss", + "minimal_roi", + "internals", + "dataformat_ohlcv", + "dataformat_trades", ] SCHEMA_BACKTEST_REQUIRED = [ - 'exchange', - 'max_open_trades', - 'stake_currency', - 'stake_amount', - 'dry_run_wallet', - 'dataformat_ohlcv', - 'dataformat_trades', + "exchange", + "max_open_trades", + "stake_currency", + "stake_amount", + "dry_run_wallet", + "dataformat_ohlcv", + "dataformat_trades", ] SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [ - 'stoploss', - 'minimal_roi', + "stoploss", + "minimal_roi", ] SCHEMA_MINIMAL_REQUIRED = [ - 'exchange', - 'dry_run', - 'dataformat_ohlcv', - 'dataformat_trades', + "exchange", + "dry_run", + "dataformat_ohlcv", + "dataformat_trades", ] CANCEL_REASON = { diff --git a/freqtrade/freqai/data_handler.py b/freqtrade/freqai/data_handler.py index 9ab47d223..94df869a1 100644 --- a/freqtrade/freqai/data_handler.py +++ b/freqtrade/freqai/data_handler.py @@ -36,6 +36,7 @@ class DataHandler: config["freqai"]["backtest_period"], ) self.data: Dict[Any, Any] = {} + self.data_dictionary: Dict[Any, Any] = {} self.config = config self.freq_config = config["freqai"] self.predictions = np.array([]) @@ -58,10 +59,6 @@ class DataHandler: save_path = Path(self.model_path) - # if not os.path.exists(self.model_path): - # os.mkdir(self.model_path) - # save_path = self.model_path + self.model_filename - # Save the trained model dump(model, save_path / str(self.model_filename + "_model.joblib")) self.data["model_path"] = self.model_path @@ -179,10 +176,8 @@ class DataHandler: (drop_index == 0) & (drop_index_labels == 0) ] # assuming the labels depend entirely on the dataframe here. logger.info( - "dropped", + "dropped %s training points due to NaNs, ensure all historical data downloaded", len(unfiltered_dataframe) - len(filtered_dataframe), - "training data points due to NaNs, ensure you have downloaded", - "all historical training data", ) self.data["filter_drop_index_training"] = drop_index @@ -197,12 +192,9 @@ class DataHandler: drop_index = ~drop_index self.do_predict = np.array(drop_index.replace(True, 1).replace(False, 0)) logger.info( - "dropped", + "dropped %s of %s prediction data points due to NaNs.", len(self.do_predict) - self.do_predict.sum(), - "of", len(filtered_dataframe), - "prediction data points due to NaNs. These are protected from prediction", - "with do_predict vector returned to strategy.", ) return filtered_dataframe, labels @@ -353,8 +345,8 @@ class DataHandler: pca2 = PCA(n_components=n_keep_components) self.data["n_kept_components"] = n_keep_components pca2 = pca2.fit(self.data_dictionary["train_features"]) - logger.info("reduced feature dimension by", n_components - n_keep_components) - logger.info("explained variance", np.sum(pca2.explained_variance_ratio_)) + logger.info("reduced feature dimension by %s", n_components - n_keep_components) + logger.info("explained variance %f", np.sum(pca2.explained_variance_ratio_)) train_components = pca2.transform(self.data_dictionary["train_features"]) test_components = pca2.transform(self.data_dictionary["test_features"]) @@ -383,7 +375,7 @@ class DataHandler: logger.info("computing average mean distance for all training points") pairwise = pairwise_distances(self.data_dictionary["train_features"], n_jobs=-1) avg_mean_dist = pairwise.mean(axis=1).mean() - logger.info("avg_mean_dist", avg_mean_dist) + logger.info("avg_mean_dist %s", avg_mean_dist) return avg_mean_dist @@ -411,9 +403,8 @@ class DataHandler: do_predict = np.array(drop_index.replace(True, 1).replace(False, 0)) logger.info( - "remove_outliers() tossed", + "remove_outliers() tossed %s predictions", len(do_predict) - do_predict.sum(), - "predictions because they were beyond 3 std deviations from training data.", ) self.do_predict += do_predict self.do_predict -= 1 @@ -475,7 +466,7 @@ class DataHandler: for p in config["freqai"]["corr_pairlist"]: features.append(p.split("/")[0] + "-" + ft + shift + "_" + tf) - logger.info("number of features", len(features)) + logger.info("number of features %s", len(features)) return features def check_if_pred_in_training_spaces(self) -> None: @@ -486,7 +477,6 @@ class DataHandler: from the training data set. """ - logger.info("checking if prediction features are in AOA") distance = pairwise_distances( self.data_dictionary["train_features"], self.data_dictionary["prediction_features"], @@ -501,9 +491,8 @@ class DataHandler: ) logger.info( - "Distance checker tossed", + "Distance checker tossed %s predictions for being too far from training data", len(do_predict) - do_predict.sum(), - "predictions for being too far from training data", ) self.do_predict += do_predict diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 05a0594f3..368ed1635 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -69,12 +69,7 @@ class IFreqaiModel(ABC): self.pair = metadata["pair"] self.dh = DataHandler(self.config, dataframe) - logger.info( - "going to train", - len(self.dh.training_timeranges), - "timeranges:", - self.dh.training_timeranges, - ) + logger.info("going to train %s timeranges", len(self.dh.training_timeranges)) # Loop enforcing the sliding window training/backtesting paragigm # tr_train is the training time range e.g. 1 historical month @@ -90,14 +85,14 @@ class IFreqaiModel(ABC): self.freqai_info["training_timerange"] = tr_train dataframe_train = self.dh.slice_dataframe(tr_train, dataframe) dataframe_backtest = self.dh.slice_dataframe(tr_backtest, dataframe) - logger.info("training", self.pair, "for", tr_train) + logger.info("training %s for %s", self.pair, tr_train) # self.dh.model_path = self.full_path + "/" + "sub-train" + "-" + str(tr_train) + "/" self.dh.model_path = Path(self.full_path / str("sub-train" + "-" + str(tr_train))) if not self.model_exists(self.pair, training_timerange=tr_train): self.model = self.train(dataframe_train, metadata) self.dh.save_data(self.model) else: - self.model = self.dh.load_data(self.dh.model_path) + self.model = self.dh.load_data() preds, do_preds = self.predict(dataframe_backtest) @@ -167,7 +162,7 @@ class IFreqaiModel(ABC): path_to_modelfile = Path(self.dh.model_path / str(self.dh.model_filename + "_model.joblib")) file_exists = path_to_modelfile.is_file() if file_exists: - logger.info("Found model at", self.dh.model_path / self.dh.model_filename) + logger.info("Found model at %s", self.dh.model_path / self.dh.model_filename) else: - logger.info("Could not find model at", self.dh.model_path / self.dh.model_filename) + logger.info("Could not find model at %s", self.dh.model_path / self.dh.model_filename) return file_exists diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5051a8db0..f976b1238 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -204,12 +204,12 @@ class Backtesting: """ self.progress.init_step(BacktestState.DATALOAD, 1) - if self.config['freqaimodel']: - self.required_startup += int((self.config['freqai']['train_period']*86400) / - timeframe_to_seconds(self.config['timeframe'])) + if self.config['freqai']['train_period'] > 0: + self.required_startup += int((self.config['freqai']['train_period'] * 86400) / + timeframe_to_seconds(self.config['timeframe'])) + logger.info("Increasing startup_candle_count for freqai to %s", self.required_startup) self.config['startup_candle_count'] = self.required_startup - data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, diff --git a/freqtrade/templates/ExamplePredictionModel.py b/freqtrade/templates/ExamplePredictionModel.py index 4906b8c04..35f25775a 100644 --- a/freqtrade/templates/ExamplePredictionModel.py +++ b/freqtrade/templates/ExamplePredictionModel.py @@ -36,7 +36,7 @@ class ExamplePredictionModel(IFreqaiModel): self.dh.data["s_mean"] = dataframe["s"].mean() self.dh.data["s_std"] = dataframe["s"].std() - logger.info("label mean", self.dh.data["s_mean"], "label std", self.dh.data["s_std"]) + # logger.info("label mean", self.dh.data["s_mean"], "label std", self.dh.data["s_std"]) return dataframe["s"] @@ -77,11 +77,10 @@ class ExamplePredictionModel(IFreqaiModel): if self.feature_parameters["DI_threshold"]: self.dh.data["avg_mean_dist"] = self.dh.compute_distances() - logger.info("length of train data", len(data_dictionary["train_features"])) + logger.info("length of train data %s", len(data_dictionary["train_features"])) model = self.fit(data_dictionary) - logger.info("Finished training") logger.info(f'--------------------done training {metadata["pair"]}--------------------') return model