From 9559f50eec5da4d9b070da248ea7ca09fccd5d52 Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 20 Mar 2018 20:09:42 +0100 Subject: [PATCH 01/57] remove obsolete helper functions and make _state a public member. --- freqtrade/freqtradebot.py | 38 ++++++------------- freqtrade/rpc/rpc.py | 18 ++++----- freqtrade/tests/rpc/test_rpc.py | 28 +++++++------- freqtrade/tests/rpc/test_rpc_telegram.py | 48 ++++++++++++------------ freqtrade/tests/test_freqtradebot.py | 14 +++---- 5 files changed, 65 insertions(+), 81 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e57f177e9..a312aed91 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -41,7 +41,7 @@ class FreqtradeBot(object): self.logger = Logger(name=__name__, level=config.get('loglevel')).get_logger() # Init bot states - self._state = State.STOPPED + self.state = State.STOPPED # Init objects self.config = config @@ -71,9 +71,9 @@ class FreqtradeBot(object): initial_state = self.config.get('initial_state') if initial_state: - self.update_state(State[initial_state.upper()]) + self.state = State[initial_state.upper()] else: - self.update_state(State.STOPPED) + self.state = State.STOPPED def clean(self) -> bool: """ @@ -82,41 +82,25 @@ class FreqtradeBot(object): """ self.rpc.send_msg('*Status:* `Stopping trader...`') self.logger.info('Stopping trader and cleaning up modules...') - self.update_state(State.STOPPED) + self.state = State.STOPPED self.rpc.cleanup() persistence.cleanup() return True - def update_state(self, state: State) -> None: - """ - Updates the application state - :param state: new state - :return: None - """ - self._state = state - - def get_state(self) -> State: - """ - Gets the current application state - :return: - """ - return self._state - def worker(self, old_state: None) -> State: """ Trading routine that must be run at each loop :param old_state: the previous service state from the previous call :return: current service state """ - new_state = self.get_state() # Log state transition - if new_state != old_state: - self.rpc.send_msg('*Status:* `{}`'.format(new_state.name.lower())) - self.logger.info('Changing state to: %s', new_state.name) + if self.state != old_state: + self.rpc.send_msg('*Status:* `{}`'.format(self.state.name.lower())) + self.logger.info('Changing state to: %s', self.state.name) - if new_state == State.STOPPED: + if self.state == State.STOPPED: time.sleep(1) - elif new_state == State.RUNNING: + elif self.state == State.RUNNING: min_secs = self.config.get('internals', {}).get( 'process_throttle_secs', Constants.PROCESS_THROTTLE_SECS @@ -130,7 +114,7 @@ class FreqtradeBot(object): self._throttle(func=self._process, min_secs=min_secs, nb_assets=nb_assets) - return new_state + return self.state def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: """ @@ -196,7 +180,7 @@ class FreqtradeBot(object): ) ) self.logger.exception('OperationalException. Stopping trader ...') - self.update_state(State.STOPPED) + self.state = State.STOPPED return state_changed @cached(TTLCache(maxsize=1, ttl=1800)) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index b4592f78a..db6ff69d1 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -41,7 +41,7 @@ class RPC(object): """ # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if self.freqtrade.get_state() != State.RUNNING: + if self.freqtrade.state != State.RUNNING: return True, '*Status:* `trader is not running`' elif not trades: return True, '*Status:* `no active trade`' @@ -87,7 +87,7 @@ class RPC(object): def rpc_status_table(self) -> Tuple[bool, Any]: trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if self.freqtrade.get_state() != State.RUNNING: + if self.freqtrade.state != State.RUNNING: return True, '*Status:* `trader is not running`' elif not trades: return True, '*Status:* `no active order`' @@ -285,18 +285,18 @@ class RPC(object): """ Handler for start. """ - if self.freqtrade.get_state() == State.RUNNING: + if self.freqtrade.state == State.RUNNING: return True, '*Status:* `already running`' - self.freqtrade.update_state(State.RUNNING) + self.freqtrade.state = State.RUNNING return False, '`Starting trader ...`' def rpc_stop(self) -> (bool, str): """ Handler for stop. """ - if self.freqtrade.get_state() == State.RUNNING: - self.freqtrade.update_state(State.STOPPED) + if self.freqtrade.state == State.RUNNING: + self.freqtrade.state = State.STOPPED return False, '`Stopping trader ...`' return True, '*Status:* `already stopped`' @@ -329,7 +329,7 @@ class RPC(object): self.freqtrade.execute_sell(trade, current_rate) # ---- EOF def _exec_forcesell ---- - if self.freqtrade.get_state() != State.RUNNING: + if self.freqtrade.state != State.RUNNING: return True, '`trader is not running`' if trade_id == 'all': @@ -357,7 +357,7 @@ class RPC(object): Handler for performance. Shows a performance statistic from finished trades """ - if self.freqtrade.get_state() != State.RUNNING: + if self.freqtrade.state != State.RUNNING: return True, '`trader is not running`' pair_rates = Trade.session.query(Trade.pair, @@ -378,7 +378,7 @@ class RPC(object): Returns the number of trades running :return: None """ - if self.freqtrade.get_state() != State.RUNNING: + if self.freqtrade.state != State.RUNNING: return True, '`trader is not running`' trades = Trade.query.filter(Trade.is_open.is_(True)).all() diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 50943b1bc..84b06bee2 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -41,12 +41,12 @@ def test_rpc_trade_status(default_conf, ticker, mocker) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) rpc = RPC(freqtradebot) - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED (error, result) = rpc.rpc_trade_status() assert error assert 'trader is not running' in result - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING (error, result) = rpc.rpc_trade_status() assert error assert 'no active trade' in result @@ -89,12 +89,12 @@ def test_rpc_status_table(default_conf, ticker, mocker) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) rpc = RPC(freqtradebot) - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED (error, result) = rpc.rpc_status_table() assert error assert '*Status:* `trader is not running`' in result - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING (error, result) = rpc.rpc_status_table() assert error assert '*Status:* `no active order`' in result @@ -344,17 +344,17 @@ def test_rpc_start(mocker, default_conf) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) rpc = RPC(freqtradebot) - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED (error, result) = rpc.rpc_start() assert not error assert '`Starting trader ...`' in result - assert freqtradebot.get_state() == State.RUNNING + assert freqtradebot.state == State.RUNNING (error, result) = rpc.rpc_start() assert error assert '*Status:* `already running`' in result - assert freqtradebot.get_state() == State.RUNNING + assert freqtradebot.state == State.RUNNING def test_rpc_stop(mocker, default_conf) -> None: @@ -372,17 +372,17 @@ def test_rpc_stop(mocker, default_conf) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) rpc = RPC(freqtradebot) - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING (error, result) = rpc.rpc_stop() assert not error assert '`Stopping trader ...`' in result - assert freqtradebot.get_state() == State.STOPPED + assert freqtradebot.state == State.STOPPED (error, result) = rpc.rpc_stop() assert error assert '*Status:* `already stopped`' in result - assert freqtradebot.get_state() == State.STOPPED + assert freqtradebot.state == State.STOPPED def test_rpc_forcesell(default_conf, ticker, mocker) -> None: @@ -410,12 +410,12 @@ def test_rpc_forcesell(default_conf, ticker, mocker) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) rpc = RPC(freqtradebot) - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED (error, res) = rpc.rpc_forcesell(None) assert error assert res == '`trader is not running`' - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING (error, res) = rpc.rpc_forcesell(None) assert error assert res == 'Invalid argument.' @@ -433,7 +433,7 @@ def test_rpc_forcesell(default_conf, ticker, mocker) -> None: assert not error assert res == '' - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED (error, res) = rpc.rpc_forcesell(None) assert error assert res == '`trader is not running`' @@ -442,7 +442,7 @@ def test_rpc_forcesell(default_conf, ticker, mocker) -> None: assert error assert res == '`trader is not running`' - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 # make an limit-buy open trade mocker.patch( diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 4796b077e..86dfd6f41 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -301,13 +301,13 @@ def test_status_handle(default_conf, update, ticker, mocker) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) telegram = Telegram(freqtradebot) - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'trader is not running' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'no active trade' in msg_mock.call_args_list[0][0][0] @@ -347,13 +347,13 @@ def test_status_table_handle(default_conf, update, ticker, mocker) -> None: freqtradebot = FreqtradeBot(conf, create_engine('sqlite://')) telegram = Telegram(freqtradebot) - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED telegram._status_table(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'trader is not running' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING telegram._status_table(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'no active order' in msg_mock.call_args_list[0][0][0] @@ -470,7 +470,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: # Try invalid data msg_mock.reset_mock() - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING update.message.text = '/daily -2' telegram._daily(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 @@ -478,7 +478,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: # Try invalid data msg_mock.reset_mock() - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING update.message.text = '/daily today' telegram._daily(bot=MagicMock(), update=update) assert str('Daily Profit over the last 7 days') in msg_mock.call_args_list[0][0][0] @@ -665,10 +665,10 @@ def test_start_handle(default_conf, update, mocker) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) telegram = Telegram(freqtradebot) - freqtradebot.update_state(State.STOPPED) - assert freqtradebot.get_state() == State.STOPPED + freqtradebot.state = State.STOPPED + assert freqtradebot.state == State.STOPPED telegram._start(bot=MagicMock(), update=update) - assert freqtradebot.get_state() == State.RUNNING + assert freqtradebot.state == State.RUNNING assert msg_mock.call_count == 0 @@ -689,10 +689,10 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) telegram = Telegram(freqtradebot) - freqtradebot.update_state(State.RUNNING) - assert freqtradebot.get_state() == State.RUNNING + freqtradebot.state = State.RUNNING + assert freqtradebot.state == State.RUNNING telegram._start(bot=MagicMock(), update=update) - assert freqtradebot.get_state() == State.RUNNING + assert freqtradebot.state == State.RUNNING assert msg_mock.call_count == 1 assert 'already running' in msg_mock.call_args_list[0][0][0] @@ -714,10 +714,10 @@ def test_stop_handle(default_conf, update, mocker) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) telegram = Telegram(freqtradebot) - freqtradebot.update_state(State.RUNNING) - assert freqtradebot.get_state() == State.RUNNING + freqtradebot.state = State.RUNNING + assert freqtradebot.state == State.RUNNING telegram._stop(bot=MagicMock(), update=update) - assert freqtradebot.get_state() == State.STOPPED + assert freqtradebot.state == State.STOPPED assert msg_mock.call_count == 1 assert 'Stopping trader' in msg_mock.call_args_list[0][0][0] @@ -739,10 +739,10 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) telegram = Telegram(freqtradebot) - freqtradebot.update_state(State.STOPPED) - assert freqtradebot.get_state() == State.STOPPED + freqtradebot.state = State.STOPPED + assert freqtradebot.state == State.STOPPED telegram._stop(bot=MagicMock(), update=update) - assert freqtradebot.get_state() == State.STOPPED + assert freqtradebot.state == State.STOPPED assert msg_mock.call_count == 1 assert 'already stopped' in msg_mock.call_args_list[0][0][0] @@ -881,7 +881,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: telegram = Telegram(freqtradebot) # Trader is not running - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED update.message.text = '/forcesell 1' telegram._forcesell(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 @@ -889,7 +889,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: # No argument msg_mock.reset_mock() - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING update.message.text = '/forcesell' telegram._forcesell(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 @@ -897,7 +897,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: # Invalid argument msg_mock.reset_mock() - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING update.message.text = '/forcesell 123456' telegram._forcesell(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 @@ -962,7 +962,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None: telegram = Telegram(freqtradebot) # Trader is not running - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED telegram._performance(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'not running' in msg_mock.call_args_list[0][0][0] @@ -989,12 +989,12 @@ def test_count_handle(default_conf, update, ticker, mocker) -> None: freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://')) telegram = Telegram(freqtradebot) - freqtradebot.update_state(State.STOPPED) + freqtradebot.state = State.STOPPED telegram._count(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 assert 'not running' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() - freqtradebot.update_state(State.RUNNING) + freqtradebot.state = State.RUNNING # Create some test data freqtradebot.create_trade() diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index d58b428da..608200071 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -104,12 +104,12 @@ def test_freqtradebot(mocker, default_conf) -> None: Test __init__, _init_modules, update_state, and get_state methods """ freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_state() is State.RUNNING + assert freqtrade.state is State.RUNNING conf = deepcopy(default_conf) conf.pop('initial_state') freqtrade = FreqtradeBot(conf) - assert freqtrade.get_state() is State.STOPPED + assert freqtrade.state is State.STOPPED def test_clean(mocker, default_conf, caplog) -> None: @@ -120,10 +120,10 @@ def test_clean(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.persistence.cleanup', mock_cleanup) freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.get_state() == State.RUNNING + assert freqtrade.state == State.RUNNING assert freqtrade.clean() - assert freqtrade.get_state() == State.STOPPED + assert freqtrade.state == State.STOPPED assert log_has('Stopping trader and cleaning up modules...', caplog.record_tuples) assert mock_cleanup.call_count == 1 @@ -152,7 +152,7 @@ def test_worker_stopped(mocker, default_conf, caplog) -> None: mock_sleep = mocker.patch('time.sleep', return_value=None) freqtrade = get_patched_freqtradebot(mocker, default_conf) - freqtrade.update_state(State.STOPPED) + freqtrade.state = State.STOPPED state = freqtrade.worker(old_state=State.RUNNING) assert state is State.STOPPED assert log_has('Changing state to: STOPPED', caplog.record_tuples) @@ -472,11 +472,11 @@ def test_process_operational_exception(default_conf, ticker, health, mocker) -> buy=MagicMock(side_effect=OperationalException) ) freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://')) - assert freqtrade.get_state() == State.RUNNING + assert freqtrade.state == State.RUNNING result = freqtrade._process() assert result is False - assert freqtrade.get_state() == State.STOPPED + assert freqtrade.state == State.STOPPED assert 'OperationalException' in msg_mock.call_args_list[-1][0][0] From 9df5e09a82bb4ac4bae4b92fdf5431052c828642 Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 20 Mar 2018 20:36:03 +0100 Subject: [PATCH 02/57] remove function assertions --- freqtrade/tests/test_freqtradebot.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 608200071..cd271d273 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -85,8 +85,6 @@ def test_freqtradebot_object() -> None: Test the FreqtradeBot object has the mandatory public methods """ assert hasattr(FreqtradeBot, 'worker') - assert hasattr(FreqtradeBot, 'get_state') - assert hasattr(FreqtradeBot, 'update_state') assert hasattr(FreqtradeBot, 'clean') assert hasattr(FreqtradeBot, 'create_trade') assert hasattr(FreqtradeBot, 'get_target_bid') From b8f322d8f67fad300bbe4817f452b1dc2ef8b3aa Mon Sep 17 00:00:00 2001 From: gcarq Date: Wed, 21 Mar 2018 19:27:30 +0100 Subject: [PATCH 03/57] revert worker() changes --- freqtrade/freqtradebot.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a312aed91..29790515e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -94,13 +94,14 @@ class FreqtradeBot(object): :return: current service state """ # Log state transition - if self.state != old_state: - self.rpc.send_msg('*Status:* `{}`'.format(self.state.name.lower())) - self.logger.info('Changing state to: %s', self.state.name) + state = self.state + if state != old_state: + self.rpc.send_msg('*Status:* `{}`'.format(state.name.lower())) + self.logger.info('Changing state to: %s', state.name) - if self.state == State.STOPPED: + if state == State.STOPPED: time.sleep(1) - elif self.state == State.RUNNING: + elif state == State.RUNNING: min_secs = self.config.get('internals', {}).get( 'process_throttle_secs', Constants.PROCESS_THROTTLE_SECS @@ -114,7 +115,7 @@ class FreqtradeBot(object): self._throttle(func=self._process, min_secs=min_secs, nb_assets=nb_assets) - return self.state + return state def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: """ From e5abc15c533c276e2a5c25b80c18b0a835d27a48 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 23 Mar 2018 05:30:54 +0100 Subject: [PATCH 04/57] Update pytest from 3.4.2 to 3.5.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 29361ecdb..f802f7ae4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.0.0 jsonschema==2.6.0 numpy==1.14.2 TA-Lib==0.4.17 -pytest==3.4.2 +pytest==3.5.0 pytest-mock==1.7.1 pytest-cov==2.5.1 hyperopt==0.1 From 71025fd374fcf362cc30735be24af010e853dbbe Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 24 Mar 2018 20:40:57 +0100 Subject: [PATCH 05/57] Update scipy from 1.0.0 to 1.0.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f802f7ae4..d228c8195 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ urllib3==1.22 wrapt==1.10.11 pandas==0.22.0 scikit-learn==0.19.1 -scipy==1.0.0 +scipy==1.0.1 jsonschema==2.6.0 numpy==1.14.2 TA-Lib==0.4.17 From 9d443b8bd837cdc7d66abeb585b5d70927f08332 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 20:54:46 +0100 Subject: [PATCH 06/57] fix reference before assignment --- freqtrade/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index d2cfc6f9f..4bd9e63f6 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -40,6 +40,7 @@ def main(sysargv: List[str]) -> None: logging.getLevelName(args.loglevel) ) + freqtrade = None try: # Load and validate configuration configuration = Configuration(args) @@ -56,7 +57,8 @@ def main(sysargv: List[str]) -> None: except BaseException: logger.exception('Fatal exception!') finally: - freqtrade.clean() + if freqtrade: + freqtrade.clean() sys.exit(0) From 4c97ee45dd3237bd55450ac19506de6322ee9f76 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 20:55:10 +0100 Subject: [PATCH 07/57] return None if subcommand has been executed --- freqtrade/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 4bd9e63f6..7b5678763 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -32,7 +32,7 @@ def main(sysargv: List[str]) -> None: # Means if Backtesting or Hyperopt have been called we exit the bot if hasattr(args, 'func'): args.func(args) - return 0 + return logger.info( 'Starting freqtrade %s (loglevel=%s)', From 3f4261ad1eea650556cef596a29cffe05605ba76 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 20:56:27 +0100 Subject: [PATCH 08/57] use correct return_code if an error occured --- freqtrade/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 7b5678763..556b70708 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -41,6 +41,7 @@ def main(sysargv: List[str]) -> None: ) freqtrade = None + return_code = 1 try: # Load and validate configuration configuration = Configuration(args) @@ -54,12 +55,13 @@ def main(sysargv: List[str]) -> None: except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') + return_code = 0 except BaseException: logger.exception('Fatal exception!') finally: if freqtrade: freqtrade.clean() - sys.exit(0) + sys.exit(return_code) def set_loggers() -> None: From ca9c5edd39331d074f8f0c2eb68fbc47676c400f Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 18:11:21 +0100 Subject: [PATCH 09/57] rename Strategy into StrategyResolver --- freqtrade/analyze.py | 4 +- .../strategy/{strategy.py => resolver.py} | 2 +- freqtrade/tests/optimize/test_hyperopt.py | 12 ++-- freqtrade/tests/strategy/test_strategy.py | 62 +++++++++---------- freqtrade/tests/test_dataframe.py | 6 +- freqtrade/tests/test_misc.py | 2 +- 6 files changed, 44 insertions(+), 44 deletions(-) rename freqtrade/strategy/{strategy.py => resolver.py} (99%) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 8bc552d74..4f0bd415a 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -11,7 +11,7 @@ from pandas import DataFrame, to_datetime from freqtrade.exchange import get_ticker_history from freqtrade.logger import Logger from freqtrade.persistence import Trade -from freqtrade.strategy.strategy import Strategy +from freqtrade.strategy.resolver import StrategyResolver class SignalType(Enum): @@ -35,7 +35,7 @@ class Analyze(object): self.logger = Logger(name=__name__, level=config.get('loglevel')).get_logger() self.config = config - self.strategy = Strategy(self.config) + self.strategy = StrategyResolver(self.config) @staticmethod def parse_ticker_dataframe(ticker: list) -> DataFrame: diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/resolver.py similarity index 99% rename from freqtrade/strategy/strategy.py rename to freqtrade/strategy/resolver.py index d7a89d1de..74391c85d 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/resolver.py @@ -17,7 +17,7 @@ from freqtrade.strategy.interface import IStrategy sys.path.insert(0, r'../../user_data/strategies') -class Strategy(object): +class StrategyResolver(object): """ This class contains all the logic to load custom strategy class """ diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 6d376471a..688a7aa8d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -8,7 +8,7 @@ import pandas as pd from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start -from freqtrade.strategy.strategy import Strategy +from freqtrade.strategy.resolver import StrategyResolver from freqtrade.tests.conftest import default_conf, log_has from freqtrade.tests.optimize.test_backtesting import get_args @@ -62,7 +62,7 @@ def test_start(mocker, default_conf, caplog) -> None: '--epochs', '5' ] args = get_args(args) - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'default_strategy'}) start(args) import pprint @@ -80,7 +80,7 @@ def test_loss_calculation_prefer_correct_trade_count() -> None: Test Hyperopt.calculate_loss() """ hyperopt = _HYPEROPT - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'default_strategy'}) correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20) over = hyperopt.calculate_loss(1, hyperopt.target_trades + 100, 20) @@ -171,7 +171,7 @@ def test_fmin_best_results(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'default_strategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) hyperopt.tickerdata_to_dataframe = MagicMock() @@ -215,7 +215,7 @@ def test_fmin_throw_value_error(mocker, default_conf, caplog) -> None: conf.update({'spaces': 'all'}) mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'default_strategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) hyperopt.tickerdata_to_dataframe = MagicMock() @@ -258,7 +258,7 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, default_conf) -> No mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'default_strategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = trials hyperopt.tickerdata_to_dataframe = MagicMock() diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 7ce9ae0ef..7dc602f58 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -2,49 +2,49 @@ import logging -from freqtrade.strategy.strategy import Strategy +from freqtrade.strategy.resolver import StrategyResolver def test_sanitize_module_name(): - assert Strategy._sanitize_module_name('default_strategy') == 'default_strategy' - assert Strategy._sanitize_module_name('default_strategy.py') == 'default_strategy' - assert Strategy._sanitize_module_name('../default_strategy.py') == 'default_strategy' - assert Strategy._sanitize_module_name('../default_strategy') == 'default_strategy' - assert Strategy._sanitize_module_name('.default_strategy') == '.default_strategy' - assert Strategy._sanitize_module_name('foo-bar') == 'foo-bar' - assert Strategy._sanitize_module_name('foo/bar') == 'bar' + assert StrategyResolver._sanitize_module_name('default_strategy') == 'default_strategy' + assert StrategyResolver._sanitize_module_name('default_strategy.py') == 'default_strategy' + assert StrategyResolver._sanitize_module_name('../default_strategy.py') == 'default_strategy' + assert StrategyResolver._sanitize_module_name('../default_strategy') == 'default_strategy' + assert StrategyResolver._sanitize_module_name('.default_strategy') == '.default_strategy' + assert StrategyResolver._sanitize_module_name('foo-bar') == 'foo-bar' + assert StrategyResolver._sanitize_module_name('foo/bar') == 'bar' def test_search_strategy(): - assert Strategy._search_strategy('default_strategy') == '.' - assert Strategy._search_strategy('test_strategy') == 'user_data.strategies.' - assert Strategy._search_strategy('super_duper') is None + assert StrategyResolver._search_strategy('default_strategy') == '.' + assert StrategyResolver._search_strategy('test_strategy') == 'user_data.strategies.' + assert StrategyResolver._search_strategy('super_duper') is None def test_strategy_structure(): - assert hasattr(Strategy, 'populate_indicators') - assert hasattr(Strategy, 'populate_buy_trend') - assert hasattr(Strategy, 'populate_sell_trend') + assert hasattr(StrategyResolver, 'populate_indicators') + assert hasattr(StrategyResolver, 'populate_buy_trend') + assert hasattr(StrategyResolver, 'populate_sell_trend') def test_load_strategy(result): - strategy = Strategy() + strategy = StrategyResolver() strategy.logger = logging.getLogger(__name__) - assert not hasattr(Strategy, 'custom_strategy') + assert not hasattr(StrategyResolver, 'custom_strategy') strategy._load_strategy('test_strategy') - assert not hasattr(Strategy, 'custom_strategy') + assert not hasattr(StrategyResolver, 'custom_strategy') assert hasattr(strategy.custom_strategy, 'populate_indicators') assert 'adx' in strategy.populate_indicators(result) def test_load_not_found_strategy(caplog): - strategy = Strategy() + strategy = StrategyResolver() strategy.logger = logging.getLogger(__name__) - assert not hasattr(Strategy, 'custom_strategy') + assert not hasattr(StrategyResolver, 'custom_strategy') strategy._load_strategy('NotFoundStrategy') error_msg = "Impossible to load Strategy 'user_data/strategies/{}.py'. This file does not " \ @@ -53,7 +53,7 @@ def test_load_not_found_strategy(caplog): def test_strategy(result): - strategy = Strategy({'strategy': 'default_strategy'}) + strategy = StrategyResolver({'strategy': 'default_strategy'}) assert hasattr(strategy.custom_strategy, 'minimal_roi') assert strategy.minimal_roi[0] == 0.04 @@ -81,11 +81,11 @@ def test_strategy_override_minimal_roi(caplog): "0": 0.5 } } - strategy = Strategy(config) + strategy = StrategyResolver(config) assert hasattr(strategy.custom_strategy, 'minimal_roi') assert strategy.minimal_roi[0] == 0.5 - assert ('freqtrade.strategy.strategy', + assert ('freqtrade.strategy.resolver', logging.INFO, 'Override strategy \'minimal_roi\' with value in config file.' ) in caplog.record_tuples @@ -97,11 +97,11 @@ def test_strategy_override_stoploss(caplog): 'strategy': 'default_strategy', 'stoploss': -0.5 } - strategy = Strategy(config) + strategy = StrategyResolver(config) assert hasattr(strategy.custom_strategy, 'stoploss') assert strategy.stoploss == -0.5 - assert ('freqtrade.strategy.strategy', + assert ('freqtrade.strategy.resolver', logging.INFO, 'Override strategy \'stoploss\' with value in config file: -0.5.' ) in caplog.record_tuples @@ -114,31 +114,31 @@ def test_strategy_override_ticker_interval(caplog): 'strategy': 'default_strategy', 'ticker_interval': 60 } - strategy = Strategy(config) + strategy = StrategyResolver(config) assert hasattr(strategy.custom_strategy, 'ticker_interval') assert strategy.ticker_interval == 60 - assert ('freqtrade.strategy.strategy', + assert ('freqtrade.strategy.resolver', logging.INFO, 'Override strategy \'ticker_interval\' with value in config file: 60.' ) in caplog.record_tuples def test_strategy_fallback_default_strategy(): - strategy = Strategy() + strategy = StrategyResolver() strategy.logger = logging.getLogger(__name__) - assert not hasattr(Strategy, 'custom_strategy') + assert not hasattr(StrategyResolver, 'custom_strategy') strategy._load_strategy('../../super_duper') - assert not hasattr(Strategy, 'custom_strategy') + assert not hasattr(StrategyResolver, 'custom_strategy') def test_strategy_singleton(): - strategy1 = Strategy({'strategy': 'default_strategy'}) + strategy1 = StrategyResolver({'strategy': 'default_strategy'}) assert hasattr(strategy1.custom_strategy, 'minimal_roi') assert strategy1.minimal_roi[0] == 0.04 - strategy2 = Strategy() + strategy2 = StrategyResolver() assert hasattr(strategy2.custom_strategy, 'minimal_roi') assert strategy2.minimal_roi[0] == 0.04 diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index 1f69a7d32..fc79cb74c 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -4,7 +4,7 @@ import pandas from freqtrade.analyze import Analyze from freqtrade.optimize import load_data -from freqtrade.strategy.strategy import Strategy +from freqtrade.strategy.resolver import StrategyResolver _pairs = ['BTC_ETH'] @@ -21,13 +21,13 @@ def load_dataframe_pair(pairs): def test_dataframe_load(): - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'default_strategy'}) dataframe = load_dataframe_pair(_pairs) assert isinstance(dataframe, pandas.core.frame.DataFrame) def test_dataframe_columns_exists(): - Strategy({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'default_strategy'}) dataframe = load_dataframe_pair(_pairs) assert 'high' in dataframe.columns assert 'low' in dataframe.columns diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 3560b2db1..4d468b589 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -47,7 +47,7 @@ def test_common_datearray(default_conf, mocker) -> None: Test common_datearray() :return: None """ - mocker.patch('freqtrade.strategy.strategy.Strategy', MagicMock()) + mocker.patch('freqtrade.strategy.resolver.StrategyResolver', MagicMock()) analyze = Analyze(default_conf) tick = load_tickerdata_file(None, 'BTC_UNITEST', 1) From 6e5c14a95b50871ef61d4e62f0df38d8dbab419f Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 18:14:05 +0100 Subject: [PATCH 10/57] fix mutable default argument --- freqtrade/strategy/resolver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 74391c85d..583f24a3f 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -7,6 +7,7 @@ import importlib import os import sys from collections import OrderedDict +from typing import Optional, Dict from pandas import DataFrame @@ -21,12 +22,14 @@ class StrategyResolver(object): """ This class contains all the logic to load custom strategy class """ - def __init__(self, config: dict = {}) -> None: + def __init__(self, config: Optional[Dict] = None) -> None: """ Load the custom class from config parameter :param config: :return: """ + config = config or {} + self.logger = Logger(name=__name__).get_logger() # Verify the strategy is in the configuration, otherwise fallback to the default strategy From b4d2a3f4950193fa5791a669ef7e6e1f81e57a62 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 20:44:04 +0100 Subject: [PATCH 11/57] refactor StrategyResolver to work with class names --- freqtrade/arguments.py | 2 +- freqtrade/constants.py | 2 +- freqtrade/strategy/default_strategy.py | 2 - freqtrade/strategy/resolver.py | 105 ++++++++---------- freqtrade/tests/optimize/test_backtesting.py | 14 +-- freqtrade/tests/optimize/test_hyperopt.py | 12 +- .../tests/strategy/test_default_strategy.py | 6 +- freqtrade/tests/strategy/test_strategy.py | 30 ++--- freqtrade/tests/test_analyze.py | 2 +- freqtrade/tests/test_configuration.py | 12 +- freqtrade/tests/test_dataframe.py | 6 +- user_data/strategies/test_strategy.py | 4 - 12 files changed, 85 insertions(+), 112 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index c69135117..5396ae682 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -82,7 +82,7 @@ class Arguments(object): '-s', '--strategy', help='specify strategy file (default: %(default)s)', dest='strategy', - default='default_strategy', + default='DefaultStrategy', type=str, metavar='PATH', ) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a3f91d774..61adf307a 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -14,7 +14,7 @@ class Constants(object): TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec - DEFAULT_STRATEGY = 'default_strategy' + DEFAULT_STRATEGY = 'DefaultStrategy' # Required json-schema for user specified config CONF_SCHEMA = { diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index ea37735b7..4b645dbd0 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -7,8 +7,6 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.indicator_helpers import fishers_inverse from freqtrade.strategy.interface import IStrategy -class_name = 'DefaultStrategy' - class DefaultStrategy(IStrategy): """ diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 583f24a3f..fd48d1f1c 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -3,11 +3,11 @@ """ This module load custom strategies """ -import importlib +import importlib.util +import inspect import os -import sys from collections import OrderedDict -from typing import Optional, Dict +from typing import Optional, Dict, Type from pandas import DataFrame @@ -15,8 +15,6 @@ from freqtrade.constants import Constants from freqtrade.logger import Logger from freqtrade.strategy.interface import IStrategy -sys.path.insert(0, r'../../user_data/strategies') - class StrategyResolver(object): """ @@ -38,7 +36,7 @@ class StrategyResolver(object): else: strategy = Constants.DEFAULT_STRATEGY - # Load the strategy + # Try to load the strategy self._load_strategy(strategy) # Set attributes @@ -72,26 +70,27 @@ class StrategyResolver(object): def _load_strategy(self, strategy_name: str) -> None: """ - Search and load the custom strategy. If no strategy found, fallback on the default strategy - Set the object into self.custom_strategy + Search and loads the specified strategy. :param strategy_name: name of the module to import :return: None """ - try: - # Start by sanitizing the file name (remove any extensions) - strategy_name = self._sanitize_module_name(filename=strategy_name) - - # Search where can be the strategy file - path = self._search_strategy(filename=strategy_name) - - # Load the strategy - self.custom_strategy = self._load_class(path + strategy_name) + current_path = os.path.dirname(os.path.realpath(__file__)) + abs_paths = [ + os.path.join(current_path, '..', '..', 'user_data', 'strategies'), + current_path, + ] + for path in abs_paths: + self.custom_strategy = self._search_strategy(path, strategy_name) + if self.custom_strategy: + self.logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) + return None + raise ImportError('not found') # Fallback to the default strategy except (ImportError, TypeError) as error: self.logger.error( - "Impossible to load Strategy 'user_data/strategies/%s.py'. This file does not exist" + "Impossible to load Strategy '%s'. This class does not exist" " or contains Python code errors", strategy_name ) @@ -100,50 +99,44 @@ class StrategyResolver(object): error ) - def _load_class(self, filename: str) -> IStrategy: - """ - Import a strategy as a module - :param filename: path to the strategy (path from freqtrade/strategy/) - :return: return the strategy class - """ - module = importlib.import_module(filename, __package__) - custom_strategy = getattr(module, module.class_name) - - self.logger.info("Load strategy class: %s (%s.py)", module.class_name, filename) - return custom_strategy() - @staticmethod - def _sanitize_module_name(filename: str) -> str: + def _get_valid_strategies(module_path: str, strategy_name: str) -> Optional[Type[IStrategy]]: """ - Remove any extension from filename - :param filename: filename to sanatize - :return: return the filename without extensions + Returns a list of all possible strategies for the given module_path + :param module_path: absolute path to the module + :param strategy_name: Class name of the strategy + :return: Tuple with (name, class) or None """ - filename = os.path.basename(filename) - filename = os.path.splitext(filename)[0] - return filename - @staticmethod - def _search_strategy(filename: str) -> str: - """ - Search for the Strategy file in different folder - 1. search into the user_data/strategies folder - 2. search into the freqtrade/strategy folder - 3. if nothing found, return None - :param strategy_name: module name to search - :return: module path where is the strategy - """ - pwd = os.path.dirname(os.path.realpath(__file__)) + '/' - user_data = os.path.join(pwd, '..', '..', 'user_data', 'strategies', filename + '.py') - strategy_folder = os.path.join(pwd, filename + '.py') + # Generate spec based on absolute path + spec = importlib.util.spec_from_file_location('user_data.strategies', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) - path = None - if os.path.isfile(user_data): - path = 'user_data.strategies.' - elif os.path.isfile(strategy_folder): - path = '.' + valid_strategies_gen = ( + obj for name, obj in inspect.getmembers(module, inspect.isclass) + if strategy_name == name and IStrategy in obj.__bases__ + ) + return next(valid_strategies_gen, None) - return path + def _search_strategy(self, directory: str, strategy_name: str) -> Optional[IStrategy]: + """ + Search for the strategy_name in the given directory + :param directory: relative or absolute directory path + :return: name of the strategy class + """ + self.logger.debug('Searching for strategy %s in \'%s\'', strategy_name, directory) + for entry in os.listdir(directory): + # Only consider python files + if not entry.endswith('.py'): + self.logger.debug('Ignoring %s', entry) + continue + strategy = StrategyResolver._get_valid_strategies( + os.path.abspath(os.path.join(directory, entry)), strategy_name + ) + if strategy: + return strategy() + return None def populate_indicators(self, dataframe: DataFrame) -> DataFrame: """ diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 021474d5c..146da8faa 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -174,7 +174,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'backtesting' ] @@ -215,7 +215,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', '--datadir', '/foo/bar', 'backtesting', '--ticker-interval', '1', @@ -277,7 +277,7 @@ def test_start(mocker, default_conf, caplog) -> None: )) args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'backtesting' ] args = get_args(args) @@ -498,7 +498,7 @@ def test_backtest_ticks(default_conf): def test_backtest_clash_buy_sell(default_conf): - # Override the default buy trend function in our default_strategy + # Override the default buy trend function in our DefaultStrategy def fun(dataframe=None): buy_value = 1 sell_value = 1 @@ -510,7 +510,7 @@ def test_backtest_clash_buy_sell(default_conf): def test_backtest_only_sell(default_conf): - # Override the default buy trend function in our default_strategy + # Override the default buy trend function in our DefaultStrategy def fun(dataframe=None): buy_value = 0 sell_value = 1 @@ -578,12 +578,12 @@ def test_backtest_start_live(default_conf, mocker, caplog): args.live = True args.datadir = None args.export = None - args.strategy = 'default_strategy' + args.strategy = 'DefaultStrategy' args.timerange = '-100' # needed due to MagicMock malleability args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'backtesting', '--ticker-interval', '1', '--live', diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 688a7aa8d..affb0a6d3 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -57,12 +57,12 @@ def test_start(mocker, default_conf, caplog) -> None: )) args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'hyperopt', '--epochs', '5' ] args = get_args(args) - StrategyResolver({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) start(args) import pprint @@ -80,7 +80,7 @@ def test_loss_calculation_prefer_correct_trade_count() -> None: Test Hyperopt.calculate_loss() """ hyperopt = _HYPEROPT - StrategyResolver({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20) over = hyperopt.calculate_loss(1, hyperopt.target_trades + 100, 20) @@ -171,7 +171,7 @@ def test_fmin_best_results(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) - StrategyResolver({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) hyperopt.tickerdata_to_dataframe = MagicMock() @@ -215,7 +215,7 @@ def test_fmin_throw_value_error(mocker, default_conf, caplog) -> None: conf.update({'spaces': 'all'}) mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) - StrategyResolver({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) hyperopt.tickerdata_to_dataframe = MagicMock() @@ -258,7 +258,7 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, default_conf) -> No mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) - StrategyResolver({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = trials hyperopt.tickerdata_to_dataframe = MagicMock() diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 2b91fbec5..5c9be0b32 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -4,7 +4,7 @@ import pytest from pandas import DataFrame from freqtrade.analyze import Analyze -from freqtrade.strategy.default_strategy import DefaultStrategy, class_name +from freqtrade.strategy.default_strategy import DefaultStrategy @pytest.fixture @@ -13,10 +13,6 @@ def result(): return Analyze.parse_ticker_dataframe(json.load(data_file)) -def test_default_strategy_class_name(): - assert class_name == DefaultStrategy.__name__ - - def test_default_strategy_structure(): assert hasattr(DefaultStrategy, 'minimal_roi') assert hasattr(DefaultStrategy, 'stoploss') diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 7dc602f58..a34aba4b5 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -5,20 +5,10 @@ import logging from freqtrade.strategy.resolver import StrategyResolver -def test_sanitize_module_name(): - assert StrategyResolver._sanitize_module_name('default_strategy') == 'default_strategy' - assert StrategyResolver._sanitize_module_name('default_strategy.py') == 'default_strategy' - assert StrategyResolver._sanitize_module_name('../default_strategy.py') == 'default_strategy' - assert StrategyResolver._sanitize_module_name('../default_strategy') == 'default_strategy' - assert StrategyResolver._sanitize_module_name('.default_strategy') == '.default_strategy' - assert StrategyResolver._sanitize_module_name('foo-bar') == 'foo-bar' - assert StrategyResolver._sanitize_module_name('foo/bar') == 'bar' - - def test_search_strategy(): - assert StrategyResolver._search_strategy('default_strategy') == '.' - assert StrategyResolver._search_strategy('test_strategy') == 'user_data.strategies.' - assert StrategyResolver._search_strategy('super_duper') is None + assert StrategyResolver._search_strategy('DefaultStrategy') == '.' + assert StrategyResolver._search_strategy('TestStrategy') == 'user_data.strategies.' + assert StrategyResolver._search_strategy('NotFoundStrategy') is None def test_strategy_structure(): @@ -32,7 +22,7 @@ def test_load_strategy(result): strategy.logger = logging.getLogger(__name__) assert not hasattr(StrategyResolver, 'custom_strategy') - strategy._load_strategy('test_strategy') + strategy._load_strategy('TestStrategy') assert not hasattr(StrategyResolver, 'custom_strategy') @@ -47,13 +37,13 @@ def test_load_not_found_strategy(caplog): assert not hasattr(StrategyResolver, 'custom_strategy') strategy._load_strategy('NotFoundStrategy') - error_msg = "Impossible to load Strategy 'user_data/strategies/{}.py'. This file does not " \ + error_msg = "Impossible to load Strategy '{}'. This class does not " \ "exist or contains Python code errors".format('NotFoundStrategy') assert ('test_strategy', logging.ERROR, error_msg) in caplog.record_tuples def test_strategy(result): - strategy = StrategyResolver({'strategy': 'default_strategy'}) + strategy = StrategyResolver({'strategy': 'DefaultStrategy'}) assert hasattr(strategy.custom_strategy, 'minimal_roi') assert strategy.minimal_roi[0] == 0.04 @@ -76,7 +66,7 @@ def test_strategy(result): def test_strategy_override_minimal_roi(caplog): caplog.set_level(logging.INFO) config = { - 'strategy': 'default_strategy', + 'strategy': 'DefaultStrategy', 'minimal_roi': { "0": 0.5 } @@ -94,7 +84,7 @@ def test_strategy_override_minimal_roi(caplog): def test_strategy_override_stoploss(caplog): caplog.set_level(logging.INFO) config = { - 'strategy': 'default_strategy', + 'strategy': 'DefaultStrategy', 'stoploss': -0.5 } strategy = StrategyResolver(config) @@ -111,7 +101,7 @@ def test_strategy_override_ticker_interval(caplog): caplog.set_level(logging.INFO) config = { - 'strategy': 'default_strategy', + 'strategy': 'DefaultStrategy', 'ticker_interval': 60 } strategy = StrategyResolver(config) @@ -134,7 +124,7 @@ def test_strategy_fallback_default_strategy(): def test_strategy_singleton(): - strategy1 = StrategyResolver({'strategy': 'default_strategy'}) + strategy1 = StrategyResolver({'strategy': 'DefaultStrategy'}) assert hasattr(strategy1.custom_strategy, 'minimal_roi') assert strategy1.minimal_roi[0] == 0.04 diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 558ea7ee5..a4f1ba549 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -16,7 +16,7 @@ from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.tests.conftest import log_has # Avoid to reinit the same object again and again -_ANALYZE = Analyze({'strategy': 'default_strategy'}) +_ANALYZE = Analyze({'strategy': 'DefaultStrategy'}) def test_signaltype_object() -> None: diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 002eac722..1085b0060 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -99,7 +99,7 @@ def test_load_config(default_conf, mocker) -> None: validated_conf = configuration.load_config() assert 'strategy' in validated_conf - assert validated_conf['strategy'] == 'default_strategy' + assert validated_conf['strategy'] == 'DefaultStrategy' assert 'dynamic_whitelist' not in validated_conf assert 'dry_run_db' not in validated_conf @@ -114,7 +114,7 @@ def test_load_config_with_params(default_conf, mocker) -> None: args = [ '--dynamic-whitelist', '10', - '--strategy', 'test_strategy', + '--strategy', 'TestStrategy', '--dry-run-db' ] args = Arguments(args, '').get_parsed_arg() @@ -125,7 +125,7 @@ def test_load_config_with_params(default_conf, mocker) -> None: assert 'dynamic_whitelist' in validated_conf assert validated_conf['dynamic_whitelist'] == 10 assert 'strategy' in validated_conf - assert validated_conf['strategy'] == 'test_strategy' + assert validated_conf['strategy'] == 'TestStrategy' assert 'dry_run_db' in validated_conf assert validated_conf['dry_run_db'] is True @@ -140,7 +140,7 @@ def test_show_info(default_conf, mocker, caplog) -> None: args = [ '--dynamic-whitelist', '10', - '--strategy', 'test_strategy', + '--strategy', 'TestStrategy', '--dry-run-db' ] args = Arguments(args, '').get_parsed_arg() @@ -184,7 +184,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', 'backtesting' ] @@ -228,7 +228,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non args = [ '--config', 'config.json', - '--strategy', 'default_strategy', + '--strategy', 'DefaultStrategy', '--datadir', '/foo/bar', 'backtesting', '--ticker-interval', '1', diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index fc79cb74c..b739ae370 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -15,19 +15,19 @@ def load_dataframe_pair(pairs): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - analyze = Analyze({'strategy': 'default_strategy'}) + analyze = Analyze({'strategy': 'DefaultStrategy'}) dataframe = analyze.analyze_ticker(dataframe) return dataframe def test_dataframe_load(): - StrategyResolver({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) dataframe = load_dataframe_pair(_pairs) assert isinstance(dataframe, pandas.core.frame.DataFrame) def test_dataframe_columns_exists(): - StrategyResolver({'strategy': 'default_strategy'}) + StrategyResolver({'strategy': 'DefaultStrategy'}) dataframe = load_dataframe_pair(_pairs) assert 'high' in dataframe.columns assert 'low' in dataframe.columns diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index a164812c4..4ba1dbe17 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -10,10 +10,6 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib import numpy # noqa -# Update this variable if you change the class name -class_name = 'TestStrategy' - - # This class is a sample. Feel free to customize it. class TestStrategy(IStrategy): """ From a38c2121cc8dce2c97dbde31369bed83fc5a9d5a Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 21:56:20 +0100 Subject: [PATCH 12/57] adapt tests --- freqtrade/strategy/resolver.py | 24 ++++++++++++----------- freqtrade/tests/strategy/test_strategy.py | 17 +++++++--------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index fd48d1f1c..2e1136fde 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -16,6 +16,9 @@ from freqtrade.logger import Logger from freqtrade.strategy.interface import IStrategy +logger = Logger(name=__name__).get_logger() + + class StrategyResolver(object): """ This class contains all the logic to load custom strategy class @@ -28,8 +31,6 @@ class StrategyResolver(object): """ config = config or {} - self.logger = Logger(name=__name__).get_logger() - # Verify the strategy is in the configuration, otherwise fallback to the default strategy if 'strategy' in config: strategy = config['strategy'] @@ -43,17 +44,17 @@ class StrategyResolver(object): # Check if we need to override configuration if 'minimal_roi' in config: self.custom_strategy.minimal_roi = config['minimal_roi'] - self.logger.info("Override strategy \'minimal_roi\' with value in config file.") + logger.info("Override strategy \'minimal_roi\' with value in config file.") if 'stoploss' in config: self.custom_strategy.stoploss = config['stoploss'] - self.logger.info( + logger.info( "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] ) if 'ticker_interval' in config: self.custom_strategy.ticker_interval = config['ticker_interval'] - self.logger.info( + logger.info( "Override strategy \'ticker_interval\' with value in config file: %s.", config['ticker_interval'] ) @@ -83,18 +84,18 @@ class StrategyResolver(object): for path in abs_paths: self.custom_strategy = self._search_strategy(path, strategy_name) if self.custom_strategy: - self.logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) + logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) return None raise ImportError('not found') # Fallback to the default strategy except (ImportError, TypeError) as error: - self.logger.error( + logger.error( "Impossible to load Strategy '%s'. This class does not exist" " or contains Python code errors", strategy_name ) - self.logger.error( + logger.error( "The error is:\n%s.", error ) @@ -119,17 +120,18 @@ class StrategyResolver(object): ) return next(valid_strategies_gen, None) - def _search_strategy(self, directory: str, strategy_name: str) -> Optional[IStrategy]: + @staticmethod + def _search_strategy(directory: str, strategy_name: str) -> Optional[IStrategy]: """ Search for the strategy_name in the given directory :param directory: relative or absolute directory path :return: name of the strategy class """ - self.logger.debug('Searching for strategy %s in \'%s\'', strategy_name, directory) + logger.debug('Searching for strategy %s in \'%s\'', strategy_name, directory) for entry in os.listdir(directory): # Only consider python files if not entry.endswith('.py'): - self.logger.debug('Ignoring %s', entry) + logger.debug('Ignoring %s', entry) continue strategy = StrategyResolver._get_valid_strategies( os.path.abspath(os.path.join(directory, entry)), strategy_name diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index a34aba4b5..b6117fd00 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -2,19 +2,16 @@ import logging +import os + +from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.resolver import StrategyResolver def test_search_strategy(): - assert StrategyResolver._search_strategy('DefaultStrategy') == '.' - assert StrategyResolver._search_strategy('TestStrategy') == 'user_data.strategies.' - assert StrategyResolver._search_strategy('NotFoundStrategy') is None - - -def test_strategy_structure(): - assert hasattr(StrategyResolver, 'populate_indicators') - assert hasattr(StrategyResolver, 'populate_buy_trend') - assert hasattr(StrategyResolver, 'populate_sell_trend') + default_location = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'strategy') + assert isinstance(StrategyResolver._search_strategy(default_location, 'DefaultStrategy'), IStrategy) + assert StrategyResolver._search_strategy(default_location, 'NotFoundStrategy') is None def test_load_strategy(result): @@ -39,7 +36,7 @@ def test_load_not_found_strategy(caplog): error_msg = "Impossible to load Strategy '{}'. This class does not " \ "exist or contains Python code errors".format('NotFoundStrategy') - assert ('test_strategy', logging.ERROR, error_msg) in caplog.record_tuples + assert ('freqtrade.strategy.resolver', logging.ERROR, error_msg) in caplog.record_tuples def test_strategy(result): From 3cee94226f2eb128fa65489b9a9202eaf2cd7698 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 22:16:42 +0100 Subject: [PATCH 13/57] fix flake8 warnings --- freqtrade/tests/strategy/test_strategy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index b6117fd00..3898a9e38 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -9,8 +9,12 @@ from freqtrade.strategy.resolver import StrategyResolver def test_search_strategy(): - default_location = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'strategy') - assert isinstance(StrategyResolver._search_strategy(default_location, 'DefaultStrategy'), IStrategy) + default_location = os.path.join(os.path.dirname( + os.path.realpath(__file__)), '..', '..', 'strategy' + ) + assert isinstance( + StrategyResolver._search_strategy(default_location, 'DefaultStrategy'), IStrategy + ) assert StrategyResolver._search_strategy(default_location, 'NotFoundStrategy') is None From 4fac61387fb215341d06deda66a59a3bd6546d5a Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 22:28:42 +0100 Subject: [PATCH 14/57] adapt docs/bot-optimization --- docs/bot-optimization.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 6911e9e20..7e64cd95a 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -14,12 +14,12 @@ Since the version `0.16.0` the bot allows using custom strategy file. This is very simple. Copy paste your strategy file into the folder `user_data/strategies`. -Let assume you have a strategy file `awesome-strategy.py`: +Let assume you have a class called `AwesomeStragety` in the file `awesome-strategy.py`: 1. Move your file into `user_data/strategies` (you should have `user_data/strategies/awesome-strategy.py` -2. Start the bot with the param `--strategy awesome-strategy` (the parameter is the name of the file without '.py') +2. Start the bot with the param `--strategy AwesomeStragety` (the parameter is the class name) ```bash -python3 ./freqtrade/main.py --strategy awesome_strategy +python3 ./freqtrade/main.py --strategy AwesomeStragety ``` ## Change your strategy @@ -35,11 +35,11 @@ A strategy file contains all the information needed to build a good strategy: - Stoploss recommended - Hyperopt parameter -The bot also include a sample strategy you can update: `user_data/strategies/test_strategy.py`. -You can test it with the parameter: `--strategy test_strategy` +The bot also include a sample strategy called `TestStrategy` you can update: `user_data/strategies/test_strategy.py`. +You can test it with the parameter: `--strategy TestStrategy` ```bash -python3 ./freqtrade/main.py --strategy awesome_strategy +python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` **For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py) From bd2a6467fe9fe01a7c0ee26cd18bc14d76d7b27d Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 22:30:21 +0100 Subject: [PATCH 15/57] adapt argument description and metavar --- freqtrade/arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 5396ae682..052b4534e 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -80,11 +80,11 @@ class Arguments(object): ) self.parser.add_argument( '-s', '--strategy', - help='specify strategy file (default: %(default)s)', + help='specify strategy class name (default: %(default)s)', dest='strategy', default='DefaultStrategy', type=str, - metavar='PATH', + metavar='NAME', ) self.parser.add_argument( '--dynamic-whitelist', From 6b47c39103657ac804745c87331c6dcf07e6d28d Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 22:51:04 +0100 Subject: [PATCH 16/57] remove invalid mock --- freqtrade/tests/test_misc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 4d468b589..837a2a66f 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -47,8 +47,6 @@ def test_common_datearray(default_conf, mocker) -> None: Test common_datearray() :return: None """ - mocker.patch('freqtrade.strategy.resolver.StrategyResolver', MagicMock()) - analyze = Analyze(default_conf) tick = load_tickerdata_file(None, 'BTC_UNITEST', 1) tickerlist = {'BTC_UNITEST': tick} From 7fe0ec540770b46237b4c88187fceee78a263df2 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 25 Mar 2018 16:39:31 +0200 Subject: [PATCH 17/57] adapt docs/bot-usage to reflect changes --- docs/bot-usage.md | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index cf3258465..b00377982 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -26,9 +26,8 @@ optional arguments: --version show program's version number and exit -c PATH, --config PATH specify configuration file (default: config.json) - -s PATH, --strategy PATH - specify strategy file (default: - freqtrade/strategy/default_strategy.py) + -s NAME, --strategy NAME + specify strategy class name (default: DefaultStrategy) --dry-run-db Force dry run to use a local DB "tradesv3.dry_run.sqlite" instead of memory DB. Work only if dry_run is enabled. @@ -48,21 +47,19 @@ python3 ./freqtrade/main.py -c path/far/far/away/config.json ``` ### How to use --strategy? -This parameter will allow you to load your custom strategy file. Per -default without `--strategy` or `-s` the bot will load the -`default_strategy` included with the bot (`freqtrade/strategy/default_strategy.py`). +This parameter will allow you to load your custom strategy class. +Per default without `--strategy` or `-s` the bot will load the +`DefaultStrategy` included with the bot (`freqtrade/strategy/default_strategy.py`). -The bot will search your strategy file into `user_data/strategies` and -`freqtrade/strategy`. +The bot will search your strategy file within `user_data/strategies` and `freqtrade/strategy`. -To load a strategy, simply pass the file name (without .py) in this -parameters. +To load a strategy, simply pass the class name (e.g.: `CustomStrategy`) in this parameter. **Example:** -In `user_data/strategies` you have a file `my_awesome_strategy.py` to -load it: +In `user_data/strategies` you have a file `my_awesome_strategy.py` which has +a strategy class called `AwesomeStrategy` to load it: ```bash -python3 ./freqtrade/main.py --strategy my_awesome_strategy +python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` If the bot does not find your strategy file, it will display in an error From 7edbae893d0da0ba3b8c377426c954e131a1f263 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 25 Mar 2018 16:42:20 +0200 Subject: [PATCH 18/57] docs: fix typos --- docs/bot-optimization.md | 6 +++--- docs/bot-usage.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 7e64cd95a..00938adbe 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -14,12 +14,12 @@ Since the version `0.16.0` the bot allows using custom strategy file. This is very simple. Copy paste your strategy file into the folder `user_data/strategies`. -Let assume you have a class called `AwesomeStragety` in the file `awesome-strategy.py`: +Let assume you have a class called `AwesomeStrategy` in the file `awesome-strategy.py`: 1. Move your file into `user_data/strategies` (you should have `user_data/strategies/awesome-strategy.py` -2. Start the bot with the param `--strategy AwesomeStragety` (the parameter is the class name) +2. Start the bot with the param `--strategy AwesomeStrategy` (the parameter is the class name) ```bash -python3 ./freqtrade/main.py --strategy AwesomeStragety +python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` ## Change your strategy diff --git a/docs/bot-usage.md b/docs/bot-usage.md index b00377982..ea9b1e4d8 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -57,7 +57,7 @@ To load a strategy, simply pass the class name (e.g.: `CustomStrategy`) in this **Example:** In `user_data/strategies` you have a file `my_awesome_strategy.py` which has -a strategy class called `AwesomeStrategy` to load it: +a strategy class called `AwesomeStrategy` to load it: ```bash python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` From 3f8d7dae390554b24e786645cc0b5dd3df509817 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 25 Mar 2018 21:05:10 +0200 Subject: [PATCH 19/57] make name a required argument and add fallback to getEffectiveLevel --- freqtrade/analyze.py | 2 +- freqtrade/configuration.py | 2 +- freqtrade/freqtradebot.py | 2 +- freqtrade/logger.py | 7 +++---- freqtrade/main.py | 4 ++-- freqtrade/optimize/__init__.py | 2 +- freqtrade/optimize/backtesting.py | 4 ++-- freqtrade/optimize/hyperopt.py | 4 ++-- freqtrade/rpc/rpc.py | 2 +- freqtrade/rpc/rpc_manager.py | 2 +- freqtrade/strategy/strategy.py | 2 +- freqtrade/tests/test_logger.py | 4 ++-- scripts/plot_dataframe.py | 2 +- scripts/plot_profit.py | 2 +- 14 files changed, 20 insertions(+), 21 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 8bc552d74..b62d8240e 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -32,7 +32,7 @@ class Analyze(object): Init Analyze :param config: Bot configuration (use the one from Configuration()) """ - self.logger = Logger(name=__name__, level=config.get('loglevel')).get_logger() + self.logger = Logger(__name__, level=config.get('loglevel')).get_logger() self.config = config self.strategy = Strategy(self.config) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 1f6cea4e6..d4f73a086 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -20,7 +20,7 @@ class Configuration(object): """ def __init__(self, args: Namespace) -> None: self.args = args - self.logging = Logger(name=__name__) + self.logging = Logger(__name__) self.logger = self.logging.get_logger() self.config = None diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 29790515e..48212b865 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -38,7 +38,7 @@ class FreqtradeBot(object): """ # Init the logger - self.logger = Logger(name=__name__, level=config.get('loglevel')).get_logger() + self.logger = Logger(__name__, level=config.get('loglevel')).get_logger() # Init bot states self.state = State.STOPPED diff --git a/freqtrade/logger.py b/freqtrade/logger.py index 95e55e477..4d051dcdc 100644 --- a/freqtrade/logger.py +++ b/freqtrade/logger.py @@ -5,13 +5,14 @@ This module contains the class for logger and logging messages """ import logging +from typing import Optional class Logger(object): """ Logging class """ - def __init__(self, name='', level=logging.INFO) -> None: + def __init__(self, name, level: Optional[int] = None) -> None: """ Init the logger class :param name: Name of the Logger scope @@ -21,9 +22,7 @@ class Logger(object): self.name = name self.logger = None - if level is None: - level = logging.INFO - self.level = level + self.level = level or logging.getLogger().getEffectiveLevel() self._init_logger() diff --git a/freqtrade/main.py b/freqtrade/main.py index 556b70708..511270a60 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -14,8 +14,6 @@ from freqtrade.configuration import Configuration from freqtrade.freqtradebot import FreqtradeBot from freqtrade.logger import Logger -logger = Logger(name='freqtrade').get_logger() - def main(sysargv: List[str]) -> None: """ @@ -28,6 +26,8 @@ def main(sysargv: List[str]) -> None: ) args = arguments.get_parsed_arg() + logger = Logger(__name__, level=args.loglevel).get_logger() + # A subcommand has been issued. # Means if Backtesting or Hyperopt have been called we exit the bot if hasattr(args, 'func'): diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index a26744691..23e8f19ad 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -10,7 +10,7 @@ from freqtrade.exchange import get_ticker_history from freqtrade.logger import Logger from user_data.hyperopt_conf import hyperopt_optimize_conf -logger = Logger(name=__name__).get_logger() +logger = Logger(__name__).get_logger() def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d8af47326..5cedb093c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -32,7 +32,7 @@ class Backtesting(object): def __init__(self, config: Dict[str, Any]) -> None: # Init the logger - self.logging = Logger(name=__name__, level=config['loglevel']) + self.logging = Logger(__name__, level=config['loglevel']) self.logger = self.logging.get_logger() self.config = config self.analyze = None @@ -301,7 +301,7 @@ def start(args: Namespace) -> None: """ # Initialize logger - logger = Logger(name=__name__).get_logger() + logger = Logger(__name__).get_logger() logger.info('Starting freqtrade in Backtesting mode') # Initialize configuration diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7dcd46fd2..551c3e860 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -44,7 +44,7 @@ class Hyperopt(Backtesting): super().__init__(config) # Rename the logging to display Hyperopt file instead of Backtesting - self.logging = Logger(name=__name__, level=config['loglevel']) + self.logging = Logger(__name__, level=config['loglevel']) self.logger = self.logging.get_logger() # set TARGET_TRADES to suit your number concurrent trades so its realistic @@ -598,7 +598,7 @@ def start(args: Namespace) -> None: logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) # Initialize logger - logger = Logger(name=__name__).get_logger() + logger = Logger(__name__).get_logger() logger.info('Starting freqtrade in Hyperopt mode') # Initialize configuration diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index db6ff69d1..b052f6775 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -29,7 +29,7 @@ class RPC(object): """ self.freqtrade = freqtrade self.logger = Logger( - name=__name__, + __name__, level=self.freqtrade.config.get('loglevel') ).get_logger() diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index fb18a8d73..6bd1f69d0 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -20,7 +20,7 @@ class RPCManager(object): # Init the logger self.logger = Logger( - name=__name__, + __name__, level=self.freqtrade.config.get('loglevel') ).get_logger() diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index d7a89d1de..c6e6e49f7 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -27,7 +27,7 @@ class Strategy(object): :param config: :return: """ - self.logger = Logger(name=__name__).get_logger() + self.logger = Logger(__name__).get_logger() # Verify the strategy is in the configuration, otherwise fallback to the default strategy if 'strategy' in config: diff --git a/freqtrade/tests/test_logger.py b/freqtrade/tests/test_logger.py index 8e094b2d1..3e38084ad 100644 --- a/freqtrade/tests/test_logger.py +++ b/freqtrade/tests/test_logger.py @@ -12,13 +12,13 @@ def test_logger_object() -> None: Test the Constants object has the mandatory Constants :return: None """ - logger = Logger() + logger = Logger('') assert logger.name == '' assert logger.level == 20 assert logger.level is logging.INFO assert hasattr(logger, 'get_logger') - logger = Logger(name='Foo', level=logging.WARNING) + logger = Logger('Foo', level=logging.WARNING) assert logger.name == 'Foo' assert logger.name != '' assert logger.level == 30 diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 285ba6d97..1bb944c7b 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -28,7 +28,7 @@ from freqtrade.logger import Logger import freqtrade.optimize as optimize -logger = Logger(name="Graph dataframe").get_logger() +logger = Logger(__name__).get_logger() def plot_analyzed_dataframe(args: Namespace) -> None: diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 022bbf33c..1c025ce1d 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -30,7 +30,7 @@ import freqtrade.optimize as optimize import freqtrade.misc as misc -logger = Logger(name="Graph profits").get_logger() +logger = Logger(__name__).get_logger() # data:: [ pair, profit-%, enter, exit, time, duration] From fa7f74b4bce52441b0e8d2cbc354a69e7eb7ff35 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 25 Mar 2018 21:37:14 +0200 Subject: [PATCH 20/57] use native python logger --- freqtrade/analyze.py | 27 +++++------ freqtrade/configuration.py | 56 ++++++++++++----------- freqtrade/freqtradebot.py | 47 +++++++++++-------- freqtrade/main.py | 16 ++----- freqtrade/optimize/__init__.py | 4 +- freqtrade/optimize/backtesting.py | 28 +++++------- freqtrade/optimize/hyperopt.py | 35 ++++++-------- freqtrade/rpc/rpc.py | 12 ++--- freqtrade/rpc/rpc_manager.py | 17 +++---- freqtrade/rpc/telegram.py | 19 ++++---- freqtrade/strategy/strategy.py | 19 ++++---- freqtrade/tests/strategy/test_strategy.py | 4 +- scripts/plot_dataframe.py | 5 +- scripts/plot_profit.py | 5 +- 14 files changed, 142 insertions(+), 152 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b62d8240e..076d3d265 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -1,6 +1,7 @@ """ Functions to analyze ticker data with indicators and produce buy and sell signals """ +import logging from datetime import datetime, timedelta from enum import Enum from typing import Dict, List, Tuple @@ -9,11 +10,13 @@ import arrow from pandas import DataFrame, to_datetime from freqtrade.exchange import get_ticker_history -from freqtrade.logger import Logger from freqtrade.persistence import Trade from freqtrade.strategy.strategy import Strategy +logger = logging.getLogger(__name__) + + class SignalType(Enum): """ Enum to distinguish between buy and sell signals @@ -32,8 +35,6 @@ class Analyze(object): Init Analyze :param config: Bot configuration (use the one from Configuration()) """ - self.logger = Logger(__name__, level=config.get('loglevel')).get_logger() - self.config = config self.strategy = Strategy(self.config) @@ -107,20 +108,20 @@ class Analyze(object): """ ticker_hist = get_ticker_history(pair, interval) if not ticker_hist: - self.logger.warning('Empty ticker history for pair %s', pair) + logger.warning('Empty ticker history for pair %s', pair) return False, False try: dataframe = self.analyze_ticker(ticker_hist) except ValueError as error: - self.logger.warning( + logger.warning( 'Unable to analyze ticker for pair %s: %s', pair, str(error) ) return False, False except Exception as error: - self.logger.exception( + logger.exception( 'Unexpected error when analyzing ticker for pair %s: %s', pair, str(error) @@ -128,7 +129,7 @@ class Analyze(object): return False, False if dataframe.empty: - self.logger.warning('Empty dataframe for pair %s', pair) + logger.warning('Empty dataframe for pair %s', pair) return False, False latest = dataframe.iloc[-1] @@ -136,7 +137,7 @@ class Analyze(object): # Check if dataframe is out of date signal_date = arrow.get(latest['date']) if signal_date < arrow.utcnow() - timedelta(minutes=(interval + 5)): - self.logger.warning( + logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', pair, (arrow.utcnow() - signal_date).seconds // 60 @@ -144,7 +145,7 @@ class Analyze(object): return False, False (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 - self.logger.debug( + logger.debug( 'trigger: %s (pair=%s) buy=%s sell=%s', latest['date'], pair, @@ -161,17 +162,17 @@ class Analyze(object): """ # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) if self.min_roi_reached(trade=trade, current_rate=rate, current_time=date): - self.logger.debug('Required profit reached. Selling..') + logger.debug('Required profit reached. Selling..') return True # Experimental: Check if the trade is profitable before selling it (avoid selling at loss) if self.config.get('experimental', {}).get('sell_profit_only', False): - self.logger.debug('Checking if trade is profitable..') + logger.debug('Checking if trade is profitable..') if trade.calc_profit(rate=rate) <= 0: return False if sell and not buy and self.config.get('experimental', {}).get('use_sell_signal', False): - self.logger.debug('Sell signal received. Selling..') + logger.debug('Sell signal received. Selling..') return True return False @@ -184,7 +185,7 @@ class Analyze(object): """ current_profit = trade.calc_profit_percent(current_rate) if self.strategy.stoploss is not None and current_profit < self.strategy.stoploss: - self.logger.debug('Stop loss hit.') + logger.debug('Stop loss hit.') return True # Check if time matches and current rate is above threshold diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index d4f73a086..1314f624e 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -3,6 +3,7 @@ This module contains the configuration class """ import json +import logging from argparse import Namespace from typing import Dict, Any @@ -10,7 +11,9 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from freqtrade.constants import Constants -from freqtrade.logger import Logger + + +logger = logging.getLogger(__name__) class Configuration(object): @@ -20,8 +23,6 @@ class Configuration(object): """ def __init__(self, args: Namespace) -> None: self.args = args - self.logging = Logger(__name__) - self.logger = self.logging.get_logger() self.config = None def load_config(self) -> Dict[str, Any]: @@ -29,7 +30,7 @@ class Configuration(object): Extract information for sys.argv and load the bot configuration :return: Configuration dictionary """ - self.logger.info('Using config: %s ...', self.args.config) + logger.info('Using config: %s ...', self.args.config) config = self._load_config_file(self.args.config) # Add the strategy file to use @@ -56,7 +57,7 @@ class Configuration(object): with open(path) as file: conf = json.load(file) except FileNotFoundError: - self.logger.critical( + logger.critical( 'Config file "%s" not found. Please create your config file', path ) @@ -64,7 +65,7 @@ class Configuration(object): if 'internals' not in conf: conf['internals'] = {} - self.logger.info('Validating configuration ...') + logger.info('Validating configuration ...') return self._validate_config(conf) @@ -77,13 +78,16 @@ class Configuration(object): # Log level if 'loglevel' in self.args and self.args.loglevel: config.update({'loglevel': self.args.loglevel}) - self.logging.set_level(self.args.loglevel) - self.logger.info('Log level set at %s', config['loglevel']) + logging.basicConfig( + level=config['loglevel'], + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + ) + logger.info('Log level set to %s', logging.getLevelName(config['loglevel'])) # Add dynamic_whitelist if found if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) - self.logger.info( + logger.info( 'Parameter --dynamic-whitelist detected. ' 'Using dynamically generated whitelist. ' '(not applicable with Backtesting and Hyperopt)' @@ -92,13 +96,13 @@ class Configuration(object): # Add dry_run_db if found and the bot in dry run if self.args.dry_run_db and config.get('dry_run', False): config.update({'dry_run_db': True}) - self.logger.info('Parameter --dry-run-db detected ...') + logger.info('Parameter --dry-run-db detected ...') if config.get('dry_run_db', False): if config.get('dry_run', False): - self.logger.info('Dry_run will use the DB file: "tradesv3.dry_run.sqlite"') + logger.info('Dry_run will use the DB file: "tradesv3.dry_run.sqlite"') else: - self.logger.info('Dry run is disabled. (--dry_run_db ignored)') + logger.info('Dry run is disabled. (--dry_run_db ignored)') return config @@ -112,39 +116,39 @@ class Configuration(object): # (that will override the strategy configuration) if 'ticker_interval' in self.args and self.args.ticker_interval: config.update({'ticker_interval': self.args.ticker_interval}) - self.logger.info('Parameter -i/--ticker-interval detected ...') - self.logger.info('Using ticker_interval: %d ...', config.get('ticker_interval')) + logger.info('Parameter -i/--ticker-interval detected ...') + logger.info('Using ticker_interval: %d ...', config.get('ticker_interval')) # If -l/--live is used we add it to the configuration if 'live' in self.args and self.args.live: config.update({'live': True}) - self.logger.info('Parameter -l/--live detected ...') + logger.info('Parameter -l/--live detected ...') # If --realistic-simulation is used we add it to the configuration if 'realistic_simulation' in self.args and self.args.realistic_simulation: config.update({'realistic_simulation': True}) - self.logger.info('Parameter --realistic-simulation detected ...') - self.logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) + logger.info('Parameter --realistic-simulation detected ...') + logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) # If --timerange is used we add it to the configuration if 'timerange' in self.args and self.args.timerange: config.update({'timerange': self.args.timerange}) - self.logger.info('Parameter --timerange detected: %s ...', self.args.timerange) + logger.info('Parameter --timerange detected: %s ...', self.args.timerange) # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self.args.datadir}) - self.logger.info('Parameter --datadir detected: %s ...', self.args.datadir) + logger.info('Parameter --datadir detected: %s ...', self.args.datadir) # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: config.update({'refresh_pairs': True}) - self.logger.info('Parameter -r/--refresh-pairs-cached detected ...') + logger.info('Parameter -r/--refresh-pairs-cached detected ...') # If --export is used we add it to the configuration if 'export' in self.args and self.args.export: config.update({'export': self.args.export}) - self.logger.info('Parameter --export detected: %s ...', self.args.export) + logger.info('Parameter --export detected: %s ...', self.args.export) return config @@ -156,18 +160,18 @@ class Configuration(object): # If --realistic-simulation is used we add it to the configuration if 'epochs' in self.args and self.args.epochs: config.update({'epochs': self.args.epochs}) - self.logger.info('Parameter --epochs detected ...') - self.logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) + logger.info('Parameter --epochs detected ...') + logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) # If --mongodb is used we add it to the configuration if 'mongodb' in self.args and self.args.mongodb: config.update({'mongodb': self.args.mongodb}) - self.logger.info('Parameter --use-mongodb detected ...') + logger.info('Parameter --use-mongodb detected ...') # If --spaces is used we add it to the configuration if 'spaces' in self.args and self.args.spaces: config.update({'spaces': self.args.spaces}) - self.logger.info('Parameter -s/--spaces detected: %s', config.get('spaces')) + logger.info('Parameter -s/--spaces detected: %s', config.get('spaces')) return config @@ -181,7 +185,7 @@ class Configuration(object): validate(conf, Constants.CONF_SCHEMA) return conf except ValidationError as exception: - self.logger.fatal( + logger.fatal( 'Invalid configuration. See config.json.example. Reason: %s', exception ) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 48212b865..3972a4382 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -4,6 +4,7 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() import copy import json +import logging import time import traceback from datetime import datetime @@ -13,16 +14,20 @@ import arrow import requests from cachetools import cached, TTLCache -from freqtrade import (DependencyException, OperationalException, exchange, persistence) +from freqtrade import ( + DependencyException, OperationalException, exchange, persistence, __version__ +) from freqtrade.analyze import Analyze from freqtrade.constants import Constants from freqtrade.fiat_convert import CryptoToFiatConverter -from freqtrade.logger import Logger from freqtrade.persistence import Trade from freqtrade.rpc.rpc_manager import RPCManager from freqtrade.state import State +logger = logging.getLogger(__name__) + + class FreqtradeBot(object): """ Freqtrade is the main class of the bot. @@ -37,8 +42,10 @@ class FreqtradeBot(object): :param db_url: database connector string for sqlalchemy (Optional) """ - # Init the logger - self.logger = Logger(__name__, level=config.get('loglevel')).get_logger() + logger.info( + 'Starting freqtrade %s', + __version__, + ) # Init bot states self.state = State.STOPPED @@ -81,7 +88,7 @@ class FreqtradeBot(object): :return: None """ self.rpc.send_msg('*Status:* `Stopping trader...`') - self.logger.info('Stopping trader and cleaning up modules...') + logger.info('Stopping trader and cleaning up modules...') self.state = State.STOPPED self.rpc.cleanup() persistence.cleanup() @@ -97,7 +104,7 @@ class FreqtradeBot(object): state = self.state if state != old_state: self.rpc.send_msg('*Status:* `{}`'.format(state.name.lower())) - self.logger.info('Changing state to: %s', state.name) + logger.info('Changing state to: %s', state.name) if state == State.STOPPED: time.sleep(1) @@ -129,7 +136,7 @@ class FreqtradeBot(object): result = func(*args, **kwargs) end = time.time() duration = max(min_secs - (end - start), 0.0) - self.logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) + logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) time.sleep(duration) return result @@ -170,7 +177,7 @@ class FreqtradeBot(object): Trade.session.flush() except (requests.exceptions.RequestException, json.JSONDecodeError) as error: - self.logger.warning('%s, retrying in 30 seconds...', error) + logger.warning('%s, retrying in 30 seconds...', error) time.sleep(Constants.RETRY_TIMEOUT) except OperationalException: self.rpc.send_msg( @@ -180,7 +187,7 @@ class FreqtradeBot(object): hint='Issue `/start` if you think it is safe to restart.' ) ) - self.logger.exception('OperationalException. Stopping trader ...') + logger.exception('OperationalException. Stopping trader ...') self.state = State.STOPPED return state_changed @@ -222,7 +229,7 @@ class FreqtradeBot(object): # Market is not active if not status['IsActive']: sanitized_whitelist.remove(pair) - self.logger.info( + logger.info( 'Ignoring %s from whitelist (reason: %s).', pair, status.get('Notice') or 'wallet is not active' ) @@ -253,7 +260,7 @@ class FreqtradeBot(object): stake_amount = self.config['stake_amount'] interval = self.analyze.get_ticker_interval() - self.logger.info( + logger.info( 'Checking buy signals to create a new trade with stake_amount: %f ...', stake_amount ) @@ -268,7 +275,7 @@ class FreqtradeBot(object): for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): if trade.pair in whitelist: whitelist.remove(trade.pair) - self.logger.debug('Ignoring %s in pair whitelist', trade.pair) + logger.debug('Ignoring %s in pair whitelist', trade.pair) if not whitelist: raise DependencyException('No currency pairs in whitelist') @@ -333,10 +340,10 @@ class FreqtradeBot(object): if self.create_trade(): return True - self.logger.info('Found no buy signals for whitelisted currencies. Trying again..') + logger.info('Found no buy signals for whitelisted currencies. Trying again..') return False except DependencyException as exception: - self.logger.warning('Unable to create trade: %s', exception) + logger.warning('Unable to create trade: %s', exception) return False def process_maybe_execute_sell(self, trade: Trade) -> bool: @@ -347,7 +354,7 @@ class FreqtradeBot(object): # Get order details for actual price per unit if trade.open_order_id: # Update trade with order values - self.logger.info('Found open order for %s', trade) + logger.info('Found open order for %s', trade) trade.update(exchange.get_order(trade.open_order_id)) if trade.is_open and trade.open_order_id is None: @@ -363,7 +370,7 @@ class FreqtradeBot(object): if not trade.is_open: raise ValueError('attempt to handle closed trade: {}'.format(trade)) - self.logger.debug('Handling %s ...', trade) + logger.debug('Handling %s ...', trade) current_rate = exchange.get_ticker(trade.pair)['bid'] (buy, sell) = (False, False) @@ -389,7 +396,7 @@ class FreqtradeBot(object): try: order = exchange.get_order(trade.open_order_id) except requests.exceptions.RequestException: - self.logger.info( + logger.info( 'Cannot query order for %s due to %s', trade, traceback.format_exc()) @@ -419,7 +426,7 @@ class FreqtradeBot(object): # FIX? do we really need to flush, caller of # check_handle_timedout will flush afterwards Trade.session.flush() - self.logger.info('Buy order timeout for %s.', trade) + logger.info('Buy order timeout for %s.', trade) self.rpc.send_msg('*Timeout:* Unfilled buy order for {} cancelled'.format( trade.pair.replace('_', '/'))) return True @@ -429,7 +436,7 @@ class FreqtradeBot(object): trade.amount = order['amount'] - order['remaining'] trade.stake_amount = trade.amount * trade.open_rate trade.open_order_id = None - self.logger.info('Partial buy order timeout for %s.', trade) + logger.info('Partial buy order timeout for %s.', trade) self.rpc.send_msg('*Timeout:* Remaining buy order for {} cancelled'.format( trade.pair.replace('_', '/'))) return False @@ -450,7 +457,7 @@ class FreqtradeBot(object): trade.open_order_id = None self.rpc.send_msg('*Timeout:* Unfilled sell order for {} cancelled'.format( trade.pair.replace('_', '/'))) - self.logger.info('Sell order timeout for %s.', trade) + logger.info('Sell order timeout for %s.', trade) return True # TODO: figure out how to handle partially complete sell orders diff --git a/freqtrade/main.py b/freqtrade/main.py index 511270a60..d62c92b9f 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -8,11 +8,11 @@ import logging import sys from typing import List -from freqtrade import (__version__) from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.logger import Logger + +logger = logging.getLogger('freqtrade') def main(sysargv: List[str]) -> None: @@ -26,28 +26,20 @@ def main(sysargv: List[str]) -> None: ) args = arguments.get_parsed_arg() - logger = Logger(__name__, level=args.loglevel).get_logger() - # A subcommand has been issued. # Means if Backtesting or Hyperopt have been called we exit the bot if hasattr(args, 'func'): args.func(args) return - logger.info( - 'Starting freqtrade %s (loglevel=%s)', - __version__, - logging.getLevelName(args.loglevel) - ) - freqtrade = None return_code = 1 try: # Load and validate configuration - configuration = Configuration(args) + config = Configuration(args).get_config() # Init the bot - freqtrade = FreqtradeBot(configuration.get_config()) + freqtrade = FreqtradeBot(config) state = None while 1: diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 23e8f19ad..fff4b72d9 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -2,15 +2,15 @@ import gzip import json +import logging import os from typing import Optional, List, Dict, Tuple from freqtrade import misc from freqtrade.exchange import get_ticker_history -from freqtrade.logger import Logger from user_data.hyperopt_conf import hyperopt_optimize_conf -logger = Logger(__name__).get_logger() +logger = logging.getLogger(__name__) def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5cedb093c..1d1c33ff9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -3,6 +3,7 @@ """ This module contains the backtesting logic """ +import logging from argparse import Namespace from typing import Dict, Tuple, Any, List, Optional @@ -16,11 +17,13 @@ from freqtrade.analyze import Analyze from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Bittrex -from freqtrade.logger import Logger from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade +logger = logging.getLogger(__name__) + + class Backtesting(object): """ Backtesting class, this class contains all the logic to run a backtest @@ -30,10 +33,6 @@ class Backtesting(object): backtesting.start() """ def __init__(self, config: Dict[str, Any]) -> None: - - # Init the logger - self.logging = Logger(__name__, level=config['loglevel']) - self.logger = self.logging.get_logger() self.config = config self.analyze = None self.ticker_interval = None @@ -200,7 +199,7 @@ class Backtesting(object): # For now export inside backtest(), maybe change so that backtest() # returns a tuple like: (dataframe, records, logs, etc) if record and record.find('trades') >= 0: - self.logger.info('Dumping backtest results') + logger.info('Dumping backtest results') file_dump_json('backtest-result.json', records) labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] return DataFrame.from_records(trades, columns=labels) @@ -212,15 +211,15 @@ class Backtesting(object): """ data = {} pairs = self.config['exchange']['pair_whitelist'] - self.logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - self.logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using stake_amount: %s ...', self.config['stake_amount']) if self.config.get('live'): - self.logger.info('Downloading data for all pairs in whitelist ...') + logger.info('Downloading data for all pairs in whitelist ...') for pair in pairs: data[pair] = exchange.get_ticker_history(pair, self.ticker_interval) else: - self.logger.info('Using local backtesting data (using whitelist in given config) ...') + logger.info('Using local backtesting data (using whitelist in given config) ...') timerange = Arguments.parse_timerange(self.config.get('timerange')) data = optimize.load_data( @@ -235,14 +234,14 @@ class Backtesting(object): if self.config.get('realistic_simulation', False): max_open_trades = self.config['max_open_trades'] else: - self.logger.info('Ignoring max_open_trades (realistic_simulation not set) ...') + logger.info('Ignoring max_open_trades (realistic_simulation not set) ...') max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) - self.logger.info( + logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), @@ -263,9 +262,7 @@ class Backtesting(object): 'record': self.config.get('export') } ) - - self.logging.set_format('%(message)s') - self.logger.info( + logger.info( '\n==================================== ' 'BACKTESTING REPORT' ' ====================================\n' @@ -301,7 +298,6 @@ def start(args: Namespace) -> None: """ # Initialize logger - logger = Logger(__name__).get_logger() logger.info('Starting freqtrade in Backtesting mode') # Initialize configuration diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 551c3e860..3e7b14610 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -31,6 +31,9 @@ from freqtrade.optimize.backtesting import Backtesting from user_data.hyperopt_conf import hyperopt_optimize_conf +logger = logging.getLogger(__name__) + + class Hyperopt(Backtesting): """ Hyperopt class, this class contains all the logic to run a hyperopt simulation @@ -42,11 +45,6 @@ class Hyperopt(Backtesting): def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - - # Rename the logging to display Hyperopt file instead of Backtesting - self.logging = Logger(__name__, level=config['loglevel']) - self.logger = self.logging.get_logger() - # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days self.target_trades = 600 @@ -194,14 +192,14 @@ class Hyperopt(Backtesting): """ Save hyperopt trials to file """ - self.logger.info('Saving Trials to \'%s\'', self.trials_file) + logger.info('Saving Trials to \'%s\'', self.trials_file) pickle.dump(self.trials, open(self.trials_file, 'wb')) def read_trials(self) -> Trials: """ Read hyperopt trials file """ - self.logger.info('Reading Trials from \'%s\'', self.trials_file) + logger.info('Reading Trials from \'%s\'', self.trials_file) trials = pickle.load(open(self.trials_file, 'rb')) os.remove(self.trials_file) return trials @@ -212,7 +210,7 @@ class Hyperopt(Backtesting): """ vals = json.dumps(self.trials.best_trial['misc']['vals'], indent=4) results = self.trials.best_trial['result']['result'] - self.logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + logger.info('Best result:\n%s\nwith values:\n%s', results, vals) def log_results(self, results) -> None: """ @@ -226,7 +224,7 @@ class Hyperopt(Backtesting): results['result'], results['loss'] ) - self.logger.info(log_msg) + logger.info(log_msg) else: print('.', end='') sys.stdout.flush() @@ -511,8 +509,8 @@ class Hyperopt(Backtesting): self.processed = self.tickerdata_to_dataframe(data) if self.config.get('mongodb'): - self.logger.info('Using mongodb ...') - self.logger.info( + logger.info('Using mongodb ...') + logger.info( 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' ) @@ -522,7 +520,7 @@ class Hyperopt(Backtesting): exp_key='exp1' ) else: - self.logger.info('Preparing Trials..') + logger.info('Preparing Trials..') signal.signal(signal.SIGINT, self.signal_handler) # read trials file if we have one if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: @@ -530,16 +528,13 @@ class Hyperopt(Backtesting): self.current_tries = len(self.trials.results) self.total_tries += self.current_tries - self.logger.info( + logger.info( 'Continuing with trials. Current: %d, Total: %d', self.current_tries, self.total_tries ) try: - # change the Logging format - self.logging.set_format('\n%(message)s') - best_parameters = fmin( fn=self.generate_optimizer, space=self.hyperopt_space(), @@ -563,11 +558,11 @@ class Hyperopt(Backtesting): best_parameters ) - self.logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) + logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) if 'roi_t1' in best_parameters: - self.logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) + logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) - self.logger.info('Best Result:\n%s', best_result) + logger.info('Best Result:\n%s', best_result) # Store trials result to file to resume next time self.save_trials() @@ -576,7 +571,7 @@ class Hyperopt(Backtesting): """ Hyperopt SIGINT handler """ - self.logger.info( + logger.info( 'Hyperopt received %s', signal.Signals(sig).name ) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index b052f6775..860d3ba43 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1,7 +1,7 @@ """ This module contains class to define a RPC communications """ - +import logging from datetime import datetime, timedelta from decimal import Decimal from typing import Tuple, Any @@ -11,12 +11,14 @@ import sqlalchemy as sql from pandas import DataFrame from freqtrade import exchange -from freqtrade.logger import Logger from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.state import State +logger = logging.getLogger(__name__) + + class RPC(object): """ RPC class can be used to have extra feature, like bot data, and access to DB data @@ -28,10 +30,6 @@ class RPC(object): :return: None """ self.freqtrade = freqtrade - self.logger = Logger( - __name__, - level=self.freqtrade.config.get('loglevel') - ).get_logger() def rpc_trade_status(self) -> Tuple[bool, Any]: """ @@ -346,7 +344,7 @@ class RPC(object): ) ).first() if not trade: - self.logger.warning('forcesell: Invalid argument received') + logger.warning('forcesell: Invalid argument received') return True, 'Invalid argument.' _exec_forcesell(trade) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 6bd1f69d0..0299c793a 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,11 +1,14 @@ """ This module contains class to manage RPC communications (Telegram, Slack, ...) """ +import logging -from freqtrade.logger import Logger from freqtrade.rpc.telegram import Telegram +logger = logging.getLogger(__name__) + + class RPCManager(object): """ Class to manage RPC objects (Telegram, Slack, ...) @@ -18,12 +21,6 @@ class RPCManager(object): """ self.freqtrade = freqtrade - # Init the logger - self.logger = Logger( - __name__, - level=self.freqtrade.config.get('loglevel') - ).get_logger() - self.registered_modules = [] self.telegram = None self._init() @@ -34,7 +31,7 @@ class RPCManager(object): :return: """ if self.freqtrade.config['telegram'].get('enabled', False): - self.logger.info('Enabling rpc.telegram ...') + logger.info('Enabling rpc.telegram ...') self.registered_modules.append('telegram') self.telegram = Telegram(self.freqtrade) @@ -44,7 +41,7 @@ class RPCManager(object): :return: None """ if 'telegram' in self.registered_modules: - self.logger.info('Cleaning up rpc.telegram ...') + logger.info('Cleaning up rpc.telegram ...') self.registered_modules.remove('telegram') self.telegram.cleanup() @@ -54,6 +51,6 @@ class RPCManager(object): :param msg: message :return: None """ - self.logger.info(msg) + logger.info(msg) if 'telegram' in self.registered_modules: self.telegram.send_msg(msg) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index fce7a81f9..81749d778 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -3,7 +3,7 @@ """ This module manage Telegram communication """ - +import logging from typing import Any, Callable from tabulate import tabulate @@ -15,6 +15,9 @@ from freqtrade.__init__ import __version__ from freqtrade.rpc.rpc import RPC +logger = logging.getLogger(__name__) + + def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[..., Any]: """ Decorator to check if the message comes from the correct chat_id @@ -31,13 +34,13 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ chat_id = int(self._config['telegram']['chat_id']) if int(update.message.chat_id) != chat_id: - self.logger.info( + logger.info( 'Rejected unauthorized message from: %s', update.message.chat_id ) return wrapper - self.logger.info( + logger.info( 'Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id @@ -45,7 +48,7 @@ def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[ try: return command_handler(self, *args, **kwargs) except BaseException: - self.logger.exception('Exception occurred within Telegram module') + logger.exception('Exception occurred within Telegram module') return wrapper @@ -101,7 +104,7 @@ class Telegram(RPC): timeout=30, read_latency=60, ) - self.logger.info( + logger.info( 'rpc.telegram is listening for following commands: %s', [h.command for h in handles] ) @@ -357,7 +360,7 @@ class Telegram(RPC): 'max': [self._config['max_open_trades']] }, headers=['current', 'max'], tablefmt='simple') message = "
{}
".format(message) - self.logger.debug(message) + logger.debug(message) self.send_msg(message, parse_mode=ParseMode.HTML) @authorized_only @@ -428,7 +431,7 @@ class Telegram(RPC): except NetworkError as network_err: # Sometimes the telegram server resets the current connection, # if this is the case we send the message again. - self.logger.warning( + logger.warning( 'Telegram NetworkError: %s! Trying one more time.', network_err.message ) @@ -439,7 +442,7 @@ class Telegram(RPC): reply_markup=reply_markup ) except TelegramError as telegram_err: - self.logger.warning( + logger.warning( 'TelegramError: %s! Giving up on that message.', telegram_err.message ) diff --git a/freqtrade/strategy/strategy.py b/freqtrade/strategy/strategy.py index c6e6e49f7..7840765bb 100644 --- a/freqtrade/strategy/strategy.py +++ b/freqtrade/strategy/strategy.py @@ -4,6 +4,7 @@ This module load custom strategies """ import importlib +import logging import os import sys from collections import OrderedDict @@ -11,12 +12,14 @@ from collections import OrderedDict from pandas import DataFrame from freqtrade.constants import Constants -from freqtrade.logger import Logger from freqtrade.strategy.interface import IStrategy sys.path.insert(0, r'../../user_data/strategies') +logger = logging.getLogger(__name__) + + class Strategy(object): """ This class contains all the logic to load custom strategy class @@ -27,8 +30,6 @@ class Strategy(object): :param config: :return: """ - self.logger = Logger(__name__).get_logger() - # Verify the strategy is in the configuration, otherwise fallback to the default strategy if 'strategy' in config: strategy = config['strategy'] @@ -42,17 +43,17 @@ class Strategy(object): # Check if we need to override configuration if 'minimal_roi' in config: self.custom_strategy.minimal_roi = config['minimal_roi'] - self.logger.info("Override strategy \'minimal_roi\' with value in config file.") + logger.info("Override strategy \'minimal_roi\' with value in config file.") if 'stoploss' in config: self.custom_strategy.stoploss = config['stoploss'] - self.logger.info( + logger.info( "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] ) if 'ticker_interval' in config: self.custom_strategy.ticker_interval = config['ticker_interval'] - self.logger.info( + logger.info( "Override strategy \'ticker_interval\' with value in config file: %s.", config['ticker_interval'] ) @@ -87,12 +88,12 @@ class Strategy(object): # Fallback to the default strategy except (ImportError, TypeError) as error: - self.logger.error( + logger.error( "Impossible to load Strategy 'user_data/strategies/%s.py'. This file does not exist" " or contains Python code errors", strategy_name ) - self.logger.error( + logger.error( "The error is:\n%s.", error ) @@ -106,7 +107,7 @@ class Strategy(object): module = importlib.import_module(filename, __package__) custom_strategy = getattr(module, module.class_name) - self.logger.info("Load strategy class: %s (%s.py)", module.class_name, filename) + logger.info("Load strategy class: %s (%s.py)", module.class_name, filename) return custom_strategy() @staticmethod diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 7ce9ae0ef..a364a59fe 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -29,7 +29,6 @@ def test_strategy_structure(): def test_load_strategy(result): strategy = Strategy() - strategy.logger = logging.getLogger(__name__) assert not hasattr(Strategy, 'custom_strategy') strategy._load_strategy('test_strategy') @@ -42,14 +41,13 @@ def test_load_strategy(result): def test_load_not_found_strategy(caplog): strategy = Strategy() - strategy.logger = logging.getLogger(__name__) assert not hasattr(Strategy, 'custom_strategy') strategy._load_strategy('NotFoundStrategy') error_msg = "Impossible to load Strategy 'user_data/strategies/{}.py'. This file does not " \ "exist or contains Python code errors".format('NotFoundStrategy') - assert ('test_strategy', logging.ERROR, error_msg) in caplog.record_tuples + assert ('freqtrade.strategy.strategy', logging.ERROR, error_msg) in caplog.record_tuples def test_strategy(result): diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 1bb944c7b..aa61fc172 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -11,7 +11,7 @@ Optional Cli parameters --timerange: specify what timerange of data to use. -l / --live: Live, to download the latest ticker for the pair """ - +import logging import sys from argparse import Namespace @@ -24,11 +24,10 @@ import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.analyze import Analyze from freqtrade import exchange -from freqtrade.logger import Logger import freqtrade.optimize as optimize -logger = Logger(__name__).get_logger() +logger = logging.getLogger(__name__) def plot_analyzed_dataframe(args: Namespace) -> None: diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 1c025ce1d..4bf220dd9 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -10,7 +10,7 @@ Optional Cli parameters -s / --strategy: strategy to use --timerange: specify what timerange of data to use. """ - +import logging import sys import json from argparse import Namespace @@ -24,13 +24,12 @@ import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.analyze import Analyze -from freqtrade.logger import Logger import freqtrade.optimize as optimize import freqtrade.misc as misc -logger = Logger(__name__).get_logger() +logger = logging.getLogger(__name__) # data:: [ pair, profit-%, enter, exit, time, duration] From f374a062e1566032fa9f1714ab39cde21262970f Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 25 Mar 2018 21:41:25 +0200 Subject: [PATCH 21/57] remove freqtrade/logger.py --- freqtrade/logger.py | 82 ------------------- freqtrade/optimize/backtesting.py | 5 +- freqtrade/optimize/hyperopt.py | 7 +- freqtrade/tests/optimize/test_hyperopt.py | 4 - freqtrade/tests/test_logger.py | 97 ----------------------- 5 files changed, 3 insertions(+), 192 deletions(-) delete mode 100644 freqtrade/logger.py delete mode 100644 freqtrade/tests/test_logger.py diff --git a/freqtrade/logger.py b/freqtrade/logger.py deleted file mode 100644 index 4d051dcdc..000000000 --- a/freqtrade/logger.py +++ /dev/null @@ -1,82 +0,0 @@ -# pragma pylint: disable=too-few-public-methods - -""" -This module contains the class for logger and logging messages -""" - -import logging -from typing import Optional - - -class Logger(object): - """ - Logging class - """ - def __init__(self, name, level: Optional[int] = None) -> None: - """ - Init the logger class - :param name: Name of the Logger scope - :param level: Logger level that should be used - :return: None - """ - self.name = name - self.logger = None - - self.level = level or logging.getLogger().getEffectiveLevel() - - self._init_logger() - - def _init_logger(self) -> None: - """ - Setup the bot logger configuration - :return: None - """ - logging.basicConfig( - level=self.level, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - ) - - self.logger = self.get_logger() - self.set_level(self.level) - - def get_logger(self) -> logging.RootLogger: - """ - Return the logger instance to use for sending message - :return: the logger instance - """ - return logging.getLogger(self.name) - - def set_name(self, name: str) -> logging.RootLogger: - """ - Set the name of the logger - :param name: Name of the logger - :return: None - """ - self.name = name - self.logger = self.get_logger() - return self.logger - - def set_level(self, level) -> None: - """ - Set the level of the logger - :param level: - :return: None - """ - self.level = level - self.logger.setLevel(self.level) - - def set_format(self, log_format: str, propagate: bool = False) -> None: - """ - Set a new logging format - :return: None - """ - handler = logging.StreamHandler() - - len_handlers = len(self.logger.handlers) - if len_handlers: - self.logger.removeHandler(handler) - - handler.setFormatter(logging.Formatter(log_format)) - self.logger.addHandler(handler) - - self.logger.propagate = propagate diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1d1c33ff9..146b69d83 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -296,12 +296,9 @@ def start(args: Namespace) -> None: :param args: Cli args from Arguments() :return: None """ - - # Initialize logger - logger.info('Starting freqtrade in Backtesting mode') - # Initialize configuration config = setup_configuration(args) + logger.info('Starting freqtrade in Backtesting mode') # Initialize backtesting object backtesting = Backtesting(config) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3e7b14610..a10bb0377 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -25,7 +25,6 @@ from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration -from freqtrade.logger import Logger from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting from user_data.hyperopt_conf import hyperopt_optimize_conf @@ -592,13 +591,11 @@ def start(args: Namespace) -> None: logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) - # Initialize logger - logger = Logger(__name__).get_logger() - logger.info('Starting freqtrade in Hyperopt mode') - # Initialize configuration # Monkey patch the configuration with hyperopt_conf.py configuration = Configuration(args) + logger.info('Starting freqtrade in Hyperopt mode') + optimize_config = hyperopt_optimize_conf() config = configuration._load_common_config(optimize_config) config = configuration._load_backtesting_config(config) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 6d376471a..26a2e388c 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -50,7 +50,6 @@ def test_start(mocker, default_conf, caplog) -> None: Test start() function """ start_mock = MagicMock() - mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) @@ -169,7 +168,6 @@ def test_fmin_best_results(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result) mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) - mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) Strategy({'strategy': 'default_strategy'}) hyperopt = Hyperopt(conf) @@ -214,7 +212,6 @@ def test_fmin_throw_value_error(mocker, default_conf, caplog) -> None: conf.update({'timerange': None}) conf.update({'spaces': 'all'}) mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) - mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) Strategy({'strategy': 'default_strategy'}) hyperopt = Hyperopt(conf) hyperopt.trials = create_trials(mocker) @@ -256,7 +253,6 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker, default_conf) -> No mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf) - mocker.patch('freqtrade.logger.Logger.set_format', MagicMock()) Strategy({'strategy': 'default_strategy'}) hyperopt = Hyperopt(conf) diff --git a/freqtrade/tests/test_logger.py b/freqtrade/tests/test_logger.py deleted file mode 100644 index 3e38084ad..000000000 --- a/freqtrade/tests/test_logger.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Unit test file for logger.py -""" - -import logging - -from freqtrade.logger import Logger - - -def test_logger_object() -> None: - """ - Test the Constants object has the mandatory Constants - :return: None - """ - logger = Logger('') - assert logger.name == '' - assert logger.level == 20 - assert logger.level is logging.INFO - assert hasattr(logger, 'get_logger') - - logger = Logger('Foo', level=logging.WARNING) - assert logger.name == 'Foo' - assert logger.name != '' - assert logger.level == 30 - assert logger.level is logging.WARNING - - -def test_get_logger() -> None: - """ - Test Logger.get_logger() and Logger._init_logger() - :return: None - """ - logger = Logger(name='get_logger', level=logging.WARNING) - get_logger = logger.get_logger() - assert logger.logger is get_logger - assert get_logger is not None - assert hasattr(get_logger, 'debug') - assert hasattr(get_logger, 'info') - assert hasattr(get_logger, 'warning') - assert hasattr(get_logger, 'critical') - assert hasattr(get_logger, 'exception') - - -def test_set_name() -> None: - """ - Test Logger.set_name() - :return: None - """ - logger = Logger(name='set_name') - assert logger.name == 'set_name' - - logger.set_name('set_name_new') - assert logger.name == 'set_name_new' - - -def test_set_level() -> None: - """ - Test Logger.set_name() - :return: None - """ - logger = Logger(name='Foo', level=logging.WARNING) - assert logger.level == logging.WARNING - assert logger.get_logger().level == logging.WARNING - - logger.set_level(logging.INFO) - assert logger.level == logging.INFO - assert logger.get_logger().level == logging.INFO - - -def test_sending_msg(caplog) -> None: - """ - Test send a logging message - :return: None - """ - logger = Logger(name='sending_msg', level=logging.WARNING).get_logger() - - logger.info('I am an INFO message') - assert('sending_msg', logging.INFO, 'I am an INFO message') not in caplog.record_tuples - - logger.warning('I am an WARNING message') - assert ('sending_msg', logging.WARNING, 'I am an WARNING message') in caplog.record_tuples - - -def test_set_format(caplog) -> None: - """ - Test Logger.set_format() - :return: None - """ - log = Logger(name='set_format') - logger = log.get_logger() - - logger.info('I am the first message') - assert ('set_format', logging.INFO, 'I am the first message') in caplog.record_tuples - - log.set_format(log_format='%(message)s', propagate=True) - logger.info('I am the second message') - assert ('set_format', logging.INFO, 'I am the second message') in caplog.record_tuples From 611bb52d1f2ceb035db3923c703d392e70dde932 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 25 Mar 2018 22:57:40 +0200 Subject: [PATCH 22/57] log hyperopt progress to stdout instead to the logger --- freqtrade/optimize/hyperopt.py | 4 ++-- freqtrade/tests/optimize/test_hyperopt.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a10bb0377..fb1d882be 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -217,13 +217,13 @@ class Hyperopt(Backtesting): """ if results['loss'] < self.current_best_loss: self.current_best_loss = results['loss'] - log_msg = '{:5d}/{}: {}. Loss {:.5f}'.format( + log_msg = '\n{:5d}/{}: {}. Loss {:.5f}'.format( results['current_tries'], results['total_tries'], results['result'], results['loss'] ) - logger.info(log_msg) + print(log_msg) else: print('.', end='') sys.stdout.flush() diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 26a2e388c..20e29c2d7 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -109,7 +109,7 @@ def test_loss_calculation_has_limited_profit() -> None: assert under > correct -def test_log_results_if_loss_improves(caplog) -> None: +def test_log_results_if_loss_improves(capsys) -> None: hyperopt = _HYPEROPT hyperopt.current_best_loss = 2 hyperopt.log_results( @@ -120,7 +120,8 @@ def test_log_results_if_loss_improves(caplog) -> None: 'result': 'foo' } ) - assert log_has(' 1/2: foo. Loss 1.00000', caplog.record_tuples) + out, err = capsys.readouterr() + assert ' 1/2: foo. Loss 1.00000'in out def test_no_log_if_loss_does_not_improve(caplog) -> None: From a182cab27f47b3b48bceb87634a94ac85128abfa Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Mon, 26 Mar 2018 20:28:51 +0200 Subject: [PATCH 23/57] fix backtest --export format reverts regression introduced in c623564 --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d8af47326..8a3d8d797 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -196,7 +196,7 @@ class Backtesting(object): records.append((pair, trade_entry[1], row.date.strftime('%s'), row2.date.strftime('%s'), - row.date, trade_entry[3])) + index, trade_entry[3])) # For now export inside backtest(), maybe change so that backtest() # returns a tuple like: (dataframe, records, logs, etc) if record and record.find('trades') >= 0: From 280886104c0354a8b2c11ef24227a01e960e3ab7 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 23:20:21 +0100 Subject: [PATCH 24/57] strategy: remove unneeded population methods in resolver --- freqtrade/analyze.py | 2 +- freqtrade/strategy/resolver.py | 55 +++++---------------- freqtrade/tests/strategy/test_strategy.py | 59 +++++++++-------------- 3 files changed, 37 insertions(+), 79 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index e6eb01e93..ccdbb139e 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -36,7 +36,7 @@ class Analyze(object): :param config: Bot configuration (use the one from Configuration()) """ self.config = config - self.strategy = StrategyResolver(self.config) + self.strategy = StrategyResolver(self.config).strategy @staticmethod def parse_ticker_dataframe(ticker: list) -> DataFrame: diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 826a50513..5709d15a7 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -10,8 +10,6 @@ import os from collections import OrderedDict from typing import Optional, Dict, Type -from pandas import DataFrame - from freqtrade.constants import Constants from freqtrade.strategy.interface import IStrategy @@ -38,42 +36,37 @@ class StrategyResolver(object): strategy = Constants.DEFAULT_STRATEGY # Try to load the strategy - self._load_strategy(strategy) + self.strategy = self._load_strategy(strategy) # Set attributes # Check if we need to override configuration if 'minimal_roi' in config: - self.custom_strategy.minimal_roi = config['minimal_roi'] + self.strategy.minimal_roi = config['minimal_roi'] logger.info("Override strategy \'minimal_roi\' with value in config file.") if 'stoploss' in config: - self.custom_strategy.stoploss = config['stoploss'] + self.strategy.stoploss = float(config['stoploss']) logger.info( "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] ) if 'ticker_interval' in config: - self.custom_strategy.ticker_interval = config['ticker_interval'] + self.strategy.ticker_interval = int(config['ticker_interval']) logger.info( "Override strategy \'ticker_interval\' with value in config file: %s.", config['ticker_interval'] ) # Minimal ROI designed for the strategy - self.minimal_roi = OrderedDict(sorted( - {int(key): value for (key, value) in self.custom_strategy.minimal_roi.items()}.items(), + self.strategy.minimal_roi = OrderedDict(sorted( + {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), key=lambda t: t[0])) # sort after converting to number - # Optimal stoploss designed for the strategy - self.stoploss = float(self.custom_strategy.stoploss) - - self.ticker_interval = int(self.custom_strategy.ticker_interval) - - def _load_strategy(self, strategy_name: str) -> None: + def _load_strategy(self, strategy_name: str) -> Optional[IStrategy]: """ Search and loads the specified strategy. :param strategy_name: name of the module to import - :return: None + :return: Strategy instance or None """ try: current_path = os.path.dirname(os.path.realpath(__file__)) @@ -82,10 +75,10 @@ class StrategyResolver(object): current_path, ] for path in abs_paths: - self.custom_strategy = self._search_strategy(path, strategy_name) - if self.custom_strategy: + strategy = self._search_strategy(path, strategy_name) + if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) - return None + return strategy raise ImportError('not found') # Fallback to the default strategy @@ -99,6 +92,7 @@ class StrategyResolver(object): "The error is:\n%s.", error ) + return None @staticmethod def _get_valid_strategies(module_path: str, strategy_name: str) -> Optional[Type[IStrategy]]: @@ -139,28 +133,3 @@ class StrategyResolver(object): if strategy: return strategy() return None - - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Populate indicators that will be used in the Buy and Sell strategy - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :return: a Dataframe with all mandatory indicators for the strategies - """ - return self.custom_strategy.populate_indicators(dataframe) - - def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - :return: - """ - return self.custom_strategy.populate_buy_trend(dataframe) - - def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame - :return: DataFrame with buy column - """ - return self.custom_strategy.populate_sell_trend(dataframe) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 075efd6ab..6e69e2b09 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -19,15 +19,15 @@ def test_search_strategy(): def test_load_strategy(result): - strategy = StrategyResolver() + resolver = StrategyResolver() assert not hasattr(StrategyResolver, 'custom_strategy') - strategy._load_strategy('TestStrategy') + resolver._load_strategy('TestStrategy') assert not hasattr(StrategyResolver, 'custom_strategy') - assert hasattr(strategy.custom_strategy, 'populate_indicators') - assert 'adx' in strategy.populate_indicators(result) + assert hasattr(resolver.strategy, 'populate_indicators') + assert 'adx' in resolver.strategy.populate_indicators(result) def test_load_not_found_strategy(caplog): @@ -42,23 +42,23 @@ def test_load_not_found_strategy(caplog): def test_strategy(result): - strategy = StrategyResolver({'strategy': 'DefaultStrategy'}) + resolver = StrategyResolver({'strategy': 'DefaultStrategy'}) - assert hasattr(strategy.custom_strategy, 'minimal_roi') - assert strategy.minimal_roi[0] == 0.04 + assert hasattr(resolver.strategy, 'minimal_roi') + assert resolver.strategy.minimal_roi[0] == 0.04 - assert hasattr(strategy.custom_strategy, 'stoploss') - assert strategy.stoploss == -0.10 + assert hasattr(resolver.strategy, 'stoploss') + assert resolver.strategy.stoploss == -0.10 - assert hasattr(strategy.custom_strategy, 'populate_indicators') - assert 'adx' in strategy.populate_indicators(result) + assert hasattr(resolver.strategy, 'populate_indicators') + assert 'adx' in resolver.strategy.populate_indicators(result) - assert hasattr(strategy.custom_strategy, 'populate_buy_trend') - dataframe = strategy.populate_buy_trend(strategy.populate_indicators(result)) + assert hasattr(resolver.strategy, 'populate_buy_trend') + dataframe = resolver.strategy.populate_buy_trend(resolver.strategy.populate_indicators(result)) assert 'buy' in dataframe.columns - assert hasattr(strategy.custom_strategy, 'populate_sell_trend') - dataframe = strategy.populate_sell_trend(strategy.populate_indicators(result)) + assert hasattr(resolver.strategy, 'populate_sell_trend') + dataframe = resolver.strategy.populate_sell_trend(resolver.strategy.populate_indicators(result)) assert 'sell' in dataframe.columns @@ -70,10 +70,10 @@ def test_strategy_override_minimal_roi(caplog): "0": 0.5 } } - strategy = StrategyResolver(config) + resolver = StrategyResolver(config) - assert hasattr(strategy.custom_strategy, 'minimal_roi') - assert strategy.minimal_roi[0] == 0.5 + assert hasattr(resolver.strategy, 'minimal_roi') + assert resolver.strategy.minimal_roi[0] == 0.5 assert ('freqtrade.strategy.resolver', logging.INFO, 'Override strategy \'minimal_roi\' with value in config file.' @@ -86,10 +86,10 @@ def test_strategy_override_stoploss(caplog): 'strategy': 'DefaultStrategy', 'stoploss': -0.5 } - strategy = StrategyResolver(config) + resolver = StrategyResolver(config) - assert hasattr(strategy.custom_strategy, 'stoploss') - assert strategy.stoploss == -0.5 + assert hasattr(resolver.strategy, 'stoploss') + assert resolver.strategy.stoploss == -0.5 assert ('freqtrade.strategy.resolver', logging.INFO, 'Override strategy \'stoploss\' with value in config file: -0.5.' @@ -103,10 +103,10 @@ def test_strategy_override_ticker_interval(caplog): 'strategy': 'DefaultStrategy', 'ticker_interval': 60 } - strategy = StrategyResolver(config) + resolver = StrategyResolver(config) - assert hasattr(strategy.custom_strategy, 'ticker_interval') - assert strategy.ticker_interval == 60 + assert hasattr(resolver.strategy, 'ticker_interval') + assert resolver.strategy.ticker_interval == 60 assert ('freqtrade.strategy.resolver', logging.INFO, 'Override strategy \'ticker_interval\' with value in config file: 60.' @@ -120,14 +120,3 @@ def test_strategy_fallback_default_strategy(): assert not hasattr(StrategyResolver, 'custom_strategy') strategy._load_strategy('../../super_duper') assert not hasattr(StrategyResolver, 'custom_strategy') - - -def test_strategy_singleton(): - strategy1 = StrategyResolver({'strategy': 'DefaultStrategy'}) - - assert hasattr(strategy1.custom_strategy, 'minimal_roi') - assert strategy1.minimal_roi[0] == 0.04 - - strategy2 = StrategyResolver() - assert hasattr(strategy2.custom_strategy, 'minimal_roi') - assert strategy2.minimal_roi[0] == 0.04 From 99e890bc99c93685d0406dcf12617d41c494c8b0 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 23:22:31 +0100 Subject: [PATCH 25/57] simplify resolver constructor --- freqtrade/strategy/resolver.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 5709d15a7..d5e9e7b5c 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -30,13 +30,7 @@ class StrategyResolver(object): config = config or {} # Verify the strategy is in the configuration, otherwise fallback to the default strategy - if 'strategy' in config: - strategy = config['strategy'] - else: - strategy = Constants.DEFAULT_STRATEGY - - # Try to load the strategy - self.strategy = self._load_strategy(strategy) + self.strategy = self._load_strategy(config.get('strategy') or Constants.DEFAULT_STRATEGY) # Set attributes # Check if we need to override configuration From 5fb6fa38aae95d8540124016a8b9a9aa544b076e Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 24 Mar 2018 23:32:17 +0100 Subject: [PATCH 26/57] apply __slots__ to resolver and reintroduce type conversations --- freqtrade/strategy/resolver.py | 21 +++++++++++---------- freqtrade/tests/strategy/test_strategy.py | 1 - 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index d5e9e7b5c..18a03ea58 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -21,6 +21,9 @@ class StrategyResolver(object): """ This class contains all the logic to load custom strategy class """ + + __slots__ = ['strategy'] + def __init__(self, config: Optional[Dict] = None) -> None: """ Load the custom class from config parameter @@ -39,22 +42,24 @@ class StrategyResolver(object): logger.info("Override strategy \'minimal_roi\' with value in config file.") if 'stoploss' in config: - self.strategy.stoploss = float(config['stoploss']) + self.strategy.stoploss = config['stoploss'] logger.info( "Override strategy \'stoploss\' with value in config file: %s.", config['stoploss'] ) if 'ticker_interval' in config: - self.strategy.ticker_interval = int(config['ticker_interval']) + self.strategy.ticker_interval = config['ticker_interval'] logger.info( "Override strategy \'ticker_interval\' with value in config file: %s.", config['ticker_interval'] ) - # Minimal ROI designed for the strategy + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), - key=lambda t: t[0])) # sort after converting to number + key=lambda t: t[0])) + self.strategy.stoploss = float(self.strategy.stoploss) + self.strategy.ticker_interval = int(self.strategy.ticker_interval) def _load_strategy(self, strategy_name: str) -> Optional[IStrategy]: """ @@ -76,16 +81,12 @@ class StrategyResolver(object): raise ImportError('not found') # Fallback to the default strategy - except (ImportError, TypeError) as error: - logger.error( + except (ImportError, TypeError): + logger.exception( "Impossible to load Strategy '%s'. This class does not exist" " or contains Python code errors", strategy_name ) - logger.error( - "The error is:\n%s.", - error - ) return None @staticmethod diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 6e69e2b09..87818b05f 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -115,7 +115,6 @@ def test_strategy_override_ticker_interval(caplog): def test_strategy_fallback_default_strategy(): strategy = StrategyResolver() - strategy.logger = logging.getLogger(__name__) assert not hasattr(StrategyResolver, 'custom_strategy') strategy._load_strategy('../../super_duper') From a356edb117c07e3a4c4311cd45d20441bcefeef6 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 25 Mar 2018 16:28:04 +0200 Subject: [PATCH 27/57] implement '--strategy-path' argument --- freqtrade/arguments.py | 7 ++++ freqtrade/configuration.py | 3 ++ freqtrade/main.py | 1 - freqtrade/strategy/resolver.py | 46 ++++++++++++----------- freqtrade/tests/strategy/test_strategy.py | 37 ++++++++++-------- freqtrade/tests/test_arguments.py | 20 ++++++++++ 6 files changed, 76 insertions(+), 38 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 052b4534e..35f8c6609 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -86,6 +86,13 @@ class Arguments(object): type=str, metavar='NAME', ) + self.parser.add_argument( + '--strategy-path', + help='specify additional strategy lookup path', + dest='strategy_path', + type=str, + metavar='PATH', + ) self.parser.add_argument( '--dynamic-whitelist', help='dynamically generate and update whitelist \ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 1314f624e..8e395e21d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -36,6 +36,9 @@ class Configuration(object): # Add the strategy file to use config.update({'strategy': self.args.strategy}) + if self.args.strategy_path: + config.update({'strategy_path': self.args.strategy_path}) + # Load Common configuration config = self._load_common_config(config) diff --git a/freqtrade/main.py b/freqtrade/main.py index d62c92b9f..9639922f9 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -3,7 +3,6 @@ Main Freqtrade bot script. Read the documentation to know what cli arguments you need. """ - import logging import sys from typing import List diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 18a03ea58..c5a7cd577 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -33,7 +33,8 @@ class StrategyResolver(object): config = config or {} # Verify the strategy is in the configuration, otherwise fallback to the default strategy - self.strategy = self._load_strategy(config.get('strategy') or Constants.DEFAULT_STRATEGY) + strategy_name = config.get('strategy') or Constants.DEFAULT_STRATEGY + self.strategy = self._load_strategy(strategy_name, extra_dir=config.get('strategy_path')) # Set attributes # Check if we need to override configuration @@ -61,33 +62,34 @@ class StrategyResolver(object): self.strategy.stoploss = float(self.strategy.stoploss) self.strategy.ticker_interval = int(self.strategy.ticker_interval) - def _load_strategy(self, strategy_name: str) -> Optional[IStrategy]: + def _load_strategy( + self, strategy_name: str, extra_dir: Optional[str] = None) -> Optional[IStrategy]: """ Search and loads the specified strategy. :param strategy_name: name of the module to import + :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ - try: - current_path = os.path.dirname(os.path.realpath(__file__)) - abs_paths = [ - os.path.join(current_path, '..', '..', 'user_data', 'strategies'), - current_path, - ] - for path in abs_paths: - strategy = self._search_strategy(path, strategy_name) - if strategy: - logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) - return strategy + current_path = os.path.dirname(os.path.realpath(__file__)) + abs_paths = [ + os.path.join(current_path, '..', '..', 'user_data', 'strategies'), + current_path, + ] - raise ImportError('not found') - # Fallback to the default strategy - except (ImportError, TypeError): - logger.exception( - "Impossible to load Strategy '%s'. This class does not exist" - " or contains Python code errors", - strategy_name - ) - return None + if extra_dir: + # Add extra strategy directory on top of search paths + abs_paths.insert(0, extra_dir) + + for path in abs_paths: + strategy = self._search_strategy(path, strategy_name) + if strategy: + logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) + return strategy + + raise ImportError( + "Impossible to load Strategy '{}'. This class does not exist" + " or contains Python code errors".format(strategy_name) + ) @staticmethod def _get_valid_strategies(module_path: str, strategy_name: str) -> Optional[Type[IStrategy]]: diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 87818b05f..0f45d8d04 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,9 +1,10 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging - import os +import pytest + from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.resolver import StrategyResolver @@ -30,15 +31,29 @@ def test_load_strategy(result): assert 'adx' in resolver.strategy.populate_indicators(result) -def test_load_not_found_strategy(caplog): - strategy = StrategyResolver() +def test_load_strategy_custom_directory(result): + resolver = StrategyResolver() assert not hasattr(StrategyResolver, 'custom_strategy') - strategy._load_strategy('NotFoundStrategy') - error_msg = "Impossible to load Strategy '{}'. This class does not " \ - "exist or contains Python code errors".format('NotFoundStrategy') - assert ('freqtrade.strategy.resolver', logging.ERROR, error_msg) in caplog.record_tuples + extra_dir = os.path.join('some', 'path') + with pytest.raises( + FileNotFoundError, + match=r".*No such file or directory: '{}'".format(extra_dir)): + resolver._load_strategy('TestStrategy', extra_dir) + + assert not hasattr(StrategyResolver, 'custom_strategy') + + assert hasattr(resolver.strategy, 'populate_indicators') + assert 'adx' in resolver.strategy.populate_indicators(result) + + +def test_load_not_found_strategy(): + strategy = StrategyResolver() + with pytest.raises(ImportError, + match=r'Impossible to load Strategy \'NotFoundStrategy\'.' + r' This class does not exist or contains Python code errors'): + strategy._load_strategy('NotFoundStrategy') def test_strategy(result): @@ -111,11 +126,3 @@ def test_strategy_override_ticker_interval(caplog): logging.INFO, 'Override strategy \'ticker_interval\' with value in config file: 60.' ) in caplog.record_tuples - - -def test_strategy_fallback_default_strategy(): - strategy = StrategyResolver() - - assert not hasattr(StrategyResolver, 'custom_strategy') - strategy._load_strategy('../../super_duper') - assert not hasattr(StrategyResolver, 'custom_strategy') diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 3e0639304..3377746b7 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -71,6 +71,26 @@ def test_parse_args_invalid() -> None: Arguments(['-c'], '').get_parsed_arg() +def test_parse_args_strategy() -> None: + args = Arguments(['--strategy', 'SomeStrategy'], '').get_parsed_arg() + assert args.strategy == 'SomeStrategy' + + +def test_parse_args_strategy_invalid() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['--strategy'], '').get_parsed_arg() + + +def test_parse_args_strategy_path() -> None: + args = Arguments(['--strategy-path', '/some/path'], '').get_parsed_arg() + assert args.strategy_path == '/some/path' + + +def test_parse_args_strategy_path_invalid() -> None: + with pytest.raises(SystemExit, match=r'2'): + Arguments(['--strategy-path'], '').get_parsed_arg() + + def test_parse_args_dynamic_whitelist() -> None: args = Arguments(['--dynamic-whitelist'], '').get_parsed_arg() assert args.dynamic_whitelist == 20 From 157f7da8cebf65da9a36763902910d717d62e977 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 25 Mar 2018 16:30:49 +0200 Subject: [PATCH 28/57] remove obsolete assertions --- freqtrade/tests/strategy/test_strategy.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 0f45d8d04..244910790 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -21,29 +21,19 @@ def test_search_strategy(): def test_load_strategy(result): resolver = StrategyResolver() - - assert not hasattr(StrategyResolver, 'custom_strategy') resolver._load_strategy('TestStrategy') - - assert not hasattr(StrategyResolver, 'custom_strategy') - assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) def test_load_strategy_custom_directory(result): resolver = StrategyResolver() - - assert not hasattr(StrategyResolver, 'custom_strategy') - extra_dir = os.path.join('some', 'path') with pytest.raises( FileNotFoundError, match=r".*No such file or directory: '{}'".format(extra_dir)): resolver._load_strategy('TestStrategy', extra_dir) - assert not hasattr(StrategyResolver, 'custom_strategy') - assert hasattr(resolver.strategy, 'populate_indicators') assert 'adx' in resolver.strategy.populate_indicators(result) From f78044da6dff834aaa14e4386ca06c841f2d91d0 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sun, 25 Mar 2018 20:24:56 +0200 Subject: [PATCH 29/57] fix method docs --- freqtrade/strategy/interface.py | 3 +-- freqtrade/strategy/resolver.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4eb73fb2e..dcf665a02 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -33,7 +33,6 @@ class IStrategy(ABC): Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame :return: DataFrame with buy column - :return: """ @abstractmethod @@ -41,5 +40,5 @@ class IStrategy(ABC): """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame - :return: DataFrame with buy column + :return: DataFrame with sell column """ diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index c5a7cd577..de38cf67a 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -27,8 +27,7 @@ class StrategyResolver(object): def __init__(self, config: Optional[Dict] = None) -> None: """ Load the custom class from config parameter - :param config: - :return: + :param config: configuration dictionary or None """ config = config or {} From df57c3207684efc8211600a0178affcf2a50d842 Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 27 Mar 2018 18:15:49 +0200 Subject: [PATCH 30/57] only override strategy if other than DEFAULT --- freqtrade/configuration.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 8e395e21d..71cbe78e9 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -33,8 +33,9 @@ class Configuration(object): logger.info('Using config: %s ...', self.args.config) config = self._load_config_file(self.args.config) - # Add the strategy file to use - config.update({'strategy': self.args.strategy}) + # Override strategy if specified + if self.args.strategy != Constants.DEFAULT_STRATEGY: + config.update({'strategy': self.args.strategy}) if self.args.strategy_path: config.update({'strategy_path': self.args.strategy_path}) From e7399b50465b734a65f1466b28a84638747e0f46 Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 27 Mar 2018 18:16:21 +0200 Subject: [PATCH 31/57] add strategy and strategy_path to config_full.json.example --- config_full.json.example | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config_full.json.example b/config_full.json.example index c74b59660..1d9a48762 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -48,5 +48,7 @@ "initial_state": "running", "internals": { "process_throttle_secs": 5 - } + }, + "strategy": "DefaultStrategy", + "strategy_path": "/some/folder/" } From 6a125912489c492c761cc2ec2dc22e5fc9e1474e Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 27 Mar 2018 18:20:15 +0200 Subject: [PATCH 32/57] change strategy override condition --- freqtrade/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 71cbe78e9..4922bf402 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -33,8 +33,8 @@ class Configuration(object): logger.info('Using config: %s ...', self.args.config) config = self._load_config_file(self.args.config) - # Override strategy if specified - if self.args.strategy != Constants.DEFAULT_STRATEGY: + # Set strategy if not specified in config and or if it's non default + if self.args.strategy != Constants.DEFAULT_STRATEGY or not config.get('strategy'): config.update({'strategy': self.args.strategy}) if self.args.strategy_path: From 872bbadded690de1f99e47d4663e7794154d2faa Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 27 Mar 2018 18:29:51 +0200 Subject: [PATCH 33/57] add test_load_custom_strategy() --- freqtrade/tests/test_configuration.py | 38 ++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 1085b0060..22a6cb005 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -98,8 +98,8 @@ def test_load_config(default_conf, mocker) -> None: configuration = Configuration(args) validated_conf = configuration.load_config() - assert 'strategy' in validated_conf - assert validated_conf['strategy'] == 'DefaultStrategy' + assert validated_conf.get('strategy') == 'DefaultStrategy' + assert validated_conf.get('strategy_path') is None assert 'dynamic_whitelist' not in validated_conf assert 'dry_run_db' not in validated_conf @@ -115,19 +115,39 @@ def test_load_config_with_params(default_conf, mocker) -> None: args = [ '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', - '--dry-run-db' + '--strategy-path', '/some/path', + '--dry-run-db', ] args = Arguments(args, '').get_parsed_arg() configuration = Configuration(args) validated_conf = configuration.load_config() - assert 'dynamic_whitelist' in validated_conf - assert validated_conf['dynamic_whitelist'] == 10 - assert 'strategy' in validated_conf - assert validated_conf['strategy'] == 'TestStrategy' - assert 'dry_run_db' in validated_conf - assert validated_conf['dry_run_db'] is True + assert validated_conf.get('dynamic_whitelist') == 10 + assert validated_conf.get('strategy') == 'TestStrategy' + assert validated_conf.get('strategy_path') == '/some/path' + assert validated_conf.get('dry_run_db') is True + + +def test_load_custom_strategy(default_conf, mocker) -> None: + """ + Test Configuration.load_config() without any cli params + """ + custom_conf = deepcopy(default_conf) + custom_conf.update({ + 'strategy': 'CustomStrategy', + 'strategy_path': '/tmp/strategies', + }) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(custom_conf) + )) + + args = Arguments([], '').get_parsed_arg() + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert validated_conf.get('strategy') == 'CustomStrategy' + assert validated_conf.get('strategy_path') == '/tmp/strategies' def test_show_info(default_conf, mocker, caplog) -> None: From ba5cbcbb3fb37ecdb922f21689e2dcf718cbef0e Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 27 Mar 2018 18:38:43 +0200 Subject: [PATCH 34/57] configuration.md: add strategy and strategy_path --- docs/configuration.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 5e3b15925..311205dad 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -35,6 +35,8 @@ The table below will list all configuration parameters. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. | `initial_state` | running | No | Defines the initial application state. More information below. +| `strategy` | DefaultStrategy | No | Defines Strategy class to use. +| `strategy_path` | null | No | Adds an additional strategy lookup path (must be a folder). | `internals.process_throttle_secs` | 5 | Yes | Set the process throttle. Value in second. The definition of each config parameters is in From 06276e1d24d6c7394bbff7395499da405431bc13 Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 27 Mar 2018 18:39:49 +0200 Subject: [PATCH 35/57] bot-optimization.md: add strategy-path --- docs/bot-optimization.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 00938adbe..b9ff3fe40 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -42,6 +42,13 @@ You can test it with the parameter: `--strategy TestStrategy` python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` +### Specify custom strategy location +If you want to use a strategy from a different folder you can pass `--strategy-path` + +```bash +python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/folder +``` + **For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/gcarq/freqtrade/blob/develop/user_data/strategies/test_strategy.py) file as reference.** From 004e0bb9a3c6c05f6cce50c8c96de9784ff69075 Mon Sep 17 00:00:00 2001 From: gcarq Date: Tue, 27 Mar 2018 18:46:42 +0200 Subject: [PATCH 36/57] bot-usage.md: add strategy-path --- docs/bot-usage.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index ea9b1e4d8..0ed073933 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -28,6 +28,7 @@ optional arguments: specify configuration file (default: config.json) -s NAME, --strategy NAME specify strategy class name (default: DefaultStrategy) + --strategy-path PATH specify additional strategy lookup path --dry-run-db Force dry run to use a local DB "tradesv3.dry_run.sqlite" instead of memory DB. Work only if dry_run is enabled. @@ -67,9 +68,16 @@ message the reason (File not found, or errors in your code). Learn more about strategy file in [optimize your bot](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md). +### How to use --strategy-path? +This parameter allows you to add an additional strategy lookup path, which gets +checked before the default locations (The passed path must be a folder!): +```bash +python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/folder +``` + #### How to install a strategy? This is very simple. Copy paste your strategy file into the folder -`user_data/strategies`. And voila, the bot is ready to use it. +`user_data/strategies` or use `--strategy-path`. And voila, the bot is ready to use it. ### How to use --dynamic-whitelist? Per default `--dynamic-whitelist` will retrieve the 20 currencies based From 02aacdd0c897cb0a6824b1d999bde2787622031f Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 17:12:49 +0200 Subject: [PATCH 37/57] parse_ticker_dataframe: group dataframe by date --- freqtrade/analyze.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index ccdbb139e..4d5412d23 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -50,6 +50,14 @@ class Analyze(object): .rename(columns=columns) if 'BV' in frame: frame.drop('BV', 1, inplace=True) + # group by date to eliminate duplicate ticks + frame.groupby('date').agg({ + 'volume': 'max', + 'open': 'first', + 'close': 'last', + 'high': 'max', + 'low': 'min', + }) frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) frame.sort_values('date', inplace=True) return frame From 4f2d3dbb418e7530446333f6c9a72410804ce040 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:14:43 +0200 Subject: [PATCH 38/57] parse_ticker_dataframe: use as_index=False to keep date column --- freqtrade/analyze.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 4d5412d23..47edd289e 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -46,20 +46,20 @@ class Analyze(object): :return: DataFrame """ columns = {'C': 'close', 'V': 'volume', 'O': 'open', 'H': 'high', 'L': 'low', 'T': 'date'} - frame = DataFrame(ticker) \ - .rename(columns=columns) + frame = DataFrame(ticker).rename(columns=columns) if 'BV' in frame: - frame.drop('BV', 1, inplace=True) - # group by date to eliminate duplicate ticks - frame.groupby('date').agg({ - 'volume': 'max', - 'open': 'first', + frame.drop('BV', axis=1, inplace=True) + + frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) + + # group by index and aggregate results to eliminate duplicate ticks + frame = frame.groupby(by='date', as_index=False, sort=True).agg({ 'close': 'last', 'high': 'max', 'low': 'min', + 'open': 'first', + 'volume': 'max', }) - frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) - frame.sort_values('date', inplace=True) return frame def populate_indicators(self, dataframe: DataFrame) -> DataFrame: From 702402e1fefbfcdd849a95c854e766978c8cb0e1 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:15:32 +0200 Subject: [PATCH 39/57] simplify download_backtesting_testdata --- freqtrade/optimize/__init__.py | 35 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index fff4b72d9..bfdfceab1 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -111,45 +111,38 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool: # FIX: 20180110, suggest rename interval to tick_interval -def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool: +def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> None: """ Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pairs: list of pairs to download - :return: bool """ path = make_testdata_path(datadir) logger.info( - 'Download the pair: "%s", Interval: %s min', - pair, - interval + 'Download the pair: "%s", Interval: %s min', pair, interval ) - filepair = pair.replace("-", "_") filename = os.path.join(path, '{pair}-{interval}.json'.format( - pair=filepair, + pair=pair.replace("-", "_"), interval=interval, )) if os.path.isfile(filename): with open(filename, "rt") as file: data = json.load(file) - logger.debug("Current Start: %s", data[1]['T']) - logger.debug("Current End: %s", data[-1:][0]['T']) else: data = [] - logger.debug("Current Start: None") - logger.debug("Current End: None") - new_data = get_ticker_history(pair=pair, tick_interval=int(interval)) - for row in new_data: - if row not in data: - data.append(row) - logger.debug("New Start: %s", data[1]['T']) - logger.debug("New End: %s", data[-1:][0]['T']) - data = sorted(data, key=lambda data: data['T']) + logger.debug('Current Start: %s', data[1]['T'] if data else None) + logger.debug('Current End: %s', data[-1:][0]['T'] if data else None) + # Extend data with new ticker history + data.extend([ + row for row in get_ticker_history(pair=pair, tick_interval=int(interval)) + if row not in data + ]) + + data = sorted(data, key=lambda _data: _data['T']) + logger.debug('New Start: %s', data[1]['T']) + logger.debug('New End: %s', data[-1:][0]['T']) misc.file_dump_json(filename, data) - - return True From fee8d0a2e1cfb40a95e66d730ac2e06def04c37c Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:16:25 +0200 Subject: [PATCH 40/57] refactor get_timeframe --- freqtrade/optimize/backtesting.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9b71dbfd9..cf140637c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -4,11 +4,12 @@ This module contains the backtesting logic """ import logging +import operator from argparse import Namespace from typing import Dict, Tuple, Any, List, Optional import arrow -from pandas import DataFrame, Series +from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize @@ -60,11 +61,12 @@ class Backtesting(object): :param data: dictionary with preprocessed backtesting data :return: tuple containing min_date, max_date """ - all_dates = Series([]) - for pair_data in data.values(): - all_dates = all_dates.append(pair_data['date']) - all_dates.sort_values(inplace=True) - return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) + timeframe = [ + (arrow.get(min(frame.date)), arrow.get(max(frame.date))) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: """ From 3775fdf9c798b450c9517eb82ed8b1248f0ba4a5 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:16:46 +0200 Subject: [PATCH 41/57] change column order assertions --- freqtrade/tests/test_analyze.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index a4f1ba549..878d97950 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -50,7 +50,7 @@ def test_dataframe_correct_length(result): def test_dataframe_correct_columns(result): assert result.columns.tolist() == \ - ['close', 'high', 'low', 'open', 'date', 'volume'] + ['date', 'close', 'high', 'low', 'open', 'volume'] def test_populates_buy_trend(result): @@ -170,7 +170,7 @@ def test_get_signal_handles_exceptions(mocker): def test_parse_ticker_dataframe(ticker_history, ticker_history_without_bv): - columns = ['close', 'high', 'low', 'open', 'date', 'volume'] + columns = ['date', 'close', 'high', 'low', 'open', 'volume'] # Test file with BV data dataframe = Analyze.parse_ticker_dataframe(ticker_history) From 24aa6a1679d63c0409b0de922afb3e8afa13fbb2 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:17:11 +0200 Subject: [PATCH 42/57] adapt test_download_backtesting_testdata --- freqtrade/tests/optimize/test_optimize.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index e26d30534..a15bff5a1 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -182,10 +182,11 @@ def test_download_backtesting_testdata(ticker_history, mocker) -> None: def test_download_backtesting_testdata2(mocker) -> None: tick = [{'T': 'bar'}, {'T': 'foo'}] - mocker.patch('freqtrade.misc.file_dump_json', return_value=None) + json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick) - assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=1) - assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3) + download_backtesting_testdata(None, pair="BTC-UNITEST", interval=1) + download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3) + assert json_dump_mock.call_count == 2 def test_load_tickerdata_file() -> None: From 5bd79546abfcc39f6011fb13a19d50828131e506 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Fri, 30 Mar 2018 22:38:09 +0200 Subject: [PATCH 43/57] Disable dynamic whitelist Revert regression introduced in refactoring for objectify --- freqtrade/freqtradebot.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3972a4382..f5de95826 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -114,10 +114,7 @@ class FreqtradeBot(object): Constants.PROCESS_THROTTLE_SECS ) - nb_assets = self.config.get( - 'dynamic_whitelist', - Constants.DYNAMIC_WHITELIST - ) + nb_assets = self.config.get('dynamic_whitelist', None) self._throttle(func=self._process, min_secs=min_secs, From 84bbe7728d7a521efadbfd6ecf1e6758be513499 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 30 Mar 2018 22:52:47 +0200 Subject: [PATCH 44/57] Update sqlalchemy from 1.2.5 to 1.2.6 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d228c8195..251723b7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ python-bittrex==0.3.0 -SQLAlchemy==1.2.5 +SQLAlchemy==1.2.6 python-telegram-bot==10.0.1 arrow==0.12.1 cachetools==2.0.1 From 9019f6492f8efb9b8eaf4bf60dab49313d16a6e2 Mon Sep 17 00:00:00 2001 From: Michael Egger Date: Mon, 2 Apr 2018 16:42:53 +0200 Subject: [PATCH 45/57] define constants on module level (#596) --- freqtrade/arguments.py | 7 +- freqtrade/configuration.py | 8 +- freqtrade/constants.py | 218 +++++++++++++++--------------- freqtrade/freqtradebot.py | 6 +- freqtrade/strategy/resolver.py | 4 +- freqtrade/tests/conftest.py | 4 +- freqtrade/tests/test_constants.py | 19 ++- 7 files changed, 129 insertions(+), 137 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 35f8c6609..791ef85f2 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -8,8 +8,7 @@ import os import re from typing import List, Tuple, Optional -from freqtrade import __version__ -from freqtrade.constants import Constants +from freqtrade import __version__, constants class Arguments(object): @@ -98,7 +97,7 @@ class Arguments(object): help='dynamically generate and update whitelist \ based on 24h BaseVolume (Default 20 currencies)', # noqa dest='dynamic_whitelist', - const=Constants.DYNAMIC_WHITELIST, + const=constants.DYNAMIC_WHITELIST, type=int, metavar='INT', nargs='?', @@ -170,7 +169,7 @@ class Arguments(object): '-e', '--epochs', help='specify number of epochs (default: %(default)d)', dest='epochs', - default=Constants.HYPEROPT_EPOCH, + default=constants.HYPEROPT_EPOCH, type=int, metavar='INT', ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 4922bf402..fd9f31ad2 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -10,7 +10,7 @@ from typing import Dict, Any from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match -from freqtrade.constants import Constants +from freqtrade import constants logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ class Configuration(object): config = self._load_config_file(self.args.config) # Set strategy if not specified in config and or if it's non default - if self.args.strategy != Constants.DEFAULT_STRATEGY or not config.get('strategy'): + if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'): config.update({'strategy': self.args.strategy}) if self.args.strategy_path: @@ -186,7 +186,7 @@ class Configuration(object): :return: Returns the config if valid, otherwise throw an exception """ try: - validate(conf, Constants.CONF_SCHEMA) + validate(conf, constants.CONF_SCHEMA) return conf except ValidationError as exception: logger.fatal( @@ -194,7 +194,7 @@ class Configuration(object): exception ) raise ValidationError( - best_match(Draft4Validator(Constants.CONF_SCHEMA).iter_errors(conf)).message + best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message ) def get_config(self) -> Dict[str, Any]: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 61adf307a..382f19ef1 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -1,122 +1,116 @@ # pragma pylint: disable=too-few-public-methods """ -List bot constants +bot constants """ +DYNAMIC_WHITELIST = 20 # pairs +PROCESS_THROTTLE_SECS = 5 # sec +TICKER_INTERVAL = 5 # min +HYPEROPT_EPOCH = 100 # epochs +RETRY_TIMEOUT = 30 # sec +DEFAULT_STRATEGY = 'DefaultStrategy' - -class Constants(object): - """ - Static class that contain all bot constants - """ - DYNAMIC_WHITELIST = 20 # pairs - PROCESS_THROTTLE_SECS = 5 # sec - TICKER_INTERVAL = 5 # min - HYPEROPT_EPOCH = 100 # epochs - RETRY_TIMEOUT = 30 # sec - DEFAULT_STRATEGY = 'DefaultStrategy' - - # Required json-schema for user specified config - CONF_SCHEMA = { - 'type': 'object', - 'properties': { - 'max_open_trades': {'type': 'integer', 'minimum': 1}, - 'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, - 'stake_amount': {'type': 'number', 'minimum': 0.0005}, - 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', - 'CLP', 'CNY', 'CZK', 'DKK', - 'EUR', 'GBP', 'HKD', 'HUF', - 'IDR', 'ILS', 'INR', 'JPY', - 'KRW', 'MXN', 'MYR', 'NOK', - 'NZD', 'PHP', 'PKR', 'PLN', - 'RUB', 'SEK', 'SGD', 'THB', - 'TRY', 'TWD', 'ZAR', 'USD']}, - 'dry_run': {'type': 'boolean'}, - 'minimal_roi': { - 'type': 'object', - 'patternProperties': { - '^[0-9.]+$': {'type': 'number'} +# Required json-schema for user specified config +CONF_SCHEMA = { + 'type': 'object', + 'properties': { + 'max_open_trades': {'type': 'integer', 'minimum': 1}, + 'ticker_interval': {'type': 'integer', 'enum': [1, 5, 30, 60, 1440]}, + 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT']}, + 'stake_amount': {'type': 'number', 'minimum': 0.0005}, + 'fiat_display_currency': {'type': 'string', 'enum': ['AUD', 'BRL', 'CAD', 'CHF', + 'CLP', 'CNY', 'CZK', 'DKK', + 'EUR', 'GBP', 'HKD', 'HUF', + 'IDR', 'ILS', 'INR', 'JPY', + 'KRW', 'MXN', 'MYR', 'NOK', + 'NZD', 'PHP', 'PKR', 'PLN', + 'RUB', 'SEK', 'SGD', 'THB', + 'TRY', 'TWD', 'ZAR', 'USD']}, + 'dry_run': {'type': 'boolean'}, + 'minimal_roi': { + 'type': 'object', + 'patternProperties': { + '^[0-9.]+$': {'type': 'number'} + }, + 'minProperties': 1 + }, + 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, + 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, + 'bid_strategy': { + 'type': 'object', + 'properties': { + 'ask_last_balance': { + 'type': 'number', + 'minimum': 0, + 'maximum': 1, + 'exclusiveMaximum': False }, - 'minProperties': 1 }, - 'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True}, - 'unfilledtimeout': {'type': 'integer', 'minimum': 0}, - 'bid_strategy': { - 'type': 'object', - 'properties': { - 'ask_last_balance': { - 'type': 'number', - 'minimum': 0, - 'maximum': 1, - 'exclusiveMaximum': False - }, - }, - 'required': ['ask_last_balance'] - }, - 'exchange': {'$ref': '#/definitions/exchange'}, - 'experimental': { - 'type': 'object', - 'properties': { - 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'} - } - }, - 'telegram': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'token': {'type': 'string'}, - 'chat_id': {'type': 'string'}, - }, - 'required': ['enabled', 'token', 'chat_id'] - }, - 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, - 'internals': { - 'type': 'object', - 'properties': { - 'process_throttle_secs': {'type': 'number'}, - 'interval': {'type': 'integer'} - } + 'required': ['ask_last_balance'] + }, + 'exchange': {'$ref': '#/definitions/exchange'}, + 'experimental': { + 'type': 'object', + 'properties': { + 'use_sell_signal': {'type': 'boolean'}, + 'sell_profit_only': {'type': 'boolean'} } }, - 'definitions': { - 'exchange': { - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'key': {'type': 'string'}, - 'secret': {'type': 'string'}, - 'pair_whitelist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' - }, - 'uniqueItems': True - }, - 'pair_blacklist': { - 'type': 'array', - 'items': { - 'type': 'string', - 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' - }, - 'uniqueItems': True - } - }, - 'required': ['name', 'key', 'secret', 'pair_whitelist'] - } + 'telegram': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'token': {'type': 'string'}, + 'chat_id': {'type': 'string'}, + }, + 'required': ['enabled', 'token', 'chat_id'] }, - 'anyOf': [ - {'required': ['exchange']} - ], - 'required': [ - 'max_open_trades', - 'stake_currency', - 'stake_amount', - 'fiat_display_currency', - 'dry_run', - 'bid_strategy', - 'telegram' - ] - } + 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, + 'internals': { + 'type': 'object', + 'properties': { + 'process_throttle_secs': {'type': 'number'}, + 'interval': {'type': 'integer'} + } + } + }, + 'definitions': { + 'exchange': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'key': {'type': 'string'}, + 'secret': {'type': 'string'}, + 'pair_whitelist': { + 'type': 'array', + 'items': { + 'type': 'string', + 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' + }, + 'uniqueItems': True + }, + 'pair_blacklist': { + 'type': 'array', + 'items': { + 'type': 'string', + 'pattern': '^[0-9A-Z]+_[0-9A-Z]+$' + }, + 'uniqueItems': True + } + }, + 'required': ['name', 'key', 'secret', 'pair_whitelist'] + } + }, + 'anyOf': [ + {'required': ['exchange']} + ], + 'required': [ + 'max_open_trades', + 'stake_currency', + 'stake_amount', + 'fiat_display_currency', + 'dry_run', + 'bid_strategy', + 'telegram' + ] +} diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f5de95826..0a730882c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade import ( DependencyException, OperationalException, exchange, persistence, __version__ ) from freqtrade.analyze import Analyze -from freqtrade.constants import Constants +from freqtrade import constants from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.persistence import Trade from freqtrade.rpc.rpc_manager import RPCManager @@ -111,7 +111,7 @@ class FreqtradeBot(object): elif state == State.RUNNING: min_secs = self.config.get('internals', {}).get( 'process_throttle_secs', - Constants.PROCESS_THROTTLE_SECS + constants.PROCESS_THROTTLE_SECS ) nb_assets = self.config.get('dynamic_whitelist', None) @@ -175,7 +175,7 @@ class FreqtradeBot(object): except (requests.exceptions.RequestException, json.JSONDecodeError) as error: logger.warning('%s, retrying in 30 seconds...', error) - time.sleep(Constants.RETRY_TIMEOUT) + time.sleep(constants.RETRY_TIMEOUT) except OperationalException: self.rpc.send_msg( '*Status:* OperationalException:\n```\n{traceback}```{hint}' diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index de38cf67a..bde852c92 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -10,7 +10,7 @@ import os from collections import OrderedDict from typing import Optional, Dict, Type -from freqtrade.constants import Constants +from freqtrade import constants from freqtrade.strategy.interface import IStrategy @@ -32,7 +32,7 @@ class StrategyResolver(object): config = config or {} # Verify the strategy is in the configuration, otherwise fallback to the default strategy - strategy_name = config.get('strategy') or Constants.DEFAULT_STRATEGY + strategy_name = config.get('strategy') or constants.DEFAULT_STRATEGY self.strategy = self._load_strategy(strategy_name, extra_dir=config.get('strategy_path')) # Set attributes diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 07dc45a3e..48bfcaf89 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -12,7 +12,7 @@ from sqlalchemy import create_engine from telegram import Chat, Message, Update from freqtrade.analyze import Analyze -from freqtrade.constants import Constants +from freqtrade import constants from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -88,7 +88,7 @@ def default_conf(): "initial_state": "running", "loglevel": logging.DEBUG } - validate(configuration, Constants.CONF_SCHEMA) + validate(configuration, constants.CONF_SCHEMA) return configuration diff --git a/freqtrade/tests/test_constants.py b/freqtrade/tests/test_constants.py index 6d544502f..541c6e533 100644 --- a/freqtrade/tests/test_constants.py +++ b/freqtrade/tests/test_constants.py @@ -2,25 +2,24 @@ Unit test file for constants.py """ -from freqtrade.constants import Constants +from freqtrade import constants def test_constant_object() -> None: """ Test the Constants object has the mandatory Constants """ - assert hasattr(Constants, 'CONF_SCHEMA') - assert hasattr(Constants, 'DYNAMIC_WHITELIST') - assert hasattr(Constants, 'PROCESS_THROTTLE_SECS') - assert hasattr(Constants, 'TICKER_INTERVAL') - assert hasattr(Constants, 'HYPEROPT_EPOCH') - assert hasattr(Constants, 'RETRY_TIMEOUT') - assert hasattr(Constants, 'DEFAULT_STRATEGY') + assert hasattr(constants, 'CONF_SCHEMA') + assert hasattr(constants, 'DYNAMIC_WHITELIST') + assert hasattr(constants, 'PROCESS_THROTTLE_SECS') + assert hasattr(constants, 'TICKER_INTERVAL') + assert hasattr(constants, 'HYPEROPT_EPOCH') + assert hasattr(constants, 'RETRY_TIMEOUT') + assert hasattr(constants, 'DEFAULT_STRATEGY') def test_conf_schema() -> None: """ Test the CONF_SCHEMA is from the right type """ - constant = Constants() - assert isinstance(constant.CONF_SCHEMA, dict) + assert isinstance(constants.CONF_SCHEMA, dict) From 55dc699d45177fac05f541f764391bbc54c3eb22 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 7 Apr 2018 06:42:10 +0200 Subject: [PATCH 46/57] Update pytest-mock from 1.7.1 to 1.8.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 251723b7f..3b6f6dcbc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ jsonschema==2.6.0 numpy==1.14.2 TA-Lib==0.4.17 pytest==3.5.0 -pytest-mock==1.7.1 +pytest-mock==1.8.0 pytest-cov==2.5.1 hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 From a26cdceb4b29b2c69d2240996588f55bc39cebc7 Mon Sep 17 00:00:00 2001 From: Matthias <5024695+xmatthias@users.noreply.github.com> Date: Sat, 7 Apr 2018 20:06:53 +0200 Subject: [PATCH 47/57] Fix tests run in random order (#599) * allow tests to run in random mode * Fix random test mode for fiat-convert * allow random test execution in persistence * fix pep8 styling * use "usefixtures" to prevent pylint "unused parameter" message * add pytest-random-order to travis --- .travis.yml | 4 ++-- freqtrade/tests/conftest.py | 2 +- freqtrade/tests/exchange/test_exchange.py | 6 +++--- freqtrade/tests/test_fiat_convert.py | 4 +++- freqtrade/tests/test_persistence.py | 14 ++++++++++++++ 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index be330dad4..6554f2095 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ addons: install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -- pip install --upgrade flake8 coveralls +- pip install --upgrade flake8 coveralls pytest-random-order - pip install -r requirements.txt - pip install -e . jobs: @@ -34,4 +34,4 @@ notifications: cache: directories: - $HOME/.cache/pip - - ta-lib \ No newline at end of file + - ta-lib diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 48bfcaf89..5e35537c9 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -46,7 +46,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: return FreqtradeBot(config, create_engine('sqlite://')) -@pytest.fixture(scope="module") +@pytest.fixture(scope="function") def default_conf(): """ Returns validated configuration suitable for most tests """ configuration = { diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index f2874f2da..578c904b3 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -16,9 +16,9 @@ from freqtrade.tests.conftest import log_has API_INIT = False -def maybe_init_api(conf, mocker): +def maybe_init_api(conf, mocker, force=False): global API_INIT - if not API_INIT: + if force or not API_INIT: mocker.patch('freqtrade.exchange.validate_pairs', side_effect=lambda s: True) init(config=conf) @@ -27,7 +27,7 @@ def maybe_init_api(conf, mocker): def test_init(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - maybe_init_api(default_conf, mocker) + maybe_init_api(default_conf, mocker, True) assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/test_fiat_convert.py index 49a2adc05..dcbc7b936 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/test_fiat_convert.py @@ -126,8 +126,10 @@ def test_fiat_convert_get_price(mocker): def test_fiat_convert_without_network(): # Because CryptoToFiatConverter is a Singleton we reset the value of _coinmarketcap - CryptoToFiatConverter._coinmarketcap = None fiat_convert = CryptoToFiatConverter() + + CryptoToFiatConverter._coinmarketcap = None + assert fiat_convert._coinmarketcap is None assert fiat_convert._find_price(crypto_symbol='BTC', fiat_symbol='USD') == 0.0 diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 70199b12a..e4e16bcd1 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -8,6 +8,11 @@ from freqtrade.exchange import Exchanges from freqtrade.persistence import Trade, init, clean_dry_run_db +@pytest.fixture(scope='function') +def init_persistence(default_conf): + init(default_conf) + + def test_init_create_session(default_conf, mocker): mocker.patch.dict('freqtrade.persistence._CONF', default_conf) @@ -90,6 +95,7 @@ def test_init_prod_db(default_conf, mocker): os.rename(prod_db_swp, prod_db) +@pytest.mark.usefixtures("init_persistence") def test_update_with_bittrex(limit_buy_order, limit_sell_order): """ On this test we will buy and sell a crypto currency. @@ -144,6 +150,7 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order): assert trade.close_date is not None +@pytest.mark.usefixtures("init_persistence") def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order): trade = Trade( pair='BTC_ETH', @@ -166,6 +173,7 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order): assert trade.calc_profit_percent() == 0.06201057 +@pytest.mark.usefixtures("init_persistence") def test_calc_close_trade_price_exception(limit_buy_order): trade = Trade( pair='BTC_ETH', @@ -179,6 +187,7 @@ def test_calc_close_trade_price_exception(limit_buy_order): assert trade.calc_close_trade_price() == 0.0 +@pytest.mark.usefixtures("init_persistence") def test_update_open_order(limit_buy_order): trade = Trade( pair='BTC_ETH', @@ -201,6 +210,7 @@ def test_update_open_order(limit_buy_order): assert trade.close_date is None +@pytest.mark.usefixtures("init_persistence") def test_update_invalid_order(limit_buy_order): trade = Trade( pair='BTC_ETH', @@ -213,6 +223,7 @@ def test_update_invalid_order(limit_buy_order): trade.update(limit_buy_order) +@pytest.mark.usefixtures("init_persistence") def test_calc_open_trade_price(limit_buy_order): trade = Trade( pair='BTC_ETH', @@ -230,6 +241,7 @@ def test_calc_open_trade_price(limit_buy_order): assert trade.calc_open_trade_price(fee=0.003) == 0.001003000 +@pytest.mark.usefixtures("init_persistence") def test_calc_close_trade_price(limit_buy_order, limit_sell_order): trade = Trade( pair='BTC_ETH', @@ -251,6 +263,7 @@ def test_calc_close_trade_price(limit_buy_order, limit_sell_order): assert trade.calc_close_trade_price(fee=0.005) == 0.0010619972 +@pytest.mark.usefixtures("init_persistence") def test_calc_profit(limit_buy_order, limit_sell_order): trade = Trade( pair='BTC_ETH', @@ -281,6 +294,7 @@ def test_calc_profit(limit_buy_order, limit_sell_order): assert trade.calc_profit(fee=0.003) == 0.00006163 +@pytest.mark.usefixtures("init_persistence") def test_calc_profit_percent(limit_buy_order, limit_sell_order): trade = Trade( pair='BTC_ETH', From 53690c5ece59ab14a19033d03ace8c027453b4f4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 10 Apr 2018 05:57:16 +0200 Subject: [PATCH 48/57] Update pytest-mock from 1.8.0 to 1.9.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3b6f6dcbc..303324f0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ jsonschema==2.6.0 numpy==1.14.2 TA-Lib==0.4.17 pytest==3.5.0 -pytest-mock==1.8.0 +pytest-mock==1.9.0 pytest-cov==2.5.1 hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 From 4b78beddddfa233f50146e581c866853a84d83d7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 17 Apr 2018 09:27:27 +0200 Subject: [PATCH 49/57] Update python-telegram-bot from 10.0.1 to 10.0.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 303324f0a..6f197597d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ python-bittrex==0.3.0 SQLAlchemy==1.2.6 -python-telegram-bot==10.0.1 +python-telegram-bot==10.0.2 arrow==0.12.1 cachetools==2.0.1 requests==2.18.4 From 0fab7f0880d4f2c837d55d5b724d8e91d74fd8dc Mon Sep 17 00:00:00 2001 From: Pan Long Date: Wed, 18 Apr 2018 19:11:37 +0800 Subject: [PATCH 50/57] Fix a typo in setup.sh --- setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.sh b/setup.sh index b38f2c645..bdcec7186 100755 --- a/setup.sh +++ b/setup.sh @@ -117,7 +117,7 @@ function config_generator () { -e "s/\"your_exchange_key\"/\"$api_key\"/g" \ -e "s/\"your_exchange_secret\"/\"$api_secret\"/g" \ -e "s/\"your_telegram_token\"/\"$token\"/g" \ - -e "s/\"your_telegram_chat_id\"/\"$chat_id\"/g" + -e "s/\"your_telegram_chat_id\"/\"$chat_id\"/g" \ -e "s/\"dry_run\": false,/\"dry_run\": true,/g" config.json.example > config.json } @@ -205,4 +205,4 @@ plot help ;; esac -exit 0 \ No newline at end of file +exit 0 From bb07ad38d35d40e875c54be595118a74a11e5c8e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 20 Apr 2018 23:35:34 +0200 Subject: [PATCH 51/57] Update sqlalchemy from 1.2.6 to 1.2.7 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f197597d..7060c8ad5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ python-bittrex==0.3.0 -SQLAlchemy==1.2.6 +SQLAlchemy==1.2.7 python-telegram-bot==10.0.2 arrow==0.12.1 cachetools==2.0.1 From 6d327658eac20784e12454aa962b2f3239d1becb Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 21 Apr 2018 19:24:53 +0300 Subject: [PATCH 52/57] docs: Add note about using telegram proxy (#611) --- docs/telegram-usage.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 7de9809ec..cfbebdca0 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -127,3 +127,14 @@ Day Profit BTC Profit USD ## /version > **Version:** `0.14.3` + +### using proxy with telegram +in [freqtrade/freqtrade/rpc/telegram.py](https://github.com/gcarq/freqtrade/blob/develop/freqtrade/rpc/telegram.py) replace +``` +self._updater = Updater(token=self._config['telegram']['token'], workers=0) +``` + +with +``` +self._updater = Updater(token=self._config['telegram']['token'], request_kwargs={'proxy_url': 'socks5://127.0.0.1:1080/'}, workers=0) +``` From 954c6e8c15095e9935dafd7f3055414687136b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Felipe=20D=C3=ADaz=20Chica?= Date: Sat, 21 Apr 2018 11:44:57 -0500 Subject: [PATCH 53/57] Write log when trying to sell opened trades (#608) --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0a730882c..9daaa7d14 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -378,7 +378,7 @@ class FreqtradeBot(object): if self.analyze.should_sell(trade, current_rate, datetime.utcnow(), buy, sell): self.execute_sell(trade, current_rate) return True - + logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False def check_handle_timedout(self, timeoutvalue: int) -> None: From 710c7daec50563ec49499734cf75aab9e22282e5 Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Sun, 22 Apr 2018 09:21:09 +0200 Subject: [PATCH 54/57] update Docker image to python-3.6.5-slim --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 918552526..afafd93c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6.4-slim-stretch +FROM python:3.6.5-slim-stretch # Install TA-lib RUN apt-get update && apt-get -y install curl build-essential && apt-get clean From 6adab0cf6bbd6a1eca7c503f3bf6256bb70e326a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 25 Apr 2018 04:54:46 +0200 Subject: [PATCH 55/57] Update pytest from 3.5.0 to 3.5.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7060c8ad5..1073e40e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.0.1 jsonschema==2.6.0 numpy==1.14.2 TA-Lib==0.4.17 -pytest==3.5.0 +pytest==3.5.1 pytest-mock==1.9.0 pytest-cov==2.5.1 hyperopt==0.1 From cec58323d4fb34cc9dff97f3ed30185bfa5670a7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 28 Apr 2018 19:19:50 +0200 Subject: [PATCH 56/57] Update numpy from 1.14.2 to 1.14.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1073e40e4..b4e8a629c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pandas==0.22.0 scikit-learn==0.19.1 scipy==1.0.1 jsonschema==2.6.0 -numpy==1.14.2 +numpy==1.14.3 TA-Lib==0.4.17 pytest==3.5.1 pytest-mock==1.9.0 From bc13b7901fdbd890c553e099ac5786e7daa6d1f7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 1 May 2018 20:12:57 +0200 Subject: [PATCH 57/57] Update pytest-mock from 1.9.0 to 1.10.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b4e8a629c..7093d783e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ jsonschema==2.6.0 numpy==1.14.3 TA-Lib==0.4.17 pytest==3.5.1 -pytest-mock==1.9.0 +pytest-mock==1.10.0 pytest-cov==2.5.1 hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325