diff --git a/docs/backtesting.md b/docs/backtesting.md index df105bd77..8c4c4180d 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -83,6 +83,8 @@ The full timerange specification: - Use tickframes till 2018/01/31: `--timerange=-20180131` - Use tickframes since 2018/01/31: `--timerange=20180131-` - Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` +- Use tickframes between POSIX timestamps 1527595200 1527618600: + `--timerange=1527595200-1527618600` **Update testdata directory** diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index fc917394e..97c3d8cb2 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -223,6 +223,9 @@ class Arguments(object): syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^-(\d{10})$', (None, 'date')), + (r'^(\d{10})-$', ('date', None)), + (r'^(\d{10})-(\d{10})$', ('date', 'date')), (r'^(-\d+)$', (None, 'line')), (r'^(\d+)-$', ('line', None)), (r'^(\d+)-(\d+)$', ('index', 'index'))] @@ -237,14 +240,16 @@ class Arguments(object): if stype[0]: starts = rvals[index] if stype[0] == 'date': - start = arrow.get(starts, 'YYYYMMDD').timestamp + start = int(starts) if len(starts) == 10 \ + else arrow.get(starts, 'YYYYMMDD').timestamp else: start = int(starts) index += 1 if stype[1]: stops = rvals[index] if stype[1] == 'date': - stop = arrow.get(stops, 'YYYYMMDD').timestamp + stop = int(stops) if len(stops) == 10 \ + else arrow.get(stops, 'YYYYMMDD').timestamp else: stop = int(stops) return stype, start, stop diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 14c29e8fd..a800bde78 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -145,7 +145,7 @@ class Configuration(object): # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self.args.datadir}) - logger.info('Parameter --datadir detected: %s ...', self.args.datadir) + logger.info('Using data folder: %s ...', self.args.datadir) # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 58a4b07fe..711adfd28 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -103,7 +103,12 @@ def load_data(datadir: str, if pairdata: result[pair] = pairdata else: - logger.warn('No data for pair %s, use --update-pairs-cached to download the data', pair) + logger.warning( + 'No data for pair: "%s", Interval: %s. ' + 'Use --refresh-pairs-cached to download the data', + pair, + ticker_interval + ) return result diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a0aee78b1..1d560d309 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -94,7 +94,7 @@ class Backtesting(object): len(results[results.profit_BTC > 0]), len(results[results.profit_BTC < 0]) ]) - return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2497d6752..74b39b445 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -455,6 +455,7 @@ class Hyperopt(Backtesting): if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: print('.', end='') + sys.stdout.flush() return { 'status': STATUS_FAIL, 'loss': float('inf') @@ -479,16 +480,16 @@ class Hyperopt(Backtesting): 'result': result_explanation, } - @staticmethod - def format_results(results: DataFrame) -> str: + def format_results(self, results: DataFrame) -> str: """ Return the format result in a string """ return ('{:6d} trades. Avg profit {: 5.2f}%. ' - 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( + 'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( len(results.index), results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), + self.config['stake_currency'], results.profit_percent.sum(), results.duration.mean(), ) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 715d38f95..f48666748 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -316,8 +316,10 @@ class RPC(object): and order['side'] == 'buy': exchange.cancel_order(trade.open_order_id, trade.pair) trade.close(order.get('price') or trade.open_rate) - # TODO: sell amount which has been bought already - return + # Do the best effort, if we don't know 'filled' amount, don't try selling + if order['filled'] is None: + return + trade.amount = order['filled'] # Ignore trades with an attached LIMIT_SELL order if order and order['status'] == 'open' \ diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 5cc41845d..1b1872404 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -181,7 +181,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config @@ -229,7 +229,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config @@ -357,16 +357,15 @@ def test_generate_text_table(default_conf, mocker): ) result_str = ( - 'pair buy count avg profit % ' - 'total profit BTC avg duration profit loss\n' - '------- ----------- -------------- ' - '------------------ -------------- -------- ------\n' - 'ETH/BTC 2 15.00 ' - '0.60000000 20.0 2 0\n' - 'TOTAL 2 15.00 ' - '0.60000000 20.0 2 0' + '| pair | buy count | avg profit % | ' + 'total profit BTC | avg duration | profit | loss |\n' + '|:--------|------------:|---------------:|' + '-------------------:|---------------:|---------:|-------:|\n' + '| ETH/BTC | 2 | 15.00 | ' + '0.60000000 | 20.0 | 2 | 0 |\n' + '| TOTAL | 2 | 15.00 | ' + '0.60000000 | 20.0 | 2 | 0 |' ) - assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str @@ -615,7 +614,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Parameter -l/--live detected ...', 'Using max_open_trades: 1 ...', 'Parameter --timerange detected: -100 ..', - 'Parameter --datadir detected: freqtrade/tests/testdata ...', + 'Using data folder: freqtrade/tests/testdata ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index f8fa66b2e..3edfe4393 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -389,10 +389,12 @@ def test_start_uses_mongotrials(mocker, init_hyperopt, default_conf) -> None: # test buy_strategy_generator def populate_buy_trend # test optimizer if 'ro_t1' in params -def test_format_results(): +def test_format_results(init_hyperopt): """ Test Hyperopt.format_results() """ + + # Test with BTC as stake_currency trades = [ ('ETH/BTC', 2, 2, 123), ('LTC/BTC', 1, 1, 123), @@ -400,8 +402,21 @@ def test_format_results(): ] labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] df = pd.DataFrame.from_records(trades, columns=labels) - x = Hyperopt.format_results(df) - assert x.find(' 66.67%') + + result = _HYPEROPT.format_results(df) + assert result.find(' 66.67%') + assert result.find('Total profit 1.00000000 BTC') + assert result.find('2.0000Σ %') + + # Test with EUR as stake_currency + trades = [ + ('ETH/EUR', 2, 2, 123), + ('LTC/EUR', 1, 1, 123), + ('XPR/EUR', -1, -2, -246) + ] + df = pd.DataFrame.from_records(trades, columns=labels) + result = _HYPEROPT.format_results(df) + assert result.find('Total profit 1.00000000 EUR') def test_signal_handler(mocker, init_hyperopt): diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 8624b500d..349fa3be3 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -105,7 +105,8 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog) -> None: refresh_pairs=False, pairs=['MEME/BTC']) assert os.path.isfile(file) is False - assert log_has('No data for pair MEME/BTC, use --update-pairs-cached to download the data', + assert log_has('No data for pair: "MEME/BTC", Interval: 1m. ' + 'Use --refresh-pairs-cached to download the data', caplog.record_tuples) # download a new pair if refresh_pairs is set diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 1d1b3b39c..1cf374b6b 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -449,20 +449,44 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 # make an limit-buy open trade + trade = Trade.query.filter(Trade.id == '1').first() + filled_amount = trade.amount / 2 mocker.patch( 'freqtrade.freqtradebot.exchange.get_order', return_value={ 'status': 'open', 'type': 'limit', - 'side': 'buy' + 'side': 'buy', + 'filled': filled_amount } ) - # check that the trade is called, which is done - # by ensuring exchange.cancel_order is called + # check that the trade is called, which is done by ensuring exchange.cancel_order is called + # and trade amount is updated (error, res) = rpc.rpc_forcesell('1') assert not error assert res == '' assert cancel_order_mock.call_count == 1 + assert trade.amount == filled_amount + + freqtradebot.create_trade() + trade = Trade.query.filter(Trade.id == '2').first() + amount = trade.amount + # make an limit-buy open trade, if there is no 'filled', don't sell it + mocker.patch( + 'freqtrade.freqtradebot.exchange.get_order', + return_value={ + 'status': 'open', + 'type': 'limit', + 'side': 'buy', + 'filled': None + } + ) + # check that the trade is called, which is done by ensuring exchange.cancel_order is called + (error, res) = rpc.rpc_forcesell('2') + assert not error + assert res == '' + assert cancel_order_mock.call_count == 2 + assert trade.amount == amount freqtradebot.create_trade() # make an limit-sell open trade @@ -474,11 +498,11 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: 'side': 'sell' } ) - (error, res) = rpc.rpc_forcesell('2') + (error, res) = rpc.rpc_forcesell('3') assert not error assert res == '' # status quo, no exchange calls - assert cancel_order_mock.call_count == 1 + assert cancel_order_mock.call_count == 2 def test_performance_handle(default_conf, ticker, limit_buy_order, fee, diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 279ace0dc..474aa2507 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -116,6 +116,12 @@ def test_parse_timerange_incorrect() -> None: timerange = Arguments.parse_timerange('20100522-20150730') assert timerange == (('date', 'date'), 1274486400, 1438214400) + # Added test for unix timestamp - BTC genesis date + assert (('date', None), 1231006505, None) == Arguments.parse_timerange('1231006505-') + assert ((None, 'date'), None, 1233360000) == Arguments.parse_timerange('-1233360000') + timerange = Arguments.parse_timerange('1231006505-1233360000') + assert timerange == (('date', 'date'), 1231006505, 1233360000) + with pytest.raises(Exception, match=r'Incorrect syntax.*'): Arguments.parse_timerange('-') diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 8ba8f8289..492fdee70 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -236,7 +236,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config @@ -287,7 +287,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Parameter --datadir detected: {} ...'.format(config['datadir']), + 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples ) assert 'ticker_interval' in config diff --git a/requirements.txt b/requirements.txt index 37b6a09f1..d96084348 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.14.73 +ccxt==1.14.120 SQLAlchemy==1.2.8 python-telegram-bot==10.1.0 arrow==0.12.1 diff --git a/setup.sh b/setup.sh index 1d06e8208..a825ca41f 100755 --- a/setup.sh +++ b/setup.sh @@ -2,16 +2,17 @@ #encoding=utf8 function updateenv () { - echo " - ------------------------- - Update your virtual env - ------------------------- - " + echo "-------------------------" + echo "Update your virtual env" + echo "-------------------------" source .env/bin/activate - pip3.6 install --upgrade pip - pip3 install -r requirements.txt --upgrade - pip3 install -r requirements.txt - pip3 install -e . + echo "pip3 install in-progress. Please wait..." + pip3.6 install --quiet --upgrade pip + pip3 install --quiet -r requirements.txt --upgrade + pip3 install --quiet -r requirements.txt + pip3 install --quiet -e . + echo "pip3 install completed" + echo } # Install tab lib @@ -29,10 +30,11 @@ function install_macos () { echo "-------------------------" echo "Install Brew" echo "-------------------------" - echo /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi brew install python3 wget ta-lib + + test_and_fix_python_on_mac } # Install bot Debian_ubuntu @@ -54,7 +56,6 @@ function reset () { echo "----------------------------" echo "Reset branch and virtual env" echo "----------------------------" - echo if [ "1" == $(git branch -vv |grep -cE "\* develop|\* master") ] then if [ -d ".env" ]; then @@ -77,18 +78,30 @@ function reset () { echo "Reset ignored because you are not on 'master' or 'develop'." fi + echo python3.6 -m venv .env updateenv } +function test_and_fix_python_on_mac() { + + if ! [ -x "$(command -v python3.6)" ] + then + echo "-------------------------" + echo "Fixing Python" + echo "-------------------------" + echo "Python 3.6 is not linked in your system. Fixing it..." + brew link --overwrite python + echo + fi +} + function config_generator () { echo "Starting to generate config.json" - - echo "-------------------------" + echo echo "General configuration" echo "-------------------------" - echo default_max_trades=3 read -p "Max open trades: (Default: $default_max_trades) " max_trades max_trades=${max_trades:-$default_max_trades} @@ -105,14 +118,13 @@ function config_generator () { read -p "Fiat currency: (Default: $default_fiat_currency) " fiat_currency fiat_currency=${fiat_currency:-$default_fiat_currency} - echo "------------------------" - echo "Bittrex config generator" - echo "------------------------" echo + echo "Exchange config generator" + echo "------------------------" read -p "Exchange API key: " api_key read -p "Exchange API Secret: " api_secret - echo "-------------------------" + echo echo "Telegram config generator" echo "-------------------------" read -p "Telegram Token: " token @@ -131,6 +143,10 @@ function config_generator () { } function config () { + + echo "-------------------------" + echo "Config file generator" + echo "-------------------------" if [ -f config.json ] then read -p "A config file already exist, do you want to override it [Y/N]? " @@ -144,22 +160,26 @@ function config () { config_generator fi + echo + echo "-------------------------" + echo "Config file generated" + echo "-------------------------" echo "Edit ./config.json to modify Pair and other configurations." + echo } function install () { echo "-------------------------" echo "Install mandatory dependencies" echo "-------------------------" - echo if [ "$(uname -s)" == "Darwin" ] then - echo "- You are on macOS" + echo "macOS detected. Setup for this system in-progress" install_macos elif [ -x "$(command -v apt-get)" ] then - echo "- You are on Debian/Ubuntu" + echo "Debian/Ubuntu detected. Setup for this system in-progress" install_debian else echo "This script does not support your OS." @@ -167,12 +187,13 @@ function install () { echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell." sleep 10 fi + echo reset - echo " - - Install complete. - " config - echo "You can now use the bot by executing 'source .env/bin/activate; python3 freqtrade/main.py'." + echo "-------------------------" + echo "Run the bot" + echo "-------------------------" + echo "You can now use the bot by executing 'source .env/bin/activate; python3.6 freqtrade/main.py'." } function plot () {