import datetime import logging import os import tempfile from base64 import urlsafe_b64encode import boto3 import simplejson as json from boto3.dynamodb.conditions import Key from freqtrade.aws.tables import get_strategy_table from freqtrade.optimize.backtesting import Backtesting from requests import post db = boto3.resource('dynamodb') def backtest(event, context): """ this method is running on the AWS server and back tests this application for us and stores the back testing results in a local database this event can be given as: :param event: { 'strategy' : 'url handle where we can find the strategy' 'stake_currency' : 'our desired stake currency' 'asset' : '[] asset we are interested in. 'username' : user who's strategy should be evaluated 'name' : name of the strategy we want to evaluate 'exchange' : name of the exchange we should be using } it should be invoked by SNS only to avoid abuse of the system! :param context: standard AWS context, so pleaes ignore for now! :return: no return """ if 'Records' in event: for x in event['Records']: if 'Sns' in x and 'Message' in x['Sns']: event['body'] = json.loads(x['Sns']['Message']) name = event['body']['name'] user = event['body']['user'] table = get_strategy_table() response = table.query( KeyConditionExpression=Key('user').eq(user) & Key('name').eq(name) ) till = datetime.datetime.today() fromDate = till - datetime.timedelta(days=1) if 'from' in event['body']: fromDate = datetime.datetime.strptime(event['body']['from'], '%Y%m%d') if 'till' in event['body']: till = datetime.datetime.strptime(event['body']['till'], '%Y%m%d') try: if "Items" in response and len(response['Items']) > 0: print("schedule backtesting from {} till {} for {} with {} vs {}".format(fromDate, till, name, event['body'][ 'stake_currency'], event['body'][ 'assets'])) configuration = _generate_configuration(event, fromDate, name, response, till) print("configuration: \n{}\n".format( urlsafe_b64encode(json.dumps(configuration).encode('utf-8')).decode('utf-8'))) # fire AWS fargate instance now run_backtest(configuration, name, user) return { "statusCode": 200 } else: return { "statusCode": 404, "body": json.dumps({ "error": "sorry we did not find any matching strategy for user {} and name {}".format( user, name)}) } except ImportError as e: return { "statusCode": 500, "body": json.dumps({"error": e}) } else: raise Exception("not a valid event: {}".format(event)) def run_backtest(configuration, name, user): backtesting = Backtesting(configuration) result = backtesting.start() for index, row in result.iterrows(): data = { "id": "{}.{}:{}:test".format(user, name, row['currency'].upper()), "trade": "{} to {}".format(row['entry'].strftime('%Y-%m-%d %H:%M:%S'), row['exit'].strftime('%Y-%m-%d %H:%M:%S')), "pair": row['currency'], "duration": row['duration'], "profit_percent": row['profit_percent'], "profit_stake": row['profit_BTC'], "entry_date": row['entry'].strftime('%Y-%m-%d %H:%M:%S'), "exit_date": row['exit'].strftime('%Y-%m-%d %H:%M:%S') } _submit_result_to_backend(data) def _submit_result_to_backend(data): """ submits the given result to the backend system for further processing and analysis :param data: :return: """ print(data) try: print(post("{}/trade".format(os.environ['BASE_URL']), data=data)) except Exception as e: print("submission ignored: {}".format(e)) def _generate_configuration(event, fromDate, name, response, till): """ generates the configuration for us on the fly :param event: :param fromDate: :param name: :param response: :param till: :return: """ content = response['Items'][0]['content'] configuration = { "max_open_trades": 1, "stake_currency": event['body']['stake_currency'].upper(), "stake_amount": 1, "fiat_display_currency": "USD", "unfilledtimeout": 600, "trailing_stop": response['Items'][0]['trailing_stop'], "bid_strategy": { "ask_last_balance": 0.0 }, "exchange": { "name": response['Items'][0]['exchange'], "enabled": True, "key": "key", "secret": "secret", "pair_whitelist": list( map(lambda x: "{}/{}".format(x, response['Items'][0]['stake_currency']).upper(), response['Items'][0]['assets'])) }, "telegram": { "enabled": False, "token": "token", "chat_id": "0" }, "initial_state": "running", "datadir": os.environ("FREQ_DATA_DIR", tempfile.gettempdir()), "experimental": { "use_sell_signal": response['Items'][0]['use_sell'], "sell_profit_only": True }, "internals": { "process_throttle_secs": 5 }, 'realistic_simulation': True, "loglevel": logging.INFO, "strategy": "{}:{}".format(name, content), "timerange": "{}-{}".format(fromDate.strftime('%Y%m%d'), till.strftime('%Y%m%d')), "refresh_pairs": True } return configuration def cron(event, context): """ this functions submits all strategies to the backtesting queue :param event: :param context: :return: """ # if topic exists, we just reuse it client = boto3.client('sns') topic_arn = client.create_topic(Name=os.environ['topic'])['TopicArn'] table = get_strategy_table() response = table.scan() def fetch(response, table): """ fetches all strategies from the server technically code duplications TODO refacture :param response: :param table: :return: """ for i in response['Items']: # fire a message to our queue message = { "user": i['user'], "name": i['name'], "assets": i['assets'], "stake_currency": i['stake_currency'] } # triggered over html, let's provide # a date range for the backtesting if 'pathParameters' in event: if 'from' in event['pathParameters']: message['from'] = event['pathParameters']['from'] else: message['from'] = datetime.datetime.today().strftime('%Y%m%d') if 'till' in event['pathParameters']: message['till'] = event['pathParameters']['till'] else: message['till'] = (datetime.datetime.today() - datetime.timedelta(days=1)).strftime('%Y%m%d') serialized = json.dumps(message, use_decimal=True) # submit item to queue for routing to the correct persistence result = client.publish( TopicArn=topic_arn, Message=json.dumps({'default': serialized}), Subject="schedule", MessageStructure='json' ) if 'LastEvaluatedKey' in response: return table.scan( ExclusiveStartKey=response['LastEvaluatedKey'] ) else: return {} # do/while simulation response = fetch(response, table) while 'LastEvaluatedKey' in response: response = fetch(response, table) return { "statusCode": 200 }