working on backtesting BASE64 encoded strategies

freqtrade/aws/backtesting_lambda.py

@@ -1,8 +1,15 @@
 import logging
 
+import boto3
+import os
+
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.optimize.backtesting import Backtesting
+import simplejson as json
+from boto3.dynamodb.conditions import Key, Attr
+
+db = boto3.resource('dynamodb')
 
 
 def backtest(event, context):
@@ -29,74 +36,92 @@ def backtest(event, context):
             no return
     """
 
-    name = "TestStrategy"
+    if 'body' in event:
-    user = "12345678"
+        event['body'] = json.loads(event['body'])
-    stake_currency = "USDT"
+        name = event['body']['name']
-    asset = ["ETH", "BTC"]
+        user = event['body']['user']
-    exchange = "binance"
+        stake_currency = event['body']['stake_currency'].upper()
+        asset = event['body']['asset']
+        exchange = event['body']['exchange']
 
-    assets = list(map(lambda x: "{}/{}".format(x, stake_currency).upper(), asset))
+        assets = list(map(lambda x: "{}/{}".format(x, stake_currency).upper(), asset))
 
-    configuration = {
+        table = db.Table(os.environ['strategyTable'])
-        "max_open_trades": 1,
-        "stake_currency": stake_currency,
-        "stake_amount": 0.001,
-        "fiat_display_currency": "USD",
-        "unfilledtimeout": 600,
-        "bid_strategy": {
-            "ask_last_balance": 0.0
-        },
-        "exchange": {
-            "name": "bittrex",
-            "enabled": True,
-            "key": "key",
-            "secret": "secret",
-            "pair_whitelist": assets
-        },
-        "telegram": {
-            "enabled": False,
-            "token": "token",
-            "chat_id": "0"
-        },
-        "initial_state": "running",
-        "datadir": ".",
-        "experimental": {
-            "use_sell_signal": True,
-            "sell_profit_only": True
-        },
-        "internals": {
-            "process_throttle_secs": 5
-        },
-        'realistic_simulation': True,
-        "loglevel": logging.DEBUG
 
-    }
+        response = table.query(
+            KeyConditionExpression=Key('user').eq(user) &
+                                   Key('name').eq(name)
 
-    print("generated configuration")
+        )
-    print(configuration)
 
-    print("initialized backtesting")
+        print(response)
-    backtesting = Backtesting(configuration)
+        if "Items" in response and len(response['Items']) > 0:
-    result = backtesting.start()
-    print("finished test")
 
-    print(result)
+            content = response['Items'][0]['content']
-    print("persist data in dynamo")
+            configuration = {
+                "max_open_trades": 1,
+                "stake_currency": stake_currency,
+                "stake_amount": 1,
+                "fiat_display_currency": "USD",
+                "unfilledtimeout": 600,
+                "bid_strategy": {
+                    "ask_last_balance": 0.0
+                },
+                "exchange": {
+                    "name": exchange,
+                    "enabled": True,
+                    "key": "key",
+                    "secret": "secret",
+                    "pair_whitelist": assets
+                },
+                "telegram": {
+                    "enabled": False,
+                    "token": "token",
+                    "chat_id": "0"
+                },
+                "initial_state": "running",
+                "datadir": ".",
+                "experimental": {
+                    "use_sell_signal": True,
+                    "sell_profit_only": True
+                },
+                "internals": {
+                    "process_throttle_secs": 5
+                },
+                'realistic_simulation': True,
+                "loglevel": logging.DEBUG,
+                "strategy": "{}:{}".format(name, content)
 
-    for index, row in result.iterrows():
+            }
-        item = {
-            "id": "{}.{}:{}".format(user, name, row['pair']),
-            "pair": row['pair'],
-            "profit": row['profit'],
-            "loss": row['loss'],
-            "duration": row['avg duration'],
-            "avg profit": row['avg profit %'],
-            "total profit": row['total profit {}'.format(stake_currency)]
 
-        }
+            print("generated configuration")
+            print(configuration)
 
-        print(item)
+            print("initialized backtesting")
-    pass
+            backtesting = Backtesting(configuration)
+            result = backtesting.start()
+            print("finished test")
+
+            print("persist data in dynamo")
+
+            for index, row in result.iterrows():
+                if row['loss'] > 0 or row['profit'] > 0:
+                    item = {
+                        "id": "{}.{}:{}".format(user, name, row['pair']),
+                        "pair": row['pair'],
+                        "count_profit": row['profit'],
+                        "count_loss": row['loss'],
+                        "avg_duration": row['avg duration'],
+                        "avg profit": row['avg profit %'],
+                        "total profit": row['total profit {}'.format(stake_currency)]
+
+                    }
+
+                print(item)
+        else:
+            raise Exception("sorry we did not find any matching strategy for user {} and name {}".format(user, name))
+    else:
+        raise Exception("no body provided")
 
 
 def submit(event, context):

freqtrade/aws/strategy.py

@@ -95,11 +95,6 @@ def code(event, context):
     :return:
     """
 
-    print("event")
-    print(event)
-    print("context")
-    print(context)
-
     user = ""
     name = ""
 

freqtrade/strategy/resolver.py

@@ -6,6 +6,7 @@ This module load custom strategies
 import importlib.util
 import inspect
 import logging
+from base64 import urlsafe_b64decode
 from collections import OrderedDict
 from typing import Optional, Dict, Type
 
@@ -88,15 +89,16 @@ class StrategyResolver(object):
             # Add extra strategy directory on top of search paths
             abs_paths.insert(0, extra_dir)
 
-        try:
+        # check if the given strategy is provided as name, value pair
-            # check if given strategy matches an url
+        # where the value is the strategy encoded in base 64
-            logger.debug("requesting remote strategy from {}".format(strategy_name))
+        if ":" in strategy_name:
-            resp = requests.get(strategy_name, stream=True)
+            strat = strategy_name.split(":")
-            if resp.status_code == 200:
-                temp = Path(tempfile.mkdtemp("freq", "strategy"))
-                name = os.path.basename(urlparse(strategy_name).path)
 
-                temp.joinpath(name).write_text(resp.text)
+            if len(strat) == 2:
+                temp = Path(tempfile.mkdtemp("freq", "strategy"))
+                name = strat[0] + ".py"
+
+                temp.joinpath(name).write_text(urlsafe_b64decode(strat[1]).decode('utf-8'))
                 temp.joinpath("__init__.py").touch()
 
                 strategy_name = os.path.splitext(name)[0]
@@ -104,8 +106,25 @@ class StrategyResolver(object):
                 # register temp path with the bot
                 abs_paths.insert(0, temp.absolute())
 
-        except requests.RequestException:
+        # check if given strategy matches an url
-            logger.debug("received error trying to fetch strategy remotely, carry on!")
+        else:
+            try:
+                logger.debug("requesting remote strategy from {}".format(strategy_name))
+                resp = requests.get(strategy_name, stream=True)
+                if resp.status_code == 200:
+                    temp = Path(tempfile.mkdtemp("freq", "strategy"))
+                    name = os.path.basename(urlparse(strategy_name).path)
+
+                    temp.joinpath(name).write_text(resp.text)
+                    temp.joinpath("__init__.py").touch()
+
+                    strategy_name = os.path.splitext(name)[0]
+
+                    # register temp path with the bot
+                    abs_paths.insert(0, temp.absolute())
+
+            except requests.RequestException:
+                logger.debug("received error trying to fetch strategy remotely, carry on!")
 
         for path in abs_paths:
             strategy = self._search_strategy(path, strategy_name)

freqtrade/tests/aws/test_backtest.py

@@ -1,7 +1,75 @@
-import pytest
+from base64 import urlsafe_b64encode
 
+import pytest
+import simplejson as json
 from freqtrade.aws.backtesting_lambda import backtest
+from freqtrade.aws.strategy import submit
 
 
 def test_backtest(lambda_context):
-    backtest({}, {})
+    content = """# --- Do not remove these libs ---
+from freqtrade.strategy.interface import IStrategy
+from typing import Dict, List
+from hyperopt import hp
+from functools import reduce
+from pandas import DataFrame
+# --------------------------------
+
+import talib.abstract as ta
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+class MyFancyTestStrategy(IStrategy):
+    minimal_roi = {
+        "0": 0.5
+    }
+    stoploss = -0.2
+    ticker_interval = '5m'
+
+    def populate_indicators(self, dataframe: DataFrame) -> DataFrame:
+        macd = ta.MACD(dataframe)
+        dataframe['maShort'] = ta.EMA(dataframe, timeperiod=8)
+        dataframe['maMedium'] = ta.EMA(dataframe, timeperiod=21)
+        return dataframe
+
+    def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
+        dataframe.loc[
+            (
+                qtpylib.crossed_above(dataframe['maShort'], dataframe['maMedium'])
+            ),
+            'buy'] = 1
+
+        return dataframe
+
+    def populate_sell_trend(self, dataframe: DataFrame) -> DataFrame:
+        dataframe.loc[
+            (
+                qtpylib.crossed_above(dataframe['maMedium'], dataframe['maShort'])
+            ),
+            'sell'] = 1
+        return dataframe
+
+
+        """
+
+    request = {
+        "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG",
+        "description": "simple test strategy",
+        "name": "MyFancyTestStrategy",
+        "content": urlsafe_b64encode(content.encode('utf-8')),
+        "public": False
+    }
+
+    # now we add an entry
+    submit({
+        "body": json.dumps(request)
+    }, {})
+
+    request = {
+        "user": "GCU4LW2XXZW3A3FM2XZJTEJHNWHTWDKY2DIJLCZJ5ULVZ4K7LZ7D23TG",
+        "name": "MyFancyTestStrategy",
+        "stake_currency": "usdt",
+        "asset": ["ETH", "BTC", "XRP", "LTC"],
+        "exchange": "binance"
+    }
+
+    backtest({"body": json.dumps(request)}, {})

freqtrade/tests/conftest.py

@@ -642,7 +642,7 @@ def lambda_context():
     # do not mock requests to these urls
     responses.add_passthru('https://api.github.com')
     responses.add_passthru('https://bittrex.com')
-
+    responses.add_passthru('https://api.binance.com')
     # here we will define required tables later
     yield
     sns.stop()