adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #29 from berlinguyinca/aws

Browse Source 
Aws
This commit is contained in:
Gert Wohlgemuth 2018-06-13 15:10:46 -07:00 committed by GitHub
commit d571ce266a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 506 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -26,8 +26,8 @@ dist/
downloads/
eggs/
.eggs/
lib/
lib64/
#lib/
#lib64/
parts/
sdist/
var/

2
Dockerfile
View File

@ -1,7 +1,7 @@
FROM python:3.6.5-slim-stretch
# Install TA-lib
RUN apt-get update && apt-get -y install curl build-essential && apt-get clean
RUN apt-get update && apt-get -y install curl build-essential git && apt-get clean
RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \
tar xzvf - && \
cd ta-lib && \

15
freqtrade/arguments.py
View File

@ -224,7 +224,7 @@ class Arguments(object):
Builds and attaches all subcommands
:return: None
"""
from freqtrade.optimize import backtesting, hyperopt
from freqtrade.optimize import backtesting
subparsers = self.parser.add_subparsers(dest='subparser')
@ -235,10 +235,14 @@ class Arguments(object):
self.backtesting_options(backtesting_cmd)
# Add hyperopt subcommand
try:
from freqtrade.optimize import hyperopt
hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
hyperopt_cmd.set_defaults(func=hyperopt.start)
self.optimizer_shared_options(hyperopt_cmd)
self.hyperopt_options(hyperopt_cmd)
except ImportError as e:
logging.warn("no hyper opt found - skipping support for it")
@staticmethod
def parse_timerange(text: Optional[str]) -> TimeRange:
@ -340,7 +344,6 @@ class Arguments(object):
default=None
)
self.parser.add_argument(
'--plot-macd',
help='Renders a macd chart of the given '
@ -383,14 +386,6 @@ class Arguments(object):
type=int
)
self.parser.add_argument(
'-db', '--db-url',
help='Show trades stored in database.',
dest='db_url',
default=None
)
def testdata_dl_options(self) -> None:
"""
Parses given arguments for testdata download

9
freqtrade/freqtradebot.py
View File

@ -253,7 +253,6 @@ class FreqtradeBot(object):
balance = self.config['bid_strategy']['ask_last_balance']
return ticker['ask'] + balance * (ticker['last'] - ticker['ask'])
def create_trade(self) -> bool:
"""
Checks the implemented trading indicator(s) for a randomly picked pair,
@ -440,7 +439,8 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \
logger.info('Using order book for selling...')
orderBook = exchange.get_order_book(trade.pair)
# logger.debug('Order book %s',orderBook)
for i in range(self.config['ask_strategy']['book_order_min'],self.config['ask_strategy']['book_order_max']+1):
for i in range(self.config['ask_strategy']['book_order_min'],
self.config['ask_strategy']['book_order_max'] + 1):
sell_rate = orderBook['asks'][i - 1][0]
if self.check_sell(trade, sell_rate, buy, sell):
return True
@ -483,8 +483,11 @@ with limit `{buy_limit:.8f} ({stake_amount:.6f} \
continue
ordertime = arrow.get(order['datetime']).datetime
print(order)
# Check if trade is still actually open
if (int(order['filled']) == 0) and (order['status']=='open'):
# this makes no real sense and causes errors!
#
# if (filled(int(order['filled']) == 0) and (order['status'] == 'open'):
if order['side'] == 'buy' and ordertime < timeoutthreashold:
self.handle_timedout_limit_buy(trade, order)
elif order['side'] == 'sell' and ordertime < timeoutthreashold:

32
freqtrade/optimize/backtesting.py
View File

@ -31,6 +31,7 @@ class Backtesting(object):
backtesting = Backtesting(config)
backtesting.start()
"""
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
self.analyze = Analyze(self.config)
@ -66,35 +67,41 @@ class Backtesting(object):
Generates and returns a text table for the given backtest data and the results dataframe
:return: pretty printed table with tabulate as str
"""
stake_currency = str(self.config.get('stake_currency'))
floatfmt = ('s', 'd', '.2f', '.8f', '.1f')
floatfmt, headers, tabular_data = self.aggregate(data, results)
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
def aggregate(self, data, results):
stake_currency = self.config.get('stake_currency')
floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.1f')
tabular_data = []
headers = ['pair', 'buy count', 'avg profit %',
headers = ['pair', 'buy count', 'avg profit %', 'cum profit %',
'total profit ' + stake_currency, 'avg duration', 'profit', 'loss']
for pair in data:
result = results[results.currency == pair]
print(results)
tabular_data.append([
pair,
len(result.index),
result.profit_percent.mean() * 100.0,
result.profit_percent.sum() * 100.0,
result.profit_BTC.sum(),
result.duration.mean(),
len(result[result.profit_BTC > 0]),
len(result[result.profit_BTC < 0])
])
# Append Total
tabular_data.append([
'TOTAL',
len(results.index),
results.profit_percent.mean() * 100.0,
results.profit_percent.sum() * 100.0,
results.profit_BTC.sum(),
results.duration.mean(),
len(results[results.profit_BTC > 0]),
len(results[results.profit_BTC < 0])
])
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
return floatfmt, headers, tabular_data
def _get_sell_trade_entry(
self, pair: str, buy_row: DataFrame,
@ -127,7 +134,9 @@ class Backtesting(object):
pair,
trade.calc_profit_percent(rate=sell_row.close),
trade.calc_profit(rate=sell_row.close),
(sell_row.date - buy_row.date).seconds // 60
(sell_row.date - buy_row.date).seconds // 60,
buy_row.date,
sell_row.date
), \
sell_row.date
return None
@ -193,6 +202,7 @@ class Backtesting(object):
if ret:
row2, trade_entry, next_date = ret
lock_pair_until = next_date
trades.append(trade_entry)
if record:
# Note, need to be json.dump friendly
@ -207,10 +217,12 @@ class Backtesting(object):
if record and record.find('trades') >= 0:
logger.info('Dumping backtest results to %s', recordfilename)
file_dump_json(recordfilename, records)
labels = ['currency', 'profit_percent', 'profit_BTC', 'duration']
file_dump_json('backtest-result.json', records)
labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'entry', 'exit']
return DataFrame.from_records(trades, columns=labels)
def start(self) -> None:
def start(self):
"""
Run a backtesting end-to-end
:return: None
@ -284,6 +296,10 @@ class Backtesting(object):
)
)
# return date for data storage
table = self.aggregate(data, results)
return (results, table)
def setup_configuration(args: Namespace) -> Dict[str, Any]:
"""

47
freqtrade/strategy/resolver.py
View File

@ -6,6 +6,7 @@ This module load custom strategies
import importlib.util
import inspect
import logging
from base64 import urlsafe_b64decode
from collections import OrderedDict
from typing import Optional, Dict, Type
@ -64,6 +65,13 @@ class StrategyResolver(object):
key=lambda t: t[0]))
self.strategy.stoploss = float(self.strategy.stoploss)
def compile(self, strategy_name: str, strategy_content: str) -> Optional[IStrategy]:
temp = Path(tempfile.mkdtemp("freq", "strategy"))
temp.joinpath(strategy_name + ".py").write_text(strategy_content)
temp.joinpath("__init__.py").touch()
return self._load_strategy(strategy_name, temp.absolute())
def _load_strategy(
self, strategy_name: str, extra_dir: Optional[str] = None) -> IStrategy:
"""
@ -82,15 +90,16 @@ class StrategyResolver(object):
# Add extra strategy directory on top of search paths
abs_paths.insert(0, extra_dir)
try:
# check if given strategy matches an url
logger.debug("requesting remote strategy from {}".format(strategy_name))
resp = requests.get(strategy_name, stream=True)
if resp.status_code == 200:
temp = Path(tempfile.mkdtemp("freq", "strategy"))
name = os.path.basename(urlparse(strategy_name).path)
# check if the given strategy is provided as name, value pair
# where the value is the strategy encoded in base 64
if ":" in strategy_name and "http" not in strategy_name:
strat = strategy_name.split(":")
temp.joinpath(name).write_text(resp.text)
if len(strat) == 2:
temp = Path(tempfile.mkdtemp("freq", "strategy"))
name = strat[0] + ".py"
temp.joinpath(name).write_text(urlsafe_b64decode(strat[1]).decode('utf-8'))
temp.joinpath("__init__.py").touch()
strategy_name = os.path.splitext(name)[0]
@ -98,6 +107,28 @@ class StrategyResolver(object):
# register temp path with the bot
abs_paths.insert(0, temp.absolute())
# check if given strategy matches an url
else:
try:
logger.debug("requesting remote strategy from {}".format(strategy_name))
resp = requests.get(strategy_name, stream=True)
if resp.status_code == 200:
temp = Path(tempfile.mkdtemp("freq", "strategy"))
if strategy_name.endswith("/code"):
strategy_name = strategy_name.replace("/code", "")
name = os.path.basename(urlparse(strategy_name).path)
temp.joinpath("{}.py".format(name)).write_text(resp.text)
temp.joinpath("__init__.py").touch()
strategy_name = os.path.splitext(name)[0]
print("stored downloaded stat at: {}".format(temp))
# register temp path with the bot
abs_paths.insert(0, temp.absolute())
except requests.RequestException:
logger.debug("received error trying to fetch strategy remotely, carry on!")

5
freqtrade/tests/conftest.py
View File

@ -15,6 +15,10 @@ from freqtrade.analyze import Analyze
from freqtrade import constants
from freqtrade.freqtradebot import FreqtradeBot
import moto
import boto3
import os
logging.getLogger('').setLevel(logging.INFO)
@ -523,6 +527,7 @@ def result():
with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file:
return Analyze.parse_ticker_dataframe(json.load(data_file))
# FIX:
# Create an fixture/function
# that inserts a trade of some type and open-status

13
freqtrade/tests/optimize/test_backtesting.py
View File

@ -365,14 +365,10 @@ def test_generate_text_table(default_conf, mocker):
)
result_str = (
'| pair | buy count | avg profit % | '
'total profit BTC | avg duration | profit | loss |\n'
'|:--------|------------:|---------------:|'
'-------------------:|---------------:|---------:|-------:|\n'
'| ETH/BTC | 2 | 15.00 | '
'0.60000000 | 20.0 | 2 | 0 |\n'
'| TOTAL | 2 | 15.00 | '
'0.60000000 | 20.0 | 2 | 0 |'
"""| pair | buy count | avg profit % | cum profit % | total profit BTC | avg duration | profit | loss |
|:--------|------------:|---------------:|---------------:|-------------------:|---------------:|---------:|-------:|
| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 | 20.0 | 2 | 0 |
| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 | 20.0 | 2 | 0 |"""
)
assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str
@ -598,6 +594,7 @@ def test_backtest_record(default_conf, fee, mocker):
results = backtesting.backtest(backtest_conf)
assert len(results) == 3
# Assert file_dump_json was only called once
print(names)
assert names == ['backtest-result.json']
records = records[0]
# Ensure records are of correct type

8
freqtrade/tests/strategy/test_strategy.py
View File

@ -28,10 +28,10 @@ def test_load_strategy(result):
def test_load_strategy_from_url(result):
resolver = StrategyResolver()
resolver._load_strategy('https://raw.githubusercontent.com/berlinguyinca'
'/freqtrade-trading-strategies'
'/master/user_data/strategies/Simple.py')
assert hasattr(resolver.strategy, 'populate_indicators')
resolver._load_strategy('https://freq.isaac.international/'
'dev/strategies/GBPAQEFGGWCMWVFU34P'
'MVGS4P2NJR4IDFNVI4LTCZAKJAD3JCXUMBI4J/AverageStrategy/code')
assert hasattr(resolver.strategy, 'minimal_roi')
assert 'adx' in resolver.strategy.populate_indicators(result)

1
freqtrade/tests/test_arguments.py
View File

@ -63,6 +63,7 @@ def test_scripts_options() -> None:
arguments = Arguments(['-p', 'ETH/BTC'], '')
arguments.scripts_options()
args = arguments.get_parsed_arg()
print(args.pair)
assert args.pair == 'ETH/BTC'

BIN
lib/libta_lib.a Normal file
View File

Binary file not shown.

35
lib/libta_lib.la Executable file
View File

@ -0,0 +1,35 @@
# libta_lib.la - a libtool library file
# Generated by ltmain.sh - GNU libtool 1.5.22 Debian 1.5.22-4 (1.1220.2.365 2005/12/18 22:14:06)
#
# Please DO NOT delete this file!
# It is necessary for linking the library.
# The name that we can dlopen(3).
dlname='libta_lib.so.0'
# Names of this library.
library_names='libta_lib.so.0.0.0 libta_lib.so.0 libta_lib.so'
# The name of the static archive.
old_library='libta_lib.a'
# Libraries that this one depends upon.
dependency_libs=' -lpthread -ldl'
# Version information for libta_lib.
current=0
age=0
revision=0
# Is this an already installed library?
installed=yes
# Should we warn about portability when linking against -modules?
shouldnotlink=no
# Files to dlopen/dlpreopen
dlopen=''
dlpreopen=''
# Directory that this library needs to be installed in:
libdir='/usr/local/lib'

1
lib/libta_lib.so.0 Symbolic link
View File

@ -0,0 +1 @@
libta_lib.so.0.0.0

BIN
lib/libta_lib.so.0.0.0 Executable file
View File

Binary file not shown.

18
requirements-aws.txt Normal file
View File

@ -0,0 +1,18 @@
ccxt==1.14.24
SQLAlchemy==1.2.7
arrow==0.12.1
cachetools==2.1.0
requests==2.18.4
urllib3==1.22
wrapt==1.10.11
pandas==0.23.0
scikit-learn==0.19.1
scipy==1.1.0
jsonschema==2.6.0
numpy==1.14.3
TA-Lib==0.4.17
git+git://github.com/berlinguyinca/networkx@v1.11
tabulate==0.8.2
coinmarketcap==5.0.3
simplejson==3.15.0
boto3

5
requirements.txt
View File

@ -17,9 +17,10 @@ pytest-mock==1.10.0
pytest-cov==2.5.1
hyperopt==0.1
# do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325
networkx==1.11 # pyup: ignore
#networkx==1.11
git+git://github.com/berlinguyinca/networkx@v1.11
tabulate==0.8.2
coinmarketcap==5.0.3
simplejson==3.15.0
# Required for plotting data
#plotly==2.3.0

337
serverless.yml Normal file
View File

@ -0,0 +1,337 @@
service: freq
frameworkVersion: ">=1.1.0 <2.0.0"
plugins:
- serverless-domain-manager
- serverless-python-requirements
############################################################################################
# configure out provider and the security guide lines
############################################################################################
provider:
name: aws
runtime: python3.6
region: us-east-2
#required permissions
iamRoleStatements:
- Effect: Allow
Action:
- dynamodb:*
Resource: "*"
- Effect: Allow
Action:
- SNS:*
Resource: { "Fn::Join" : [":", ["arn:aws:sns:${self:custom.region}", "*:*" ] ] }
- Effect: "Allow"
Action:
- ecs:RunTask
Resource: "*"
- Effect: Allow
Action:
- iam:PassRole
Resource: "*"
memorySize: 128
timeout: 90
versionFunctions: false
logRetentionInDays: 3
#where to store out data, needs to be manually created!
deploymentBucket:
name: lambdas-freq
# limit the invocations a bit to avoid overloading the server
usagePlan:
throttle:
burstLimit: 100
rateLimit: 50
############################################################################################
#custom configuration settings
############################################################################################
custom:
stage: ${opt:stage, self:provider.stage}
region: ${opt:region, self:provider.region}
snsTopic: "FreqQueue-${self:custom.stage}"
snsTradeTopic: "FreqTradeQueue-${self:custom.stage}"
tradeTable: "FreqTradesTable-${self:custom.stage}"
strategyTable: "FreqStrategyTable-${self:custom.stage}"
###
# custom domain management
###
customDomain:
basePath: "${self:custom.stage}"
domainName: "freq.isaac.international"
stage: "${self:custom.stage}"
createRoute53Record: true
pythonRequirements:
slim: true
invalidateCaches: true
dockerizePip: false
fileName: requirements-aws.txt
noDeploy:
- pytest
- moto
- plotly
- boto3
- pytest-mock
- pytest-cov
- pymongo
package:
exclude:
- test/**
- node_modules/**
- doc/**
- scripts/**
- bin
- freqtrade/tests/**
############################################################################################
# this section defines all lambda function and triggers
############################################################################################
functions:
#returns all known strategy names from the server
#and if they are private or not
strategies:
memorySize: 128
handler: freqtrade/aws/strategy.names
events:
- http:
path: strategies
method: get
cors: true
environment:
strategyTable: ${self:custom.strategyTable}
reservedConcurrency: 5
#returns the source code of this given strategy
#unless it's private
code:
memorySize: 128
handler: freqtrade/aws/strategy.code
events:
- http:
path: strategies/{user}/{name}/code
method: get
cors: true
integration: lambda
request:
parameter:
paths:
user: true
name: true
response:
headers:
Content-Type: "'text/plain'"
template: $input.path('$')
environment:
strategyTable: ${self:custom.strategyTable}
reservedConcurrency: 5
# loads the details of the specific strategy
get:
memorySize: 128
handler: freqtrade/aws/strategy.get
events:
- http:
path: strategies/{user}/{name}
method: get
cors: true
request:
parameter:
paths:
user: true
name: true
environment:
strategyTable: ${self:custom.strategyTable}
reservedConcurrency: 5
# loads the aggregation report for the given strategy based on different tickers
get_aggregate_interval:
memorySize: 128
handler: freqtrade/aws/aggregate/strategy.ticker
events:
- http:
path: strategies/{user}/{name}/aggregate/ticker
method: get
cors: true
request:
parameter:
paths:
user: true
name: true
environment:
strategyTable: ${self:custom.strategyTable}
tradeTable: ${self:custom.tradeTable}
reservedConcurrency: 5
# loads the aggregation report for the given strategy based on different tickers
get_aggregate_timeframe:
memorySize: 128
handler: freqtrade/aws/aggregate/strategy.timeframe
events:
- http:
path: strategies/{user}/{name}/aggregate/timeframe
method: get
cors: true
request:
parameter:
paths:
user: true
name: true
environment:
strategyTable: ${self:custom.strategyTable}
tradeTable: ${self:custom.tradeTable}
reservedConcurrency: 5
#submits a new strategy to the system
submit:
memorySize: 128
handler: freqtrade/aws/strategy.submit
events:
- http:
path: strategies/submit
method: post
cors: true
environment:
topic: ${self:custom.snsTopic}
strategyTable: ${self:custom.strategyTable}
BASE_URL: ${self:custom.customDomain.domainName}/${self:custom.customDomain.stage}
reservedConcurrency: 5
#submits a new strategy to the system
submit_github:
memorySize: 128
handler: freqtrade/aws/strategy.submit_github
events:
- http:
path: strategies/submit/github
method: post
cors: true
environment:
topic: ${self:custom.snsTopic}
strategyTable: ${self:custom.strategyTable}
reservedConcurrency: 1
### TRADE REQUESTS
# loads all trades for a strategy and it's associated pairs
trades:
memorySize: 128
handler: freqtrade/aws/trade.get_trades
events:
- http:
path: strategies/{user}/{name}/{stake}/{asset}
method: get
cors: true
request:
parameter:
paths:
user: true
name: true
stake: true
asset: true
environment:
strategyTable: ${self:custom.strategyTable}
tradeTable: ${self:custom.tradeTable}
reservedConcurrency: 5
# submits a new trade to the system
trade:
memorySize: 128
handler: freqtrade/aws/trade.submit
events:
- http:
path: trade
method: post
cors: true
environment:
tradeTopic: ${self:custom.snsTradeTopic}
reservedConcurrency: 5
# query aggregates by day and ticker for all strategies
trade-aggregate:
memorySize: 128
handler: freqtrade/aws/trade.get_aggregated_trades
events:
- http:
path: trades/aggregate/{ticker}/{days}
method: get
cors: true
request:
parameter:
paths:
ticker: true
days: true
environment:
tradeTable: ${self:custom.tradeTable}
reservedConcurrency: 5
### SNS TRIGGERED FUNCTIONS
# stores the received message in the trade table
trade-store:
memorySize: 128
handler: freqtrade/aws/trade.store
events:
- sns: ${self:custom.snsTradeTopic}
environment:
tradeTable: ${self:custom.tradeTable}
reservedConcurrency: 1
#backtests the strategy
#should be switched to utilze aws fargate instead
#and running a container
#so that we can evaluate long running tasks
backtest:
memorySize: 128
handler: freqtrade/aws/backtesting_lambda.backtest
events:
- sns: ${self:custom.snsTopic}
environment:
topic: ${self:custom.snsTopic}
tradeTable: ${self:custom.tradeTable}
strategyTable: ${self:custom.strategyTable}
BASE_URL: https://${self:custom.customDomain.domainName}/${self:custom.customDomain.stage}
reservedConcurrency: 1
# schedules all registered strategies on a daily base
schedule:
memorySize: 128
handler: freqtrade/aws/backtesting_lambda.cron
events:
- schedule:
rate: rate(1440 minutes)
enabled: true
environment:
topic: ${self:custom.snsTopic}
tradeTable: ${self:custom.tradeTable}
strategyTable: ${self:custom.strategyTable}
reservedConcurrency: 1

6
setup.py
View File

@ -19,7 +19,7 @@ setup(name='freqtrade',
packages=['freqtrade'],
scripts=['bin/freqtrade'],
setup_requires=['pytest-runner'],
tests_require=['pytest', 'pytest-mock', 'pytest-cov'],
tests_require=['pytest', 'pytest-mock', 'pytest-cov', 'moto'],
install_requires=[
'ccxt',
'SQLAlchemy',
@ -35,7 +35,9 @@ setup(name='freqtrade',
'TA-Lib',
'tabulate',
'cachetools',
'coinmarketcap'
'coinmarketcap',
'boto3'
],
include_package_data=True,
zip_safe=False,