adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Do not use ticker where it's not a ticker

Browse Source
This commit is contained in:
hroff-1902
2020-03-08 13:35:31 +03:00
parent 77944175e2
commit 3208faf7ed
43 changed files with 459 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
freqtrade/data/btanalysis.py
View File

@@ -151,17 +151,17 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> p
return trades
def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame],
column: str = "close") -> pd.DataFrame:
def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
column: str = "close") -> pd.DataFrame:
"""
Combine multiple dataframes "column"
:param tickers: Dict of Dataframes, dict key should be pair.
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return: DataFrame with the column renamed to the dict key, and a column
named mean, containing the mean of all pairs.
"""
df_comb = pd.concat([tickers[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in tickers], axis=1)
df_comb = pd.concat([data[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in data], axis=1)
df_comb['mean'] = df_comb.mean(axis=1)

55
freqtrade/data/converter.py
View File

@@ -13,12 +13,12 @@ from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
logger = logging.getLogger(__name__)
def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *,
fill_missing: bool = True,
drop_incomplete: bool = True) -> DataFrame:
def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *,
fill_missing: bool = True, drop_incomplete: bool = True) -> DataFrame:
"""
Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe
:param ticker: ticker list, as returned by exchange.async_get_candle_history
Converts a list with candle (OHLCV) data (in format returned by ccxt.fetch_ohlcv)
to a Dataframe
:param ohlcv: list with candle (OHLCV) data, as returned by exchange.async_get_candle_history
:param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data
:param pair: Pair this data is for (used to warn if fillup was necessary)
:param fill_missing: fill up missing candles with 0 candles
@@ -26,21 +26,18 @@ def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *,
:param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete
:return: DataFrame
"""
logger.debug("Parsing tickerlist to dataframe")
logger.debug(f"Converting candle (OHLCV) data to dataframe for pair {pair}.")
cols = DEFAULT_DATAFRAME_COLUMNS
frame = DataFrame(ticker, columns=cols)
df = DataFrame(ohlcv, columns=cols)
frame['date'] = to_datetime(frame['date'],
unit='ms',
utc=True,
infer_datetime_format=True)
df['date'] = to_datetime(df['date'], unit='ms', utc=True, infer_datetime_format=True)
# Some exchanges return int values for volume and even for ohlc.
# Some exchanges return int values for Volume and even for OHLC.
# Convert them since TA-LIB indicators used in the strategy assume floats
# and fail with exception...
frame = frame.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float',
'volume': 'float'})
return clean_ohlcv_dataframe(frame, timeframe, pair,
df = df.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float',
'volume': 'float'})
return clean_ohlcv_dataframe(df, timeframe, pair,
fill_missing=fill_missing,
drop_incomplete=drop_incomplete)
@@ -49,11 +46,11 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *,
fill_missing: bool = True,
drop_incomplete: bool = True) -> DataFrame:
"""
Clense a ohlcv dataframe by
Clense a OHLCV dataframe by
* Grouping it by date (removes duplicate tics)
* dropping last candles if requested
* Filling up missing data (if requested)
:param data: DataFrame containing ohlcv data.
:param data: DataFrame containing candle (OHLCV) data.
:param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data
:param pair: Pair this data is for (used to warn if fillup was necessary)
:param fill_missing: fill up missing candles with 0 candles
@@ -88,16 +85,16 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
"""
from freqtrade.exchange import timeframe_to_minutes
ohlc_dict = {
ohlcv_dict = {
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'volume': 'sum'
}
ticker_minutes = timeframe_to_minutes(timeframe)
timeframe_minutes = timeframe_to_minutes(timeframe)
# Resample to create "NAN" values
df = dataframe.resample(f'{ticker_minutes}min', on='date').agg(ohlc_dict)
df = dataframe.resample(f'{timeframe_minutes}min', on='date').agg(ohlcv_dict)
# Forwardfill close for missing columns
df['close'] = df['close'].fillna(method='ffill')
@@ -159,20 +156,20 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
"""
Converts trades list to ohlcv list
Converts trades list to OHLCV list
TODO: This should get a dedicated test
:param trades: List of trades, as returned by ccxt.fetch_trades.
:param timeframe: Ticker timeframe to resample data to
:return: ohlcv Dataframe.
:param timeframe: Timeframe to resample data to
:return: OHLCV Dataframe.
"""
from freqtrade.exchange import timeframe_to_minutes
ticker_minutes = timeframe_to_minutes(timeframe)
timeframe_minutes = timeframe_to_minutes(timeframe)
df = pd.DataFrame(trades)
df['datetime'] = pd.to_datetime(df['datetime'])
df = df.set_index('datetime')
df_new = df['price'].resample(f'{ticker_minutes}min').ohlc()
df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum()
df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()
df_new['date'] = df_new.index
# Drop 0 volume rows
df_new = df_new.dropna()
@@ -206,7 +203,7 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to:
def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool):
"""
Convert ohlcv from one format to another format.
Convert OHLCV from one format to another
:param config: Config dictionary
:param convert_from: Source format
:param convert_to: Target format
@@ -216,7 +213,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
src = get_datahandler(config['datadir'], convert_from)
trg = get_datahandler(config['datadir'], convert_to)
timeframes = config.get('timeframes', [config.get('ticker_interval')])
logger.info(f"Converting OHLCV for timeframe {timeframes}")
logger.info(f"Converting candle (OHLCV) for timeframe {timeframes}")
if 'pairs' not in config:
config['pairs'] = []
@@ -224,7 +221,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
for timeframe in timeframes:
config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'],
timeframe))
logger.info(f"Converting OHLCV for {config['pairs']}")
logger.info(f"Converting candle (OHLCV) data for {config['pairs']}")
for timeframe in timeframes:
for pair in config['pairs']:

14
freqtrade/data/dataprovider.py
View File

@@ -1,7 +1,7 @@
"""
Dataprovider
Responsible to provide data to the bot
including Klines, tickers, historic data
including ticker and orderbook data, live and historical candle (OHLCV) data
Common Interface for bot and strategy to access data.
"""
import logging
@@ -43,10 +43,10 @@ class DataProvider:
def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
"""
Get ohlcv data for the given pair as DataFrame
Get candle (OHLCV) data for the given pair as DataFrame
Please use the `available_pairs` method to verify which pairs are currently cached.
:param pair: pair to get the data for
:param timeframe: Ticker timeframe to get data for
:param timeframe: Timeframe to get data for
:param copy: copy dataframe before returning if True.
Use False only for read-only operations (where the dataframe is not modified)
"""
@@ -58,7 +58,7 @@ class DataProvider:
def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame:
"""
Get stored historic ohlcv data
Get stored historical candle (OHLCV) data
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
"""
@@ -69,17 +69,17 @@ class DataProvider:
def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame:
"""
Return pair ohlcv data, either live or cached historical -- depending
Return pair candle (OHLCV) data, either live or cached historical -- depending
on the runmode.
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
:return: Dataframe for this pair
"""
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
# Get live ohlcv data.
# Get live OHLCV data.
data = self.ohlcv(pair=pair, timeframe=timeframe)
else:
# Get historic ohlcv data (cached on disk).
# Get historical OHLCV data (cached on disk).
data = self.historic_ohlcv(pair=pair, timeframe=timeframe)
if len(data) == 0:
logger.warning(f"No data found for ({pair}, {timeframe}).")

22
freqtrade/data/history/history_utils.py
View File

@@ -9,7 +9,7 @@ from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
from freqtrade.data.converter import ohlcv_to_dataframe, trades_to_ohlcv
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
@@ -28,10 +28,10 @@ def load_pair_history(pair: str,
data_handler: IDataHandler = None,
) -> DataFrame:
"""
Load cached ticker history for the given pair.
Load cached ohlcv history for the given pair.
:param pair: Pair to load data for
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param datadir: Path to the data storage location.
:param data_format: Format of the data. Ignored if data_handler is set.
:param timerange: Limit data to be loaded to this timerange
@@ -63,10 +63,10 @@ def load_data(datadir: Path,
data_format: str = 'json',
) -> Dict[str, DataFrame]:
"""
Load ticker history data for a list of pairs.
Load ohlcv history data for a list of pairs.
:param datadir: Path to the data storage location.
:param timeframe: Ticker Timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param pairs: List of pairs to load
:param timerange: Limit data to be loaded to this timerange
:param fill_up_missing: Fill missing values with "No action"-candles
@@ -104,10 +104,10 @@ def refresh_data(datadir: Path,
timerange: Optional[TimeRange] = None,
) -> None:
"""
Refresh ticker history data for a list of pairs.
Refresh ohlcv history data for a list of pairs.
:param datadir: Path to the data storage location.
:param timeframe: Ticker Timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param pairs: List of pairs to load
:param exchange: Exchange object
:param timerange: Limit data to be loaded to this timerange
@@ -165,7 +165,7 @@ def _download_pair_history(datadir: Path,
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param timeframe: Ticker Timeframe (e.g 5m)
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
:return: bool with success state
"""
@@ -194,8 +194,8 @@ def _download_pair_history(datadir: Path,
days=-30).float_timestamp) * 1000
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = parse_ticker_dataframe(new_data, timeframe, pair,
fill_missing=False, drop_incomplete=True)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
fill_missing=False, drop_incomplete=True)
if data.empty:
data = new_dataframe
else:
@@ -362,7 +362,7 @@ def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime,
:param pair: pair used for log output.
:param min_date: start-date of the data
:param max_date: end-date of the data
:param timeframe_min: ticker Timeframe in minutes
:param timeframe_min: Timeframe in minutes
"""
# total difference in minutes / timeframe-minutes
expected_frames = int((max_date - min_date).total_seconds() // 60 // timeframe_min)

25
freqtrade/data/history/idatahandler.py
View File

@@ -55,7 +55,7 @@ class IDataHandler(ABC):
Implements the loading and conversion to a Pandas dataframe.
Timerange trimming and dataframe validation happens outside of this method.
:param pair: Pair to load data
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange.
Optionally implemented by subclasses to avoid loading
all data where possible.
@@ -67,7 +67,7 @@ class IDataHandler(ABC):
"""
Remove data for this pair
:param pair: Delete data for this pair.
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:return: True when deleted, false if file did not exist.
"""
@@ -129,10 +129,10 @@ class IDataHandler(ABC):
warn_no_data: bool = True
) -> DataFrame:
"""
Load cached ticker history for the given pair.
Load cached candle (OHLCV) data for the given pair.
:param pair: Pair to load data for
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange
:param fill_missing: Fill missing values with "No action"-candles
:param drop_incomplete: Drop last candle assuming it may be incomplete.
@@ -145,28 +145,27 @@ class IDataHandler(ABC):
if startup_candles > 0 and timerange_startup:
timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles)
pairdf = self._ohlcv_load(pair, timeframe,
timerange=timerange_startup)
if pairdf.empty:
df = self._ohlcv_load(pair, timeframe, timerange=timerange_startup)
if df.empty:
if warn_no_data:
logger.warning(
f'No history data for pair: "{pair}", timeframe: {timeframe}. '
'Use `freqtrade download-data` to download the data'
)
return pairdf
return df
else:
enddate = pairdf.iloc[-1]['date']
enddate = df.iloc[-1]['date']
if timerange_startup:
self._validate_pairdata(pair, pairdf, timerange_startup)
pairdf = trim_dataframe(pairdf, timerange_startup)
self._validate_pairdata(pair, df, timerange_startup)
df = trim_dataframe(df, timerange_startup)
# incomplete candles should only be dropped if we didn't trim the end beforehand.
return clean_ohlcv_dataframe(pairdf, timeframe,
return clean_ohlcv_dataframe(df, timeframe,
pair=pair,
fill_missing=fill_missing,
drop_incomplete=(drop_incomplete and
enddate == pairdf.iloc[-1]['date']))
enddate == df.iloc[-1]['date']))
def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange):
"""

4
freqtrade/data/history/jsondatahandler.py
View File

@@ -60,7 +60,7 @@ class JsonDataHandler(IDataHandler):
Implements the loading and conversion to a Pandas dataframe.
Timerange trimming and dataframe validation happens outside of this method.
:param pair: Pair to load data
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange.
Optionally implemented by subclasses to avoid loading
all data where possible.
@@ -83,7 +83,7 @@ class JsonDataHandler(IDataHandler):
"""
Remove data for this pair
:param pair: Delete data for this pair.
:param timeframe: Ticker timeframe (e.g. "5m")
:param timeframe: Timeframe (e.g. "5m")
:return: True when deleted, false if file did not exist.
"""
filename = self._pair_data_filename(self._datadir, pair, timeframe)