flake8 fix
This commit is contained in:
parent
404df7ae20
commit
3b9052247f
@ -121,16 +121,17 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
|
||||
logger.debug(message)
|
||||
return df
|
||||
|
||||
|
||||
def reduce_mem_usage(pair: str, df: DataFrame) -> DataFrame:
|
||||
""" iterate through all the columns of a dataframe and modify the data type
|
||||
to reduce memory usage.
|
||||
to reduce memory usage.
|
||||
"""
|
||||
# start_mem = df.memory_usage().sum() / 1024**2
|
||||
# logger.info(f"Memory usage of dataframe for {pair} is {start_mem:.2f} MB")
|
||||
|
||||
|
||||
for col in df.columns[1:]:
|
||||
col_type = df[col].dtype
|
||||
|
||||
|
||||
if col_type != object:
|
||||
c_min = df[col].min()
|
||||
c_max = df[col].max()
|
||||
@ -142,7 +143,7 @@ def reduce_mem_usage(pair: str, df: DataFrame) -> DataFrame:
|
||||
elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
|
||||
df[col] = df[col].astype(np.int32)
|
||||
elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
|
||||
df[col] = df[col].astype(np.int64)
|
||||
df[col] = df[col].astype(np.int64)
|
||||
elif str(col_type)[:5] == "float":
|
||||
if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
|
||||
df[col] = df[col].astype(np.float16)
|
||||
@ -158,9 +159,10 @@ def reduce_mem_usage(pair: str, df: DataFrame) -> DataFrame:
|
||||
# end_mem = df.memory_usage().sum() / 1024**2
|
||||
# logger.info("Memory usage after optimization is: {:.2f} MB".format(end_mem))
|
||||
# logger.info("Decreased by {:.1f}%".format(100 * (start_mem - end_mem) / start_mem))
|
||||
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
|
||||
startup_candles: int = 0) -> DataFrame:
|
||||
"""
|
||||
@ -196,10 +198,10 @@ def trim_dataframes(preprocessed: Dict[str, DataFrame], timerange,
|
||||
trimed_df = trim_dataframe(df, timerange, startup_candles=startup_candles)
|
||||
if not trimed_df.empty:
|
||||
# start_mem = trimed_df.memory_usage().sum() / 1024**2
|
||||
# logger.info(f"Memory usage of dataframe for {pair} before reduced is {start_mem:.2f} MB")
|
||||
# logger.info(f"Memory usage of df for {pair} before reduced is {start_mem:.2f} MB")
|
||||
trimed_df = reduce_mem_usage(pair, trimed_df)
|
||||
# end_mem = trimed_df.memory_usage().sum() / 1024**2
|
||||
# logger.info(f"Memory usage of dataframe for {pair} after reduced is {end_mem:.2f} MB")
|
||||
# logger.info(f"Memory usage of df for {pair} after reduced is {end_mem:.2f} MB")
|
||||
processed[pair] = trimed_df
|
||||
else:
|
||||
logger.warning(f'{pair} has no data left after adjusting for startup candles, '
|
||||
|
Loading…
Reference in New Issue
Block a user