Fix imports and handling of trade regarding changes from PR#9065

freqtrade/data/converter/__init__.py

@@ -1,7 +1,8 @@
-from freqtrade.data.converter.converter import (clean_ohlcv_dataframe, convert_ohlcv_format,
+from freqtrade.data.converter.converter import (clean_duplicate_trades, clean_ohlcv_dataframe, convert_ohlcv_format,
  ohlcv_fill_up_missing_data, ohlcv_to_dataframe,
- order_book_to_dataframe, reduce_dataframe_footprint,
- trim_dataframe, trim_dataframes)
+ order_book_to_dataframe,
+ populate_dataframe_with_trades, public_trades_to_dataframe,
+ reduce_dataframe_footprint, trim_dataframe, trim_dataframes)
 from freqtrade.data.converter.trade_converter import (convert_trades_format,
  convert_trades_to_ohlcv, trades_convert_types,
  trades_df_remove_duplicates,

freqtrade/data/converter/converter.py

@@ -9,9 +9,11 @@
 import numpy as np
 import pandas as pd
 from pandas import DataFrame, to_datetime
+import itertools
 
 from freqtrade.constants import DEFAULT_ORDERFLOW_COLUMNS, DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, Config, TradeList
 from freqtrade.enums import CandleType, TradingMode
+from freqtrade.data.converter.trade_converter import trades_df_remove_duplicates
 
 
 logger = logging.getLogger(__name__)
@@ -76,10 +78,11 @@ def _calculate_ohlcv_candle_start_and_end(df: DataFrame, timeframe: str):
  timeframe_frequency, timeframe_minutes = _convert_timeframe_to_pandas_frequency(
  timeframe)
  # calculate ohlcv candle start and end
- df['datetime'] = pd.to_datetime(df['date'], unit='ms')
- df['candle_start'] = df['datetime'].dt.floor(timeframe_frequency)
- df['candle_end'] = df['candle_start'] + pd.Timedelta(timeframe_minutes)
- df.drop(columns=['datetime'], inplace=True)
+ if df is not None and not df.empty:
+ df['datetime'] = pd.to_datetime(df['date'], unit='ms')
+ df['candle_start'] = df['datetime'].dt.floor(timeframe_frequency)
+ df['candle_end'] = df['candle_start'] + pd.Timedelta(timeframe_minutes)
+ df.drop(columns=['datetime'], inplace=True)
 
 
 def populate_dataframe_with_trades(config: Config, dataframe: DataFrame, trades: DataFrame, *, pair: str) -> DataFrame:

freqtrade/exchange/exchange.py

@@ -9,6 +9,7 @@
 import arrow
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
+import arrow
 from math import floor
 from threading import Lock
 from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union, Callable
@@ -2320,7 +2321,7 @@ def refresh_latest_trades(self,
  results_df = {}
  for pair, timeframe, candle_type in set(pair_list):
  new_ticks = []
- all_stored_ticks = []
+ all_stored_ticks_df = DataFrame(columns=DEFAULT_TRADES_COLUMNS + ['date'])
  first_candle_ms = self.needed_candle_ms(timeframe, candle_type)
  # refresh, if
  # a. not in _trades
@@ -2341,19 +2342,20 @@ def refresh_latest_trades(self,
 
  else: 
  until = int(timeframe_to_prev_date(timeframe).timestamp()) * 1000
- all_stored_ticks = data_handler.trades_load(f"{pair}-cached")
- if all_stored_ticks:
- if all_stored_ticks[0][0] <= first_candle_ms:
- last_cached_ms = all_stored_ticks[-1][0]
+ all_stored_ticks_df = data_handler.trades_load(f"{pair}-cached")
+
+ if not all_stored_ticks_df.empty:
+ if all_stored_ticks_df.iloc[0]['timestamp'] <= first_candle_ms:
+ last_cached_ms = all_stored_ticks_df.iloc[-1]['timestamp']
  # only use cached if it's closer than first_candle_ms
  since_ms = last_cached_ms if last_cached_ms > first_candle_ms else first_candle_ms
  # doesn't go far enough
  else: 
- all_stored_ticks = []
+ all_stored_ticks_df = DataFrame(columns=DEFAULT_TRADES_COLUMNS + ['date'])
 
  # from_id overrules with exchange set to id paginate
  # TODO: DEBUG:
- # since_ms = 1682609520000
+ # since_ms = 1695832200000
  # from_id = None
  # TODO: /DEBUG
  [ticks_pair, new_ticks]=self._download_trades_history(pair,
@@ -2368,12 +2370,14 @@ def refresh_latest_trades(self,
 
  if new_ticks:
  drop_incomplete = False # TODO: remove, no incomplete trades
- all_stored_ticks.extend(new_ticks)
+ # drop 'date' column from stored ticks
+ all_stored_ticks_list = all_stored_ticks_df[DEFAULT_TRADES_COLUMNS].values.tolist()
+ all_stored_ticks_list.extend(new_ticks)
  # NOTE: only process new trades
  # self._trades = until_first_candle(stored_trades) + fetch_trades
- trades_df = self._process_trades_df(pair, timeframe, candle_type, all_stored_ticks, cache, drop_incomplete, first_candle_ms)
+ trades_df = self._process_trades_df(pair, timeframe, candle_type, all_stored_ticks_list, cache, drop_incomplete, first_candle_ms)
  results_df[(pair, timeframe, candle_type)] = trades_df
- data_handler.trades_store(f"{pair}-cached", trades_df[DEFAULT_TRADES_COLUMNS].values.tolist())
+ data_handler.trades_store(f"{pair}-cached", trades_df[DEFAULT_TRADES_COLUMNS])
 
  return results_df