This commit is contained in:
7LZL
2025-12-02 22:05:54 +08:00
commit 7fa249a767
18 changed files with 1045 additions and 0 deletions

0
sync/__init__.py Normal file
View File

183
sync/account_sync.py Normal file
View File

@@ -0,0 +1,183 @@
from .base_sync import BaseSync
from loguru import logger
from typing import List, Dict
import json
import time
from datetime import datetime, timedelta
class AccountSync(BaseSync):
"""账户信息同步器"""
async def sync(self):
"""同步账户信息数据"""
try:
# 获取所有账号
accounts = self.get_accounts_from_redis()
for k_id_str, account_info in accounts.items():
try:
k_id = int(k_id_str)
st_id = account_info.get('st_id', 0)
exchange_id = account_info['exchange_id']
if k_id <= 0 or st_id <= 0:
continue
# 从Redis获取账户信息数据
account_data = await self._get_account_info_from_redis(k_id, st_id, exchange_id)
# 同步到数据库
if account_data:
success = self._sync_account_info_to_db(account_data)
if success:
logger.debug(f"账户信息同步成功: k_id={k_id}")
except Exception as e:
logger.error(f"同步账号 {k_id_str} 账户信息失败: {e}")
continue
logger.info("账户信息同步完成")
except Exception as e:
logger.error(f"账户信息同步失败: {e}")
async def _get_account_info_from_redis(self, k_id: int, st_id: int, exchange_id: str) -> List[Dict]:
"""从Redis获取账户信息数据"""
try:
redis_key = f"{exchange_id}:balance:{k_id}"
redis_funds = self.redis_client.client.hgetall(redis_key)
if not redis_funds:
return []
# 按天统计数据
from config.settings import SYNC_CONFIG
recent_days = SYNC_CONFIG['recent_days']
today = datetime.now()
date_stats = {}
# 收集所有日期的数据
for fund_key, fund_json in redis_funds.items():
try:
fund_data = json.loads(fund_json)
date_str = fund_data.get('lz_time', '')
lz_type = fund_data.get('lz_type', '')
if not date_str or lz_type not in ['lz_balance', 'deposit', 'withdrawal']:
continue
# 只处理最近N天的数据
date_obj = datetime.strptime(date_str, '%Y-%m-%d')
if (today - date_obj).days > recent_days:
continue
if date_str not in date_stats:
date_stats[date_str] = {
'balance': 0.0,
'deposit': 0.0,
'withdrawal': 0.0,
'has_balance': False
}
lz_amount = float(fund_data.get('lz_amount', 0))
if lz_type == 'lz_balance':
date_stats[date_str]['balance'] = lz_amount
date_stats[date_str]['has_balance'] = True
elif lz_type == 'deposit':
date_stats[date_str]['deposit'] += lz_amount
elif lz_type == 'withdrawal':
date_stats[date_str]['withdrawal'] += lz_amount
except (json.JSONDecodeError, ValueError) as e:
logger.debug(f"解析Redis数据失败: {fund_key}, error={e}")
continue
# 转换为账户信息数据
account_data_list = []
sorted_dates = sorted(date_stats.keys())
prev_balance = 0.0
for date_str in sorted_dates:
stats = date_stats[date_str]
# 如果没有余额数据但有充提数据,仍然处理
if not stats['has_balance'] and stats['deposit'] == 0 and stats['withdrawal'] == 0:
continue
balance = stats['balance']
deposit = stats['deposit']
withdrawal = stats['withdrawal']
# 计算利润
profit = balance - deposit - withdrawal - prev_balance
# 转换时间戳
date_obj = datetime.strptime(date_str, '%Y-%m-%d')
time_timestamp = int(date_obj.timestamp())
account_data = {
'st_id': st_id,
'k_id': k_id,
'balance': balance,
'withdrawal': withdrawal,
'deposit': deposit,
'other': 0.0, # 暂时为0
'profit': profit,
'time': time_timestamp
}
account_data_list.append(account_data)
# 更新前一天的余额
if stats['has_balance']:
prev_balance = balance
return account_data_list
except Exception as e:
logger.error(f"获取Redis账户信息失败: k_id={k_id}, error={e}")
return []
def _sync_account_info_to_db(self, account_data_list: List[Dict]) -> bool:
"""同步账户信息到数据库"""
session = self.db_manager.get_session()
try:
with session.begin():
for account_data in account_data_list:
try:
# 查询是否已存在
existing = session.execute(
select(StrategyKX).where(
and_(
StrategyKX.k_id == account_data['k_id'],
StrategyKX.st_id == account_data['st_id'],
StrategyKX.time == account_data['time']
)
)
).scalar_one_or_none()
if existing:
# 更新
existing.balance = account_data['balance']
existing.withdrawal = account_data['withdrawal']
existing.deposit = account_data['deposit']
existing.other = account_data['other']
existing.profit = account_data['profit']
else:
# 插入
new_account = StrategyKX(**account_data)
session.add(new_account)
except Exception as e:
logger.error(f"处理账户数据失败: {account_data}, error={e}")
continue
return True
except Exception as e:
logger.error(f"同步账户信息到数据库失败: error={e}")
return False
finally:
session.close()

90
sync/base_sync.py Normal file
View File

@@ -0,0 +1,90 @@
from abc import ABC, abstractmethod
from loguru import logger
from typing import List, Dict, Any
import json
from utils.redis_client import RedisClient
from utils.database_manager import DatabaseManager
class BaseSync(ABC):
"""同步基类"""
def __init__(self):
self.redis_client = RedisClient()
self.db_manager = DatabaseManager()
self.computer_name = None # 从配置读取
@abstractmethod
async def sync(self):
"""执行同步"""
pass
def get_accounts_from_redis(self) -> Dict[str, Dict]:
"""从Redis获取账号配置"""
try:
if self.computer_name is None:
from config.settings import COMPUTER_NAME
self.computer_name = COMPUTER_NAME
# 从Redis获取数据
result = self.redis_client.client.hgetall(f"{self.computer_name}_strategy_api")
if not result:
logger.warning(f"未找到 {self.computer_name} 的策略API配置")
return {}
accounts_dict = {}
for exchange_name, accounts_json in result.items():
try:
accounts = json.loads(accounts_json)
if not accounts:
continue
# 格式化交易所ID
exchange_id = self.format_exchange_id(exchange_name)
for account_id, account_info in accounts.items():
parsed_account = self.parse_account(exchange_id, account_id, account_info)
if parsed_account:
accounts_dict[account_id] = parsed_account
except json.JSONDecodeError as e:
logger.error(f"解析交易所 {exchange_name} 的JSON数据失败: {e}")
continue
return accounts_dict
except Exception as e:
logger.error(f"获取账户信息失败: {e}")
return {}
def format_exchange_id(self, key: str) -> str:
"""格式化交易所ID"""
key = key.lower().strip()
# 交易所名称映射
exchange_mapping = {
'metatrader': 'mt5',
'binance_spot_test': 'binance',
'binance_spot': 'binance',
'binance': 'binance',
'gate_spot': 'gate',
'okex': 'okx'
}
return exchange_mapping.get(key, key)
def parse_account(self, exchange_id: str, account_id: str, account_info: str) -> Dict:
"""解析账号信息"""
try:
source_account_info = json.loads(account_info)
account_data = {
'exchange_id': exchange_id,
'k_id': account_id,
'st_id': int(source_account_info.get('st_id', 0)),
'add_time': int(source_account_info.get('add_time', 0))
}
return {**source_account_info, **account_data}
except json.JSONDecodeError as e:
logger.error(f"解析账号 {account_id} 数据失败: {e}")
return {}

66
sync/manager.py Normal file
View File

@@ -0,0 +1,66 @@
import asyncio
from loguru import logger
from typing import List, Dict
import signal
import sys
from config.settings import SYNC_CONFIG
from .position_sync import PositionSync
from .order_sync import OrderSync
from .account_sync import AccountSync
class SyncManager:
"""同步管理器"""
def __init__(self):
self.is_running = True
self.sync_interval = SYNC_CONFIG['interval']
# 初始化同步器
self.syncers = []
if SYNC_CONFIG['enable_position_sync']:
self.syncers.append(PositionSync())
logger.info("启用持仓同步")
if SYNC_CONFIG['enable_order_sync']:
self.syncers.append(OrderSync())
logger.info("启用订单同步")
if SYNC_CONFIG['enable_account_sync']:
self.syncers.append(AccountSync())
logger.info("启用账户信息同步")
# 注册信号处理器
signal.signal(signal.SIGINT, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)
def signal_handler(self, signum, frame):
"""信号处理器"""
logger.info(f"接收到信号 {signum},正在关闭...")
self.is_running = False
async def start(self):
"""启动同步服务"""
logger.info(f"同步服务启动,间隔 {self.sync_interval}")
while self.is_running:
try:
# 执行所有同步器
tasks = [syncer.sync() for syncer in self.syncers]
await asyncio.gather(*tasks, return_exceptions=True)
logger.debug(f"同步完成,等待 {self.sync_interval}")
await asyncio.sleep(self.sync_interval)
except asyncio.CancelledError:
logger.info("同步任务被取消")
break
except Exception as e:
logger.error(f"同步任务异常: {e}")
await asyncio.sleep(30) # 出错后等待30秒
async def stop(self):
"""停止同步服务"""
self.is_running = False
logger.info("同步服务停止")

166
sync/order_sync.py Normal file
View File

@@ -0,0 +1,166 @@
from .base_sync import BaseSync
from loguru import logger
from typing import List, Dict
import json
import time
from datetime import datetime, timedelta
class OrderSync(BaseSync):
"""订单数据同步器"""
async def sync(self):
"""同步订单数据"""
try:
# 获取所有账号
accounts = self.get_accounts_from_redis()
for k_id_str, account_info in accounts.items():
try:
k_id = int(k_id_str)
st_id = account_info.get('st_id', 0)
exchange_id = account_info['exchange_id']
if k_id <= 0 or st_id <= 0:
continue
# 从Redis获取最近N天的订单数据
orders = await self._get_recent_orders_from_redis(k_id, exchange_id)
# 同步到数据库
if orders:
success = self._sync_orders_to_db(k_id, st_id, orders)
if success:
logger.debug(f"订单同步成功: k_id={k_id}, 订单数={len(orders)}")
except Exception as e:
logger.error(f"同步账号 {k_id_str} 订单失败: {e}")
continue
logger.info("订单数据同步完成")
except Exception as e:
logger.error(f"订单同步失败: {e}")
async def _get_recent_orders_from_redis(self, k_id: int, exchange_id: str) -> List[Dict]:
"""从Redis获取最近N天的订单数据"""
try:
redis_key = f"{exchange_id}:orders:{k_id}"
# 计算最近N天的日期
from config.settings import SYNC_CONFIG
recent_days = SYNC_CONFIG['recent_days']
today = datetime.now()
recent_dates = []
for i in range(recent_days):
date = today - timedelta(days=i)
date_format = date.strftime('%Y-%m-%d')
recent_dates.append(date_format)
# 获取所有key
all_keys = self.redis_client.client.hkeys(redis_key)
orders_list = []
for key in all_keys:
key_str = key.decode('utf-8') if isinstance(key, bytes) else key
if key_str == 'positions':
continue
# 检查是否以最近N天的日期开头
for date_format in recent_dates:
if key_str.startswith(date_format + '_'):
try:
order_json = self.redis_client.client.hget(redis_key, key_str)
if order_json:
order = json.loads(order_json)
# 验证时间
order_time = order.get('time', 0)
if order_time >= int(time.time()) - recent_days * 24 * 3600:
orders_list.append(order)
break
except:
break
return orders_list
except Exception as e:
logger.error(f"获取Redis订单数据失败: k_id={k_id}, error={e}")
return []
def _sync_orders_to_db(self, k_id: int, st_id: int, orders_data: List[Dict]) -> bool:
"""同步订单数据到数据库"""
session = self.db_manager.get_session()
try:
# 准备批量数据
insert_data = []
for order_data in orders_data:
try:
order_dict = self._convert_order_data(order_data)
# 检查完整性
required_fields = ['order_id', 'symbol', 'side', 'time']
if not all(order_dict.get(field) for field in required_fields):
continue
insert_data.append(order_dict)
except Exception as e:
logger.error(f"转换订单数据失败: {order_data}, error={e}")
continue
if not insert_data:
return True
with session.begin():
# 使用参数化批量插入
sql = """
INSERT INTO deh_strategy_order_new
(st_id, k_id, asset, order_id, symbol, side, price, time,
order_qty, last_qty, avg_price, exchange_id)
VALUES
(:st_id, :k_id, :asset, :order_id, :symbol, :side, :price, :time,
:order_qty, :last_qty, :avg_price, :exchange_id)
ON DUPLICATE KEY UPDATE
side = VALUES(side),
price = VALUES(price),
time = VALUES(time),
order_qty = VALUES(order_qty),
last_qty = VALUES(last_qty),
avg_price = VALUES(avg_price)
"""
# 分块执行
from config.settings import SYNC_CONFIG
chunk_size = SYNC_CONFIG['chunk_size']
for i in range(0, len(insert_data), chunk_size):
chunk = insert_data[i:i + chunk_size]
session.execute(text(sql), chunk)
return True
except Exception as e:
logger.error(f"同步订单到数据库失败: k_id={k_id}, error={e}")
return False
finally:
session.close()
def _convert_order_data(self, data: Dict) -> Dict:
"""转换订单数据格式"""
return {
'st_id': int(data.get('st_id', 0)),
'k_id': int(data.get('k_id', 0)),
'asset': 'USDT',
'order_id': str(data.get('order_id', '')),
'symbol': data.get('symbol', ''),
'side': data.get('side', ''),
'price': float(data.get('price', 0)) if data.get('price') is not None else None,
'time': int(data.get('time', 0)) if data.get('time') is not None else None,
'order_qty': float(data.get('order_qty', 0)) if data.get('order_qty') is not None else None,
'last_qty': float(data.get('last_qty', 0)) if data.get('last_qty') is not None else None,
'avg_price': float(data.get('avg_price', 0)) if data.get('avg_price') is not None else None,
'exchange_id': None # 忽略该字段
}

174
sync/position_sync.py Normal file
View File

@@ -0,0 +1,174 @@
from .base_sync import BaseSync
from loguru import logger
from typing import List, Dict
import json
from datetime import datetime, timedelta
class PositionSync(BaseSync):
"""持仓数据同步器"""
async def sync(self):
"""同步持仓数据"""
try:
# 获取所有账号
accounts = self.get_accounts_from_redis()
for k_id_str, account_info in accounts.items():
try:
k_id = int(k_id_str)
st_id = account_info.get('st_id', 0)
exchange_id = account_info['exchange_id']
if k_id <= 0 or st_id <= 0:
continue
# 从Redis获取持仓数据
positions = await self._get_positions_from_redis(k_id, exchange_id)
# 同步到数据库
if positions:
success = self._sync_positions_to_db(k_id, st_id, positions)
if success:
logger.debug(f"持仓同步成功: k_id={k_id}, 持仓数={len(positions)}")
except Exception as e:
logger.error(f"同步账号 {k_id_str} 持仓失败: {e}")
continue
logger.info("持仓数据同步完成")
except Exception as e:
logger.error(f"持仓同步失败: {e}")
async def _get_positions_from_redis(self, k_id: int, exchange_id: str) -> List[Dict]:
"""从Redis获取持仓数据"""
try:
redis_key = f"{exchange_id}:positions:{k_id}"
redis_data = self.redis_client.client.hget(redis_key, 'positions')
if not redis_data:
return []
positions = json.loads(redis_data)
# 添加账号信息
for position in positions:
position['k_id'] = k_id
return positions
except Exception as e:
logger.error(f"获取Redis持仓数据失败: k_id={k_id}, error={e}")
return []
def _sync_positions_to_db(self, k_id: int, st_id: int, positions_data: List[Dict]) -> bool:
"""同步持仓数据到数据库"""
session = self.db_manager.get_session()
try:
# 使用批量优化方案
from sqlalchemy.dialects.mysql import insert
# 准备数据
insert_data = []
keep_keys = set() # 需要保留的(symbol, side)组合
for pos_data in positions_data:
try:
# 转换数据(这里需要实现转换逻辑)
pos_dict = self._convert_position_data(pos_data)
if not all([pos_dict.get('symbol'), pos_dict.get('side')]):
continue
# 重命名qty为sum
if 'qty' in pos_dict:
pos_dict['sum'] = pos_dict.pop('qty')
insert_data.append(pos_dict)
keep_keys.add((pos_dict['symbol'], pos_dict['side']))
except Exception as e:
logger.error(f"转换持仓数据失败: {pos_data}, error={e}")
continue
with session.begin():
if not insert_data:
# 清空该账号持仓
session.execute(
delete(StrategyPosition).where(
and_(
StrategyPosition.k_id == k_id,
StrategyPosition.st_id == st_id
)
)
)
return True
# 批量插入/更新
stmt = insert(StrategyPosition.__table__).values(insert_data)
update_dict = {
'price': stmt.inserted.price,
'sum': stmt.inserted.sum,
'asset_num': stmt.inserted.asset_num,
'asset_profit': stmt.inserted.asset_profit,
'leverage': stmt.inserted.leverage,
'uptime': stmt.inserted.uptime,
'profit_price': stmt.inserted.profit_price,
'stop_price': stmt.inserted.stop_price,
'liquidation_price': stmt.inserted.liquidation_price
}
stmt = stmt.on_duplicate_key_update(**update_dict)
session.execute(stmt)
# 删除多余持仓
if keep_keys:
existing_positions = session.execute(
select(StrategyPosition).where(
and_(
StrategyPosition.k_id == k_id,
StrategyPosition.st_id == st_id
)
)
).scalars().all()
to_delete_ids = []
for existing in existing_positions:
key = (existing.symbol, existing.side)
if key not in keep_keys:
to_delete_ids.append(existing.id)
if to_delete_ids:
session.execute(
delete(StrategyPosition).where(
StrategyPosition.id.in_(to_delete_ids)
)
)
return True
except Exception as e:
logger.error(f"同步持仓到数据库失败: k_id={k_id}, error={e}")
return False
finally:
session.close()
def _convert_position_data(self, data: Dict) -> Dict:
"""转换持仓数据格式"""
# 这里实现具体的转换逻辑
return {
'st_id': int(data.get('st_id', 0)),
'k_id': int(data.get('k_id', 0)),
'asset': 'USDT',
'symbol': data.get('symbol', ''),
'side': data.get('side', ''),
'price': float(data.get('price', 0)) if data.get('price') is not None else None,
'qty': float(data.get('qty', 0)) if data.get('qty') is not None else None,
'asset_num': float(data.get('asset_num', 0)) if data.get('asset_num') is not None else None,
'asset_profit': float(data.get('asset_profit', 0)) if data.get('asset_profit') is not None else None,
'leverage': int(data.get('leverage', 0)) if data.get('leverage') is not None else None,
'uptime': int(data.get('uptime', 0)) if data.get('uptime') is not None else None,
'profit_price': float(data.get('profit_price', 0)) if data.get('profit_price') is not None else None,
'stop_price': float(data.get('stop_price', 0)) if data.get('stop_price') is not None else None,
'liquidation_price': float(data.get('liquidation_price', 0)) if data.get('liquidation_price') is not None else None
}