Spaces:
Running
Running
Update data_manager.py
Browse files- data_manager.py +17 -45
data_manager.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# data_manager.py (Updated to V7.
|
| 2 |
import os
|
| 3 |
import asyncio
|
| 4 |
import httpx
|
|
@@ -52,7 +52,7 @@ class DataManager:
|
|
| 52 |
async def initialize(self):
|
| 53 |
self.http_client = httpx.AsyncClient(timeout=30.0)
|
| 54 |
await self._load_markets()
|
| 55 |
-
print("✅ DataManager initialized - V7.
|
| 56 |
|
| 57 |
async def _load_markets(self):
|
| 58 |
try:
|
|
@@ -80,7 +80,6 @@ class DataManager:
|
|
| 80 |
except Exception as e:
|
| 81 |
print(f" ⚠️ DataManager: Error closing ccxt.kucoin: {e}")
|
| 82 |
|
| 83 |
-
# (الدوال المساعدة لسياق السوق وأسعار BTC/ETH تبقى كما هي - لا تغيير)
|
| 84 |
async def get_market_context_async(self):
|
| 85 |
try:
|
| 86 |
sentiment_data = await self.get_sentiment_safe_async()
|
|
@@ -198,7 +197,6 @@ class DataManager:
|
|
| 198 |
'data_quality': 'LOW'
|
| 199 |
}
|
| 200 |
|
| 201 |
-
# 🔴 --- START OF REFACTOR (V7.2 - Fixed Bugs) --- 🔴
|
| 202 |
|
| 203 |
def _create_dataframe(self, candles: List) -> pd.DataFrame:
|
| 204 |
"""(V7.1) دالة مساعدة لإنشاء DataFrame لتحليل 1H"""
|
|
@@ -220,18 +218,15 @@ class DataManager:
|
|
| 220 |
يحتوي الآن على "واقي العملات المستقرة"
|
| 221 |
"""
|
| 222 |
try:
|
| 223 |
-
#
|
| 224 |
-
# (يتم تشغيله قبل أي حساب آخر)
|
| 225 |
if 'ohlcv_1h' in analysis and '1h' in analysis['ohlcv_1h']:
|
| 226 |
closes_1h = [c[4] for c in analysis['ohlcv_1h']['1h']]
|
| 227 |
if len(closes_1h) > 20:
|
| 228 |
-
# (نستخدم آخر 20 شمعة للتحقق من التقلب)
|
| 229 |
std_dev = np.std(closes_1h[-20:])
|
| 230 |
-
if std_dev < 1e-5:
|
| 231 |
print(f" - {analysis.get('symbol', 'N/A')}: تم الاستبعاد (عملة مستقرة)")
|
| 232 |
-
return 0.0
|
| 233 |
-
|
| 234 |
-
|
| 235 |
# 1. درجة الأنماط (Pattern Score)
|
| 236 |
pattern_confidence = analysis.get('pattern_analysis', {}).get('pattern_confidence', 0)
|
| 237 |
|
|
@@ -239,7 +234,7 @@ class DataManager:
|
|
| 239 |
mc_distribution = analysis.get('monte_carlo_distribution')
|
| 240 |
monte_carlo_score = 0
|
| 241 |
|
| 242 |
-
if mc_distribution and mc_distribution.get('error') is None:
|
| 243 |
prob_gain = mc_distribution.get('probability_of_gain', 0)
|
| 244 |
var_95_value = mc_distribution.get('risk_metrics', {}).get('VaR_95_value', 0)
|
| 245 |
current_price = analysis.get('current_price', 1)
|
|
@@ -262,10 +257,8 @@ class DataManager:
|
|
| 262 |
ema_9 = indicators.get('ema_9', 0)
|
| 263 |
ema_21 = indicators.get('ema_21', 0)
|
| 264 |
|
| 265 |
-
# ملمح الزخم
|
| 266 |
if rsi > 55 and macd_hist > 0 and ema_9 > ema_21:
|
| 267 |
indicator_score = min(0.5 + (rsi - 55) / 50 + (macd_hist / (analysis.get('current_price', 1) * 0.001)), 1.0)
|
| 268 |
-
# ملمح الانعكاس
|
| 269 |
elif rsi < 35:
|
| 270 |
indicator_score = min(0.4 + (35 - rsi) / 35, 0.8)
|
| 271 |
|
|
@@ -291,9 +284,9 @@ class DataManager:
|
|
| 291 |
|
| 292 |
async def layer1_rapid_screening(self) -> List[Dict[str, Any]]:
|
| 293 |
"""
|
| 294 |
-
الطبقة 1: فحص سريع - (محدث بالكامل V7.
|
| 295 |
"""
|
| 296 |
-
print("📊 الطبقة 1 (V7.
|
| 297 |
|
| 298 |
# الخطوة 1: جلب أفضل 100 عملة حسب الحجم
|
| 299 |
volume_data = await self._get_volume_data_optimal()
|
|
@@ -332,7 +325,6 @@ class DataManager:
|
|
| 332 |
if isinstance(candles, Exception) or not candles or len(candles) < 50:
|
| 333 |
continue
|
| 334 |
|
| 335 |
-
# (إعداد البيانات للمحللات)
|
| 336 |
ohlcv_1h_only = {'1h': candles}
|
| 337 |
symbol_data['ohlcv_1h'] = ohlcv_1h_only
|
| 338 |
symbol_data['current_price'] = candles[-1][4]
|
|
@@ -352,15 +344,15 @@ class DataManager:
|
|
| 352 |
print(f" - {symbol}: فشل الكاشف المصغر ({analysis_output})")
|
| 353 |
continue
|
| 354 |
|
| 355 |
-
# (تمرير ohlcv_1h إلى دالة حساب النقاط من أجل واقي العملات المستقرة)
|
| 356 |
analysis_output['ohlcv_1h'] = symbol_data['ohlcv_1h']
|
| 357 |
analysis_output['symbol'] = symbol
|
| 358 |
|
| 359 |
-
# الخطوة 4: حساب النتيجة النهائية للفلتر
|
| 360 |
filter_score = self._calculate_1h_filter_score(analysis_output)
|
| 361 |
|
| 362 |
-
#
|
| 363 |
-
|
|
|
|
|
|
|
| 364 |
print(f" ✅ {symbol}: نجح (الدرجة: {filter_score:.2f})")
|
| 365 |
symbol_data['layer1_score'] = filter_score
|
| 366 |
symbol_data['reasons_for_candidacy'] = [f'1H_DETECTOR_PASS']
|
|
@@ -368,11 +360,8 @@ class DataManager:
|
|
| 368 |
if 'ohlcv_1h' in symbol_data: del symbol_data['ohlcv_1h']
|
| 369 |
|
| 370 |
final_candidates.append(symbol_data)
|
| 371 |
-
# else:
|
| 372 |
-
# (تم إيقاف طباعة الفشل لتقليل التشويش)
|
| 373 |
-
# print(f" - {symbol}: فشل (الدرجة: {filter_score:.2f})")
|
| 374 |
|
| 375 |
-
print(f"🎯 اكتملت الغربلة (V7.
|
| 376 |
|
| 377 |
print("🏆 المرشحون الناجحون:")
|
| 378 |
for k, candidate in enumerate(final_candidates[:15]):
|
|
@@ -393,7 +382,6 @@ class DataManager:
|
|
| 393 |
|
| 394 |
analysis_dict = {'current_price': current_price}
|
| 395 |
|
| 396 |
-
# (ملاحظة: جعلنا دوال التحليل async في ملفاتها لتكون متوافقة)
|
| 397 |
task_indicators = self.technical_analyzer.calculate_all_indicators(df, '1h')
|
| 398 |
task_mc = self.monte_carlo_analyzer.generate_1h_price_distribution(ohlcv_1h)
|
| 399 |
task_pattern = self.pattern_analyzer.detect_chart_patterns(ohlcv_1h)
|
|
@@ -421,7 +409,6 @@ class DataManager:
|
|
| 421 |
except Exception:
|
| 422 |
return None
|
| 423 |
|
| 424 |
-
# (دوال جلب الحجم تبقى كما هي لأنها فعالة)
|
| 425 |
async def _get_volume_data_optimal(self) -> List[Dict[str, Any]]:
|
| 426 |
try:
|
| 427 |
if not self.exchange: return []
|
|
@@ -440,11 +427,8 @@ class DataManager:
|
|
| 440 |
dollar_volume = base_volume * current_price
|
| 441 |
if dollar_volume is None or dollar_volume < 50000: continue
|
| 442 |
|
| 443 |
-
# 🔴 --- (FIX V7.2) إصلاح نسبة التغير المئوية --- 🔴
|
| 444 |
-
# (ccxt 'percentage' هي النسبة الفعلية، لا تحتاج للضرب في 100)
|
| 445 |
price_change_24h = ticker.get('percentage', 0) or 0
|
| 446 |
if price_change_24h is None: price_change_24h = 0
|
| 447 |
-
# 🔴 --- نهاية الإصلاح --- 🔴
|
| 448 |
|
| 449 |
volume_data.append({
|
| 450 |
'symbol': symbol, 'dollar_volume': dollar_volume,
|
|
@@ -479,10 +463,7 @@ class DataManager:
|
|
| 479 |
if vol_value is None or last_price is None or change_rate is None or vol is None: continue
|
| 480 |
dollar_volume = float(vol_value) if vol_value else 0
|
| 481 |
current_price = float(last_price) if last_price else 0
|
| 482 |
-
|
| 483 |
-
# (الإصلاح هنا أيضاً: 'changeRate' هو النسبة العشرية، لذا نضرب في 100)
|
| 484 |
price_change = (float(change_rate) * 100) if change_rate else 0
|
| 485 |
-
|
| 486 |
volume_24h = float(vol) if vol else 0
|
| 487 |
if dollar_volume >= 50000 and current_price > 0:
|
| 488 |
volume_data.append({
|
|
@@ -497,7 +478,6 @@ class DataManager:
|
|
| 497 |
print(f"❌ خطأ في جلب بيانات الحجم المباشر: {e}")
|
| 498 |
return []
|
| 499 |
|
| 500 |
-
# (دالة تدفق الشموع تبقى كما هي - لا تغيير)
|
| 501 |
async def stream_ohlcv_data(self, symbols: List[Dict[str, Any]], queue: asyncio.Queue):
|
| 502 |
"""
|
| 503 |
(محدث V7.2)
|
|
@@ -515,13 +495,11 @@ class DataManager:
|
|
| 515 |
|
| 516 |
batch_tasks = []
|
| 517 |
|
| 518 |
-
#
|
| 519 |
for symbol_data in batch:
|
| 520 |
-
symbol_str = symbol_data['symbol']
|
| 521 |
-
# (تمرير النص فقط إلى الدالة التي تتوقع نصاً)
|
| 522 |
task = asyncio.create_task(self._fetch_complete_ohlcv_parallel(symbol_str))
|
| 523 |
batch_tasks.append(task)
|
| 524 |
-
# 🔴 --- نهاية الإصلاح --- 🔴
|
| 525 |
|
| 526 |
batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True)
|
| 527 |
|
|
@@ -529,15 +507,12 @@ class DataManager:
|
|
| 529 |
successful_count = 0
|
| 530 |
for i, result in enumerate(batch_results):
|
| 531 |
|
| 532 |
-
# 🔴 --- (FIX V7.2) مطابقة النتائج بالبيانات الأصلية --- 🔴
|
| 533 |
original_symbol_data = batch[i]
|
| 534 |
symbol_str = original_symbol_data['symbol']
|
| 535 |
-
# 🔴 --- نهاية الإصلاح --- 🔴
|
| 536 |
|
| 537 |
if isinstance(result, Exception):
|
| 538 |
print(f" ❌ [المنتج] فشل جلب {symbol_str}: {result}")
|
| 539 |
elif result is not None:
|
| 540 |
-
# (دمج الدرجة الأولية مع البيانات الكاملة)
|
| 541 |
result.update(original_symbol_data)
|
| 542 |
successful_data_for_batch.append(result)
|
| 543 |
successful_count += 1
|
|
@@ -580,7 +555,6 @@ class DataManager:
|
|
| 580 |
|
| 581 |
timeframe_tasks = []
|
| 582 |
for timeframe, limit in timeframes:
|
| 583 |
-
# (هنا 'symbol' هو نص، وهو صحيح)
|
| 584 |
task = asyncio.create_task(self._fetch_single_timeframe_improved(symbol, timeframe, limit))
|
| 585 |
timeframe_tasks.append(task)
|
| 586 |
|
|
@@ -624,7 +598,6 @@ class DataManager:
|
|
| 624 |
retry_delay = 2
|
| 625 |
for attempt in range(max_retries):
|
| 626 |
try:
|
| 627 |
-
# (هنا 'symbol' هو نص، وهو صحيح)
|
| 628 |
ohlcv_data = self.exchange.fetch_ohlcv(symbol, timeframe, limit=limit)
|
| 629 |
if ohlcv_data and len(ohlcv_data) > 0:
|
| 630 |
return ohlcv_data
|
|
@@ -648,7 +621,6 @@ class DataManager:
|
|
| 648 |
return float(current_price)
|
| 649 |
except Exception as e: return None
|
| 650 |
|
| 651 |
-
# (دوال دعم بيانات الحيتان تبقى كما هي - لا تغيير)
|
| 652 |
async def get_whale_data_for_symbol(self, symbol):
|
| 653 |
try:
|
| 654 |
if self.whale_monitor:
|
|
@@ -666,4 +638,4 @@ class DataManager:
|
|
| 666 |
except Exception as e:
|
| 667 |
return {'action': 'HOLD', 'confidence': 0.3, 'reason': f'Error: {str(e)}', 'source': 'whale_analysis'}
|
| 668 |
|
| 669 |
-
print("✅ DataManager loaded - V7.
|
|
|
|
| 1 |
+
# data_manager.py (Updated to V7.3 - Increased L1 Threshold to 0.50)
|
| 2 |
import os
|
| 3 |
import asyncio
|
| 4 |
import httpx
|
|
|
|
| 52 |
async def initialize(self):
|
| 53 |
self.http_client = httpx.AsyncClient(timeout=30.0)
|
| 54 |
await self._load_markets()
|
| 55 |
+
print("✅ DataManager initialized - V7.3 (L1 Threshold @ 0.50)")
|
| 56 |
|
| 57 |
async def _load_markets(self):
|
| 58 |
try:
|
|
|
|
| 80 |
except Exception as e:
|
| 81 |
print(f" ⚠️ DataManager: Error closing ccxt.kucoin: {e}")
|
| 82 |
|
|
|
|
| 83 |
async def get_market_context_async(self):
|
| 84 |
try:
|
| 85 |
sentiment_data = await self.get_sentiment_safe_async()
|
|
|
|
| 197 |
'data_quality': 'LOW'
|
| 198 |
}
|
| 199 |
|
|
|
|
| 200 |
|
| 201 |
def _create_dataframe(self, candles: List) -> pd.DataFrame:
|
| 202 |
"""(V7.1) دالة مساعدة لإنشاء DataFrame لتحليل 1H"""
|
|
|
|
| 218 |
يحتوي الآن على "واقي العملات المستقرة"
|
| 219 |
"""
|
| 220 |
try:
|
| 221 |
+
# (V7.2) واقي العملات المستقرة
|
|
|
|
| 222 |
if 'ohlcv_1h' in analysis and '1h' in analysis['ohlcv_1h']:
|
| 223 |
closes_1h = [c[4] for c in analysis['ohlcv_1h']['1h']]
|
| 224 |
if len(closes_1h) > 20:
|
|
|
|
| 225 |
std_dev = np.std(closes_1h[-20:])
|
| 226 |
+
if std_dev < 1e-5:
|
| 227 |
print(f" - {analysis.get('symbol', 'N/A')}: تم الاستبعاد (عملة مستقرة)")
|
| 228 |
+
return 0.0
|
| 229 |
+
|
|
|
|
| 230 |
# 1. درجة الأنماط (Pattern Score)
|
| 231 |
pattern_confidence = analysis.get('pattern_analysis', {}).get('pattern_confidence', 0)
|
| 232 |
|
|
|
|
| 234 |
mc_distribution = analysis.get('monte_carlo_distribution')
|
| 235 |
monte_carlo_score = 0
|
| 236 |
|
| 237 |
+
if mc_distribution and mc_distribution.get('error') is None:
|
| 238 |
prob_gain = mc_distribution.get('probability_of_gain', 0)
|
| 239 |
var_95_value = mc_distribution.get('risk_metrics', {}).get('VaR_95_value', 0)
|
| 240 |
current_price = analysis.get('current_price', 1)
|
|
|
|
| 257 |
ema_9 = indicators.get('ema_9', 0)
|
| 258 |
ema_21 = indicators.get('ema_21', 0)
|
| 259 |
|
|
|
|
| 260 |
if rsi > 55 and macd_hist > 0 and ema_9 > ema_21:
|
| 261 |
indicator_score = min(0.5 + (rsi - 55) / 50 + (macd_hist / (analysis.get('current_price', 1) * 0.001)), 1.0)
|
|
|
|
| 262 |
elif rsi < 35:
|
| 263 |
indicator_score = min(0.4 + (35 - rsi) / 35, 0.8)
|
| 264 |
|
|
|
|
| 284 |
|
| 285 |
async def layer1_rapid_screening(self) -> List[Dict[str, Any]]:
|
| 286 |
"""
|
| 287 |
+
الطبقة 1: فحص سريع - (محدث بالكامل V7.3)
|
| 288 |
"""
|
| 289 |
+
print("📊 الطبقة 1 (V7.3): بدء الغربلة (الكاشف المصغر 1H)...")
|
| 290 |
|
| 291 |
# الخطوة 1: جلب أفضل 100 عملة حسب الحجم
|
| 292 |
volume_data = await self._get_volume_data_optimal()
|
|
|
|
| 325 |
if isinstance(candles, Exception) or not candles or len(candles) < 50:
|
| 326 |
continue
|
| 327 |
|
|
|
|
| 328 |
ohlcv_1h_only = {'1h': candles}
|
| 329 |
symbol_data['ohlcv_1h'] = ohlcv_1h_only
|
| 330 |
symbol_data['current_price'] = candles[-1][4]
|
|
|
|
| 344 |
print(f" - {symbol}: فشل الكاشف المصغر ({analysis_output})")
|
| 345 |
continue
|
| 346 |
|
|
|
|
| 347 |
analysis_output['ohlcv_1h'] = symbol_data['ohlcv_1h']
|
| 348 |
analysis_output['symbol'] = symbol
|
| 349 |
|
|
|
|
| 350 |
filter_score = self._calculate_1h_filter_score(analysis_output)
|
| 351 |
|
| 352 |
+
# 🔴 --- START OF CHANGE (V7.3) --- 🔴
|
| 353 |
+
# (رفع العتبة من 0.20 إلى 0.50)
|
| 354 |
+
if filter_score >= 0.50:
|
| 355 |
+
# 🔴 --- END OF CHANGE --- 🔴
|
| 356 |
print(f" ✅ {symbol}: نجح (الدرجة: {filter_score:.2f})")
|
| 357 |
symbol_data['layer1_score'] = filter_score
|
| 358 |
symbol_data['reasons_for_candidacy'] = [f'1H_DETECTOR_PASS']
|
|
|
|
| 360 |
if 'ohlcv_1h' in symbol_data: del symbol_data['ohlcv_1h']
|
| 361 |
|
| 362 |
final_candidates.append(symbol_data)
|
|
|
|
|
|
|
|
|
|
| 363 |
|
| 364 |
+
print(f"🎯 اكتملت الغربلة (V7.3). تم تأهيل {len(final_candidates)} عملة من أصل 100 للطبقة 2.")
|
| 365 |
|
| 366 |
print("🏆 المرشحون الناجحون:")
|
| 367 |
for k, candidate in enumerate(final_candidates[:15]):
|
|
|
|
| 382 |
|
| 383 |
analysis_dict = {'current_price': current_price}
|
| 384 |
|
|
|
|
| 385 |
task_indicators = self.technical_analyzer.calculate_all_indicators(df, '1h')
|
| 386 |
task_mc = self.monte_carlo_analyzer.generate_1h_price_distribution(ohlcv_1h)
|
| 387 |
task_pattern = self.pattern_analyzer.detect_chart_patterns(ohlcv_1h)
|
|
|
|
| 409 |
except Exception:
|
| 410 |
return None
|
| 411 |
|
|
|
|
| 412 |
async def _get_volume_data_optimal(self) -> List[Dict[str, Any]]:
|
| 413 |
try:
|
| 414 |
if not self.exchange: return []
|
|
|
|
| 427 |
dollar_volume = base_volume * current_price
|
| 428 |
if dollar_volume is None or dollar_volume < 50000: continue
|
| 429 |
|
|
|
|
|
|
|
| 430 |
price_change_24h = ticker.get('percentage', 0) or 0
|
| 431 |
if price_change_24h is None: price_change_24h = 0
|
|
|
|
| 432 |
|
| 433 |
volume_data.append({
|
| 434 |
'symbol': symbol, 'dollar_volume': dollar_volume,
|
|
|
|
| 463 |
if vol_value is None or last_price is None or change_rate is None or vol is None: continue
|
| 464 |
dollar_volume = float(vol_value) if vol_value else 0
|
| 465 |
current_price = float(last_price) if last_price else 0
|
|
|
|
|
|
|
| 466 |
price_change = (float(change_rate) * 100) if change_rate else 0
|
|
|
|
| 467 |
volume_24h = float(vol) if vol else 0
|
| 468 |
if dollar_volume >= 50000 and current_price > 0:
|
| 469 |
volume_data.append({
|
|
|
|
| 478 |
print(f"❌ خطأ في جلب بيانات الحجم المباشر: {e}")
|
| 479 |
return []
|
| 480 |
|
|
|
|
| 481 |
async def stream_ohlcv_data(self, symbols: List[Dict[str, Any]], queue: asyncio.Queue):
|
| 482 |
"""
|
| 483 |
(محدث V7.2)
|
|
|
|
| 495 |
|
| 496 |
batch_tasks = []
|
| 497 |
|
| 498 |
+
# (V7.2 FIX)
|
| 499 |
for symbol_data in batch:
|
| 500 |
+
symbol_str = symbol_data['symbol']
|
|
|
|
| 501 |
task = asyncio.create_task(self._fetch_complete_ohlcv_parallel(symbol_str))
|
| 502 |
batch_tasks.append(task)
|
|
|
|
| 503 |
|
| 504 |
batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True)
|
| 505 |
|
|
|
|
| 507 |
successful_count = 0
|
| 508 |
for i, result in enumerate(batch_results):
|
| 509 |
|
|
|
|
| 510 |
original_symbol_data = batch[i]
|
| 511 |
symbol_str = original_symbol_data['symbol']
|
|
|
|
| 512 |
|
| 513 |
if isinstance(result, Exception):
|
| 514 |
print(f" ❌ [المنتج] فشل جلب {symbol_str}: {result}")
|
| 515 |
elif result is not None:
|
|
|
|
| 516 |
result.update(original_symbol_data)
|
| 517 |
successful_data_for_batch.append(result)
|
| 518 |
successful_count += 1
|
|
|
|
| 555 |
|
| 556 |
timeframe_tasks = []
|
| 557 |
for timeframe, limit in timeframes:
|
|
|
|
| 558 |
task = asyncio.create_task(self._fetch_single_timeframe_improved(symbol, timeframe, limit))
|
| 559 |
timeframe_tasks.append(task)
|
| 560 |
|
|
|
|
| 598 |
retry_delay = 2
|
| 599 |
for attempt in range(max_retries):
|
| 600 |
try:
|
|
|
|
| 601 |
ohlcv_data = self.exchange.fetch_ohlcv(symbol, timeframe, limit=limit)
|
| 602 |
if ohlcv_data and len(ohlcv_data) > 0:
|
| 603 |
return ohlcv_data
|
|
|
|
| 621 |
return float(current_price)
|
| 622 |
except Exception as e: return None
|
| 623 |
|
|
|
|
| 624 |
async def get_whale_data_for_symbol(self, symbol):
|
| 625 |
try:
|
| 626 |
if self.whale_monitor:
|
|
|
|
| 638 |
except Exception as e:
|
| 639 |
return {'action': 'HOLD', 'confidence': 0.3, 'reason': f'Error: {str(e)}', 'source': 'whale_analysis'}
|
| 640 |
|
| 641 |
+
print("✅ DataManager loaded - V7.3 (L1 Threshold @ 0.50)")
|