hiepnd11 commited on
Commit
cb84d67
·
verified ·
1 Parent(s): 0068cb7

update miner_fn

Browse files
Files changed (1) hide show
  1. miner.py +0 -224
miner.py CHANGED
@@ -6,100 +6,9 @@ from typing import List, Tuple, Optional
6
  import numpy as np
7
  import cv2
8
  from sklearn.cluster import KMeans
9
- import base64
10
- import boto3
11
- import json
12
- import uuid
13
 
14
 
15
 
16
- ########################################
17
- # Helper utilities for R2 storage
18
- ########################################
19
-
20
- def init_r2_client():
21
- """
22
- Khởi tạo S3 client cho Cloudflare R2.
23
- Returns:
24
- tuple: (s3_client, bucket_name, can_upload)
25
- """
26
- try:
27
- r2_account_id = "f5ac691bc782b80f90edb38eba5534ad"
28
- r2_access_key_id = "54f3343f68621c563d7ca29d3b356122"
29
- r2_secret_access_key = "41484baa8a10838e197f528b7eefbb824e1f38ffe13abc4e6b5fa7b68ad6d82d"
30
- bucket_name = "my-miner-sn44"
31
-
32
- can_upload = all([r2_account_id, r2_access_key_id, r2_secret_access_key, bucket_name])
33
-
34
- if can_upload:
35
- s3_client = boto3.client(
36
- 's3',
37
- endpoint_url=f"https://{r2_account_id}.r2.cloudflarestorage.com",
38
- aws_access_key_id=r2_access_key_id,
39
- aws_secret_access_key=r2_secret_access_key,
40
- region_name='auto'
41
- )
42
- print(f"✅ R2 client initialized for bucket: {bucket_name}")
43
- return s3_client, bucket_name, True
44
- else:
45
- print("⚠️ Thiếu một hoặc nhiều secret của R2, sẽ không lưu frames.")
46
- return None, None, False
47
-
48
- except Exception as e:
49
- print(f"⚠️ Không thể khởi tạo S3 client: {e}")
50
- return None, None, False
51
-
52
-
53
- def image_to_base64(image: np.ndarray, quality: int = 85) -> str:
54
- """
55
- Convert numpy image array to base64 string.
56
- Args:
57
- image: numpy array (BGR format from OpenCV)
58
- quality: JPEG quality (1-100, default 85)
59
- Returns:
60
- str: base64 encoded string
61
- """
62
- # Encode image as JPEG
63
- encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), quality]
64
- _, buffer = cv2.imencode('.jpg', image, encode_param)
65
- # Convert to base64
66
- base64_str = base64.b64encode(buffer).decode('utf-8')
67
- return base64_str
68
-
69
-
70
- def upload_frames_to_r2(
71
- s3_client,
72
- bucket_name: str,
73
- frames_base64: List[dict],
74
- challenge_id: str
75
- ) -> bool:
76
- """
77
- Upload danh sách frames (base64) lên Cloudflare R2 dưới dạng JSON.
78
- Args:
79
- s3_client: boto3 S3 client
80
- bucket_name: Tên bucket R2
81
- frames_base64: List of dicts with frame_id and base64 data
82
- challenge_id: ID của challenge (dùng làm tên file)
83
- Returns:
84
- bool: True nếu upload thành công
85
- """
86
- try:
87
- json_filename = f"{challenge_id}_frames.json"
88
- json_data = json.dumps(frames_base64)
89
-
90
- s3_client.put_object(
91
- Bucket=bucket_name,
92
- Key=json_filename,
93
- Body=json_data.encode('utf-8'),
94
- ContentType='application/json'
95
- )
96
- print(f"✅ {len(frames_base64)} frames đã được lưu vào R2: {json_filename}")
97
- return True
98
- except Exception as e:
99
- print(f"⚠️ Lỗi khi tải frames lên R2: {e}")
100
- return False
101
-
102
-
103
  ########################################
104
  # Helper utilities for grass & color clustering
105
  ########################################
@@ -195,21 +104,7 @@ class Miner:
195
 
196
  Args:
197
  path_hf_repo: Path to HuggingFace repo with models
198
- enable_frame_storage: If True, collect frames as base64 for R2 upload
199
- storage_quality: JPEG quality for stored frames (1-100)
200
- challenge_id: Challenge ID for R2 upload (required if enable_frame_storage=True)
201
  """
202
- enable_frame_storage = True
203
- storage_quality = 85
204
-
205
- challenge_id = f"challenge_{uuid.uuid4().hex[:12]}"
206
-
207
- # Option 2: Timestamp-based (unique theo thời gian)
208
- # challenge_id = f"challenge_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
209
-
210
- print(f"✅ Auto-generated challenge_id: {challenge_id}")
211
-
212
-
213
  self.bbox_model = YOLO(path_hf_repo / "251110-football-detection.pt")
214
  print("✅ BBox Model Loaded")
215
  self.keypoints_model = YOLO(path_hf_repo / "17112025_keypoint.pt")
@@ -219,22 +114,6 @@ class Miner:
219
  self.left_team_label = 0
220
  self.grass_hsv = None
221
  self.team_classifier_fitted = False
222
-
223
- # Frame storage setup
224
- self.enable_frame_storage = enable_frame_storage
225
- self.storage_quality = storage_quality
226
- self.stored_frames: List[dict] = [] # Store frames as base64
227
- self.challenge_id = challenge_id
228
-
229
- # R2 client setup
230
- if enable_frame_storage:
231
- self.s3_client, self.r2_bucket, self.can_upload = init_r2_client()
232
- if not challenge_id:
233
- print("⚠️ WARNING: enable_frame_storage=True nhưng chưa set challenge_id")
234
- else:
235
- self.s3_client = None
236
- self.r2_bucket = None
237
- self.can_upload = False
238
 
239
  def __repr__(self) -> str:
240
  return (
@@ -274,83 +153,6 @@ class Miner:
274
 
275
  grass_color = get_grass_color(frame)
276
  self.grass_hsv = cv2.cvtColor(np.uint8([[list(grass_color)]]), cv2.COLOR_BGR2HSV)
277
-
278
- def _auto_upload_frames(self) -> None:
279
- """Internal method to auto-upload frames after last batch."""
280
- if not self.challenge_id:
281
- print("❌ Không thể upload: challenge_id chưa được set!")
282
- return
283
-
284
- total_frames = len(self.stored_frames)
285
- size_mb = self.get_stored_frames_size_mb()
286
-
287
- print(f"📊 Tổng frames đã lưu: {total_frames}")
288
- print(f"💾 Size trong memory: {size_mb:.2f} MB")
289
- print(f"📤 Đang upload lên R2...")
290
-
291
- success = upload_frames_to_r2(
292
- self.s3_client,
293
- self.r2_bucket,
294
- self.stored_frames,
295
- self.challenge_id
296
- )
297
-
298
- if success:
299
- print(f"✅ Upload thành công {total_frames} frames!")
300
- print(f"📁 File trên R2: {self.challenge_id}_frames.json")
301
- # Clear frames after successful upload
302
- self.clear_stored_frames()
303
- else:
304
- print(f"❌ Upload thất bại!")
305
- print(f"💡 Frames vẫn còn trong memory. Có thể retry bằng: miner.upload_stored_frames('{self.challenge_id}')")
306
-
307
- def upload_stored_frames(self, challenge_id: str) -> bool:
308
- """
309
- Upload all stored frames to R2.
310
- Args:
311
- challenge_id: ID của challenge để đặt tên file
312
- Returns:
313
- bool: True nếu upload thành công
314
- """
315
- if not self.can_upload:
316
- print("⚠️ R2 client chưa được khởi tạo, không thể upload frames.")
317
- return False
318
-
319
- if len(self.stored_frames) == 0:
320
- print("⚠️ Không có frames nào để upload.")
321
- return False
322
-
323
- print(f"📤 Đang upload {len(self.stored_frames)} frames lên R2...")
324
- success = upload_frames_to_r2(
325
- self.s3_client,
326
- self.r2_bucket,
327
- self.stored_frames,
328
- challenge_id
329
- )
330
-
331
- if success:
332
- print(f"✅ Đã upload thành công {len(self.stored_frames)} frames")
333
- return True
334
- else:
335
- print("Chưa upload được.")
336
- return False
337
-
338
- def clear_stored_frames(self) -> None:
339
- """Clear all stored frames from memory."""
340
- self.stored_frames = []
341
- print("🗑️ Đã xóa stored frames khỏi memory")
342
-
343
- def get_stored_frames_count(self) -> int:
344
- """Get number of stored frames."""
345
- return len(self.stored_frames)
346
-
347
- def get_stored_frames_size_mb(self) -> float:
348
- """Get approximate size of stored frames in MB."""
349
- if len(self.stored_frames) == 0:
350
- return 0.0
351
- total_size = sum(len(frame["data"]) for frame in self.stored_frames)
352
- # Base64 encoding adds ~33% overhead, but we calculate as-is
353
- return total_size / (1024 * 1024)
354
 
355
  def predict_batch(
356
  self,
@@ -360,7 +162,6 @@ class Miner:
360
  ) -> list[TVFrameResult]:
361
  """
362
  Run predictions and return structured results.
363
- Auto-upload when frame_id reaches 750.
364
 
365
  Args:
366
  batch_images: List of image arrays (numpy)
@@ -374,17 +175,6 @@ class Miner:
374
 
375
  for i, frame in enumerate(batch_images):
376
  frame_id = offset + i
377
-
378
- # Store frame as base64 if enabled
379
- if self.enable_frame_storage and self.can_upload:
380
- try:
381
- frame_base64 = image_to_base64(frame, quality=self.storage_quality)
382
- self.stored_frames.append({
383
- "frame_id": frame_id,
384
- "data": frame_base64
385
- })
386
- except Exception as e:
387
- print(f"⚠️ Lỗi khi convert frame {frame_id} sang base64: {e}")
388
 
389
  # Fit KMeans on first frame if not done
390
  if not self.team_classifier_fitted:
@@ -487,19 +277,5 @@ class Miner:
487
  frame_keypoints = filtered_keypoints
488
 
489
  results.append(TVFrameResult(frame_id=frame_id, boxes=boxes, keypoints=frame_keypoints))
490
-
491
- # Auto-upload when reaching frame 750
492
- if frame_id == 749 and self.enable_frame_storage and self.can_upload:
493
- try:
494
- if len(self.stored_frames) > 0:
495
- print(f"\n{'='*60}")
496
- print(f"🏁 FRAME 750 REACHED - Tự động upload {len(self.stored_frames)} frames lên R2")
497
- print(f"{'='*60}")
498
- self._auto_upload_frames()
499
- else:
500
- print("⚠️ Frame 750 reached nhưng không có frames nào để upload.")
501
- except Exception as e:
502
- print(f"⚠️ Lỗi khi upload R2: {e}")
503
- print(f"💡 Tiếp tục trả về results. Frames vẫn còn trong memory.")
504
 
505
  return results
 
6
  import numpy as np
7
  import cv2
8
  from sklearn.cluster import KMeans
 
 
 
 
9
 
10
 
11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  ########################################
13
  # Helper utilities for grass & color clustering
14
  ########################################
 
104
 
105
  Args:
106
  path_hf_repo: Path to HuggingFace repo with models
 
 
 
107
  """
 
 
 
 
 
 
 
 
 
 
 
108
  self.bbox_model = YOLO(path_hf_repo / "251110-football-detection.pt")
109
  print("✅ BBox Model Loaded")
110
  self.keypoints_model = YOLO(path_hf_repo / "17112025_keypoint.pt")
 
114
  self.left_team_label = 0
115
  self.grass_hsv = None
116
  self.team_classifier_fitted = False
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
117
 
118
  def __repr__(self) -> str:
119
  return (
 
153
 
154
  grass_color = get_grass_color(frame)
155
  self.grass_hsv = cv2.cvtColor(np.uint8([[list(grass_color)]]), cv2.COLOR_BGR2HSV)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
156
 
157
  def predict_batch(
158
  self,
 
162
  ) -> list[TVFrameResult]:
163
  """
164
  Run predictions and return structured results.
 
165
 
166
  Args:
167
  batch_images: List of image arrays (numpy)
 
175
 
176
  for i, frame in enumerate(batch_images):
177
  frame_id = offset + i
 
 
 
 
 
 
 
 
 
 
 
178
 
179
  # Fit KMeans on first frame if not done
180
  if not self.team_classifier_fitted:
 
277
  frame_keypoints = filtered_keypoints
278
 
279
  results.append(TVFrameResult(frame_id=frame_id, boxes=boxes, keypoints=frame_keypoints))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
280
 
281
  return results