diff --git a/api_v1/utils/tencentcloud_tiia.py b/api_v1/utils/tencentcloud_tiia.py index f57bc9d..7394430 100644 --- a/api_v1/utils/tencentcloud_tiia.py +++ b/api_v1/utils/tencentcloud_tiia.py @@ -10,6 +10,7 @@ SDK reference: https://github.com/TencentCloud/tencentcloud-sdk-python """ import json +import time import uuid from urllib.parse import urlparse @@ -24,6 +25,27 @@ def _truncate(s: str, max_len: int) -> str: return s[:max_len] +class SimpleRateLimiter: + """ + Very small in-process rate limiter (token-bucket-ish) to keep API calls under QPS. + + This is used by the scheduled task to respect TencentCloud API limits (e.g. 10 req/s). + """ + + def __init__(self, *, qps: float): + if qps <= 0: + raise ValueError('qps 必须 > 0') + self._min_interval = 1.0 / float(qps) + self._last_ts = 0.0 + + def wait(self) -> None: + now = time.monotonic() + elapsed = now - self._last_ts + if self._last_ts > 0 and elapsed < self._min_interval: + time.sleep(self._min_interval - elapsed) + self._last_ts = time.monotonic() + + def upload_image_url_to_tencent_tiia(*, image_url: str, entity_id: str) -> dict: """ Upload an image by URL to Tencent Cloud TIIA gallery using CreateImage. @@ -157,7 +179,7 @@ def _extract_urls_from_plate_image(plate_image) -> list[str]: return deduped -def upload_plate_order_images_to_tencent_tiia(*, plate_order_id: int) -> list[dict]: +def upload_plate_order_images_to_tencent_tiia(*, plate_order_id: int, rate_limiter: SimpleRateLimiter | None = None) -> list[dict]: """ 包装函数：给定 plate_order_id，读取 PlateOrder.plate_image(JSONField) 中所有图片并上传到腾讯云图库。 @@ -205,6 +227,8 @@ def upload_plate_order_images_to_tencent_tiia(*, plate_order_id: int) -> list[di }) continue try: + if rate_limiter is not None: + rate_limiter.wait() resp = upload_image_url_to_tencent_tiia(image_url=url_str, entity_id=entity_id) results.append({'image_url': url_str, 'ok': True, 'response': resp, 'error': None}) except Exception as e: @@ -212,5 +236,9 @@ def upload_plate_order_images_to_tencent_tiia(*, plate_order_id: int) -> list[di return results -__all__ = ['upload_image_url_to_tencent_tiia', 'upload_plate_order_images_to_tencent_tiia'] +__all__ = [ + 'SimpleRateLimiter', + 'upload_image_url_to_tencent_tiia', + 'upload_plate_order_images_to_tencent_tiia', +] diff --git a/flower/settings.py b/flower/settings.py index 138504d..128205c 100644 --- a/flower/settings.py +++ b/flower/settings.py @@ -74,6 +74,7 @@ TENCENTCLOUD_TIIA_GROUP_ID = env('TENCENTCLOUD_TIIA_GROUP_ID', default='168') TENCENTCLOUD_TIIA_REGION = env('TENCENTCLOUD_TIIA_REGION', default='ap-guangzhou') TENCENTCLOUD_TIIA_ENDPOINT = env('TENCENTCLOUD_TIIA_ENDPOINT', default='tiia.tencentcloudapi.com') TENCENTCLOUD_TIIA_PIC_NAME_PREFIX = env('TENCENTCLOUD_TIIA_PIC_NAME_PREFIX', default='plate_order') +TENCENTCLOUD_TIIA_QPS = env.int('TENCENTCLOUD_TIIA_QPS', default=10) # Tencent API 限速：每秒最多 10 次 # CORS 配置 @@ -473,6 +474,10 @@ CELERY_BEAT_SCHEDULE = { 'filename_prefix': 'db-backup', }, }, + 'daily_plate_order_tiia_image_upload': { + 'task': 'printing.tasks.upload_yesterday_plate_order_images_to_tencent_tiia', + 'schedule': crontab(hour=3, minute=0), + }, 'mdy_product_sync': { 'task': 'api_v1.tasks.sync_mdy_products', 'schedule': crontab(minute='*/10'), diff --git a/printing/admin.py b/printing/admin.py index 957288c..3ca183e 100644 --- a/printing/admin.py +++ b/printing/admin.py @@ -199,7 +199,7 @@ class PrintingOrderAdmin(admin.ModelAdmin): 'craft', 'description', ) - inlines = [PrintingJobInline] + def save_formset(self, request, form, formset, change): """保存 formset 后，为新创建的 PrintingJob 创建 BusinessObject""" @@ -468,3 +468,11 @@ class PrintingJobBatchAdvanceRecordAdmin(admin.ModelAdmin): @admin.display(description='涉及任务数') def jobs_count(self, obj: models.PrintingJobBatchAdvanceRecord) -> int: return obj.printing_jobs.count() + + +@admin.register(models.PlateOrderTiiaUploadFailure) +class PlateOrderTiiaUploadFailureAdmin(admin.ModelAdmin): + list_display = ('id', 'run_date', 'plate_order_id', 'attempts', 'last_attempt_at', 'created_at') + list_filter = ('run_date', 'created_at') + search_fields = ('plate_order_id', 'error') + readonly_fields = ('created_at', 'updated_at') diff --git a/printing/migrations/0031_plateorder_tiia_upload_failure.py b/printing/migrations/0031_plateorder_tiia_upload_failure.py new file mode 100644 index 0000000..9347161 --- /dev/null +++ b/printing/migrations/0031_plateorder_tiia_upload_failure.py @@ -0,0 +1,37 @@ +# Generated by Django 5.2.8 on 2026-01-16 19:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('printing', '0030_add_view_all_permissions'), + ] + + operations = [ + migrations.CreateModel( + name='PlateOrderTiiaUploadFailure', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='更新时间')), + ('run_date', models.DateField(help_text='以任务处理的日期为准（通常为“昨天”）', verbose_name='任务日期')), + ('plate_order_id', models.PositiveBigIntegerField(db_index=True, help_text='PlateOrder.id（不使用外键，避免历史数据/删除带来的约束问题）', verbose_name='开版订单ID')), + ('error', models.TextField(blank=True, default='', verbose_name='错误摘要')), + ('details', models.JSONField(blank=True, default=list, help_text='通常存储 upload_plate_order_images_to_tencent_tiia 返回的失败条目列表', verbose_name='错误详情')), + ('attempts', models.PositiveIntegerField(default=0, verbose_name='尝试次数')), + ('last_attempt_at', models.DateTimeField(blank=True, null=True, verbose_name='最后尝试时间')), + ], + options={ + 'verbose_name': '开版订单图片上传失败记录', + 'verbose_name_plural': '开版订单图片上传失败记录', + 'ordering': ['-created_at'], + }, + ), + migrations.AddConstraint( + model_name='plateordertiiauploadfailure', + constraint=models.UniqueConstraint(fields=('run_date', 'plate_order_id'), name='uniq_plateorder_tiia_fail_run_date_po'), + ), + ] + diff --git a/printing/models.py b/printing/models.py index e142722..baa4212 100644 --- a/printing/models.py +++ b/printing/models.py @@ -255,6 +255,54 @@ class PlateOrder(ModelBase): return self.business_object.get_progress_percentage() +class PlateOrderTiiaUploadFailure(ModelBase): + """ + PlateOrder 图片上传腾讯云图库失败记录（用于定时任务/人工重试追踪）。 + + 设计目标：最小可用——记录“哪天跑的任务 + 哪个 plate_order_id + 失败详情/次数”。 + """ + run_date = models.DateField( + verbose_name='任务日期', + help_text='以任务处理的日期为准（通常为“昨天”）', + ) + plate_order_id = models.PositiveBigIntegerField( + verbose_name='开版订单ID', + db_index=True, + help_text='PlateOrder.id（不使用外键，避免历史数据/删除带来的约束问题）', + ) + error = models.TextField( + blank=True, + default='', + verbose_name='错误摘要', + ) + details = models.JSONField( + default=list, + blank=True, + verbose_name='错误详情', + help_text='通常存储 upload_plate_order_images_to_tencent_tiia 返回的失败条目列表', + ) + attempts = models.PositiveIntegerField( + default=0, + verbose_name='尝试次数', + ) + last_attempt_at = models.DateTimeField( + null=True, + blank=True, + verbose_name='最后尝试时间', + ) + + class Meta: + verbose_name = '开版订单图片上传失败记录' + verbose_name_plural = '开版订单图片上传失败记录' + ordering = ['-created_at'] + constraints = [ + models.UniqueConstraint( + fields=['run_date', 'plate_order_id'], + name='uniq_plateorder_tiia_fail_run_date_po', + ) + ] + + class PrintingOrder(ModelBase): """PrintingOrder model representing a printing order.""" diff --git a/printing/tasks.py b/printing/tasks.py new file mode 100644 index 0000000..9ccda43 --- /dev/null +++ b/printing/tasks.py @@ -0,0 +1,118 @@ +import logging +from datetime import datetime, time, timedelta + +from celery import shared_task +from django.conf import settings +from django.db import transaction +from django.utils import timezone + +from printing.models import PlateOrder, PlateOrderTiiaUploadFailure + + +logger = logging.getLogger(__name__) + + +def _get_day_range(day) -> tuple[datetime, datetime]: + """ + Build [start, end) datetimes for a local date. + """ + tz = timezone.get_current_timezone() + start = datetime.combine(day, time.min) + end = start + timedelta(days=1) + if timezone.is_naive(start): + start = timezone.make_aware(start, tz) + if timezone.is_naive(end): + end = timezone.make_aware(end, tz) + return start, end + + +@transaction.atomic +def _record_failure(*, run_date, plate_order_id: int, error: str, details): + """ + Upsert failure record for (run_date, plate_order_id). + """ + now = timezone.now() + obj, created = PlateOrderTiiaUploadFailure.objects.select_for_update().get_or_create( + run_date=run_date, + plate_order_id=plate_order_id, + defaults={ + 'error': error or '', + 'details': details or [], + 'attempts': 1, + 'last_attempt_at': now, + }, + ) + if not created: + obj.error = error or obj.error or '' + obj.details = details or obj.details or [] + obj.attempts = (obj.attempts or 0) + 1 + obj.last_attempt_at = now + obj.save(update_fields=['error', 'details', 'attempts', 'last_attempt_at', 'updated_at']) + return obj + + +@shared_task(bind=True) +def upload_yesterday_plate_order_images_to_tencent_tiia(self): + """ + 每天凌晨 03:00 运行： + - 扫描“昨天创建”的 PlateOrder + - 将 plate_image(JSONField) 中的所有图片 URL 上传到腾讯云图库 + - 遇错：记录失败的 plate_order_id（落库），继续处理下一个 + + 注意：腾讯云调用限速 10 QPS（每秒 10 次）。 + """ + from api_v1.utils.tencentcloud_tiia import ( + SimpleRateLimiter, + upload_plate_order_images_to_tencent_tiia, + ) + + today = timezone.localdate() + run_date = today - timedelta(days=1) + start, end = _get_day_range(run_date) + + qps = int(getattr(settings, 'TENCENTCLOUD_TIIA_QPS', 10) or 10) + limiter = SimpleRateLimiter(qps=float(qps)) + + qs = ( + PlateOrder.objects.filter(created_at__gte=start, created_at__lt=end) + .only('id', 'plate_image') + .order_by('id') + ) + + total_orders = 0 + failed_ids: list[int] = [] + processed_images = 0 + + for po in qs.iterator(chunk_size=200): + total_orders += 1 + try: + results = upload_plate_order_images_to_tencent_tiia( + plate_order_id=po.id, + rate_limiter=limiter, + ) + except Exception as e: + failed_ids.append(po.id) + _record_failure(run_date=run_date, plate_order_id=po.id, error=str(e), details=[{'error': str(e)}]) + logger.exception('[TIIA] PlateOrder %s 上传异常（任务继续）', po.id) + continue + + processed_images += len(results) + bad = [r for r in results if not r.get('ok')] + if bad: + failed_ids.append(po.id) + summary = '; '.join([(b.get('error') or '') for b in bad][:5]) + _record_failure(run_date=run_date, plate_order_id=po.id, error=summary, details=bad) + logger.warning('[TIIA] PlateOrder %s 部分图片上传失败: %s', po.id, summary) + + payload = { + 'task_id': getattr(getattr(self, 'request', None), 'id', None), + 'run_date': str(run_date), + 'created_range': {'start': start.isoformat(), 'end': end.isoformat()}, + 'total_orders': total_orders, + 'processed_images': processed_images, + 'failed_count': len(failed_ids), + 'failed_ids': failed_ids, + } + logger.info('[TIIA] 昨日 PlateOrder 图片上传任务完成: %s', payload) + return payload + diff --git a/printing/test_tiia_upload_task.py b/printing/test_tiia_upload_task.py new file mode 100644 index 0000000..9e583e1 --- /dev/null +++ b/printing/test_tiia_upload_task.py @@ -0,0 +1,64 @@ +from datetime import datetime, time, timedelta +from unittest.mock import patch + +from django.test import TestCase +from django.utils import timezone + +from basic_info import models as basic_models +from printing import models as printing_models + + +class PlateOrderTiiaUploadTaskTestCase(TestCase): + def setUp(self): + self.merchant = basic_models.Merchant.objects.create( + name='测试商户', + type=basic_models.MerchantTypeEnum.FACTORY, + ) + self.customer = basic_models.Customer.objects.create( + merchant=self.merchant, + name='客户A', + mobile='13900000000', + area='A', + ) + + @patch('api_v1.utils.tencentcloud_tiia.time.sleep', lambda *_args, **_kwargs: None) + @patch('api_v1.utils.tencentcloud_tiia.upload_plate_order_images_to_tencent_tiia') + def test_task_records_failure_and_continues(self, mocked_upload_plate_order): + # 准备：创建 2 个 plate_order，并把 created_at 改到“昨天” + po_ok = printing_models.PlateOrder.objects.create( + merchant=self.merchant, + customer=self.customer, + plate_image=[{'url': 'https://example.com/a.png'}], + ) + po_bad = printing_models.PlateOrder.objects.create( + merchant=self.merchant, + customer=self.customer, + plate_image=[{'url': 'https://example.com/b.png'}], + ) + + yesterday = timezone.localdate() - timedelta(days=1) + dt = timezone.make_aware(datetime.combine(yesterday, time.min)) + printing_models.PlateOrder.objects.filter(id__in=[po_ok.id, po_bad.id]).update(created_at=dt) + + def side_effect(*, plate_order_id: int, rate_limiter=None): + if plate_order_id == po_bad.id: + raise ValueError('boom') + return [{'image_url': 'https://example.com/a.png', 'ok': True, 'response': {'ok': 1}, 'error': None}] + + mocked_upload_plate_order.side_effect = side_effect + + from printing.tasks import upload_yesterday_plate_order_images_to_tencent_tiia + + payload = upload_yesterday_plate_order_images_to_tencent_tiia() + + self.assertEqual(payload['total_orders'], 2) + self.assertEqual(payload['failed_count'], 1) + self.assertIn(po_bad.id, payload['failed_ids']) + + # failure 表应落库 + from printing.models import PlateOrderTiiaUploadFailure + + rec = PlateOrderTiiaUploadFailure.objects.filter(run_date=yesterday, plate_order_id=po_bad.id).first() + self.assertIsNotNone(rec) + self.assertGreaterEqual(rec.attempts, 1) +