feat: added mdy_product_sync new task for merchant_id isolated

api_v1/tasks.py

@@ -103,8 +103,8 @@ def backup_database(self, output_dir: str | None = None, filename_prefix: str =
     return payload
 
 
-def _get_mdy_merchant():
-    merchant_id = getattr(settings, 'MDY_MERCHANT_ID', None)
+def _get_mdy_merchant(merchant_id: int | None = None):
+    merchant_id = merchant_id or getattr(settings, 'MDY_MERCHANT_ID', None)
     qs = basic_models.Merchant.objects.all()
     if merchant_id:
         qs = qs.filter(id=merchant_id)
@@ -230,7 +230,12 @@ def _upsert_product(product_data, merchant, category):
 
 
 @shared_task(bind=True)
-def sync_mdy_products(self, page_size: int = 300, max_pages: int | None = None, max_records: int | None = None):
+def sync_mdy_products(
+    self,
+    page_size: int = 300,
+    max_pages: int | None = None,
+    max_records: int | None = None,
+):
     """
     从明道云同步产品数据（按 ctime 升序分页扫描）。
 
@@ -243,7 +248,8 @@ def sync_mdy_products(self, page_size: int = 300, max_pages: int | None = None,
     max_records = max_records or 0  # 0 表示不限制
 
     last_sync = api_models.DataSync.objects.filter(
-        table_name=api_models.DataSync.TableName.PRODUCT
+        table_name=api_models.DataSync.TableName.PRODUCT,
+        merchant__isnull=True,
     ).order_by('-created_at').first()
     last_ctime = last_sync.last_ctime if last_sync else None
     last_rowid = last_sync.last_rowid if last_sync else ''
@@ -307,6 +313,7 @@ def sync_mdy_products(self, page_size: int = 300, max_pages: int | None = None,
     record_last_rowid = latest_rowid or last_rowid
     api_models.DataSync.objects.create(
         table_name=api_models.DataSync.TableName.PRODUCT,
+        merchant=None,
         page_index=page_index,
         page_size=page_size,
         synced_rows=synced_rows,
@@ -362,7 +369,12 @@ def _upsert_customer(customer_data, merchant):
 
 
 @shared_task(bind=True)
-def sync_mdy_customers(self, page_size: int = 300, max_pages: int | None = None, max_records: int | None = None):
+def sync_mdy_customers(
+    self,
+    page_size: int = 300,
+    max_pages: int | None = None,
+    max_records: int | None = None,
+):
     """
     从明道云同步客户数据（按 ctime 升序分页扫描）。
 
@@ -374,7 +386,8 @@ def sync_mdy_customers(self, page_size: int = 300, max_pages: int | None = None,
     max_records = max_records or 0  # 0 表示不限制
 
     last_sync = api_models.DataSync.objects.filter(
-        table_name=api_models.DataSync.TableName.CUSTOMER
+        table_name=api_models.DataSync.TableName.CUSTOMER,
+        merchant__isnull=True,
     ).order_by('-created_at').first()
     last_ctime = last_sync.last_ctime if last_sync else None
     last_rowid = last_sync.last_rowid if last_sync else ''
@@ -434,6 +447,7 @@ def sync_mdy_customers(self, page_size: int = 300, max_pages: int | None = None,
     record_last_rowid = latest_rowid or last_rowid
     api_models.DataSync.objects.create(
         table_name=api_models.DataSync.TableName.CUSTOMER,
+        merchant=None,
         page_index=page_index,
         page_size=page_size,
         synced_rows=synced_rows,
@@ -474,11 +488,234 @@ def sync_mdy_plate_orders(
         with_related=with_related,
         max_related_per_type=max_related_per_type,
         request_interval_seconds=request_interval_seconds,
+        merchant_id=None,
     )
     payload["task_id"] = self.request.id
     return payload
 
 
+@shared_task(bind=True)
+def sync_mdy_products_for_merchant(
+    self,
+    *,
+    merchant_id: int,
+    page_size: int = 300,
+    max_pages: int | None = None,
+    max_records: int | None = None,
+):
+    """按商户隔离同步产品数据（独立游标）。"""
+    merchant = _get_mdy_merchant(merchant_id=merchant_id)
+    category = _get_mdy_category(merchant)
+    max_records = max_records or 0
+
+    last_sync = api_models.DataSync.objects.filter(
+        table_name=api_models.DataSync.TableName.PRODUCT,
+        merchant=merchant,
+    ).order_by('-created_at').first()
+    last_ctime = last_sync.last_ctime if last_sync else None
+    last_rowid = last_sync.last_rowid if last_sync else ''
+    start_page_index = last_sync.page_index if last_sync else 1
+
+    synced_rows = 0
+    page_index = max(1, start_page_index)
+    pages_processed = 0
+    total_count = 0
+    latest_ctime = last_ctime
+    latest_rowid = last_rowid
+
+    while True:
+        if max_pages is not None and pages_processed >= max_pages:
+            break
+        if max_records and synced_rows >= max_records:
+            break
+
+        products, total = _run_fetch(page_index, page_size)
+        total_count = total
+        if not products:
+            break
+
+        hit_max_records = False
+        for item in products:
+            if max_records and synced_rows >= max_records:
+                hit_max_records = True
+                break
+            product_ctime = _parse_mdy_datetime(item.created_at)
+
+            if last_ctime and product_ctime:
+                if product_ctime < last_ctime:
+                    continue
+                if product_ctime == last_ctime and last_rowid and item.rowid == last_rowid:
+                    continue
+
+            changed = _upsert_product(item, merchant, category)
+            if changed:
+                synced_rows += 1
+            if product_ctime:
+                if latest_ctime is None or product_ctime > latest_ctime:
+                    latest_ctime = product_ctime
+                    latest_rowid = item.rowid
+                elif product_ctime == latest_ctime:
+                    latest_rowid = item.rowid
+
+        pages_processed += 1
+        if hit_max_records:
+            break
+
+        if len(products) < page_size:
+            break
+        page_index += 1
+
+    record_last_ctime = latest_ctime or last_ctime
+    record_last_rowid = latest_rowid or last_rowid
+    api_models.DataSync.objects.create(
+        table_name=api_models.DataSync.TableName.PRODUCT,
+        merchant=merchant,
+        page_index=page_index,
+        page_size=page_size,
+        synced_rows=synced_rows,
+        total_count=total_count,
+        last_ctime=record_last_ctime,
+        last_rowid=record_last_rowid,
+        note='asc scan (merchant isolated)',
+    )
+
+    payload = {
+        'task_id': self.request.id,
+        'merchant_id': merchant.id,
+        'synced_rows': synced_rows,
+        'page_index': page_index,
+        'page_size': page_size,
+        'total_count': total_count,
+        'last_ctime': record_last_ctime.isoformat() if record_last_ctime else None,
+    }
+    logger.info('明道云产品同步完成(merchant=%s): %s', merchant.id, payload)
+    return payload
+
+
+@shared_task(bind=True)
+def sync_mdy_customers_for_merchant(
+    self,
+    *,
+    merchant_id: int,
+    page_size: int = 300,
+    max_pages: int | None = None,
+    max_records: int | None = None,
+):
+    """按商户隔离同步客户数据（独立游标）。"""
+    merchant = _get_mdy_merchant(merchant_id=merchant_id)
+    max_records = max_records or 0
+
+    last_sync = api_models.DataSync.objects.filter(
+        table_name=api_models.DataSync.TableName.CUSTOMER,
+        merchant=merchant,
+    ).order_by('-created_at').first()
+    last_ctime = last_sync.last_ctime if last_sync else None
+    last_rowid = last_sync.last_rowid if last_sync else ''
+    start_page_index = last_sync.page_index if last_sync else 1
+
+    synced_rows = 0
+    page_index = max(1, start_page_index)
+    pages_processed = 0
+    total_count = 0
+    latest_ctime = last_ctime
+    latest_rowid = last_rowid
+
+    while True:
+        if max_pages is not None and pages_processed >= max_pages:
+            break
+        if max_records and synced_rows >= max_records:
+            break
+
+        customers, total = _run_fetch_customers(page_index, page_size)
+        total_count = total
+        if not customers:
+            break
+
+        hit_max_records = False
+        for item in customers:
+            if max_records and synced_rows >= max_records:
+                hit_max_records = True
+                break
+            record_ctime = _parse_mdy_datetime(item.created_at)
+
+            if last_ctime and record_ctime:
+                if record_ctime < last_ctime:
+                    continue
+                if record_ctime == last_ctime and last_rowid and item.rowid == last_rowid:
+                    continue
+
+            changed = _upsert_customer(item, merchant)
+            if changed:
+                synced_rows += 1
+            if record_ctime:
+                if latest_ctime is None or record_ctime > latest_ctime:
+                    latest_ctime = record_ctime
+                    latest_rowid = item.rowid
+                elif record_ctime == latest_ctime:
+                    latest_rowid = item.rowid
+
+        pages_processed += 1
+        if hit_max_records:
+            break
+
+        if len(customers) < page_size:
+            break
+        page_index += 1
+
+    record_last_ctime = latest_ctime or last_ctime
+    record_last_rowid = latest_rowid or last_rowid
+    api_models.DataSync.objects.create(
+        table_name=api_models.DataSync.TableName.CUSTOMER,
+        merchant=merchant,
+        page_index=page_index,
+        page_size=page_size,
+        synced_rows=synced_rows,
+        total_count=total_count,
+        last_ctime=record_last_ctime,
+        last_rowid=record_last_rowid,
+        note='asc scan (merchant isolated)',
+    )
+
+    payload = {
+        'task_id': self.request.id,
+        'merchant_id': merchant.id,
+        'synced_rows': synced_rows,
+        'page_index': page_index,
+        'page_size': page_size,
+        'total_count': total_count,
+        'last_ctime': record_last_ctime.isoformat() if record_last_ctime else None,
+    }
+    logger.info('明道云客户同步完成(merchant=%s): %s', merchant.id, payload)
+    return payload
+
+
+@shared_task(bind=True)
+def sync_mdy_plate_orders_for_merchant(
+    self,
+    *,
+    merchant_id: int,
+    page_size: int = 300,
+    max_pages: int | None = None,
+    max_records: int | None = None,
+    with_related: bool = True,
+    max_related_per_type: int = 5,
+    request_interval_seconds: float = 0.02,
+):
+    """按商户隔离同步开版暂存表（独立游标）。"""
+    payload = sync_mdy_plate_orders_to_staging(
+        page_size=page_size,
+        max_pages=max_pages,
+        max_records=max_records,
+        with_related=with_related,
+        max_related_per_type=max_related_per_type,
+        request_interval_seconds=request_interval_seconds,
+        merchant_id=merchant_id,
+    )
+    payload["task_id"] = self.request.id
+    payload["merchant_id"] = merchant_id
+    return payload
+
+
 def _record_mdy_plate_order_staging_tiia_failure(
     *,
     run_date,