1
0
forked from erp-dev/erp

feat: batch update for plate order

This commit is contained in:
2025-12-18 17:36:43 +08:00
parent 98d797221a
commit 1a6ce32441
33 changed files with 32087 additions and 8 deletions

View File

@@ -35,7 +35,7 @@ class DataSyncAdmin(admin.ModelAdmin):
@admin.register(MDYPlateOrderStaging)
class MDYPlateOrderStagingAdmin(admin.ModelAdmin):
list_display = ['id', 'mdy_rowid', 'created_at', 'updated_at']
list_display = ['id', 'mdy_rowid', 'ctime', 'utime', 'created_at', 'updated_at']
list_filter = ['created_at']
search_fields = ['mdy_rowid']
readonly_fields = ['created_at', 'updated_at']

View File

View File

View File

@@ -0,0 +1,35 @@
from django.core.management.base import BaseCommand
from api_v1.mdy_plate_order_sync import sync_mdy_plate_orders_to_staging
class Command(BaseCommand):
help = '同步明道云开版数据表到暂存表(可选抓取跨表关联数据)'
def add_arguments(self, parser):
parser.add_argument('--page-size', type=int, default=100)
parser.add_argument('--max-pages', type=int, default=None)
parser.add_argument('--max-records', type=int, default=None)
# 默认抓取关联表数据;如需关闭可显式传 --without-related
parser.add_argument('--with-related', dest='with_related', action='store_true', default=True)
parser.add_argument('--without-related', dest='with_related', action='store_false')
parser.add_argument('--max-related-per-type', type=int, default=5)
parser.add_argument(
'--request-interval-seconds',
type=float,
default=0.02,
help='每次明道云请求之间的最小间隔用于限流50qps 建议 >= 0.02',
)
def handle(self, *args, **options):
payload = sync_mdy_plate_orders_to_staging(
page_size=options['page_size'],
max_pages=options['max_pages'],
max_records=options['max_records'],
with_related=options['with_related'],
max_related_per_type=options['max_related_per_type'],
request_interval_seconds=options['request_interval_seconds'],
)
self.stdout.write(self.style.SUCCESS(str(payload)))

View File

@@ -0,0 +1,232 @@
import asyncio
import logging
from datetime import datetime
import time
from django.utils import timezone
from api_v1 import models as api_models
from flower.utils.mingdaoyun.fetch import (
fetch_plate_orders_from_mingdaoyun,
fetch_row_by_rowid_from_mingdaoyun,
)
from flower.utils.mingdaoyun.mappings import plate_order_related_worksheet_map
from flower.utils.mingdaoyun.relations import (
extract_plate_order_related_rowids,
flatten_plate_order_related_row,
)
logger = logging.getLogger(__name__)
def _parse_mdy_datetime(value: str | None):
if not value:
return None
try:
dt = datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
except ValueError:
return None
if timezone.is_naive(dt):
dt = timezone.make_aware(dt, timezone.get_current_timezone())
return dt
async def _fetch_related_flattened(
plate_order_row: dict,
*,
max_related_per_type: int = 5,
request_interval_seconds: float = 0.0,
) -> list[dict]:
"""根据开版主表 row 的 Relation 字段,拉取并碾平关联表数据。"""
related_rowids_map = extract_plate_order_related_rowids(plate_order_row)
if not related_rowids_map:
return []
flattened: list[dict] = []
for related_key, rowids in related_rowids_map.items():
wsid = plate_order_related_worksheet_map.get(related_key)
if not wsid:
continue
for rid in rowids[: max_related_per_type or 0]:
related_row = await fetch_row_by_rowid_from_mingdaoyun(
worksheet_id=wsid,
rowid=rid,
)
if not related_row:
continue
flattened.append(flatten_plate_order_related_row(related_key, related_row))
if request_interval_seconds:
await asyncio.sleep(request_interval_seconds)
return flattened
async def _fetch_related_for_rows(
rows: list[dict],
*,
max_related_per_type: int,
request_interval_seconds: float = 0.0,
) -> dict[str, list[dict]]:
"""批量拉取并碾平关联表数据(仅网络请求,不涉及 ORM"""
result: dict[str, list[dict]] = {}
for row in rows:
rowid = row.get("rowid")
if not rowid:
continue
result[rowid] = await _fetch_related_flattened(
row,
max_related_per_type=max_related_per_type,
request_interval_seconds=request_interval_seconds,
)
return result
def sync_mdy_plate_orders_to_staging(
*,
page_size: int = 100,
max_pages: int | None = None,
max_records: int | None = None,
with_related: bool = True,
max_related_per_type: int = 5,
request_interval_seconds: float = 0.02,
) -> dict:
"""同步明道云开版数据表到暂存表(含可选跨表关联数据)。
- 写入 `api_v1.MDYPlateOrderStaging`
- `raw`:整行原始数据
- `related`:跨表数据(碾平 list[dict]
- 进度记录到 `api_v1.DataSync`table_name=plate_order
说明:
- max_pages 表示“单次任务最多处理多少页”(不是最大页码)
- max_records 表示“单次任务最多处理多少条记录”0/None 表示不限制)
"""
max_records = max_records or 0
last_sync = api_models.DataSync.objects.filter(
table_name=api_models.DataSync.TableName.PLATE_ORDER
).order_by("-created_at").first()
last_ctime = last_sync.last_ctime if last_sync else None
last_rowid = last_sync.last_rowid if last_sync else ""
start_page_index = last_sync.page_index if last_sync else 1
synced_rows = 0
page_index = max(1, start_page_index)
pages_processed = 0
total_count = 0
latest_ctime = last_ctime
latest_rowid = last_rowid
while True:
if max_pages is not None and pages_processed >= max_pages:
break
if max_records and synced_rows >= max_records:
break
rows, total = asyncio.run(
fetch_plate_orders_from_mingdaoyun(
page=page_index,
page_size=page_size,
sort_id="ctime",
is_asc=True,
)
)
if request_interval_seconds:
time.sleep(request_interval_seconds)
total_count = total
if not rows:
break
hit_max_records = False
to_process: list[dict] = []
for row in rows:
if max_records and synced_rows + len(to_process) >= max_records:
hit_max_records = True
break
rowid = row.get("rowid")
if not rowid:
continue
record_ctime = _parse_mdy_datetime(row.get("ctime"))
# 跳过已同步到的游标
if last_ctime and record_ctime:
if record_ctime < last_ctime:
continue
if record_ctime == last_ctime and last_rowid and rowid == last_rowid:
continue
to_process.append(row)
related_map: dict[str, list[dict]] = {}
if with_related and to_process:
related_map = asyncio.run(
_fetch_related_for_rows(
to_process,
max_related_per_type=max_related_per_type,
request_interval_seconds=request_interval_seconds,
)
)
for row in to_process:
rowid = row.get("rowid")
if not rowid:
continue
api_models.MDYPlateOrderStaging.objects.update_or_create(
mdy_rowid=rowid,
defaults={
"ctime": _parse_mdy_datetime(row.get("ctime")),
"utime": _parse_mdy_datetime(row.get("utime")),
"raw": row,
"related": related_map.get(rowid, []),
},
)
synced_rows += 1
record_ctime = _parse_mdy_datetime(row.get("ctime"))
if record_ctime:
if latest_ctime is None or record_ctime > latest_ctime:
latest_ctime = record_ctime
latest_rowid = rowid
elif record_ctime == latest_ctime:
latest_rowid = rowid
pages_processed += 1
if hit_max_records:
break
if len(rows) < page_size:
break
page_index += 1
record_last_ctime = latest_ctime or last_ctime
record_last_rowid = latest_rowid or last_rowid
api_models.DataSync.objects.create(
table_name=api_models.DataSync.TableName.PLATE_ORDER,
page_index=page_index,
page_size=page_size,
synced_rows=synced_rows,
total_count=total_count,
last_ctime=record_last_ctime,
last_rowid=record_last_rowid,
note=f"asc scan; with_related={with_related}",
)
payload = {
"synced_rows": synced_rows,
"page_index": page_index,
"page_size": page_size,
"total_count": total_count,
"last_ctime": record_last_ctime.isoformat() if record_last_ctime else None,
"with_related": with_related,
}
logger.info("明道云开版暂存同步完成: %s", payload)
return payload

View File

@@ -0,0 +1,21 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('api_v1', '0004_mdy_plate_order_staging'),
]
operations = [
migrations.AddField(
model_name='mdyplateorderstaging',
name='related',
field=models.JSONField(
blank=True,
default=list,
help_text='开版相关跨表数据(碾平 list[dict]),用于排查与后续属性化提取',
verbose_name='关联表数据',
),
),
]

View File

@@ -0,0 +1,19 @@
from django.db import migrations, models
from django.db.models.fields.json import KeyTextTransform
class Migration(migrations.Migration):
dependencies = [
('api_v1', '0005_mdy_plate_order_staging_related'),
]
operations = [
migrations.AddIndex(
model_name='mdyplateorderstaging',
index=models.Index(
KeyTextTransform('62d52f4b8d2972284492dd0e', 'raw'),
name='api_mdy_po_raw_id_idx',
),
),
]

View File

@@ -0,0 +1,33 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('api_v1', '0006_mdy_plate_order_staging_raw_id_index'),
]
operations = [
migrations.AddField(
model_name='mdyplateorderstaging',
name='ctime',
field=models.DateTimeField(
blank=True,
db_index=True,
help_text='明道云系统字段 ctime用于排序/增量定位)',
null=True,
verbose_name='明道云创建时间',
),
),
migrations.AddField(
model_name='mdyplateorderstaging',
name='utime',
field=models.DateTimeField(
blank=True,
db_index=True,
help_text='明道云系统字段 utime用于排序/增量定位)',
null=True,
verbose_name='明道云更新时间',
),
),
]

View File

@@ -5,6 +5,7 @@ import os
import uuid
from django.db import models
from django.contrib.auth import get_user_model
from django.db.models.fields.json import KeyTextTransform
from flower.common import ModelBase
User = get_user_model()
@@ -90,6 +91,7 @@ class DataSync(ModelBase):
class TableName(models.TextChoices):
PRODUCT = 'product', '产品'
CUSTOMER = 'customer', '客户'
PLATE_ORDER = 'plate_order', '开版(明道云)'
table_name = models.CharField(max_length=50, choices=TableName.choices, verbose_name='同步目标')
page_index = models.PositiveIntegerField(default=1, verbose_name='页码')
@@ -126,18 +128,46 @@ class MDYPlateOrderStaging(ModelBase):
verbose_name='明道云 RowID',
help_text='明道云行记录的 rowid全局唯一',
)
ctime = models.DateTimeField(
null=True,
blank=True,
db_index=True,
verbose_name='明道云创建时间',
help_text='明道云系统字段 ctime用于排序/增量定位)',
)
utime = models.DateTimeField(
null=True,
blank=True,
db_index=True,
verbose_name='明道云更新时间',
help_text='明道云系统字段 utime用于排序/增量定位)',
)
raw = models.JSONField(
default=dict,
blank=True,
verbose_name='原始行数据',
help_text='getFilterRows 返回的整行原始数据(包含各 controlId 字段)',
)
related = models.JSONField(
default=list,
blank=True,
verbose_name='关联表数据',
help_text='开版相关跨表数据(碾平 list[dict]),用于排查与后续属性化提取',
)
class Meta:
db_table = 'api_mdy_plate_order_staging'
verbose_name = '明道云开版暂存'
verbose_name_plural = '明道云开版暂存'
ordering = ['-created_at']
indexes = [
# 明道云字段62d52f4b8d2972284492dd0e开版表“订单id/设计编号”)
# 用于在暂存表中按该字段快速检索
models.Index(
KeyTextTransform('62d52f4b8d2972284492dd0e', 'raw'),
name='api_mdy_po_raw_id_idx',
),
]
def __str__(self):
return f'{self.mdy_rowid}'

View File

@@ -18,6 +18,7 @@ from flower.utils import (
fetch_products_from_mingdaoyun,
fetch_customers_from_mingdaoyun,
)
from api_v1.mdy_plate_order_sync import sync_mdy_plate_orders_to_staging
logger = logging.getLogger(__name__)
@@ -441,3 +442,27 @@ def sync_mdy_customers(self, page_size: int = 300, max_pages: int | None = None,
}
logger.info('明道云客户同步完成: %s', payload)
return payload
@shared_task(bind=True)
def sync_mdy_plate_orders(
self,
page_size: int = 300,
max_pages: int | None = None,
max_records: int | None = None,
with_related: bool = True,
max_related_per_type: int = 5,
request_interval_seconds: float = 0.02,
):
"""从明道云同步开版数据表到暂存表(可选抓取跨表关联数据)。"""
payload = sync_mdy_plate_orders_to_staging(
page_size=page_size,
max_pages=max_pages,
max_records=max_records,
with_related=with_related,
max_related_per_type=max_related_per_type,
request_interval_seconds=request_interval_seconds,
)
payload["task_id"] = self.request.id
return payload

View File

@@ -24,6 +24,7 @@ from .views.upload import UploadFileViewSet
from .views.products import ProductQuickViewSet
from .views.parameters import StateParameterViewSet
from .views.users import CreateUserWithProfileView
from .views.mingdaoyun import MDYPlateOrderStagingViewSet
# 创建 DRF Router for Stateflow
stateflow_router = DefaultRouter()
@@ -39,6 +40,7 @@ main_router.register(r'plate-orders', PlateOrderViewSet, basename='plate-order')
main_router.register(r'upload', UploadFileViewSet, basename='upload')
main_router.register(r'products/quick', ProductQuickViewSet, basename='product-quick')
main_router.register(r'parameters', StateParameterViewSet, basename='parameter')
main_router.register(r'mdy-plate-order-staging', MDYPlateOrderStagingViewSet, basename='mdy-plate-order-staging')
urlpatterns = [
# 库存变动相关API

View File

@@ -0,0 +1,5 @@
"""MingdaoYun related API views."""
from .plate_order_staging import MDYPlateOrderStagingViewSet
__all__ = ["MDYPlateOrderStagingViewSet"]

View File

@@ -0,0 +1,102 @@
"""开版暂存(明道云)查询 API。
需求:
- 默认按 -ctime 排序
- 支持 raw__62d52f4b8d2972284492dd0e设计编号过滤
- 支持 LimitOffset 分页
- 返回时把 raw / related 两个 JSONField 做“可读化/碾平”
"""
from __future__ import annotations
from typing import Any
from django.db.models import F
from rest_framework import serializers, viewsets
from rest_framework.pagination import LimitOffsetPagination
from rest_framework.permissions import DjangoModelPermissions
from api_v1.models import MDYPlateOrderStaging
from flower.utils.mingdaoyun.mappings import plate_order_field_definitions
from flower.utils.mingdaoyun.relations import flatten_row_by_field_definitions, normalize_mdy_value
_PLATE_ORDER_DESIGN_NO_CONTROL_ID = "62d52f4b8d2972284492dd0e"
class MDYPlateOrderStagingSerializer(serializers.ModelSerializer):
"""把 raw/related 输出为扁平的可读结构。"""
design_no = serializers.SerializerMethodField()
raw = serializers.SerializerMethodField()
related = serializers.SerializerMethodField()
class Meta:
model = MDYPlateOrderStaging
fields = [
"id",
"mdy_rowid",
"ctime",
"utime",
"created_at",
"updated_at",
"design_no",
"raw",
"related",
]
def get_design_no(self, obj: MDYPlateOrderStaging) -> Any:
if isinstance(obj.raw, dict):
return obj.raw.get(_PLATE_ORDER_DESIGN_NO_CONTROL_ID)
return None
def get_raw(self, obj: MDYPlateOrderStaging) -> dict[str, Any]:
if not isinstance(obj.raw, dict):
return {}
# 1) 已知字段:按映射转换为中文 key更可读
out = flatten_row_by_field_definitions(obj.raw, plate_order_field_definitions, include_system_keys=True)
# 2) 未映射字段:为了排查方便,不静默丢弃,保留 controlId 原样
known_control_ids = set(plate_order_field_definitions.keys())
for k, v in obj.raw.items():
if k in {"rowid", "ctime", "utime"}:
continue
if k in known_control_ids:
continue
out[str(k)] = normalize_mdy_value(v)
return out
def get_related(self, obj: MDYPlateOrderStaging) -> list[dict[str, Any]]:
if not obj.related:
return []
if not isinstance(obj.related, list):
return []
out: list[dict[str, Any]] = []
for item in obj.related:
if isinstance(item, dict):
out.append({k: normalize_mdy_value(v) for k, v in item.items()})
else:
# 兜底:若数据被写成非 dict 元素,仍然返回原值,便于排查
out.append({"value": item})
return out
class MDYPlateOrderStagingViewSet(viewsets.ReadOnlyModelViewSet):
"""开版暂存数据查询(只读)。"""
serializer_class = MDYPlateOrderStagingSerializer
# permission_classes = [DjangoModelPermissions]
pagination_class = LimitOffsetPagination
def get_queryset(self):
qs = MDYPlateOrderStaging.objects.all()
# 支持按 raw 中的“设计编号/订单id”过滤按你要求的 query param 名)
design_no = self.request.query_params.get(f"raw__{_PLATE_ORDER_DESIGN_NO_CONTROL_ID}")
if design_no:
qs = qs.filter(**{f"raw__{_PLATE_ORDER_DESIGN_NO_CONTROL_ID}": design_no})
# 默认 -ctimectime 为空的放最后,避免老数据(未回填)跑到前面
return qs.order_by(F("ctime").desc(nulls_last=True), F("id").desc())

View File

@@ -0,0 +1,101 @@
"""MDYPlateOrderStaging 查询 API 测试"""
from datetime import timedelta
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Permission
from django.test import TestCase
from django.utils import timezone
from rest_framework.test import APIClient
from api_v1.models import MDYPlateOrderStaging
User = get_user_model()
class MDYPlateOrderStagingAPITestCase(TestCase):
def setUp(self):
self.client = APIClient()
self.user = User.objects.create_user(
username="testuser",
password="testpass123",
email="test@example.com",
)
self.client.force_authenticate(user=self.user)
view_perm = Permission.objects.get(codename="view_mdyplateorderstaging")
self.user.user_permissions.add(view_perm)
now = timezone.now()
# older
self.row1 = MDYPlateOrderStaging.objects.create(
mdy_rowid="row1",
ctime=now - timedelta(days=1),
utime=now - timedelta(days=1, minutes=1),
raw={
"rowid": "row1",
"ctime": "2025-01-01 00:00:00",
"utime": "2025-01-01 00:01:00",
"62d52f4b8d2972284492dd0e": "A001", # 设计编号
"62d52f4b8d2972284492dd2c": "款号A", # 款号名称
},
related=[
{
"rowid": "rel1",
"source": "drawing",
"source_name": "画图",
"设计师名字": "张三",
}
],
)
# newer
self.row2 = MDYPlateOrderStaging.objects.create(
mdy_rowid="row2",
ctime=now,
utime=now,
raw={
"rowid": "row2",
"ctime": "2025-01-02 00:00:00",
"utime": "2025-01-02 00:01:00",
"62d52f4b8d2972284492dd0e": "A002", # 设计编号
"62d52f4b8d2972284492dd2c": "款号B", # 款号名称
},
related=[],
)
def test_list_default_order_and_flatten(self):
resp = self.client.get("/api/v1/mdy-plate-order-staging/?limit=10&offset=0")
self.assertEqual(resp.status_code, 200)
payload = resp.json()
self.assertEqual(payload["count"], 2)
first = payload["results"][0]
self.assertEqual(first["mdy_rowid"], "row2") # -ctime 排序
# raw 已按字段定义“可读化”
self.assertIn("raw", first)
self.assertIn("设计编号", first["raw"])
self.assertEqual(first["raw"]["设计编号"], "A002")
self.assertEqual(first["raw"]["款号名称"], "款号B")
# related 以 list[dict] 返回
self.assertIn("related", first)
self.assertEqual(first["related"], [])
def test_filter_by_raw_design_no(self):
resp = self.client.get(
"/api/v1/mdy-plate-order-staging/?limit=10&offset=0&raw__62d52f4b8d2972284492dd0e=A001"
)
self.assertEqual(resp.status_code, 200)
payload = resp.json()
self.assertEqual(payload["count"], 1)
self.assertEqual(payload["results"][0]["mdy_rowid"], "row1")
self.assertEqual(payload["results"][0]["raw"]["设计编号"], "A001")
self.assertEqual(payload["results"][0]["related"][0]["source"], "drawing")

View File

@@ -236,7 +236,7 @@ class PrintingJobListSerializer(serializers.ModelSerializer):
class Meta:
model = models.PrintingJob
fields = [
'id', 'printing_order', 'printing_order_id', 'product', 'product_name',
'id', 'original_id', 'printing_order', 'printing_order_id', 'product', 'product_name',
'quantity', 'unit', 'size', 'pieces', 'description',
'work_state', 'work_state_display',
'status', 'is_completed', 'business_object_id',
@@ -280,7 +280,7 @@ class PrintingJobDetailSerializer(serializers.ModelSerializer):
class Meta:
model = models.PrintingJob
fields = [
'id', 'printing_order', 'printing_order_id', 'product', 'product_name', 'product_code',
'id', 'original_id', 'printing_order', 'printing_order_id', 'product', 'product_name', 'product_code',
'quantity', 'unit', 'size', 'pieces', 'description',
'work_state', 'work_state_display',
'status', 'status_id', 'is_completed', 'has_started', 'business_object_id',
@@ -312,7 +312,7 @@ class PrintingJobCreateUpdateSerializer(serializers.ModelSerializer):
class Meta:
model = models.PrintingJob
fields = [
'id', 'printing_order', 'product', 'quantity', 'unit', 'size', 'pieces', 'description',
'id', 'original_id', 'printing_order', 'product', 'quantity', 'unit', 'size', 'pieces', 'description',
'work_state',
'batch_advance_records',
]
@@ -431,7 +431,7 @@ class PlateOrderListSerializer(PlateOrderDesignCodeMixin, serializers.ModelSeria
model = models.PlateOrder
fields = [
'id', 'design_code', 'plate_type', 'plate_date', 'plate_method',
'id', 'original_id', 'design_code', 'plate_type', 'plate_date', 'plate_method',
'plate_image', 'plate_image_url', 'image_name', 'plate_notes', 'reprint_reason',
'urgency_level', 'is_invalid',
'customer', 'customer_name', 'area', 'default_address',
@@ -504,7 +504,7 @@ class PlateOrderDetailSerializer(PlateOrderDesignCodeMixin, serializers.ModelSer
class Meta:
model = models.PlateOrder
fields = [
'id', 'design_code', 'plate_type', 'plate_date', 'plate_method',
'id', 'original_id', 'design_code', 'plate_type', 'plate_date', 'plate_method',
'plate_image', 'plate_image_url', 'image_name', 'plate_notes', 'reprint_reason',
'urgency_level', 'is_invalid',
'customer', 'customer_name', 'customer_phone', 'area', 'default_address',
@@ -562,7 +562,7 @@ class PlateOrderCreateUpdateSerializer(serializers.ModelSerializer):
class Meta:
model = models.PlateOrder
fields = [
"id", "design_code", "plate_type", "plate_date", "plate_method",
"id", "original_id", "design_code", "plate_type", "plate_date", "plate_method",
"plate_image", "image_name", "plate_notes", "reprint_reason",
"urgency_level", "is_invalid",
"customer", "area", "default_address",