forked from erp-dev/erp
feat: upload mdy stagging to qiniu beta
This commit is contained in:
@@ -0,0 +1,192 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
|
||||
from api_v1.mdy_plate_order_staging_qiniu_upload import (
|
||||
DEFAULT_DOWNLOAD_CHUNK_SIZE,
|
||||
DEFAULT_DOWNLOAD_TIMEOUT,
|
||||
DEFAULT_MAX_IMAGE_SIZE,
|
||||
DEFAULT_SPOOL_MAX_SIZE,
|
||||
build_default_qiniu_rate_limiter,
|
||||
is_qiniu_url,
|
||||
iter_plate_image_work_items,
|
||||
process_plate_image_work_item,
|
||||
should_process_audit,
|
||||
)
|
||||
from api_v1.models import (
|
||||
MDYPlateOrderStaging,
|
||||
MDYPlateOrderStagingQiniuImageUploadAudit,
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = '将明道云开版暂存 raw.开版图 图片流式上传到七牛，并记录逐图审计'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('--dry-run', action='store_true', help='只扫描和汇总，不写数据库、不下载、不上传')
|
||||
parser.add_argument('--limit', type=int, default=None, help='最多处理多少张候选图片；不传则处理全部')
|
||||
parser.add_argument('--start-id', type=int, default=None, help='仅扫描 id >= start-id 的暂存记录')
|
||||
parser.add_argument('--db-chunk-size', type=int, default=500, help='Django iterator 每批读取的 staging 行数')
|
||||
parser.add_argument('--qps', type=float, default=None, help='下载/上传限速；默认 settings 或 2 qps；<=0 表示不限速')
|
||||
parser.add_argument('--retry-failed', action='store_true', help='重试 status=failed 的审计记录')
|
||||
parser.add_argument(
|
||||
'--retry-stale-processing',
|
||||
action='store_true',
|
||||
help='重试长时间停留在 processing 的审计记录',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--stale-after-minutes',
|
||||
type=int,
|
||||
default=120,
|
||||
help='processing 超过多少分钟视为可重试的陈旧记录',
|
||||
)
|
||||
parser.add_argument('--max-attempts', type=int, default=3, help='单张图最多尝试次数；不限制请传 0')
|
||||
parser.add_argument(
|
||||
'--max-image-size-mb',
|
||||
type=int,
|
||||
default=DEFAULT_MAX_IMAGE_SIZE // 1024 // 1024,
|
||||
help='单张图片最大下载体积，超过则失败落审计',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--spool-max-size-mb',
|
||||
type=int,
|
||||
default=DEFAULT_SPOOL_MAX_SIZE // 1024 // 1024,
|
||||
help='单张图内存缓冲上限，超过后临时落盘',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--download-chunk-size-mb',
|
||||
type=int,
|
||||
default=DEFAULT_DOWNLOAD_CHUNK_SIZE // 1024 // 1024,
|
||||
help='流式下载的单块大小',
|
||||
)
|
||||
parser.add_argument('--timeout', type=int, default=DEFAULT_DOWNLOAD_TIMEOUT, help='单次下载超时时间（秒）')
|
||||
parser.add_argument('--progress-every', type=int, default=100, help='每处理多少张候选图片输出一次进度')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
dry_run = options['dry_run']
|
||||
limit = options['limit']
|
||||
start_id = options['start_id']
|
||||
retry_failed = options['retry_failed']
|
||||
retry_stale_processing = options['retry_stale_processing']
|
||||
stale_after_minutes = options['stale_after_minutes']
|
||||
max_attempts = options['max_attempts'] or None
|
||||
db_chunk_size = options['db_chunk_size']
|
||||
progress_every = options['progress_every']
|
||||
|
||||
stale_before = timezone.now() - timedelta(minutes=stale_after_minutes)
|
||||
rate_limiter = None if dry_run else build_default_qiniu_rate_limiter(options['qps'])
|
||||
|
||||
qs = MDYPlateOrderStaging.objects.only('id', 'mdy_rowid', 'raw').order_by('id')
|
||||
if start_id is not None:
|
||||
qs = qs.filter(id__gte=start_id)
|
||||
|
||||
stats = {
|
||||
'staging_rows': 0,
|
||||
'image_items': 0,
|
||||
'planned': 0,
|
||||
'uploaded': 0,
|
||||
'reused': 0,
|
||||
'failed': 0,
|
||||
'skipped': 0,
|
||||
'already_qiniu': 0,
|
||||
}
|
||||
last_progress_count = 0
|
||||
|
||||
for staging in qs.iterator(chunk_size=db_chunk_size):
|
||||
stats['staging_rows'] += 1
|
||||
for work_item in iter_plate_image_work_items(staging):
|
||||
stats['image_items'] += 1
|
||||
if dry_run:
|
||||
action, reason = self._classify_dry_run(
|
||||
work_item,
|
||||
retry_failed=retry_failed,
|
||||
retry_stale_processing=retry_stale_processing,
|
||||
stale_before=stale_before,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
if action == 'planned':
|
||||
stats['planned'] += 1
|
||||
elif reason == 'original_url_already_qiniu':
|
||||
stats['already_qiniu'] += 1
|
||||
stats['skipped'] += 1
|
||||
else:
|
||||
stats['skipped'] += 1
|
||||
processed_count = stats['planned']
|
||||
else:
|
||||
result = process_plate_image_work_item(
|
||||
work_item,
|
||||
retry_failed=retry_failed,
|
||||
retry_stale_processing=retry_stale_processing,
|
||||
stale_before=stale_before,
|
||||
max_attempts=max_attempts,
|
||||
rate_limiter=rate_limiter,
|
||||
max_image_size=options['max_image_size_mb'] * 1024 * 1024,
|
||||
spool_max_size=options['spool_max_size_mb'] * 1024 * 1024,
|
||||
chunk_size=options['download_chunk_size_mb'] * 1024 * 1024,
|
||||
timeout=options['timeout'],
|
||||
)
|
||||
if result.action == 'uploaded':
|
||||
stats['uploaded'] += 1
|
||||
elif result.action == 'reused':
|
||||
stats['reused'] += 1
|
||||
elif result.action == 'failed':
|
||||
stats['failed'] += 1
|
||||
elif result.reason == 'original_url_already_qiniu':
|
||||
stats['already_qiniu'] += 1
|
||||
stats['skipped'] += 1
|
||||
else:
|
||||
stats['skipped'] += 1
|
||||
processed_count = stats['uploaded'] + stats['reused'] + stats['failed']
|
||||
|
||||
if (
|
||||
progress_every > 0
|
||||
and processed_count > 0
|
||||
and processed_count % progress_every == 0
|
||||
and processed_count != last_progress_count
|
||||
):
|
||||
self.stdout.write(f'progress: {stats}')
|
||||
last_progress_count = processed_count
|
||||
|
||||
if limit is not None and processed_count >= limit:
|
||||
self.stdout.write(self.style.SUCCESS(f'limited stop: {stats}'))
|
||||
return
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(str(stats)))
|
||||
|
||||
def _classify_dry_run(
|
||||
self,
|
||||
work_item,
|
||||
*,
|
||||
retry_failed: bool,
|
||||
retry_stale_processing: bool,
|
||||
stale_before,
|
||||
max_attempts: int | None,
|
||||
) -> tuple[str, str]:
|
||||
if is_qiniu_url(work_item.original_url):
|
||||
return 'skipped', 'original_url_already_qiniu'
|
||||
|
||||
audit = (
|
||||
MDYPlateOrderStagingQiniuImageUploadAudit.objects.filter(
|
||||
staging=work_item.staging,
|
||||
source=work_item.source,
|
||||
attachment_index=work_item.attachment_index,
|
||||
source_field=work_item.source_field,
|
||||
original_url=work_item.original_url,
|
||||
)
|
||||
.only('id', 'status', 'attempts', 'last_attempt_at')
|
||||
.first()
|
||||
)
|
||||
if audit is None:
|
||||
return 'planned', 'new_audit'
|
||||
|
||||
can_process, reason = should_process_audit(
|
||||
audit,
|
||||
retry_failed=retry_failed,
|
||||
retry_stale_processing=retry_stale_processing,
|
||||
stale_before=stale_before,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
if can_process:
|
||||
return 'planned', reason
|
||||
return 'skipped', reason
|
||||
Reference in New Issue
Block a user