""" 美图API服务模块 - 处理与美图API的交互 """ import os import time import uuid import json import asyncio import logging import aiohttp import aiofiles from pathlib import Path from typing import Optional, Dict, Any, Callable, List, Tuple # 配置日志 logger = logging.getLogger(__name__) # 统计信息 class MeituServiceStats: def __init__(self): self.total_requests = 0 self.successful_requests = 0 self.failed_requests = 0 self.timeout_requests = 0 self.network_error_requests = 0 self.last_success_time = None self.last_error_time = None self.last_error_message = None def record_success(self): self.total_requests += 1 self.successful_requests += 1 self.last_success_time = time.time() def record_failure(self, error_type="general", message=""): self.total_requests += 1 self.failed_requests += 1 self.last_error_time = time.time() self.last_error_message = message if error_type == "timeout": self.timeout_requests += 1 elif error_type == "network": self.network_error_requests += 1 def get_success_rate(self): if self.total_requests == 0: return 0.0 return self.successful_requests / self.total_requests * 100 def get_stats(self): return { "total_requests": self.total_requests, "successful_requests": self.successful_requests, "failed_requests": self.failed_requests, "timeout_requests": self.timeout_requests, "network_error_requests": self.network_error_requests, "success_rate": self.get_success_rate(), "last_success_time": self.last_success_time, "last_error_time": self.last_error_time, "last_error_message": self.last_error_message } # 全局统计实例 _service_stats = MeituServiceStats() class MeituServiceError(Exception): """美图服务异常""" pass class MeituTimeoutError(MeituServiceError): """美图服务超时异常""" pass class MeituNetworkError(MeituServiceError): """美图服务网络异常""" pass class MeituAPIService: """美图API服务类，处理与美图API的所有交互""" # 服务状态 _service_status = { "available": False, "last_check": 0, "error": None } # 支持的处理模式 SUPPORTED_MODES = { "crystal": "极速重绘", "standard": "标准处理", "enhance": "增强处理", "hdr": "HDR处理", "portrait": "人像优化" } def __init__(self, api_url: str = None): """ 初始化美图API服务 :param api_url: API基础URL，如果为None则使用环境变量或默认值 """ # self.api_url = api_url or os.environ.get('MEITU_API_URL', 'http://89358zi786.goho.co:38226') self.api_url = api_url or os.environ.get('MEITU_API_URL', 'https://127.0.0.1:6668') # 本地 self.stats = _service_stats # 使用全局统计实例 self._active_tasks = set() # 追踪活跃任务，确保高并发安全 self._task_cancellation_tokens = {} # 任务取消令牌 async def process_image(self, image_path: str, mode: str, output_dir: Path, progress_callback: Optional[Callable[[int, str], None]] = None) -> Dict[str, Any]: """ 处理图片 :param image_path: 图片路径 :param mode: 处理模式（crystal, standard等） :param output_dir: 输出目录 :param progress_callback: 进度回调函数 :return: 处理结果，包含任务ID和结果图片路径 """ # 检查模式是否支持 if mode not in self.SUPPORTED_MODES: supported_modes = ", ".join(self.SUPPORTED_MODES.keys()) raise MeituServiceError(f"不支持的处理模式: {mode}，支持的模式: {supported_modes}") # 检查图片文件是否存在 if not os.path.exists(image_path): raise MeituServiceError(f"图片文件不存在: {image_path}") try: # 生成唯一文件名 unique_filename = os.path.basename(image_path) file_extension = os.path.splitext(unique_filename)[1] # 上传图片并获取任务ID task_id = await self._upload_image(image_path, unique_filename, mode) if not task_id: raise MeituServiceError("美图API上传失败，未获取到任务ID") # 记录任务ID类型和值 logger.info(f"获取到任务ID: {task_id}, 类型: {type(task_id)}") # 注册活跃任务 self._active_tasks.add(task_id) self._task_cancellation_tokens[task_id] = False # 轮询等待处理完成 - 确保异常正确传播 try: await self._wait_for_completion(task_id, progress_callback) except (MeituTimeoutError, MeituServiceError) as e: logger.error(f"美图处理被终止: {str(e)}") # 立即清理任务并重新抛出异常 await self.cleanup_failed_task(task_id) raise finally: # 确保任务总是从活跃列表中移除 self._active_tasks.discard(task_id) self._task_cancellation_tokens.pop(task_id, None) logger.info(f"任务{task_id}已从活跃列表中移除") # 下载处理后的图片 processed_filename = f"processed_{task_id}_{int(time.time())}{file_extension}" processed_path = output_dir / processed_filename # 确保输出目录存在 output_dir.mkdir(exist_ok=True, parents=True) # 下载结果图片 try: await self._download_result(task_id, processed_path) except Exception as download_error: logger.error(f"下载处理结果失败: {str(download_error)}") raise MeituServiceError(f"下载处理结果失败: {str(download_error)}") # 安全处理processing_time计算 processing_time = 0 try: if '_' in task_id: # 尝试从task_id中提取时间戳 timestamp_str = task_id.split('_')[-1] logger.info(f"从task_id提取时间戳: {timestamp_str}, 类型: {type(timestamp_str)}") # 安全转换为整数 try: timestamp = int(timestamp_str) processing_time = time.time() - timestamp logger.info(f"计算处理时间: {processing_time}秒") except (ValueError, TypeError) as e: logger.warning(f"时间戳转换失败: {e}") else: logger.warning(f"任务ID不包含下划线分隔符: {task_id}") except Exception as time_error: logger.warning(f"处理时间计算错误: {str(time_error)}") # 记录成功 self.stats.record_success() logger.info(f"美图服务处理成功 - 任务ID: {task_id}, 耗时: {processing_time:.2f}秒") return { "task_id": task_id, "processed_path": processed_path, "processed_filename": processed_filename, "mode": mode, "mode_name": self.SUPPORTED_MODES.get(mode, "未知模式"), "processing_time": processing_time } except MeituTimeoutError as e: self.stats.record_failure("timeout", str(e)) logger.error(f"美图服务超时: {str(e)}") raise except MeituNetworkError as e: self.stats.record_failure("network", str(e)) logger.error(f"美图服务网络错误: {str(e)}") raise except MeituServiceError as e: self.stats.record_failure("general", str(e)) logger.error(f"美图服务错误: {str(e)}") raise except Exception as e: self.stats.record_failure("unknown", str(e)) logger.error(f"美图API处理失败: {str(e)}") import traceback logger.error(f"异常堆栈: {traceback.format_exc()}") raise MeituServiceError(f"美图API处理失败: {str(e)}") async def process_batch(self, image_paths: List[str], mode: str, output_dir: Path, max_concurrent: int = 3) -> List[Dict[str, Any]]: """ 批量处理图片 :param image_paths: 图片路径列表 :param mode: 处理模式 :param output_dir: 输出目录 :param max_concurrent: 最大并发数 :return: 处理结果列表 """ results = [] semaphore = asyncio.Semaphore(max_concurrent) async def process_single(image_path): async with semaphore: try: result = await self.process_image(image_path, mode, output_dir) return { "success": True, "result": result, "image_path": image_path } except Exception as e: logger.error(f"处理图片失败: {image_path}, 错误: {str(e)}") return { "success": False, "error": str(e), "image_path": image_path } # 创建任务 tasks = [process_single(path) for path in image_paths] # 等待所有任务完成 try: results = await asyncio.gather(*tasks) except Exception as e: logger.error(f"批量处理任务异常: {str(e)}") # 即使有异常，也返回已完成的结果 results = [{"success": False, "error": f"批量处理异常: {str(e)}", "image_path": path} for path in image_paths] return results async def _upload_image(self, image_path: str, filename: str, mode: str) -> str: """ 上传图片到美图API :param image_path: 图片路径 :param filename: 文件名 :param mode: 处理模式 :return: 任务ID """ logger.info(f"上传图片到美图API - 文件: {filename}, 模式: {mode}") # 检查文件大小 file_size = os.path.getsize(image_path) if file_size > 10 * 1024 * 1024: # 10MB raise MeituServiceError(f"文件太大: {file_size / 1024 / 1024:.2f}MB，最大允许10MB") # 检查文件类型 content_type = await self._get_content_type(filename) if not content_type.startswith('image/'): raise MeituServiceError(f"不支持的文件类型: {content_type}") # 重试逻辑 max_retries = 3 retry_count = 0 last_error = None while retry_count < max_retries: try: # 设置上传专用的超时配置 upload_timeout = aiohttp.ClientTimeout(total=60, connect=10) # 上传允许更长时间 async with aiohttp.ClientSession(timeout=upload_timeout) as session: with open(image_path, 'rb') as f: data = aiohttp.FormData() data.add_field(filename, f.read(), filename=filename, content_type=content_type) async with session.post(f'{self.api_url}/add_task?scene={mode}', data=data, ssl=False) as resp: if resp.status != 200: error_text = await resp.text() logger.error(f"美图API上传失败 - 状态码: {resp.status}, 响应: {error_text}") raise MeituServiceError(f"美图API上传失败: {error_text}") response_text = await resp.text() logger.info(f"美图API上传响应: {response_text}") try: result = json.loads(response_text) if result.get('code') != 0: raise MeituServiceError(result.get('message', '美图API上传失败')) task_id = result.get('id') logger.info(f"美图API上传成功 - 任务ID: {task_id}") return task_id except json.JSONDecodeError: logger.error(f"美图API响应解析失败: {response_text}") raise MeituServiceError("美图API响应格式错误") except Exception as e: retry_count += 1 last_error = e wait_time = 2 ** retry_count # 指数退避 logger.warning(f"上传失败，正在重试 ({retry_count}/{max_retries})，等待 {wait_time} 秒: {str(e)}") await asyncio.sleep(wait_time) # 所有重试都失败 raise last_error or MeituServiceError("上传图片失败，已达到最大重试次数") async def _wait_for_completion(self, task_id: str, progress_callback: Optional[Callable[[int, str], None]] = None, max_wait_time: int = 180, # 最长等待3分钟（缩短超时时间） check_interval: int = 2 # 每2秒检查一次 ) -> None: """ 轮询等待处理完成 :param task_id: 任务ID :param progress_callback: 进度回调函数 :param max_wait_time: 最长等待时间（秒） :param check_interval: 检查间隔（秒） """ logger.info(f"开始等待美图API处理完成 - 任务ID: {task_id}") start_time = time.time() progress = 0 # 连续失败计数 consecutive_failures = 0 max_consecutive_failures = 3 # 减少连续失败次数，更快失败 # 早期检测到的致命错误类型 detected_fatal_errors = set() # 创建当前任务的取消令牌 is_cancelled = False async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: for i in range(max_wait_time // check_interval): # 检查是否被取消 - 支持外部取消令牌 if is_cancelled or self._task_cancellation_tokens.get(task_id, False): logger.error(f"美图处理任务被取消: {task_id}") await self.cleanup_failed_task(task_id) raise MeituTimeoutError("美图处理任务被取消") # 检查任务是否还在活跃列表中 if task_id not in self._active_tasks: logger.warning(f"任务{task_id}不在活跃列表中，可能已被清理") raise MeituTimeoutError("任务已被停止") # 计算预估进度 elapsed = time.time() - start_time if elapsed < max_wait_time: # 假设进度在90%以内线性增长 progress = min(90, int(90 * elapsed / max_wait_time)) if progress_callback: progress_callback(progress, f"美图API处理中 - {progress}%") try: # 在每次API调用前再次检查取消状态 if self._task_cancellation_tokens.get(task_id, False): logger.info(f"任务{task_id}在API调用前被取消") raise MeituTimeoutError("任务被取消") async with session.get(f'{self.api_url}/try_get?taskid={task_id}', ssl=False) as resp: if resp.status != 200: consecutive_failures += 1 error_text = await resp.text() logger.error(f"美图API状态检查失败 - 状态码: {resp.status}, 响应: {error_text}") # 如果连续失败次数过多，抛出异常 if consecutive_failures >= max_consecutive_failures: logger.error(f"美图API连续失败{consecutive_failures}次，尝试清理任务") await self.cleanup_failed_task(task_id) raise MeituTimeoutError(f"美图API连续失败{consecutive_failures}次: {error_text}") # 否则继续等待 await asyncio.sleep(check_interval * 2) # 失败时等待更长时间 continue # 重置失败计数 consecutive_failures = 0 # 获取响应内容 try: response_text = await resp.text() logger.debug(f"美图API状态检查响应: {response_text}") # 解析JSON响应 try: result = json.loads(response_text) logger.debug(f"解析的JSON响应: {result}") # 获取code字段 if 'code' not in result: logger.warning(f"美图API响应中缺少code字段: {result}") # 继续等待，不抛出异常 await asyncio.sleep(check_interval) continue # 安全获取code值 try: code = result['code'] # 确保code是整数 if not isinstance(code, (int, float)): code = int(code) if str(code).isdigit() else -1 except (ValueError, TypeError) as e: logger.warning(f"无法转换code为整数: {e}, 原始值: {result.get('code')}") code = -1 # 检查处理状态 if code < 0: error_message = str(result.get('message', '美图API处理失败')) logger.error(f"美图API处理失败: {error_message}") # 检查是否是超时或严重错误，如果是则立即终止 timeout_indicators = [ "TimeoutException", "selenium.common.exceptions.TimeoutException", "WebDriver", "已终止处理", "处理超时", "连接超时", "响应超时" ] is_timeout = any(indicator in error_message for indicator in timeout_indicators) if is_timeout: logger.error(f"检测到严重错误/超时，立即终止美图处理: {error_message}") raise MeituTimeoutError("美图处理超时或遇到严重错误，已终止处理") raise MeituServiceError(error_message) elif code == 0: logger.info(f"美图API处理完成 - 任务ID: {task_id}") if progress_callback: progress_callback(100, "美图API处理完成") return else: # 其他状态码，继续等待 logger.info(f"美图API处理中 - 状态码: {code}") except json.JSONDecodeError as e: logger.warning(f"JSON解析失败: {e}, 响应内容: {response_text[:100]}...") # 继续等待，不抛出异常 except Exception as e: error_str = str(e) logger.error(f"处理响应内容异常: {error_str}") import traceback logger.error(f"异常堆栈: {traceback.format_exc()}") # 检查是否是致命错误，如果是则立即终止 fatal_error_indicators = [ "MeituTimeoutError", "美图处理超时", "已终止处理", "WebDriver", "TimeoutException" ] is_fatal = any(indicator in error_str for indicator in fatal_error_indicators) if is_fatal: # 记录检测到的致命错误类型 for indicator in fatal_error_indicators: if indicator in error_str: detected_fatal_errors.add(indicator) logger.error(f"检测到致命错误，立即终止处理: {error_str}") logger.error(f"已检测到的致命错误类型: {detected_fatal_errors}") is_cancelled = True # 设置取消标志 await self.cleanup_failed_task(task_id) # 立即清理 raise MeituTimeoutError(f"美图处理遇到致命错误: {error_str}") # 增加连续失败计数 consecutive_failures += 1 if consecutive_failures >= max_consecutive_failures: logger.error(f"连续失败{consecutive_failures}次，终止处理") raise MeituServiceError(f"美图处理连续失败{consecutive_failures}次") except asyncio.TimeoutError: consecutive_failures += 1 logger.error(f"美图API请求超时 - 第{consecutive_failures}次") # 如果连续超时次数过多，抛出异常 if consecutive_failures >= max_consecutive_failures: logger.error(f"美图API连续超时{consecutive_failures}次，尝试清理任务") await self.cleanup_failed_task(task_id) raise MeituTimeoutError(f"美图API连续超时{consecutive_failures}次，任务可能已失败") await asyncio.sleep(check_interval * 2) continue except aiohttp.ClientError as e: consecutive_failures += 1 logger.error(f"网络请求异常: {str(e)} - 第{consecutive_failures}次") # 如果连续失败次数过多，抛出异常 if consecutive_failures >= max_consecutive_failures: logger.error(f"美图API网络连接连续失败{consecutive_failures}次，尝试清理任务") await self.cleanup_failed_task(task_id) raise MeituNetworkError(f"美图API网络连接连续失败{consecutive_failures}次") await asyncio.sleep(check_interval * 2) continue except Exception as e: if isinstance(e, MeituServiceError): raise consecutive_failures += 1 logger.error(f"美图API状态检查异常: {str(e)} - 第{consecutive_failures}次") import traceback logger.error(f"异常堆栈: {traceback.format_exc()}") # 如果连续失败次数过多，抛出异常 if consecutive_failures >= max_consecutive_failures: logger.error(f"美图API状态检查连续异常{consecutive_failures}次，尝试清理任务") await self.cleanup_failed_task(task_id) raise MeituServiceError(f"美图API状态检查连续异常{consecutive_failures}次: {str(e)}") await asyncio.sleep(check_interval * 2) continue await asyncio.sleep(check_interval) # 超时 logger.error(f"美图API处理超时({max_wait_time}秒)，尝试清理任务") await self.cleanup_failed_task(task_id) raise MeituTimeoutError(f"美图API处理超时({max_wait_time}秒) - 任务ID: {task_id}") async def _download_result(self, task_id: str, output_path: Path) -> None: """ 下载处理结果 :param task_id: 任务ID :param output_path: 输出路径 """ logger.info(f"开始下载美图API处理结果 - 任务ID: {task_id}, 输出路径: {output_path}") # 确保输出目录存在 output_path.parent.mkdir(exist_ok=True, parents=True) # 重试逻辑 max_retries = 3 retry_count = 0 last_error = None while retry_count < max_retries: try: # 设置下载专用的超时配置 download_timeout = aiohttp.ClientTimeout(total=120, connect=10) # 下载允许更长时间 async with aiohttp.ClientSession(timeout=download_timeout) as session: async with session.get(f'{self.api_url}/get_image?taskid={task_id}', ssl=False) as resp: if resp.status != 200: error_text = await resp.text() logger.error(f"美图API下载失败 - 状态码: {resp.status}, 响应: {error_text}") raise MeituServiceError(f"美图API下载失败: {error_text}") # 读取图片数据 image_data = await resp.read() if not image_data: raise MeituServiceError("美图API返回空图片数据") # 保存图片 async with aiofiles.open(output_path, 'wb') as f: await f.write(image_data) # 安全获取文件大小 try: file_size = len(image_data) logger.info(f"美图API结果下载成功 - 任务ID: {task_id}, 文件大小: {file_size} 字节") except Exception as size_error: logger.warning(f"获取文件大小失败: {str(size_error)}") logger.info(f"美图API结果下载成功 - 任务ID: {task_id}") # 验证文件是否成功保存 if not output_path.exists() or output_path.stat().st_size == 0: raise MeituServiceError("图片保存失败或文件大小为0") return except MeituServiceError: # 直接抛出服务特定错误 raise except Exception as e: retry_count += 1 last_error = e wait_time = 2 ** retry_count # 指数退避 logger.warning(f"下载失败，正在重试 ({retry_count}/{max_retries})，等待 {wait_time} 秒: {str(e)}") await asyncio.sleep(wait_time) # 所有重试都失败 error_msg = str(last_error) if last_error else "未知错误" logger.error(f"下载结果失败，已达到最大重试次数: {error_msg}") raise MeituServiceError(f"下载结果失败，已达到最大重试次数: {error_msg}") @staticmethod async def _get_content_type(filename: str) -> str: """ 根据文件名获取内容类型 :param filename: 文件名 :return: 内容类型 """ ext = os.path.splitext(filename)[1].lower() content_types = { '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png', '.gif': 'image/gif', '.webp': 'image/webp', '.bmp': 'image/bmp' } return content_types.get(ext, 'application/octet-stream') @classmethod async def test_connection(cls, api_url: str = None) -> Tuple[bool, str]: """ 测试与美图API的连接 :param api_url: API URL，如果为None则使用默认值 :return: (连接是否成功, 状态消息) """ # 如果最后检查时间在30秒内，直接返回缓存的状态 if time.time() - cls._service_status["last_check"] < 30: return ( cls._service_status["available"], "服务在线" if cls._service_status["available"] else f"服务离线: {cls._service_status['error']}" ) service = MeituAPIService(api_url) try: async with aiohttp.ClientSession() as session: # 尝试健康检查端点 try: async with session.get(f'{service.api_url}/health', ssl=False, timeout=5) as resp: if resp.status == 200: cls._service_status = { "available": True, "last_check": time.time(), "error": None } return True, "服务在线" except: # 健康检查失败，尝试其他端点 pass # 尝试调用API的其他端点 try: async with session.get(f'{service.api_url}/', ssl=False, timeout=5) as resp: cls._service_status = { "available": resp.status < 500, "last_check": time.time(), "error": f"状态码: {resp.status}" if resp.status >= 400 else None } return cls._service_status["available"], "服务可能在线，但健康检查失败" except Exception as e: cls._service_status = { "available": False, "last_check": time.time(), "error": str(e) } return False, f"服务离线: {str(e)}" except Exception as e: cls._service_status = { "available": False, "last_check": time.time(), "error": str(e) } logger.error(f"美图API连接测试失败: {str(e)}") return False, f"服务离线: {str(e)}" async def cancel_task(self, task_id: str) -> bool: """取消处理任务""" try: # 设置取消任务的超时配置 cancel_timeout = aiohttp.ClientTimeout(total=15, connect=5) # 取消操作超时短一些 async with aiohttp.ClientSession(timeout=cancel_timeout) as session: async with session.post(f'{self.api_url}/cancel_task', data={'taskid': task_id}, ssl=False) as resp: if resp.status == 200: logger.info(f"任务取消成功: {task_id}") return True else: logger.warning(f"任务取消失败: {task_id}, 状态码: {resp.status}") return False except Exception as e: logger.error(f"取消任务异常: {task_id}, 错误: {str(e)}") return False async def cancel_all_tasks(self) -> None: """取消所有活跃任务""" logger.warning(f"取消所有活跃任务，共{len(self._active_tasks)}个") for task_id in list(self._active_tasks): self._task_cancellation_tokens[task_id] = True await self.cleanup_failed_task(task_id) self._active_tasks.clear() self._task_cancellation_tokens.clear() async def cleanup_failed_task(self, task_id: str) -> None: """清理失败的任务""" try: # 尝试取消任务 await self.cancel_task(task_id) # 清理可能的临时文件 # 这里可以添加更多清理逻辑 logger.info(f"失败任务清理完成: {task_id}") except Exception as e: logger.warning(f"清理失败任务异常: {task_id}, 错误: {str(e)}") @classmethod def get_service_stats(cls) -> Dict[str, Any]: """获取服务统计信息""" return _service_stats.get_stats() @classmethod def reset_stats(cls) -> None: """重置服务统计信息""" global _service_stats _service_stats = MeituServiceStats() @classmethod def get_supported_modes(cls) -> Dict[str, str]: """获取支持的处理模式""" return cls.SUPPORTED_MODES