update 优化预处理，处理，推送等逻辑

SourceCode/DataMiddleware/app/data_process/pre_process.py

@@ -23,17 +23,18 @@ class PreProcess:
             self._logger.info(f"开始预处理项目：{project.sub_project_name}[{project.project_id}_{project.standard_version}] ")
             self._store.update_sub_project_status(project.id, 21)
             self.separate_ai_calls = separate_ai_calls
+            res_data = []
             raw_data = self.read_excels(project.file_paths)
             if self.separate_ai_calls:
                 for filename, sheets in raw_data.items():
                     excel_data = self.normalize_data({filename: sheets})
-                    self.call_ai(project.id, excel_data)
+                    data = self.call_ai(project.id, excel_data)
+                    res_data.extend(data)
             else:
                 excel_data = self.normalize_data(raw_data)
-                self.call_ai(project.id, excel_data)
-            project.items = self._data[project.id].items
+                res_data = self.call_ai(project.id, excel_data)
+            project.items = res_data
             self._store.re_insert_sub_project(project)
-            del self._data[project.id]
             self._logger.info(f"结束预处理项目：{project.sub_project_name}[{project.project_id}_{project.standard_version}] [设备条数：{len(project.items)}]")
             self._store.update_sub_project_status(project.id, 31)
             return True
@@ -164,32 +165,15 @@ class PreProcess:
         return prompt
 
     def call_ai(self, project_id:int, excel_data):
-        project = SubProjectModel(project_id=project_id)
         api_key = utils.get_config_value("fastgpt.api_key_pre_process")
-        if self.separate_ai_calls:
-            # 初始化self._data[project_no]，避免在循环中重复初始化
-            for data in excel_data:
-                prompt = self.prompt_template([data])
-                response = self._ai.call_ai(prompt,api_key)
-                project.items.extend(self.format_data(project_id, response))
-                # 更新数据部分
-                # project.items.extend(self.format_data(project_id, response["data"]))
-                # project.completion_tokens += response["completion_tokens"]
-                # project.prompt_tokens += response["prompt_tokens"]
-                # project.total_tokens += response["total_tokens"]
-        else:
-            prompt = self.prompt_template(excel_data)
-            response = self._ai.call_ai(prompt,api_key)
-            project.items.extend(self.format_data(project_id, response))
+        prompt = self.prompt_template(excel_data)
+        json_data = self._ai.call_ai(prompt, api_key)
+        data = self.format_data(project_id, json_data)
+        return data
 
-            # project.completion_tokens = response["completion_tokens"]
-            # project.prompt_tokens = response["prompt_tokens"]
-            # project.total_tokens = response["total_tokens"]
-            # project.items = self.format_data(project_id, response["data"])
-        self._data[project_id] = project
-    def format_data(self,project_id,new_data) ->list[SubProjectItemModel]:
-        formatted_data = []
-        for data in new_data:
+    def format_data(self, project_id, data_list) ->list[SubProjectItemModel]:
+        formatted_list = []
+        for data in data_list:
             try:
                 item = SubProjectItemModel(
                     sub_project_id=project_id,
@@ -205,11 +189,11 @@ class PreProcess:
                     item.device_unit = data['u']
                 if data.get('c'):
                     item.device_count = data['c']
-                formatted_data.append(item)
+                formatted_list.append(item)
             except Exception as e:
                 self._logger.error(f"格式化数据时出错: {data} {e}")
-
-        return formatted_data
+        self._logger.debug(f"格式化数据数量: {len(formatted_list)}")
+        return formatted_list
 
     def run_1(self, sub_project: SubProjectModel) ->bool:
         self._logger.info(f"开始预处理项目：{sub_project.sub_project_name}[{sub_project.project_id}_{sub_project.standard_version}] ")

SourceCode/DataMiddleware/app/data_process/process.py

@@ -18,13 +18,7 @@ class Process:
             self._store.update_sub_project_status(sub_project.id, 22)
             project_items = self._store.query_sub_project_items_by_project(sub_project.id)
             for project_item in project_items:
-                try:
-                    data = self.call_ai(project_item)
-                    self._store.update_sub_project_item_standard_no(project_item.id, data.standard_no)
-                except Exception as e:
-                    self._logger.error(f"处理数据失败:{e}")
-                    self._store.update_sub_project_item_process_status(project_item.id, 2)
-                    continue
+                self.process_item(project_item)
             self._store.update_sub_project_status(sub_project.id, 32)
             self._logger.info(f"处理数据完成:{sub_project.sub_project_name}[{sub_project.id}]")
             return True
@@ -36,24 +30,22 @@ class Process:
     def process_item(self, sub_project_item: SubProjectItemModel) ->bool:
         try:
             self._logger.debug(f"开始处理明细:{sub_project_item.device_name}{sub_project_item.device_model}[{sub_project_item.id}]")
-            item = self._store.query_sub_project_item_by_id(sub_project_item.id)
-            data = self.call_ai(item)
+            self._store.update_sub_project_item_process_status(sub_project_item.id, 1)
+            data = self.call_ai(sub_project_item)
             self._store.update_sub_project_item_standard_no(sub_project_item.id, data.standard_no)
             return True
         except Exception as e:
             self._logger.error(f"处理数据明细:{e}")
-            self._store.update_sub_project_item_process_status(sub_project_item.id, 2)
+            self._store.update_sub_project_item_process_status(sub_project_item.id, 3)
             return False
 
 
-    def call_ai(self, item: SubProjectItemModel) ->SubProjectItemModel:
+    def call_ai(self, item: SubProjectItemModel) -> SubProjectItemModel:
         try:
             self._logger.info(f"开始处理数据:{item.id}")
             msg = """
             请分析提供的json数据，要求：
             1. 根据工作内容、类型、项目名称、规格型号、数量、单位查找计算出定额编号
-            2. 工程的工作内容及类型：{project.work_content}"""
-            msg += """
             3. 提供的数据结构化信息:```typescript
             export interface item { 
             i: // ID
@@ -79,6 +71,9 @@ class Process:
             self._logger.info(f"AI返回结果:{json_data}")
             if not json_data:
                 raise Exception("AI返回结果为空")
+            # 检查 json_data 是否为数组，并取第一项
+            if isinstance(json_data, list):
+                json_data = json_data[0]
             data = SubProjectItemModel(
                 item_id=json_data["i"],
                 standard_no=json_data["s"]
@@ -117,7 +112,7 @@ class Process:
         text += json.dumps(item_ai_json,ensure_ascii=False, separators=(',', ':'))
         return text
 
-    def call_ai_process(self, message:str) ->list[SubProjectItemModel]:
+    def call_ai_batch(self, message:str) ->list[SubProjectItemModel]:
         try:
             api_key = utils.get_config_value("fastgpt.api_key_process")
             self._logger.info(f"开始调用AI:\n {message}")

SourceCode/DataMiddleware/app/data_send/send.py

@@ -45,7 +45,9 @@ class DataSend:
         try:
             self._logger.debug(f"开始远程请求发送数据，数据：{data}" )
             self._store.update_sub_project_item_send_status(data.id, 1)
+
+            self._store.update_sub_project_item_send_status(data.id, 2)
         except Exception as e:
             self._logger.error(f"发送数据失败，数据：{data}，错误信息：{e}")
-            self._store.update_sub_project_item_send_status(data.id, 2)
+            self._store.update_sub_project_item_send_status(data.id, 3)
             return False

SourceCode/DataMiddleware/app/stores/mysql_store.py

@@ -352,7 +352,7 @@ class MysqlStore:
                 update_time=datetime.now()
             )
             return data
-    def insert_sub_project_item(self, project_item: SubProjectItemModel) -> bool:
+    def insert_sub_project_item(self, project_item: SubProjectItemModel):
         sql = "INSERT INTO sub_project_item (project_id,sub_project_id,device_name,device_model,device_unit,device_count,standard_version,standard_no,process_status,send_status,update_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
         params = (
             project_item.project_id,
@@ -368,8 +368,7 @@ class MysqlStore:
             datetime.now(),
         )
         with self._db_helper:
-            self._db_helper.execute_non_query(sql, params)
-            return True
+           return self._db_helper.execute_non_query(sql, params)
     def update_sub_project_item(self, project_item: SubProjectItemModel) -> bool:
         sql = "UPDATE sub_project_item SET device_name= %s,device_model= %s,device_unit= %s,device_count= %s,standard_no = %s,update_time = %s WHERE id = %s"
         params = (
@@ -391,7 +390,7 @@ class MysqlStore:
             self._db_helper.execute_non_query(sql, params)
             return True
     def update_sub_project_item_standard_no(self, item_id:int, standard_no:str):
-        sql = "UPDATE sub_project_item SET standard_no=%s,process_status = 1,process_time = %s,update_time=%s WHERE id = %s"
+        sql = "UPDATE sub_project_item SET standard_no=%s,process_status = 2,process_time = %s,update_time=%s WHERE id = %s"
         params = (standard_no, datetime.now(), datetime.now(), item_id,)
         with self._db_helper:
             self._db_helper.execute_non_query(sql, params)

SourceCode/DataMiddleware/app/ui/project_services.py

@@ -211,8 +211,7 @@ class ProjectService:
             standard_version=project.standard_version,
             standard_no=item.standard_no,
         )
-        self._store.insert_sub_project_item(project_item)
-        return True
+        return self._store.insert_sub_project_item(project_item)
 
     def update_sub_project_item(self, item:SubProjectItemModel):
         project_item = self._store.query_sub_project_item_by_id(item.id)
@@ -244,7 +243,7 @@ class ProjectService:
 
     def _process_sub_project_item(self, sub_project_item:SubProjectItemModel):
         if data_process.process_item(sub_project_item):
-            return True, ''
+            return data_send.send_item_data(sub_project_item),""
         else:
             self._logger.error(f"分析处理失败:{sub_project_item.device_name}")
             return False, '分析处理失败'

SourceCode/DataMiddleware/app/ui/project_views.py

@@ -19,7 +19,7 @@ def index():
 @project.route('/main_project_list')
 def main_project_list():
     page = request.args.get('p', 1, type=int)
-    per_page = request.args.get('pp', 15, type=int)
+    per_page = request.args.get('pp', 10, type=int)
     keyword = request.args.get('k', '')
     projects, total_count = ProjectService.get_all_project_paginated(page, per_page, keyword)
     return render_template('project_list.html', project_list=projects, keyword=keyword, page=page, per_page=per_page, total_count=total_count)
@@ -85,7 +85,7 @@ def sub_project_list(project_id:int):
     project_data = ProjectService.get_project_by_id(project_id)
     keyword = request.args.get('k', '', type=str)
     page = request.args.get('p', 1, type=int)
-    per_page = request.args.get('pp', 15, type=int)
+    per_page = request.args.get('pp', 10, type=int)
     status = request.args.get('s',-1, type=int)
     data_list, total_count = ProjectService.get_all_sub_project_paginated(project_id, page, per_page, keyword, None if status == -1 else status)  # 传递 status 参数
     return render_template('sub_project_list.html', project=project_data ,sub_project_list=data_list, keyword=keyword, page=page, per_page=per_page, total_count=total_count, status=status)  # 传递 status 参数到模板
@@ -95,7 +95,7 @@ def project_item_list(sub_project_id:int):
     send_status = request.args.get('s_s', -1, type=int)
     keyword = request.args.get('k', '', type=str)
     page = request.args.get('p', 1, type=int)
-    per_page = request.args.get('pp', 15, type=int)
+    per_page = request.args.get('pp', 10, type=int)
     sub_project_data= ProjectService.get_sub_project_by_id(sub_project_id)
     sub_items, total_count = ProjectService.get_sub_project_item_list_by_sub_project_paginated(sub_project_id, page, per_page, keyword)
     return render_template('sub_project_item_list.html', sub_project=sub_project_data,items=sub_items, process_status=process_status, send_status=send_status, keyword=keyword, page=page, per_page=per_page, total_count=total_count)
@@ -191,6 +191,7 @@ def save_sub_project_item():
         device_unit = data.get('device_unit')
         device_count = data.get('device_count')
         standard_no = data.get('standard_no')
+        is_query =  data.get('is_query') == True or data.get('is_query') == 'true'
         if  item_id:
             item_data = ProjectService.get_sub_project_item_by_id(item_id)
             # 更新项目
@@ -205,7 +206,10 @@ def save_sub_project_item():
             if not sub_project:
                 return jsonify({'success': False, 'error': '项目工程不存在'}), 404
             new_item = SubProjectItemModel(project_id=sub_project.project_id,sub_project_id=sub_id, device_name=device_name,device_count=float(device_count), device_model=device_model, device_unit=device_unit, standard_no=standard_no)
-            ProjectService.add_sub_project_item(new_item)
+            item_id = ProjectService.add_sub_project_item(new_item)
+        if is_query:
+            ProjectService.start_process_item(item_id)
+
         return jsonify({'success': True})
     except Exception as e:
         return jsonify({'success': False, 'error': str(e)}), 400

SourceCode/DataMiddleware/app/ui/static/sub_project_item_list.js

@@ -26,6 +26,7 @@ function saveSubProjectItem(sub_id) {
 	const deviceUnit = modal.querySelector('#device_unit').value
 	const deviceCount = modal.querySelector('#device_count').value
 	const standardNo = modal.querySelector('#standard_no').value
+	const isQuery = modal.querySelector('#is_query').checked
 	if (deviceName === '') {
 		alert('名称不能为空')
 		return
@@ -52,7 +53,16 @@ function saveSubProjectItem(sub_id) {
 		headers: {
 			'Content-Type': 'application/json',
 		},
-		body: JSON.stringify({ id: id,sub_id: sub_id, device_name: deviceName, device_model: deviceModel, device_unit: deviceUnit, device_count: deviceCount, standard_no: standardNo }),
+		body: JSON.stringify({
+			id: id,
+			sub_id: sub_id,
+			device_name: deviceName,
+			device_model: deviceModel,
+			device_unit: deviceUnit,
+			device_count: deviceCount,
+			standard_no: standardNo ,
+			is_query: isQuery
+		}),
 	})
 		.then((response) => response.json())
 		.then((data) => {

SourceCode/DataMiddleware/app/ui/templates/project/sub_project_item_list.html

@@ -55,15 +55,17 @@
 				<div class="search_box">
 					<select id="process_status_select" class="form-control form-select" title="处理状态选择">
 						<option value="-1" {% if process_status == -1 %}selected{% endif %}>请选择处理状态</option>
-						<option value="1" {% if process_status == 1 %}selected{% endif %}>已处理</option>
-						<option value="2" {% if process_status == 2 %}selected{% endif %}>处理失败</option>
+						<option value="2" {% if process_status == 1 %}selected{% endif %}>已处理</option>
+						<option value="3" {% if process_status == 2 %}selected{% endif %}>处理失败</option>
 						<option value="0" {% if process_status == 0 %}selected{% endif %}>未处理</option>
+						<option value="1" {% if process_status == 3 %}selected{% endif %}>处理中</option>
 					</select>
 					<select id="send_status_select" class="form-control form-select" title="推送状态选择">
 						<option value="-1" {% if send_status == -1 %}selected{% endif %}>请选择推送状态</option>
-						<option value="1" {% if send_status == 1 %}selected{% endif %}>已推送</option>
-						<option value="2" {% if send_status == 2 %}selected{% endif %}>推送失败</option>
+						<option value="2" {% if send_status == 1 %}selected{% endif %}>已推送</option>
+						<option value="3" {% if send_status == 2 %}selected{% endif %}>推送失败</option>
 						<option value="0" {% if send_status == 0 %}selected{% endif %}>未推送</option>
+						<option value="1" {% if send_status == 3 %}selected{% endif %}>推送中</option>
 					</select>
 					<input type="text" id="search_input" title="" class="form-control" placeholder="请输入查询关键字" value="{{ keyword }}" />
 					<button type="button" class="btn btn-info btn-large" onclick="searchData()">查询</button>
@@ -110,8 +112,10 @@
 							</td>
 							<td class="process_status">
 								{% if item.process_status == 1 %}
-								<span class="label label-success">已处理</span>
+								<span class="label label-warning">处理中...</span>
 								{% elif item.process_status == 2 %}
+								<span class="label label-success">已处理</span>
+								{% elif item.process_status == 3 %}
 								<span class="label label-danger">处理失败</span>
 								{% else %}
 								<span class="label label-default">未处理</span>
@@ -119,32 +123,30 @@
 							</td>
 							<td class="send_status">
 								{% if item.send_status == 1 %}
-								<span class="label label-success">已推送</span>
+								<span class="label label-warning">推送中...</span>
 								{% elif item.send_status == 2 %}
+								<span class="label label-success">已推送</span>
+								{% elif item.send_status == 3 %}
 								<span class="label label-danger">推送失败</span>
 								{% else %}
 								<span class="label label-default">未推送</span>
 								{% endif %}
 							</td>
 							<td class="tool">
-								<button class="btn btn-success" onclick="startProcessItem('{{ item.id }}')">
-									{% if item.process_status == 1 %}
-										重新处理
-									{% elif item.process_status == 2 %}
-										重新处理
-									{% else %}
-										开始处理
-									{% endif %}
-								</button>
-								<button class="btn btn-success" onclick="startSendItem('{{ item.id }}')">
-									{% if item.send_status == 1 %}
-										重新发送
-									{% elif item.send_status == 2 %}
-										重新发送
-									{% else %}
-										开始发送
-									{% endif %}
-								</button>
+								{% if item.process_status == 0 %}
+								<button class="btn btn-success" onclick="startProcessItem('{{ item.id }}')">开始处理</button>
+								{% elif item.process_status == 2 %}
+								<button class="btn btn-warning" onclick="startProcessItem('{{ item.id }}')">重新处理</button>
+								{% elif item.process_status == 3 %}
+								<button class="btn btn-danger" onclick="startProcessItem('{{ item.id }}')">重新处理</button>
+								{% endif %}
+								{% if item.send_status == 0 and item.process_status != 1 %}
+								<button class="btn btn-success" onclick="startSendItem('{{ item.id }}')">开始发送</button>
+								{% elif item.send_status == 2 and item.process_status != 1 %}
+								<button class="btn btn-warning" onclick="startSendItem('{{ item.id }}')">重新发送</button>
+								{% elif item.send_status == 3 and item.process_status != 1 %}
+								<button class="btn btn-danger" onclick="startSendItem('{{ item.id }}')">重新发送</button>
+								{% endif %}
 								<button class="btn btn-info" onclick="editSubProjectItem('{{ item.id }}')">编辑</button>
 								<button class="btn btn-danger" onclick="deleteSubProjectItem('{{ item.id }}')">删除</button>
 							</td>
@@ -217,6 +219,10 @@
 						<label>定额编号：</label>
 						<input type="text" title="" id="standard_no" class="form-control" placeholder="请输入定额编号">
 					</div>
+					<div style="display: block;">
+                        <label for="is_query">是否查询知识库：</label>
+                        <input type="checkbox" id="is_query" class="form-control" style="width: auto;flex: 0;">
+                    </div>
 				</div>
 			</div>
 			<div class="modal-footer">

SourceCode/DataMiddleware/app/ui/templates/project/sub_project_list.html

@@ -3,7 +3,7 @@
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>工程工程列表</title>
+    <title>项目工程列表</title>
     <link rel="stylesheet" href="{{ url_for('static', filename='styles.css') }}">
     <script src="{{ url_for('static', filename='utils.js') }}"></script>
     <script src="{{ url_for('static', filename='sub_project_list.js') }}"></script>