Merge branch '1.0.0' of yue/DataMiddleware into dev

SourceCode/DataMiddleware/app/ai/anything_llm.py

@@ -0,0 +1,48 @@
+import requests,utils
+
+class AnythingLLM:
+
+
+    def __init__(self, api_key: str=None, api_url: str=None):
+        self._api_key = api_key or utils.get_config_value('anythingLLm.api_key')
+        self._api_url = api_url or utils.get_config_value('anythingLLm.api_url')
+        self._logger = utils.get_logger()
+        self._headers ={}
+
+    def build_header(self):
+        if self._api_key is None:
+            self._logger.error("anythingLLm.api_key 没有配置")
+            raise Exception("anythingLLm.api_key 没有配置")
+        self._headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json",
+            "accept": "application/json"
+        }
+
+    def call_ai(self, msg: str, workspace:str=None) -> str|None:
+        try:
+            self.build_header()
+            if workspace is None:
+                workspace = utils.get_config_value('anythingLLm.workspace')
+            url = f"{self._api_url}/workspace/{workspace}/chat"
+            data = {
+                "message": msg,
+                "mode": "query"  # 可选chat/query模式
+            }
+            response = requests.post(url, headers=self._headers, json=data)
+            if response.status_code == 200:
+                result = response.json()
+                self._logger.info(f"Response: {result}")
+                # 提取有效回答（去除思考过程）
+                answer = result['textResponse'].split('</think>')[-1].strip()
+                self._logger.info(f"Answer: {answer}")
+                sources = result.get('sources', [])
+                for source in sources:
+                    self._logger.info(f"Source: {source}")
+                return answer
+            else:
+                return f"Error: {response.text}"
+        except Exception as e:
+            self._logger.error(f"Error: {str(e)}")
+            return None
+

SourceCode/DataMiddleware/app/ai/fast_gpt.py

@@ -0,0 +1,63 @@
+from httpx import stream
+
+import utils,json,requests
+import re  # 添加正则表达式库
+
+class FastGPTAi:
+
+    def __init__(self, api_key: str=None, api_url: str=None):
+        self._api_key = api_key or utils.get_config_value('fastgpt.api_key')
+        self._api_url = api_url or utils.get_config_value('fastgpt.api_url')
+        self._logger = utils.get_logger()
+        self._headers ={}
+
+    def call_ai(self, msg: str) -> json:
+        try:
+            if self._api_key is None:
+                self._logger.error("fastgpt.api_key 没有配置")
+                raise Exception("fastgpt.api_key 没有配置")
+
+            self._headers = {
+                "Authorization": f"Bearer {self._api_key}",
+                "Content-Type": "application/json"
+            }
+            url = f"{self._api_url}/v1/chat/completions"
+            data = {
+                # "model":"",
+                "stream": False,
+                "detail": False,
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": msg
+                    }
+                ],
+                "response_format": {
+                    "type": "json_object"
+                }
+            }
+
+            response = requests.post(url, headers=self._headers, json=data)
+            if response.status_code == 200:
+                result = response.json()
+                self._logger.info(f"Response: {result}")
+                content = result.get("choices", [{}])[0].get("message", {}).get("content", "")
+                if content:
+                    # 使用正则表达式去除 content 前后的特定字符串
+                    content = re.sub(r'^```json\n', '', content.strip())
+                    content = re.sub(r'\n```$', '', content.strip())
+                    try:
+                        data = json.loads(content)
+                        self._logger.info(f"Response_JSON: {data}")
+                        return data
+                    except json.JSONDecodeError as e:
+                        self._logger.error(f"Failed to decode JSON: {e}")
+                        return None
+                return None
+            else:
+                error_msg = f"Error: {response.status_code} - {response.text}"
+                self._logger.error(error_msg)
+                return None
+        except Exception as e:
+            self._logger.error(f"Error: {str(e)}")
+            return None

SourceCode/DataMiddleware/app/config.yml

@@ -19,3 +19,12 @@ ai:
   max_tokens: 1024
 file:
   source_path: './temp_files'
+anythingLLm:
+  api_key: KN9ZCNM-ET04KGB-QPAQ02C-X1P1VN9
+  api_url: http://192.168.0.104:8011/v1
+  workspace: datamiddleware
+  thread: fd218d56-8717-4519-a6e0-09c64b732091
+fastgpt:
+  api_key: fastgpt-wdJruTQNjqeH7oRdf54Ilb12pHAspEdW7Io3hcsuaifx5U1OOFzW8Qrc
+  api_url: http://192.168.0.104:8020/api
+  app_id: 67c6be7d686fc1d3f0cc1cce

SourceCode/DataMiddleware/app/data_process/pre_process.py

@@ -135,8 +135,8 @@ class PreProcess:
             # 初始化self._data[project_no]，避免在循环中重复初始化
             for data in excel_data:
                 prompt = self.prompt_template([data])
-                # response = self._ai_helper.call_openai(self._ai_sys_prompt, prompt)
-                response = {'data': [{'n': '阻燃型导线穿管敷设', 'm': 'WDZB1-BYJ-750V-2.5mm2', 'u': '米', 'c': 900.0}, {'n': '阻燃型导线穿管敷设', 'm': 'WDZB1-BYJ-750V-4mm2', 'u': '米', 'c': 800.0}, {'n': '耐火型导线穿管敷设', 'm': 'WDZB1N-BYJ-750V-2.5mm2', 'u': '米', 'c': 200.0}, {'n': '防火堵料', 'm': '', 'u': '公斤', 'c': 10.0}, {'n': '防火漆', 'm': '10kg/', '': '桶', 'c': 2.0}, {'n': '镀锌钢管', 'm': 'SC20', 'u': '米', 'c': 580.0}, {'n': '接地线', 'm': '热浸镀锌扁钢25x4', 'u': '米', 'c': 50.0}, {'n': '局部等电位端子箱', 'm': '', 'u': '个', 'c': 2.0}, {'n': '双联单控照明开关', 'm': '~250V 10A', 'u': '个', 'c': 4.0}, {'n': '密闭双联单控照明开关', 'm': '~250V 10A', 'u': '个', 'c': 4.0}, {'n': '配合空调室外机移位', 'm': '', 'u': '项', 'c': 1.0}, {'n': '应急照明灯', 'm': '220V，10W', 'u': '套', 'c': 4.0}, {'n': '门禁', 'm': '', 'u': '套', 'c': 4.0}, {'n': '配电线路改移', 'm': '开槽、移点位等', 'u': '项', 'c': 1.0}, {'n': '烘手器插座', 'm': '220V，10A，密闭型', 'u': '个', 'c': 2.0}], 'completion_tokens': 439, 'prompt_tokens': 766, 'total_tokens': 1205}
+                response = self._ai_helper.call_openai(self._ai_sys_prompt, prompt)
+                # response = {'data': [{'n': '阻燃型导线穿管敷设', 'm': 'WDZB1-BYJ-750V-2.5mm2', 'u': '米', 'c': 900.0}, {'n': '阻燃型导线穿管敷设', 'm': 'WDZB1-BYJ-750V-4mm2', 'u': '米', 'c': 800.0}, {'n': '耐火型导线穿管敷设', 'm': 'WDZB1N-BYJ-750V-2.5mm2', 'u': '米', 'c': 200.0}, {'n': '防火堵料', 'm': '', 'u': '公斤', 'c': 10.0}, {'n': '防火漆', 'm': '10kg/', '': '桶', 'c': 2.0}, {'n': '镀锌钢管', 'm': 'SC20', 'u': '米', 'c': 580.0}, {'n': '接地线', 'm': '热浸镀锌扁钢25x4', 'u': '米', 'c': 50.0}, {'n': '局部等电位端子箱', 'm': '', 'u': '个', 'c': 2.0}, {'n': '双联单控照明开关', 'm': '~250V 10A', 'u': '个', 'c': 4.0}, {'n': '密闭双联单控照明开关', 'm': '~250V 10A', 'u': '个', 'c': 4.0}, {'n': '配合空调室外机移位', 'm': '', 'u': '项', 'c': 1.0}, {'n': '应急照明灯', 'm': '220V，10W', 'u': '套', 'c': 4.0}, {'n': '门禁', 'm': '', 'u': '套', 'c': 4.0}, {'n': '配电线路改移', 'm': '开槽、移点位等', 'u': '项', 'c': 1.0}, {'n': '烘手器插座', 'm': '220V，10A，密闭型', 'u': '个', 'c': 2.0}], 'completion_tokens': 439, 'prompt_tokens': 766, 'total_tokens': 1205}
                 # 更新数据部分
                 project.items.extend(self.format_data(project_id, response["data"]))
                 project.completion_tokens += response["completion_tokens"]

SourceCode/DataMiddleware/app/data_process/process.py

@@ -1,18 +1,15 @@
-from click import prompt
-
 import utils,json
 
 from stores.mysql_store import MysqlStore
 from models.project_data import ProjectModel, ProjectItemModel
-from ui.project_views import project_item_list
+from ai.fast_gpt import FastGPTAi
 
 
 class Process:
     def __init__(self):
         self._logger= utils.get_logger()
         self._store= MysqlStore()
-        self._ai_helper = utils.AiHelper()
-        self._ai_sys_prompt = "查询给定数据的标准编号信息，返回压缩的json字符传"
+        self._ai = FastGPTAi()
         self._data={}
 
     def run(self, project: ProjectModel) ->bool:
@@ -20,9 +17,10 @@ class Process:
             self._logger.info(f"开始处理数据:{project.project_no}")
             self._store.update_project_status(project.id,22)
             project_items = self._store.query_project_items_by_project(project.id)
-            text = self.prompt_template(project_items)
-            self._logger.info(f"开始调用AI:{text}")
-            self._store.update_project_status(project.id,32)
+            text = self.prompt_template(project,project_items)
+            items = self.call_ai_process(text)
+            self._store.update_project_item_standard_no_batch(items)
+            self._store.update_project_status(project.id,12)
             self._logger.info(f"处理数据完成:{project.project_no}")
             return True
         except Exception as e:
@@ -31,12 +29,14 @@ class Process:
             return False
 
     @staticmethod
-    def prompt_template( items) ->str:
-        text = """
-        请分析提供的json数据。要求：
-        1. 根据项目名称，规格型号，数量，单位 查找计算出 定额编号
-        2. 提供的数据结构化信息:```typescript
-        export interface device { 
+    def prompt_template(project, items) ->str:
+        text = f"""
+        请分析提供的json数据，要求：
+        1. 根据工作内容、类型、项目名称、规格型号、数量、单位查找计算出定额编号
+        2. 工程的工作内容及类型：{project.work_content}"""
+        text += """
+        3. 提供的数据结构化信息:```typescript
+        export interface item { 
         i: // ID
         n: string; //物料名称
         m: string; //型号规格
@@ -44,20 +44,34 @@ class Process:
         c: float; //数量
         }
         ```
-        3. 需返回的结构体:```typescript
-        export interface device { 
-        i: int; // ID
+        4. 需返回的结构体:```typescript
+        export interface item { 
+        i: int; // ID 与提供的ID保持一致
         s: string; // 定额编号
         }
         ```
-        4. 返回结构体device的数组的json数组格式，压缩成一行
-        数据如下：
+        5. 返回结构体item的数组的json数组格式，压缩成一行。
+        6. 数据如下：
         """
         item_ai_json = ProjectItemModel.list_to_ai_json(items)
         text += json.dumps(item_ai_json,ensure_ascii=False, separators=(',', ':'))
         return text
 
-    def call_ai(self, system_prompt:str, prompt:str) ->list[ProjectItemModel]:
-        pass
+    def call_ai_process(self, message:str) ->list[ProjectItemModel]:
+        try:
+
+            self._logger.info(f"开始调用AI:\n {message}")
+            json_data = self._ai.call_ai(message)
+            self._logger.info(f"AI返回结果:{json_data}")
+            data=[]
+            for item in json_data:
+                data.append(ProjectItemModel(
+                    item_id=item["i"],
+                    standard_no=item["s"]
+                ))
+            return data
+        except Exception as e:
+            self._logger.error(f"AI调用失败:{e}")
+            return []
 
 

SourceCode/DataMiddleware/app/stores/mysql_store.py

@@ -311,6 +311,14 @@ class MysqlStore:
             self._db_helper.execute_non_query(query, params)
             return True
 
+    def update_project_item_standard_no_batch(self,items:list[ProjectItemModel]):
+        data =datetime.now()
+        for item in items:
+            query = "UPDATE project_item SET standard_no = %s,update_time = %s WHERE id = %s"
+            params = (item.standard_no, data, item.id)
+            with self._db_helper:
+                self._db_helper.execute_non_query(query, params)
+
     def query_standard_data_by_no(self, standard_no: str,version:str):
         query = "SELECT device_model,device_name,device_unit,standard_version,standard_no FROM standard_data WHERE standard_no = %s AND standard_version = %s"
         params = (standard_no,version,)

SourceCode/DataMiddleware/app/ui/project_services.py

@@ -97,9 +97,9 @@ class SourceDataService:
         project = self._store.query_project(project_id)
         if project:
             self._store.delete_project(project_id)
-            return True
+            return True,''
         else:
-            return False
+            return False, '项目不存在'
 
     def get_project_item_list_by_project_paginated(self, project_id: int, page: int, per_page: int, keyword: str = None) -> (list[ProjectItemModel], int):
         return self._store.query_project_items_by_project_paginated(project_id, page, per_page, keyword)

SourceCode/DataMiddleware/app/ui/project_views.py

@@ -131,16 +131,16 @@ def start_project_task(project_id:int):
 
 @project.route('/process_project/<project_id>', methods=['POST'])
 def process_project(project_id:int):
-    result, err_msg = SourceDataService.start_task(project_id)
+    result, err_msg = SourceDataService.process_project(project_id)
     return jsonify({'success': result, 'error': err_msg})
 @project.route('/send_project/<project_id>', methods=['POST'])
 def send_project(project_id:int):
     result, err_msg = SourceDataService.start_send_data(project_id)
     return jsonify({'success': result, 'error': err_msg})
 @project.route('/delete_project/<project_id>', methods=['POST'])
-def delete_project(project_id):
-    SourceDataService.delete_project(project_id)  # 使用实例方法
-    return redirect(url_for('project.project_list'))
+def delete_project(project_id:int):
+    result, err_msg =  SourceDataService.delete_project(project_id)
+    return jsonify({'success': result, 'error': err_msg})
 
 @project.route('/add_project_item', methods=['POST'])
 def add_project_item():

SourceCode/DataMiddleware/app/ui/static/project_list.js

@@ -4,6 +4,11 @@ function addProject() {
 	inputs.forEach((input) => {
 		input.value = ''
 	})
+	// 移除已存在的delete_old_data复选框
+	const existingCheckbox = document.getElementById('delete_old_data')
+	if (existingCheckbox) {
+		existingCheckbox.parentNode.remove()
+	}
 	const file = document.getElementById('new_project_file').parentNode
 	file.innerHTML = `<label for="new_project_file">项目数据：</label><input type="file" id="new_project_file" class="form-control" accept=".xlsx,.xls,.csv" multiple>`
 
@@ -22,7 +27,7 @@ function updateProject(row, id) {
 		existingCheckbox.parentNode.remove()
 	}
 	const checkboxDiv = document.createElement('div')
-	checkboxDiv.innerHTML = `<label for="delete_old_data">删除原数据：</label><input type="checkbox" id="delete_old_data" class="form-control" style="width: auto;">`
+	checkboxDiv.innerHTML = `<label for="delete_old_data">删除原数据：</label><input type="checkbox" id="delete_old_data" class="form-control" style="width: auto;flex: 0;">`
 	file.parentNode.insertBefore(checkboxDiv, file.nextSibling)
 	document.getElementById('new_work_catalog').value = row.querySelector('.work_catalog .form-control').value
 	document.getElementById('new_work_content').value = row.querySelector('.work_content .form-control').value
@@ -70,7 +75,9 @@ function saveProject() {
 	formData.append('work_catalog', catalog)
 	formData.append('work_content', content)
 	formData.append('standard_version', version)
-	formData.append('delete_old_data', document.getElementById('delete_old_data').checked)
+	if (document.getElementById('delete_old_data')){
+		formData.append('delete_old_data', document.getElementById('delete_old_data').checked)
+	}
 
 	fetch('/save_project', {
 		method: 'POST',
@@ -142,51 +149,25 @@ function saveProjectUpdate(row, id) {
 		})
 }
 
-function confirmCollectData(id) {
-	if (confirm('确定要开始采集数据吗？')) {
-		fetch(`/collect_project/${id}`, {
-			method: 'POST',
-			headers: {
-				'Content-Type': 'application/json',
-			},
-		})
-			.then((response) => response.json())
-			.then((data) => {
-				if (data.success) {
-					alert('操作成功')
-					window.location.reload()
-				} else {
-					alert('采集失败:' + data.error)
-				}
-			})
-			.catch((error) => {})
-	}
+function confirmStartTask(id){
+	_confirm('确定要开始任务吗？',`/start_project_task/${id}`)
 }
 
-function confirmProcessData(id) {
-	if (confirm('确定要开始分析处理数据吗？')) {
-		fetch(`/process_project/${id}`, {
-			method: 'POST',
-			headers: {
-				'Content-Type': 'application/json',
-			},
-		})
-			.then((response) => response.json())
-			.then((data) => {
-				if (data.success) {
-					alert('操作成功')
-					window.location.reload()
-				} else {
-					alert('分析处理失败:' + data.error)
-				}
-			})
-			.catch((error) => {})
-	}
+function confirmReStartTask(id){
+	_confirm('确定要重新采集数据吗？',`/start_project_task/${id}`)
+}
+
+function confirmReProcessData(id){
+	_confirm('确定要重新处理数据吗？',`/process_project/${id}`)
 }
 
 function confirmSendData(id) {
-	if (confirm('确定要开始上传数据吗？')) {
-		fetch(`/send_project/${id}`, {
+	_confirm('确定要开始上传数据吗？',`/send_project/${id}`)
+}
+
+function _confirm(text,url){
+	if (confirm(text)) {
+		fetch(url, {
 			method: 'POST',
 			headers: {
 				'Content-Type': 'application/json',
@@ -198,13 +179,14 @@ function confirmSendData(id) {
 					alert('操作成功')
 					window.location.reload()
 				} else {
-					alert('分析处理失败:' + data.error)
+					alert('操作失败:' + data.error)
 				}
 			})
 			.catch((error) => {})
 	}
 }
 
+
 function confirmProjectDelete(id) {
 	if (confirm('确定要删除该项目吗？')) {
 		fetch(`/delete_project/${id}`, {

SourceCode/DataMiddleware/app/ui/templates/project/project_list.html

@@ -116,7 +116,7 @@
                             <td class="editable work_catalog">
                                 <span class="show">{{ project.work_catalog if project.work_catalog else '-' }}</span>
                                 <span class="edit">
-                                    <input type="text" class="form-control" title="工作目录" name="work_catalogvalue="{{ project.work_catalog if project.work_catalog else '' }}">
+                                    <input type="text" class="form-control" title="工作目录" name="work_catalog" value="{{ project.work_catalog if project.work_catalog else '' }}">
                                 </span>
                             </td>
                             <td class="editable work_content">

SourceCode/DataMiddleware/requirements.txt

@@ -4,8 +4,7 @@ Requests==2.32.3
 openai==1.58.1
 pandas~=2.2.3
 Flask~=3.1.0
-click~=8.1.8
 PyPDF2~=3.0.1
 pillow~=11.1.0
 pathlib~=1.0.1
-PyMuPDF~=1.25.3
+PyMuPDF~=1.25.3