Update 完善数据处理

SourceCode/DataMiddleware/app/ai/anything_llm.py

@@ -0,0 +1,48 @@
+import requests,utils
+
+class AnythingLLM:
+
+
+    def __init__(self, api_key: str=None, api_url: str=None):
+        self._api_key = api_key or utils.get_config_value('anythingLLm.api_key')
+        self._api_url = api_url or utils.get_config_value('anythingLLm.api_url')
+        self._logger = utils.get_logger()
+        self._headers ={}
+
+    def build_header(self):
+        if self._api_key is None:
+            self._logger.error("anythingLLm.api_key 没有配置")
+            raise Exception("anythingLLm.api_key 没有配置")
+        self._headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json",
+            "accept": "application/json"
+        }
+
+    def call_ai(self, msg: str, workspace:str=None) -> str|None:
+        try:
+            self.build_header()
+            if workspace is None:
+                workspace = utils.get_config_value('anythingLLm.workspace')
+            url = f"{self._api_url}/workspace/{workspace}/chat"
+            data = {
+                "message": msg,
+                "mode": "query"  # 可选chat/query模式
+            }
+            response = requests.post(url, headers=self._headers, json=data)
+            if response.status_code == 200:
+                result = response.json()
+                self._logger.info(f"Response: {result}")
+                # 提取有效回答（去除思考过程）
+                answer = result['textResponse'].split('</think>')[-1].strip()
+                self._logger.info(f"Answer: {answer}")
+                sources = result.get('sources', [])
+                for source in sources:
+                    self._logger.info(f"Source: {source}")
+                return answer
+            else:
+                return f"Error: {response.text}"
+        except Exception as e:
+            self._logger.error(f"Error: {str(e)}")
+            return None
+

SourceCode/DataMiddleware/app/ai/fast_gpt.py

@@ -0,0 +1,63 @@
+from httpx import stream
+
+import utils,json,requests
+import re  # 添加正则表达式库
+
+class FastGPTAi:
+
+    def __init__(self, api_key: str=None, api_url: str=None):
+        self._api_key = api_key or utils.get_config_value('fastgpt.api_key')
+        self._api_url = api_url or utils.get_config_value('fastgpt.api_url')
+        self._logger = utils.get_logger()
+        self._headers ={}
+
+    def call_ai(self, msg: str) -> json:
+        try:
+            if self._api_key is None:
+                self._logger.error("fastgpt.api_key 没有配置")
+                raise Exception("fastgpt.api_key 没有配置")
+
+            self._headers = {
+                "Authorization": f"Bearer {self._api_key}",
+                "Content-Type": "application/json"
+            }
+            url = f"{self._api_url}/v1/chat/completions"
+            data = {
+                # "model":"",
+                "stream": False,
+                "detail": False,
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": msg
+                    }
+                ],
+                "response_format": {
+                    "type": "json_object"
+                }
+            }
+
+            response = requests.post(url, headers=self._headers, json=data)
+            if response.status_code == 200:
+                result = response.json()
+                self._logger.info(f"Response: {result}")
+                content = result.get("choices", [{}])[0].get("message", {}).get("content", "")
+                if content:
+                    # 使用正则表达式去除 content 前后的特定字符串
+                    content = re.sub(r'^```json\n', '', content.strip())
+                    content = re.sub(r'\n```$', '', content.strip())
+                    try:
+                        data = json.loads(content)
+                        self._logger.info(f"Response_JSON: {data}")
+                        return data
+                    except json.JSONDecodeError as e:
+                        self._logger.error(f"Failed to decode JSON: {e}")
+                        return None
+                return None
+            else:
+                error_msg = f"Error: {response.status_code} - {response.text}"
+                self._logger.error(error_msg)
+                return None
+        except Exception as e:
+            self._logger.error(f"Error: {str(e)}")
+            return None

SourceCode/DataMiddleware/app/config.yml

@@ -19,3 +19,12 @@ ai:
   max_tokens: 1024
 file:
   source_path: './temp_files'
+anythingLLm:
+  api_key: KN9ZCNM-ET04KGB-QPAQ02C-X1P1VN9
+  api_url: http://192.168.0.104:8011/v1
+  workspace: datamiddleware
+  thread: fd218d56-8717-4519-a6e0-09c64b732091
+fastgpt:
+  api_key: fastgpt-wdJruTQNjqeH7oRdf54Ilb12pHAspEdW7Io3hcsuaifx5U1OOFzW8Qrc
+  api_url: http://192.168.0.104:8020/api
+  app_id: 67c6be7d686fc1d3f0cc1cce

SourceCode/DataMiddleware/app/data_process/process.py

@@ -1,18 +1,15 @@
-from click import prompt
-
 import utils,json
 
 from stores.mysql_store import MysqlStore
 from models.project_data import ProjectModel, ProjectItemModel
-from ui.project_views import project_item_list
+from ai.fast_gpt import FastGPTAi
 
 
 class Process:
     def __init__(self):
         self._logger= utils.get_logger()
         self._store= MysqlStore()
-        self._ai_helper = utils.AiHelper()
-        self._ai_sys_prompt = "查询给定数据的标准编号信息，返回压缩的json字符传"
+        self._ai = FastGPTAi()
         self._data={}
 
     def run(self, project: ProjectModel) ->bool:
@@ -20,9 +17,10 @@ class Process:
             self._logger.info(f"开始处理数据:{project.project_no}")
             self._store.update_project_status(project.id,22)
             project_items = self._store.query_project_items_by_project(project.id)
-            text = self.prompt_template(project_items)
-            self._logger.info(f"开始调用AI:{text}")
-            self._store.update_project_status(project.id,32)
+            text = self.prompt_template(project,project_items)
+            items = self.call_ai_process(text)
+            self._store.update_project_item_standard_no_batch(items)
+            self._store.update_project_status(project.id,12)
             self._logger.info(f"处理数据完成:{project.project_no}")
             return True
         except Exception as e:
@@ -31,12 +29,14 @@ class Process:
             return False
 
     @staticmethod
-    def prompt_template( items) ->str:
-        text = """
-        请分析提供的json数据。要求：
-        1. 根据项目名称，规格型号，数量，单位 查找计算出 定额编号
-        2. 提供的数据结构化信息:```typescript
-        export interface device { 
+    def prompt_template(project, items) ->str:
+        text = f"""
+        请分析提供的json数据，要求：
+        1. 根据工作内容、类型、项目名称、规格型号、数量、单位查找计算出定额编号
+        2. 工程的工作内容及类型：{project.work_content}"""
+        text += """
+        3. 提供的数据结构化信息:```typescript
+        export interface item { 
         i: // ID
         n: string; //物料名称
         m: string; //型号规格
@@ -44,20 +44,34 @@ class Process:
         c: float; //数量
         }
         ```
-        3. 需返回的结构体:```typescript
-        export interface device { 
-        i: int; // ID
+        4. 需返回的结构体:```typescript
+        export interface item { 
+        i: int; // ID 与提供的ID保持一致
         s: string; // 定额编号
         }
         ```
-        4. 返回结构体device的数组的json数组格式，压缩成一行
-        数据如下：
+        5. 返回结构体item的数组的json数组格式，压缩成一行。
+        6. 数据如下：
         """
         item_ai_json = ProjectItemModel.list_to_ai_json(items)
         text += json.dumps(item_ai_json,ensure_ascii=False, separators=(',', ':'))
         return text
 
-    def call_ai(self, system_prompt:str, prompt:str) ->list[ProjectItemModel]:
-        pass
+    def call_ai_process(self, message:str) ->list[ProjectItemModel]:
+        try:
+
+            self._logger.info(f"开始调用AI:\n {message}")
+            json_data = self._ai.call_ai(message)
+            self._logger.info(f"AI返回结果:{json_data}")
+            data=[]
+            for item in json_data:
+                data.append(ProjectItemModel(
+                    item_id=item["i"],
+                    standard_no=item["s"]
+                ))
+            return data
+        except Exception as e:
+            self._logger.error(f"AI调用失败:{e}")
+            return []
 
 

SourceCode/DataMiddleware/app/stores/mysql_store.py

@@ -311,6 +311,14 @@ class MysqlStore:
             self._db_helper.execute_non_query(query, params)
             return True
 
+    def update_project_item_standard_no_batch(self,items:list[ProjectItemModel]):
+        data =datetime.now()
+        for item in items:
+            query = "UPDATE project_item SET standard_no = %s,update_time = %s WHERE id = %s"
+            params = (item.standard_no, data, item.id)
+            with self._db_helper:
+                self._db_helper.execute_non_query(query, params)
+
     def query_standard_data_by_no(self, standard_no: str,version:str):
         query = "SELECT device_model,device_name,device_unit,standard_version,standard_no FROM standard_data WHERE standard_no = %s AND standard_version = %s"
         params = (standard_no,version,)

SourceCode/DataMiddleware/requirements.txt

@@ -4,8 +4,7 @@ Requests==2.32.3
 openai==1.58.1
 pandas~=2.2.3
 Flask~=3.1.0
-click~=8.1.8
 PyPDF2~=3.0.1
 pillow~=11.1.0
 pathlib~=1.0.1
-PyMuPDF~=1.25.3
+PyMuPDF~=1.25.3