Change embedding (labring#1463)

* rebuild embedding queue

* dataset menu

* feat: rebuild data api

* feat: ui change embedding model

* dataset ui

* feat: rebuild index ui

* rename collection

.vscode/nextapi.code-snippets

@@ -10,20 +10,50 @@
 		"scope": "javascript,typescript",
 		"prefix": "nextapi",
 		"body": [
-			"import type { NextApiRequest, NextApiResponse } from 'next';",
+			"import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';",
 			"import { NextAPI } from '@/service/middle/entry';",
 			"",
-			"type Props = {};",
+			"export type ${TM_FILENAME_BASE}Query = {};",
 			"",
-			"type Response = {};",
+			"export type ${TM_FILENAME_BASE}Body = {};",
 			"",
-			"async function handler(req: NextApiRequest, res: NextApiResponse<any>): Promise<Response> {",
+			"export type ${TM_FILENAME_BASE}Response = {};",
+			"",
+			"async function handler(",
+			"  req: ApiRequestProps<getDatasetTrainingQueueBody, getDatasetTrainingQueueQuery>,",
+			"  res: ApiResponseType<any>",
+			"): Promise<getDatasetTrainingQueueResponse> {",
 			"    $1",
 			"    return {}",
 			"}",
 			"",
 			"export default NextAPI(handler);"
 		],
 		"description": "FastGPT Next API template"
+	},
+	"use context template": {
+		"scope": "typescriptreact",
+		"prefix": "context",
+		"body": [
+			"import { ReactNode } from 'react';",
+			"import { createContext } from 'use-context-selector';",
+			"",
+			"type ContextType = {$1};",
+			"",
+			"type ContextValueType = {};",
+			"",
+			"export const Context = createContext<ContextType>({});",
+			"",
+			"export const ContextProvider = ({",
+			"  children,",
+			"  value",
+			"}: {",
+			"  children: ReactNode;",
+			"  value: ContextValueType;",
+			"}) => {",
+			"  return <Context.Provider value={value}>{children}</Context.Provider>;",
+			"};",
+		],
+		"description": "FastGPT usecontext template"
 	}
 }

.vscode/settings.json

@@ -11,5 +11,6 @@
     "i18n-ally.sortKeys": true,
     "i18n-ally.keepFulfilled": false,
     "i18n-ally.sourceLanguage": "zh", // 根据此语言文件翻译其他语言文件的变量和内容
-    "i18n-ally.displayLanguage": "zh" // 显示语言
+    "i18n-ally.displayLanguage": "zh", // 显示语言
+    "i18n-ally.extract.targetPickingStrategy": "most-similar-by-key"
 }

docSite/content/docs/development/upgrading/48.md

@@ -1,5 +1,5 @@
 ---
-title: 'V4.8(开发中)'
+title: 'V4.8'
 description: 'FastGPT V4.8 更新说明'
 icon: 'upgrade'
 draft: false

docSite/content/docs/development/upgrading/481.md

@@ -0,0 +1,38 @@
+---
+title: 'V4.8.1(进行中)'
+description: 'FastGPT V4.8.1 更新说明'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 825
+---
+
+## 初始化脚本
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成FastGPT的域名。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv481' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+由于之前集合名不规范，该初始化会重置表名。请在初始化前，确保 dataset.trainings 表没有数据。
+最好更新该版本时，暂停所有进行中业务，再进行初始化，避免数据冲突。
+
+## 执行脏数据清理
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成FastGPT的域名。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化完后，可以执行这个命令。之前定时清理的定时器有些问题，部分数据没被清理，可以手动执行清理。
+
+## V4.8.1 更新说明
+
+1. 新增 - 知识库重新选择向量模型重建
+2. 修复 - 定时器清理脏数据任务

packages/global/core/dataset/type.d.ts

@@ -80,6 +80,7 @@ export type DatasetDataSchemaType = {
   a: string; // answer or custom content
   fullTextToken: string;
   indexes: DatasetDataIndexItemType[];
+  rebuilding?: boolean;
 };
 
 export type DatasetTrainingSchemaType = {
@@ -95,6 +96,7 @@ export type DatasetTrainingSchemaType = {
   mode: `${TrainingModeEnum}`;
   model: string;
   prompt: string;
+  dataId?: string;
   q: string;
   a: string;
   chunkIndex: number;

packages/global/support/user/team/constant.ts

@@ -1,6 +1,6 @@
 export const TeamCollectionName = 'teams';
-export const TeamMemberCollectionName = 'team.members';
-export const TeamTagsCollectionName = 'team.tags';
+export const TeamMemberCollectionName = 'team_members';
+export const TeamTagsCollectionName = 'team_tags';
 
 export enum TeamMemberRoleEnum {
   owner = 'owner',

packages/service/common/buffer/rawText/schema.ts

@@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../mongo';
 const { Schema, model, models } = connectionMongo;
 import { RawTextBufferSchemaType } from './type';
 
-export const collectionName = 'buffer.rawText';
+export const collectionName = 'buffer_rawtexts';
 
 const RawTextBufferSchema = new Schema({
   sourceId: {

packages/service/common/buffer/tts/schema.ts

@@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { TTSBufferSchemaType } from './type.d';
 
-export const collectionName = 'buffer.tts';
+export const collectionName = 'buffer_tts';
 
 const TTSBufferSchema = new Schema({
   bufferId: {

packages/service/common/mongo/sessionRun.ts

@@ -12,8 +12,6 @@ export const mongoSessionRun = async <T = unknown>(fn: (session: ClientSession)
 
     return result as T;
   } catch (error) {
-    console.log(error);
-
     await session.abortTransaction();
     await session.endSession();
     return Promise.reject(error);

packages/service/common/vectorStore/pg/controller.ts

@@ -98,12 +98,15 @@ export const deleteDatasetDataVector = async (
       return `${teamIdWhere} ${datasetIdWhere}`;
     }
 
-    if ('idList' in props && props.idList) {
+    if ('idList' in props && Array.isArray(props.idList)) {
+      if (props.idList.length === 0) return;
       return `${teamIdWhere} id IN (${props.idList.map((id) => `'${String(id)}'`).join(',')})`;
     }
     return Promise.reject('deleteDatasetData: no where');
   })();
 
+  if (!where) return;
+
   try {
     await PgClient.delete(PgDatasetTableName, {
       where: [where]

packages/service/core/app/versionSchema.ts

@@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
 
-export const AppVersionCollectionName = 'app.versions';
+export const AppVersionCollectionName = 'app_versions';
 
 const AppVersionSchema = new Schema({
   appId: {

packages/service/core/dataset/collection/schema.ts

@@ -8,7 +8,7 @@ import {
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-export const DatasetColCollectionName = 'dataset.collections';
+export const DatasetColCollectionName = 'dataset_collections';
 
 const DatasetCollectionSchema = new Schema({
   parentId: {

packages/service/core/dataset/data/schema.ts

@@ -8,7 +8,7 @@ import {
 import { DatasetCollectionName } from '../schema';
 import { DatasetColCollectionName } from '../collection/schema';
 
-export const DatasetDataCollectionName = 'dataset.datas';
+export const DatasetDataCollectionName = 'dataset_datas';
 
 const DatasetDataSchema = new Schema({
   teamId: {
@@ -73,7 +73,8 @@ const DatasetDataSchema = new Schema({
   },
   inited: {
     type: Boolean
-  }
+  },
+  rebuilding: Boolean
 });
 
 try {
@@ -90,10 +91,13 @@ try {
     { background: true }
   );
   DatasetDataSchema.index({ updateTime: 1 }, { background: true });
+  // rebuild data
+  DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }, { background: true });
 } catch (error) {
   console.log(error);
 }
 
 export const MongoDatasetData: Model<DatasetDataSchemaType> =
   models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);
+
 MongoDatasetData.syncIndexes();

packages/service/core/dataset/training/schema.ts

@@ -10,7 +10,7 @@ import {
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-export const DatasetTrainingCollectionName = 'dataset.trainings';
+export const DatasetTrainingCollectionName = 'dataset_trainings';
 
 const TrainingDataSchema = new Schema({
   teamId: {
@@ -35,8 +35,7 @@ const TrainingDataSchema = new Schema({
   },
   billId: {
     // concat bill
-    type: String,
-    default: ''
+    type: Schema.Types.ObjectId
   },
   mode: {
     type: String,
@@ -78,6 +77,9 @@ const TrainingDataSchema = new Schema({
     type: Number,
     default: 0
   },
+  dataId: {
+    type: Schema.Types.ObjectId
+  },
   indexes: {
     type: [
       {

packages/service/type/next.d.ts

@@ -0,0 +1,8 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+
+export type ApiRequestProps<Body = any, Query = any> = Omit<NextApiRequest, 'query' | 'body'> & {
+  query: Query;
+  body: Body;
+};
+
+export type { NextApiResponse as ApiResponseType } from 'next';

packages/web/components/common/MyDivider/index.tsx

@@ -0,0 +1,9 @@
+import React from 'react';
+import { Divider, type DividerProps } from '@chakra-ui/react';
+
+const MyDivider = (props: DividerProps) => {
+  const { h } = props;
+  return <Divider my={4} borderBottomWidth={h || '1x'} {...props}></Divider>;
+};
+
+export default MyDivider;

packages/web/components/common/MyTooltip/index.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { Tooltip, TooltipProps, css, useMediaQuery } from '@chakra-ui/react';
+import { Box, Tooltip, TooltipProps, css, useMediaQuery } from '@chakra-ui/react';
 
 interface Props extends TooltipProps {
   forceShow?: boolean;
@@ -9,24 +9,32 @@ const MyTooltip = ({ children, forceShow = false, shouldWrapChildren = true, ...
   const [isPc] = useMediaQuery('(min-width: 900px)');
 
   return isPc || forceShow ? (
-    <Tooltip
-      className="tooltip"
-      bg={'white'}
-      arrowShadowColor={'rgba(0,0,0,0.05)'}
-      hasArrow
-      arrowSize={12}
-      offset={[-15, 15]}
-      color={'myGray.800'}
-      px={4}
-      py={2}
-      borderRadius={'8px'}
-      whiteSpace={'pre-wrap'}
-      boxShadow={'1px 1px 10px rgba(0,0,0,0.2)'}
-      shouldWrapChildren={shouldWrapChildren}
-      {...props}
+    <Box
+      css={css({
+        '& span': {
+          display: 'block'
+        }
+      })}
     >
-      {children}
-    </Tooltip>
+      <Tooltip
+        className="tooltip"
+        bg={'white'}
+        arrowShadowColor={'rgba(0,0,0,0.05)'}
+        hasArrow
+        arrowSize={12}
+        offset={[-15, 15]}
+        color={'myGray.800'}
+        px={4}
+        py={2}
+        borderRadius={'8px'}
+        whiteSpace={'pre-wrap'}
+        boxShadow={'1px 1px 10px rgba(0,0,0,0.2)'}
+        shouldWrapChildren={shouldWrapChildren}
+        {...props}
+      >
+        {children}
+      </Tooltip>
+    </Box>
   ) : (
     <>{children}</>
   );

projects/app/.eslintrc.json

@@ -1,6 +1,12 @@
+
 {
+  "parser": "@typescript-eslint/parser", // 确保使用了 TypeScript 解析器
+  "plugins": ["@typescript-eslint"], // 引入 TypeScript 插件
+
   "extends": "next/core-web-vitals",
   "rules": {
-    "react-hooks/rules-of-hooks": 0
+    "react-hooks/rules-of-hooks": 0,
+    "@typescript-eslint/consistent-type-imports": "warn" // 或者 "error" 来强制执行
+
   }
 }

projects/app/i18n/en/dataset.json

@@ -0,0 +1,6 @@
+{
+  "Confirm to rebuild embedding tip": "Are you sure to switch the knowledge base index? Switching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.",
+  "Rebuild embedding start tip": "The task of switching index models has begun",
+  "Rebuilding index count": "Rebuilding count: {{count}}",
+  "The knowledge base has indexes that are being trained or being rebuilt": "The knowledge base has indexes that are being trained or being rebuilt"
+}

projects/app/i18n/zh/dataset.json

@@ -0,0 +1,6 @@
+{
+  "Confirm to rebuild embedding tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。",
+  "Rebuild embedding start tip": "切换索引模型任务已开始",
+  "Rebuilding index count": "重建中索引数量: {{count}}",
+  "The knowledge base has indexes that are being trained or being rebuilt": "知识库有训练中或正在重建的索引"
+}

projects/app/src/components/Select/AIModelSelector.tsx

@@ -8,8 +8,13 @@ import MySelect, { SelectProps } from '@fastgpt/web/components/common/MySelect';
 import { HUGGING_FACE_ICON, LOGO_ICON } from '@fastgpt/global/common/system/constants';
 import { Box, Flex } from '@chakra-ui/react';
 import Avatar from '../Avatar';
+import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 
-const AIModelSelector = ({ list, onchange, ...props }: SelectProps) => {
+type Props = SelectProps & {
+  disableTip?: string;
+};
+
+const AIModelSelector = ({ list, onchange, disableTip, ...props }: Props) => {
   const { t } = useTranslation();
   const { feConfigs, llmModelList, vectorModelList } = useSystemStore();
   const router = useRouter();
@@ -62,9 +67,9 @@ const AIModelSelector = ({ list, onchange, ...props }: SelectProps) => {
   );
 
   return (
-    <>
-      <MySelect list={expandList} {...props} onchange={onSelect} />
-    </>
+    <MyTooltip label={disableTip}>
+      <MySelect isDisabled={!!disableTip} list={expandList} {...props} onchange={onSelect} />
+    </MyTooltip>
   );
 };