diff --git a/app-builder/plugins/data-mate-knowledge/pom.xml b/app-builder/plugins/data-mate-knowledge/pom.xml new file mode 100644 index 000000000..1a637780c --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/pom.xml @@ -0,0 +1,142 @@ + + + 4.0.0 + + modelengine.fit.jade + app-builder-plugin-parent + 1.0.0-SNAPSHOT + + + modelengine.jade.plugin + data-mate-knowledge + + + 17 + 17 + UTF-8 + + + + + + org.fitframework + fit-api + + + + + modelengine.fit.jade.service + knowledge-service + + + + + org.projectlombok + lombok + + + + + org.mapstruct + mapstruct + + + org.mapstruct + mapstruct-processor + + + + + com.fasterxml.jackson.core + jackson-databind + + + + + org.fitframework + fit-test-framework + + + org.junit.jupiter + junit-jupiter + + + org.mockito + mockito-core + + + org.assertj + assertj-core + + + + + + + org.fitframework + fit-build-maven-plugin + ${fit.version} + + + build-plugin + + build-plugin + + + + package-plugin + + package-plugin + + + + + + org.fitframework + fit-dependency-maven-plugin + ${fit.version} + + + dependency + compile + + dependency + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.version} + + + + FIT Lab + + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven.antrun.version} + + + install + + + + + + + run + + + + + + + \ No newline at end of file diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/convertor/ParamConvertor.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/convertor/ParamConvertor.java new file mode 100644 index 000000000..14cfaa7e5 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/convertor/ParamConvertor.java @@ -0,0 +1,140 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.convertor; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import modelengine.fitframework.util.StringUtils; +import modelengine.jade.knowledge.KnowledgeRepo; +import modelengine.jade.knowledge.ReferenceLimit; +import modelengine.jade.knowledge.document.KnowledgeDocument; +import modelengine.fit.jade.datamate.knowledge.knowledge.dto.DataMateRetrievalParam; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateKnowledgeEntity; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateRetrievalChunksEntity; +import modelengine.jade.knowledge.support.FlatKnowledgeOption; + +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.Named; +import org.mapstruct.factory.Mappers; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Map; + +/** + * DataMate 内部数据的转换器接口。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Mapper +public interface ParamConvertor { + ParamConvertor INSTANCE = Mappers.getMapper(ParamConvertor.class); + int TOP = 400; + + /** + * 将 {@link DataMateKnowledgeEntity} 转换为 {@link KnowledgeRepo}。 + * + * @param entity 表示待转换的 {@link DataMateKnowledgeEntity}。 + * @return 转换完成的 {@link KnowledgeRepo}。 + */ + @Mapping(target = "type", source = "entity", qualifiedByName = "mapIndexTypeToType") + @Mapping(target = "createdAt", source = "entity", qualifiedByName = "stringToLocalDateTime") + KnowledgeRepo convertToKnowledgeRepo(DataMateKnowledgeEntity entity); + + /** + * 将 DataMate 知识库的检索 type 映射为 平台知识库元数据 type。 + * + * @param entity 表示待转换的 {@link DataMateKnowledgeEntity}。 + * @return 表示转换完成的 {@link String}。 + */ + @Named("mapIndexTypeToType") + default String mapIndexTypeToType(DataMateKnowledgeEntity entity) { + return entity == null ? null : entity.getEmbeddingModel(); + } + + /** + * 将 DataMate 知识库的 createdAt 映射为 平台知识库元数据 createdAt。 + * + * @param entity 表示待转换的 {@link DataMateKnowledgeEntity}。 + * @return 表示转换完成的 {@link LocalDateTime}。 + */ + @Named("stringToLocalDateTime") + default LocalDateTime stringToLocalDateTime(DataMateKnowledgeEntity entity) { + String dateStr = entity.getCreatedAt(); + if (dateStr == null || StringUtils.isEmpty(dateStr)) { + return null; + } + return LocalDateTime.parse(dateStr, DateTimeFormatter.ISO_DATE_TIME); + } + + /** + * 将 {@link FlatKnowledgeOption} 转换为 {@link DataMateRetrievalParam}。 + * + * @param option 表示待转换的 {@link FlatKnowledgeOption}。 + * @return 转换完成的 {@link DataMateRetrievalParam}。 + */ + @Mapping(target = "knowledgeBaseIds", source = "repoIds") + @Mapping(target = "query", source = "query") + @Mapping(target = "topK", source = "referenceLimit", qualifiedByName = "mapReferenceLimitToTop") + @Mapping(target = "threshold", source = "similarityThreshold") + DataMateRetrievalParam convertToRetrievalParam(FlatKnowledgeOption option); + + /** + * 将平台检索请求 ReferenceLimit 映射为 DataMate 检索请求 top。 + * + * @param limit 表示待转换的 {@link ReferenceLimit}。 + * @return 转换完成的 {@link int}。 + */ + @Named("mapReferenceLimitToTop") + default int mapReferenceLimitToTop(ReferenceLimit limit) { + if (limit == null) { + return TOP; + } + return limit.getValue(); + } + + /** + * 将 {@link DataMateRetrievalChunksEntity} 转换为 {@link KnowledgeDocument}。 + * + * @param entity 表示待转换的 {@link DataMateRetrievalChunksEntity}。 + * @return 转换完成的 {@link KnowledgeDocument}。 + */ + @Mapping(target = "id", expression = "java(entity.chunkId())") + @Mapping(target = "text", expression = "java(entity.content())") + @Mapping(target = "score", expression = "java(entity.retrievalScore())") + @Mapping(target = "metadata", source = ".", qualifiedByName = "mapChunksEntityToMetadata") + KnowledgeDocument convertToKnowledgeDocument(DataMateRetrievalChunksEntity entity); + + /** + * 将 DataMate 检索结果 entity 映射为 平台检索结果 metadata。 + * + * @param entity 表示待转换的 {@link DataMateRetrievalChunksEntity}。 + * @return 转换完成的 {@link Map}{@code <}{@link String}{@code , }{@link Object}{@code >}。 + */ + @Named("mapChunksEntityToMetadata") + default Map mapChunksEntityToMetadata(DataMateRetrievalChunksEntity entity) { + Map metadata = new HashMap<>(); + if (entity == null || entity.getEntity() == null) { + return metadata; + } + metadata.put("primaryKey", entity.getPrimaryKey()); + String rawMetadata = entity.getEntity().getMetadata(); + if (!StringUtils.isEmpty(rawMetadata)) { + try { + Map parsed = new ObjectMapper().readValue(rawMetadata, Map.class); + metadata.putAll(parsed); + } catch (JsonProcessingException ex) { + metadata.put("metadata", rawMetadata); + } + } + return metadata; + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/dto/DataMateKnowledgeListQueryParam.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/dto/DataMateKnowledgeListQueryParam.java new file mode 100644 index 000000000..efd1d363e --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/dto/DataMateKnowledgeListQueryParam.java @@ -0,0 +1,43 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.dto; + +import lombok.Builder; +import lombok.Data; +import modelengine.fitframework.serialization.annotation.SerializeStrategy; + +/** + * DataMate 知识库列表查询参数。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +@SerializeStrategy(include = SerializeStrategy.Include.NON_NULL) +public class DataMateKnowledgeListQueryParam { + /** + * 页码，从0开始。 + */ + private Integer page; + + /** + * 每页大小。 + */ + private Integer size; + + /** + * 知识库名称过滤。 + */ + private String name; + + /** + * 知识库描述过滤。 + */ + private String description; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/dto/DataMateRetrievalParam.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/dto/DataMateRetrievalParam.java new file mode 100644 index 000000000..0af237480 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/dto/DataMateRetrievalParam.java @@ -0,0 +1,44 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.dto; + +import lombok.Builder; +import lombok.Data; +import modelengine.fitframework.annotation.Property; + +import java.util.List; + +/** + * DataMate 知识库检索查询参数。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +public class DataMateRetrievalParam { + /** + * 检索 query。 + */ + private String query; + /** + * 返回前多少的条目。 + */ + @Property(description = "topK", name = "topK") + private Integer topK; + /** + * 相关性阈值。 + */ + @Property(description = "threshold", name = "threshold") + private Double threshold; + /** + * 指定知识库的id集合。 + */ + @Property(description = "knowledgeBaseIds", name = "knowledgeBaseIds") + private List knowledgeBaseIds; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateKnowledgeEntity.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateKnowledgeEntity.java new file mode 100644 index 000000000..823d68b72 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateKnowledgeEntity.java @@ -0,0 +1,98 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.entity; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * DataMate 知识库 Entity。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataMateKnowledgeEntity { + /** + * 知识库id。 + */ + private String id; + /** + * 知识库名称。 + */ + private String name; + /** + * 知识库描述。 + */ + private String description; + /** + * 知识库创建时间。 + */ + private String createdAt; + /** + * 知识库更新时间。 + */ + private String updatedAt; + /** + * 创建人。 + */ + private String createdBy; + /** + * 更新人。 + */ + private String updatedBy; + /** + * 嵌入模型名称。 + */ + private String embeddingModel; + /** + * 聊天模型名称。 + */ + private String chatModel; + /** + * 文件数量。 + */ + private Long fileCount; + /** + * chunk 数量。 + */ + private Long chunkCount; + /** + * 嵌入模型配置。 + */ + private ModelConfig embedding; + /** + * 聊天模型配置。 + */ + private ModelConfig chat; + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ModelConfig { + private String id; + private String createdAt; + private String updatedAt; + private String createdBy; + private String updatedBy; + private String modelName; + private String provider; + private String baseUrl; + private String apiKey; + private String type; + private Boolean isEnabled; + private Boolean isDefault; + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateKnowledgeListEntity.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateKnowledgeListEntity.java new file mode 100644 index 000000000..6cd4e8e44 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateKnowledgeListEntity.java @@ -0,0 +1,53 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.entity; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * DataMate 知识库列表 Entity。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataMateKnowledgeListEntity { + /** + * 当前页码（从 0 开始）。 + */ + private Integer page; + /** + * 每页数量。 + */ + private Integer size; + /** + * 总条目数。 + */ + private Integer totalElements; + /** + * 总页数。 + */ + private Integer totalPages; + /** + * 知识库列表查询数据。 + */ + @JsonProperty("content") + private List content; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateResponse.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateResponse.java new file mode 100644 index 000000000..2fb686999 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateResponse.java @@ -0,0 +1,45 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.entity; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.Data; +import modelengine.fitframework.util.ObjectUtils; + +import java.util.Map; + +/** + * DataMate 接口返回值结构。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +public class DataMateResponse { + private T data; + private String code; + private String message; + + /** + * 表示 DataMate 知识库请求结构体。 + * + * @param context 表示 http 响应数据内容的 {@link Map}{@code <}{@link String}{@code ,}{@link Object}{@code >}。 + * @param type 表示响应数据期望解析类型的 {@link Class}{@code <}{@link T}{@code >}。 + * @return 表示 DataMate 知识库请求的 {@link DataMateResponse}。 + * @param 表示泛型的 {@code <}{@link T}{@code >}。 + */ + public static DataMateResponse from(Map context, Class type) { + DataMateResponse response = new DataMateResponse<>(); + ObjectMapper objectMapper = new ObjectMapper(); + response.data = objectMapper.convertValue(context.get("data"), type); + response.code = ObjectUtils.cast(context.get("code")); + response.message = ObjectUtils.cast(context.get("message")); + return response; + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateRetrievalChunksEntity.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateRetrievalChunksEntity.java new file mode 100644 index 000000000..87657a2d3 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateRetrievalChunksEntity.java @@ -0,0 +1,55 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.entity; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import lombok.Data; + +/** + * DataMate 知识库检索 chunk 结果。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataMateRetrievalChunksEntity { + /** + * 查询结果实体。 + */ + private RagChunk entity; + /** + * 匹配分值。 + */ + private Double score; + /** + * 主键字段名。 + */ + private String primaryKey; + + public String chunkId() { + return this.entity != null ? this.entity.getId() : null; + } + + public String content() { + return this.entity != null ? this.entity.getText() : null; + } + + public double retrievalScore() { + return this.score == null ? 0 : this.score; + } + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class RagChunk { + private String id; + private String text; + private String metadata; + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateRetrievalResult.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateRetrievalResult.java new file mode 100644 index 000000000..176c57ceb --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/DataMateRetrievalResult.java @@ -0,0 +1,29 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.entity; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import lombok.Data; + +import java.util.List; + +/** + * DataMate 知识库检索结果。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataMateRetrievalResult { + /** + * 检索结果集合。 + */ + private List data; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/PageVoKnowledgeList.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/PageVoKnowledgeList.java new file mode 100644 index 000000000..c6e977176 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/entity/PageVoKnowledgeList.java @@ -0,0 +1,33 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.entity; + +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * 表示知识库列表分页数据对象。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +public class PageVoKnowledgeList { + /** + * 知识库列表查询数据。 + */ + private List knowledgeEntityList; + + /** + * 知识库总数。 + */ + private int total; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/external/DataMateKnowledgeBaseManager.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/external/DataMateKnowledgeBaseManager.java new file mode 100644 index 000000000..4d64888d8 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/external/DataMateKnowledgeBaseManager.java @@ -0,0 +1,140 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.external; + +import modelengine.fit.http.client.HttpClassicClient; +import modelengine.fit.http.client.HttpClassicClientFactory; +import modelengine.fit.http.client.HttpClassicClientRequest; +import modelengine.fit.http.client.HttpClientResponseException; +import modelengine.fit.http.entity.Entity; +import modelengine.fit.http.protocol.HttpRequestMethod; +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.annotation.Value; +import modelengine.fitframework.exception.ClientException; +import modelengine.fitframework.inspection.Validation; +import modelengine.fitframework.log.Logger; +import modelengine.fitframework.util.LazyLoader; +import modelengine.fitframework.util.MapBuilder; +import modelengine.fitframework.util.ObjectUtils; +import modelengine.jade.knowledge.code.KnowledgeManagerRetCode; +import modelengine.fit.jade.datamate.knowledge.knowledge.dto.DataMateKnowledgeListQueryParam; +import modelengine.fit.jade.datamate.knowledge.knowledge.dto.DataMateRetrievalParam; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateKnowledgeListEntity; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateResponse; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateRetrievalResult; +import modelengine.jade.knowledge.exception.KnowledgeException; + +import java.util.HashMap; +import java.util.Map; + +import static modelengine.fit.http.protocol.MessageHeaderNames.AUTHORIZATION; +import static modelengine.fit.http.protocol.MessageHeaderNames.CONTENT_TYPE; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.AUTHENTICATION_ERROR; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.CLIENT_REQUEST_ERROR; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.INTERNAL_SERVICE_ERROR; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.NOT_FOUND; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.QUERY_KNOWLEDGE_ERROR; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.QUERY_KNOWLEDGE_LIST_ERROR; + +/** + * 表示 DataMate 知识库的调用工具。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Component +public class DataMateKnowledgeBaseManager { + private static final Logger log = Logger.get(DataMateKnowledgeBaseManager.class); + private static final String BEARER = "Bearer "; + private static final String CONTENT_TYPE_JSON = "application/json"; + + private final Map dataMateUrls; + private final HttpClassicClientFactory httpClientFactory; + private final LazyLoader httpClient; + private final Map exceptionMap = new HashMap<>(); + + public DataMateKnowledgeBaseManager(@Value("${datamate.url}") Map dataMateUrls, + HttpClassicClientFactory httpClientFactory) { + this.dataMateUrls = dataMateUrls; + this.httpClientFactory = httpClientFactory; + this.httpClient = new LazyLoader<>(this::getHttpClient); + this.exceptionMap.put(500, INTERNAL_SERVICE_ERROR); + this.exceptionMap.put(401, AUTHENTICATION_ERROR); + this.exceptionMap.put(404, NOT_FOUND); + this.exceptionMap.put(400, CLIENT_REQUEST_ERROR); + } + + /** + * 获取 DataMate 知识库列表。 + * + * @param apiKey 表示知识库接口鉴权 api key 的 {@link String}。 + * @param param 表示知识库列表查询参数的 {@link DataMateKnowledgeListQueryParam}。 + * @return 表示知识库列表的 {@link DataMateKnowledgeListEntity}。 + */ + public DataMateKnowledgeListEntity listRepos(String apiKey, DataMateKnowledgeListQueryParam param) { + HttpClassicClientRequest request = + this.httpClient.get().createRequest(HttpRequestMethod.POST, this.dataMateUrls.get("list")); + request.entity(Entity.createObject(request, param)); + request.headers().set(AUTHORIZATION, BEARER + apiKey); + try { + Object object = this.httpClient.get().exchangeForEntity(request, Object.class); + Map response = + ObjectUtils.toCustomObject(Validation.notNull(object, "The response body is abnormal."), Map.class); + DataMateResponse resp = + DataMateResponse.from(response, DataMateKnowledgeListEntity.class); + return Validation.notNull(resp.getData(), "The response body is abnormal."); + } catch (ClientException ex) { + log.error(QUERY_KNOWLEDGE_LIST_ERROR.getMsg(), ex.getMessage()); + throw new KnowledgeException(QUERY_KNOWLEDGE_LIST_ERROR, ex.getMessage()); + } catch (HttpClientResponseException ex) { + throw this.handleException(ex); + } + } + + /** + * DataMate 知识库检索。 + * + * @param apiKey 表示知识库接口鉴权 api key 的 {@link String}。 + * @param param 表示知识库检索查询参数的 {@link DataMateRetrievalParam}。 + * @return 表示知识库检索结果的 {@link DataMateRetrievalResult}。 + */ + public DataMateRetrievalResult retrieve(String apiKey, DataMateRetrievalParam param) { + HttpClassicClientRequest request = + this.httpClient.get().createRequest(HttpRequestMethod.POST, this.dataMateUrls.get("retrieve")); + request.entity(Entity.createObject(request, param)); + request.headers().set(AUTHORIZATION, BEARER + apiKey); + request.headers().set(CONTENT_TYPE, CONTENT_TYPE_JSON); + try { + Object object = this.httpClient.get().exchangeForEntity(request, Object.class); + Map response = + ObjectUtils.toCustomObject(Validation.notNull(object, "The response body is abnormal."), Map.class); + DataMateResponse resp = + DataMateResponse.from(response, DataMateRetrievalResult.class); + return Validation.notNull(resp.getData(), "The response body is abnormal."); + } catch (ClientException ex) { + log.error(QUERY_KNOWLEDGE_ERROR.getMsg(), ex.getMessage()); + throw new KnowledgeException(QUERY_KNOWLEDGE_ERROR, ex.getMessage()); + } catch (HttpClientResponseException ex) { + throw this.handleException(ex); + } + } + + private KnowledgeException handleException(HttpClientResponseException ex) { + int statusCode = ex.statusCode(); + log.error(this.exceptionMap.get(statusCode).getMsg(), ex); + return new KnowledgeException(this.exceptionMap.get(statusCode), ex.getSimpleMessage()); + } + + private HttpClassicClient getHttpClient() { + Map custom = MapBuilder.get() + .put("client.http.secure.ignore-trust", true) + .put("client.http.secure.ignore-hostname", true) + .build(); + return this.httpClientFactory.create(HttpClassicClientFactory.Config.builder().custom(custom).build()); + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/service/DataMateKnowledgeRepoServiceImpl.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/service/DataMateKnowledgeRepoServiceImpl.java new file mode 100644 index 000000000..77398a950 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/service/DataMateKnowledgeRepoServiceImpl.java @@ -0,0 +1,155 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.knowledge.service; + +import modelengine.fit.jade.datamate.knowledge.knowledge.convertor.ParamConvertor; +import modelengine.fit.jade.datamate.knowledge.knowledge.dto.DataMateKnowledgeListQueryParam; +import modelengine.fit.jade.datamate.knowledge.knowledge.dto.DataMateRetrievalParam; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateKnowledgeEntity; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateKnowledgeListEntity; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateRetrievalChunksEntity; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateRetrievalResult; +import modelengine.fit.jade.datamate.knowledge.knowledge.entity.PageVoKnowledgeList; +import modelengine.fit.jade.datamate.knowledge.knowledge.external.DataMateKnowledgeBaseManager; +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.annotation.Fitable; +import modelengine.fitframework.inspection.Validation; +import modelengine.jade.common.vo.PageVo; +import modelengine.jade.knowledge.FilterConfig; +import modelengine.jade.knowledge.KnowledgeI18nInfo; +import modelengine.jade.knowledge.KnowledgeI18nService; +import modelengine.jade.knowledge.KnowledgeProperty; +import modelengine.jade.knowledge.KnowledgeRepo; +import modelengine.jade.knowledge.KnowledgeRepoService; +import modelengine.jade.knowledge.ListRepoQueryParam; +import modelengine.jade.knowledge.document.KnowledgeDocument; +import modelengine.jade.knowledge.enums.FilterType; +import modelengine.jade.knowledge.enums.IndexType; +import modelengine.jade.knowledge.support.FlatFilterConfig; +import modelengine.jade.knowledge.support.FlatKnowledgeOption; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 表示知识库服务在 DataMate 中的实现。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Component +public class DataMateKnowledgeRepoServiceImpl implements KnowledgeRepoService { + /** + * DataMate 知识库的服务唯一标识。 + */ + public static final String FITABLE_ID_DEFAULT = "dataMateKnowledge"; + + private static final int DEFAULT_TOP_K = 3; + private static final int MAX_TOP_K = 10; + private static final float DEFAULT_THRESHOLD = 0.1f; + + private final DataMateKnowledgeBaseManager knowledgeBaseManager; + private final KnowledgeI18nService knowledgeI18nService; + + public DataMateKnowledgeRepoServiceImpl(DataMateKnowledgeBaseManager knowledgeBaseManager, + KnowledgeI18nService knowledgeI18nService) { + this.knowledgeBaseManager = knowledgeBaseManager; + this.knowledgeI18nService = knowledgeI18nService; + } + + @Override + @Fitable(FITABLE_ID_DEFAULT) + public PageVo listRepos(String apiKey, ListRepoQueryParam param) { + Validation.notNull(param, "The query param cannot be null."); + PageVoKnowledgeList pageVoKnowledgeList = this.queryKnowledgeList(apiKey, param); + List repos = pageVoKnowledgeList.getKnowledgeEntityList() + .stream() + .map(ParamConvertor.INSTANCE::convertToKnowledgeRepo) + .toList(); + return PageVo.of(pageVoKnowledgeList.getTotal(), repos); + } + + @Override + @Fitable(FITABLE_ID_DEFAULT) + public KnowledgeProperty getProperty(String apiKey) { + KnowledgeI18nInfo semanticInfo = this.knowledgeI18nService.localizeText(IndexType.SEMANTIC); + KnowledgeProperty.IndexInfo semanticIndex = new KnowledgeProperty.IndexInfo(IndexType.SEMANTIC, + semanticInfo.getName(), + semanticInfo.getDescription()); + KnowledgeI18nInfo fullTextInfo = this.knowledgeI18nService.localizeText(IndexType.FULL_TEXT); + KnowledgeProperty.IndexInfo fullTextIndex = new KnowledgeProperty.IndexInfo(IndexType.FULL_TEXT, + fullTextInfo.getName(), + fullTextInfo.getDescription()); + KnowledgeI18nInfo hybridInfo = this.knowledgeI18nService.localizeText(IndexType.HYBRID); + KnowledgeProperty.IndexInfo hybridIndex = + new KnowledgeProperty.IndexInfo(IndexType.HYBRID, hybridInfo.getName(), hybridInfo.getDescription()); + KnowledgeI18nInfo referenceInfo = this.knowledgeI18nService.localizeText(FilterType.REFERENCE_TOP_K); + FlatFilterConfig topKFilter = new FlatFilterConfig(FilterConfig.custom() + .name(referenceInfo.getName()) + .description(referenceInfo.getDescription()) + .type(FilterType.REFERENCE_TOP_K) + .minimum(1) + .maximum(MAX_TOP_K) + .defaultValue(DEFAULT_TOP_K) + .build()); + KnowledgeI18nInfo relevancyInfo = this.knowledgeI18nService.localizeText(FilterType.SIMILARITY_THRESHOLD); + FlatFilterConfig similarityFilter = new FlatFilterConfig(FilterConfig.custom() + .name(relevancyInfo.getName()) + .description(relevancyInfo.getDescription()) + .type(FilterType.SIMILARITY_THRESHOLD) + .minimum(0) + .maximum(1) + .defaultValue(DEFAULT_THRESHOLD) + .build()); + KnowledgeI18nInfo rerankInfo = new KnowledgeI18nInfo(this.knowledgeI18nService.localizeText("rerankParam"), + this.knowledgeI18nService.localizeText("rerankParam.description")); + KnowledgeProperty.RerankConfig rerankConfig = + new KnowledgeProperty.RerankConfig("boolean", rerankInfo.getName(), rerankInfo.getDescription(), false); + return new KnowledgeProperty(Arrays.asList(semanticIndex, fullTextIndex, hybridIndex), + Arrays.asList(topKFilter, similarityFilter), + Collections.singletonList(rerankConfig)); + } + + @Override + @Fitable(FITABLE_ID_DEFAULT) + public List retrieve(String apiKey, FlatKnowledgeOption option) { + Validation.notNull(option, "The knowledge option cannot be null."); + DataMateRetrievalParam param = ParamConvertor.INSTANCE.convertToRetrievalParam(option); + DataMateRetrievalResult result = this.knowledgeBaseManager.retrieve(apiKey, param); + List chunks = result.getData() == null + ? Collections.emptyList() + : result.getData(); + return chunks + .stream() + .map(ParamConvertor.INSTANCE::convertToKnowledgeDocument) + .collect(Collectors.toList()); + } + + private PageVoKnowledgeList queryKnowledgeList(String apiKey, ListRepoQueryParam param) { + int page = Math.max(param.getPageIndex() - 1, 0); + int size = param.getPageSize(); + DataMateKnowledgeListEntity listEntity = this.executeQuery(apiKey, param.getRepoName(), page, size); + List content = listEntity.getContent() == null + ? Collections.emptyList() + : listEntity.getContent(); + return PageVoKnowledgeList.builder() + .knowledgeEntityList(content) + .total(listEntity.getTotalElements() == null ? 0 : listEntity.getTotalElements()) + .build(); + } + + private DataMateKnowledgeListEntity executeQuery(String apiKey, String repoName, int page, int size) { + DataMateKnowledgeListQueryParam queryParam = DataMateKnowledgeListQueryParam.builder() + .name(repoName) + .page(page) + .size(size) + .build(); + return this.knowledgeBaseManager.listRepos(apiKey, queryParam); + } +} diff --git a/app-builder/plugins/data-mate-knowledge/src/main/resources/application.yml b/app-builder/plugins/data-mate-knowledge/src/main/resources/application.yml new file mode 100644 index 000000000..1078a87d2 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/resources/application.yml @@ -0,0 +1,9 @@ +fit: + beans: + packages: + - 'modelengine.fit.jade.datamate.knowledge' +datamate: + url: + list: '${datamate.knowledge-list-url:https://datamate.example.com/api/knowledge-base/list}' + retrieve: '${datamate.knowledge-retrieve-url:https://datamate.example.com/api/knowledge-base/retrieve}' + diff --git a/app-builder/plugins/pom.xml b/app-builder/plugins/pom.xml index d89361406..e16f10165 100644 --- a/app-builder/plugins/pom.xml +++ b/app-builder/plugins/pom.xml @@ -39,6 +39,7 @@ app-base app-metrics app-metrics-influxdb + data-mate-knowledge eval-dataset eval-task flow-graph-db-driver