diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java index 6fe82437da..7d0967455d 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java @@ -1,13 +1,10 @@ package io.dataease.dataset.manage; -import io.dataease.extensions.view.dto.ChartExtFilterDTO; import io.dataease.api.chart.dto.ColumnPermissionItem; import io.dataease.api.chart.dto.DeSortField; -import io.dataease.extensions.view.dto.ChartExtRequest; import io.dataease.api.dataset.dto.*; import io.dataease.api.dataset.union.DatasetGroupInfoDTO; import io.dataease.api.dataset.union.DatasetTableInfoDTO; -import io.dataease.extensions.view.model.SQLMeta; import io.dataease.api.ds.vo.TableField; import io.dataease.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO; import io.dataease.auth.bo.TokenUserBO; @@ -16,16 +13,13 @@ import io.dataease.chart.utils.ChartDataBuild; import io.dataease.commons.utils.SqlparserUtils; import io.dataease.dataset.constant.DatasetTableType; import io.dataease.dataset.dto.DatasourceSchemaDTO; -import io.dataease.dataset.utils.FieldUtils; -import io.dataease.dataset.utils.SqlUtils; -import io.dataease.dataset.utils.TableUtils; +import io.dataease.dataset.utils.*; import io.dataease.datasource.dao.auto.entity.CoreDatasource; import io.dataease.datasource.dao.auto.mapper.CoreDatasourceMapper; import io.dataease.datasource.manage.EngineManage; import io.dataease.datasource.provider.CalciteProvider; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.datasource.utils.DatasourceUtils; -import io.dataease.extensions.view.dto.DatasetTableFieldDTO; import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SqlPlaceholderConstants; @@ -34,11 +28,16 @@ import io.dataease.engine.trans.*; import io.dataease.engine.utils.SQLUtils; import io.dataease.engine.utils.Utils; import io.dataease.exception.DEException; +import io.dataease.extensions.view.dto.ChartExtFilterDTO; +import io.dataease.extensions.view.dto.ChartExtRequest; +import io.dataease.extensions.view.dto.DatasetTableFieldDTO; import io.dataease.extensions.view.dto.SqlVariableDetails; +import io.dataease.extensions.view.model.SQLMeta; import io.dataease.i18n.Translator; import io.dataease.utils.AuthUtils; import io.dataease.utils.BeanUtils; import io.dataease.utils.JsonUtil; +import io.dataease.utils.TreeUtils; import jakarta.annotation.Resource; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -733,4 +732,118 @@ public class DatasetDataManage { } return previewData; } + + public List getFieldValueTree(List ids) throws Exception { + if (ids.isEmpty()) { + DEException.throwException("no field selected."); + } + // 根据前端传的查询组件field ids，获取所有字段枚举值并去重合并 + List> list = new ArrayList<>(); + List fields = new ArrayList<>(); + + // 根据图表计算字段，获取数据集 + List allFields = new ArrayList<>(); + DatasetTableFieldDTO field = datasetTableFieldManage.selectById(ids.getFirst()); + Long datasetGroupId = field.getDatasetGroupId(); + if (field.getChartId() != null) { + allFields.addAll(datasetTableFieldManage.getChartCalcFields(field.getChartId())); + } + DatasetGroupInfoDTO datasetGroupInfoDTO = datasetGroupManage.get(datasetGroupId, null); + + Map sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, new ChartExtRequest()); + String sql = (String) sqlMap.get("sql"); + + allFields.addAll(datasetGroupInfoDTO.getAllFields()); + + Map dsMap = (Map) sqlMap.get("dsMap"); + boolean crossDs = Utils.isCrossDs(dsMap); + if (!crossDs) { + sql = Utils.replaceSchemaAlias(sql, dsMap); + } + + // build query sql + SQLMeta sqlMeta = new SQLMeta(); + Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs); + + for (Long id : ids) { + DatasetTableFieldDTO f = datasetTableFieldManage.selectById(id); + if (f == null) { + DEException.throwException(Translator.get("i18n_no_field")); + } + // 获取allFields + fields.add(f); + } + + Map desensitizationList = new HashMap<>(); + fields = permissionManage.filterColumnPermissions(fields, desensitizationList, datasetGroupInfoDTO.getId(), null); + if (ObjectUtils.isEmpty(fields)) { + DEException.throwException(Translator.get("i18n_no_column_permission")); + } + buildFieldName(sqlMap, fields); + + List dsList = new ArrayList<>(); + for (Map.Entry next : dsMap.entrySet()) { + dsList.add(next.getValue().getType()); + } + boolean needOrder = Utils.isNeedOrder(dsList); + + List rowPermissionsTree = new ArrayList<>(); + TokenUserBO user = AuthUtils.getUser(); + if (user != null) { + rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId()); + } + + Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap); + WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap); + Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap); + String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, 1000); + querySQL = SqlUtils.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap); + logger.info("filter tree sql: " + querySQL); + + // 通过数据源请求数据 + // 调用数据源的calcite获得data + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setQuery(querySQL); + datasourceRequest.setDsList(dsMap); + Map data = calciteProvider.fetchResultField(datasourceRequest); + List rows = (List) data.get("data"); + + // 重新构造data + Set pkSet = new HashSet<>(); +// if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(rows) && existExtSortField && originSize > 0) { +// rows = rows.stream().map(row -> ArrayUtils.subarray(row, 0, originSize)).collect(Collectors.toList()); +// } + rows = rows.stream().filter(row -> { + int length = row.length; + boolean allEmpty = true; + for (String s : row) { + if (StringUtils.isNotBlank(s)) { + allEmpty = false; + } + } + return !allEmpty; + }).toList(); + List treeNodes = rows.stream().map(row -> buildTreeNode(row, pkSet)).flatMap(Collection::stream).collect(Collectors.toList()); + List tree = DatasetUtils.mergeDuplicateTree(treeNodes, "root"); + return tree; + } + + private List buildTreeNode(String[] row, Set pkSet) { + List nodes = new ArrayList<>(); + List parentPkList = new ArrayList<>(); + for (int i = 0; i < row.length; i++) { + String text = row[i]; + + parentPkList.add(text); + String val = String.join(TreeUtils.SEPARATOR, parentPkList); + String parentVal = i == 0 ? TreeUtils.DEFAULT_ROOT : row[i - 1]; + String pk = String.join(TreeUtils.SEPARATOR, parentPkList); + if (pkSet.contains(pk)) continue; + pkSet.add(pk); + BaseTreeNodeDTO node = new BaseTreeNodeDTO(val, parentVal, StringUtils.isNotBlank(text) ? text.trim() : text, pk + TreeUtils.SEPARATOR + i); + nodes.add(node); + } + return nodes; + + } } diff --git a/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetDataServer.java b/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetDataServer.java index 60e9740c32..7f13ed268c 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetDataServer.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetDataServer.java @@ -1,6 +1,7 @@ package io.dataease.dataset.server; import io.dataease.api.dataset.DatasetDataApi; +import io.dataease.api.dataset.dto.BaseTreeNodeDTO; import io.dataease.api.dataset.dto.DatasetTableDTO; import io.dataease.api.dataset.dto.EnumValueRequest; import io.dataease.api.dataset.dto.PreviewSqlDTO; @@ -70,4 +71,15 @@ public class DatasetDataServer implements DatasetDataApi { public Long getDatasetCount(DatasetGroupInfoDTO datasetGroupInfoDTO) throws Exception { return datasetDataManage.getDatasetTotal(datasetGroupInfoDTO.getId()); } + + @Override + public List getFieldValueTree(List ids) throws Exception { + try { + return datasetDataManage.getFieldValueTree(ids); + } catch (Exception e) { + e.printStackTrace(); + LogUtil.error(e); + return null; + } + } } diff --git a/core/core-backend/src/main/java/io/dataease/dataset/utils/DatasetUtils.java b/core/core-backend/src/main/java/io/dataease/dataset/utils/DatasetUtils.java new file mode 100644 index 0000000000..d11571a87e --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/dataset/utils/DatasetUtils.java @@ -0,0 +1,56 @@ +package io.dataease.dataset.utils; + +import io.dataease.api.dataset.dto.BaseTreeNodeDTO; +import io.dataease.utils.TreeUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.util.Assert; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @Author Junjun + */ +public class DatasetUtils { + public final static String SEPARATOR = "-de-"; + + public static List mergeDuplicateTree(List tree, String... rootPid) { + Assert.notNull(rootPid, "Root Pid cannot be null"); + if (CollectionUtils.isEmpty(tree)) { + return null; + } + List result = new ArrayList<>(); + // 构建id-节点map映射 + Map treePidMap = tree.stream().collect(Collectors.toMap(node -> node.getNodeType(), t -> t)); + tree.stream().filter(item -> ObjectUtils.isNotEmpty(item.getId())).forEach(node -> { + + String nodeType = node.getNodeType(); + String[] links = nodeType.split(SEPARATOR); + int length = links.length; + int level = Integer.parseInt(links[length - 1]); + // 判断根节点 + if (Arrays.asList(rootPid).contains(node.getPid()) && 0 == level) { + result.add(node); + } else { + //找到父元素 + String[] pLinks = new String[level]; + System.arraycopy(links, 0, pLinks, 0, level); + String parentType = Arrays.stream(pLinks).collect(Collectors.joining(SEPARATOR)) + TreeUtils.SEPARATOR + (level - 1); + BaseTreeNodeDTO parentNode = treePidMap.get(parentType); + if (parentNode == null) { + // 可能出现 rootPid 更高的节点 这个操作相当于截断 + return; + } + if (parentNode.getChildren() == null) { + parentNode.setChildren(new ArrayList()); + } + parentNode.getChildren().add(node); + } + }); + return result; + } +} diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DatasetDataApi.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DatasetDataApi.java index 3ea305171c..fde19bcaa7 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DatasetDataApi.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DatasetDataApi.java @@ -1,6 +1,7 @@ package io.dataease.api.dataset; import com.github.xiaoymin.knife4j.annotations.ApiSupport; +import io.dataease.api.dataset.dto.BaseTreeNodeDTO; import io.dataease.api.dataset.dto.DatasetTableDTO; import io.dataease.api.dataset.dto.EnumValueRequest; import io.dataease.api.dataset.dto.PreviewSqlDTO; @@ -47,4 +48,8 @@ public interface DatasetDataApi { @Operation(summary = "获取数据集总数据量", hidden = true) @PostMapping("getDatasetCount") Long getDatasetCount(@RequestBody DatasetGroupInfoDTO datasetGroupInfoDTO) throws Exception; + + @Operation(summary = "获取下拉树数据", hidden = true) + @PostMapping("getFieldTree") + List getFieldValueTree(@RequestBody List ids) throws Exception; } diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/dto/BaseTreeNodeDTO.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/dto/BaseTreeNodeDTO.java new file mode 100644 index 0000000000..c6768e00c4 --- /dev/null +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/dto/BaseTreeNodeDTO.java @@ -0,0 +1,28 @@ +package io.dataease.api.dataset.dto; + + +import lombok.Data; + +import java.util.List; + +@Data +public class BaseTreeNodeDTO { + + private String id; + + private String pid; + + private String text; + + private String nodeType; + + private List children; + + public BaseTreeNodeDTO(String id, String pid, String text, String nodeType) { + this.id = id; + this.pid = pid; + this.text = text; + this.nodeType = nodeType; + } + +}