From 2bd5c795af239492d8eba0f91aa7c5a4aa48f7de Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 15 May 2024 14:56:58 +0800 Subject: [PATCH] =?UTF-8?q?feat(=E5=9B=BE=E8=A1=A8):=20=E5=9B=BE=E8=A1=A8?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=E5=99=A8=E6=94=AF=E6=8C=81=E5=A4=8D=E6=9D=82?= =?UTF-8?q?=E7=9A=84=E4=B8=8E=E6=88=96=E6=9D=A1=E4=BB=B6=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chart/manage/ChartDataManage.java | 32 +-- .../chart/manage/ChartFilterTreeService.java | 66 +++++ .../chart/manage/ChartViewManege.java | 5 +- .../manage/ChartViewOldDataMergeService.java | 150 ++++++++++ .../engine/trans/CustomWhere2Str.java | 268 ++++++++++-------- .../listener/ChartFilterMergeListener.java | 45 +++ .../dao/auto/entity/CoreSysStartupJob.java | 66 +++++ .../auto/mapper/CoreSysStartupJobMapper.java | 18 ++ .../main/resources/db/desktop/V2.7__ddl.sql | 15 +- .../main/resources/db/migration/V2.7__ddl.sql | 15 +- .../api/chart/dto/ChartViewBaseDTO.java | 3 +- .../api/chart/filter/FilterTreeItem.java | 24 ++ .../api/chart/filter/FilterTreeObj.java | 15 + 13 files changed, 575 insertions(+), 147 deletions(-) create mode 100644 core/core-backend/src/main/java/io/dataease/chart/manage/ChartFilterTreeService.java create mode 100644 core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewOldDataMergeService.java create mode 100644 core/core-backend/src/main/java/io/dataease/listener/ChartFilterMergeListener.java create mode 100644 core/core-backend/src/main/java/io/dataease/startup/dao/auto/entity/CoreSysStartupJob.java create mode 100644 core/core-backend/src/main/java/io/dataease/startup/dao/auto/mapper/CoreSysStartupJobMapper.java create mode 100644 sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeItem.java create mode 100644 sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeObj.java diff --git a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java index 90a8228300..308bc4940d 100644 --- a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java @@ -1,6 +1,7 @@ package io.dataease.chart.manage; import io.dataease.api.chart.dto.*; +import io.dataease.api.chart.filter.FilterTreeObj; import io.dataease.api.chart.request.ChartDrillRequest; import io.dataease.api.chart.request.ChartExtRequest; import io.dataease.api.dataset.dto.SqlVariableDetails; @@ -65,6 +66,8 @@ public class ChartDataManage { private ChartViewManege chartViewManege; @Resource private PermissionManage permissionManage; + @Resource + private ChartFilterTreeService chartFilterTreeService; @Resource private CorePermissionManage corePermissionManage; @@ -165,7 +168,7 @@ public class ChartDataManage { List extTooltip = new ArrayList<>(view.getExtTooltip()); yAxis.addAll(extTooltip); } - List fieldCustomFilter = new ArrayList<>(view.getCustomFilter()); + FilterTreeObj fieldCustomFilter = view.getCustomFilter(); List drill = new ArrayList<>(view.getDrillFields()); DatasetGroupInfoDTO table = datasetGroupManage.get(view.getTableId(), null); @@ -187,16 +190,14 @@ public class ChartDataManage { //将没有权限的列删掉 List dataeaseNames = columnPermissionFields.stream().map(DatasetTableFieldDTO::getDataeaseName).collect(Collectors.toList()); dataeaseNames.add("*"); - fieldCustomFilter = fieldCustomFilter.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList()); +// fieldCustomFilter = fieldCustomFilter.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList()); extStack = extStack.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList()); extBubble = extBubble.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList()); drill = drill.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList()); // row permission List rowPermissionsTree = permissionManage.getRowPermissionsTree(table.getId(), chartExtRequest.getUser()); - for (ChartFieldCustomFilterDTO ele : fieldCustomFilter) { - ele.setField(datasetTableFieldManage.selectById(ele.getId())); - } + chartFilterTreeService.searchFieldAndSet(fieldCustomFilter); if (ObjectUtils.isEmpty(xAxis) && ObjectUtils.isEmpty(yAxis)) { return emptyChartViewDTO(view); @@ -483,16 +484,7 @@ public class ChartDataManage { } // 处理过滤条件中的单引号 - fieldCustomFilter = fieldCustomFilter.stream().peek(ele -> { - if (ObjectUtils.isNotEmpty(ele.getEnumCheckField())) { - List collect = ele.getEnumCheckField().stream().map(SQLUtils::transKeyword).collect(Collectors.toList()); - ele.setEnumCheckField(collect); - } - if (ObjectUtils.isNotEmpty(ele.getFilter())) { - List collect = ele.getFilter().stream().peek(f -> f.setValue(SQLUtils.transKeyword(f.getValue()))).collect(Collectors.toList()); - ele.setFilter(collect); - } - }).collect(Collectors.toList()); + fieldCustomFilter = chartFilterTreeService.charReplace(fieldCustomFilter); extFilterList = extFilterList.stream().peek(ele -> { if (ObjectUtils.isNotEmpty(ele.getValue())) { @@ -1269,17 +1261,15 @@ public class ChartDataManage { } List extStack = new ArrayList<>(view.getExtStack()); List extBubble = new ArrayList<>(view.getExtBubble()); - List fieldCustomFilter = new ArrayList<>(view.getCustomFilter()); + FilterTreeObj fieldCustomFilter = view.getCustomFilter(); List drill = new ArrayList<>(view.getDrillFields()); // 获取数据集,需校验权限 - DatasetGroupInfoDTO table = datasetGroupManage.get(view.getTableId(), null);// todo - Map desensitizationList = new HashMap<>();// todo + DatasetGroupInfoDTO table = datasetGroupManage.get(view.getTableId(), null); + Map desensitizationList = new HashMap<>(); List rowPermissionsTree = permissionManage.getRowPermissionsTree(table.getId(), view.getChartExtRequest().getUser()); - for (ChartFieldCustomFilterDTO ele : fieldCustomFilter) { - ele.setField(datasetTableFieldManage.selectById(ele.getId())); - } + chartFilterTreeService.searchFieldAndSet(fieldCustomFilter); if (ObjectUtils.isEmpty(xAxis) && ObjectUtils.isEmpty(yAxis)) { return new ArrayList(); diff --git a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartFilterTreeService.java b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartFilterTreeService.java new file mode 100644 index 0000000000..05e3f3e839 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartFilterTreeService.java @@ -0,0 +1,66 @@ +package io.dataease.chart.manage; + +import io.dataease.api.chart.filter.FilterTreeItem; +import io.dataease.api.chart.filter.FilterTreeObj; +import io.dataease.dataset.dao.auto.entity.CoreDatasetTableField; +import io.dataease.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper; +import io.dataease.dto.dataset.DatasetTableFieldDTO; +import io.dataease.engine.utils.SQLUtils; +import io.dataease.utils.BeanUtils; +import jakarta.annotation.Resource; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * @Author Junjun + */ +@Service +public class ChartFilterTreeService { + @Resource + private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper; + + public void searchFieldAndSet(FilterTreeObj tree) { + if (ObjectUtils.isNotEmpty(tree)) { + if (ObjectUtils.isNotEmpty(tree.getItems())) { + for (FilterTreeItem item : tree.getItems()) { + if (ObjectUtils.isNotEmpty(item)) { + if (StringUtils.equalsIgnoreCase(item.getType(), "item") || ObjectUtils.isEmpty(item.getSubTree())) { + CoreDatasetTableField coreDatasetTableField = coreDatasetTableFieldMapper.selectById(item.getFieldId()); + DatasetTableFieldDTO dto = new DatasetTableFieldDTO(); + BeanUtils.copyBean(dto, coreDatasetTableField); + item.setField(dto); + } else if (StringUtils.equalsIgnoreCase(item.getType(), "tree") || (ObjectUtils.isNotEmpty(item.getSubTree()) && StringUtils.isNotEmpty(item.getSubTree().getLogic()))) { + searchFieldAndSet(item.getSubTree()); + } + } + } + } + } + } + + public FilterTreeObj charReplace(FilterTreeObj tree) { + if (ObjectUtils.isNotEmpty(tree)) { + if (ObjectUtils.isNotEmpty(tree.getItems())) { + for (FilterTreeItem item : tree.getItems()) { + if (ObjectUtils.isNotEmpty(item)) { + if (StringUtils.equalsIgnoreCase(item.getType(), "item") || ObjectUtils.isEmpty(item.getSubTree())) { + if (CollectionUtils.isNotEmpty(item.getEnumValue())) { + List collect = item.getEnumValue().stream().map(SQLUtils::transKeyword).collect(Collectors.toList()); + item.setEnumValue(collect); + } + item.setValue(SQLUtils.transKeyword(item.getValue())); + } else if (StringUtils.equalsIgnoreCase(item.getType(), "tree") || (ObjectUtils.isNotEmpty(item.getSubTree()) && StringUtils.isNotEmpty(item.getSubTree().getLogic()))) { + charReplace(item.getSubTree()); + } + } + } + } + } + return tree; + } +} diff --git a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewManege.java b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewManege.java index 58c003e400..b4b6427766 100644 --- a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewManege.java +++ b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewManege.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import io.dataease.api.chart.dto.*; +import io.dataease.api.chart.filter.FilterTreeObj; import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.chart.dao.auto.entity.CoreChartView; import io.dataease.chart.dao.auto.mapper.CoreChartViewMapper; @@ -238,8 +239,6 @@ public class ChartViewManege { TypeReference> tokenType = new TypeReference<>() { }; - TypeReference> filterTokenType = new TypeReference<>() { - }; dto.setXAxis(JsonUtil.parseList(record.getxAxis(), tokenType)); dto.setXAxisExt(JsonUtil.parseList(record.getxAxisExt(), tokenType)); @@ -253,7 +252,7 @@ public class ChartViewManege { dto.setCustomStyle(JsonUtil.parse(record.getCustomStyle(), Map.class)); dto.setSenior(JsonUtil.parse(record.getSenior(), Map.class)); dto.setDrillFields(JsonUtil.parseList(record.getDrillFields(), tokenType)); - dto.setCustomFilter(JsonUtil.parseList(record.getCustomFilter(), filterTokenType)); + dto.setCustomFilter(JsonUtil.parse(record.getCustomFilter(), FilterTreeObj.class)); dto.setViewFields(JsonUtil.parseList(record.getViewFields(), tokenType)); return dto; diff --git a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewOldDataMergeService.java b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewOldDataMergeService.java new file mode 100644 index 0000000000..63c265fd9c --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewOldDataMergeService.java @@ -0,0 +1,150 @@ +package io.dataease.chart.manage; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.fasterxml.jackson.core.type.TypeReference; +import io.dataease.api.chart.dto.ChartCustomFilterItemDTO; +import io.dataease.api.chart.dto.ChartFieldCustomFilterDTO; +import io.dataease.api.chart.filter.FilterTreeItem; +import io.dataease.api.chart.filter.FilterTreeObj; +import io.dataease.chart.dao.auto.entity.CoreChartView; +import io.dataease.chart.dao.auto.mapper.CoreChartViewMapper; +import io.dataease.utils.JsonUtil; +import jakarta.annotation.Resource; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; + +/** + * @Author Junjun + */ +@Service +public class ChartViewOldDataMergeService { + + @Resource + private CoreChartViewMapper coreChartViewMapper; + + /** + * 视图过滤器重构，合并老数据，将list变成tree + */ + public void mergeOldData() { + // 获取所有视图数据 + // 把一个视图中的过滤器，即customFilter字段进行重构 + // 之前是以list形式储存，一个字段是一个item + // 现在把一个字段当做tree中的一个节点 + // 节点中如果是logic且length>1，保留and或or，每一条都变成一个子节点；如果是枚举或只有1条，则当做and处理并保留值 + // 最后把字段之间通过and的逻辑合并 + List chartViewWithBLOBs = coreChartViewMapper.selectList(new QueryWrapper<>()); + if (CollectionUtils.isEmpty(chartViewWithBLOBs)) { + return; + } + + for (CoreChartView view : chartViewWithBLOBs) { + TypeReference> tokenType = new TypeReference<>() { + }; + List fieldCustomFilter; + // 尝试将历史数据转成list，如果转换出现异常，则忽略该视图继续执行下一个 + try { + fieldCustomFilter = JsonUtil.parseList(view.getCustomFilter(), tokenType); + } catch (Exception e) { + continue; + } + + if (CollectionUtils.isEmpty(fieldCustomFilter)) { + // 将 '[]' 转换成 '{}' + view.setCustomFilter("{}"); + } else { + // array -> tree + FilterTreeObj tree = transArr2Obj(fieldCustomFilter); + view.setCustomFilter((String) JsonUtil.toJSONString(tree)); + } + + try { + coreChartViewMapper.updateById(view); + } catch (Exception e) { + // do nothing,to continue + e.printStackTrace(); + } + } + } + + public FilterTreeObj transArr2Obj(List fieldCustomFilter) { + FilterTreeObj tree = new FilterTreeObj(); + tree.setItems(new ArrayList<>()); + if (fieldCustomFilter.size() == 1) { + ChartFieldCustomFilterDTO filterDTO = fieldCustomFilter.get(0); + tree.setLogic(filterDTO.getLogic()); + if (StringUtils.equalsIgnoreCase(filterDTO.getFilterType(), "enum")) { + FilterTreeItem item = new FilterTreeItem(); + item.setType("item"); + item.setFieldId(filterDTO.getId()); + item.setFilterType(filterDTO.getFilterType()); + item.setEnumValue(filterDTO.getEnumCheckField()); + tree.getItems().add(item); + } else { + List filter = filterDTO.getFilter(); + if (CollectionUtils.isNotEmpty(filter)) { + for (ChartCustomFilterItemDTO f : filter) { + FilterTreeItem item = new FilterTreeItem(); + item.setType("item"); + item.setFieldId(filterDTO.getId()); + item.setFilterType(filterDTO.getFilterType()); + item.setTerm(f.getTerm()); + item.setValue(f.getValue()); + item.setEnumValue(new ArrayList<>()); + tree.getItems().add(item); + } + } + } + } else { + tree.setLogic("and"); + for (ChartFieldCustomFilterDTO dto : fieldCustomFilter) { + if (StringUtils.equalsIgnoreCase(dto.getFilterType(), "enum")) { + FilterTreeItem item = new FilterTreeItem(); + item.setType("item"); + item.setFieldId(dto.getId()); + item.setFilterType(dto.getFilterType()); + item.setEnumValue(dto.getEnumCheckField()); + tree.getItems().add(item); + } else { + List filter = dto.getFilter(); + if (CollectionUtils.isNotEmpty(filter)) { + if (filter.size() == 1) { + ChartCustomFilterItemDTO f = filter.get(0); + FilterTreeItem item = new FilterTreeItem(); + item.setType("item"); + item.setFieldId(dto.getId()); + item.setFilterType(dto.getFilterType()); + item.setTerm(f.getTerm()); + item.setValue(f.getValue()); + item.setEnumValue(new ArrayList<>()); + tree.getItems().add(item); + } else { + FilterTreeItem item = new FilterTreeItem(); + item.setType("tree"); + item.setEnumValue(new ArrayList<>()); + FilterTreeObj subTree = new FilterTreeObj(); + subTree.setLogic(dto.getLogic()); + subTree.setItems(new ArrayList<>()); + for (ChartCustomFilterItemDTO f : filter) { + FilterTreeItem itemTree = new FilterTreeItem(); + itemTree.setType("item"); + itemTree.setFieldId(dto.getId()); + itemTree.setFilterType(dto.getFilterType()); + itemTree.setTerm(f.getTerm()); + itemTree.setValue(f.getValue()); + itemTree.setEnumValue(new ArrayList<>()); + subTree.getItems().add(itemTree); + } + item.setSubTree(subTree); + tree.getItems().add(item); + } + } + } + } + } + return tree; + } +} diff --git a/core/core-backend/src/main/java/io/dataease/engine/trans/CustomWhere2Str.java b/core/core-backend/src/main/java/io/dataease/engine/trans/CustomWhere2Str.java index c333932be8..b9913f0625 100644 --- a/core/core-backend/src/main/java/io/dataease/engine/trans/CustomWhere2Str.java +++ b/core/core-backend/src/main/java/io/dataease/engine/trans/CustomWhere2Str.java @@ -1,7 +1,7 @@ package io.dataease.engine.trans; -import io.dataease.api.chart.dto.ChartCustomFilterItemDTO; -import io.dataease.api.chart.dto.ChartFieldCustomFilterDTO; +import io.dataease.api.chart.filter.FilterTreeItem; +import io.dataease.api.chart.filter.FilterTreeObj; import io.dataease.api.dataset.union.model.SQLMeta; import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.dataset.dto.DatasourceSchemaDTO; @@ -9,6 +9,7 @@ import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SqlPlaceholderConstants; import io.dataease.engine.utils.Utils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -22,132 +23,159 @@ import java.util.Map; */ public class CustomWhere2Str { - public static void customWhere2sqlObj(SQLMeta meta, List fields, List originFields, boolean isCross, Map dsMap) { + public static void customWhere2sqlObj(SQLMeta meta, FilterTreeObj tree, List originFields, boolean isCross, Map dsMap) { SQLObj tableObj = meta.getTable(); if (ObjectUtils.isEmpty(tableObj)) { return; } List res = new ArrayList<>(); - Map fieldsDialect = new HashMap<>(); - if (ObjectUtils.isNotEmpty(fields)) { - for (ChartFieldCustomFilterDTO request : fields) { - List list = new ArrayList<>(); - DatasetTableFieldDTO field = request.getField(); - - if (ObjectUtils.isEmpty(field)) { - continue; - } - String whereName = ""; - String originName; - if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { - // 解析origin name中有关联的字段生成sql表达式 - String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields, isCross, dsMap); - // 给计算字段处加一个占位符，后续SQL方言转换后再替换 - originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId()); - fieldsDialect.put(originName, calcFieldExp); - if (isCross) { - originName = calcFieldExp; - } - } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { - originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); - } else { - originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); - } - if (field.getDeType() == 1) { - if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { - // 此处获取标准格式的日期 - whereName = String.format(SQLConstants.DE_STR_TO_DATE, originName, StringUtils.isNotEmpty(field.getDateFormat()) ? field.getDateFormat() : SQLConstants.DEFAULT_DATE_FORMAT); - } - if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { - String cast = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_INT_FORMAT); - // 此处获取标准格式的日期 - whereName = String.format(SQLConstants.FROM_UNIXTIME, cast, SQLConstants.DEFAULT_DATE_FORMAT); - } - if (field.getDeExtractType() == 1) { - // 此处获取标准格式的日期 - whereName = originName; - } - } else if (field.getDeType() == 2 || field.getDeType() == 3) { - if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { - whereName = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_FLOAT_FORMAT); - } - if (field.getDeExtractType() == 1) { - whereName = String.format(SQLConstants.UNIX_TIMESTAMP, originName); - } - if (field.getDeExtractType() == 2 || field.getDeExtractType() == 4) { - whereName = originName; - } - if (field.getDeExtractType() == 3) { - whereName = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_FLOAT_FORMAT); - } - } else { - whereName = originName; - } - - if (StringUtils.equalsIgnoreCase(request.getFilterType(), "enum")) { - if (ObjectUtils.isNotEmpty(request.getEnumCheckField())) { - res.add("(" + whereName + " IN ('" + String.join("','", request.getEnumCheckField()) + "'))"); - } - } else { - if (field.getDeType() == 1) { - // 规定几种日期格式，一一匹配，匹配到就是该格式 - whereName = String.format(SQLConstants.UNIX_TIMESTAMP, whereName); - } - - List filter = request.getFilter(); - for (ChartCustomFilterItemDTO filterItemDTO : filter) { - String value = filterItemDTO.getValue(); - String whereTerm = Utils.transFilterTerm(filterItemDTO.getTerm()); - String whereValue = ""; - - if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "null")) { - whereValue = ""; - } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_null")) { - whereValue = ""; - } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "empty")) { - whereValue = "''"; - } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) { - whereValue = "''"; - } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) { - whereValue = "('" + String.join("','", value.split(",")) + "')"; - } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) { - whereValue = "'%" + value + "%'"; - } else { - // 如果是时间字段过滤，当条件是等于和不等于的时候转换成between和not between - if (field.getDeType() == 1) { - if (StringUtils.equalsIgnoreCase(whereTerm, " = ")) { - whereTerm = " BETWEEN "; - // 把value类似过滤组件处理，获得start time和end time - Map stringLongMap = Utils.parseDateTimeValue(value); - whereValue = String.format(SQLConstants.WHERE_VALUE_BETWEEN, stringLongMap.get("startTime"), stringLongMap.get("endTime")); - } else if (StringUtils.equalsIgnoreCase(whereTerm, " <> ")) { - whereTerm = " NOT BETWEEN "; - Map stringLongMap = Utils.parseDateTimeValue(value); - whereValue = String.format(SQLConstants.WHERE_VALUE_BETWEEN, stringLongMap.get("startTime"), stringLongMap.get("endTime")); - } else { - value = Utils.allDateFormat2Long(value) + ""; - whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value); - } - } else { - whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value); - } - } - list.add(SQLObj.builder() - .whereField(whereName) - .whereTermAndValue(whereTerm + whereValue) - .build()); - } - List strList = new ArrayList<>(); - list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); - if (ObjectUtils.isNotEmpty(list)) { - res.add("(" + String.join(" " + Utils.getLogic(request.getLogic()) + " ", strList) + ")"); - } - } - } - meta.setCustomWheres(ObjectUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null); + // permission trees + // 解析每个tree，然后多个tree之间用and拼接 + // 每个tree，如果是sub tree节点，则使用递归合并成一组条件 + if (ObjectUtils.isEmpty(tree)) { + return; } + Map fieldsDialect = new HashMap<>(); + String treeExp = transTreeToWhere(tableObj, tree, fieldsDialect, originFields, isCross, dsMap); + if (StringUtils.isNotEmpty(treeExp)) { + res.add(treeExp); + } + meta.setCustomWheres(ObjectUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null); meta.setCustomWheresDialect(fieldsDialect); } + private static String transTreeToWhere(SQLObj tableObj, FilterTreeObj tree, Map fieldsDialect, List originFields, boolean isCross, Map dsMap) { + if (ObjectUtils.isEmpty(tree)) { + return null; + } + String logic = tree.getLogic(); + List items = tree.getItems(); + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(items)) { + // type is item or tree + for (FilterTreeItem item : items) { + String exp = null; + if (StringUtils.equalsIgnoreCase(item.getType(), "item")) { + // 单个item拼接SQL，最后根据logic汇总 + exp = transTreeItem(tableObj, item, fieldsDialect, originFields, isCross, dsMap); + } else if (StringUtils.equalsIgnoreCase(item.getType(), "tree")) { + // 递归tree + exp = transTreeToWhere(tableObj, item.getSubTree(), fieldsDialect, originFields, isCross, dsMap); + } + if (StringUtils.isNotEmpty(exp)) { + list.add(exp); + } + } + } + return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + logic + " ", list) + ")" : null; + } + private static String transTreeItem(SQLObj tableObj, FilterTreeItem item, Map fieldsDialect, List originFields, boolean isCross, Map dsMap) { + String res = null; + DatasetTableFieldDTO field = item.getField(); + + if (ObjectUtils.isEmpty(field)) { + return null; + } + String whereName = ""; + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields, isCross, dsMap); + // 给计算字段处加一个占位符，后续SQL方言转换后再替换 + originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId()); + fieldsDialect.put(originName, calcFieldExp); + if (isCross) { + originName = calcFieldExp; + } + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { + originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); + } else { + originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); + } + if (field.getDeType() == 1) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + // 此处获取标准格式的日期 + whereName = String.format(SQLConstants.DE_STR_TO_DATE, originName, StringUtils.isNotEmpty(field.getDateFormat()) ? field.getDateFormat() : SQLConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + String cast = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_INT_FORMAT); + // 此处获取标准格式的日期 + whereName = String.format(SQLConstants.FROM_UNIXTIME, cast, SQLConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == 1) { + // 此处获取标准格式的日期 + whereName = originName; + } + } else if (field.getDeType() == 2 || field.getDeType() == 3) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_FLOAT_FORMAT); + } + if (field.getDeExtractType() == 1) { + whereName = String.format(SQLConstants.UNIX_TIMESTAMP, originName); + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 4) { + whereName = originName; + } + if (field.getDeExtractType() == 3) { + whereName = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_FLOAT_FORMAT); + } + } else { + whereName = originName; + } + + if (StringUtils.equalsIgnoreCase(item.getFilterType(), "enum")) { + if (ObjectUtils.isNotEmpty(item.getEnumValue())) { + res = "(" + whereName + " IN ('" + String.join("','", item.getEnumValue()) + "'))"; + } + } else { + if (field.getDeType() == 1) { + // 规定几种日期格式，一一匹配，匹配到就是该格式 + whereName = String.format(SQLConstants.UNIX_TIMESTAMP, whereName); + } + + String value = item.getValue(); + String whereTerm = Utils.transFilterTerm(item.getTerm()); + String whereValue = ""; + + if (StringUtils.equalsIgnoreCase(item.getTerm(), "null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(item.getTerm(), "not_null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(item.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(item.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(item.getTerm(), "in") || StringUtils.containsIgnoreCase(item.getTerm(), "not in")) { + whereValue = "('" + String.join("','", value.split(",")) + "')"; + } else if (StringUtils.containsIgnoreCase(item.getTerm(), "like")) { + whereValue = "'%" + value + "%'"; + } else { + // 如果是时间字段过滤，当条件是等于和不等于的时候转换成between和not between + if (field.getDeType() == 1) { + if (StringUtils.equalsIgnoreCase(whereTerm, " = ")) { + whereTerm = " BETWEEN "; + // 把value类似过滤组件处理，获得start time和end time + Map stringLongMap = Utils.parseDateTimeValue(value); + whereValue = String.format(SQLConstants.WHERE_VALUE_BETWEEN, stringLongMap.get("startTime"), stringLongMap.get("endTime")); + } else if (StringUtils.equalsIgnoreCase(whereTerm, " <> ")) { + whereTerm = " NOT BETWEEN "; + Map stringLongMap = Utils.parseDateTimeValue(value); + whereValue = String.format(SQLConstants.WHERE_VALUE_BETWEEN, stringLongMap.get("startTime"), stringLongMap.get("endTime")); + } else { + value = Utils.allDateFormat2Long(value) + ""; + whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value); + } + } else { + whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value); + } + } + SQLObj build = SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build(); + res = build.getWhereField() + " " + build.getWhereTermAndValue(); + } + return res; + } } diff --git a/core/core-backend/src/main/java/io/dataease/listener/ChartFilterMergeListener.java b/core/core-backend/src/main/java/io/dataease/listener/ChartFilterMergeListener.java new file mode 100644 index 0000000000..6620b5e3a4 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/listener/ChartFilterMergeListener.java @@ -0,0 +1,45 @@ +package io.dataease.listener; + +import io.dataease.chart.manage.ChartViewOldDataMergeService; +import io.dataease.startup.dao.auto.entity.CoreSysStartupJob; +import io.dataease.startup.dao.auto.mapper.CoreSysStartupJobMapper; +import jakarta.annotation.Resource; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + + +/** + * @Author Junjun + */ +@Component +@Order(value = 4) +public class ChartFilterMergeListener implements ApplicationListener { + private final Logger logger = LoggerFactory.getLogger(ChartFilterMergeListener.class); + public static final String JOB_ID = "chartFilterMerge"; + @Resource + private CoreSysStartupJobMapper coreSysStartupJobMapper; + @Resource + private ChartViewOldDataMergeService chartViewOldDataMergeService; + + @Override + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + logger.info("====chart filter merge [start]===="); + + CoreSysStartupJob sysStartupJob = coreSysStartupJobMapper.selectById(JOB_ID); + if (ObjectUtils.isNotEmpty(sysStartupJob) && StringUtils.equalsIgnoreCase(sysStartupJob.getStatus(), "ready")) { + logger.info("====chart filter merge [doing]===="); + + chartViewOldDataMergeService.mergeOldData(); + + sysStartupJob.setStatus("done"); + coreSysStartupJobMapper.updateById(sysStartupJob); + } + logger.info("====chart filter merge [end]===="); + } +} diff --git a/core/core-backend/src/main/java/io/dataease/startup/dao/auto/entity/CoreSysStartupJob.java b/core/core-backend/src/main/java/io/dataease/startup/dao/auto/entity/CoreSysStartupJob.java new file mode 100644 index 0000000000..29065a7d90 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/startup/dao/auto/entity/CoreSysStartupJob.java @@ -0,0 +1,66 @@ +package io.dataease.startup.dao.auto.entity; + +import com.baomidou.mybatisplus.annotation.TableName; +import java.io.Serializable; + +/** + *

+ * 项目启动任务 + *

+ * + * @author fit2cloud + * @since 2024-05-15 + */ +@TableName("core_sys_startup_job") +public class CoreSysStartupJob implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * ID + */ + private String id; + + /** + * 任务名称 + */ + private String name; + + /** + * 任务状态 + */ + private String status; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + @Override + public String toString() { + return "CoreSysStartupJob{" + + "id = " + id + + ", name = " + name + + ", status = " + status + + "}"; + } +} diff --git a/core/core-backend/src/main/java/io/dataease/startup/dao/auto/mapper/CoreSysStartupJobMapper.java b/core/core-backend/src/main/java/io/dataease/startup/dao/auto/mapper/CoreSysStartupJobMapper.java new file mode 100644 index 0000000000..b2679ec755 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/startup/dao/auto/mapper/CoreSysStartupJobMapper.java @@ -0,0 +1,18 @@ +package io.dataease.startup.dao.auto.mapper; + +import io.dataease.startup.dao.auto.entity.CoreSysStartupJob; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; + +/** + *

+ * 项目启动任务 Mapper 接口 + *

+ * + * @author fit2cloud + * @since 2024-05-15 + */ +@Mapper +public interface CoreSysStartupJobMapper extends BaseMapper { + +} diff --git a/core/core-backend/src/main/resources/db/desktop/V2.7__ddl.sql b/core/core-backend/src/main/resources/db/desktop/V2.7__ddl.sql index 76dbdd4a62..81f0a52efc 100644 --- a/core/core-backend/src/main/resources/db/desktop/V2.7__ddl.sql +++ b/core/core-backend/src/main/resources/db/desktop/V2.7__ddl.sql @@ -1,2 +1,15 @@ alter table `core_chart_view` - add aggregate bit null comment '区间条形图开启时间纬度开启聚合'; \ No newline at end of file + add aggregate bit null comment '区间条形图开启时间纬度开启聚合'; + +DROP TABLE IF EXISTS `core_sys_startup_job`; +CREATE TABLE `core_sys_startup_job` +( + `id` varchar(64) NOT NULL COMMENT 'ID', + `name` varchar(255) DEFAULT NULL COMMENT '任务名称', + `status` varchar(255) DEFAULT NULL COMMENT '任务状态', + PRIMARY KEY (`id`) +) COMMENT ='项目启动任务'; + +BEGIN; +INSERT INTO `core_sys_startup_job` VALUES ('chartFilterMerge', 'chartFilterMerge', 'ready'); +COMMIT; diff --git a/core/core-backend/src/main/resources/db/migration/V2.7__ddl.sql b/core/core-backend/src/main/resources/db/migration/V2.7__ddl.sql index 76dbdd4a62..81f0a52efc 100644 --- a/core/core-backend/src/main/resources/db/migration/V2.7__ddl.sql +++ b/core/core-backend/src/main/resources/db/migration/V2.7__ddl.sql @@ -1,2 +1,15 @@ alter table `core_chart_view` - add aggregate bit null comment '区间条形图开启时间纬度开启聚合'; \ No newline at end of file + add aggregate bit null comment '区间条形图开启时间纬度开启聚合'; + +DROP TABLE IF EXISTS `core_sys_startup_job`; +CREATE TABLE `core_sys_startup_job` +( + `id` varchar(64) NOT NULL COMMENT 'ID', + `name` varchar(255) DEFAULT NULL COMMENT '任务名称', + `status` varchar(255) DEFAULT NULL COMMENT '任务状态', + PRIMARY KEY (`id`) +) COMMENT ='项目启动任务'; + +BEGIN; +INSERT INTO `core_sys_startup_job` VALUES ('chartFilterMerge', 'chartFilterMerge', 'ready'); +COMMIT; diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/chart/dto/ChartViewBaseDTO.java b/sdk/api/api-base/src/main/java/io/dataease/api/chart/dto/ChartViewBaseDTO.java index 631dd264eb..335ec085ef 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/chart/dto/ChartViewBaseDTO.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/chart/dto/ChartViewBaseDTO.java @@ -3,6 +3,7 @@ package io.dataease.api.chart.dto; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import io.dataease.api.chart.filter.FilterTreeObj; import lombok.Data; import java.io.Serializable; @@ -120,7 +121,7 @@ public class ChartViewBaseDTO implements Serializable { /** * 结果过滤 */ - private List customFilter; + private FilterTreeObj customFilter; /** * 钻取字段 diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeItem.java b/sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeItem.java new file mode 100644 index 0000000000..bc04c4cae2 --- /dev/null +++ b/sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeItem.java @@ -0,0 +1,24 @@ +package io.dataease.api.chart.filter; + +import io.dataease.dto.dataset.DatasetTableFieldDTO; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * @Author Junjun + */ +@Data +public class FilterTreeItem implements Serializable { + private String type;// 'item' or 'tree' + // item + private Long fieldId; + private DatasetTableFieldDTO field;// field object + private String filterType;// 'logic' or 'enum' + private String term;//'eq','not_eq','lt','le','gt','ge','in','not in','like','not like','null','not_null','empty','not_empty','between' + private String value;// 'a' + private List enumValue;// ['a','b'] + // tree + private FilterTreeObj subTree; +} diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeObj.java b/sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeObj.java new file mode 100644 index 0000000000..6841a150e5 --- /dev/null +++ b/sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeObj.java @@ -0,0 +1,15 @@ +package io.dataease.api.chart.filter; + +import lombok.Data; + +import java.util.List; + + +/** + * @Author Junjun + */ +@Data +public class FilterTreeObj { + private String logic; + private List items; +}