center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat(图表): 图表过滤器支持复杂的与或条件设置

Browse Source
This commit is contained in:
junjun 2024-05-15 14:56:58 +08:00
parent 39ffbf638c
commit 2bd5c795af
13 changed files with 575 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java
View File

@ -1,6 +1,7 @@
package io.dataease.chart.manage;
import io.dataease.api.chart.dto.*;
import io.dataease.api.chart.filter.FilterTreeObj;
import io.dataease.api.chart.request.ChartDrillRequest;
import io.dataease.api.chart.request.ChartExtRequest;
import io.dataease.api.dataset.dto.SqlVariableDetails;
@ -65,6 +66,8 @@ public class ChartDataManage {
private ChartViewManege chartViewManege;
@Resource
private PermissionManage permissionManage;
@Resource
private ChartFilterTreeService chartFilterTreeService;
@Resource
private CorePermissionManage corePermissionManage;
@ -165,7 +168,7 @@ public class ChartDataManage {
List<ChartViewFieldDTO> extTooltip = new ArrayList<>(view.getExtTooltip());
yAxis.addAll(extTooltip);
}
List<ChartFieldCustomFilterDTO> fieldCustomFilter = new ArrayList<>(view.getCustomFilter());
FilterTreeObj fieldCustomFilter = view.getCustomFilter();
List<ChartViewFieldDTO> drill = new ArrayList<>(view.getDrillFields());
DatasetGroupInfoDTO table = datasetGroupManage.get(view.getTableId(), null);
@ -187,16 +190,14 @@ public class ChartDataManage {
//将没有权限的列删掉
List<String> dataeaseNames = columnPermissionFields.stream().map(DatasetTableFieldDTO::getDataeaseName).collect(Collectors.toList());
dataeaseNames.add("*");
fieldCustomFilter = fieldCustomFilter.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList());
// fieldCustomFilter = fieldCustomFilter.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList());
extStack = extStack.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList());
extBubble = extBubble.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList());
drill = drill.stream().filter(item -> !desensitizationList.keySet().contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList());
// row permission
List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = permissionManage.getRowPermissionsTree(table.getId(), chartExtRequest.getUser());
for (ChartFieldCustomFilterDTO ele : fieldCustomFilter) {
ele.setField(datasetTableFieldManage.selectById(ele.getId()));
}
chartFilterTreeService.searchFieldAndSet(fieldCustomFilter);
if (ObjectUtils.isEmpty(xAxis) && ObjectUtils.isEmpty(yAxis)) {
return emptyChartViewDTO(view);
@ -483,16 +484,7 @@ public class ChartDataManage {
}
// 处理过滤条件中的单引号
fieldCustomFilter = fieldCustomFilter.stream().peek(ele -> {
if (ObjectUtils.isNotEmpty(ele.getEnumCheckField())) {
List<String> collect = ele.getEnumCheckField().stream().map(SQLUtils::transKeyword).collect(Collectors.toList());
ele.setEnumCheckField(collect);
}
if (ObjectUtils.isNotEmpty(ele.getFilter())) {
List<ChartCustomFilterItemDTO> collect = ele.getFilter().stream().peek(f -> f.setValue(SQLUtils.transKeyword(f.getValue()))).collect(Collectors.toList());
ele.setFilter(collect);
}
}).collect(Collectors.toList());
fieldCustomFilter = chartFilterTreeService.charReplace(fieldCustomFilter);
extFilterList = extFilterList.stream().peek(ele -> {
if (ObjectUtils.isNotEmpty(ele.getValue())) {
@ -1269,17 +1261,15 @@ public class ChartDataManage {
}
List<ChartViewFieldDTO> extStack = new ArrayList<>(view.getExtStack());
List<ChartViewFieldDTO> extBubble = new ArrayList<>(view.getExtBubble());
List<ChartFieldCustomFilterDTO> fieldCustomFilter = new ArrayList<>(view.getCustomFilter());
FilterTreeObj fieldCustomFilter = view.getCustomFilter();
List<ChartViewFieldDTO> drill = new ArrayList<>(view.getDrillFields());
// 获取数据集,需校验权限
DatasetGroupInfoDTO table = datasetGroupManage.get(view.getTableId(), null);// todo
Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();// todo
DatasetGroupInfoDTO table = datasetGroupManage.get(view.getTableId(), null);
Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = permissionManage.getRowPermissionsTree(table.getId(), view.getChartExtRequest().getUser());
for (ChartFieldCustomFilterDTO ele : fieldCustomFilter) {
ele.setField(datasetTableFieldManage.selectById(ele.getId()));
}
chartFilterTreeService.searchFieldAndSet(fieldCustomFilter);
if (ObjectUtils.isEmpty(xAxis) && ObjectUtils.isEmpty(yAxis)) {
return new ArrayList<String[]>();

66
core/core-backend/src/main/java/io/dataease/chart/manage/ChartFilterTreeService.java Normal file
View File

@ -0,0 +1,66 @@
package io.dataease.chart.manage;
import io.dataease.api.chart.filter.FilterTreeItem;
import io.dataease.api.chart.filter.FilterTreeObj;
import io.dataease.dataset.dao.auto.entity.CoreDatasetTableField;
import io.dataease.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper;
import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.utils.SQLUtils;
import io.dataease.utils.BeanUtils;
import jakarta.annotation.Resource;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.stream.Collectors;
/**
* @Author Junjun
*/
@Service
public class ChartFilterTreeService {
@Resource
private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper;
public void searchFieldAndSet(FilterTreeObj tree) {
if (ObjectUtils.isNotEmpty(tree)) {
if (ObjectUtils.isNotEmpty(tree.getItems())) {
for (FilterTreeItem item : tree.getItems()) {
if (ObjectUtils.isNotEmpty(item)) {
if (StringUtils.equalsIgnoreCase(item.getType(), "item") || ObjectUtils.isEmpty(item.getSubTree())) {
CoreDatasetTableField coreDatasetTableField = coreDatasetTableFieldMapper.selectById(item.getFieldId());
DatasetTableFieldDTO dto = new DatasetTableFieldDTO();
BeanUtils.copyBean(dto, coreDatasetTableField);
item.setField(dto);
} else if (StringUtils.equalsIgnoreCase(item.getType(), "tree") || (ObjectUtils.isNotEmpty(item.getSubTree()) && StringUtils.isNotEmpty(item.getSubTree().getLogic()))) {
searchFieldAndSet(item.getSubTree());
}
}
}
}
}
}
public FilterTreeObj charReplace(FilterTreeObj tree) {
if (ObjectUtils.isNotEmpty(tree)) {
if (ObjectUtils.isNotEmpty(tree.getItems())) {
for (FilterTreeItem item : tree.getItems()) {
if (ObjectUtils.isNotEmpty(item)) {
if (StringUtils.equalsIgnoreCase(item.getType(), "item") || ObjectUtils.isEmpty(item.getSubTree())) {
if (CollectionUtils.isNotEmpty(item.getEnumValue())) {
List<String> collect = item.getEnumValue().stream().map(SQLUtils::transKeyword).collect(Collectors.toList());
item.setEnumValue(collect);
}
item.setValue(SQLUtils.transKeyword(item.getValue()));
} else if (StringUtils.equalsIgnoreCase(item.getType(), "tree") || (ObjectUtils.isNotEmpty(item.getSubTree()) && StringUtils.isNotEmpty(item.getSubTree().getLogic()))) {
charReplace(item.getSubTree());
}
}
}
}
}
return tree;
}
}

5
core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewManege.java
View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.dataease.api.chart.dto.*;
import io.dataease.api.chart.filter.FilterTreeObj;
import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.chart.dao.auto.entity.CoreChartView;
import io.dataease.chart.dao.auto.mapper.CoreChartViewMapper;
@ -238,8 +239,6 @@ public class ChartViewManege {
TypeReference<List<ChartViewFieldDTO>> tokenType = new TypeReference<>() {
};
TypeReference<List<ChartFieldCustomFilterDTO>> filterTokenType = new TypeReference<>() {
};
dto.setXAxis(JsonUtil.parseList(record.getxAxis(), tokenType));
dto.setXAxisExt(JsonUtil.parseList(record.getxAxisExt(), tokenType));
@ -253,7 +252,7 @@ public class ChartViewManege {
dto.setCustomStyle(JsonUtil.parse(record.getCustomStyle(), Map.class));
dto.setSenior(JsonUtil.parse(record.getSenior(), Map.class));
dto.setDrillFields(JsonUtil.parseList(record.getDrillFields(), tokenType));
dto.setCustomFilter(JsonUtil.parseList(record.getCustomFilter(), filterTokenType));
dto.setCustomFilter(JsonUtil.parse(record.getCustomFilter(), FilterTreeObj.class));
dto.setViewFields(JsonUtil.parseList(record.getViewFields(), tokenType));
return dto;

150
core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewOldDataMergeService.java Normal file
View File

@ -0,0 +1,150 @@
package io.dataease.chart.manage;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import io.dataease.api.chart.dto.ChartCustomFilterItemDTO;
import io.dataease.api.chart.dto.ChartFieldCustomFilterDTO;
import io.dataease.api.chart.filter.FilterTreeItem;
import io.dataease.api.chart.filter.FilterTreeObj;
import io.dataease.chart.dao.auto.entity.CoreChartView;
import io.dataease.chart.dao.auto.mapper.CoreChartViewMapper;
import io.dataease.utils.JsonUtil;
import jakarta.annotation.Resource;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
/**
* @Author Junjun
*/
@Service
public class ChartViewOldDataMergeService {
@Resource
private CoreChartViewMapper coreChartViewMapper;
/**
* 视图过滤器重构，合并老数据，将list变成tree
*/
public void mergeOldData() {
// 获取所有视图数据
// 把一个视图中的过滤器，即customFilter字段进行重构
// 之前是以list形式储存，一个字段是一个item
// 现在把一个字段当做tree中的一个节点
// 节点中如果是logic且length>1，保留and或or，每一条都变成一个子节点；如果是枚举或只有1条，则当做and处理并保留值
// 最后把字段之间通过and的逻辑合并
List<CoreChartView> chartViewWithBLOBs = coreChartViewMapper.selectList(new QueryWrapper<>());
if (CollectionUtils.isEmpty(chartViewWithBLOBs)) {
return;
}
for (CoreChartView view : chartViewWithBLOBs) {
TypeReference<List<ChartFieldCustomFilterDTO>> tokenType = new TypeReference<>() {
};
List<ChartFieldCustomFilterDTO> fieldCustomFilter;
// 尝试将历史数据转成list，如果转换出现异常，则忽略该视图继续执行下一个
try {
fieldCustomFilter = JsonUtil.parseList(view.getCustomFilter(), tokenType);
} catch (Exception e) {
continue;
}
if (CollectionUtils.isEmpty(fieldCustomFilter)) {
// 将 '[]' 转换成 '{}'
view.setCustomFilter("{}");
} else {
// array -> tree
FilterTreeObj tree = transArr2Obj(fieldCustomFilter);
view.setCustomFilter((String) JsonUtil.toJSONString(tree));
}
try {
coreChartViewMapper.updateById(view);
} catch (Exception e) {
// do nothing,to continue
e.printStackTrace();
}
}
}
public FilterTreeObj transArr2Obj(List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
FilterTreeObj tree = new FilterTreeObj();
tree.setItems(new ArrayList<>());
if (fieldCustomFilter.size() == 1) {
ChartFieldCustomFilterDTO filterDTO = fieldCustomFilter.get(0);
tree.setLogic(filterDTO.getLogic());
if (StringUtils.equalsIgnoreCase(filterDTO.getFilterType(), "enum")) {
FilterTreeItem item = new FilterTreeItem();
item.setType("item");
item.setFieldId(filterDTO.getId());
item.setFilterType(filterDTO.getFilterType());
item.setEnumValue(filterDTO.getEnumCheckField());
tree.getItems().add(item);
} else {
List<ChartCustomFilterItemDTO> filter = filterDTO.getFilter();
if (CollectionUtils.isNotEmpty(filter)) {
for (ChartCustomFilterItemDTO f : filter) {
FilterTreeItem item = new FilterTreeItem();
item.setType("item");
item.setFieldId(filterDTO.getId());
item.setFilterType(filterDTO.getFilterType());
item.setTerm(f.getTerm());
item.setValue(f.getValue());
item.setEnumValue(new ArrayList<>());
tree.getItems().add(item);
}
}
}
} else {
tree.setLogic("and");
for (ChartFieldCustomFilterDTO dto : fieldCustomFilter) {
if (StringUtils.equalsIgnoreCase(dto.getFilterType(), "enum")) {
FilterTreeItem item = new FilterTreeItem();
item.setType("item");
item.setFieldId(dto.getId());
item.setFilterType(dto.getFilterType());
item.setEnumValue(dto.getEnumCheckField());
tree.getItems().add(item);
} else {
List<ChartCustomFilterItemDTO> filter = dto.getFilter();
if (CollectionUtils.isNotEmpty(filter)) {
if (filter.size() == 1) {
ChartCustomFilterItemDTO f = filter.get(0);
FilterTreeItem item = new FilterTreeItem();
item.setType("item");
item.setFieldId(dto.getId());
item.setFilterType(dto.getFilterType());
item.setTerm(f.getTerm());
item.setValue(f.getValue());
item.setEnumValue(new ArrayList<>());
tree.getItems().add(item);
} else {
FilterTreeItem item = new FilterTreeItem();
item.setType("tree");
item.setEnumValue(new ArrayList<>());
FilterTreeObj subTree = new FilterTreeObj();
subTree.setLogic(dto.getLogic());
subTree.setItems(new ArrayList<>());
for (ChartCustomFilterItemDTO f : filter) {
FilterTreeItem itemTree = new FilterTreeItem();
itemTree.setType("item");
itemTree.setFieldId(dto.getId());
itemTree.setFilterType(dto.getFilterType());
itemTree.setTerm(f.getTerm());
itemTree.setValue(f.getValue());
itemTree.setEnumValue(new ArrayList<>());
subTree.getItems().add(itemTree);
}
item.setSubTree(subTree);
tree.getItems().add(item);
}
}
}
}
}
return tree;
}
}

268
core/core-backend/src/main/java/io/dataease/engine/trans/CustomWhere2Str.java
View File

@ -1,7 +1,7 @@
package io.dataease.engine.trans;
import io.dataease.api.chart.dto.ChartCustomFilterItemDTO;
import io.dataease.api.chart.dto.ChartFieldCustomFilterDTO;
import io.dataease.api.chart.filter.FilterTreeItem;
import io.dataease.api.chart.filter.FilterTreeObj;
import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
@ -9,6 +9,7 @@ import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -22,132 +23,159 @@ import java.util.Map;
*/
public class CustomWhere2Str {
public static void customWhere2sqlObj(SQLMeta meta, List<ChartFieldCustomFilterDTO> fields, List<DatasetTableFieldDTO> originFields, boolean isCross, Map<Long, DatasourceSchemaDTO> dsMap) {
public static void customWhere2sqlObj(SQLMeta meta, FilterTreeObj tree, List<DatasetTableFieldDTO> originFields, boolean isCross, Map<Long, DatasourceSchemaDTO> dsMap) {
SQLObj tableObj = meta.getTable();
if (ObjectUtils.isEmpty(tableObj)) {
return;
}
List<String> res = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (ObjectUtils.isNotEmpty(fields)) {
for (ChartFieldCustomFilterDTO request : fields) {
List<SQLObj> list = new ArrayList<>();
DatasetTableFieldDTO field = request.getField();
if (ObjectUtils.isEmpty(field)) {
continue;
}
String whereName = "";
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields, isCross, dsMap);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, calcFieldExp);
if (isCross) {
originName = calcFieldExp;
}
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
}
if (field.getDeType() == 1) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
// 此处获取标准格式的日期
whereName = String.format(SQLConstants.DE_STR_TO_DATE, originName, StringUtils.isNotEmpty(field.getDateFormat()) ? field.getDateFormat() : SQLConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
String cast = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_INT_FORMAT);
// 此处获取标准格式的日期
whereName = String.format(SQLConstants.FROM_UNIXTIME, cast, SQLConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == 1) {
// 此处获取标准格式的日期
whereName = originName;
}
} else if (field.getDeType() == 2 || field.getDeType() == 3) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
whereName = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_FLOAT_FORMAT);
}
if (field.getDeExtractType() == 1) {
whereName = String.format(SQLConstants.UNIX_TIMESTAMP, originName);
}
if (field.getDeExtractType() == 2 || field.getDeExtractType() == 4) {
whereName = originName;
}
if (field.getDeExtractType() == 3) {
whereName = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_FLOAT_FORMAT);
}
} else {
whereName = originName;
}
if (StringUtils.equalsIgnoreCase(request.getFilterType(), "enum")) {
if (ObjectUtils.isNotEmpty(request.getEnumCheckField())) {
res.add("(" + whereName + " IN ('" + String.join("','", request.getEnumCheckField()) + "'))");
}
} else {
if (field.getDeType() == 1) {
// 规定几种日期格式，一一匹配，匹配到就是该格式
whereName = String.format(SQLConstants.UNIX_TIMESTAMP, whereName);
}
List<ChartCustomFilterItemDTO> filter = request.getFilter();
for (ChartCustomFilterItemDTO filterItemDTO : filter) {
String value = filterItemDTO.getValue();
String whereTerm = Utils.transFilterTerm(filterItemDTO.getTerm());
String whereValue = "";
if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "empty")) {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {
// 如果是时间字段过滤，当条件是等于和不等于的时候转换成between和not between
if (field.getDeType() == 1) {
if (StringUtils.equalsIgnoreCase(whereTerm, " = ")) {
whereTerm = " BETWEEN ";
// 把value类似过滤组件处理，获得start time和end time
Map<String, Long> stringLongMap = Utils.parseDateTimeValue(value);
whereValue = String.format(SQLConstants.WHERE_VALUE_BETWEEN, stringLongMap.get("startTime"), stringLongMap.get("endTime"));
} else if (StringUtils.equalsIgnoreCase(whereTerm, " <> ")) {
whereTerm = " NOT BETWEEN ";
Map<String, Long> stringLongMap = Utils.parseDateTimeValue(value);
whereValue = String.format(SQLConstants.WHERE_VALUE_BETWEEN, stringLongMap.get("startTime"), stringLongMap.get("endTime"));
} else {
value = Utils.allDateFormat2Long(value) + "";
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value);
}
} else {
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value);
}
}
list.add(SQLObj.builder()
.whereField(whereName)
.whereTermAndValue(whereTerm + whereValue)
.build());
}
List<String> strList = new ArrayList<>();
list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue()));
if (ObjectUtils.isNotEmpty(list)) {
res.add("(" + String.join(" " + Utils.getLogic(request.getLogic()) + " ", strList) + ")");
}
}
}
meta.setCustomWheres(ObjectUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null);
// permission trees
// 解析每个tree，然后多个tree之间用and拼接
// 每个tree，如果是sub tree节点，则使用递归合并成一组条件
if (ObjectUtils.isEmpty(tree)) {
return;
}
Map<String, String> fieldsDialect = new HashMap<>();
String treeExp = transTreeToWhere(tableObj, tree, fieldsDialect, originFields, isCross, dsMap);
if (StringUtils.isNotEmpty(treeExp)) {
res.add(treeExp);
}
meta.setCustomWheres(ObjectUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null);
meta.setCustomWheresDialect(fieldsDialect);
}
private static String transTreeToWhere(SQLObj tableObj, FilterTreeObj tree, Map<String, String> fieldsDialect, List<DatasetTableFieldDTO> originFields, boolean isCross, Map<Long, DatasourceSchemaDTO> dsMap) {
if (ObjectUtils.isEmpty(tree)) {
return null;
}
String logic = tree.getLogic();
List<FilterTreeItem> items = tree.getItems();
List<String> list = new ArrayList<>();
if (CollectionUtils.isNotEmpty(items)) {
// type is item or tree
for (FilterTreeItem item : items) {
String exp = null;
if (StringUtils.equalsIgnoreCase(item.getType(), "item")) {
// 单个item拼接SQL，最后根据logic汇总
exp = transTreeItem(tableObj, item, fieldsDialect, originFields, isCross, dsMap);
} else if (StringUtils.equalsIgnoreCase(item.getType(), "tree")) {
// 递归tree
exp = transTreeToWhere(tableObj, item.getSubTree(), fieldsDialect, originFields, isCross, dsMap);
}
if (StringUtils.isNotEmpty(exp)) {
list.add(exp);
}
}
}
return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + logic + " ", list) + ")" : null;
}
private static String transTreeItem(SQLObj tableObj, FilterTreeItem item, Map<String, String> fieldsDialect, List<DatasetTableFieldDTO> originFields, boolean isCross, Map<Long, DatasourceSchemaDTO> dsMap) {
String res = null;
DatasetTableFieldDTO field = item.getField();
if (ObjectUtils.isEmpty(field)) {
return null;
}
String whereName = "";
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields, isCross, dsMap);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, calcFieldExp);
if (isCross) {
originName = calcFieldExp;
}
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
}
if (field.getDeType() == 1) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
// 此处获取标准格式的日期
whereName = String.format(SQLConstants.DE_STR_TO_DATE, originName, StringUtils.isNotEmpty(field.getDateFormat()) ? field.getDateFormat() : SQLConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
String cast = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_INT_FORMAT);
// 此处获取标准格式的日期
whereName = String.format(SQLConstants.FROM_UNIXTIME, cast, SQLConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == 1) {
// 此处获取标准格式的日期
whereName = originName;
}
} else if (field.getDeType() == 2 || field.getDeType() == 3) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
whereName = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_FLOAT_FORMAT);
}
if (field.getDeExtractType() == 1) {
whereName = String.format(SQLConstants.UNIX_TIMESTAMP, originName);
}
if (field.getDeExtractType() == 2 || field.getDeExtractType() == 4) {
whereName = originName;
}
if (field.getDeExtractType() == 3) {
whereName = String.format(SQLConstants.CAST, originName, SQLConstants.DEFAULT_FLOAT_FORMAT);
}
} else {
whereName = originName;
}
if (StringUtils.equalsIgnoreCase(item.getFilterType(), "enum")) {
if (ObjectUtils.isNotEmpty(item.getEnumValue())) {
res = "(" + whereName + " IN ('" + String.join("','", item.getEnumValue()) + "'))";
}
} else {
if (field.getDeType() == 1) {
// 规定几种日期格式，一一匹配，匹配到就是该格式
whereName = String.format(SQLConstants.UNIX_TIMESTAMP, whereName);
}
String value = item.getValue();
String whereTerm = Utils.transFilterTerm(item.getTerm());
String whereValue = "";
if (StringUtils.equalsIgnoreCase(item.getTerm(), "null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(item.getTerm(), "not_null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(item.getTerm(), "empty")) {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(item.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(item.getTerm(), "in") || StringUtils.containsIgnoreCase(item.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(item.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {
// 如果是时间字段过滤，当条件是等于和不等于的时候转换成between和not between
if (field.getDeType() == 1) {
if (StringUtils.equalsIgnoreCase(whereTerm, " = ")) {
whereTerm = " BETWEEN ";
// 把value类似过滤组件处理，获得start time和end time
Map<String, Long> stringLongMap = Utils.parseDateTimeValue(value);
whereValue = String.format(SQLConstants.WHERE_VALUE_BETWEEN, stringLongMap.get("startTime"), stringLongMap.get("endTime"));
} else if (StringUtils.equalsIgnoreCase(whereTerm, " <> ")) {
whereTerm = " NOT BETWEEN ";
Map<String, Long> stringLongMap = Utils.parseDateTimeValue(value);
whereValue = String.format(SQLConstants.WHERE_VALUE_BETWEEN, stringLongMap.get("startTime"), stringLongMap.get("endTime"));
} else {
value = Utils.allDateFormat2Long(value) + "";
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value);
}
} else {
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value);
}
}
SQLObj build = SQLObj.builder()
.whereField(whereName)
.whereTermAndValue(whereTerm + whereValue)
.build();
res = build.getWhereField() + " " + build.getWhereTermAndValue();
}
return res;
}
}

45
core/core-backend/src/main/java/io/dataease/listener/ChartFilterMergeListener.java Normal file
View File

@ -0,0 +1,45 @@
package io.dataease.listener;
import io.dataease.chart.manage.ChartViewOldDataMergeService;
import io.dataease.startup.dao.auto.entity.CoreSysStartupJob;
import io.dataease.startup.dao.auto.mapper.CoreSysStartupJobMapper;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
/**
* @Author Junjun
*/
@Component
@Order(value = 4)
public class ChartFilterMergeListener implements ApplicationListener<ApplicationReadyEvent> {
private final Logger logger = LoggerFactory.getLogger(ChartFilterMergeListener.class);
public static final String JOB_ID = "chartFilterMerge";
@Resource
private CoreSysStartupJobMapper coreSysStartupJobMapper;
@Resource
private ChartViewOldDataMergeService chartViewOldDataMergeService;
@Override
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
logger.info("====chart filter merge [start]====");
CoreSysStartupJob sysStartupJob = coreSysStartupJobMapper.selectById(JOB_ID);
if (ObjectUtils.isNotEmpty(sysStartupJob) && StringUtils.equalsIgnoreCase(sysStartupJob.getStatus(), "ready")) {
logger.info("====chart filter merge [doing]====");
chartViewOldDataMergeService.mergeOldData();
sysStartupJob.setStatus("done");
coreSysStartupJobMapper.updateById(sysStartupJob);
}
logger.info("====chart filter merge [end]====");
}
}

66
core/core-backend/src/main/java/io/dataease/startup/dao/auto/entity/CoreSysStartupJob.java Normal file
View File

@ -0,0 +1,66 @@
package io.dataease.startup.dao.auto.entity;
import com.baomidou.mybatisplus.annotation.TableName;
import java.io.Serializable;
/**
* <p>
* 项目启动任务
* </p>
*
* @author fit2cloud
* @since 2024-05-15
*/
@TableName("core_sys_startup_job")
public class CoreSysStartupJob implements Serializable {
private static final long serialVersionUID = 1L;
/**
* ID
*/
private String id;
/**
* 任务名称
*/
private String name;
/**
* 任务状态
*/
private String status;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
@Override
public String toString() {
return "CoreSysStartupJob{" +
"id = " + id +
", name = " + name +
", status = " + status +
"}";
}
}

18
core/core-backend/src/main/java/io/dataease/startup/dao/auto/mapper/CoreSysStartupJobMapper.java Normal file
View File

@ -0,0 +1,18 @@
package io.dataease.startup.dao.auto.mapper;
import io.dataease.startup.dao.auto.entity.CoreSysStartupJob;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
/**
* <p>
* 项目启动任务 Mapper 接口
* </p>
*
* @author fit2cloud
* @since 2024-05-15
*/
@Mapper
public interface CoreSysStartupJobMapper extends BaseMapper<CoreSysStartupJob> {
}

13
core/core-backend/src/main/resources/db/desktop/V2.7__ddl.sql
View File

@ -1,2 +1,15 @@
alter table `core_chart_view`
add aggregate bit null comment '区间条形图开启时间纬度开启聚合';
DROP TABLE IF EXISTS `core_sys_startup_job`;
CREATE TABLE `core_sys_startup_job`
(
`id` varchar(64) NOT NULL COMMENT 'ID',
`name` varchar(255) DEFAULT NULL COMMENT '任务名称',
`status` varchar(255) DEFAULT NULL COMMENT '任务状态',
PRIMARY KEY (`id`)
) COMMENT ='项目启动任务';
BEGIN;
INSERT INTO `core_sys_startup_job` VALUES ('chartFilterMerge', 'chartFilterMerge', 'ready');
COMMIT;

13
core/core-backend/src/main/resources/db/migration/V2.7__ddl.sql
View File

@ -1,2 +1,15 @@
alter table `core_chart_view`
add aggregate bit null comment '区间条形图开启时间纬度开启聚合';
DROP TABLE IF EXISTS `core_sys_startup_job`;
CREATE TABLE `core_sys_startup_job`
(
`id` varchar(64) NOT NULL COMMENT 'ID',
`name` varchar(255) DEFAULT NULL COMMENT '任务名称',
`status` varchar(255) DEFAULT NULL COMMENT '任务状态',
PRIMARY KEY (`id`)
) COMMENT ='项目启动任务';
BEGIN;
INSERT INTO `core_sys_startup_job` VALUES ('chartFilterMerge', 'chartFilterMerge', 'ready');
COMMIT;

3
sdk/api/api-base/src/main/java/io/dataease/api/chart/dto/ChartViewBaseDTO.java
View File

@ -3,6 +3,7 @@ package io.dataease.api.chart.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import io.dataease.api.chart.filter.FilterTreeObj;
import lombok.Data;
import java.io.Serializable;
@ -120,7 +121,7 @@ public class ChartViewBaseDTO implements Serializable {
/**
* 结果过滤
*/
private List<ChartFieldCustomFilterDTO> customFilter;
private FilterTreeObj customFilter;
/**
* 钻取字段

24
sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeItem.java Normal file
View File

@ -0,0 +1,24 @@
package io.dataease.api.chart.filter;
import io.dataease.dto.dataset.DatasetTableFieldDTO;
import lombok.Data;
import java.io.Serializable;
import java.util.List;
/**
* @Author Junjun
*/
@Data
public class FilterTreeItem implements Serializable {
private String type;// 'item' or 'tree'
// item
private Long fieldId;
private DatasetTableFieldDTO field;// field object
private String filterType;// 'logic' or 'enum'
private String term;//'eq','not_eq','lt','le','gt','ge','in','not in','like','not like','null','not_null','empty','not_empty','between'
private String value;// 'a'
private List<String> enumValue;// ['a','b']
// tree
private FilterTreeObj subTree;
}

15
sdk/api/api-base/src/main/java/io/dataease/api/chart/filter/FilterTreeObj.java Normal file
View File

@ -0,0 +1,15 @@
package io.dataease.api.chart.filter;
import lombok.Data;
import java.util.List;
/**
* @Author Junjun
*/
@Data
public class FilterTreeObj {
private String logic;
private List<FilterTreeItem> items;
}