center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

refactor: 单一数据源的数据集、图表使用对应数据库语法和函数

Browse Source
This commit is contained in:
junjun 2024-04-08 10:43:23 +08:00
parent 9732a99ace
commit 29ae4c563b
20 changed files with 617 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java
View File

@ -16,6 +16,7 @@ import io.dataease.dataset.manage.DatasetGroupManage;
import io.dataease.dataset.manage.DatasetSQLManage;
import io.dataease.dataset.manage.DatasetTableFieldManage;
import io.dataease.dataset.manage.PermissionManage;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.dto.dataset.DatasetTableFieldDTO;
@ -423,6 +424,10 @@ public class ChartDataManage {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
// 调用数据源的calcite获得data
DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -494,7 +499,7 @@ public class ChartDataManage {
}
SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")");
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
CustomWhere2Str.customWhere2sqlObj(sqlMeta, fieldCustomFilter, transFields(allFields));
ExtWhere2Str.extWhere2sqlOjb(sqlMeta, extFilterList, transFields(allFields));
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, transFields(allFields));
@ -538,6 +543,7 @@ public class ChartDataManage {
}
if (StringUtils.isNotEmpty(totalPageSql) && StringUtils.equalsIgnoreCase((String) mapSize.get("tablePageMode"), "page")) {
totalPageSql = SqlUtils.rebuildSQL(totalPageSql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(totalPageSql);
datasourceRequest.setTotalPageFlag(true);
List<String[]> tmpData = (List<String[]>) calciteProvider.fetchResultField(datasourceRequest).get("data");
@ -548,6 +554,7 @@ public class ChartDataManage {
totalPage = (totalItems / pageInfo.getPageSize()) + (totalItems % pageInfo.getPageSize() > 0 ? 1 : 0);
}
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql);
logger.info("calcite chart sql: " + querySql);
@ -1266,6 +1273,10 @@ public class ChartDataManage {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
// 调用数据源的calcite获得data
DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -1287,7 +1298,7 @@ public class ChartDataManage {
}
SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")");
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, transFields(allFields));
if (StringUtils.equalsAnyIgnoreCase(view.getType(), "indicator", "gauge", "liquid")) {
@ -1316,6 +1327,7 @@ public class ChartDataManage {
querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
}
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql);
logger.info("calcite chart get field enum sql: " + querySql);

2
core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java
View File

@ -55,7 +55,7 @@ public class SqlparserUtils {
}
// 递归遍历语法树
getDependencies(sqlNode, false);
return sqlNode.toString();
return sqlNode.toString().replaceAll("`", "");
}
private static void getDependencies(SqlNode sqlNode, Boolean fromOrJoin) {

81
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java
View File

@ -28,6 +28,7 @@ import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.func.FunctionConstant;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Field2SQLObj;
@ -94,27 +95,32 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
String sql;
if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)) {
// add table schema
datasourceRequest.setQuery(TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0");
sql = TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0";
// replace schema alias, trans dialect
sql = Utils.replaceSchemaAlias(sql, datasourceRequest.getDsList());
sql = SqlUtils.transSqlDialect(sql, datasourceRequest.getDsList());
} else {
// parser sql params and replace default value
String sql = SqlparserUtils.handleVariableDefaultValue(new String(Base64.getDecoder().decode(tableInfoDTO.getSql())), datasetTableDTO.getSqlVariableDetails(), false, false, null);
String originSql = SqlparserUtils.handleVariableDefaultValue(new String(Base64.getDecoder().decode(tableInfoDTO.getSql())), datasetTableDTO.getSqlVariableDetails(), false, false, null);
// add sql table schema
sql = SqlUtils.addSchema(sql, datasourceSchemaDTO.getSchemaAlias());
sql = SQLUtils.buildOriginPreviewSql(sql, 0, 0);
datasourceRequest.setQuery(sql);
sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);
sql = SqlUtils.transSqlDialect(sql, datasourceRequest.getDsList());
// replace placeholder
sql = SqlUtils.replaceTablePlaceHolder(sql, originSql);
}
datasourceRequest.setQuery(sql.replaceAll("\r\n", " ")
.replaceAll("\n", " "));
logger.info("calcite data table field sql: " + datasourceRequest.getQuery());
// 获取数据源表的原始字段
if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)
&& !StringUtils.equalsIgnoreCase("excel", coreDatasource.getType())
&& !StringUtils.equalsIgnoreCase("api", coreDatasource.getType())) {
datasourceRequest.setDatasource(coreDatasource);
if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)) {
datasourceRequest.setTable(tableInfoDTO.getTable());
tableFields = calciteProvider.fetchTableField(datasourceRequest);
} else {
tableFields = (List<TableField>) calciteProvider.fetchResultField(datasourceRequest).get("fields");
tableFields = calciteProvider.fetchTableField(datasourceRequest);
}
} else {
// excel,api
@ -125,9 +131,13 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
datasourceRequest.setQuery(TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0");
String sql = TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0";
// replace schema alias, trans dialect
sql = Utils.replaceSchemaAlias(sql, datasourceRequest.getDsList());
sql = SqlUtils.transSqlDialect(sql, datasourceRequest.getDsList());
datasourceRequest.setQuery(sql);
logger.info("calcite data table field sql: " + datasourceRequest.getQuery());
tableFields = (List<TableField>) calciteProvider.fetchResultField(datasourceRequest).get("fields");
tableFields = calciteProvider.fetchTableField(datasourceRequest);
}
return transFields(tableFields, true);
}
@ -175,6 +185,10 @@ public class DatasetDataManage {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>();
TokenUserBO user = AuthUtils.getUser();
@ -184,7 +198,7 @@ public class DatasetDataManage {
// build query sql
SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")");
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
Field2SQLObj.field2sqlObj(sqlMeta, fields);
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields);
Order2SQLObj.getOrders(sqlMeta, fields, datasetGroupInfoDTO.getSortFields());
@ -194,6 +208,7 @@ public class DatasetDataManage {
} else {
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, start, count);
}
querySQL = SqlUtils.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
logger.info("calcite data preview sql: " + querySQL);
// 通过数据源请求数据
@ -230,9 +245,9 @@ public class DatasetDataManage {
String sql = (String) sqlMap.get("sql");
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
List<String> dsList = new ArrayList<>();
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
dsList.add(next.getValue().getType());
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
String querySQL = "SELECT COUNT(*) FROM (" + sql + ") t_a_0";
@ -277,7 +292,7 @@ public class DatasetDataManage {
return map;
}
public Map<String, Object> previewSql(PreviewSqlDTO dto) {
public Map<String, Object> previewSql(PreviewSqlDTO dto) throws DEException {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(dto.getDatasourceId());
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
if (coreDatasource.getType().equalsIgnoreCase("API") || coreDatasource.getType().equalsIgnoreCase("Excel")) {
@ -288,27 +303,35 @@ public class DatasetDataManage {
String alias = String.format(SQLConstants.SCHEMA, datasourceSchemaDTO.getId());
datasourceSchemaDTO.setSchemaAlias(alias);
// parser sql params and replace default value
String sql = SqlparserUtils.handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(new String(Base64.getDecoder().decode(dto.getSql()))), dto.getSqlVariableDetails(), true, true, null);
sql = SqlUtils.addSchema(sql, alias);
String originSql = SqlparserUtils.handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(new String(Base64.getDecoder().decode(dto.getSql()))), dto.getSqlVariableDetails(), true, true, null);
Map<Long, DatasourceSchemaDTO> dsMap = new LinkedHashMap<>();
dsMap.put(datasourceSchemaDTO.getId(), datasourceSchemaDTO);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(dsMap);
// sql 作为临时表，外层加上limit
String sql;
if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) {
// 先根据sql获取表字段
String sqlField = SQLUtils.buildOriginPreviewSql(sql, 0, 0);
String sqlField = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);
sqlField = SqlUtils.transSqlDialect(sqlField, datasourceRequest.getDsList());
// replace placeholder
sqlField = SqlUtils.replaceTablePlaceHolder(sqlField, originSql);
datasourceRequest.setQuery(sqlField);
// 获取数据源表的原始字段
List<TableField> list = (List<TableField>) calciteProvider.fetchResultField(datasourceRequest).get("fields");
List<TableField> list = calciteProvider.fetchTableField(datasourceRequest);
if (ObjectUtils.isEmpty(list)) {
return null;
}
sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(sql, 100, 0, String.format(SQLConstants.FIELD_DOT, list.get(0).getOriginName()) + " ASC ");
sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0, String.format(SQLConstants.FIELD_DOT, list.get(0).getOriginName()) + " ASC ");
} else {
sql = SQLUtils.buildOriginPreviewSql(sql, 100, 0);
sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0);
}
sql = SqlUtils.transSqlDialect(sql, datasourceRequest.getDsList());
// replace placeholder
sql = SqlUtils.replaceTablePlaceHolder(sql, originSql);
logger.info("calcite data preview sql: " + sql);
datasourceRequest.setQuery(sql);
Map<String, Object> data = calciteProvider.fetchResultField(datasourceRequest);
@ -331,7 +354,7 @@ public class DatasetDataManage {
String[] row = dataList.get(i);
LinkedHashMap<String, Object> obj = new LinkedHashMap<>();
if (row.length > 0) {
for (int j = 0; j < row.length; j++) {
for (int j = 0; j < fields.size(); j++) {
if (desensitizationList.keySet().contains(fields.get(j).getDataeaseName())) {
obj.put(fields.get(j).getDataeaseName(), ChartDataBuild.desensitizationValue(desensitizationList.get(fields.get(j).getDataeaseName()), String.valueOf(row[j])));
} else {
@ -402,9 +425,15 @@ public class DatasetDataManage {
allFields.addAll(datasetGroupInfoDTO.getAllFields());
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
// build query sql
SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")");
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
// 计算字段先完成内容替换
if (Objects.equals(field.getExtField(), ExtFieldConstant.EXT_CALC)) {
String originField = Utils.calcFieldRegex(field.getOriginName(), sqlMeta.getTable(), allFields);
@ -427,7 +456,6 @@ public class DatasetDataManage {
}
buildFieldName(sqlMap, fields);
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
List<String> dsList = new ArrayList<>();
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
dsList.add(next.getValue().getType());
@ -444,6 +472,7 @@ public class DatasetDataManage {
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields);
Order2SQLObj.getOrders(sqlMeta, fields, datasetGroupInfoDTO.getSortFields());
String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, true, 0, 1000);
querySQL = SqlUtils.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
// 通过数据源请求数据
// 调用数据源的calcite获得data

124
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java
View File

@ -6,6 +6,7 @@ import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.dataset.dto.SqlVariableDetails;
import io.dataease.api.dataset.union.*;
import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.api.permissions.auth.dto.BusiPerCheckDTO;
import io.dataease.commons.utils.SqlparserUtils;
import io.dataease.constant.AuthEnum;
@ -93,9 +94,10 @@ public class DatasetSQLManage {
String tableSchema = putObj2Map(dsMap, currentDs);
// get table
DatasetTableInfoDTO infoDTO = JsonUtil.parseObject(currentDs.getInfo(), DatasetTableInfoDTO.class);
Set<Long> allDs = getAllDs(union);
boolean isCross = allDs.size() > 1;
SQLObj tableName = getUnionTable(currentDs, infoDTO, tableSchema, 0, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null);
SQLObj tableName = getUnionTable(currentDs, infoDTO, tableSchema, 0, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null, isCross);
for (int i = 0; i < union.size(); i++) {
UnionDTO unionDTO = union.get(i);
@ -108,7 +110,7 @@ public class DatasetSQLManage {
} else {
schema = putObj2Map(dsMap, datasetTable);
}
SQLObj table = getUnionTable(datasetTable, tableInfo, schema, i, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null);
SQLObj table = getUnionTable(datasetTable, tableInfo, schema, i, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null, isCross);
// 获取前端传过来选中的字段
List<DatasetTableFieldDTO> fields = unionDTO.getCurrentDsFields();
@ -127,17 +129,25 @@ public class DatasetSQLManage {
f.setDataeaseName(f.getFieldShortName());
f.setDatasetTableId(datasetTable.getId());
String prefix = "";
String suffix = "";
if (Objects.equals(f.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
prefix = "`";
if (isCross) {
prefix = "`";
suffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, datasetTable.getDatasourceId());
prefix = datasourceType.getPrefix();
suffix = datasourceType.getSuffix();
}
}
return table.getTableAlias() + "." + prefix + f.getOriginName() + prefix + " AS " + alias;
return table.getTableAlias() + "." + prefix + f.getOriginName() + suffix + " AS " + prefix + alias + suffix;
})
.toArray(String[]::new);
checkedInfo.put(table.getTableAlias(), array);
checkedFields.addAll(fields);
// 获取child的fields和union
if (!CollectionUtils.isEmpty(unionDTO.getChildrenDs())) {
getUnionForEdit(datasetTable, table, unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields, dsMap, chartExtRequest);
getUnionForEdit(datasetTable, table, unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields, dsMap, chartExtRequest, isCross);
}
}
// build sql
@ -158,17 +168,28 @@ public class DatasetSQLManage {
SQLObj parentSQLObj = unionParamDTO.getParentSQLObj();
SQLObj currentSQLObj = unionParamDTO.getCurrentSQLObj();
DatasetTableDTO parentDs = unionParamDTO.getParentDs();
DatasetTableDTO currentDs1 = unionParamDTO.getCurrentDs();
String ts = "";
String tablePrefix = "";
String tableSuffix = "";
if (ObjectUtils.isNotEmpty(currentSQLObj.getTableSchema())) {
ts = currentSQLObj.getTableSchema() + ".";
tablePrefix = "`";
if (isCross) {
tablePrefix = "`";
tableSuffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, currentDs1.getDatasourceId());
tablePrefix = datasourceType.getPrefix();
tableSuffix = datasourceType.getSuffix();
}
}
// build join
join.append(" ").append(joinType).append(" ")
.append(ts)
.append(tablePrefix + currentSQLObj.getTableName() + tablePrefix)
.append(tablePrefix + currentSQLObj.getTableName() + tableSuffix)
.append(" ").append(currentSQLObj.getTableAlias()).append(" ")
.append(" ON ");
if (unionParamDTO.getUnionFields().size() == 0) {
@ -180,18 +201,34 @@ public class DatasetSQLManage {
DatasetTableFieldDTO parentField = unionItemDTO.getParentField();
DatasetTableFieldDTO currentField = unionItemDTO.getCurrentField();
String pPrefix = "";
String pSuffix = "";
if (Objects.equals(parentField.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
pPrefix = "`";
if (isCross) {
pPrefix = "`";
pSuffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, parentDs.getDatasourceId());
pPrefix = datasourceType.getPrefix();
pSuffix = datasourceType.getSuffix();
}
}
String cPrefix = "";
String cSuffix = "";
if (Objects.equals(currentField.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
cPrefix = "`";
if (isCross) {
cPrefix = "`";
cSuffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, currentDs1.getDatasourceId());
cPrefix = datasourceType.getPrefix();
cSuffix = datasourceType.getSuffix();
}
}
join.append(parentSQLObj.getTableAlias()).append(".")
.append(pPrefix + parentField.getOriginName() + pPrefix)
.append(pPrefix + parentField.getOriginName() + pSuffix)
.append(" = ")
.append(currentSQLObj.getTableAlias()).append(".")
.append(cPrefix + currentField.getOriginName() + cPrefix);
.append(cPrefix + currentField.getOriginName() + cSuffix);
if (i < unionParamDTO.getUnionFields().size() - 1) {
join.append(" AND ");
}
@ -200,13 +237,13 @@ public class DatasetSQLManage {
if (StringUtils.isEmpty(f)) {
DEException.throwException(Translator.get("i18n_union_ds_no_checked"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.getTableAndAlias(tableName)) + join.toString();
sql = MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.getTableAndAlias(tableName, getDatasourceType(dsMap, currentDs.getDatasourceId()), isCross)) + join.toString();
} else {
String f = StringUtils.join(checkedInfo.get(tableName.getTableAlias()), ",");
if (StringUtils.isEmpty(f)) {
DEException.throwException(Translator.get("i18n_union_ds_no_checked"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.getTableAndAlias(tableName));
sql = MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.getTableAndAlias(tableName, getDatasourceType(dsMap, currentDs.getDatasourceId()), isCross));
}
logger.info("calcite origin sql: " + sql);
Map<String, Object> map = new HashMap<>();
@ -221,7 +258,8 @@ public class DatasetSQLManage {
private void getUnionForEdit(DatasetTableDTO parentTable, SQLObj parentSQLObj,
List<UnionDTO> childrenDs, Map<String, String[]> checkedInfo,
List<UnionParamDTO> unionList, List<DatasetTableFieldDTO> checkedFields,
Map<Long, DatasourceSchemaDTO> dsMap, ChartExtRequest chartExtRequest) throws Exception {
Map<Long, DatasourceSchemaDTO> dsMap, ChartExtRequest chartExtRequest,
boolean isCross) throws Exception {
for (int i = 0; i < childrenDs.size(); i++) {
int index = unionList.size() + 1;
@ -235,7 +273,7 @@ public class DatasetSQLManage {
} else {
schema = putObj2Map(dsMap, datasetTable);
}
SQLObj table = getUnionTable(datasetTable, tableInfo, schema, index, filterParameters(chartExtRequest, datasetTable.getId()), chartExtRequest == null);
SQLObj table = getUnionTable(datasetTable, tableInfo, schema, index, filterParameters(chartExtRequest, datasetTable.getId()), chartExtRequest == null, isCross);
List<DatasetTableFieldDTO> fields = unionDTO.getCurrentDsFields();
fields = fields.stream().filter(DatasetTableFieldDTO::getChecked).collect(Collectors.toList());
@ -253,10 +291,18 @@ public class DatasetSQLManage {
f.setDataeaseName(f.getFieldShortName());
f.setDatasetTableId(datasetTable.getId());
String prefix = "";
String suffix = "";
if (Objects.equals(f.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
prefix = "`";
if (isCross) {
prefix = "`";
suffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, datasetTable.getDatasourceId());
prefix = datasourceType.getPrefix();
suffix = datasourceType.getSuffix();
}
}
return table.getTableAlias() + "." + prefix + f.getOriginName() + prefix + " AS " + alias;
return table.getTableAlias() + "." + prefix + f.getOriginName() + suffix + " AS " + prefix + alias + suffix;
})
.toArray(String[]::new);
checkedInfo.put(table.getTableAlias(), array);
@ -270,11 +316,45 @@ public class DatasetSQLManage {
unionToParent.setCurrentSQLObj(table);
unionList.add(unionToParent);
if (!CollectionUtils.isEmpty(unionDTO.getChildrenDs())) {
getUnionForEdit(datasetTable, table, unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields, dsMap, chartExtRequest);
getUnionForEdit(datasetTable, table, unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields, dsMap, chartExtRequest, isCross);
}
}
}
private Set<Long> getAllDs(List<UnionDTO> union) {
Set<Long> set = new HashSet<>();
for (UnionDTO unionDTO : union) {
Long datasourceId = unionDTO.getCurrentDs().getDatasourceId();
set.add(datasourceId);
getChildrenDs(unionDTO.getChildrenDs(), set);
}
return set;
}
private void getChildrenDs(List<UnionDTO> childrenDs, Set<Long> set) {
for (UnionDTO unionDTO : childrenDs) {
set.add(unionDTO.getCurrentDs().getDatasourceId());
if (!CollectionUtils.isEmpty(unionDTO.getChildrenDs())) {
getChildrenDs(unionDTO.getChildrenDs(), set);
}
}
}
private DatasourceConfiguration.DatasourceType getDatasourceType(Map<Long, DatasourceSchemaDTO> dsMap, Long datasourceId) {
DatasourceSchemaDTO datasourceSchemaDTO = dsMap.get(datasourceId);
String type;
if (datasourceSchemaDTO == null) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasourceId);
if (coreDatasource == null) {
DEException.throwException(Translator.get("i18n_dataset_ds_error") + ",ID:" + datasourceId);
}
type = engineManage.getDeEngine().getType();
} else {
type = datasourceSchemaDTO.getType();
}
return DatasourceConfiguration.DatasourceType.valueOf(type);
}
public String subPrefixSuffixChar(String str) {
while (StringUtils.startsWith(str, ",")) {
str = str.substring(1, str.length());
@ -304,7 +384,7 @@ public class DatasetSQLManage {
}
}
private SQLObj getUnionTable(DatasetTableDTO currentDs, DatasetTableInfoDTO infoDTO, String tableSchema, int index, List<SqlVariableDetails> parameters, boolean isFromDataSet) {
private SQLObj getUnionTable(DatasetTableDTO currentDs, DatasetTableInfoDTO infoDTO, String tableSchema, int index, List<SqlVariableDetails> parameters, boolean isFromDataSet, boolean isCross) {
SQLObj tableObj;
String tableAlias = String.format(SQLConstants.TABLE_ALIAS_PREFIX, index);
if (StringUtils.equalsIgnoreCase(currentDs.getType(), DatasetTableTypeConstants.DATASET_TABLE_DB)) {
@ -313,7 +393,9 @@ public class DatasetSQLManage {
// parser sql params and replace default value
String sql = SqlparserUtils.handleVariableDefaultValue(new String(Base64.getDecoder().decode(infoDTO.getSql())), currentDs.getSqlVariableDetails(), false, isFromDataSet, parameters);
// add table schema
sql = SqlUtils.addSchema(sql, tableSchema);
if (isCross) {
sql = SqlUtils.addSchema(sql, tableSchema);
}
tableObj = SQLObj.builder().tableSchema("").tableName("(" + sql + ")").tableAlias(tableAlias).build();
} else {
// excel,api

32
core/core-backend/src/main/java/io/dataease/dataset/utils/FieldUtils.java
View File

@ -5,6 +5,7 @@ package io.dataease.dataset.utils;
*/
public class FieldUtils {
public static int transType2DeType(String type) {
type = type.replaceAll("\\((.*?)\\)","");
switch (type) {
case "CHAR":
case "VARCHAR":
@ -14,12 +15,19 @@ public class FieldUtils {
case "LONGTEXT":
case "ENUM":
case "ANY":
case "STRING":
return 0;// 文本
case "DATE":
case "TIME":
case "YEAR":
case "DATETIME":
case "TIMESTAMP":
case "DATEV2":
case "DATETIMEV2":
case "DATETIME2":
case "DATETIMEOFFSET":
case "SMALLDATETIME":
case "DATETIME64":
return 1;// 时间
case "INT":
case "SMALLINT":
@ -27,14 +35,38 @@ public class FieldUtils {
case "INTEGER":
case "BIGINT":
case "LONG": //增加了LONG类型
case "INT2":
case "INT4":
case "INT8":
case "int2":
case "int4":
case "int8":
case "INT16":
case "INT32":
case "INT64":
case "UINT8":
case "UINT16":
case "UINT32":
case "UINT64":
return 2;// 整型
case "FLOAT":
case "DOUBLE":
case "DECIMAL":
case "REAL":
case "MONEY":
case "NUMERIC":
case "float4":
case "float8":
case "FLOAT4":
case "FLOAT8":
case "DECFLOAT":
case "FLOAT32":
case "FLOAT64":
return 3;// 浮点
case "BIT":
case "TINYINT":
case "BOOL":
case "BOOLEAN":
return 4;// 布尔
default:
return 0;

111
core/core-backend/src/main/java/io/dataease/dataset/utils/SqlUtils.java
View File

@ -1,13 +1,21 @@
package io.dataease.dataset.utils;
import com.google.common.collect.ImmutableList;
import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.exception.DEException;
import org.apache.calcite.config.Lex;
import org.apache.calcite.sql.*;
import org.apache.calcite.sql.dialect.*;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.commons.collections4.MapUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.calcite.sql.SqlKind.*;
@ -38,7 +46,7 @@ public class SqlUtils {
String sqlRender = sqlNode.toString();
// 处理sql中多余的`都替换成1个
sqlRender = sqlRender.replaceAll("(`+)", "`");
return sqlRender;
return sqlRender.replaceAll("`", "");
}
private static void addTableSchema(SqlNode sqlNode, Boolean fromOrJoin, String schema, SqlParser.Config config) {
@ -107,4 +115,105 @@ public class SqlUtils {
DEException.throwException("使用 Calcite 进行语法分析发生了异常:" + e);
}
}
public static String rebuildSQL(String sql, SQLMeta sqlMeta, boolean crossDs, Map<Long, DatasourceSchemaDTO> dsMap) {
if (crossDs) {
return sql;
}
String s = transSqlDialect(sql, dsMap);
String tableDialect = sqlMeta.getTableDialect();
s = replaceTablePlaceHolder(s, tableDialect);
return replaceCalcFieldPlaceHolder(s, sqlMeta);
}
public static String transSqlDialect(String sql, Map<Long, DatasourceSchemaDTO> dsMap) throws DEException {
try {
DatasourceSchemaDTO value = dsMap.entrySet().iterator().next().getValue();
SqlParser parser = SqlParser.create(sql, SqlParser.Config.DEFAULT.withLex(Lex.JAVA));
SqlNode sqlNode = parser.parseStmt();
return sqlNode.toSqlString(getDialect(value)).toString();
} catch (Exception e) {
DEException.throwException(e.getMessage());
}
return null;
}
public static String replaceTablePlaceHolder(String s, String placeholder) {
s = s.replaceAll("\r\n", " ")
.replaceAll("\n", " ")
.replaceAll(SqlPlaceholderConstants.TABLE_PLACEHOLDER_REGEX, placeholder)
.replaceAll("ASYMMETRIC", "")
.replaceAll("SYMMETRIC", "");
return s;
}
public static String replaceCalcFieldPlaceHolder(String s, SQLMeta sqlMeta) {
Map<String, String> fieldsDialect = new HashMap<>();
if (MapUtils.isNotEmpty(sqlMeta.getXFieldsDialect())) {
fieldsDialect.putAll(sqlMeta.getXFieldsDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getYFieldsDialect())) {
fieldsDialect.putAll(sqlMeta.getYFieldsDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getCustomWheresDialect())) {
fieldsDialect.putAll(sqlMeta.getCustomWheresDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getExtWheresDialect())) {
fieldsDialect.putAll(sqlMeta.getExtWheresDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getWhereTreesDialect())) {
fieldsDialect.putAll(sqlMeta.getWhereTreesDialect());
}
if (MapUtils.isNotEmpty(fieldsDialect)) {
for (Map.Entry<String, String> ele : fieldsDialect.entrySet()) {
s = s.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + ele.getKey() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, ele.getValue());
}
}
return s;
}
private static SqlDialect getDialect(DatasourceSchemaDTO coreDatasource) {
SqlDialect sqlDialect = null;
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(coreDatasource.getType());
switch (datasourceType) {
case mysql:
case mongo:
case StarRocks:
case doris:
case TiDB:
case mariadb:
sqlDialect = MysqlSqlDialect.DEFAULT;
break;
case impala:
sqlDialect = ImpalaSqlDialect.DEFAULT;
break;
case sqlServer:
sqlDialect = MssqlSqlDialect.DEFAULT;
break;
case oracle:
sqlDialect = OracleSqlDialect.DEFAULT;
break;
case db2:
sqlDialect = Db2SqlDialect.DEFAULT;
break;
case pg:
sqlDialect = PostgresqlSqlDialect.DEFAULT;
break;
case redshift:
sqlDialect = RedshiftSqlDialect.DEFAULT;
break;
case ck:
sqlDialect = ClickHouseSqlDialect.DEFAULT;
break;
case h2:
sqlDialect = H2SqlDialect.DEFAULT;
break;
default:
sqlDialect = MysqlSqlDialect.DEFAULT;
}
return sqlDialect;
}
}

14
core/core-backend/src/main/java/io/dataease/dataset/utils/TableUtils.java
View File

@ -1,6 +1,7 @@
package io.dataease.dataset.utils;
import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.utils.Md5Utils;
import org.apache.calcite.avatica.util.Quoting;
@ -38,14 +39,21 @@ public class TableUtils {
return "C_" + Md5Utils.md5(fieldName);
}
public static String getTableAndAlias(SQLObj sqlObj) {
public static String getTableAndAlias(SQLObj sqlObj, DatasourceConfiguration.DatasourceType datasourceType, boolean isCross) {
String schema = "";
String prefix = "";
String suffix = "";
if (StringUtils.isNotEmpty(sqlObj.getTableSchema())) {
schema = sqlObj.getTableSchema() + ".";
prefix = "`";
if (isCross) {
prefix = "`";
suffix = "`";
} else {
prefix = datasourceType.getPrefix();
suffix = datasourceType.getSuffix();
}
}
return schema + prefix + sqlObj.getTableName() + prefix + " " + sqlObj.getTableAlias();
return schema + prefix + sqlObj.getTableName() + suffix + " " + sqlObj.getTableAlias();
}
public static String tableName2Sql(DatasourceSchemaDTO ds, String tableName) {

181
core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java
View File

@ -35,6 +35,7 @@ import org.springframework.util.CollectionUtils;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URL;
import java.sql.*;
import java.util.*;
@ -147,6 +148,11 @@ public class CalciteProvider {
}
public Map<String, Object> fetchResultField(DatasourceRequest datasourceRequest) throws DEException {
// 不跨数据源
if (datasourceRequest.getDsList().size() == 1) {
return jdbcFetchResultField(datasourceRequest);
}
List<TableField> datasetTableFields = new ArrayList<>();
List<String[]> list = new LinkedList<>();
PreparedStatement statement = null;
@ -191,6 +197,119 @@ public class CalciteProvider {
return map;
}
public Map<String, Object> jdbcFetchResultField(DatasourceRequest datasourceRequest) throws DEException {
DatasourceSchemaDTO value = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
datasourceRequest.setDatasource(value);
DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
Map<String, Object> map = new LinkedHashMap<>();
List<TableField> fieldList = new ArrayList<>();
List<String[]> dataList = new LinkedList<>();
// schema
ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, datasourceConfiguration.getQueryTimeout())) {
if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(datasourceRequest.getQuery());
fieldList = getField(resultSet, datasourceRequest);
dataList = getData(resultSet, datasourceRequest);
} catch (SQLException e) {
DEException.throwException("SQL ERROR: " + e.getMessage());
} catch (Exception e) {
DEException.throwException("Data source connection exception: " + e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
map.put("fields", fieldList);
map.put("data", dataList);
return map;
}
private List<TableField> getField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
List<TableField> fieldList = new ArrayList<>();
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
for (int j = 0; j < columnCount; j++) {
String f = metaData.getColumnName(j + 1);
if (StringUtils.equalsIgnoreCase(f, "DE_ROWNUM")) {
continue;
}
String l = StringUtils.isNotEmpty(metaData.getColumnLabel(j + 1)) ? metaData.getColumnLabel(j + 1) : f;
String t = metaData.getColumnTypeName(j + 1).toUpperCase();
TableField field = new TableField();
field.setOriginName(l);
field.setName(l);
field.setFieldType(t);
field.setType(t);
fieldList.add(field);
}
return fieldList;
}
private List<String[]> getData(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
String charset = null;
String targetCharset = "UTF-8";
if (datasourceRequest != null && datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) {
DatasourceConfiguration jdbcConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset()) && !jdbcConfiguration.getCharset().equalsIgnoreCase("Default")) {
charset = jdbcConfiguration.getCharset();
}
if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset()) && !jdbcConfiguration.getTargetCharset().equalsIgnoreCase("Default")) {
targetCharset = jdbcConfiguration.getTargetCharset();
}
}
List<String[]> list = new LinkedList<>();
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
while (rs.next()) {
String[] row = new String[columnCount];
for (int j = 0; j < columnCount; j++) {
int columnType = metaData.getColumnType(j + 1);
switch (columnType) {
case Types.DATE:
if (rs.getDate(j + 1) != null) {
row[j] = rs.getDate(j + 1).toString();
}
break;
case Types.BOOLEAN:
row[j] = rs.getBoolean(j + 1) ? "1" : "0";
break;
case Types.NUMERIC:
BigDecimal bigDecimal = rs.getBigDecimal(j + 1);
row[j] = bigDecimal == null ? null : bigDecimal.toString();
break;
default:
if (metaData.getColumnTypeName(j + 1).toLowerCase().equalsIgnoreCase("blob")) {
row[j] = rs.getBlob(j + 1) == null ? "" : rs.getBlob(j + 1).toString();
} else {
if (charset != null && StringUtils.isNotEmpty(rs.getString(j + 1))) {
String originStr = new String(rs.getString(j + 1).getBytes(charset), targetCharset);
row[j] = new String(originStr.getBytes("UTF-8"), "UTF-8");
} else {
row[j] = rs.getString(j + 1);
}
}
break;
}
}
list.add(row);
}
return list;
}
private String getTableFiledSql(DatasourceRequest datasourceRequest) {
String sql = "";
DatasourceConfiguration configuration = null;
@ -295,7 +414,8 @@ public class CalciteProvider {
private TableField getTableFieldDesc(DatasourceRequest datasourceRequest, ResultSet resultSet) throws SQLException {
TableField tableField = new TableField();
tableField.setOriginName(resultSet.getString(1));
tableField.setType(resultSet.getString(2));
tableField.setType(resultSet.getString(2).toUpperCase());
tableField.setFieldType(tableField.getType());
int deType = FieldUtils.transType2DeType(tableField.getType());
tableField.setDeExtractType(deType);
tableField.setDeType(deType);
@ -306,24 +426,57 @@ public class CalciteProvider {
public List<TableField> fetchTableField(DatasourceRequest datasourceRequest) throws DEException {
List<TableField> datasetTableFields = new ArrayList<>();
try (Connection con = getConnection(datasourceRequest.getDatasource()); Statement statement = getStatement(con, 30); ResultSet resultSet = statement.executeQuery(getTableFiledSql(datasourceRequest))) {
while (resultSet.next()) {
datasetTableFields.add(getTableFieldDesc(datasourceRequest, resultSet));
}
} catch (Exception e) {
DEException.throwException(e.getMessage());
}
DatasourceSchemaDTO datasourceSchemaDTO = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
datasourceRequest.setDatasource(datasourceSchemaDTO);
List<TableField> tableFields = (List<TableField>) fetchResultField(datasourceRequest).get("fields");
for (TableField tableField : tableFields) {
for (TableField datasetTableField : datasetTableFields) {
if(tableField.getOriginName().equalsIgnoreCase(datasetTableField.getOriginName())){
tableField.setName(datasetTableField.getName());
DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
String table = datasourceRequest.getTable();
if (StringUtils.isEmpty(table)) {
ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, 30)) {
if (DatasourceConfiguration.DatasourceType.valueOf(datasourceSchemaDTO.getType()) == DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(datasourceRequest.getQuery());
datasetTableFields.addAll(getField(resultSet, datasourceRequest));
} catch (Exception e) {
DEException.throwException(e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
} else {
ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, 30)) {
if (DatasourceConfiguration.DatasourceType.valueOf(datasourceSchemaDTO.getType()) == DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(getTableFiledSql(datasourceRequest));
while (resultSet.next()) {
datasetTableFields.add(getTableFieldDesc(datasourceRequest, resultSet));
}
} catch (Exception e) {
DEException.throwException(e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
return tableFields;
return datasetTableFields;
}

2
core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java
View File

@ -878,7 +878,7 @@ public class DatasourceServer implements DatasourceApi {
}
@Override
public Map<String, Object> previewDataWithLimit(Map<String, Object> req) {
public Map<String, Object> previewDataWithLimit(Map<String, Object> req) throws DEException {
String tableName = req.get("table").toString();
Long id = Long.valueOf(req.get("id").toString());
if (ObjectUtils.isEmpty(tableName) || ObjectUtils.isEmpty(id)) {

9
core/core-backend/src/main/java/io/dataease/engine/trans/CustomWhere2Str.java
View File

@ -6,11 +6,13 @@ import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -25,6 +27,7 @@ public class CustomWhere2Str {
return;
}
List<String> res = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (ObjectUtils.isNotEmpty(fields)) {
for (ChartFieldCustomFilterDTO request : fields) {
List<SQLObj> list = new ArrayList<>();
@ -37,7 +40,10 @@ public class CustomWhere2Str {
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originName = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields);
String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
} else {
@ -137,6 +143,7 @@ public class CustomWhere2Str {
}
meta.setCustomWheres(ObjectUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null);
}
meta.setCustomWheresDialect(fieldsDialect);
}

12
core/core-backend/src/main/java/io/dataease/engine/trans/Dimension2SQLObj.java
View File

@ -7,14 +7,13 @@ import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.*;
/**
* @Author Junjun
@ -28,13 +27,17 @@ public class Dimension2SQLObj {
}
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (!CollectionUtils.isEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
ChartViewFieldDTO x = fields.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式
originField = Utils.calcFieldRegex(x.getOriginName(), tableObj, originFields);
String calcFieldExp = Utils.calcFieldRegex(x.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, x.getId());
fieldsDialect.put(originField, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_COPY)) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
} else {
@ -56,6 +59,7 @@ public class Dimension2SQLObj {
}
meta.setXFields(xFields);
meta.setXOrders(xOrders);
meta.setXFieldsDialect(fieldsDialect);
}
private static SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) {

10
core/core-backend/src/main/java/io/dataease/engine/trans/ExtWhere2Str.java
View File

@ -5,12 +5,15 @@ import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @Author Junjun
@ -23,6 +26,7 @@ public class ExtWhere2Str {
return;
}
List<SQLObj> list = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (ObjectUtils.isNotEmpty(fields)) {
for (ChartExtFilterDTO request : fields) {
List<String> value = request.getValue();
@ -44,7 +48,10 @@ public class ExtWhere2Str {
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originName = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields);
String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
} else {
@ -133,6 +140,7 @@ public class ExtWhere2Str {
list.forEach(ele -> strList.add("(" + ele.getWhereField() + " " + ele.getWhereTermAndValue() + ")"));
meta.setExtWheres(ObjectUtils.isNotEmpty(list) ? "(" + String.join(" AND ", strList) + ")" : null);
}
meta.setExtWheresDialect(fieldsDialect);
}
}

12
core/core-backend/src/main/java/io/dataease/engine/trans/Field2SQLObj.java
View File

@ -6,14 +6,13 @@ import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.func.FunctionConstant;
import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.*;
/**
* @Author Junjun
@ -26,13 +25,17 @@ public class Field2SQLObj {
return;
}
List<SQLObj> xFields = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (ObjectUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
DatasetTableFieldDTO x = fields.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式
originField = Utils.calcFieldRegex(x.getOriginName(), tableObj, fields);
String calcFieldExp = Utils.calcFieldRegex(x.getOriginName(), tableObj, fields);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, x.getId());
fieldsDialect.put(originField, calcFieldExp);
// 此处是数据集预览，获取数据库原始字段枚举值等操作使用，如果遇到聚合函数则将originField设置为null
for (String func : FunctionConstant.AGG_FUNC) {
if (Utils.matchFunction(func, originField)) {
@ -51,6 +54,7 @@ public class Field2SQLObj {
}
}
meta.setXFields(xFields);
meta.setXFieldsDialect(fieldsDialect);
}
public static SQLObj getXFields(DatasetTableFieldDTO f, String originField, String fieldAlias) {

12
core/core-backend/src/main/java/io/dataease/engine/trans/Quota2SQLObj.java
View File

@ -7,14 +7,13 @@ import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.*;
/**
* @Author Junjun
@ -29,13 +28,17 @@ public class Quota2SQLObj {
List<SQLObj> yFields = new ArrayList<>();
List<String> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (!CollectionUtils.isEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
ChartViewFieldDTO y = fields.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && Objects.equals(y.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式
originField = Utils.calcFieldRegex(y.getOriginName(), tableObj, originFields);
String calcFieldExp = Utils.calcFieldRegex(y.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, y.getId());
fieldsDialect.put(originField, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && Objects.equals(y.getExtField(), ExtFieldConstant.EXT_COPY)) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName());
} else {
@ -62,6 +65,7 @@ public class Quota2SQLObj {
meta.setYFields(yFields);
meta.setYWheres(yWheres);
meta.setYOrders(yOrders);
meta.setYFieldsDialect(fieldsDialect);
}
private static SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) {

12
core/core-backend/src/main/java/io/dataease/engine/trans/Table2SQLObj.java
View File

@ -3,14 +3,22 @@ package io.dataease.engine.trans;
import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
/**
* @Author Junjun
*/
public class Table2SQLObj {
public static void table2sqlobj(SQLMeta meta, String tablePrefix, String table) {
public static void table2sqlobj(SQLMeta meta, String tablePrefix, String table, boolean crossDs) {
String sql;
if (table.startsWith("(") && table.endsWith(")") && !crossDs) {// SQL片段和关联
meta.setTableDialect(table.substring(1, table.length() - 1));
sql = "(" + SqlPlaceholderConstants.TABLE_PLACEHOLDER + ")";
} else {
sql = table;
}
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(SQLConstants.TABLE_NAME, tablePrefix, table))
.tableName((table.startsWith("(") && table.endsWith(")")) ? sql : String.format(SQLConstants.TABLE_NAME, tablePrefix, table))
.tableAlias(String.format(SQLConstants.TABLE_ALIAS_PREFIX, 0))
.build();
meta.setTable(tableObj);

23
core/core-backend/src/main/java/io/dataease/engine/trans/WhereTree2Str.java
View File

@ -8,15 +8,13 @@ import io.dataease.api.permissions.dataset.dto.DatasetRowPermissionsTreeObj;
import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
/**
* @Author Junjun
@ -31,6 +29,7 @@ public class WhereTree2Str {
return;
}
List<String> res = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
// permission trees
// 解析每个tree，然后多个tree之间用and拼接
// 每个tree，如果是sub tree节点，则使用递归合并成一组条件
@ -39,15 +38,16 @@ public class WhereTree2Str {
if (ObjectUtils.isEmpty(tree)) {
continue;
}
String treeExp = transTreeToWhere(tableObj, tree, originFields);
String treeExp = transTreeToWhere(tableObj, tree, originFields, fieldsDialect);
if (StringUtils.isNotEmpty(treeExp)) {
res.add(treeExp);
}
}
meta.setWhereTrees(CollectionUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null);
meta.setWhereTreesDialect(fieldsDialect);
}
private static String transTreeToWhere(SQLObj tableObj, DatasetRowPermissionsTreeObj tree, List<DatasetTableFieldDTO> originFields) {
private static String transTreeToWhere(SQLObj tableObj, DatasetRowPermissionsTreeObj tree, List<DatasetTableFieldDTO> originFields, Map<String, String> fieldsDialect) {
if (ObjectUtils.isEmpty(tree)) {
return null;
}
@ -60,10 +60,10 @@ public class WhereTree2Str {
String exp = null;
if (StringUtils.equalsIgnoreCase(item.getType(), "item")) {
// 单个item拼接SQL，最后根据logic汇总
exp = transTreeItem(tableObj, item, originFields);
exp = transTreeItem(tableObj, item, originFields, fieldsDialect);
} else if (StringUtils.equalsIgnoreCase(item.getType(), "tree")) {
// 递归tree
exp = transTreeToWhere(tableObj, item.getSubTree(), originFields);
exp = transTreeToWhere(tableObj, item.getSubTree(), originFields, fieldsDialect);
}
if (StringUtils.isNotEmpty(exp)) {
list.add(exp);
@ -73,7 +73,7 @@ public class WhereTree2Str {
return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + logic + " ", list) + ")" : null;
}
public static String transTreeItem(SQLObj tableObj, DatasetRowPermissionsTreeItem item, List<DatasetTableFieldDTO> originFields) {
public static String transTreeItem(SQLObj tableObj, DatasetRowPermissionsTreeItem item, List<DatasetTableFieldDTO> originFields, Map<String, String> fieldsDialect) {
String res = null;
DatasetTableFieldDTO field = item.getField();
if (ObjectUtils.isEmpty(field)) {
@ -83,7 +83,10 @@ public class WhereTree2Str {
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式
originName = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields);
String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_COPY)) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
} else {

11
core/core-backend/src/main/java/io/dataease/engine/utils/Utils.java
View File

@ -1,9 +1,11 @@
package io.dataease.engine.utils;
import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.exception.DEException;
import io.dataease.i18n.Translator;
import org.apache.commons.lang3.ObjectUtils;
@ -219,6 +221,15 @@ public class Utils {
return ObjectUtils.isNotEmpty(collect);
}
public static boolean isCrossDs(Map<Long, DatasourceSchemaDTO> dsMap) {
return dsMap.size() != 1;
}
public static String replaceSchemaAlias(String sql, Map<Long, DatasourceSchemaDTO> dsMap) {
DatasourceSchemaDTO value = dsMap.entrySet().iterator().next().getValue();
return sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + value.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX + "\\.", "");
}
public static long allDateFormat2Long(String value) {
String split = "-";
if (value != null && value.contains("/")) {

2
pom.xml
View File

@ -25,7 +25,7 @@
<mybatis-plus.version>3.5.3.1</mybatis-plus.version>
<h2.version>2.2.220</h2.version>
<knife4j.version>4.4.0</knife4j.version>
<calcite-core.version>1.35.6</calcite-core.version>
<calcite-core.version>1.35.7</calcite-core.version>
<commons-dbcp2.version>2.6.0</commons-dbcp2.version>
<antlr.version>3.5.2</antlr.version>
<java-jwt.version>3.12.1</java-jwt.version>

16
sdk/api/api-base/src/main/java/io/dataease/api/dataset/union/model/SQLMeta.java
View File

@ -3,6 +3,7 @@ package io.dataease.api.dataset.union.model;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* @Author Junjun
@ -12,14 +13,23 @@ public class SQLMeta {
private SQLObj table;
/**
* SQL片段占位符
*/
private String tableDialect;
private List<SQLObj> xFields;
private Map<String, String> xFieldsDialect;
private List<String> xWheres;
private List<SQLObj> xOrders;
private List<SQLObj> yFields;
private Map<String, String> yFieldsDialect;
private List<String> yWheres;
private List<SQLObj> yOrders;
@ -29,14 +39,20 @@ public class SQLMeta {
*/
private String customWheres;
private Map<String, String> customWheresDialect;
/**
* 仪表板过滤字段
*/
private String extWheres;
private Map<String, String> extWheresDialect;
/**
* 行权限过滤
*/
private String whereTrees;
private Map<String, String> whereTreesDialect;
}

50
sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/DatasourceConfiguration.java
View File

@ -5,39 +5,43 @@ import lombok.Data;
import java.util.List;
@Data
public class DatasourceConfiguration extends Configuration{
public class DatasourceConfiguration extends Configuration {
private List<String> illegalParameters;
private List<String> showTableSqls;
static public enum DatasourceType {
folder("folder", "folder", "folder"),
API("API", "API", "API"),
Excel("Excel", "Excel", "LOCALFILE"),
mysql("mysql", "Mysql", "OLTP"),
impala("impala", "Apache Impala", "OLAP"),
mariadb("mariadb", "Mariadb", "OLTP"),
StarRocks("StarRocks", "StarRocks", "OLAP"),
doris("doris", "Apache Doris", "OLAP"),
TiDB("TiDB", "TiDB", "OLTP"),
oracle("oracle", "ORACLE", "OLTP"),
pg("pg", "PostgreSQL", "OLTP"),
redshift("redshift", "AWS Redshift", "OLTP"),
db2("db2", "Db2", "OLTP"),
ck("ck", "Clickhouse", "OLAP"),
h2("h2", "H2", "OLAP"),
sqlServer("sqlServer", "Sqlserver", "DL"),
mongo("mongo", "MongoDB", "DL");
folder("folder", "folder", "folder", null, null),
API("API", "API", "API", "`", "`"),
Excel("Excel", "Excel", "LOCALFILE", "`", "`"),
mysql("mysql", "Mysql", "OLTP", "`", "`"),
impala("impala", "Apache Impala", "OLAP", "`", "`"),
mariadb("mariadb", "Mariadb", "OLTP", "`", "`"),
StarRocks("StarRocks", "StarRocks", "OLAP", "`", "`"),
doris("doris", "Apache Doris", "OLAP", "`", "`"),
TiDB("TiDB", "TiDB", "OLTP", "`", "`"),
oracle("oracle", "ORACLE", "OLTP", "\"", "\""),
pg("pg", "PostgreSQL", "OLTP", "\"", "\""),
redshift("redshift", "AWS Redshift", "OLTP", "\"", "\""),
db2("db2", "Db2", "OLTP", "", ""),
ck("ck", "Clickhouse", "OLAP", "`", "`"),
h2("h2", "H2", "OLAP", "\"", "\""),
sqlServer("sqlServer", "Sqlserver", "DL", "[", "]"),
mongo("mongo", "MongoDB", "DL", "`", "`");
private String type;
private String name;
private String catalog;
private String prefix;
private String suffix;
DatasourceType(String type, String name, String catalog) {
DatasourceType(String type, String name, String catalog, String prefix, String suffix) {
this.type = type;
this.name = name;
this.catalog = catalog;
this.prefix = prefix;
this.suffix = suffix;
}
public String getType() {
@ -51,5 +55,13 @@ public class DatasourceConfiguration extends Configuration{
public String getCatalog() {
return catalog;
}
public String getPrefix() {
return prefix;
}
public String getSuffix() {
return suffix;
}
}
}