refactor: 单一数据源的数据集、图表使用对应数据库语法和函数

This commit is contained in:
junjun 2024-04-08 10:43:23 +08:00
parent 9732a99ace
commit 29ae4c563b
20 changed files with 617 additions and 115 deletions

View File

@ -16,6 +16,7 @@ import io.dataease.dataset.manage.DatasetGroupManage;
import io.dataease.dataset.manage.DatasetSQLManage; import io.dataease.dataset.manage.DatasetSQLManage;
import io.dataease.dataset.manage.DatasetTableFieldManage; import io.dataease.dataset.manage.DatasetTableFieldManage;
import io.dataease.dataset.manage.PermissionManage; import io.dataease.dataset.manage.PermissionManage;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.datasource.provider.CalciteProvider; import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest; import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.dto.dataset.DatasetTableFieldDTO;
@ -423,6 +424,10 @@ public class ChartDataManage {
dsList.add(next.getValue().getType()); dsList.add(next.getValue().getType());
} }
boolean needOrder = Utils.isNeedOrder(dsList); boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
// 调用数据源的calcite获得data // 调用数据源的calcite获得data
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -494,7 +499,7 @@ public class ChartDataManage {
} }
SQLMeta sqlMeta = new SQLMeta(); SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")"); Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
CustomWhere2Str.customWhere2sqlObj(sqlMeta, fieldCustomFilter, transFields(allFields)); CustomWhere2Str.customWhere2sqlObj(sqlMeta, fieldCustomFilter, transFields(allFields));
ExtWhere2Str.extWhere2sqlOjb(sqlMeta, extFilterList, transFields(allFields)); ExtWhere2Str.extWhere2sqlOjb(sqlMeta, extFilterList, transFields(allFields));
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, transFields(allFields)); WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, transFields(allFields));
@ -538,6 +543,7 @@ public class ChartDataManage {
} }
if (StringUtils.isNotEmpty(totalPageSql) && StringUtils.equalsIgnoreCase((String) mapSize.get("tablePageMode"), "page")) { if (StringUtils.isNotEmpty(totalPageSql) && StringUtils.equalsIgnoreCase((String) mapSize.get("tablePageMode"), "page")) {
totalPageSql = SqlUtils.rebuildSQL(totalPageSql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(totalPageSql); datasourceRequest.setQuery(totalPageSql);
datasourceRequest.setTotalPageFlag(true); datasourceRequest.setTotalPageFlag(true);
List<String[]> tmpData = (List<String[]>) calciteProvider.fetchResultField(datasourceRequest).get("data"); List<String[]> tmpData = (List<String[]>) calciteProvider.fetchResultField(datasourceRequest).get("data");
@ -548,6 +554,7 @@ public class ChartDataManage {
totalPage = (totalItems / pageInfo.getPageSize()) + (totalItems % pageInfo.getPageSize() > 0 ? 1 : 0); totalPage = (totalItems / pageInfo.getPageSize()) + (totalItems % pageInfo.getPageSize() > 0 ? 1 : 0);
} }
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql); datasourceRequest.setQuery(querySql);
logger.info("calcite chart sql: " + querySql); logger.info("calcite chart sql: " + querySql);
@ -1266,6 +1273,10 @@ public class ChartDataManage {
dsList.add(next.getValue().getType()); dsList.add(next.getValue().getType());
} }
boolean needOrder = Utils.isNeedOrder(dsList); boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
// 调用数据源的calcite获得data // 调用数据源的calcite获得data
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -1287,7 +1298,7 @@ public class ChartDataManage {
} }
SQLMeta sqlMeta = new SQLMeta(); SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")"); Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, transFields(allFields)); WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, transFields(allFields));
if (StringUtils.equalsAnyIgnoreCase(view.getType(), "indicator", "gauge", "liquid")) { if (StringUtils.equalsAnyIgnoreCase(view.getType(), "indicator", "gauge", "liquid")) {
@ -1316,6 +1327,7 @@ public class ChartDataManage {
querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view); querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
} }
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql); datasourceRequest.setQuery(querySql);
logger.info("calcite chart get field enum sql: " + querySql); logger.info("calcite chart get field enum sql: " + querySql);

View File

@ -55,7 +55,7 @@ public class SqlparserUtils {
} }
// 递归遍历语法树 // 递归遍历语法树
getDependencies(sqlNode, false); getDependencies(sqlNode, false);
return sqlNode.toString(); return sqlNode.toString().replaceAll("`", "");
} }
private static void getDependencies(SqlNode sqlNode, Boolean fromOrJoin) { private static void getDependencies(SqlNode sqlNode, Boolean fromOrJoin) {

View File

@ -28,6 +28,7 @@ import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.func.FunctionConstant; import io.dataease.engine.func.FunctionConstant;
import io.dataease.engine.sql.SQLProvider; import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Field2SQLObj; import io.dataease.engine.trans.Field2SQLObj;
@ -94,27 +95,32 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO)); datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
String sql;
if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)) { if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)) {
// add table schema // add table schema
datasourceRequest.setQuery(TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0"); sql = TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0";
// replace schema alias, trans dialect
sql = Utils.replaceSchemaAlias(sql, datasourceRequest.getDsList());
sql = SqlUtils.transSqlDialect(sql, datasourceRequest.getDsList());
} else { } else {
// parser sql params and replace default value // parser sql params and replace default value
String sql = SqlparserUtils.handleVariableDefaultValue(new String(Base64.getDecoder().decode(tableInfoDTO.getSql())), datasetTableDTO.getSqlVariableDetails(), false, false, null); String originSql = SqlparserUtils.handleVariableDefaultValue(new String(Base64.getDecoder().decode(tableInfoDTO.getSql())), datasetTableDTO.getSqlVariableDetails(), false, false, null);
// add sql table schema // add sql table schema
sql = SqlUtils.addSchema(sql, datasourceSchemaDTO.getSchemaAlias());
sql = SQLUtils.buildOriginPreviewSql(sql, 0, 0); sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);
datasourceRequest.setQuery(sql); sql = SqlUtils.transSqlDialect(sql, datasourceRequest.getDsList());
// replace placeholder
sql = SqlUtils.replaceTablePlaceHolder(sql, originSql);
} }
datasourceRequest.setQuery(sql.replaceAll("\r\n", " ")
.replaceAll("\n", " "));
logger.info("calcite data table field sql: " + datasourceRequest.getQuery()); logger.info("calcite data table field sql: " + datasourceRequest.getQuery());
// 获取数据源表的原始字段 // 获取数据源表的原始字段
if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB) if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)) {
&& !StringUtils.equalsIgnoreCase("excel", coreDatasource.getType())
&& !StringUtils.equalsIgnoreCase("api", coreDatasource.getType())) {
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setTable(tableInfoDTO.getTable()); datasourceRequest.setTable(tableInfoDTO.getTable());
tableFields = calciteProvider.fetchTableField(datasourceRequest); tableFields = calciteProvider.fetchTableField(datasourceRequest);
} else { } else {
tableFields = (List<TableField>) calciteProvider.fetchResultField(datasourceRequest).get("fields"); tableFields = calciteProvider.fetchTableField(datasourceRequest);
} }
} else { } else {
// excel,api // excel,api
@ -125,9 +131,13 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO)); datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
datasourceRequest.setQuery(TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0"); String sql = TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0";
// replace schema alias, trans dialect
sql = Utils.replaceSchemaAlias(sql, datasourceRequest.getDsList());
sql = SqlUtils.transSqlDialect(sql, datasourceRequest.getDsList());
datasourceRequest.setQuery(sql);
logger.info("calcite data table field sql: " + datasourceRequest.getQuery()); logger.info("calcite data table field sql: " + datasourceRequest.getQuery());
tableFields = (List<TableField>) calciteProvider.fetchResultField(datasourceRequest).get("fields"); tableFields = calciteProvider.fetchTableField(datasourceRequest);
} }
return transFields(tableFields, true); return transFields(tableFields, true);
} }
@ -175,6 +185,10 @@ public class DatasetDataManage {
dsList.add(next.getValue().getType()); dsList.add(next.getValue().getType());
} }
boolean needOrder = Utils.isNeedOrder(dsList); boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>(); List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>();
TokenUserBO user = AuthUtils.getUser(); TokenUserBO user = AuthUtils.getUser();
@ -184,7 +198,7 @@ public class DatasetDataManage {
// build query sql // build query sql
SQLMeta sqlMeta = new SQLMeta(); SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")"); Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
Field2SQLObj.field2sqlObj(sqlMeta, fields); Field2SQLObj.field2sqlObj(sqlMeta, fields);
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields); WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields);
Order2SQLObj.getOrders(sqlMeta, fields, datasetGroupInfoDTO.getSortFields()); Order2SQLObj.getOrders(sqlMeta, fields, datasetGroupInfoDTO.getSortFields());
@ -194,6 +208,7 @@ public class DatasetDataManage {
} else { } else {
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, start, count); querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, start, count);
} }
querySQL = SqlUtils.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
logger.info("calcite data preview sql: " + querySQL); logger.info("calcite data preview sql: " + querySQL);
// 通过数据源请求数据 // 通过数据源请求数据
@ -230,9 +245,9 @@ public class DatasetDataManage {
String sql = (String) sqlMap.get("sql"); String sql = (String) sqlMap.get("sql");
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap"); Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
List<String> dsList = new ArrayList<>(); boolean crossDs = Utils.isCrossDs(dsMap);
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) { if (!crossDs) {
dsList.add(next.getValue().getType()); sql = Utils.replaceSchemaAlias(sql, dsMap);
} }
String querySQL = "SELECT COUNT(*) FROM (" + sql + ") t_a_0"; String querySQL = "SELECT COUNT(*) FROM (" + sql + ") t_a_0";
@ -277,7 +292,7 @@ public class DatasetDataManage {
return map; return map;
} }
public Map<String, Object> previewSql(PreviewSqlDTO dto) { public Map<String, Object> previewSql(PreviewSqlDTO dto) throws DEException {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(dto.getDatasourceId()); CoreDatasource coreDatasource = coreDatasourceMapper.selectById(dto.getDatasourceId());
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
if (coreDatasource.getType().equalsIgnoreCase("API") || coreDatasource.getType().equalsIgnoreCase("Excel")) { if (coreDatasource.getType().equalsIgnoreCase("API") || coreDatasource.getType().equalsIgnoreCase("Excel")) {
@ -288,27 +303,35 @@ public class DatasetDataManage {
String alias = String.format(SQLConstants.SCHEMA, datasourceSchemaDTO.getId()); String alias = String.format(SQLConstants.SCHEMA, datasourceSchemaDTO.getId());
datasourceSchemaDTO.setSchemaAlias(alias); datasourceSchemaDTO.setSchemaAlias(alias);
// parser sql params and replace default value // parser sql params and replace default value
String sql = SqlparserUtils.handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(new String(Base64.getDecoder().decode(dto.getSql()))), dto.getSqlVariableDetails(), true, true, null); String originSql = SqlparserUtils.handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(new String(Base64.getDecoder().decode(dto.getSql()))), dto.getSqlVariableDetails(), true, true, null);
sql = SqlUtils.addSchema(sql, alias);
Map<Long, DatasourceSchemaDTO> dsMap = new LinkedHashMap<>(); Map<Long, DatasourceSchemaDTO> dsMap = new LinkedHashMap<>();
dsMap.put(datasourceSchemaDTO.getId(), datasourceSchemaDTO); dsMap.put(datasourceSchemaDTO.getId(), datasourceSchemaDTO);
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(dsMap); datasourceRequest.setDsList(dsMap);
// sql 作为临时表外层加上limit // sql 作为临时表外层加上limit
String sql;
if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) { if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) {
// 先根据sql获取表字段 // 先根据sql获取表字段
String sqlField = SQLUtils.buildOriginPreviewSql(sql, 0, 0); String sqlField = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);
sqlField = SqlUtils.transSqlDialect(sqlField, datasourceRequest.getDsList());
// replace placeholder
sqlField = SqlUtils.replaceTablePlaceHolder(sqlField, originSql);
datasourceRequest.setQuery(sqlField); datasourceRequest.setQuery(sqlField);
// 获取数据源表的原始字段 // 获取数据源表的原始字段
List<TableField> list = (List<TableField>) calciteProvider.fetchResultField(datasourceRequest).get("fields"); List<TableField> list = calciteProvider.fetchTableField(datasourceRequest);
if (ObjectUtils.isEmpty(list)) { if (ObjectUtils.isEmpty(list)) {
return null; return null;
} }
sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(sql, 100, 0, String.format(SQLConstants.FIELD_DOT, list.get(0).getOriginName()) + " ASC "); sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0, String.format(SQLConstants.FIELD_DOT, list.get(0).getOriginName()) + " ASC ");
} else { } else {
sql = SQLUtils.buildOriginPreviewSql(sql, 100, 0); sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0);
} }
sql = SqlUtils.transSqlDialect(sql, datasourceRequest.getDsList());
// replace placeholder
sql = SqlUtils.replaceTablePlaceHolder(sql, originSql);
logger.info("calcite data preview sql: " + sql); logger.info("calcite data preview sql: " + sql);
datasourceRequest.setQuery(sql); datasourceRequest.setQuery(sql);
Map<String, Object> data = calciteProvider.fetchResultField(datasourceRequest); Map<String, Object> data = calciteProvider.fetchResultField(datasourceRequest);
@ -331,7 +354,7 @@ public class DatasetDataManage {
String[] row = dataList.get(i); String[] row = dataList.get(i);
LinkedHashMap<String, Object> obj = new LinkedHashMap<>(); LinkedHashMap<String, Object> obj = new LinkedHashMap<>();
if (row.length > 0) { if (row.length > 0) {
for (int j = 0; j < row.length; j++) { for (int j = 0; j < fields.size(); j++) {
if (desensitizationList.keySet().contains(fields.get(j).getDataeaseName())) { if (desensitizationList.keySet().contains(fields.get(j).getDataeaseName())) {
obj.put(fields.get(j).getDataeaseName(), ChartDataBuild.desensitizationValue(desensitizationList.get(fields.get(j).getDataeaseName()), String.valueOf(row[j]))); obj.put(fields.get(j).getDataeaseName(), ChartDataBuild.desensitizationValue(desensitizationList.get(fields.get(j).getDataeaseName()), String.valueOf(row[j])));
} else { } else {
@ -402,9 +425,15 @@ public class DatasetDataManage {
allFields.addAll(datasetGroupInfoDTO.getAllFields()); allFields.addAll(datasetGroupInfoDTO.getAllFields());
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
// build query sql // build query sql
SQLMeta sqlMeta = new SQLMeta(); SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")"); Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
// 计算字段先完成内容替换 // 计算字段先完成内容替换
if (Objects.equals(field.getExtField(), ExtFieldConstant.EXT_CALC)) { if (Objects.equals(field.getExtField(), ExtFieldConstant.EXT_CALC)) {
String originField = Utils.calcFieldRegex(field.getOriginName(), sqlMeta.getTable(), allFields); String originField = Utils.calcFieldRegex(field.getOriginName(), sqlMeta.getTable(), allFields);
@ -427,7 +456,6 @@ public class DatasetDataManage {
} }
buildFieldName(sqlMap, fields); buildFieldName(sqlMap, fields);
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
List<String> dsList = new ArrayList<>(); List<String> dsList = new ArrayList<>();
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) { for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
dsList.add(next.getValue().getType()); dsList.add(next.getValue().getType());
@ -444,6 +472,7 @@ public class DatasetDataManage {
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields); WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields);
Order2SQLObj.getOrders(sqlMeta, fields, datasetGroupInfoDTO.getSortFields()); Order2SQLObj.getOrders(sqlMeta, fields, datasetGroupInfoDTO.getSortFields());
String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, true, 0, 1000); String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, true, 0, 1000);
querySQL = SqlUtils.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
// 通过数据源请求数据 // 通过数据源请求数据
// 调用数据源的calcite获得data // 调用数据源的calcite获得data

View File

@ -6,6 +6,7 @@ import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.dataset.dto.SqlVariableDetails; import io.dataease.api.dataset.dto.SqlVariableDetails;
import io.dataease.api.dataset.union.*; import io.dataease.api.dataset.union.*;
import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.api.permissions.auth.dto.BusiPerCheckDTO; import io.dataease.api.permissions.auth.dto.BusiPerCheckDTO;
import io.dataease.commons.utils.SqlparserUtils; import io.dataease.commons.utils.SqlparserUtils;
import io.dataease.constant.AuthEnum; import io.dataease.constant.AuthEnum;
@ -93,9 +94,10 @@ public class DatasetSQLManage {
String tableSchema = putObj2Map(dsMap, currentDs); String tableSchema = putObj2Map(dsMap, currentDs);
// get table // get table
DatasetTableInfoDTO infoDTO = JsonUtil.parseObject(currentDs.getInfo(), DatasetTableInfoDTO.class); DatasetTableInfoDTO infoDTO = JsonUtil.parseObject(currentDs.getInfo(), DatasetTableInfoDTO.class);
Set<Long> allDs = getAllDs(union);
boolean isCross = allDs.size() > 1;
SQLObj tableName = getUnionTable(currentDs, infoDTO, tableSchema, 0, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null, isCross);
SQLObj tableName = getUnionTable(currentDs, infoDTO, tableSchema, 0, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null);
for (int i = 0; i < union.size(); i++) { for (int i = 0; i < union.size(); i++) {
UnionDTO unionDTO = union.get(i); UnionDTO unionDTO = union.get(i);
@ -108,7 +110,7 @@ public class DatasetSQLManage {
} else { } else {
schema = putObj2Map(dsMap, datasetTable); schema = putObj2Map(dsMap, datasetTable);
} }
SQLObj table = getUnionTable(datasetTable, tableInfo, schema, i, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null); SQLObj table = getUnionTable(datasetTable, tableInfo, schema, i, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null, isCross);
// 获取前端传过来选中的字段 // 获取前端传过来选中的字段
List<DatasetTableFieldDTO> fields = unionDTO.getCurrentDsFields(); List<DatasetTableFieldDTO> fields = unionDTO.getCurrentDsFields();
@ -127,17 +129,25 @@ public class DatasetSQLManage {
f.setDataeaseName(f.getFieldShortName()); f.setDataeaseName(f.getFieldShortName());
f.setDatasetTableId(datasetTable.getId()); f.setDatasetTableId(datasetTable.getId());
String prefix = ""; String prefix = "";
String suffix = "";
if (Objects.equals(f.getExtField(), ExtFieldConstant.EXT_NORMAL)) { if (Objects.equals(f.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
prefix = "`"; if (isCross) {
prefix = "`";
suffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, datasetTable.getDatasourceId());
prefix = datasourceType.getPrefix();
suffix = datasourceType.getSuffix();
}
} }
return table.getTableAlias() + "." + prefix + f.getOriginName() + prefix + " AS " + alias; return table.getTableAlias() + "." + prefix + f.getOriginName() + suffix + " AS " + prefix + alias + suffix;
}) })
.toArray(String[]::new); .toArray(String[]::new);
checkedInfo.put(table.getTableAlias(), array); checkedInfo.put(table.getTableAlias(), array);
checkedFields.addAll(fields); checkedFields.addAll(fields);
// 获取child的fields和union // 获取child的fields和union
if (!CollectionUtils.isEmpty(unionDTO.getChildrenDs())) { if (!CollectionUtils.isEmpty(unionDTO.getChildrenDs())) {
getUnionForEdit(datasetTable, table, unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields, dsMap, chartExtRequest); getUnionForEdit(datasetTable, table, unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields, dsMap, chartExtRequest, isCross);
} }
} }
// build sql // build sql
@ -158,17 +168,28 @@ public class DatasetSQLManage {
SQLObj parentSQLObj = unionParamDTO.getParentSQLObj(); SQLObj parentSQLObj = unionParamDTO.getParentSQLObj();
SQLObj currentSQLObj = unionParamDTO.getCurrentSQLObj(); SQLObj currentSQLObj = unionParamDTO.getCurrentSQLObj();
DatasetTableDTO parentDs = unionParamDTO.getParentDs();
DatasetTableDTO currentDs1 = unionParamDTO.getCurrentDs();
String ts = ""; String ts = "";
String tablePrefix = ""; String tablePrefix = "";
String tableSuffix = "";
if (ObjectUtils.isNotEmpty(currentSQLObj.getTableSchema())) { if (ObjectUtils.isNotEmpty(currentSQLObj.getTableSchema())) {
ts = currentSQLObj.getTableSchema() + "."; ts = currentSQLObj.getTableSchema() + ".";
tablePrefix = "`";
if (isCross) {
tablePrefix = "`";
tableSuffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, currentDs1.getDatasourceId());
tablePrefix = datasourceType.getPrefix();
tableSuffix = datasourceType.getSuffix();
}
} }
// build join // build join
join.append(" ").append(joinType).append(" ") join.append(" ").append(joinType).append(" ")
.append(ts) .append(ts)
.append(tablePrefix + currentSQLObj.getTableName() + tablePrefix) .append(tablePrefix + currentSQLObj.getTableName() + tableSuffix)
.append(" ").append(currentSQLObj.getTableAlias()).append(" ") .append(" ").append(currentSQLObj.getTableAlias()).append(" ")
.append(" ON "); .append(" ON ");
if (unionParamDTO.getUnionFields().size() == 0) { if (unionParamDTO.getUnionFields().size() == 0) {
@ -180,18 +201,34 @@ public class DatasetSQLManage {
DatasetTableFieldDTO parentField = unionItemDTO.getParentField(); DatasetTableFieldDTO parentField = unionItemDTO.getParentField();
DatasetTableFieldDTO currentField = unionItemDTO.getCurrentField(); DatasetTableFieldDTO currentField = unionItemDTO.getCurrentField();
String pPrefix = ""; String pPrefix = "";
String pSuffix = "";
if (Objects.equals(parentField.getExtField(), ExtFieldConstant.EXT_NORMAL)) { if (Objects.equals(parentField.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
pPrefix = "`"; if (isCross) {
pPrefix = "`";
pSuffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, parentDs.getDatasourceId());
pPrefix = datasourceType.getPrefix();
pSuffix = datasourceType.getSuffix();
}
} }
String cPrefix = ""; String cPrefix = "";
String cSuffix = "";
if (Objects.equals(currentField.getExtField(), ExtFieldConstant.EXT_NORMAL)) { if (Objects.equals(currentField.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
cPrefix = "`"; if (isCross) {
cPrefix = "`";
cSuffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, currentDs1.getDatasourceId());
cPrefix = datasourceType.getPrefix();
cSuffix = datasourceType.getSuffix();
}
} }
join.append(parentSQLObj.getTableAlias()).append(".") join.append(parentSQLObj.getTableAlias()).append(".")
.append(pPrefix + parentField.getOriginName() + pPrefix) .append(pPrefix + parentField.getOriginName() + pSuffix)
.append(" = ") .append(" = ")
.append(currentSQLObj.getTableAlias()).append(".") .append(currentSQLObj.getTableAlias()).append(".")
.append(cPrefix + currentField.getOriginName() + cPrefix); .append(cPrefix + currentField.getOriginName() + cSuffix);
if (i < unionParamDTO.getUnionFields().size() - 1) { if (i < unionParamDTO.getUnionFields().size() - 1) {
join.append(" AND "); join.append(" AND ");
} }
@ -200,13 +237,13 @@ public class DatasetSQLManage {
if (StringUtils.isEmpty(f)) { if (StringUtils.isEmpty(f)) {
DEException.throwException(Translator.get("i18n_union_ds_no_checked")); DEException.throwException(Translator.get("i18n_union_ds_no_checked"));
} }
sql = MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.getTableAndAlias(tableName)) + join.toString(); sql = MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.getTableAndAlias(tableName, getDatasourceType(dsMap, currentDs.getDatasourceId()), isCross)) + join.toString();
} else { } else {
String f = StringUtils.join(checkedInfo.get(tableName.getTableAlias()), ","); String f = StringUtils.join(checkedInfo.get(tableName.getTableAlias()), ",");
if (StringUtils.isEmpty(f)) { if (StringUtils.isEmpty(f)) {
DEException.throwException(Translator.get("i18n_union_ds_no_checked")); DEException.throwException(Translator.get("i18n_union_ds_no_checked"));
} }
sql = MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.getTableAndAlias(tableName)); sql = MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.getTableAndAlias(tableName, getDatasourceType(dsMap, currentDs.getDatasourceId()), isCross));
} }
logger.info("calcite origin sql: " + sql); logger.info("calcite origin sql: " + sql);
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
@ -221,7 +258,8 @@ public class DatasetSQLManage {
private void getUnionForEdit(DatasetTableDTO parentTable, SQLObj parentSQLObj, private void getUnionForEdit(DatasetTableDTO parentTable, SQLObj parentSQLObj,
List<UnionDTO> childrenDs, Map<String, String[]> checkedInfo, List<UnionDTO> childrenDs, Map<String, String[]> checkedInfo,
List<UnionParamDTO> unionList, List<DatasetTableFieldDTO> checkedFields, List<UnionParamDTO> unionList, List<DatasetTableFieldDTO> checkedFields,
Map<Long, DatasourceSchemaDTO> dsMap, ChartExtRequest chartExtRequest) throws Exception { Map<Long, DatasourceSchemaDTO> dsMap, ChartExtRequest chartExtRequest,
boolean isCross) throws Exception {
for (int i = 0; i < childrenDs.size(); i++) { for (int i = 0; i < childrenDs.size(); i++) {
int index = unionList.size() + 1; int index = unionList.size() + 1;
@ -235,7 +273,7 @@ public class DatasetSQLManage {
} else { } else {
schema = putObj2Map(dsMap, datasetTable); schema = putObj2Map(dsMap, datasetTable);
} }
SQLObj table = getUnionTable(datasetTable, tableInfo, schema, index, filterParameters(chartExtRequest, datasetTable.getId()), chartExtRequest == null); SQLObj table = getUnionTable(datasetTable, tableInfo, schema, index, filterParameters(chartExtRequest, datasetTable.getId()), chartExtRequest == null, isCross);
List<DatasetTableFieldDTO> fields = unionDTO.getCurrentDsFields(); List<DatasetTableFieldDTO> fields = unionDTO.getCurrentDsFields();
fields = fields.stream().filter(DatasetTableFieldDTO::getChecked).collect(Collectors.toList()); fields = fields.stream().filter(DatasetTableFieldDTO::getChecked).collect(Collectors.toList());
@ -253,10 +291,18 @@ public class DatasetSQLManage {
f.setDataeaseName(f.getFieldShortName()); f.setDataeaseName(f.getFieldShortName());
f.setDatasetTableId(datasetTable.getId()); f.setDatasetTableId(datasetTable.getId());
String prefix = ""; String prefix = "";
String suffix = "";
if (Objects.equals(f.getExtField(), ExtFieldConstant.EXT_NORMAL)) { if (Objects.equals(f.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
prefix = "`"; if (isCross) {
prefix = "`";
suffix = "`";
} else {
DatasourceConfiguration.DatasourceType datasourceType = getDatasourceType(dsMap, datasetTable.getDatasourceId());
prefix = datasourceType.getPrefix();
suffix = datasourceType.getSuffix();
}
} }
return table.getTableAlias() + "." + prefix + f.getOriginName() + prefix + " AS " + alias; return table.getTableAlias() + "." + prefix + f.getOriginName() + suffix + " AS " + prefix + alias + suffix;
}) })
.toArray(String[]::new); .toArray(String[]::new);
checkedInfo.put(table.getTableAlias(), array); checkedInfo.put(table.getTableAlias(), array);
@ -270,11 +316,45 @@ public class DatasetSQLManage {
unionToParent.setCurrentSQLObj(table); unionToParent.setCurrentSQLObj(table);
unionList.add(unionToParent); unionList.add(unionToParent);
if (!CollectionUtils.isEmpty(unionDTO.getChildrenDs())) { if (!CollectionUtils.isEmpty(unionDTO.getChildrenDs())) {
getUnionForEdit(datasetTable, table, unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields, dsMap, chartExtRequest); getUnionForEdit(datasetTable, table, unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields, dsMap, chartExtRequest, isCross);
} }
} }
} }
private Set<Long> getAllDs(List<UnionDTO> union) {
Set<Long> set = new HashSet<>();
for (UnionDTO unionDTO : union) {
Long datasourceId = unionDTO.getCurrentDs().getDatasourceId();
set.add(datasourceId);
getChildrenDs(unionDTO.getChildrenDs(), set);
}
return set;
}
private void getChildrenDs(List<UnionDTO> childrenDs, Set<Long> set) {
for (UnionDTO unionDTO : childrenDs) {
set.add(unionDTO.getCurrentDs().getDatasourceId());
if (!CollectionUtils.isEmpty(unionDTO.getChildrenDs())) {
getChildrenDs(unionDTO.getChildrenDs(), set);
}
}
}
private DatasourceConfiguration.DatasourceType getDatasourceType(Map<Long, DatasourceSchemaDTO> dsMap, Long datasourceId) {
DatasourceSchemaDTO datasourceSchemaDTO = dsMap.get(datasourceId);
String type;
if (datasourceSchemaDTO == null) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasourceId);
if (coreDatasource == null) {
DEException.throwException(Translator.get("i18n_dataset_ds_error") + ",ID:" + datasourceId);
}
type = engineManage.getDeEngine().getType();
} else {
type = datasourceSchemaDTO.getType();
}
return DatasourceConfiguration.DatasourceType.valueOf(type);
}
public String subPrefixSuffixChar(String str) { public String subPrefixSuffixChar(String str) {
while (StringUtils.startsWith(str, ",")) { while (StringUtils.startsWith(str, ",")) {
str = str.substring(1, str.length()); str = str.substring(1, str.length());
@ -304,7 +384,7 @@ public class DatasetSQLManage {
} }
} }
private SQLObj getUnionTable(DatasetTableDTO currentDs, DatasetTableInfoDTO infoDTO, String tableSchema, int index, List<SqlVariableDetails> parameters, boolean isFromDataSet) { private SQLObj getUnionTable(DatasetTableDTO currentDs, DatasetTableInfoDTO infoDTO, String tableSchema, int index, List<SqlVariableDetails> parameters, boolean isFromDataSet, boolean isCross) {
SQLObj tableObj; SQLObj tableObj;
String tableAlias = String.format(SQLConstants.TABLE_ALIAS_PREFIX, index); String tableAlias = String.format(SQLConstants.TABLE_ALIAS_PREFIX, index);
if (StringUtils.equalsIgnoreCase(currentDs.getType(), DatasetTableTypeConstants.DATASET_TABLE_DB)) { if (StringUtils.equalsIgnoreCase(currentDs.getType(), DatasetTableTypeConstants.DATASET_TABLE_DB)) {
@ -313,7 +393,9 @@ public class DatasetSQLManage {
// parser sql params and replace default value // parser sql params and replace default value
String sql = SqlparserUtils.handleVariableDefaultValue(new String(Base64.getDecoder().decode(infoDTO.getSql())), currentDs.getSqlVariableDetails(), false, isFromDataSet, parameters); String sql = SqlparserUtils.handleVariableDefaultValue(new String(Base64.getDecoder().decode(infoDTO.getSql())), currentDs.getSqlVariableDetails(), false, isFromDataSet, parameters);
// add table schema // add table schema
sql = SqlUtils.addSchema(sql, tableSchema); if (isCross) {
sql = SqlUtils.addSchema(sql, tableSchema);
}
tableObj = SQLObj.builder().tableSchema("").tableName("(" + sql + ")").tableAlias(tableAlias).build(); tableObj = SQLObj.builder().tableSchema("").tableName("(" + sql + ")").tableAlias(tableAlias).build();
} else { } else {
// excel,api // excel,api

View File

@ -5,6 +5,7 @@ package io.dataease.dataset.utils;
*/ */
public class FieldUtils { public class FieldUtils {
public static int transType2DeType(String type) { public static int transType2DeType(String type) {
type = type.replaceAll("\\((.*?)\\)","");
switch (type) { switch (type) {
case "CHAR": case "CHAR":
case "VARCHAR": case "VARCHAR":
@ -14,12 +15,19 @@ public class FieldUtils {
case "LONGTEXT": case "LONGTEXT":
case "ENUM": case "ENUM":
case "ANY": case "ANY":
case "STRING":
return 0;// 文本 return 0;// 文本
case "DATE": case "DATE":
case "TIME": case "TIME":
case "YEAR": case "YEAR":
case "DATETIME": case "DATETIME":
case "TIMESTAMP": case "TIMESTAMP":
case "DATEV2":
case "DATETIMEV2":
case "DATETIME2":
case "DATETIMEOFFSET":
case "SMALLDATETIME":
case "DATETIME64":
return 1;// 时间 return 1;// 时间
case "INT": case "INT":
case "SMALLINT": case "SMALLINT":
@ -27,14 +35,38 @@ public class FieldUtils {
case "INTEGER": case "INTEGER":
case "BIGINT": case "BIGINT":
case "LONG": //增加了LONG类型 case "LONG": //增加了LONG类型
case "INT2":
case "INT4":
case "INT8":
case "int2":
case "int4":
case "int8":
case "INT16":
case "INT32":
case "INT64":
case "UINT8":
case "UINT16":
case "UINT32":
case "UINT64":
return 2;// 整型 return 2;// 整型
case "FLOAT": case "FLOAT":
case "DOUBLE": case "DOUBLE":
case "DECIMAL": case "DECIMAL":
case "REAL": case "REAL":
case "MONEY":
case "NUMERIC":
case "float4":
case "float8":
case "FLOAT4":
case "FLOAT8":
case "DECFLOAT":
case "FLOAT32":
case "FLOAT64":
return 3;// 浮点 return 3;// 浮点
case "BIT": case "BIT":
case "TINYINT": case "TINYINT":
case "BOOL":
case "BOOLEAN":
return 4;// 布尔 return 4;// 布尔
default: default:
return 0; return 0;

View File

@ -1,13 +1,21 @@
package io.dataease.dataset.utils; package io.dataease.dataset.utils;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.exception.DEException; import io.dataease.exception.DEException;
import org.apache.calcite.config.Lex; import org.apache.calcite.config.Lex;
import org.apache.calcite.sql.*; import org.apache.calcite.sql.*;
import org.apache.calcite.sql.dialect.*;
import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParser;
import org.apache.commons.collections4.MapUtils;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import static org.apache.calcite.sql.SqlKind.*; import static org.apache.calcite.sql.SqlKind.*;
@ -38,7 +46,7 @@ public class SqlUtils {
String sqlRender = sqlNode.toString(); String sqlRender = sqlNode.toString();
// 处理sql中多余的`都替换成1个 // 处理sql中多余的`都替换成1个
sqlRender = sqlRender.replaceAll("(`+)", "`"); sqlRender = sqlRender.replaceAll("(`+)", "`");
return sqlRender; return sqlRender.replaceAll("`", "");
} }
private static void addTableSchema(SqlNode sqlNode, Boolean fromOrJoin, String schema, SqlParser.Config config) { private static void addTableSchema(SqlNode sqlNode, Boolean fromOrJoin, String schema, SqlParser.Config config) {
@ -107,4 +115,105 @@ public class SqlUtils {
DEException.throwException("使用 Calcite 进行语法分析发生了异常:" + e); DEException.throwException("使用 Calcite 进行语法分析发生了异常:" + e);
} }
} }
public static String rebuildSQL(String sql, SQLMeta sqlMeta, boolean crossDs, Map<Long, DatasourceSchemaDTO> dsMap) {
if (crossDs) {
return sql;
}
String s = transSqlDialect(sql, dsMap);
String tableDialect = sqlMeta.getTableDialect();
s = replaceTablePlaceHolder(s, tableDialect);
return replaceCalcFieldPlaceHolder(s, sqlMeta);
}
public static String transSqlDialect(String sql, Map<Long, DatasourceSchemaDTO> dsMap) throws DEException {
try {
DatasourceSchemaDTO value = dsMap.entrySet().iterator().next().getValue();
SqlParser parser = SqlParser.create(sql, SqlParser.Config.DEFAULT.withLex(Lex.JAVA));
SqlNode sqlNode = parser.parseStmt();
return sqlNode.toSqlString(getDialect(value)).toString();
} catch (Exception e) {
DEException.throwException(e.getMessage());
}
return null;
}
public static String replaceTablePlaceHolder(String s, String placeholder) {
s = s.replaceAll("\r\n", " ")
.replaceAll("\n", " ")
.replaceAll(SqlPlaceholderConstants.TABLE_PLACEHOLDER_REGEX, placeholder)
.replaceAll("ASYMMETRIC", "")
.replaceAll("SYMMETRIC", "");
return s;
}
public static String replaceCalcFieldPlaceHolder(String s, SQLMeta sqlMeta) {
Map<String, String> fieldsDialect = new HashMap<>();
if (MapUtils.isNotEmpty(sqlMeta.getXFieldsDialect())) {
fieldsDialect.putAll(sqlMeta.getXFieldsDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getYFieldsDialect())) {
fieldsDialect.putAll(sqlMeta.getYFieldsDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getCustomWheresDialect())) {
fieldsDialect.putAll(sqlMeta.getCustomWheresDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getExtWheresDialect())) {
fieldsDialect.putAll(sqlMeta.getExtWheresDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getWhereTreesDialect())) {
fieldsDialect.putAll(sqlMeta.getWhereTreesDialect());
}
if (MapUtils.isNotEmpty(fieldsDialect)) {
for (Map.Entry<String, String> ele : fieldsDialect.entrySet()) {
s = s.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + ele.getKey() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, ele.getValue());
}
}
return s;
}
private static SqlDialect getDialect(DatasourceSchemaDTO coreDatasource) {
SqlDialect sqlDialect = null;
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(coreDatasource.getType());
switch (datasourceType) {
case mysql:
case mongo:
case StarRocks:
case doris:
case TiDB:
case mariadb:
sqlDialect = MysqlSqlDialect.DEFAULT;
break;
case impala:
sqlDialect = ImpalaSqlDialect.DEFAULT;
break;
case sqlServer:
sqlDialect = MssqlSqlDialect.DEFAULT;
break;
case oracle:
sqlDialect = OracleSqlDialect.DEFAULT;
break;
case db2:
sqlDialect = Db2SqlDialect.DEFAULT;
break;
case pg:
sqlDialect = PostgresqlSqlDialect.DEFAULT;
break;
case redshift:
sqlDialect = RedshiftSqlDialect.DEFAULT;
break;
case ck:
sqlDialect = ClickHouseSqlDialect.DEFAULT;
break;
case h2:
sqlDialect = H2SqlDialect.DEFAULT;
break;
default:
sqlDialect = MysqlSqlDialect.DEFAULT;
}
return sqlDialect;
}
} }

View File

@ -1,6 +1,7 @@
package io.dataease.dataset.utils; package io.dataease.dataset.utils;
import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.dataset.dto.DatasourceSchemaDTO; import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.utils.Md5Utils; import io.dataease.utils.Md5Utils;
import org.apache.calcite.avatica.util.Quoting; import org.apache.calcite.avatica.util.Quoting;
@ -38,14 +39,21 @@ public class TableUtils {
return "C_" + Md5Utils.md5(fieldName); return "C_" + Md5Utils.md5(fieldName);
} }
public static String getTableAndAlias(SQLObj sqlObj) { public static String getTableAndAlias(SQLObj sqlObj, DatasourceConfiguration.DatasourceType datasourceType, boolean isCross) {
String schema = ""; String schema = "";
String prefix = ""; String prefix = "";
String suffix = "";
if (StringUtils.isNotEmpty(sqlObj.getTableSchema())) { if (StringUtils.isNotEmpty(sqlObj.getTableSchema())) {
schema = sqlObj.getTableSchema() + "."; schema = sqlObj.getTableSchema() + ".";
prefix = "`"; if (isCross) {
prefix = "`";
suffix = "`";
} else {
prefix = datasourceType.getPrefix();
suffix = datasourceType.getSuffix();
}
} }
return schema + prefix + sqlObj.getTableName() + prefix + " " + sqlObj.getTableAlias(); return schema + prefix + sqlObj.getTableName() + suffix + " " + sqlObj.getTableAlias();
} }
public static String tableName2Sql(DatasourceSchemaDTO ds, String tableName) { public static String tableName2Sql(DatasourceSchemaDTO ds, String tableName) {

View File

@ -35,6 +35,7 @@ import org.springframework.util.CollectionUtils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.math.BigDecimal;
import java.net.URL; import java.net.URL;
import java.sql.*; import java.sql.*;
import java.util.*; import java.util.*;
@ -147,6 +148,11 @@ public class CalciteProvider {
} }
public Map<String, Object> fetchResultField(DatasourceRequest datasourceRequest) throws DEException { public Map<String, Object> fetchResultField(DatasourceRequest datasourceRequest) throws DEException {
// 不跨数据源
if (datasourceRequest.getDsList().size() == 1) {
return jdbcFetchResultField(datasourceRequest);
}
List<TableField> datasetTableFields = new ArrayList<>(); List<TableField> datasetTableFields = new ArrayList<>();
List<String[]> list = new LinkedList<>(); List<String[]> list = new LinkedList<>();
PreparedStatement statement = null; PreparedStatement statement = null;
@ -191,6 +197,119 @@ public class CalciteProvider {
return map; return map;
} }
public Map<String, Object> jdbcFetchResultField(DatasourceRequest datasourceRequest) throws DEException {
DatasourceSchemaDTO value = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
datasourceRequest.setDatasource(value);
DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
Map<String, Object> map = new LinkedHashMap<>();
List<TableField> fieldList = new ArrayList<>();
List<String[]> dataList = new LinkedList<>();
// schema
ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, datasourceConfiguration.getQueryTimeout())) {
if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(datasourceRequest.getQuery());
fieldList = getField(resultSet, datasourceRequest);
dataList = getData(resultSet, datasourceRequest);
} catch (SQLException e) {
DEException.throwException("SQL ERROR: " + e.getMessage());
} catch (Exception e) {
DEException.throwException("Data source connection exception: " + e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
map.put("fields", fieldList);
map.put("data", dataList);
return map;
}
private List<TableField> getField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
List<TableField> fieldList = new ArrayList<>();
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
for (int j = 0; j < columnCount; j++) {
String f = metaData.getColumnName(j + 1);
if (StringUtils.equalsIgnoreCase(f, "DE_ROWNUM")) {
continue;
}
String l = StringUtils.isNotEmpty(metaData.getColumnLabel(j + 1)) ? metaData.getColumnLabel(j + 1) : f;
String t = metaData.getColumnTypeName(j + 1).toUpperCase();
TableField field = new TableField();
field.setOriginName(l);
field.setName(l);
field.setFieldType(t);
field.setType(t);
fieldList.add(field);
}
return fieldList;
}
private List<String[]> getData(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
String charset = null;
String targetCharset = "UTF-8";
if (datasourceRequest != null && datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) {
DatasourceConfiguration jdbcConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset()) && !jdbcConfiguration.getCharset().equalsIgnoreCase("Default")) {
charset = jdbcConfiguration.getCharset();
}
if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset()) && !jdbcConfiguration.getTargetCharset().equalsIgnoreCase("Default")) {
targetCharset = jdbcConfiguration.getTargetCharset();
}
}
List<String[]> list = new LinkedList<>();
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
while (rs.next()) {
String[] row = new String[columnCount];
for (int j = 0; j < columnCount; j++) {
int columnType = metaData.getColumnType(j + 1);
switch (columnType) {
case Types.DATE:
if (rs.getDate(j + 1) != null) {
row[j] = rs.getDate(j + 1).toString();
}
break;
case Types.BOOLEAN:
row[j] = rs.getBoolean(j + 1) ? "1" : "0";
break;
case Types.NUMERIC:
BigDecimal bigDecimal = rs.getBigDecimal(j + 1);
row[j] = bigDecimal == null ? null : bigDecimal.toString();
break;
default:
if (metaData.getColumnTypeName(j + 1).toLowerCase().equalsIgnoreCase("blob")) {
row[j] = rs.getBlob(j + 1) == null ? "" : rs.getBlob(j + 1).toString();
} else {
if (charset != null && StringUtils.isNotEmpty(rs.getString(j + 1))) {
String originStr = new String(rs.getString(j + 1).getBytes(charset), targetCharset);
row[j] = new String(originStr.getBytes("UTF-8"), "UTF-8");
} else {
row[j] = rs.getString(j + 1);
}
}
break;
}
}
list.add(row);
}
return list;
}
private String getTableFiledSql(DatasourceRequest datasourceRequest) { private String getTableFiledSql(DatasourceRequest datasourceRequest) {
String sql = ""; String sql = "";
DatasourceConfiguration configuration = null; DatasourceConfiguration configuration = null;
@ -295,7 +414,8 @@ public class CalciteProvider {
private TableField getTableFieldDesc(DatasourceRequest datasourceRequest, ResultSet resultSet) throws SQLException { private TableField getTableFieldDesc(DatasourceRequest datasourceRequest, ResultSet resultSet) throws SQLException {
TableField tableField = new TableField(); TableField tableField = new TableField();
tableField.setOriginName(resultSet.getString(1)); tableField.setOriginName(resultSet.getString(1));
tableField.setType(resultSet.getString(2)); tableField.setType(resultSet.getString(2).toUpperCase());
tableField.setFieldType(tableField.getType());
int deType = FieldUtils.transType2DeType(tableField.getType()); int deType = FieldUtils.transType2DeType(tableField.getType());
tableField.setDeExtractType(deType); tableField.setDeExtractType(deType);
tableField.setDeType(deType); tableField.setDeType(deType);
@ -306,24 +426,57 @@ public class CalciteProvider {
public List<TableField> fetchTableField(DatasourceRequest datasourceRequest) throws DEException { public List<TableField> fetchTableField(DatasourceRequest datasourceRequest) throws DEException {
List<TableField> datasetTableFields = new ArrayList<>(); List<TableField> datasetTableFields = new ArrayList<>();
try (Connection con = getConnection(datasourceRequest.getDatasource()); Statement statement = getStatement(con, 30); ResultSet resultSet = statement.executeQuery(getTableFiledSql(datasourceRequest))) { DatasourceSchemaDTO datasourceSchemaDTO = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
while (resultSet.next()) { datasourceRequest.setDatasource(datasourceSchemaDTO);
datasetTableFields.add(getTableFieldDesc(datasourceRequest, resultSet));
}
} catch (Exception e) {
DEException.throwException(e.getMessage());
}
List<TableField> tableFields = (List<TableField>) fetchResultField(datasourceRequest).get("fields"); DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
for (TableField tableField : tableFields) {
for (TableField datasetTableField : datasetTableFields) { String table = datasourceRequest.getTable();
if(tableField.getOriginName().equalsIgnoreCase(datasetTableField.getOriginName())){ if (StringUtils.isEmpty(table)) {
tableField.setName(datasetTableField.getName()); ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, 30)) {
if (DatasourceConfiguration.DatasourceType.valueOf(datasourceSchemaDTO.getType()) == DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(datasourceRequest.getQuery());
datasetTableFields.addAll(getField(resultSet, datasourceRequest));
} catch (Exception e) {
DEException.throwException(e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
} else {
ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, 30)) {
if (DatasourceConfiguration.DatasourceType.valueOf(datasourceSchemaDTO.getType()) == DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(getTableFiledSql(datasourceRequest));
while (resultSet.next()) {
datasetTableFields.add(getTableFieldDesc(datasourceRequest, resultSet));
}
} catch (Exception e) {
DEException.throwException(e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
} }
} }
} }
return tableFields; return datasetTableFields;
} }

View File

@ -878,7 +878,7 @@ public class DatasourceServer implements DatasourceApi {
} }
@Override @Override
public Map<String, Object> previewDataWithLimit(Map<String, Object> req) { public Map<String, Object> previewDataWithLimit(Map<String, Object> req) throws DEException {
String tableName = req.get("table").toString(); String tableName = req.get("table").toString();
Long id = Long.valueOf(req.get("id").toString()); Long id = Long.valueOf(req.get("id").toString());
if (ObjectUtils.isEmpty(tableName) || ObjectUtils.isEmpty(id)) { if (ObjectUtils.isEmpty(tableName) || ObjectUtils.isEmpty(id)) {

View File

@ -6,11 +6,13 @@ import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils; import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -25,6 +27,7 @@ public class CustomWhere2Str {
return; return;
} }
List<String> res = new ArrayList<>(); List<String> res = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (ObjectUtils.isNotEmpty(fields)) { if (ObjectUtils.isNotEmpty(fields)) {
for (ChartFieldCustomFilterDTO request : fields) { for (ChartFieldCustomFilterDTO request : fields) {
List<SQLObj> list = new ArrayList<>(); List<SQLObj> list = new ArrayList<>();
@ -37,7 +40,10 @@ public class CustomWhere2Str {
String originName; String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originName = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields); String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
} else { } else {
@ -137,6 +143,7 @@ public class CustomWhere2Str {
} }
meta.setCustomWheres(ObjectUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null); meta.setCustomWheres(ObjectUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null);
} }
meta.setCustomWheresDialect(fieldsDialect);
} }

View File

@ -7,14 +7,13 @@ import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants; import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils; import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Objects;
/** /**
* @Author Junjun * @Author Junjun
@ -28,13 +27,17 @@ public class Dimension2SQLObj {
} }
List<SQLObj> xFields = new ArrayList<>(); List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>(); List<SQLObj> xOrders = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (!CollectionUtils.isEmpty(fields)) { if (!CollectionUtils.isEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) { for (int i = 0; i < fields.size(); i++) {
ChartViewFieldDTO x = fields.get(i); ChartViewFieldDTO x = fields.get(i);
String originField; String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_CALC)) { if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = Utils.calcFieldRegex(x.getOriginName(), tableObj, originFields); String calcFieldExp = Utils.calcFieldRegex(x.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, x.getId());
fieldsDialect.put(originField, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_COPY)) { } else if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_COPY)) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName()); originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
} else { } else {
@ -56,6 +59,7 @@ public class Dimension2SQLObj {
} }
meta.setXFields(xFields); meta.setXFields(xFields);
meta.setXOrders(xOrders); meta.setXOrders(xOrders);
meta.setXFieldsDialect(fieldsDialect);
} }
private static SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) { private static SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) {

View File

@ -5,12 +5,15 @@ import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils; import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* @Author Junjun * @Author Junjun
@ -23,6 +26,7 @@ public class ExtWhere2Str {
return; return;
} }
List<SQLObj> list = new ArrayList<>(); List<SQLObj> list = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (ObjectUtils.isNotEmpty(fields)) { if (ObjectUtils.isNotEmpty(fields)) {
for (ChartExtFilterDTO request : fields) { for (ChartExtFilterDTO request : fields) {
List<String> value = request.getValue(); List<String> value = request.getValue();
@ -44,7 +48,10 @@ public class ExtWhere2Str {
String originName; String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originName = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields); String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
} else { } else {
@ -133,6 +140,7 @@ public class ExtWhere2Str {
list.forEach(ele -> strList.add("(" + ele.getWhereField() + " " + ele.getWhereTermAndValue() + ")")); list.forEach(ele -> strList.add("(" + ele.getWhereField() + " " + ele.getWhereTermAndValue() + ")"));
meta.setExtWheres(ObjectUtils.isNotEmpty(list) ? "(" + String.join(" AND ", strList) + ")" : null); meta.setExtWheres(ObjectUtils.isNotEmpty(list) ? "(" + String.join(" AND ", strList) + ")" : null);
} }
meta.setExtWheresDialect(fieldsDialect);
} }
} }

View File

@ -6,14 +6,13 @@ import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants; import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.func.FunctionConstant; import io.dataease.engine.func.FunctionConstant;
import io.dataease.engine.utils.Utils; import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Objects;
/** /**
* @Author Junjun * @Author Junjun
@ -26,13 +25,17 @@ public class Field2SQLObj {
return; return;
} }
List<SQLObj> xFields = new ArrayList<>(); List<SQLObj> xFields = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (ObjectUtils.isNotEmpty(fields)) { if (ObjectUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) { for (int i = 0; i < fields.size(); i++) {
DatasetTableFieldDTO x = fields.get(i); DatasetTableFieldDTO x = fields.get(i);
String originField; String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_CALC)) { if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = Utils.calcFieldRegex(x.getOriginName(), tableObj, fields); String calcFieldExp = Utils.calcFieldRegex(x.getOriginName(), tableObj, fields);
// 给计算字段处加一个占位符后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, x.getId());
fieldsDialect.put(originField, calcFieldExp);
// 此处是数据集预览获取数据库原始字段枚举值等操作使用如果遇到聚合函数则将originField设置为null // 此处是数据集预览获取数据库原始字段枚举值等操作使用如果遇到聚合函数则将originField设置为null
for (String func : FunctionConstant.AGG_FUNC) { for (String func : FunctionConstant.AGG_FUNC) {
if (Utils.matchFunction(func, originField)) { if (Utils.matchFunction(func, originField)) {
@ -51,6 +54,7 @@ public class Field2SQLObj {
} }
} }
meta.setXFields(xFields); meta.setXFields(xFields);
meta.setXFieldsDialect(fieldsDialect);
} }
public static SQLObj getXFields(DatasetTableFieldDTO f, String originField, String fieldAlias) { public static SQLObj getXFields(DatasetTableFieldDTO f, String originField, String fieldAlias) {

View File

@ -7,14 +7,13 @@ import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.engine.constant.DeTypeConstants; import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils; import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Objects;
/** /**
* @Author Junjun * @Author Junjun
@ -29,13 +28,17 @@ public class Quota2SQLObj {
List<SQLObj> yFields = new ArrayList<>(); List<SQLObj> yFields = new ArrayList<>();
List<String> yWheres = new ArrayList<>(); List<String> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>(); List<SQLObj> yOrders = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
if (!CollectionUtils.isEmpty(fields)) { if (!CollectionUtils.isEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) { for (int i = 0; i < fields.size(); i++) {
ChartViewFieldDTO y = fields.get(i); ChartViewFieldDTO y = fields.get(i);
String originField; String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && Objects.equals(y.getExtField(), ExtFieldConstant.EXT_CALC)) { if (ObjectUtils.isNotEmpty(y.getExtField()) && Objects.equals(y.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = Utils.calcFieldRegex(y.getOriginName(), tableObj, originFields); String calcFieldExp = Utils.calcFieldRegex(y.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, y.getId());
fieldsDialect.put(originField, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && Objects.equals(y.getExtField(), ExtFieldConstant.EXT_COPY)) { } else if (ObjectUtils.isNotEmpty(y.getExtField()) && Objects.equals(y.getExtField(), ExtFieldConstant.EXT_COPY)) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName()); originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName());
} else { } else {
@ -62,6 +65,7 @@ public class Quota2SQLObj {
meta.setYFields(yFields); meta.setYFields(yFields);
meta.setYWheres(yWheres); meta.setYWheres(yWheres);
meta.setYOrders(yOrders); meta.setYOrders(yOrders);
meta.setYFieldsDialect(fieldsDialect);
} }
private static SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) { private static SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) {

View File

@ -3,14 +3,22 @@ package io.dataease.engine.trans;
import io.dataease.api.dataset.union.model.SQLMeta; import io.dataease.api.dataset.union.model.SQLMeta;
import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
/** /**
* @Author Junjun * @Author Junjun
*/ */
public class Table2SQLObj { public class Table2SQLObj {
public static void table2sqlobj(SQLMeta meta, String tablePrefix, String table) { public static void table2sqlobj(SQLMeta meta, String tablePrefix, String table, boolean crossDs) {
String sql;
if (table.startsWith("(") && table.endsWith(")") && !crossDs) {// SQL片段和关联
meta.setTableDialect(table.substring(1, table.length() - 1));
sql = "(" + SqlPlaceholderConstants.TABLE_PLACEHOLDER + ")";
} else {
sql = table;
}
SQLObj tableObj = SQLObj.builder() SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(SQLConstants.TABLE_NAME, tablePrefix, table)) .tableName((table.startsWith("(") && table.endsWith(")")) ? sql : String.format(SQLConstants.TABLE_NAME, tablePrefix, table))
.tableAlias(String.format(SQLConstants.TABLE_ALIAS_PREFIX, 0)) .tableAlias(String.format(SQLConstants.TABLE_ALIAS_PREFIX, 0))
.build(); .build();
meta.setTable(tableObj); meta.setTable(tableObj);

View File

@ -8,15 +8,13 @@ import io.dataease.api.permissions.dataset.dto.DatasetRowPermissionsTreeObj;
import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils; import io.dataease.engine.utils.Utils;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/** /**
* @Author Junjun * @Author Junjun
@ -31,6 +29,7 @@ public class WhereTree2Str {
return; return;
} }
List<String> res = new ArrayList<>(); List<String> res = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
// permission trees // permission trees
// 解析每个tree然后多个tree之间用and拼接 // 解析每个tree然后多个tree之间用and拼接
// 每个tree如果是sub tree节点则使用递归合并成一组条件 // 每个tree如果是sub tree节点则使用递归合并成一组条件
@ -39,15 +38,16 @@ public class WhereTree2Str {
if (ObjectUtils.isEmpty(tree)) { if (ObjectUtils.isEmpty(tree)) {
continue; continue;
} }
String treeExp = transTreeToWhere(tableObj, tree, originFields); String treeExp = transTreeToWhere(tableObj, tree, originFields, fieldsDialect);
if (StringUtils.isNotEmpty(treeExp)) { if (StringUtils.isNotEmpty(treeExp)) {
res.add(treeExp); res.add(treeExp);
} }
} }
meta.setWhereTrees(CollectionUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null); meta.setWhereTrees(CollectionUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null);
meta.setWhereTreesDialect(fieldsDialect);
} }
private static String transTreeToWhere(SQLObj tableObj, DatasetRowPermissionsTreeObj tree, List<DatasetTableFieldDTO> originFields) { private static String transTreeToWhere(SQLObj tableObj, DatasetRowPermissionsTreeObj tree, List<DatasetTableFieldDTO> originFields, Map<String, String> fieldsDialect) {
if (ObjectUtils.isEmpty(tree)) { if (ObjectUtils.isEmpty(tree)) {
return null; return null;
} }
@ -60,10 +60,10 @@ public class WhereTree2Str {
String exp = null; String exp = null;
if (StringUtils.equalsIgnoreCase(item.getType(), "item")) { if (StringUtils.equalsIgnoreCase(item.getType(), "item")) {
// 单个item拼接SQL最后根据logic汇总 // 单个item拼接SQL最后根据logic汇总
exp = transTreeItem(tableObj, item, originFields); exp = transTreeItem(tableObj, item, originFields, fieldsDialect);
} else if (StringUtils.equalsIgnoreCase(item.getType(), "tree")) { } else if (StringUtils.equalsIgnoreCase(item.getType(), "tree")) {
// 递归tree // 递归tree
exp = transTreeToWhere(tableObj, item.getSubTree(), originFields); exp = transTreeToWhere(tableObj, item.getSubTree(), originFields, fieldsDialect);
} }
if (StringUtils.isNotEmpty(exp)) { if (StringUtils.isNotEmpty(exp)) {
list.add(exp); list.add(exp);
@ -73,7 +73,7 @@ public class WhereTree2Str {
return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + logic + " ", list) + ")" : null; return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + logic + " ", list) + ")" : null;
} }
public static String transTreeItem(SQLObj tableObj, DatasetRowPermissionsTreeItem item, List<DatasetTableFieldDTO> originFields) { public static String transTreeItem(SQLObj tableObj, DatasetRowPermissionsTreeItem item, List<DatasetTableFieldDTO> originFields, Map<String, String> fieldsDialect) {
String res = null; String res = null;
DatasetTableFieldDTO field = item.getField(); DatasetTableFieldDTO field = item.getField();
if (ObjectUtils.isEmpty(field)) { if (ObjectUtils.isEmpty(field)) {
@ -83,7 +83,10 @@ public class WhereTree2Str {
String originName; String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_CALC)) { if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originName = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields); String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields);
// 给计算字段处加一个占位符后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, calcFieldExp);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_COPY)) { } else if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_COPY)) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
} else { } else {

View File

@ -1,9 +1,11 @@
package io.dataease.engine.utils; package io.dataease.engine.utils;
import io.dataease.api.dataset.union.model.SQLObj; import io.dataease.api.dataset.union.model.SQLObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.exception.DEException; import io.dataease.exception.DEException;
import io.dataease.i18n.Translator; import io.dataease.i18n.Translator;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
@ -219,6 +221,15 @@ public class Utils {
return ObjectUtils.isNotEmpty(collect); return ObjectUtils.isNotEmpty(collect);
} }
public static boolean isCrossDs(Map<Long, DatasourceSchemaDTO> dsMap) {
return dsMap.size() != 1;
}
public static String replaceSchemaAlias(String sql, Map<Long, DatasourceSchemaDTO> dsMap) {
DatasourceSchemaDTO value = dsMap.entrySet().iterator().next().getValue();
return sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + value.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX + "\\.", "");
}
public static long allDateFormat2Long(String value) { public static long allDateFormat2Long(String value) {
String split = "-"; String split = "-";
if (value != null && value.contains("/")) { if (value != null && value.contains("/")) {

View File

@ -25,7 +25,7 @@
<mybatis-plus.version>3.5.3.1</mybatis-plus.version> <mybatis-plus.version>3.5.3.1</mybatis-plus.version>
<h2.version>2.2.220</h2.version> <h2.version>2.2.220</h2.version>
<knife4j.version>4.4.0</knife4j.version> <knife4j.version>4.4.0</knife4j.version>
<calcite-core.version>1.35.6</calcite-core.version> <calcite-core.version>1.35.7</calcite-core.version>
<commons-dbcp2.version>2.6.0</commons-dbcp2.version> <commons-dbcp2.version>2.6.0</commons-dbcp2.version>
<antlr.version>3.5.2</antlr.version> <antlr.version>3.5.2</antlr.version>
<java-jwt.version>3.12.1</java-jwt.version> <java-jwt.version>3.12.1</java-jwt.version>

View File

@ -3,6 +3,7 @@ package io.dataease.api.dataset.union.model;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* @Author Junjun * @Author Junjun
@ -12,14 +13,23 @@ public class SQLMeta {
private SQLObj table; private SQLObj table;
/**
* SQL片段占位符
*/
private String tableDialect;
private List<SQLObj> xFields; private List<SQLObj> xFields;
private Map<String, String> xFieldsDialect;
private List<String> xWheres; private List<String> xWheres;
private List<SQLObj> xOrders; private List<SQLObj> xOrders;
private List<SQLObj> yFields; private List<SQLObj> yFields;
private Map<String, String> yFieldsDialect;
private List<String> yWheres; private List<String> yWheres;
private List<SQLObj> yOrders; private List<SQLObj> yOrders;
@ -29,14 +39,20 @@ public class SQLMeta {
*/ */
private String customWheres; private String customWheres;
private Map<String, String> customWheresDialect;
/** /**
* 仪表板过滤字段 * 仪表板过滤字段
*/ */
private String extWheres; private String extWheres;
private Map<String, String> extWheresDialect;
/** /**
* 行权限过滤 * 行权限过滤
*/ */
private String whereTrees; private String whereTrees;
private Map<String, String> whereTreesDialect;
} }

View File

@ -5,39 +5,43 @@ import lombok.Data;
import java.util.List; import java.util.List;
@Data @Data
public class DatasourceConfiguration extends Configuration{ public class DatasourceConfiguration extends Configuration {
private List<String> illegalParameters; private List<String> illegalParameters;
private List<String> showTableSqls; private List<String> showTableSqls;
static public enum DatasourceType { static public enum DatasourceType {
folder("folder", "folder", "folder"), folder("folder", "folder", "folder", null, null),
API("API", "API", "API"), API("API", "API", "API", "`", "`"),
Excel("Excel", "Excel", "LOCALFILE"), Excel("Excel", "Excel", "LOCALFILE", "`", "`"),
mysql("mysql", "Mysql", "OLTP"), mysql("mysql", "Mysql", "OLTP", "`", "`"),
impala("impala", "Apache Impala", "OLAP"), impala("impala", "Apache Impala", "OLAP", "`", "`"),
mariadb("mariadb", "Mariadb", "OLTP"), mariadb("mariadb", "Mariadb", "OLTP", "`", "`"),
StarRocks("StarRocks", "StarRocks", "OLAP"), StarRocks("StarRocks", "StarRocks", "OLAP", "`", "`"),
doris("doris", "Apache Doris", "OLAP"), doris("doris", "Apache Doris", "OLAP", "`", "`"),
TiDB("TiDB", "TiDB", "OLTP"), TiDB("TiDB", "TiDB", "OLTP", "`", "`"),
oracle("oracle", "ORACLE", "OLTP"), oracle("oracle", "ORACLE", "OLTP", "\"", "\""),
pg("pg", "PostgreSQL", "OLTP"), pg("pg", "PostgreSQL", "OLTP", "\"", "\""),
redshift("redshift", "AWS Redshift", "OLTP"), redshift("redshift", "AWS Redshift", "OLTP", "\"", "\""),
db2("db2", "Db2", "OLTP"), db2("db2", "Db2", "OLTP", "", ""),
ck("ck", "Clickhouse", "OLAP"), ck("ck", "Clickhouse", "OLAP", "`", "`"),
h2("h2", "H2", "OLAP"), h2("h2", "H2", "OLAP", "\"", "\""),
sqlServer("sqlServer", "Sqlserver", "DL"), sqlServer("sqlServer", "Sqlserver", "DL", "[", "]"),
mongo("mongo", "MongoDB", "DL"); mongo("mongo", "MongoDB", "DL", "`", "`");
private String type; private String type;
private String name; private String name;
private String catalog; private String catalog;
private String prefix;
private String suffix;
DatasourceType(String type, String name, String catalog) { DatasourceType(String type, String name, String catalog, String prefix, String suffix) {
this.type = type; this.type = type;
this.name = name; this.name = name;
this.catalog = catalog; this.catalog = catalog;
this.prefix = prefix;
this.suffix = suffix;
} }
public String getType() { public String getType() {
@ -51,5 +55,13 @@ public class DatasourceConfiguration extends Configuration{
public String getCatalog() { public String getCatalog() {
return catalog; return catalog;
} }
public String getPrefix() {
return prefix;
}
public String getSuffix() {
return suffix;
}
} }
} }