center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

chore: 数据源接口抽象

Browse Source
This commit is contained in:
junjun 2024-07-03 14:07:24 +08:00
parent aa53d6ea8f
commit bc582d773f
9 changed files with 42 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/core-backend/src/main/java/io/dataease/chart/charts/impl/DefaultChartHandler.java
View File

@ -6,7 +6,6 @@ import io.dataease.chart.manage.ChartDataManage;
import io.dataease.chart.manage.ChartViewManege;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.dataset.manage.DatasetTableFieldManage;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Dimension2SQLObj;
@ -109,7 +108,7 @@ public class DefaultChartHandler extends AbstractChartPlugin {
Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql);
logger.info("calcite chart sql: " + querySql);
List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");

3
core/core-backend/src/main/java/io/dataease/chart/charts/impl/YoyChartHandler.java
View File

@ -2,7 +2,6 @@ package io.dataease.chart.charts.impl;
import com.fasterxml.jackson.core.type.TypeReference;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.ExtWhere2Str;
import io.dataease.engine.utils.Utils;
@ -84,7 +83,7 @@ public class YoyChartHandler extends DefaultChartHandler {
var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
ExtWhere2Str.extWhere2sqlOjb(sqlMeta, originFilter, FieldUtil.transFields(allFields), crossDs, dsMap);
var originSql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
originSql = SqlUtils.rebuildSQL(originSql, sqlMeta, crossDs, dsMap);
originSql = provider.rebuildSQL(originSql, sqlMeta, crossDs, dsMap);
var request = new DatasourceRequest();
request.setDsList(dsMap);
request.setQuery(originSql);

5
core/core-backend/src/main/java/io/dataease/chart/charts/impl/map/SymbolicMapHandler.java
View File

@ -2,7 +2,6 @@ package io.dataease.chart.charts.impl.map;
import io.dataease.chart.charts.impl.GroupChartHandler;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Dimension2SQLObj;
import io.dataease.engine.trans.Quota2SQLObj;
@ -70,7 +69,7 @@ public class SymbolicMapHandler extends GroupChartHandler {
yAxis.addAll(yFields);
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql);
logger.info("calcite chart sql: " + querySql);
List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");
@ -86,7 +85,7 @@ public class SymbolicMapHandler extends GroupChartHandler {
String defaultResultMode = view.getResultMode();
view.setResultMode("");
String originSql = SQLProvider.createQuerySQL(sqlMeta1, false, needOrder, view);
originSql = SqlUtils.rebuildSQL(originSql, sqlMeta, crossDs, dsMap);
originSql = provider.rebuildSQL(originSql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(originSql);
logger.info("calcite detail field sql: " + querySql);
detailData = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");

3
core/core-backend/src/main/java/io/dataease/chart/charts/impl/numeric/NumericalChartHandler.java
View File

@ -2,7 +2,6 @@ package io.dataease.chart.charts.impl.numeric;
import io.dataease.chart.charts.impl.DefaultChartHandler;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Quota2SQLObj;
import io.dataease.engine.utils.Utils;
@ -40,7 +39,7 @@ public class NumericalChartHandler extends DefaultChartHandler {
var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql);
logger.info("calcite chart sql: " + querySql);
List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");

5
core/core-backend/src/main/java/io/dataease/chart/charts/impl/table/TableInfoHandler.java
View File

@ -2,7 +2,6 @@ package io.dataease.chart.charts.impl.table;
import io.dataease.api.chart.dto.PageInfo;
import io.dataease.chart.charts.impl.DefaultChartHandler;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Dimension2SQLObj;
import io.dataease.engine.utils.Utils;
@ -93,7 +92,7 @@ public class TableInfoHandler extends DefaultChartHandler {
var tablePageMode = (String) filterResult.getContext().get("tablePageMode");
var totalPageSql = "SELECT COUNT(*) FROM (" + SQLProvider.createQuerySQL(sqlMeta, false, false, view) + ") COUNT_TEMP";
if (StringUtils.isNotEmpty(totalPageSql) && StringUtils.equalsIgnoreCase(tablePageMode, "page")) {
totalPageSql = SqlUtils.rebuildSQL(totalPageSql, sqlMeta, crossDs, dsMap);
totalPageSql = provider.rebuildSQL(totalPageSql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(totalPageSql);
datasourceRequest.setTotalPageFlag(true);
logger.info("calcite total sql: " + totalPageSql);
@ -107,7 +106,7 @@ public class TableInfoHandler extends DefaultChartHandler {
view.setTotalPage(totalPage);
}
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql);
logger.info("calcite chart sql: " + querySql);
List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");

29
core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java
View File

@ -10,8 +10,6 @@ import io.dataease.dataset.manage.DatasetGroupManage;
import io.dataease.dataset.manage.DatasetSQLManage;
import io.dataease.dataset.manage.DatasetTableFieldManage;
import io.dataease.dataset.manage.PermissionManage;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.*;
import io.dataease.engine.utils.SQLUtils;
@ -20,7 +18,9 @@ import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.factory.ProviderFactory;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.factory.PluginsChartFactory;
import io.dataease.extensions.view.filter.FilterTreeObj;
@ -56,8 +56,6 @@ public class ChartDataManage {
@Resource
private DatasetSQLManage datasetSQLManage;
@Resource
private CalciteProvider calciteProvider;
@Resource
private ChartViewManege chartViewManege;
@Resource
private PermissionManage permissionManage;
@ -342,6 +340,13 @@ public class ChartDataManage {
}
}
Provider provider;
if (crossDs) {
provider = ProviderFactory.getDefaultProvider();
} else {
provider = ProviderFactory.getProvider(dsMap.entrySet().iterator().next().getValue().getType());
}
SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
CustomWhere2Str.customWhere2sqlObj(sqlMeta, fieldCustomFilter, transFields(allFields), crossDs, dsMap);
@ -357,10 +362,11 @@ public class ChartDataManage {
Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
filterResult.getContext().put("querySql", querySql);
}
ChartCalcDataResult calcResult = chartHandler.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, calciteProvider);
ChartCalcDataResult calcResult = chartHandler.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
return chartHandler.buildChart(view, calcResult, formatResult, filterResult);
}
@ -661,6 +667,13 @@ public class ChartDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(dsMap);
Provider provider;
if (crossDs) {
provider = ProviderFactory.getDefaultProvider();
} else {
provider = ProviderFactory.getProvider(dsMap.entrySet().iterator().next().getValue().getType());
}
List<String[]> data = new ArrayList<>();
String querySql = null;
@ -706,11 +719,11 @@ public class ChartDataManage {
querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
}
querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql);
logger.info("calcite chart get field enum sql: " + querySql);
data = (List<String[]>) calciteProvider.fetchResultField(datasourceRequest).get("data");
data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");
}
return data;
}

6
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetTableFieldManage.java
View File

@ -2,10 +2,8 @@ package io.dataease.dataset.manage;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import io.dataease.dataset.dao.auto.entity.CoreDatasetTableField;
import io.dataease.dataset.dao.auto.mapper.CoreDatasetGroupMapper;
import io.dataease.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper;
import io.dataease.dataset.utils.TableUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.func.FunctionConstant;
import io.dataease.engine.utils.Utils;
@ -37,10 +35,6 @@ public class DatasetTableFieldManage {
private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper;
@Resource
private PermissionManage permissionManage;
@Resource
private CoreDatasetGroupMapper coreDatasetGroupMapper;
@Resource
private CalciteProvider calciteProvider;
public void save(CoreDatasetTableField coreDatasetTableField) {
checkNameLength(coreDatasetTableField.getName());

112
core/core-backend/src/main/java/io/dataease/dataset/utils/SqlUtils.java
View File

@ -2,23 +2,14 @@ package io.dataease.dataset.utils;
import com.google.common.collect.ImmutableList;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import org.apache.calcite.config.Lex;
import org.apache.calcite.sql.*;
import org.apache.calcite.sql.dialect.*;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.commons.collections4.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import static org.apache.calcite.sql.SqlKind.*;
@ -121,107 +112,4 @@ public class SqlUtils {
}
}
public static String rebuildSQL(String sql, SQLMeta sqlMeta, boolean crossDs, Map<Long, DatasourceSchemaDTO> dsMap) {
logger.info("calcite sql: " + sql);
if (crossDs) {
return sql;
}
String s = transSqlDialect(sql, dsMap);
String tableDialect = sqlMeta.getTableDialect();
s = replaceTablePlaceHolder(s, tableDialect);
return replaceCalcFieldPlaceHolder(s, sqlMeta);
}
public static String transSqlDialect(String sql, Map<Long, DatasourceSchemaDTO> dsMap) throws DEException {
try {
DatasourceSchemaDTO value = dsMap.entrySet().iterator().next().getValue();
SqlParser parser = SqlParser.create(sql, SqlParser.Config.DEFAULT.withLex(Lex.JAVA));
SqlNode sqlNode = parser.parseStmt();
return sqlNode.toSqlString(getDialect(value)).toString();
} catch (Exception e) {
DEException.throwException(e.getMessage());
}
return null;
}
public static String replaceTablePlaceHolder(String s, String placeholder) {
s = s.replaceAll("\r\n", " ")
.replaceAll("\n", " ")
.replaceAll(SqlPlaceholderConstants.TABLE_PLACEHOLDER_REGEX, Matcher.quoteReplacement(placeholder))
.replaceAll("ASYMMETRIC", "")
.replaceAll("SYMMETRIC", "");
return s;
}
public static String replaceCalcFieldPlaceHolder(String s, SQLMeta sqlMeta) {
Map<String, String> fieldsDialect = new HashMap<>();
if (MapUtils.isNotEmpty(sqlMeta.getXFieldsDialect())) {
fieldsDialect.putAll(sqlMeta.getXFieldsDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getYFieldsDialect())) {
fieldsDialect.putAll(sqlMeta.getYFieldsDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getCustomWheresDialect())) {
fieldsDialect.putAll(sqlMeta.getCustomWheresDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getExtWheresDialect())) {
fieldsDialect.putAll(sqlMeta.getExtWheresDialect());
}
if (MapUtils.isNotEmpty(sqlMeta.getWhereTreesDialect())) {
fieldsDialect.putAll(sqlMeta.getWhereTreesDialect());
}
if (MapUtils.isNotEmpty(fieldsDialect)) {
for (Map.Entry<String, String> ele : fieldsDialect.entrySet()) {
s = s.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + ele.getKey() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, Matcher.quoteReplacement(ele.getValue()));
}
}
return s;
}
private static SqlDialect getDialect(DatasourceSchemaDTO coreDatasource) {
SqlDialect sqlDialect = null;
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(coreDatasource.getType());
switch (datasourceType) {
case mysql:
case mongo:
case StarRocks:
case TiDB:
case mariadb:
sqlDialect = MysqlSqlDialect.DEFAULT;
break;
case doris:
sqlDialect = DorisSqlDialect.DEFAULT;
break;
case impala:
sqlDialect = ImpalaSqlDialect.DEFAULT;
break;
case sqlServer:
sqlDialect = MssqlSqlDialect.DEFAULT;
break;
case oracle:
sqlDialect = OracleSqlDialect.DEFAULT;
break;
case db2:
sqlDialect = Db2SqlDialect.DEFAULT;
break;
case pg:
sqlDialect = PostgresqlSqlDialect.DEFAULT;
break;
case redshift:
sqlDialect = RedshiftSqlDialect.DEFAULT;
break;
case ck:
sqlDialect = ClickHouseSqlDialect.DEFAULT;
break;
case h2:
sqlDialect = H2SqlDialect.DEFAULT;
break;
default:
sqlDialect = MysqlSqlDialect.DEFAULT;
}
return sqlDialect;
}
}

21
core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java
View File

@ -6,13 +6,14 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.dataease.api.dataset.dto.PreviewSqlDTO;
import io.dataease.api.ds.DatasourceApi;
import io.dataease.api.ds.vo.*;
import io.dataease.api.ds.vo.ApiDefinition;
import io.dataease.api.ds.vo.CoreDatasourceTaskLogDTO;
import io.dataease.api.ds.vo.ExcelFileData;
import io.dataease.api.ds.vo.ExcelSheetData;
import io.dataease.commons.constants.TaskStatus;
import io.dataease.commons.utils.CommonThreadPool;
import io.dataease.constant.DataSourceType;
import io.dataease.constant.LogOT;
import io.dataease.constant.LogST;
import io.dataease.dataset.manage.DatasetDataManage;
@ -32,6 +33,8 @@ import io.dataease.datasource.provider.ExcelUtils;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.*;
import io.dataease.extensions.datasource.factory.ProviderFactory;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import io.dataease.i18n.Translator;
import io.dataease.job.schedule.CheckDsStatusJob;
@ -492,7 +495,8 @@ public class DatasourceServer implements DatasourceApi {
BeanUtils.copyBean(coreDatasource, dataSourceDTO);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dataSourceDTO);
return calciteProvider.getSchema(datasourceRequest);
Provider provider = ProviderFactory.getProvider(dataSourceDTO.getType());
return provider.getSchema(datasourceRequest);
}
@Override
@ -724,7 +728,8 @@ public class DatasourceServer implements DatasourceApi {
if (coreDatasource.getType().equals("Excel")) {
return ExcelUtils.getTables(datasourceRequest);
}
return calciteProvider.getTables(datasourceRequest);
Provider provider = ProviderFactory.getProvider(datasourceDTO.getType());
return provider.getTables(datasourceRequest);
}
@Override
@ -742,7 +747,8 @@ public class DatasourceServer implements DatasourceApi {
datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
datasourceRequest.setQuery(TableUtils.tableName2Sql(datasourceSchemaDTO, tableName) + " LIMIT 0 OFFSET 0");
datasourceRequest.setTable(tableName);
List<TableField> tableFields = (List<TableField>) calciteProvider.fetchTableField(datasourceRequest);
Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType());
List<TableField> tableFields = (List<TableField>) provider.fetchTableField(datasourceRequest);
return tableFields.stream().filter(tableField -> {
return !tableField.getOriginName().equalsIgnoreCase("dataease_uuid");
}).collect(Collectors.toList());
@ -754,7 +760,8 @@ public class DatasourceServer implements DatasourceApi {
datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
datasourceRequest.setQuery(TableUtils.tableName2Sql(datasourceSchemaDTO, tableName) + " LIMIT 0 OFFSET 0");
datasourceRequest.setTable(tableName);
return (List<TableField>) calciteProvider.fetchTableField(datasourceRequest);
Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType());
return (List<TableField>) provider.fetchTableField(datasourceRequest);
}
@Override