chore: 数据源接口抽象

core/core-backend/src/main/java/io/dataease/chart/charts/impl/DefaultChartHandler.java

@@ -6,7 +6,6 @@ import io.dataease.chart.manage.ChartDataManage;
 import io.dataease.chart.manage.ChartViewManege;
 import io.dataease.chart.utils.ChartDataBuild;
 import io.dataease.dataset.manage.DatasetTableFieldManage;
-import io.dataease.dataset.utils.SqlUtils;
 import io.dataease.engine.constant.SQLConstants;
 import io.dataease.engine.sql.SQLProvider;
 import io.dataease.engine.trans.Dimension2SQLObj;
@@ -109,7 +108,7 @@ public class DefaultChartHandler extends AbstractChartPlugin {
         Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
         Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
         String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
-        querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
+        querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
         datasourceRequest.setQuery(querySql);
         logger.info("calcite chart sql: " + querySql);
         List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");

core/core-backend/src/main/java/io/dataease/chart/charts/impl/YoyChartHandler.java

@@ -2,7 +2,6 @@ package io.dataease.chart.charts.impl;
 
 
 import com.fasterxml.jackson.core.type.TypeReference;
-import io.dataease.dataset.utils.SqlUtils;
 import io.dataease.engine.sql.SQLProvider;
 import io.dataease.engine.trans.ExtWhere2Str;
 import io.dataease.engine.utils.Utils;
@@ -84,7 +83,7 @@ public class YoyChartHandler extends DefaultChartHandler {
             var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
             ExtWhere2Str.extWhere2sqlOjb(sqlMeta, originFilter, FieldUtil.transFields(allFields), crossDs, dsMap);
             var originSql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
-            originSql = SqlUtils.rebuildSQL(originSql, sqlMeta, crossDs, dsMap);
+            originSql = provider.rebuildSQL(originSql, sqlMeta, crossDs, dsMap);
             var request = new DatasourceRequest();
             request.setDsList(dsMap);
             request.setQuery(originSql);

core/core-backend/src/main/java/io/dataease/chart/charts/impl/map/SymbolicMapHandler.java

@@ -2,7 +2,6 @@ package io.dataease.chart.charts.impl.map;
 
 import io.dataease.chart.charts.impl.GroupChartHandler;
 import io.dataease.chart.utils.ChartDataBuild;
-import io.dataease.dataset.utils.SqlUtils;
 import io.dataease.engine.sql.SQLProvider;
 import io.dataease.engine.trans.Dimension2SQLObj;
 import io.dataease.engine.trans.Quota2SQLObj;
@@ -70,7 +69,7 @@ public class SymbolicMapHandler extends GroupChartHandler {
         yAxis.addAll(yFields);
         Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
         String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
-        querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
+        querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
         datasourceRequest.setQuery(querySql);
         logger.info("calcite chart sql: " + querySql);
         List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");
@@ -86,7 +85,7 @@ public class SymbolicMapHandler extends GroupChartHandler {
             String defaultResultMode = view.getResultMode();
             view.setResultMode("");
             String originSql = SQLProvider.createQuerySQL(sqlMeta1, false, needOrder, view);
-            originSql = SqlUtils.rebuildSQL(originSql, sqlMeta, crossDs, dsMap);
+            originSql = provider.rebuildSQL(originSql, sqlMeta, crossDs, dsMap);
             datasourceRequest.setQuery(originSql);
             logger.info("calcite detail field sql: " + querySql);
             detailData = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");

core/core-backend/src/main/java/io/dataease/chart/charts/impl/numeric/NumericalChartHandler.java

@@ -2,7 +2,6 @@ package io.dataease.chart.charts.impl.numeric;
 
 import io.dataease.chart.charts.impl.DefaultChartHandler;
 import io.dataease.chart.utils.ChartDataBuild;
-import io.dataease.dataset.utils.SqlUtils;
 import io.dataease.engine.sql.SQLProvider;
 import io.dataease.engine.trans.Quota2SQLObj;
 import io.dataease.engine.utils.Utils;
@@ -40,7 +39,7 @@ public class NumericalChartHandler extends DefaultChartHandler {
         var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
         Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
         String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
-        querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
+        querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
         datasourceRequest.setQuery(querySql);
         logger.info("calcite chart sql: " + querySql);
         List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");

core/core-backend/src/main/java/io/dataease/chart/charts/impl/table/TableInfoHandler.java

@@ -2,7 +2,6 @@ package io.dataease.chart.charts.impl.table;
 
 import io.dataease.api.chart.dto.PageInfo;
 import io.dataease.chart.charts.impl.DefaultChartHandler;
-import io.dataease.dataset.utils.SqlUtils;
 import io.dataease.engine.sql.SQLProvider;
 import io.dataease.engine.trans.Dimension2SQLObj;
 import io.dataease.engine.utils.Utils;
@@ -93,7 +92,7 @@ public class TableInfoHandler extends DefaultChartHandler {
         var tablePageMode = (String) filterResult.getContext().get("tablePageMode");
         var totalPageSql = "SELECT COUNT(*) FROM (" + SQLProvider.createQuerySQL(sqlMeta, false, false, view) + ") COUNT_TEMP";
         if (StringUtils.isNotEmpty(totalPageSql) && StringUtils.equalsIgnoreCase(tablePageMode, "page")) {
-            totalPageSql = SqlUtils.rebuildSQL(totalPageSql, sqlMeta, crossDs, dsMap);
+            totalPageSql = provider.rebuildSQL(totalPageSql, sqlMeta, crossDs, dsMap);
             datasourceRequest.setQuery(totalPageSql);
             datasourceRequest.setTotalPageFlag(true);
             logger.info("calcite total sql: " + totalPageSql);
@@ -107,7 +106,7 @@ public class TableInfoHandler extends DefaultChartHandler {
             view.setTotalPage(totalPage);
         }
 
-        querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
+        querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
         datasourceRequest.setQuery(querySql);
         logger.info("calcite chart sql: " + querySql);
         List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");

core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java

@@ -10,8 +10,6 @@ import io.dataease.dataset.manage.DatasetGroupManage;
 import io.dataease.dataset.manage.DatasetSQLManage;
 import io.dataease.dataset.manage.DatasetTableFieldManage;
 import io.dataease.dataset.manage.PermissionManage;
-import io.dataease.dataset.utils.SqlUtils;
-import io.dataease.datasource.provider.CalciteProvider;
 import io.dataease.engine.sql.SQLProvider;
 import io.dataease.engine.trans.*;
 import io.dataease.engine.utils.SQLUtils;
@@ -20,7 +18,9 @@ import io.dataease.exception.DEException;
 import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
 import io.dataease.extensions.datasource.dto.DatasourceRequest;
 import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
+import io.dataease.extensions.datasource.factory.ProviderFactory;
 import io.dataease.extensions.datasource.model.SQLMeta;
+import io.dataease.extensions.datasource.provider.Provider;
 import io.dataease.extensions.view.dto.*;
 import io.dataease.extensions.view.factory.PluginsChartFactory;
 import io.dataease.extensions.view.filter.FilterTreeObj;
@@ -56,8 +56,6 @@ public class ChartDataManage {
     @Resource
     private DatasetSQLManage datasetSQLManage;
     @Resource
-    private CalciteProvider calciteProvider;
-    @Resource
     private ChartViewManege chartViewManege;
     @Resource
     private PermissionManage permissionManage;
@@ -342,6 +340,13 @@ public class ChartDataManage {
             }
         }
 
+        Provider provider;
+        if (crossDs) {
+            provider = ProviderFactory.getDefaultProvider();
+        } else {
+            provider = ProviderFactory.getProvider(dsMap.entrySet().iterator().next().getValue().getType());
+        }
+
         SQLMeta sqlMeta = new SQLMeta();
         Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
         CustomWhere2Str.customWhere2sqlObj(sqlMeta, fieldCustomFilter, transFields(allFields), crossDs, dsMap);
@@ -357,10 +362,11 @@ public class ChartDataManage {
             Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
             Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap);
             String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
-            querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
+            querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
             filterResult.getContext().put("querySql", querySql);
         }
-        ChartCalcDataResult calcResult = chartHandler.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, calciteProvider);
+
+        ChartCalcDataResult calcResult = chartHandler.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
         return chartHandler.buildChart(view, calcResult, formatResult, filterResult);
     }
 
@@ -661,6 +667,13 @@ public class ChartDataManage {
         DatasourceRequest datasourceRequest = new DatasourceRequest();
         datasourceRequest.setDsList(dsMap);
 
+        Provider provider;
+        if (crossDs) {
+            provider = ProviderFactory.getDefaultProvider();
+        } else {
+            provider = ProviderFactory.getProvider(dsMap.entrySet().iterator().next().getValue().getType());
+        }
+
         List<String[]> data = new ArrayList<>();
 
         String querySql = null;
@@ -706,11 +719,11 @@ public class ChartDataManage {
                 querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
             }
 
-            querySql = SqlUtils.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
+            querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
             datasourceRequest.setQuery(querySql);
             logger.info("calcite chart get field enum sql: " + querySql);
 
-            data = (List<String[]>) calciteProvider.fetchResultField(datasourceRequest).get("data");
+            data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");
         }
         return data;
     }

core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetTableFieldManage.java

@@ -2,10 +2,8 @@ package io.dataease.dataset.manage;
 
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import io.dataease.dataset.dao.auto.entity.CoreDatasetTableField;
-import io.dataease.dataset.dao.auto.mapper.CoreDatasetGroupMapper;
 import io.dataease.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper;
 import io.dataease.dataset.utils.TableUtils;
-import io.dataease.datasource.provider.CalciteProvider;
 import io.dataease.engine.constant.ExtFieldConstant;
 import io.dataease.engine.func.FunctionConstant;
 import io.dataease.engine.utils.Utils;
@@ -37,10 +35,6 @@ public class DatasetTableFieldManage {
     private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper;
     @Resource
     private PermissionManage permissionManage;
-    @Resource
-    private CoreDatasetGroupMapper coreDatasetGroupMapper;
-    @Resource
-    private CalciteProvider calciteProvider;
 
     public void save(CoreDatasetTableField coreDatasetTableField) {
         checkNameLength(coreDatasetTableField.getName());

core/core-backend/src/main/java/io/dataease/dataset/utils/SqlUtils.java

@@ -2,23 +2,14 @@ package io.dataease.dataset.utils;
 
 import com.google.common.collect.ImmutableList;
 import io.dataease.exception.DEException;
-import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
-import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
-import io.dataease.extensions.datasource.model.SQLMeta;
-import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
 import org.apache.calcite.config.Lex;
 import org.apache.calcite.sql.*;
-import org.apache.calcite.sql.dialect.*;
 import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.calcite.sql.parser.SqlParser;
-import org.apache.commons.collections4.MapUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
 
 import static org.apache.calcite.sql.SqlKind.*;
 
@@ -121,107 +112,4 @@ public class SqlUtils {
         }
     }
 
-    public static String rebuildSQL(String sql, SQLMeta sqlMeta, boolean crossDs, Map<Long, DatasourceSchemaDTO> dsMap) {
-        logger.info("calcite sql: " + sql);
-        if (crossDs) {
-            return sql;
-        }
-
-        String s = transSqlDialect(sql, dsMap);
-        String tableDialect = sqlMeta.getTableDialect();
-        s = replaceTablePlaceHolder(s, tableDialect);
-        return replaceCalcFieldPlaceHolder(s, sqlMeta);
-    }
-
-    public static String transSqlDialect(String sql, Map<Long, DatasourceSchemaDTO> dsMap) throws DEException {
-        try {
-            DatasourceSchemaDTO value = dsMap.entrySet().iterator().next().getValue();
-
-            SqlParser parser = SqlParser.create(sql, SqlParser.Config.DEFAULT.withLex(Lex.JAVA));
-            SqlNode sqlNode = parser.parseStmt();
-            return sqlNode.toSqlString(getDialect(value)).toString();
-        } catch (Exception e) {
-            DEException.throwException(e.getMessage());
-        }
-        return null;
-    }
-
-    public static String replaceTablePlaceHolder(String s, String placeholder) {
-        s = s.replaceAll("\r\n", " ")
-                .replaceAll("\n", " ")
-                .replaceAll(SqlPlaceholderConstants.TABLE_PLACEHOLDER_REGEX, Matcher.quoteReplacement(placeholder))
-                .replaceAll("ASYMMETRIC", "")
-                .replaceAll("SYMMETRIC", "");
-        return s;
-    }
-
-    public static String replaceCalcFieldPlaceHolder(String s, SQLMeta sqlMeta) {
-        Map<String, String> fieldsDialect = new HashMap<>();
-        if (MapUtils.isNotEmpty(sqlMeta.getXFieldsDialect())) {
-            fieldsDialect.putAll(sqlMeta.getXFieldsDialect());
-        }
-        if (MapUtils.isNotEmpty(sqlMeta.getYFieldsDialect())) {
-            fieldsDialect.putAll(sqlMeta.getYFieldsDialect());
-        }
-        if (MapUtils.isNotEmpty(sqlMeta.getCustomWheresDialect())) {
-            fieldsDialect.putAll(sqlMeta.getCustomWheresDialect());
-        }
-        if (MapUtils.isNotEmpty(sqlMeta.getExtWheresDialect())) {
-            fieldsDialect.putAll(sqlMeta.getExtWheresDialect());
-        }
-        if (MapUtils.isNotEmpty(sqlMeta.getWhereTreesDialect())) {
-            fieldsDialect.putAll(sqlMeta.getWhereTreesDialect());
-        }
-
-        if (MapUtils.isNotEmpty(fieldsDialect)) {
-            for (Map.Entry<String, String> ele : fieldsDialect.entrySet()) {
-                s = s.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + ele.getKey() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, Matcher.quoteReplacement(ele.getValue()));
-            }
-        }
-        return s;
-    }
-
-    private static SqlDialect getDialect(DatasourceSchemaDTO coreDatasource) {
-        SqlDialect sqlDialect = null;
-        DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(coreDatasource.getType());
-        switch (datasourceType) {
-            case mysql:
-            case mongo:
-            case StarRocks:
-            case TiDB:
-            case mariadb:
-                sqlDialect = MysqlSqlDialect.DEFAULT;
-                break;
-            case doris:
-                sqlDialect = DorisSqlDialect.DEFAULT;
-                break;
-            case impala:
-                sqlDialect = ImpalaSqlDialect.DEFAULT;
-                break;
-            case sqlServer:
-                sqlDialect = MssqlSqlDialect.DEFAULT;
-                break;
-            case oracle:
-                sqlDialect = OracleSqlDialect.DEFAULT;
-                break;
-            case db2:
-                sqlDialect = Db2SqlDialect.DEFAULT;
-                break;
-            case pg:
-                sqlDialect = PostgresqlSqlDialect.DEFAULT;
-                break;
-            case redshift:
-                sqlDialect = RedshiftSqlDialect.DEFAULT;
-                break;
-            case ck:
-                sqlDialect = ClickHouseSqlDialect.DEFAULT;
-                break;
-            case h2:
-                sqlDialect = H2SqlDialect.DEFAULT;
-                break;
-            default:
-                sqlDialect = MysqlSqlDialect.DEFAULT;
-        }
-        return sqlDialect;
-    }
 }

core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java

@@ -6,13 +6,14 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.dataease.api.dataset.dto.PreviewSqlDTO;
 import io.dataease.api.ds.DatasourceApi;
-import io.dataease.api.ds.vo.*;
+import io.dataease.api.ds.vo.ApiDefinition;
+import io.dataease.api.ds.vo.CoreDatasourceTaskLogDTO;
+import io.dataease.api.ds.vo.ExcelFileData;
+import io.dataease.api.ds.vo.ExcelSheetData;
 import io.dataease.commons.constants.TaskStatus;
 import io.dataease.commons.utils.CommonThreadPool;
-import io.dataease.constant.DataSourceType;
 import io.dataease.constant.LogOT;
 import io.dataease.constant.LogST;
 import io.dataease.dataset.manage.DatasetDataManage;
@@ -32,6 +33,8 @@ import io.dataease.datasource.provider.ExcelUtils;
 import io.dataease.engine.constant.SQLConstants;
 import io.dataease.exception.DEException;
 import io.dataease.extensions.datasource.dto.*;
+import io.dataease.extensions.datasource.factory.ProviderFactory;
+import io.dataease.extensions.datasource.provider.Provider;
 import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
 import io.dataease.i18n.Translator;
 import io.dataease.job.schedule.CheckDsStatusJob;
@@ -492,7 +495,8 @@ public class DatasourceServer implements DatasourceApi {
         BeanUtils.copyBean(coreDatasource, dataSourceDTO);
         DatasourceRequest datasourceRequest = new DatasourceRequest();
         datasourceRequest.setDatasource(dataSourceDTO);
-        return calciteProvider.getSchema(datasourceRequest);
+        Provider provider = ProviderFactory.getProvider(dataSourceDTO.getType());
+        return provider.getSchema(datasourceRequest);
     }
 
     @Override
@@ -724,7 +728,8 @@ public class DatasourceServer implements DatasourceApi {
         if (coreDatasource.getType().equals("Excel")) {
             return ExcelUtils.getTables(datasourceRequest);
         }
-        return calciteProvider.getTables(datasourceRequest);
+        Provider provider = ProviderFactory.getProvider(datasourceDTO.getType());
+        return provider.getTables(datasourceRequest);
     }
 
     @Override
@@ -742,7 +747,8 @@ public class DatasourceServer implements DatasourceApi {
             datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
             datasourceRequest.setQuery(TableUtils.tableName2Sql(datasourceSchemaDTO, tableName) + " LIMIT 0 OFFSET 0");
             datasourceRequest.setTable(tableName);
-            List<TableField> tableFields = (List<TableField>) calciteProvider.fetchTableField(datasourceRequest);
+            Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType());
+            List<TableField> tableFields = (List<TableField>) provider.fetchTableField(datasourceRequest);
             return tableFields.stream().filter(tableField -> {
                 return !tableField.getOriginName().equalsIgnoreCase("dataease_uuid");
             }).collect(Collectors.toList());
@@ -754,7 +760,8 @@ public class DatasourceServer implements DatasourceApi {
         datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
         datasourceRequest.setQuery(TableUtils.tableName2Sql(datasourceSchemaDTO, tableName) + " LIMIT 0 OFFSET 0");
         datasourceRequest.setTable(tableName);
-        return (List<TableField>) calciteProvider.fetchTableField(datasourceRequest);
+        Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType());
+        return (List<TableField>) provider.fetchTableField(datasourceRequest);
     }
 
     @Override