diff --git a/backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java b/backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java index 281466dbbd..1e279482d7 100644 --- a/backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java +++ b/backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java @@ -7,6 +7,7 @@ import io.dataease.auth.annotation.DeLog; import io.dataease.auth.annotation.DePermission; import io.dataease.commons.constants.DePermissionType; import io.dataease.commons.constants.ResourceAuthLevel; +import io.dataease.commons.constants.SysAuthConstants; import io.dataease.commons.constants.SysLogConstants; import io.dataease.commons.utils.AuthUtils; import io.dataease.commons.utils.DeLogUtils; @@ -18,10 +19,15 @@ import io.dataease.dto.DatasourceDTO; import io.dataease.dto.SysLogDTO; import io.dataease.dto.datasource.DBTableDTO; import io.dataease.plugins.common.base.domain.Datasource; +import io.dataease.plugins.common.base.domain.DatasourceExample; +import io.dataease.plugins.common.dto.datasource.DataSourceType; +import io.dataease.plugins.datasource.provider.Provider; +import io.dataease.provider.ProviderFactory; import io.dataease.service.datasource.DatasourceService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.shiro.authz.annotation.RequiresPermissions; import org.springframework.web.bind.annotation.*; import springfox.documentation.annotations.ApiIgnore; @@ -30,6 +36,7 @@ import javax.annotation.Resource; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.stream.Collectors; @Api(tags = "数据源：数据源管理") @@ -65,8 +72,21 @@ public class DatasourceController { positionIndex = 0, positionKey = "type", value = "id" ) - public void updateDatasource(@RequestBody UpdataDsRequest dsRequest) throws Exception { - datasourceService.updateDatasource(dsRequest); + public void updateDatasource(@RequestBody UpdataDsRequest updataDsRequest) throws Exception { + DatasourceDTO datasource = new DatasourceDTO(); + datasource.setConfigurationEncryption(updataDsRequest.isConfigurationEncryption()); + datasource.setName(updataDsRequest.getName()); + datasource.setDesc(updataDsRequest.getDesc()); + datasource.setConfiguration(updataDsRequest.getConfiguration()); + datasource.setCreateTime(null); + datasource.setType(updataDsRequest.getType()); + datasource.setUpdateTime(System.currentTimeMillis()); + datasourceService.preCheckDs(datasource); + if (StringUtils.isNotEmpty(updataDsRequest.getId())) { + datasourceService.updateDatasource(updataDsRequest.getId(), datasource); + } else { + datasourceService.insert(datasource); + } } @RequiresPermissions("datasource:read") diff --git a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java index cb6d9dae10..71c0b6ba97 100644 --- a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java @@ -824,7 +824,7 @@ public class CKQueryProvider extends QueryProvider { if(isTable){ return "SELECT COUNT(*) from " + String.format(CKConstants.KEYWORD_TABLE, sql); }else { - return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + return "SELECT COUNT(*) from ( " + sqlFix(sql) + " ) DE_COUNT_TEMP"; } } diff --git a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java index f68b796c11..a0940c08ab 100644 --- a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java @@ -810,12 +810,12 @@ public class Db2QueryProvider extends QueryProvider { public String getTotalCount(boolean isTable, String sql, Datasource ds) { - if(isTable){ + if (isTable) { String schema = new Gson().fromJson(ds.getConfiguration(), JdbcConfiguration.class).getSchema(); schema = String.format(Db2Constants.KEYWORD_TABLE, schema); return "SELECT COUNT(*) from " + schema + "." + String.format(Db2Constants.KEYWORD_TABLE, sql); - }else { - return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + } else { + return "SELECT COUNT(*) from ( " + sqlFix(sql) + " ) DE_COUNT_TEMP"; } } @@ -828,7 +828,7 @@ public class Db2QueryProvider extends QueryProvider { }).toArray(String[]::new); if (ds != null) { Db2Configuration db2Configuration = new Gson().fromJson(ds.getConfiguration(), Db2Configuration.class); - return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_OFFSET, DE_PAGE_SIZE ", StringUtils.join(array, ","), db2Configuration.getSchema() + String.format(Db2Constants.KEYWORD_TABLE, table)); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_OFFSET, DE_PAGE_SIZE ", StringUtils.join(array, ","), String.format(Db2Constants.KEYWORD_TABLE, db2Configuration.getSchema()) + "." + String.format(Db2Constants.KEYWORD_TABLE, table)); } else { return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_OFFSET, DE_PAGE_SIZE ", StringUtils.join(array, ","), table); } diff --git a/backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java index 9ac6b663de..3087ca2f89 100644 --- a/backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java @@ -1,15 +1,19 @@ package io.dataease.provider.query.doris; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; +import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.Datasource; +import io.dataease.plugins.common.constants.datasource.MySQLConstants; import io.dataease.plugins.common.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.plugins.common.dto.chart.ChartViewFieldDTO; import io.dataease.plugins.common.request.chart.ChartExtFilterRequest; import io.dataease.plugins.common.request.permission.DataSetRowPermissionsTreeDTO; import io.dataease.plugins.datasource.entity.PageInfo; import io.dataease.provider.query.mysql.MysqlQueryProvider; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; +import java.text.MessageFormat; import java.util.List; @@ -17,9 +21,9 @@ import java.util.List; public class DorisQueryProvider extends MysqlQueryProvider { public String getSQLWithPage(boolean isTable, String sql, List xAxis, List fieldCustomFilter, List rowPermissionsTree, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view, PageInfo pageInfo) { - if(isTable){ + if (isTable) { return getSQLTableInfo(sql, xAxis, fieldCustomFilter, rowPermissionsTree, extFilterRequestList, ds, view); - }else { + } else { return getSQLAsTmpTableInfo(sql, xAxis, fieldCustomFilter, rowPermissionsTree, extFilterRequestList, ds, view); } } @@ -28,4 +32,23 @@ public class DorisQueryProvider extends MysqlQueryProvider { return null; } + @Override + public String createRawQuerySQL(String table, List fields, Datasource ds) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + if (f.getDeExtractType() == 4) { // 处理 tinyint + stringBuilder.append("concat(`").append(f.getOriginName()).append("`,'') AS ").append(f.getDataeaseName()); + } else if (f.getDeExtractType() == 1 && f.getType().equalsIgnoreCase("YEAR")) { // 处理 YEAR + stringBuilder.append("").append(String.format(MySQLConstants.DATE_FORMAT, "CONCAT(" + f.getOriginName() + ",'-01-01')", MySQLConstants.DEFAULT_DATE_FORMAT)).append(" AS ").append(f.getDataeaseName()); + } else { + stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); + } + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), table); + } + + public String getTotalCount(boolean isTable, String sql, Datasource ds) { + return null; + } } diff --git a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java index 77ecd69bab..f5e1ed62a0 100644 --- a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java @@ -774,7 +774,7 @@ public class HiveQueryProvider extends QueryProvider { if(isTable){ return "SELECT COUNT(*) from " + String.format(HiveConstants.KEYWORD_TABLE, sql); }else { - return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + return "SELECT COUNT(*) from ( " + sqlFix(sql) + " ) DE_COUNT_TEMP"; } } diff --git a/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java index 7229e15ee5..fbdcfb0a8a 100644 --- a/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java @@ -772,7 +772,7 @@ public class ImpalaQueryProvider extends QueryProvider { if(isTable){ return "SELECT COUNT(*) from " + String.format(ImpalaConstants.KEYWORD_TABLE, sql); }else { - return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + return "SELECT COUNT(*) from ( " + sqlFix(sql) + " ) DE_COUNT_TEMP"; } } diff --git a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java index 2b8d8ff350..d7bc42a7f2 100644 --- a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java @@ -815,7 +815,7 @@ public class MysqlQueryProvider extends QueryProvider { if(isTable){ return "SELECT COUNT(*) from " + String.format(MySQLConstants.KEYWORD_TABLE, sql); }else { - return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + return "SELECT COUNT(*) from ( " + sqlFix(sql) + " ) DE_COUNT_TEMP"; } } @@ -1123,6 +1123,7 @@ public class MysqlQueryProvider extends QueryProvider { } private String sqlFix(String sql) { + sql = sql.trim(); if (sql.lastIndexOf(";") == (sql.length() - 1)) { sql = sql.substring(0, sql.length() - 1); } diff --git a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java index 11c5fd238b..e5dada23c6 100644 --- a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java @@ -866,7 +866,7 @@ public class OracleQueryProvider extends QueryProvider { schema = String.format(OracleConstants.KEYWORD_TABLE, schema); return "SELECT COUNT(*) from " + schema + "." + String.format(OracleConstants.KEYWORD_TABLE, sql); } else { - return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + return "SELECT COUNT(*) from ( " + sqlFix(sql) + " ) DE_COUNT_TEMP"; } } diff --git a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java index a3d9e9fe43..6acb16b903 100644 --- a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java @@ -806,7 +806,7 @@ public class PgQueryProvider extends QueryProvider { String tableWithSchema = String.format(PgConstants.KEYWORD_TABLE, schema) + "." + String.format(PgConstants.KEYWORD_TABLE, sql); return "SELECT COUNT(*) from " + tableWithSchema; }else { - return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + return "SELECT COUNT(*) from ( " + sqlFix(sql) + " ) DE_COUNT_TEMP"; } } diff --git a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java index d0e2d0b549..61aec7505f 100644 --- a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java @@ -813,7 +813,7 @@ public class RedshiftQueryProvider extends QueryProvider { String tableWithSchema = String.format(SqlServerSQLConstants.KEYWORD_TABLE, schema) + "." + String.format(SqlServerSQLConstants.KEYWORD_TABLE, sql); return "SELECT COUNT(*) from " + String.format(ImpalaConstants.KEYWORD_TABLE, tableWithSchema); }else { - return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + return "SELECT COUNT(*) from ( " + sqlFix(sql) + " ) DE_COUNT_TEMP"; } } @@ -826,7 +826,7 @@ public class RedshiftQueryProvider extends QueryProvider { }).toArray(String[]::new); if (ds != null) { String schema = new Gson().fromJson(ds.getConfiguration(), JdbcConfiguration.class).getSchema(); - String tableWithSchema = String.format(SqlServerSQLConstants.KEYWORD_TABLE, schema) + "." + String.format(SqlServerSQLConstants.KEYWORD_TABLE, table); + String tableWithSchema = String.format(RedshiftConstants.KEYWORD_TABLE, schema) + "." + String.format(RedshiftConstants.KEYWORD_TABLE, table); return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", StringUtils.join(array, ","), tableWithSchema); } else { return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", StringUtils.join(array, ","), table); diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index ee68b2e33e..b867ebed3c 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -105,16 +105,12 @@ public class DatasourceService { @DeCleaner(DePermissionType.DATASOURCE) @Transactional(rollbackFor = Exception.class) public Datasource addDatasource(DatasourceDTO datasource) throws Exception { - if (!types().stream().map(DataSourceType::getType).collect(Collectors.toList()).contains(datasource.getType())) { - throw new Exception("Datasource type not supported."); - } - if(datasource.isConfigurationEncryption()){ - datasource.setConfiguration(new String(java.util.Base64.getDecoder().decode(datasource.getConfiguration()))); - } - Provider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); - datasourceProvider.checkConfiguration(datasource); + preCheckDs(datasource); + return insert(datasource); + } - checkName(datasource.getName(), datasource.getType(), datasource.getId()); + @DeCleaner(DePermissionType.DATASOURCE) + public DatasourceDTO insert(DatasourceDTO datasource) { long currentTimeMillis = System.currentTimeMillis(); datasource.setId(UUID.randomUUID().toString()); datasource.setUpdateTime(currentTimeMillis); @@ -127,6 +123,17 @@ public class DatasourceService { return datasource; } + public void preCheckDs(DatasourceDTO datasource) throws Exception { + if (!types().stream().map(DataSourceType::getType).collect(Collectors.toList()).contains(datasource.getType())) { + throw new Exception("Datasource type not supported."); + } + if (datasource.isConfigurationEncryption()) { + datasource.setConfiguration(new String(java.util.Base64.getDecoder().decode(datasource.getConfiguration()))); + } + Provider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); + datasourceProvider.checkConfiguration(datasource); + checkName(datasource.getName(), datasource.getType(), datasource.getId()); + } public void handleConnectionPool(String datasourceId, String type) { Datasource datasource = datasourceMapper.selectByPrimaryKey(datasourceId); @@ -159,7 +166,7 @@ public class DatasourceService { return datasourceDTOS; } - private void datasourceTrans(DatasourceDTO datasourceDTO){ + private void datasourceTrans(DatasourceDTO datasourceDTO) { types().forEach(dataSourceType -> { if (dataSourceType.getType().equalsIgnoreCase(datasourceDTO.getType())) { datasourceDTO.setTypeDesc(dataSourceType.getName()); @@ -214,23 +221,24 @@ public class DatasourceService { } } } - if(StringUtils.isNotEmpty(datasourceDTO.getConfiguration())){ + if (StringUtils.isNotEmpty(datasourceDTO.getConfiguration())) { datasourceDTO.setConfiguration(new String(java.util.Base64.getEncoder().encode(datasourceDTO.getConfiguration().getBytes()))); } - if(CollectionUtils.isNotEmpty(datasourceDTO.getApiConfiguration())){ + if (CollectionUtils.isNotEmpty(datasourceDTO.getApiConfiguration())) { String config = new Gson().toJson(datasourceDTO.getApiConfiguration()); datasourceDTO.setApiConfigurationStr(new String(java.util.Base64.getEncoder().encode(config.getBytes()))); datasourceDTO.setApiConfiguration(null); } } - public DatasourceDTO getDataSourceDetails(String datasourceId){ - DatasourceDTO result = extDataSourceMapper.queryDetails(datasourceId,String.valueOf(AuthUtils.getUser().getUserId())); - if(result != null){ + public DatasourceDTO getDataSourceDetails(String datasourceId) { + DatasourceDTO result = extDataSourceMapper.queryDetails(datasourceId, String.valueOf(AuthUtils.getUser().getUserId())); + if (result != null) { this.datasourceTrans(result); } return result; } + public List gridQuery(BaseGridRequest request) { //如果没有查询条件增加一个默认的条件 if (CollectionUtils.isEmpty(request.getConditions())) { @@ -260,12 +268,6 @@ public class DatasourceService { } public void updateDatasource(UpdataDsRequest updataDsRequest) throws Exception { - if (!types().stream().map(DataSourceType::getType).collect(Collectors.toList()).contains(updataDsRequest.getType())) { - throw new Exception("Datasource type not supported."); - } - if(updataDsRequest.isConfigurationEncryption()){ - updataDsRequest.setConfiguration(new String(java.util.Base64.getDecoder().decode(updataDsRequest.getConfiguration()))); - } checkName(updataDsRequest.getName(), updataDsRequest.getType(), updataDsRequest.getId()); Datasource datasource = new Datasource(); datasource.setName(updataDsRequest.getName()); @@ -277,19 +279,15 @@ public class DatasourceService { Provider datasourceProvider = ProviderFactory.getProvider(updataDsRequest.getType()); datasourceProvider.checkConfiguration(datasource); checkAndUpdateDatasourceStatus(datasource); - if(StringUtils.isNotEmpty(updataDsRequest.getId())){ - DatasourceExample example = new DatasourceExample(); - example.createCriteria().andIdEqualTo(updataDsRequest.getId()); - datasourceMapper.updateByExampleSelective(datasource, example); - handleConnectionPool(updataDsRequest.getId()); - }else { - datasource.setId(UUID.randomUUID().toString()); - datasource.setCreateTime(System.currentTimeMillis()); - datasourceMapper.insert(datasource); - handleConnectionPool(datasource, "add"); - sysAuthService.copyAuth(datasource.getId(), SysAuthConstants.AUTH_SOURCE_TYPE_DATASOURCE); - } + updateDatasource(updataDsRequest.getId(), datasource); + } + + public void updateDatasource(String id, Datasource datasource) { + DatasourceExample example = new DatasourceExample(); + example.createCriteria().andIdEqualTo(id); + datasourceMapper.updateByExampleSelective(datasource, example); + handleConnectionPool(id); } private void handleConnectionPool(String datasourceId) { @@ -304,7 +302,7 @@ public class DatasourceService { } public ResultHolder validate(DatasourceDTO datasource) throws Exception { - if(datasource.isConfigurationEncryption()){ + if (datasource.isConfigurationEncryption()) { datasource.setConfiguration(new String(java.util.Base64.getDecoder().decode(datasource.getConfiguration()))); } DatasourceDTO datasourceDTO = new DatasourceDTO(); @@ -395,7 +393,7 @@ public class DatasourceService { } public List getSchema(DatasourceDTO datasource) throws Exception { - if(datasource.isConfigurationEncryption()){ + if (datasource.isConfigurationEncryption()) { datasource.setConfiguration(new String(java.util.Base64.getDecoder().decode(datasource.getConfiguration()))); } Provider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); @@ -519,7 +517,7 @@ public class DatasourceService { return datasourceMapper.selectByExampleWithBLOBs(example); } - private void checkAndUpdateDatasourceStatus(Datasource datasource) { + public void checkAndUpdateDatasourceStatus(Datasource datasource) { try { Provider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest();