From 1b3f739f266d6d4bae29203f3222d5dc45d291a0 Mon Sep 17 00:00:00 2001 From: xiajunjie <752696511@qq.com> Date: Wed, 19 May 2021 22:31:26 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(backend):=E8=8E=B7=E5=8F=96=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=BA=90Mapper=20SQL=20bug=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/dataease/base/mapper/ext/ExtDataSourceMapper.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.xml b/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.xml index 2aa2a6c9ee..4178d702c5 100644 --- a/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.xml @@ -27,12 +27,12 @@ select id , name , `desc` ,`type` , configuration ,create_time ,update_time from (select GET_V_AUTH_MODEL_ID_P_USE (#{userId}, 'link') cids) t,datasource - FIND_IN_SET(datasource.id,cids)) + FIND_IN_SET(datasource.id,cids) id = #{id,jdbcType=VARCHAR} - `name` = concat('%',#{name,jdbcType=VARCHAR},'%') + `name` like concat('%',#{name,jdbcType=VARCHAR},'%') `desc` = #{desc,jdbcType=VARCHAR} From bc5cbad7bd88a121f6da78481d8facbe335444f3 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Wed, 19 May 2021 23:28:08 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81sqlserver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasource/provider/JdbcProvider.java | 3 + .../datasource/provider/ProviderFactory.java | 2 +- .../sqlserver/SqlserverQueryProvider.java | 311 ++++++++++++++++++ .../service/dataset/ExtractDataService.java | 13 +- 4 files changed, 327 insertions(+), 2 deletions(-) create mode 100644 backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index 19e404cd56..533edf96b9 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -202,6 +202,9 @@ public class JdbcProvider extends DatasourceProvider { tableFiled.setFieldSize(Integer.valueOf(resultSet.getString("COLUMN_SIZE"))); String dbType = resultSet.getString("TYPE_NAME"); tableFiled.setFieldType(dbType); + if(StringUtils.isNotEmpty(dbType) && dbType.equalsIgnoreCase("date") && tableFiled.getFieldSize() < 50 ){ + tableFiled.setFieldSize(50); + } list.add(tableFiled); } } diff --git a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java index 72f68b631d..bc609b2e75 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java @@ -41,7 +41,7 @@ public class ProviderFactory implements ApplicationContextAware { case doris: return context.getBean("dorisQuery", QueryProvider.class); case sqlServer: - return context.getBean("mysqlQuery", QueryProvider.class); + return context.getBean("sqlserverQuery", QueryProvider.class); default: return context.getBean("mysqlQuery", QueryProvider.class); } diff --git a/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java b/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java new file mode 100644 index 0000000000..83a2b11b8c --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java @@ -0,0 +1,311 @@ +package io.dataease.provider.sqlserver; + +import io.dataease.base.domain.DatasetTableField; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.provider.QueryProvider; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.List; + +/** + * @Author gin + * @Date 2021/5/17 2:43 下午 + */ +@Service("sqlserverQuery") +public class SqlserverQueryProvider extends QueryProvider { + @Override + public Integer transFieldType(String field) { + switch (field.toUpperCase()) { + case "CHAR": + case "VARCHAR": + case "TEXT": + case "TINYTEXT": + case "MEDIUMTEXT": + case "LONGTEXT": + case "NCHAR": + case "NVARCHAR": + case "NTEXT": + case "ENUM": + return 0;// 文本 + case "DATE": + case "TIME": + case "YEAR": + case "DATETIME": + case "TIMESTAMP": + return 1;// 时间 + case "INT": + case "SMALLINT": + case "MEDIUMINT": + case "INTEGER": + case "BIGINT": + return 2;// 整型 + case "FLOAT": + case "DOUBLE": + case "DECIMAL": + return 3;// 浮点 + case "BIT": + case "TINYINT": + return 4;// 布尔 + default: + return 0; + } + } + + @Override + public String createQueryCountSQL(String table) { + return MessageFormat.format("SELECT count(*) FROM {0}", table); + } + + @Override + public String createQueryCountSQLAsTmp(String sql) { + return createQueryCountSQL(" (" + sqlFix(sql) + ") AS tmp "); + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp ORDER BY " + orderBy + " LIMIT 0,1000"; + } + + @Override + public String createQuerySQL(String table, List fields) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + // 如果原始类型为时间 + if (f.getDeExtractType() == 1) { + if (f.getDeType() == 2 || f.getDeType() == 3) { + stringBuilder.append("unix_timestamp(").append(f.getDataeaseName()).append(")*1000 as ").append(f.getDataeaseName()); + } else { + stringBuilder.append(f.getDataeaseName()); + } + } else { + if (f.getDeType() == 1) { + stringBuilder.append("FROM_UNIXTIME(cast(").append(f.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(f.getDataeaseName()); + } else { + stringBuilder.append(f.getDataeaseName()); + } + } + return stringBuilder.toString(); + }).toArray(String[]::new); + + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table); + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields) { + return createQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields); + } + + @Override + public String createQuerySQLWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize) { + return createQuerySQL(table, fields) + " offset " + (page - 1) * pageSize + " rows fetch next " + realSize + " rows only"; + } + + @Override + public String createQuerySQLAsTmpWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize) { + return createQuerySQLAsTmp(sql, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List extFilterRequestList) { + // 字段汇总 排序等 + String[] field = yAxis.stream().map(y -> { + StringBuilder f = new StringBuilder(); + if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) { + f.append(y.getSummary()).append("(").append(y.getDataeaseName()).append(")"); + } else { + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + f.append("CAST(") + .append(y.getSummary()).append("(") + .append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") + .append(") AS DECIMAL(20,2)").append(")"); + } else { + f.append(y.getSummary()).append("(") + .append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") + .append(")"); + } + } + f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); + return f.toString(); + }).toArray(String[]::new); + String[] groupField = xAxis.stream().map(x -> { + StringBuilder stringBuilder = new StringBuilder(); + // 如果原始类型为时间 + if (x.getDeExtractType() == 1) { + if (x.getDeType() == 2 || x.getDeType() == 3) { + stringBuilder.append("unix_timestamp(").append(x.getDataeaseName()).append(")*1000 as ").append(x.getDataeaseName()); + } else { + stringBuilder.append(x.getDataeaseName()); + } + } else { + if (x.getDeType() == 1) { + stringBuilder.append("FROM_UNIXTIME(cast(").append(x.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(x.getDataeaseName()); + } else { + stringBuilder.append(x.getDataeaseName()); + } + } + return stringBuilder.toString(); + }).toArray(String[]::new); + String[] group = xAxis.stream().map(ChartViewFieldDTO::getDataeaseName).toArray(String[]::new); + String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) + .map(f -> f.getDataeaseName() + " " + f.getSort()).toArray(String[]::new); + String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) + .map(f -> "_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getDataeaseName(), "*") ? "" : f.getDataeaseName()) + " " + f.getSort()).toArray(String[]::new); + String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length); + System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length); + + String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) + .map(x -> { + String[] s = x.getFilter().stream().map(f -> { + StringBuilder filter = new StringBuilder(); + if (x.getDeType() == 1 && x.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast(") + .append(x.getDataeaseName()) + .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND ").append(x.getDataeaseName()); + } + filter.append(transMysqlFilterTerm(f.getTerm())); + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + filter.append("%").append(f.getValue()).append("%"); + } else { + filter.append("'").append(f.getValue()).append("'"); + } + return filter.toString(); + }).toArray(String[]::new); + return StringUtils.join(s, " "); + }).toArray(String[]::new); + + String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}", + StringUtils.join(groupField, ","), + StringUtils.join(field, ","), + table, + (xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "") + transMysqlExtFilter(extFilterRequestList),// origin field filter and panel field filter + StringUtils.join(group, ","), + StringUtils.join(order, ",")); + if (sql.endsWith(",")) { + sql = sql.substring(0, sql.length() - 1); + } + // 如果是对结果字段过滤，则再包裹一层sql + String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) + .map(y -> { + String[] s = y.getFilter().stream().map(f -> { + StringBuilder filter = new StringBuilder(); + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (y.getDeType() == 1 && y.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast(_") + .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()) + .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); + } + filter.append(transMysqlFilterTerm(f.getTerm())); + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + filter.append("%").append(f.getValue()).append("%"); + } else { + filter.append("'").append(f.getValue()).append("'"); + } + return filter.toString(); + }).toArray(String[]::new); + return StringUtils.join(s, " "); + }).toArray(String[]::new); + if (resultFilter.length == 0) { + return sql; + } else { + String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1}", + "(" + sql + ") AS tmp", + StringUtils.join(resultFilter, " ")); + return filterSql; + } + } + + @Override + public String getSQLAsTmp(String sql, List xAxis, List yAxis, List extFilterRequestList) { + return getSQL(" (" + sqlFix(sql) + ") AS tmp ", xAxis, yAxis, extFilterRequestList); + } + + @Override + public String searchTable(String table) { + return "SELECT table_name FROM information_schema.TABLES WHERE table_name ='" + table + "'"; + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IS NULL "; + case "not_null": + return " IS NOT NULL "; + default: + return ""; + } + } + + public String transMysqlExtFilter(List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return ""; + } + StringBuilder filter = new StringBuilder(); + for (ChartExtFilterRequest request : requestList) { + List value = request.getValue(); + if (CollectionUtils.isEmpty(value)) { + continue; + } + DatasetTableField field = request.getDatasetTableField(); + if (field.getDeType() == 1 && field.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast(") + .append(field.getDataeaseName()) + .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND ").append(field.getDataeaseName()); + } + filter.append(" ") + .append(transMysqlFilterTerm(request.getOperator())) + .append(" "); + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + filter.append("('").append(StringUtils.join(value, "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + filter.append("'%").append(value.get(0)).append("%'"); + } else { + filter.append("'").append(value.get(0)).append("'"); + } + } + return filter.toString(); + } + + private String sqlFix(String sql) { + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } +} diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index acf72d5034..ef53e8fed1 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -12,6 +12,7 @@ import io.dataease.commons.utils.LogUtil; import io.dataease.datasource.constants.DatasourceTypes; import io.dataease.datasource.dto.DorisConfigration; import io.dataease.datasource.dto.MysqlConfigration; +import io.dataease.datasource.dto.SqlServerConfigration; import io.dataease.datasource.dto.TableFiled; import io.dataease.datasource.provider.DatasourceProvider; import io.dataease.datasource.provider.JdbcProvider; @@ -188,7 +189,6 @@ public class ExtractDataService { return o1.getColumnIndex().compareTo(o2.getColumnIndex()); }); String dorisTablColumnSql = createDorisTablColumnSql(datasetTableFields); - System.out.println(dorisTablColumnSql); switch (updateType) { // 全量更新 case all_scope: @@ -443,6 +443,17 @@ public class ExtractDataService { } inputStep = inputStep(transMeta, selectSQL); break; + case sqlServer: + SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasource.getConfiguration(), SqlServerConfigration.class); + dataMeta = new DatabaseMeta("db", "MSSQLNATIVE", "Native", sqlServerConfigration.getHost(), sqlServerConfigration.getDataBase(), sqlServerConfigration.getPort().toString(), sqlServerConfigration.getUsername(), sqlServerConfigration.getPassword()); + transMeta.addDatabase(dataMeta); + if (extractType.equalsIgnoreCase("all_scope")) { + String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); + QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); + selectSQL = qp.createQuerySQL(tableName, datasetTableFields); + } + inputStep = inputStep(transMeta, selectSQL); + break; case excel: String filePath = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData(); inputStep = excelInputStep(filePath, datasetTableFields);