center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat(backend):数据集 视图，sql重新整理

Browse Source
This commit is contained in:
junjie 2021-05-17 16:43:02 +08:00
parent 8fadc27a29
commit e98f95604e
13 changed files with 785 additions and 470 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/dataease/config/CommonConfig.java
View File

@ -39,7 +39,7 @@ public class CommonConfig {
datasource.setId("doris");
datasource.setName("doris");
datasource.setDesc("doris");
datasource.setType("mysql");
datasource.setType("doris");
datasource.setConfiguration(jsonObject.toJSONString());
return datasource;
}

2
backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java
View File

@ -1,5 +1,5 @@
package io.dataease.datasource.constants;
public enum DatasourceTypes {
mysql, sqlServer, excel
mysql, sqlServer, excel, doris
}

37
backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java
View File

@ -1,13 +1,14 @@
package io.dataease.datasource.provider;
import io.dataease.datasource.constants.DatasourceTypes;
import io.dataease.provider.DDLProvider;
import io.dataease.provider.QueryProvider;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
@Component
public class ProviderFactory implements ApplicationContextAware {
@ -18,18 +19,46 @@ public class ProviderFactory implements ApplicationContextAware {
this.context = applicationContext;
}
public static DatasourceProvider getProvider(String type){
public static DatasourceProvider getProvider(String type) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType){
switch (datasourceType) {
case mysql:
return context.getBean("jdbc", DatasourceProvider.class);
case doris:
return context.getBean("jdbc", DatasourceProvider.class);
case sqlServer:
return context.getBean("jdbc", DatasourceProvider.class);
default:
return context.getBean("jdbc", DatasourceProvider.class);
}
}
public static QueryProvider getQueryProvider(String type) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case mysql:
return context.getBean("mysqlQuery", QueryProvider.class);
case doris:
return context.getBean("dorisQuery", QueryProvider.class);
case sqlServer:
return context.getBean("mysqlQuery", QueryProvider.class);
default:
return context.getBean("mysqlQuery", QueryProvider.class);
}
}
public static DDLProvider getDDLProvider(String type) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case mysql:
return context.getBean("mysqlDDL", DDLProvider.class);
case doris:
return context.getBean("dorisDDL", DDLProvider.class);
case sqlServer:
return context.getBean("mysqlDDL", DDLProvider.class);
default:
return context.getBean("mysqlDDL", DDLProvider.class);
}
}
}

11
backend/src/main/java/io/dataease/provider/DDLProvider.java Normal file
View File

@ -0,0 +1,11 @@
package io.dataease.provider;
/**
* @Author gin
* @Date 2021/5/17 4:19 下午
*/
public abstract class DDLProvider {
public abstract String createView(String name, String viewSQL);
public abstract String dropTableOrView(String name);
}

33
backend/src/main/java/io/dataease/provider/QueryProvider.java Normal file
View File

@ -0,0 +1,33 @@
package io.dataease.provider;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.controller.request.chart.ChartExtFilterRequest;
import io.dataease.dto.chart.ChartViewFieldDTO;
import java.util.List;
/**
* @Author gin
* @Date 2021/5/17 2:42 下午
*/
public abstract class QueryProvider {
public abstract Integer transFieldType( String field);
public abstract String createQueryCountSQL(String table);
public abstract String createQueryCountSQLAsTmp(String sql);
public abstract String createSQLPreview(String sql, String orderBy);
public abstract String createQuerySQL(String table, List<DatasetTableField> fields);
public abstract String createQuerySQLAsTmp(String sql, List<DatasetTableField> fields);
public abstract String createQuerySQLWithPage(String table, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize);
public abstract String createQuerySQLAsTmpWithPage(String sql, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize);
public abstract String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartExtFilterRequest> extFilterRequestList);
public abstract String getSQLAsTmp(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartExtFilterRequest> extFilterRequestList);
}

21
backend/src/main/java/io/dataease/provider/doris/DorisDDLProvider.java Normal file
View File

@ -0,0 +1,21 @@
package io.dataease.provider.doris;
import io.dataease.provider.DDLProvider;
import org.springframework.stereotype.Service;
/**
* @Author gin
* @Date 2021/5/17 4:27 下午
*/
@Service("dorisDDL")
public class DorisDDLProvider extends DDLProvider {
@Override
public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
}
@Override
public String dropTableOrView(String name) {
return "DROP TABLE IF EXISTS " + name;
}
}

289
backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java Normal file
View File

@ -0,0 +1,289 @@
package io.dataease.provider.doris;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.controller.request.chart.ChartExtFilterRequest;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.provider.QueryProvider;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.text.MessageFormat;
import java.util.Arrays;
import java.util.List;
/**
* @Author gin
* @Date 2021/5/17 4:11 下午
*/
@Service("dorisQuery")
public class DorisQueryProvider extends QueryProvider {
@Override
public Integer transFieldType(String field) {
switch (field) {
case "CHAR":
case "VARCHAR":
case "TEXT":
case "TINYTEXT":
case "MEDIUMTEXT":
case "LONGTEXT":
case "ENUM":
return 0;// 文本
case "DATE":
case "TIME":
case "YEAR":
case "DATETIME":
case "TIMESTAMP":
return 1;// 时间
case "INT":
case "SMALLINT":
case "MEDIUMINT":
case "INTEGER":
case "BIGINT":
return 2;// 整型
case "FLOAT":
case "DOUBLE":
case "DECIMAL":
return 3;// 浮点
case "BIT":
case "TINYINT":
return 4;// 布尔
default:
return 0;
}
}
@Override
public String createQueryCountSQL(String table) {
return MessageFormat.format("SELECT count(*) FROM {0}", table);
}
@Override
public String createQueryCountSQLAsTmp(String sql) {
return createQueryCountSQL(" (" + sql + ") AS tmp ");
}
@Override
public String createSQLPreview(String sql, String orderBy) {
return "SELECT * FROM (" + sql + ") AS tmp ORDER BY " + orderBy + " LIMIT 0,1000";
}
@Override
public String createQuerySQL(String table, List<DatasetTableField> fields) {
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
// 如果原始类型为时间
if (f.getDeExtractType() == 1) {
if (f.getDeType() == 2 || f.getDeType() == 3) {
stringBuilder.append("unix_timestamp(").append(f.getDataeaseName()).append(")*1000 as ").append(f.getDataeaseName());
} else {
stringBuilder.append(f.getDataeaseName());
}
} else {
if (f.getDeType() == 1) {
stringBuilder.append("FROM_UNIXTIME(cast(").append(f.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(f.getDataeaseName());
} else {
stringBuilder.append(f.getDataeaseName());
}
}
return stringBuilder.toString();
}).toArray(String[]::new);
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table);
}
@Override
public String createQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return createQuerySQL(" (" + sql + ") AS tmp ", fields);
}
@Override
public String createQuerySQLWithPage(String table, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize) {
return createQuerySQL(table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize;
}
@Override
public String createQuerySQLAsTmpWithPage(String sql, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize) {
return createQuerySQLAsTmp(sql, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize;
}
@Override
public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartExtFilterRequest> extFilterRequestList) {
// 字段汇总 排序等
String[] field = yAxis.stream().map(y -> {
StringBuilder f = new StringBuilder();
if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) {
f.append(y.getSummary()).append("(").append(y.getDataeaseName()).append(")");
} else {
f.append("ROUND(")
.append(y.getSummary()).append("(")
.append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")")
.append("),2").append(")");
}
f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName());
return f.toString();
}).toArray(String[]::new);
String[] groupField = xAxis.stream().map(x -> {
StringBuilder stringBuilder = new StringBuilder();
// 如果原始类型为时间
if (x.getDeExtractType() == 1) {
if (x.getDeType() == 2 || x.getDeType() == 3) {
stringBuilder.append("unix_timestamp(").append(x.getDataeaseName()).append(")*1000 as ").append(x.getDataeaseName());
} else {
stringBuilder.append(x.getDataeaseName());
}
} else {
if (x.getDeType() == 1) {
stringBuilder.append("FROM_UNIXTIME(cast(").append(x.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(x.getDataeaseName());
} else {
stringBuilder.append(x.getDataeaseName());
}
}
return stringBuilder.toString();
}).toArray(String[]::new);
String[] group = xAxis.stream().map(ChartViewFieldDTO::getDataeaseName).toArray(String[]::new);
String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none"))
.map(f -> f.getDataeaseName() + " " + f.getSort()).toArray(String[]::new);
String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none"))
.map(f -> "_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getDataeaseName(), "*") ? "" : f.getDataeaseName()) + " " + f.getSort()).toArray(String[]::new);
String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length);
System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length);
String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0)
.map(x -> {
String[] s = x.getFilter().stream().map(f -> {
StringBuilder filter = new StringBuilder();
if (x.getDeType() == 1 && x.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(")
.append(x.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND ").append(x.getDataeaseName());
}
filter.append(transMysqlFilterTerm(f.getTerm()));
if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) {
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')");
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
filter.append("%").append(f.getValue()).append("%");
} else {
filter.append("'" + f.getValue() + "'");
}
return filter.toString();
}).toArray(String[]::new);
return StringUtils.join(s, " ");
}).toArray(String[]::new);
String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}",
StringUtils.join(groupField, ","),
StringUtils.join(field, ","),
table,
xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "" + transMysqlExtFilter(extFilterRequestList),// origin field filter and panel field filter
StringUtils.join(group, ","),
StringUtils.join(order, ","));
if (sql.endsWith(",")) {
sql = sql.substring(0, sql.length() - 1);
}
// 如果是对结果字段过滤，则再包裹一层sql
String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0)
.map(y -> {
String[] s = y.getFilter().stream().map(f -> {
StringBuilder filter = new StringBuilder();
if (y.getDeType() == 1 && y.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(_")
.append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName());
}
filter.append(transMysqlFilterTerm(f.getTerm()));
if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) {
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')");
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
filter.append("%").append(f.getValue()).append("%");
} else {
filter.append("'" + f.getValue() + "'");
}
return filter.toString();
}).toArray(String[]::new);
return StringUtils.join(s, " ");
}).toArray(String[]::new);
if (resultFilter.length == 0) {
return sql;
} else {
String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1}",
"(" + sql + ") AS tmp",
StringUtils.join(resultFilter, " "));
return filterSql;
}
}
@Override
public String getSQLAsTmp(String sql, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartExtFilterRequest> extFilterRequestList) {
return getSQL(" (" + sql + ") AS tmp ", xAxis, yAxis, extFilterRequestList);
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":
return " = ";
case "not_eq":
return " <> ";
case "lt":
return " < ";
case "le":
return " <= ";
case "gt":
return " > ";
case "ge":
return " >= ";
case "in":
return " IN ";
case "not in":
return " NOT IN ";
case "like":
return " LIKE ";
case "not like":
return " NOT LIKE ";
case "null":
return " IS NULL ";
case "not_null":
return " IS NOT NULL ";
default:
return "";
}
}
public String transMysqlExtFilter(List<ChartExtFilterRequest> requestList) {
if (CollectionUtils.isEmpty(requestList)) {
return "";
}
StringBuilder filter = new StringBuilder();
for (ChartExtFilterRequest request : requestList) {
List<String> value = request.getValue();
if (CollectionUtils.isEmpty(value)) {
continue;
}
DatasetTableField field = request.getDatasetTableField();
if (field.getDeType() == 1 && field.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(")
.append(field.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND ").append(field.getDataeaseName());
}
filter.append(" ")
.append(transMysqlFilterTerm(request.getOperator()))
.append(" ");
if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) {
filter.append("('").append(StringUtils.join(value, "','")).append("')");
} else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) {
filter.append("'%").append(value.get(0)).append("%'");
} else {
filter.append("'").append(value.get(0)).append("'");
}
}
return filter.toString();
}
}

21
backend/src/main/java/io/dataease/provider/mysql/MysqlDDLProvider.java Normal file
View File

@ -0,0 +1,21 @@
package io.dataease.provider.mysql;
import io.dataease.provider.DDLProvider;
import org.springframework.stereotype.Service;
/**
* @Author gin
* @Date 2021/5/17 4:27 下午
*/
@Service("mysqlDDL")
public class MysqlDDLProvider extends DDLProvider {
@Override
public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
}
@Override
public String dropTableOrView(String name) {
return "DROP TABLE IF EXISTS " + name;
}
}

296
backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java Normal file
View File

@ -0,0 +1,296 @@
package io.dataease.provider.mysql;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.controller.request.chart.ChartExtFilterRequest;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.provider.QueryProvider;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.text.MessageFormat;
import java.util.Arrays;
import java.util.List;
/**
* @Author gin
* @Date 2021/5/17 2:43 下午
*/
@Service("mysqlQuery")
public class MysqlQueryProvider extends QueryProvider {
@Override
public Integer transFieldType(String field) {
switch (field) {
case "CHAR":
case "VARCHAR":
case "TEXT":
case "TINYTEXT":
case "MEDIUMTEXT":
case "LONGTEXT":
case "ENUM":
return 0;// 文本
case "DATE":
case "TIME":
case "YEAR":
case "DATETIME":
case "TIMESTAMP":
return 1;// 时间
case "INT":
case "SMALLINT":
case "MEDIUMINT":
case "INTEGER":
case "BIGINT":
return 2;// 整型
case "FLOAT":
case "DOUBLE":
case "DECIMAL":
return 3;// 浮点
case "BIT":
case "TINYINT":
return 4;// 布尔
default:
return 0;
}
}
@Override
public String createQueryCountSQL(String table) {
return MessageFormat.format("SELECT count(*) FROM {0}", table);
}
@Override
public String createQueryCountSQLAsTmp(String sql) {
return createQueryCountSQL(" (" + sql + ") AS tmp ");
}
@Override
public String createSQLPreview(String sql, String orderBy) {
return "SELECT * FROM (" + sql + ") AS tmp ORDER BY " + orderBy + " LIMIT 0,1000";
}
@Override
public String createQuerySQL(String table, List<DatasetTableField> fields) {
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
// 如果原始类型为时间
if (f.getDeExtractType() == 1) {
if (f.getDeType() == 2 || f.getDeType() == 3) {
stringBuilder.append("unix_timestamp(").append(f.getDataeaseName()).append(")*1000 as ").append(f.getDataeaseName());
} else {
stringBuilder.append(f.getDataeaseName());
}
} else {
if (f.getDeType() == 1) {
stringBuilder.append("FROM_UNIXTIME(cast(").append(f.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(f.getDataeaseName());
} else {
stringBuilder.append(f.getDataeaseName());
}
}
return stringBuilder.toString();
}).toArray(String[]::new);
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table);
}
@Override
public String createQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return createQuerySQL(" (" + sql + ") AS tmp ", fields);
}
@Override
public String createQuerySQLWithPage(String table, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize) {
return createQuerySQL(table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize;
}
@Override
public String createQuerySQLAsTmpWithPage(String sql, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize) {
return createQuerySQLAsTmp(sql, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize;
}
@Override
public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartExtFilterRequest> extFilterRequestList) {
// 字段汇总 排序等
String[] field = yAxis.stream().map(y -> {
StringBuilder f = new StringBuilder();
if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) {
f.append(y.getSummary()).append("(").append(y.getDataeaseName()).append(")");
} else {
if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) {
f.append("CAST(")
.append(y.getSummary()).append("(")
.append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")")
.append(") AS DECIMAL(20,2)").append(")");
} else {
f.append(y.getSummary()).append("(")
.append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")")
.append(")");
}
}
f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName());
return f.toString();
}).toArray(String[]::new);
String[] groupField = xAxis.stream().map(x -> {
StringBuilder stringBuilder = new StringBuilder();
// 如果原始类型为时间
if (x.getDeExtractType() == 1) {
if (x.getDeType() == 2 || x.getDeType() == 3) {
stringBuilder.append("unix_timestamp(").append(x.getDataeaseName()).append(")*1000 as ").append(x.getDataeaseName());
} else {
stringBuilder.append(x.getDataeaseName());
}
} else {
if (x.getDeType() == 1) {
stringBuilder.append("FROM_UNIXTIME(cast(").append(x.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(x.getDataeaseName());
} else {
stringBuilder.append(x.getDataeaseName());
}
}
return stringBuilder.toString();
}).toArray(String[]::new);
String[] group = xAxis.stream().map(ChartViewFieldDTO::getDataeaseName).toArray(String[]::new);
String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none"))
.map(f -> f.getDataeaseName() + " " + f.getSort()).toArray(String[]::new);
String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none"))
.map(f -> "_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getDataeaseName(), "*") ? "" : f.getDataeaseName()) + " " + f.getSort()).toArray(String[]::new);
String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length);
System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length);
String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0)
.map(x -> {
String[] s = x.getFilter().stream().map(f -> {
StringBuilder filter = new StringBuilder();
if (x.getDeType() == 1 && x.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(")
.append(x.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND ").append(x.getDataeaseName());
}
filter.append(transMysqlFilterTerm(f.getTerm()));
if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) {
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')");
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
filter.append("%").append(f.getValue()).append("%");
} else {
filter.append("'").append(f.getValue()).append("'");
}
return filter.toString();
}).toArray(String[]::new);
return StringUtils.join(s, " ");
}).toArray(String[]::new);
String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}",
StringUtils.join(groupField, ","),
StringUtils.join(field, ","),
table,
xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "" + transMysqlExtFilter(extFilterRequestList),// origin field filter and panel field filter
StringUtils.join(group, ","),
StringUtils.join(order, ","));
if (sql.endsWith(",")) {
sql = sql.substring(0, sql.length() - 1);
}
// 如果是对结果字段过滤，则再包裹一层sql
String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0)
.map(y -> {
String[] s = y.getFilter().stream().map(f -> {
StringBuilder filter = new StringBuilder();
// 原始类型不是时间，在de中被转成时间的字段做处理
if (y.getDeType() == 1 && y.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(_")
.append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName());
}
filter.append(transMysqlFilterTerm(f.getTerm()));
if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) {
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')");
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
filter.append("%").append(f.getValue()).append("%");
} else {
filter.append("'").append(f.getValue()).append("'");
}
return filter.toString();
}).toArray(String[]::new);
return StringUtils.join(s, " ");
}).toArray(String[]::new);
if (resultFilter.length == 0) {
return sql;
} else {
String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1}",
"(" + sql + ") AS tmp",
StringUtils.join(resultFilter, " "));
return filterSql;
}
}
@Override
public String getSQLAsTmp(String sql, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartExtFilterRequest> extFilterRequestList) {
return getSQL(" (" + sql + ") AS tmp ", xAxis, yAxis, extFilterRequestList);
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":
return " = ";
case "not_eq":
return " <> ";
case "lt":
return " < ";
case "le":
return " <= ";
case "gt":
return " > ";
case "ge":
return " >= ";
case "in":
return " IN ";
case "not in":
return " NOT IN ";
case "like":
return " LIKE ";
case "not like":
return " NOT LIKE ";
case "null":
return " IS NULL ";
case "not_null":
return " IS NOT NULL ";
default:
return "";
}
}
public String transMysqlExtFilter(List<ChartExtFilterRequest> requestList) {
if (CollectionUtils.isEmpty(requestList)) {
return "";
}
StringBuilder filter = new StringBuilder();
for (ChartExtFilterRequest request : requestList) {
List<String> value = request.getValue();
if (CollectionUtils.isEmpty(value)) {
continue;
}
DatasetTableField field = request.getDatasetTableField();
if (field.getDeType() == 1 && field.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(")
.append(field.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND ").append(field.getDataeaseName());
}
filter.append(" ")
.append(transMysqlFilterTerm(request.getOperator()))
.append(" ");
if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) {
filter.append("('").append(StringUtils.join(value, "','")).append("')");
} else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) {
filter.append("'%").append(value.get(0)).append("%'");
} else {
filter.append("'").append(value.get(0)).append("'");
}
}
return filter.toString();
}
}

326
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
View File

@ -1,6 +1,5 @@
package io.dataease.service.chart;
import com.alibaba.fastjson.JSONObject;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import io.dataease.base.domain.*;
@ -8,11 +7,9 @@ import io.dataease.base.mapper.ChartViewMapper;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.commons.utils.LogUtil;
import io.dataease.controller.request.chart.ChartExtFilterRequest;
import io.dataease.controller.request.chart.ChartExtRequest;
import io.dataease.controller.request.chart.ChartViewRequest;
import io.dataease.datasource.constants.DatasourceTypes;
import io.dataease.datasource.provider.DatasourceProvider;
import io.dataease.datasource.provider.ProviderFactory;
import io.dataease.datasource.request.DatasourceRequest;
@ -21,6 +18,7 @@ import io.dataease.dto.chart.ChartViewDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.chart.Series;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.provider.QueryProvider;
import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService;
import org.apache.commons.collections4.CollectionUtils;
@ -31,7 +29,6 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.text.MessageFormat;
import java.util.*;
/**
@ -46,8 +43,6 @@ public class ChartViewService {
private DataSetTableService dataSetTableService;
@Resource
private DatasourceService datasourceService;
// @Resource
// private SparkCalc sparkCalc;
@Resource
private DataSetTableFieldsService dataSetTableFieldsService;
@ -105,10 +100,6 @@ public class ChartViewService {
BeanUtils.copyBean(dto, view);
return dto;
}
// List<String> xIds = xAxis.stream().map(DatasetTableField::getId).collect(Collectors.toList());
// List<String> yIds = yAxis.stream().map(DatasetTableField::getId).collect(Collectors.toList());
// List<DatasetTableField> xList = dataSetTableFieldsService.getListByIds(xIds);
// List<DatasetTableField> yList = dataSetTableFieldsService.getListByIds(yIds);
// 过滤来自仪表盘的条件
List<ChartExtFilterRequest> extFilterList = new ArrayList<>();
@ -138,18 +129,15 @@ public class ChartViewService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class);
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
if (StringUtils.equalsIgnoreCase(table.getType(), "db")) {
datasourceRequest.setTable(dataTableInfoDTO.getTable());
datasourceRequest.setQuery(getSQL(ds.getType(), dataTableInfoDTO.getTable(), xAxis, yAxis, extFilterList));
datasourceRequest.setQuery(qp.getSQL(dataTableInfoDTO.getTable(), xAxis, yAxis, extFilterList));
} else if (StringUtils.equalsIgnoreCase(table.getType(), "sql")) {
datasourceRequest.setQuery(getSQL(ds.getType(), " (" + dataTableInfoDTO.getSql() + ") AS tmp ", xAxis, yAxis, extFilterList));
datasourceRequest.setQuery(qp.getSQLAsTmp(dataTableInfoDTO.getSql(), xAxis, yAxis, extFilterList));
}
data = datasourceProvider.getData(datasourceRequest);
} else if (table.getMode() == 1) {// 抽取
// 获取数据集de字段
// List<DatasetTableField> fields = dataSetTableFieldsService.getFieldsByTableId(table.getId());
// data = sparkCalc.getData(table.getId(), fields, xAxis, yAxis, "tmp_" + view.getId().split("-")[0], extFilterList);
// 连接doris，构建doris数据源查询
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
@ -157,7 +145,8 @@ public class ChartViewService {
datasourceRequest.setDatasource(ds);
String tableName = "ds_" + table.getId().replaceAll("-", "_");
datasourceRequest.setTable(tableName);
datasourceRequest.setQuery(transDorisSQL(tableName, xAxis, yAxis, extFilterList));
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.getSQL(tableName, xAxis, yAxis, extFilterList));
data = datasourceProvider.getData(datasourceRequest);
}
@ -220,309 +209,6 @@ public class ChartViewService {
return dto;
}
public String transMysqlExtFilter(List<ChartExtFilterRequest> requestList) {
if (CollectionUtils.isEmpty(requestList)) {
return "";
}
StringBuilder filter = new StringBuilder();
for (ChartExtFilterRequest request : requestList) {
List<String> value = request.getValue();
if (CollectionUtils.isEmpty(value)) {
continue;
}
DatasetTableField field = request.getDatasetTableField();
if (field.getDeType() == 1 && field.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(")
.append(field.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND ").append(field.getDataeaseName());
}
filter.append(" ")
.append(transMysqlFilterTerm(request.getOperator()))
.append(" ");
if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) {
filter.append("('").append(StringUtils.join(value, "','")).append("')");
} else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) {
filter.append("'%").append(value.get(0)).append("%'");
} else {
filter.append("'").append(value.get(0)).append("'");
}
}
return filter.toString();
}
public String getSQL(String type, String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartExtFilterRequest> extFilterRequestList) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case mysql:
return transMysqlSQL(table, xAxis, yAxis, extFilterRequestList);
case sqlServer:
default:
return "";
}
}
public String transDorisSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartExtFilterRequest> extFilterRequestList) {
// 字段汇总 排序等
String[] field = yAxis.stream().map(y -> {
StringBuilder f = new StringBuilder();
if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) {
f.append(y.getSummary()).append("(").append(y.getDataeaseName()).append(")");
} else {
f.append("ROUND(")
.append(y.getSummary()).append("(")
.append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")")
.append("),2").append(")");
}
f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName());
return f.toString();
}).toArray(String[]::new);
String[] groupField = xAxis.stream().map(x -> {
StringBuilder stringBuilder = new StringBuilder();
// 如果原始类型为时间
if (x.getDeExtractType() == 1) {
if (x.getDeType() == 2 || x.getDeType() == 3) {
stringBuilder.append("unix_timestamp(").append(x.getDataeaseName()).append(")*1000 as ").append(x.getDataeaseName());
} else {
stringBuilder.append(x.getDataeaseName());
}
} else {
if (x.getDeType() == 1) {
stringBuilder.append("FROM_UNIXTIME(cast(").append(x.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(x.getDataeaseName());
} else {
stringBuilder.append(x.getDataeaseName());
}
}
return stringBuilder.toString();
}).toArray(String[]::new);
String[] group = xAxis.stream().map(ChartViewFieldDTO::getDataeaseName).toArray(String[]::new);
String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none"))
.map(f -> f.getDataeaseName() + " " + f.getSort()).toArray(String[]::new);
String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none"))
.map(f -> "_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getDataeaseName(), "*") ? "" : f.getDataeaseName()) + " " + f.getSort()).toArray(String[]::new);
String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length);
System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length);
String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0)
.map(x -> {
String[] s = x.getFilter().stream().map(f -> {
StringBuilder filter = new StringBuilder();
if (x.getDeType() == 1 && x.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(")
.append(x.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND ").append(x.getDataeaseName());
}
filter.append(transMysqlFilterTerm(f.getTerm()));
if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) {
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')");
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
filter.append("%").append(f.getValue()).append("%");
} else {
filter.append("'" + f.getValue() + "'");
}
return filter.toString();
}).toArray(String[]::new);
return StringUtils.join(s, " ");
}).toArray(String[]::new);
String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}",
StringUtils.join(groupField, ","),
StringUtils.join(field, ","),
table,
xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "" + transMysqlExtFilter(extFilterRequestList),// origin field filter and panel field filter
StringUtils.join(group, ","),
StringUtils.join(order, ","));
if (sql.endsWith(",")) {
sql = sql.substring(0, sql.length() - 1);
}
// 如果是对结果字段过滤，则再包裹一层sql
String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0)
.map(y -> {
String[] s = y.getFilter().stream().map(f -> {
StringBuilder filter = new StringBuilder();
if (y.getDeType() == 1 && y.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(_")
.append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName());
}
filter.append(transMysqlFilterTerm(f.getTerm()));
if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) {
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')");
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
filter.append("%").append(f.getValue()).append("%");
} else {
filter.append("'" + f.getValue() + "'");
}
return filter.toString();
}).toArray(String[]::new);
return StringUtils.join(s, " ");
}).toArray(String[]::new);
if (resultFilter.length == 0) {
return sql;
} else {
String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1}",
"(" + sql + ") AS tmp",
StringUtils.join(resultFilter, " "));
return filterSql;
}
}
public String transMysqlSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartExtFilterRequest> extFilterRequestList) {
// 字段汇总 排序等
String[] field = yAxis.stream().map(y -> {
StringBuilder f = new StringBuilder();
if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) {
f.append(y.getSummary()).append("(").append(y.getDataeaseName()).append(")");
} else {
if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) {
f.append("CAST(")
.append(y.getSummary()).append("(")
.append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")")
.append(") AS DECIMAL(20,2)").append(")");
} else {
f.append(y.getSummary()).append("(")
.append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")")
.append(")");
}
}
f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName());
return f.toString();
}).toArray(String[]::new);
String[] groupField = xAxis.stream().map(x -> {
StringBuilder stringBuilder = new StringBuilder();
// 如果原始类型为时间
if (x.getDeExtractType() == 1) {
if (x.getDeType() == 2 || x.getDeType() == 3) {
stringBuilder.append("unix_timestamp(").append(x.getDataeaseName()).append(")*1000 as ").append(x.getDataeaseName());
} else {
stringBuilder.append(x.getDataeaseName());
}
} else {
if (x.getDeType() == 1) {
stringBuilder.append("FROM_UNIXTIME(cast(").append(x.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(x.getDataeaseName());
} else {
stringBuilder.append(x.getDataeaseName());
}
}
return stringBuilder.toString();
}).toArray(String[]::new);
String[] group = xAxis.stream().map(ChartViewFieldDTO::getDataeaseName).toArray(String[]::new);
String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none"))
.map(f -> f.getDataeaseName() + " " + f.getSort()).toArray(String[]::new);
String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none"))
.map(f -> "_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getDataeaseName(), "*") ? "" : f.getDataeaseName()) + " " + f.getSort()).toArray(String[]::new);
String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length);
System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length);
String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0)
.map(x -> {
String[] s = x.getFilter().stream().map(f -> {
StringBuilder filter = new StringBuilder();
if (x.getDeType() == 1 && x.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(")
.append(x.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND ").append(x.getDataeaseName());
}
filter.append(transMysqlFilterTerm(f.getTerm()));
if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) {
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')");
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
filter.append("%").append(f.getValue()).append("%");
} else {
filter.append("'").append(f.getValue()).append("'");
}
return filter.toString();
}).toArray(String[]::new);
return StringUtils.join(s, " ");
}).toArray(String[]::new);
String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}",
StringUtils.join(groupField, ","),
StringUtils.join(field, ","),
table,
xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "" + transMysqlExtFilter(extFilterRequestList),// origin field filter and panel field filter
StringUtils.join(group, ","),
StringUtils.join(order, ","));
if (sql.endsWith(",")) {
sql = sql.substring(0, sql.length() - 1);
}
// 如果是对结果字段过滤，则再包裹一层sql
String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0)
.map(y -> {
String[] s = y.getFilter().stream().map(f -> {
StringBuilder filter = new StringBuilder();
// 原始类型不是时间，在de中被转成时间的字段做处理
if (y.getDeType() == 1 && y.getDeExtractType() != 1) {
filter.append(" AND FROM_UNIXTIME(cast(_")
.append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName())
.append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') ");
} else {
filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName());
}
filter.append(transMysqlFilterTerm(f.getTerm()));
if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) {
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')");
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
filter.append("%").append(f.getValue()).append("%");
} else {
filter.append("'").append(f.getValue()).append("'");
}
return filter.toString();
}).toArray(String[]::new);
return StringUtils.join(s, " ");
}).toArray(String[]::new);
if (resultFilter.length == 0) {
return sql;
} else {
String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1}",
"(" + sql + ") AS tmp",
StringUtils.join(resultFilter, " "));
return filterSql;
}
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":
return " = ";
case "not_eq":
return " <> ";
case "lt":
return " < ";
case "le":
return " <= ";
case "gt":
return " > ";
case "ge":
return " >= ";
case "in":
return " IN ";
case "not in":
return " NOT IN ";
case "like":
return " LIKE ";
case "not like":
return " NOT LIKE ";
case "null":
return " IS NULL ";
case "not_null":
return " IS NOT NULL ";
default:
return "";
}
}
private void checkName(ChartViewWithBLOBs chartView) {
if (StringUtils.isEmpty(chartView.getId())) {
return;

131
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -8,7 +8,6 @@ import io.dataease.base.mapper.DatasetTableMapper;
import io.dataease.base.mapper.DatasourceMapper;
import io.dataease.commons.utils.*;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.datasource.constants.DatasourceTypes;
import io.dataease.datasource.dto.TableFiled;
import io.dataease.datasource.provider.DatasourceProvider;
import io.dataease.datasource.provider.JdbcProvider;
@ -18,6 +17,8 @@ import io.dataease.dto.dataset.DataSetPreviewPage;
import io.dataease.dto.dataset.DataSetTableUnionDTO;
import io.dataease.dto.dataset.DataTableInfoCustomUnion;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.provider.DDLProvider;
import io.dataease.provider.QueryProvider;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -126,9 +127,10 @@ public class DataSetTableService {
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery("drop table if exists " + dorisTableName);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(dorisDatasource.getType());
datasourceRequest.setQuery(ddlProvider.dropTableOrView(dorisTableName));
jdbcProvider.exec(datasourceRequest);
datasourceRequest.setQuery("drop table if exists " + DorisTableUtils.dorisTmpName(dorisTableName));
datasourceRequest.setQuery(ddlProvider.dropTableOrView(DorisTableUtils.dorisTmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
}
@ -224,14 +226,15 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
String table = dataTableInfoDTO.getTable();
datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize);
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize));
try {
data.addAll(datasourceProvider.getData(datasourceRequest));
} catch (Exception e) {
e.printStackTrace();
}
try {
datasourceRequest.setQuery(createQueryCountSQL(ds.getType(), table));
datasourceRequest.setQuery(qp.createQueryCountSQL(table));
dataSetPreviewPage.setTotal(Integer.valueOf(datasourceProvider.getData(datasourceRequest).get(0)[0]));
} catch (Exception e) {
e.printStackTrace();
@ -243,14 +246,15 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
String sql = dataTableInfoDTO.getSql();
datasourceRequest.setQuery(createQuerySQL(ds.getType(), " (" + sql + ") AS tmp ", fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize);
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLAsTmpWithPage(sql, fields, page, pageSize, realSize));
try {
data.addAll(datasourceProvider.getData(datasourceRequest));
} catch (Exception e) {
e.printStackTrace();
}
try {
datasourceRequest.setQuery(createQueryCountSQL(ds.getType(), " (" + sql + ") AS tmp "));
datasourceRequest.setQuery(qp.createQueryCountSQLAsTmp(sql));
dataSetPreviewPage.setTotal(Integer.valueOf(datasourceProvider.getData(datasourceRequest).get(0)[0]));
} catch (Exception e) {
e.printStackTrace();
@ -261,7 +265,8 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize);
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize));
try {
data.addAll(jdbcProvider.getData(datasourceRequest));
} catch (Exception e) {
@ -269,7 +274,7 @@ public class DataSetTableService {
}
try {
datasourceRequest.setQuery(createQueryCountSQL(ds.getType(), table));
datasourceRequest.setQuery(qp.createQueryCountSQL(table));
dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0]));
} catch (Exception e) {
e.printStackTrace();
@ -280,7 +285,8 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize);
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize));
try {
data.addAll(jdbcProvider.getData(datasourceRequest));
} catch (Exception e) {
@ -288,7 +294,7 @@ public class DataSetTableService {
}
try {
datasourceRequest.setQuery(createQueryCountSQL(ds.getType(), table));
datasourceRequest.setQuery(qp.createQueryCountSQL(table));
dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0]));
} catch (Exception e) {
e.printStackTrace();
@ -324,7 +330,8 @@ public class DataSetTableService {
datasourceRequest.setQuery(sql);
List<TableFiled> previewFields = datasourceProvider.fetchResultField(datasourceRequest);
// 正式执行
datasourceRequest.setQuery("SELECT * FROM (" + sql + ") AS tmp ORDER BY " + previewFields.get(0).getFieldName() + " LIMIT 0,1000");
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createSQLPreview(sql, previewFields.get(0).getFieldName()));
Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
List<String[]> data = result.get("dataList");
List<TableFiled> fields = result.get("fieldList");
@ -362,7 +369,8 @@ public class DataSetTableService {
datasourceRequest.setQuery(sql);
List<TableFiled> previewFields = jdbcProvider.fetchResultField(datasourceRequest);
datasourceRequest.setQuery("SELECT * FROM (" + sql + ") AS tmp ORDER BY " + previewFields.get(0).getFieldName() + " LIMIT 0,1000");
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createSQLPreview(sql, previewFields.get(0).getFieldName()));
Map<String, List> result = jdbcProvider.fetchResultAndField(datasourceRequest);
List<String[]> data = result.get("dataList");
List<TableFiled> fields = result.get("fieldList");
@ -400,6 +408,7 @@ public class DataSetTableService {
return map;
}
// 自助数据集从doris里预览数据
private String getCustomSQL(DataTableInfoDTO dataTableInfoDTO, List<DataSetTableUnionDTO> list) {
Map<String, String[]> customInfo = new TreeMap<>();
dataTableInfoDTO.getList().forEach(ele -> {
@ -499,6 +508,7 @@ public class DataSetTableService {
createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), getCustomSQL(dataTableInfoDTO, dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId())));
return;
}
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
if (CollectionUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
TableFiled filed = fields.get(i);
@ -516,7 +526,7 @@ public class DataSetTableService {
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(filed.getFieldType()));
} else {
Integer fieldType = transFieldType(ds.getType(), filed.getFieldType());
Integer fieldType = qp.transFieldType(filed.getFieldType());
datasetTableField.setDeType(fieldType == 4 ? 2 : fieldType);
datasetTableField.setDeExtractType(fieldType);
}
@ -534,56 +544,14 @@ public class DataSetTableService {
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(dorisDatasource.getType());
// 先删除表
datasourceRequest.setQuery("DROP VIEW IF EXISTS " + dorisTableName);
datasourceRequest.setQuery(ddlProvider.dropTableOrView(dorisTableName));
jdbcProvider.exec(datasourceRequest);
datasourceRequest.setQuery("CREATE VIEW IF NOT EXISTS " + dorisTableName + " AS (" + customSql + ")");
datasourceRequest.setQuery(ddlProvider.createView(dorisTableName, customSql));
jdbcProvider.exec(datasourceRequest);
}
public String createQueryCountSQL(String type, String table) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case mysql:
return MessageFormat.format("SELECT count(*) FROM {0}", table);
case sqlServer:
return MessageFormat.format("SELECT count(*) FROM {0}", table);
default:
return MessageFormat.format("SELECT count(*) FROM {0}", table);
}
}
public String createQuerySQL(String type, String table, List<DatasetTableField> fields) {
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
// 如果原始类型为时间
if (f.getDeExtractType() == 1) {
if (f.getDeType() == 2 || f.getDeType() == 3) {
stringBuilder.append("unix_timestamp(").append(f.getDataeaseName()).append(")*1000 as ").append(f.getDataeaseName());
} else {
stringBuilder.append(f.getDataeaseName());
}
} else {
if (f.getDeType() == 1) {
stringBuilder.append("FROM_UNIXTIME(cast(").append(f.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(f.getDataeaseName());
} else {
stringBuilder.append(f.getDataeaseName());
}
}
return stringBuilder.toString();
}).toArray(String[]::new);
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case mysql:
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table);
case sqlServer:
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table);
default:
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table);
}
}
public Integer transFieldType(String field) {
switch (field) {
case "TEXT":
@ -599,51 +567,6 @@ public class DataSetTableService {
}
}
public Integer transFieldType(String type, String field) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case mysql:
return transMysqlField(field);
case sqlServer:
default:
return 0;
}
}
public Integer transMysqlField(String field) {
switch (field) {
case "CHAR":
case "VARCHAR":
case "TEXT":
case "TINYTEXT":
case "MEDIUMTEXT":
case "LONGTEXT":
case "ENUM":
return 0;// 文本
case "DATE":
case "TIME":
case "YEAR":
case "DATETIME":
case "TIMESTAMP":
return 1;// 时间
case "INT":
case "SMALLINT":
case "MEDIUMINT":
case "INTEGER":
case "BIGINT":
return 2;// 整型
case "FLOAT":
case "DOUBLE":
case "DECIMAL":
return 3;// 浮点
case "BIT":
case "TINYINT":
return 4;// 布尔
default:
return 0;
}
}
public DatasetTableIncrementalConfig incrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) {
if (StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) {
return new DatasetTableIncrementalConfig();

82
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -19,6 +19,7 @@ import io.dataease.datasource.provider.ProviderFactory;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.dto.dataset.DataSetTaskLogDTO;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.provider.QueryProvider;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@ -109,15 +110,15 @@ public class ExtractDataService {
private static String shellScript = "curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load\n" +
"rm -rf %s\n";
private String createDorisTablColumnSql( List<DatasetTableField> datasetTableFields){
private String createDorisTablColumnSql(List<DatasetTableField> datasetTableFields) {
String Column_Fields = "dataease_uuid varchar(50), `";
for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` ";
switch (datasetTableField.getDeExtractType()){
switch (datasetTableField.getDeExtractType()) {
case 0:
if(datasetTableField.getSize() > 65533){
if (datasetTableField.getSize() > 65533) {
Column_Fields = Column_Fields + "String" + ",`";
}else {
} else {
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
}
break;
@ -138,23 +139,25 @@ public class ExtractDataService {
break;
}
}
Column_Fields = Column_Fields.substring(0, Column_Fields.length() -2);
Column_Fields = "(" + Column_Fields + ")\n";
Column_Fields = Column_Fields.substring(0, Column_Fields.length() - 2);
Column_Fields = "(" + Column_Fields + ")\n";
return Column_Fields;
}
private void createDorisTable(String dorisTableName, String dorisTablColumnSql) throws Exception{
Datasource dorisDatasource = (Datasource)CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);;
private void createDorisTable(String dorisTableName, String dorisTablColumnSql) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
;
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTablColumnSql));
jdbcProvider.exec(datasourceRequest);
}
private void replaceTable (String dorisTableName) throws Exception{
Datasource dorisDatasource = (Datasource)CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);;
private void replaceTable(String dorisTableName) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
;
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery("ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false');".replace("DORIS_TABLE", dorisTableName).replace("DORIS_TMP_TABLE", DorisTableUtils.dorisTmpName(dorisTableName)));
@ -168,9 +171,9 @@ public class ExtractDataService {
Datasource datasource = new Datasource();
try {
datasetTable = dataSetTableService.get(datasetTableId);
if(StringUtils.isNotEmpty(datasetTable.getDataSourceId())){
if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) {
datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
}else {
} else {
datasource.setType(datasetTable.getType());
}
@ -288,10 +291,10 @@ public class ExtractDataService {
String lastCarteObjectId = Job.sendToSlaveServer(jobMeta, jobExecutionConfiguration, repository, null);
SlaveServerJobStatus jobStatus = null;
boolean executing = true;
while(executing) {
while (executing) {
jobStatus = remoteSlaveServer.getJobStatus(jobMeta.getName(), lastCarteObjectId, 0);
executing = jobStatus.isRunning() || jobStatus.isWaiting();
if(!executing)
if (!executing)
break;
Thread.sleep(1000);
}
@ -320,7 +323,7 @@ public class ExtractDataService {
String dorisOutputTable = null;
String jobName = null;
String script = null;
Datasource dorisDatasource = (Datasource)CommonBeanFactory.getBean("DorisDatasource");
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
DorisConfigration dorisConfigration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfigration.class);
String columns = columnFeilds + ",dataease_uuid";
String transName = null;
@ -329,18 +332,18 @@ public class ExtractDataService {
transName = "trans_" + datasetTable.getId();
dorisOutputTable = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()));
jobName = "job_" + datasetTable.getId();
script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "APPEND",root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(),dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), dorisOutputTable, root_path + dorisOutputTable + "." + extention);
script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "APPEND", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(), dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), dorisOutputTable, root_path + dorisOutputTable + "." + extention);
break;
case "incremental_add":
transName = "trans_add_" + datasetTable.getId();
dorisOutputTable = DorisTableUtils.dorisName(datasetTable.getId());
jobName = "job_add_" + datasetTable.getId();
script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "APPEND", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(),dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), dorisOutputTable, root_path + dorisOutputTable + "." + extention);
script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "APPEND", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(), dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), dorisOutputTable, root_path + dorisOutputTable + "." + extention);
break;
case "incremental_delete":
transName = "trans_delete_" + datasetTable.getId();
dorisOutputTable = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId()));
script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "DELETE", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(),dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + dorisOutputTable + "." + extention);
script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "DELETE", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(), dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + dorisOutputTable + "." + extention);
jobName = "job_delete_" + datasetTable.getId();
break;
default:
@ -402,14 +405,14 @@ public class ExtractDataService {
FileUtils.writeStringToFile(file, jobXml, "UTF-8");
}
private String fetchSqlField(String sql, Datasource ds)throws Exception{
private String fetchSqlField(String sql, Datasource ds) throws Exception {
String tmpSql = sql;
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
if(tmpSql.trim().endsWith(";")){
tmpSql = tmpSql.substring(0, tmpSql.length() -1 ) + " limit 0";
}else {
if (tmpSql.trim().endsWith(";")) {
tmpSql = tmpSql.substring(0, tmpSql.length() - 1) + " limit 0";
} else {
tmpSql = tmpSql + " limit 0";
}
datasourceRequest.setQuery(tmpSql);
@ -433,9 +436,10 @@ public class ExtractDataService {
MysqlConfigration mysqlConfigration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfigration.class);
dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword());
transMeta.addDatabase(dataMeta);
if(extractType.equalsIgnoreCase("all_scope")){
if (extractType.equalsIgnoreCase("all_scope")) {
String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
selectSQL = dataSetTableService.createQuerySQL(datasource.getType(), tableName, datasetTableFields);
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
selectSQL = qp.createQuerySQL(tableName, datasetTableFields);
}
inputStep = inputStep(transMeta, selectSQL);
break;
@ -481,7 +485,7 @@ public class ExtractDataService {
FileUtils.writeStringToFile(file, transXml, "UTF-8");
}
private StepMeta inputStep(TransMeta transMeta, String selectSQL){
private StepMeta inputStep(TransMeta transMeta, String selectSQL) {
TableInputMeta tableInput = new TableInputMeta();
DatabaseMeta database = transMeta.findDatabase("db");
tableInput.setDatabaseMeta(database);
@ -492,7 +496,7 @@ public class ExtractDataService {
return fromStep;
}
private StepMeta excelInputStep(String filePath, List<DatasetTableField> datasetTableFields){
private StepMeta excelInputStep(String filePath, List<DatasetTableField> datasetTableFields) {
ExcelInputMeta excelInputMeta = new ExcelInputMeta();
excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI);
excelInputMeta.setPassword("Encrypted");
@ -500,7 +504,7 @@ public class ExtractDataService {
excelInputMeta.setStartsWithHeader(true);
excelInputMeta.setIgnoreEmptyRows(true);
ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()];
for (int i=0; i<datasetTableFields.size();i++) {
for (int i = 0; i < datasetTableFields.size(); i++) {
ExcelInputField field = new ExcelInputField();
field.setName(datasetTableFields.get(i).getOriginName());
field.setType("String");
@ -514,7 +518,7 @@ public class ExtractDataService {
return fromStep;
}
private StepMeta outputStep(String dorisOutputTable){
private StepMeta outputStep(String dorisOutputTable) {
TextFileOutputMeta textFileOutputMeta = new TextFileOutputMeta();
textFileOutputMeta.setEncoding("UTF-8");
textFileOutputMeta.setHeaderEnabled(false);
@ -528,10 +532,10 @@ public class ExtractDataService {
return outputStep;
}
private StepMeta udjc(List<DatasetTableField> datasetTableFields){
private StepMeta udjc(List<DatasetTableField> datasetTableFields) {
String needToChangeolumnType = "";
for (DatasetTableField datasetTableField : datasetTableFields) {
if(datasetTableField.getDeExtractType() != null && datasetTableField.getDeExtractType() == 4){
if (datasetTableField.getDeExtractType() != null && datasetTableField.getDeExtractType() == 4) {
needToChangeolumnType = needToChangeolumnType + alterColumnTypeCode.replace("FILED", datasetTableField.getOriginName());
}
}
@ -552,28 +556,28 @@ public class ExtractDataService {
StepMeta userDefinedJavaClassStep = new StepMeta("UserDefinedJavaClass", "UserDefinedJavaClass", userDefinedJavaClassMeta);
userDefinedJavaClassStep.setLocation(300, 100);
userDefinedJavaClassStep.setDraw(true);
return userDefinedJavaClassStep;
return userDefinedJavaClassStep;
}
public boolean isKettleRunning(){
public boolean isKettleRunning() {
try {
if (!InetAddress.getByName(carte).isReachable(1000)) {
return false;
}
HttpClient httpClient;
HttpGet getMethod = new HttpGet( "http://" + carte + ":" + port);
HttpGet getMethod = new HttpGet("http://" + carte + ":" + port);
HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder();
clientBuilder.setConnectionTimeout(1);
clientBuilder.setCredentials(user, passwd);
httpClient = clientBuilder.build();
HttpResponse httpResponse = httpClient.execute( getMethod );
HttpResponse httpResponse = httpClient.execute(getMethod);
int statusCode = httpResponse.getStatusLine().getStatusCode();
if ( statusCode != -1 && statusCode < 400) {
if (statusCode != -1 && statusCode < 400) {
return true;
}else {
} else {
return false;
}
}catch (Exception e){
} catch (Exception e) {
return false;
}
}

4
backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java
View File

@ -7,6 +7,7 @@ import io.dataease.datasource.provider.DatasourceProvider;
import io.dataease.datasource.provider.ProviderFactory;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.datasource.service.DatasourceService;
import io.dataease.provider.QueryProvider;
import io.dataease.service.dataset.DataSetFieldService;
import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService;
@ -58,7 +59,8 @@ public class DirectFieldService implements DataSetFieldService {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String querySQL = dataSetTableService.createQuerySQL(ds.getType(), tableName, Collections.singletonList(field));
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
String querySQL = qp.createQuerySQL(tableName, Collections.singletonList(field));
datasourceRequest.setQuery(querySQL);
try {
List<String[]> rows = datasourceProvider.getData(datasourceRequest);