center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat: pg

Browse Source
This commit is contained in:
taojinlong 2021-08-10 17:01:20 +08:00
parent 23f85147d8
commit 347b64eff2
9 changed files with 1041 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java
View File

@ -3,6 +3,7 @@ package io.dataease.datasource.constants;
public enum DatasourceTypes {
excel("excel", "excel", "", "", "", "", ""),
mysql("mysql", "mysql", "com.mysql.jdbc.Driver", "`", "`", "'", "'"),
pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""),
doris("doris", "doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\"");

21
backend/src/main/java/io/dataease/datasource/dto/PgConfigration.java Normal file
View File

@ -0,0 +1,21 @@
package io.dataease.datasource.dto;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class PgConfigration extends JdbcDTO {
private String driver = "org.postgresql.Driver";
private String schema;
public String getJdbc() {
// 连接参数先写死，后边要把编码、时区等参数放到数据源的设置中
return "jdbc:postgresql://HOSTNAME:PORT/DATABASE"
.replace("HOSTNAME", getHost())
.replace("PORT", getPort().toString())
.replace("DATABASE", getDataBase());
}
}

33
backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java
View File

@ -3,10 +3,7 @@ package io.dataease.datasource.provider;
import com.google.gson.Gson;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import io.dataease.datasource.constants.DatasourceTypes;
import io.dataease.datasource.dto.MysqlConfigration;
import io.dataease.datasource.dto.OracleConfigration;
import io.dataease.datasource.dto.SqlServerConfigration;
import io.dataease.datasource.dto.TableFiled;
import io.dataease.datasource.dto.*;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator;
@ -222,8 +219,10 @@ public class JdbcProvider extends DatasourceProvider {
public List<String> getSchema(DatasourceRequest datasourceRequest) throws Exception {
List<String> schemas = new ArrayList<>();
String queryStr = getSchemaSql(datasourceRequest);
System.out.println(queryStr);
Connection con = null;
try {
System.out.println(new Gson().toJson(datasourceRequest));
con = getConnection(datasourceRequest);
Statement statement = con.createStatement();
ResultSet resultSet = statement.executeQuery(queryStr);
@ -309,7 +308,7 @@ public class JdbcProvider extends DatasourceProvider {
resultSet.close();
ps.close();
} catch (Exception e) {
DataEaseException.throwException(Translator.get("i18n_datasource_connect_error") + e.getMessage());
DataEaseException.throwException(e.getMessage());
} finally {
if(con != null){con.close();}
}
@ -433,6 +432,12 @@ public class JdbcProvider extends DatasourceProvider {
props.put( "oracle.net.CONNECT_TIMEOUT" , "5000") ;
// props.put( "oracle.jdbc.ReadTimeout" , "5000" ) ;
break;
case pg:
PgConfigration pgConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfigration.class);
username = pgConfigration.getUsername();
password = pgConfigration.getPassword();
driver = pgConfigration.getDriver();
jdbcurl = pgConfigration.getJdbc();
default:
break;
}
@ -477,6 +482,13 @@ public class JdbcProvider extends DatasourceProvider {
dataSource.setPassword(oracleConfigration.getPassword());
dataSource.setJdbcUrl(oracleConfigration.getJdbc());
break;
case pg:
PgConfigration pgConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfigration.class);
dataSource.setUser(pgConfigration.getUsername());
dataSource.setDriverClass(pgConfigration.getDriver());
dataSource.setPassword(pgConfigration.getPassword());
dataSource.setJdbcUrl(pgConfigration.getJdbc());
break;
default:
break;
}
@ -494,6 +506,9 @@ public class JdbcProvider extends DatasourceProvider {
case sqlServer:
SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfigration.class);
return sqlServerConfigration.getDataBase();
case pg:
PgConfigration pgConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfigration.class);
return pgConfigration.getDataBase();
default:
return null;
}
@ -517,6 +532,12 @@ public class JdbcProvider extends DatasourceProvider {
throw new Exception(Translator.get("i18n_schema_is_empty"));
}
return "select table_name, owner from all_tables where owner='" + oracleConfigration.getSchema() + "'";
case pg:
PgConfigration pgConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfigration.class);
if(StringUtils.isEmpty(pgConfigration.getSchema())){
throw new Exception(Translator.get("i18n_schema_is_empty"));
}
return "SELECT tablename FROM pg_tables WHERE tablename NOT LIKE 'pg%' AND tablename NOT LIKE 'sql_%' AND schemaname='SCHEMA' ;".replace("SCHEMA", pgConfigration.getSchema());
default:
return "show tables;";
}
@ -529,6 +550,8 @@ public class JdbcProvider extends DatasourceProvider {
return "select * from all_users";
case sqlServer:
return "select name from sys.schemas;";
case pg:
return "SELECT nspname FROM pg_namespace;";
default:
return "show tables;";
}

4
backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java
View File

@ -28,6 +28,8 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("jdbc", DatasourceProvider.class);
case sqlServer:
return context.getBean("jdbc", DatasourceProvider.class);
case pg:
return context.getBean("jdbc", DatasourceProvider.class);
default:
return context.getBean("jdbc", DatasourceProvider.class);
}
@ -42,6 +44,8 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("dorisQuery", QueryProvider.class);
case sqlServer:
return context.getBean("sqlserverQuery", QueryProvider.class);
case pg:
return context.getBean("pgQuery", QueryProvider.class);
case oracle:
return context.getBean("oracleQuery", QueryProvider.class);
default:

43
backend/src/main/java/io/dataease/provider/pg/PgConstants.java Normal file
View File

@ -0,0 +1,43 @@
package io.dataease.provider.pg;
import io.dataease.provider.SQLConstants;
import static io.dataease.datasource.constants.DatasourceTypes.pg;
/**
* @Author gin
* @Date 2021/7/8 7:22 下午
*/
public class PgConstants extends SQLConstants {
public static final String KEYWORD_TABLE = pg.getKeywordPrefix() + "%s" + pg.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + pg.getKeywordPrefix() + "%s" + pg.getKeywordSuffix();
public static final String UNIX_TIMESTAMP = "floor(extract(epoch from(( %s - timestamp '1970-01-01 00:00:00')*1000))) ";
public static final String DATE_FORMAT = "to_char(%s, %s)";
public static final String FROM_UNIXTIME = "to_timestamp(%s)";
public static final String CAST = "CAST(%s AS %s)";
public static final String DEFAULT_DATE_FORMAT = "'YYYY-MM-DD HH24:MI:SS'";
public static final String DEFAULT_INT_FORMAT = "numeric(18,0)";
public static final String DEFAULT_FLOAT_FORMAT = "numeric(18,2)";
public static final String WHERE_VALUE_NULL = "(NULL,'')";
public static final String WHERE_VALUE_VALUE = "'%s'";
public static final String AGG_COUNT = "COUNT(*)";
public static final String AGG_FIELD = "%s(%s)";
public static final String WHERE_BETWEEN = "'%s' AND '%s'";
public static final String BRACKETS = "(%s)";
}

911
backend/src/main/java/io/dataease/provider/pg/PgQueryProvider.java Normal file
View File

@ -0,0 +1,911 @@
package io.dataease.provider.pg;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableFieldExample;
import io.dataease.base.domain.Datasource;
import io.dataease.base.mapper.DatasetTableFieldMapper;
import io.dataease.commons.constants.DeTypeConstants;
import io.dataease.controller.request.chart.ChartExtFilterRequest;
import io.dataease.dto.chart.ChartCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupFile;
import javax.annotation.Resource;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("pgQuery")
public class PgQueryProvider extends QueryProvider {
@Resource
private DatasetTableFieldMapper datasetTableFieldMapper;
@Override
public Integer transFieldType(String field) {
field = field.toLowerCase();
switch (field) {
case "bpchar":
case "varchar":
case "text":
case "tsquery":
case "tsvector":
case "uuid":
case "xml":
case "json":
case "bit":
case "jsonb":
case "cidr":
case "inet":
case "macaddr":
case "txid_snapshot":
case "box":
case "circle":
case "line":
case "lseg":
case "path":
case "point":
case "polygon":
case "bool":
return DeTypeConstants.DE_STRING;// 文本
case "date":
case "time":
case "timestamp":
case "timestamptz":
case "interval":
return DeTypeConstants.DE_TIME;// 时间
case "int2":
case "int4":
case "int8":
case "INTEGER":
case "BIGINT":
return DeTypeConstants.DE_INT;// 整型
case "numeric":
case "float4":
case "float8":
case "money":
return DeTypeConstants.DE_FLOAT;// 浮点
// case "bool":
case "TINYINT":
return DeTypeConstants.DE_BOOL;// 布尔
case "bytea":
return DeTypeConstants.DE_Binary;// 二进制
default:
return DeTypeConstants.DE_STRING;
}
}
@Override
public String createSQLPreview(String sql, String orderBy) {
return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp " + " LIMIT 1000 offset 0";
}
@Override
public String createQuerySQL(String table, List<DatasetTableField> fields, boolean isGroup, Datasource ds) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
if (CollectionUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
DatasetTableField f = fields.get(i);
String originField;
if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(f.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
} else {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
String fieldName = "";
// 处理横轴字段
if (f.getDeExtractType() == DeTypeConstants.DE_TIME) {
if (f.getDeType() == DeTypeConstants.DE_INT || f.getDeType() == DeTypeConstants.DE_FLOAT) {
fieldName = String.format( PgConstants.UNIX_TIMESTAMP, originField);
} else {
fieldName = originField;
}
} else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) {
if (f.getDeType() == DeTypeConstants.DE_INT) {
fieldName = String.format( PgConstants.CAST, originField, PgConstants.DEFAULT_INT_FORMAT);
} else if (f.getDeType() == DeTypeConstants.DE_FLOAT) {
fieldName = String.format( PgConstants.CAST, originField, PgConstants.DEFAULT_FLOAT_FORMAT);
} else if (f.getDeType() == DeTypeConstants.DE_TIME) {
fieldName = String.format( PgConstants.CAST, originField, "timestamp");
} else {
fieldName = originField;
}
} else {
if (f.getDeType() == DeTypeConstants.DE_TIME) {
String cast = String.format( PgConstants.CAST, originField, "bigint");
fieldName = String.format( PgConstants.FROM_UNIXTIME, cast );
} else if (f.getDeType() == DeTypeConstants.DE_INT) {
fieldName = String.format( PgConstants.CAST, originField, PgConstants.DEFAULT_INT_FORMAT);
} else {
fieldName = originField;
}
}
xFields.add(SQLObj.builder()
.fieldName(fieldName)
.fieldAlias(fieldAlias)
.build());
}
}
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("previewSql");
st_sql.add("isGroup", isGroup);
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
return st_sql.render();
}
@Override
public String createQuerySQLAsTmp(String sql, List<DatasetTableField> fields, boolean isGroup) {
return createQuerySQL("(" + sqlFix(sql) + ")", fields, isGroup, null);
}
@Override
public String createQuerySQLWithPage(String table, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds) {
return createQuerySQL(table, fields, isGroup, null) + " LIMIT " + realSize + " offset " + (page - 1) * pageSize;
}
@Override
public String createQueryTableWithLimit(String table, List<DatasetTableField> fields, Integer limit, boolean isGroup, Datasource ds) {
return createQuerySQL(table, fields, isGroup, null) + " LIMIT " + limit + " offset 0";
}
@Override
public String createQuerySqlWithLimit(String sql, List<DatasetTableField> fields, Integer limit, boolean isGroup) {
return createQuerySQLAsTmp(sql, fields, isGroup) + " LIMIT " + limit + " offset 0";
}
@Override
public String createQuerySQLAsTmpWithPage(String sql, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup) {
return createQuerySQLAsTmp(sql, fields, isGroup) + " LIMIT " + realSize + " offset " + (page - 1) * pageSize;
}
@Override
public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xWheres = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(xAxis)) {
for (int i = 0; i < xAxis.size(); i++) {
ChartViewFieldDTO x = xAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴过滤
// xWheres.addAll(getXWheres(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
List<SQLObj> yFields = new ArrayList<>();
List<SQLObj> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(yAxis)) {
for (int i = 0; i < yAxis.size(); i++) {
ChartViewFieldDTO y = yAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.addAll(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
List<SQLObj> customWheres = transCustomFilterList(tableObj, customFilter);
// 处理仪表板字段过滤
List<SQLObj> extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
fields.addAll(yFields);
List<SQLObj> wheres = new ArrayList<>();
wheres.addAll(xWheres);
if (customWheres != null) wheres.addAll(customWheres);
if (extWheres != null) wheres.addAll(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
orders.addAll(yOrders);
List<SQLObj> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres);
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format( PgConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return st.render();
}
@Override
public String getSQLAsTmp(String sql, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) {
return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList);
}
@Override
public String getSQLStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xWheres = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
List<ChartViewFieldDTO> xList = new ArrayList<>();
xList.addAll(xAxis);
xList.addAll(extStack);
if (CollectionUtils.isNotEmpty(xList)) {
for (int i = 0; i < xList.size(); i++) {
ChartViewFieldDTO x = xList.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴过滤
// xWheres.addAll(getXWheres(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
List<SQLObj> yFields = new ArrayList<>();
List<SQLObj> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(yAxis)) {
for (int i = 0; i < yAxis.size(); i++) {
ChartViewFieldDTO y = yAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.addAll(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
List<SQLObj> customWheres = transCustomFilterList(tableObj, customFilter);
// 处理仪表板字段过滤
List<SQLObj> extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
fields.addAll(yFields);
List<SQLObj> wheres = new ArrayList<>();
wheres.addAll(xWheres);
if (customWheres != null) wheres.addAll(customWheres);
if (extWheres != null) wheres.addAll(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
orders.addAll(yOrders);
List<SQLObj> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres);
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format( PgConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return st.render();
}
@Override
public String getSQLAsTmpStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack) {
return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack);
}
@Override
public String getSQLScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xWheres = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(xAxis)) {
for (int i = 0; i < xAxis.size(); i++) {
ChartViewFieldDTO x = xAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴过滤
// xWheres.addAll(getXWheres(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
List<SQLObj> yFields = new ArrayList<>();
List<SQLObj> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
List<ChartViewFieldDTO> yList = new ArrayList<>();
yList.addAll(yAxis);
yList.addAll(extBubble);
if (CollectionUtils.isNotEmpty(yList)) {
for (int i = 0; i < yList.size(); i++) {
ChartViewFieldDTO y = yList.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.addAll(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
List<SQLObj> customWheres = transCustomFilterList(tableObj, customFilter);
// 处理仪表板字段过滤
List<SQLObj> extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
fields.addAll(yFields);
List<SQLObj> wheres = new ArrayList<>();
wheres.addAll(xWheres);
if (customWheres != null) wheres.addAll(customWheres);
if (extWheres != null) wheres.addAll(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
orders.addAll(yOrders);
List<SQLObj> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres);
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format( PgConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return st.render();
}
@Override
public String getSQLAsTmpScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble) {
return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble);
}
@Override
public String searchTable(String table) {
return "SELECT table_name FROM information_schema.TABLES WHERE table_name ='" + table + "'";
}
@Override
public String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) {
// 字段汇总 排序等
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> yFields = new ArrayList<>();
List<SQLObj> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(yAxis)) {
for (int i = 0; i < yAxis.size(); i++) {
ChartViewFieldDTO y = yAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.addAll(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
List<SQLObj> customWheres = transCustomFilterList(tableObj, customFilter);
// 处理仪表板字段过滤
List<SQLObj> extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(yFields);
List<SQLObj> wheres = new ArrayList<>();
if (customWheres != null) wheres.addAll(customWheres);
if (extWheres != null) wheres.addAll(extWheres);
List<SQLObj> groups = new ArrayList<>();
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(yOrders);
List<SQLObj> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres);
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format( PgConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return st.render();
}
@Override
public String getSQLSummaryAsTmp(String sql, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) {
return getSQLSummary("(" + sqlFix(sql) + ")", yAxis, customFilter, extFilterRequestList);
}
@Override
public String wrapSql(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0 ";
return tmpSql;
}
@Override
public String createRawQuerySQL(String table, List<DatasetTableField> fields) {
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
// if (f.getDeExtractType() == 4) { // 处理 tinyint
// stringBuilder.append("concat(`").append(f.getOriginName()).append("`,'') AS ").append(f.getDataeaseName());
// } else {
// stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName());
// }
stringBuilder.append("\"").append(f.getOriginName()).append("\" AS ").append(f.getDataeaseName());
return stringBuilder.toString();
}).toArray(String[]::new);
return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), table);
}
@Override
public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields);
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":
return " = ";
case "not_eq":
return " <> ";
case "lt":
return " < ";
case "le":
return " <= ";
case "gt":
return " > ";
case "ge":
return " >= ";
case "in":
return " IN ";
case "not in":
return " NOT IN ";
case "like":
return " LIKE ";
case "not like":
return " NOT LIKE ";
case "null":
return " IN ";
case "not_null":
return " IS NOT NULL AND %s <> ''";
case "between":
return " BETWEEN ";
default:
return "";
}
}
public List<SQLObj> transCustomFilterList(SQLObj tableObj, List<ChartCustomFilterDTO> requestList) {
if (CollectionUtils.isEmpty(requestList)) {
return null;
}
List<SQLObj> list = new ArrayList<>();
for (ChartCustomFilterDTO request : requestList) {
DatasetTableField field = request.getField();
if (ObjectUtils.isEmpty(field)) {
continue;
}
String value = request.getValue();
String whereName = "";
String whereTerm = transMysqlFilterTerm(request.getTerm());
String whereValue = "";
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originName = calcFieldRegex(field.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
}
if (field.getDeType() == 1 && field.getDeExtractType() != 1) {
String cast = String.format( PgConstants.CAST, originName, "bigint");
whereName = String.format( PgConstants.FROM_UNIXTIME, cast);
} else {
whereName = originName;
}
if (StringUtils.equalsIgnoreCase(request.getTerm(), "null")) {
whereValue = PgConstants.WHERE_VALUE_NULL;
} else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_null")) {
whereTerm = String.format(whereTerm, originName);
} else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {
whereValue = String.format( PgConstants.WHERE_VALUE_VALUE, value);
}
list.add(SQLObj.builder()
.whereField(whereName)
.whereTermAndValue(whereTerm + whereValue)
.build());
}
return list;
}
public List<SQLObj> transExtFilterList(SQLObj tableObj, List<ChartExtFilterRequest> requestList) {
if (CollectionUtils.isEmpty(requestList)) {
return null;
}
List<SQLObj> list = new ArrayList<>();
for (ChartExtFilterRequest request : requestList) {
List<String> value = request.getValue();
DatasetTableField field = request.getDatasetTableField();
if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) {
continue;
}
String whereName = "";
String whereTerm = transMysqlFilterTerm(request.getOperator());
String whereValue = "";
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originName = calcFieldRegex(field.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
}
if (field.getDeType() == 1 && field.getDeExtractType() != 1) {
String cast = String.format( PgConstants.CAST, originName, "bigint");
whereName = String.format( PgConstants.FROM_UNIXTIME, cast);
} else {
whereName = originName;
}
if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) {
whereValue = "'%" + value.get(0) + "%'";
} else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) {
if (request.getDatasetTableField().getDeType() == 1) {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0))));
String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1))));
whereValue = String.format( PgConstants.WHERE_BETWEEN, startTime, endTime);
} else {
whereValue = String.format( PgConstants.WHERE_BETWEEN, value.get(0), value.get(1));
}
} else {
whereValue = String.format( PgConstants.WHERE_VALUE_VALUE, value.get(0));
}
list.add(SQLObj.builder()
.whereField(whereName)
.whereTermAndValue(whereTerm + whereValue)
.build());
}
return list;
}
private String sqlFix(String sql) {
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
return sql;
}
private String transDateFormat(String dateStyle, String datePattern) {
String split = "-";
if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) {
split = "-";
} else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) {
split = "/";
}
switch (dateStyle) {
case "y":
return "'YYYY'";
case "y_M":
return "'YYYY" + split + "MM'";
case "y_M_d":
return "'YYYY" + split + "MM" + split + "DD'";
case "H_m_s":
return "'HH24:MI:SS'";
case "y_M_d_H_m":
return "'YYYY" + split + "MM" + split + "DD" + " HH24:MI'";
case "y_M_d_H_m_s":
return "'YYYY" + split + "MM" + split + "DD" + " %H:%i:%S'";
default:
return "'YYYY-MM-DD HH24:MI:SS'";
}
}
private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) {
String fieldName = "";
if (x.getDeExtractType() == DeTypeConstants.DE_TIME) {
if (x.getDeType() == 2 || x.getDeType() == 3) {
fieldName = String.format( PgConstants.UNIX_TIMESTAMP, originField);
} else if (x.getDeType() == DeTypeConstants.DE_TIME) {
String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
fieldName = String.format( PgConstants.DATE_FORMAT, originField, format);
} else {
fieldName = originField;
}
} else {
if (x.getDeType() == DeTypeConstants.DE_TIME) {
String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
if (x.getDeExtractType() == DeTypeConstants.DE_STRING) {
fieldName = String.format( PgConstants.DATE_FORMAT, originField, format);
} else {
String cast = String.format( PgConstants.CAST, originField, "bigint");
String from_unixtime = String.format( PgConstants.FROM_UNIXTIME, cast);
fieldName = String.format( PgConstants.DATE_FORMAT, from_unixtime, format);
}
} else {
fieldName = originField;
}
}
return SQLObj.builder()
.fieldName(fieldName)
.fieldAlias(fieldAlias)
.build();
}
private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) {
String fieldName = "";
if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) {
fieldName = PgConstants.AGG_COUNT;
} else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) {
fieldName = String.format( PgConstants.AGG_FIELD, y.getSummary(), originField);
} else {
if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) {
String cast = String.format( PgConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? PgConstants.DEFAULT_INT_FORMAT : PgConstants.DEFAULT_FLOAT_FORMAT);
String agg = String.format( PgConstants.AGG_FIELD, y.getSummary(), cast);
fieldName = String.format( PgConstants.CAST, agg, PgConstants.DEFAULT_FLOAT_FORMAT);
} else {
String cast = String.format( PgConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? PgConstants.DEFAULT_INT_FORMAT : PgConstants.DEFAULT_FLOAT_FORMAT);
fieldName = String.format( PgConstants.AGG_FIELD, y.getSummary(), cast);
}
}
return SQLObj.builder()
.fieldName(fieldName)
.fieldAlias(fieldAlias)
.build();
}
private List<SQLObj> getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) {
List<SQLObj> list = new ArrayList<>();
if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) {
y.getFilter().forEach(f -> {
String whereTerm = transMysqlFilterTerm(f.getTerm());
String whereValue = "";
// 原始类型不是时间，在de中被转成时间的字段做处理
if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) {
whereValue = PgConstants.WHERE_VALUE_NULL;
} else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) {
whereTerm = String.format(whereTerm, originField);
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')";
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
whereValue = "'%" + f.getValue() + "%'";
} else {
whereValue = String.format( PgConstants.WHERE_VALUE_VALUE, f.getValue());
}
list.add(SQLObj.builder()
.whereField(fieldAlias)
.whereAlias(fieldAlias)
.whereTermAndValue(whereTerm + whereValue)
.build());
});
}
return list;
}
private String calcFieldRegex(String originField, SQLObj tableObj) {
originField = originField.replaceAll("[\\t\\n\\r]]", "");
// 正则提取[xxx]
String regex = "\\[(.*?)]";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(originField);
Set<String> ids = new HashSet<>();
while (matcher.find()) {
String id = matcher.group(1);
ids.add(id);
}
if (CollectionUtils.isEmpty(ids)) {
return originField;
}
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids));
List<DatasetTableField> calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample);
for (DatasetTableField ele : calcFields) {
originField = originField.replaceAll("\\[" + ele.getId() + "]",
String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName()));
}
return originField;
}
}

5
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -379,6 +379,7 @@ public class DataSetTableService {
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
map.put("sql", datasourceRequest.getQuery());
System.out.println(datasourceRequest.getQuery());
try {
data.addAll(datasourceProvider.getData(datasourceRequest));
} catch (Exception e) {
@ -903,7 +904,9 @@ public class DataSetTableService {
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(filed.getFieldType()));
} else {
System.out.println(new Gson().toJson(filed));
Integer fieldType = qp.transFieldType(filed.getFieldType());
System.out.println(fieldType);
datasetTableField.setDeType(fieldType == 4 ? 2 : fieldType);
datasetTableField.setDeExtractType(fieldType);
}
@ -1087,6 +1090,8 @@ public class DataSetTableService {
List<TableFiled> fields = (List<TableFiled>) fileMap.get("fields");
List<String> newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList());
List<String> oldFields = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList());
System.out.println("oldFields: " + oldFields);
System.out.println("newFields: "+ newFields);
if (!oldFields.equals(newFields)) {
DataEaseException.throwException(Translator.get("i18n_excel_colume_change"));
}

16
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -192,7 +192,7 @@ public class ExtractDataService {
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
} finally {
deleteFile("all_scope", datasetTableId);
deleteFile(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData());
// deleteFile(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData());
}
break;
@ -930,13 +930,15 @@ public class ExtractDataService {
}
if (datasourceType.equals(DatasourceTypes.excel)) {
System.out.println(Column_Fields);
tmp_code = tmp_code.replace("handleExcelIntColumn", handleExcelIntColumn).replace("Column_Fields", Column_Fields);
} else {
tmp_code = tmp_code.replace("handleExcelIntColumn", "").replace("Column_Fields", Column_Fields);
}
tmp_code = tmp_code.replace("handleBinaryType", handleBinaryTypeCode);
System.out.println(tmp_code);
UserDefinedJavaClassDef userDefinedJavaClassDef = new UserDefinedJavaClassDef(UserDefinedJavaClassDef.ClassType.TRANSFORM_CLASS, "Processor", tmp_code);
userDefinedJavaClassDef.setActive(true);
@ -1019,13 +1021,13 @@ public class ExtractDataService {
}
}
private static String handleBinaryType = " \t\tif(\"FEILD\".equalsIgnoreCase(filed)){\n" +
private final static String handleBinaryType = " \t\tif(\"FEILD\".equalsIgnoreCase(filed)){\n" +
" get(Fields.Out, filed).setValue(r, \"\");\n" +
" get(Fields.Out, filed).getValueMeta().setType(2);\n" +
" \t}";
private static String alterColumnTypeCode = " if(\"FILED\".equalsIgnoreCase(filed)){\n" +
private final static String alterColumnTypeCode = " if(\"FILED\".equalsIgnoreCase(filed)){\n" +
"\t if(tmp != null && tmp.equalsIgnoreCase(\"Y\")){\n" +
" get(Fields.Out, filed).setValue(r, 1);\n" +
" get(Fields.Out, filed).getValueMeta().setType(2);\n" +
@ -1035,7 +1037,7 @@ public class ExtractDataService {
" }\n" +
" }\n";
private static String handleExcelIntColumn = " \t\tif(tmp != null && tmp.endsWith(\".0\")){\n" +
private final static String handleExcelIntColumn = " \t\tif(tmp != null && tmp.endsWith(\".0\")){\n" +
" try {\n" +
" Long.valueOf(tmp.substring(0, tmp.length()-2));\n" +
" get(Fields.Out, filed).setValue(r, tmp.substring(0, tmp.length()-2));\n" +
@ -1043,14 +1045,14 @@ public class ExtractDataService {
" }catch (Exception e){}\n" +
" }";
private static String handleWraps = " if(tmp != null && ( tmp.contains(\"\\r\") || tmp.contains(\"\\n\"))){\n" +
private final static String handleWraps = " if(tmp != null && ( tmp.contains(\"\\r\") || tmp.contains(\"\\n\"))){\n" +
"\t\t\ttmp = tmp.trim();\n" +
" tmp = tmp.replaceAll(\"\\r\",\" \");\n" +
" tmp = tmp.replaceAll(\"\\n\",\" \");\n" +
" get(Fields.Out, filed).setValue(r, tmp);\n" +
" } ";
private static String code = "import org.pentaho.di.core.row.ValueMetaInterface;\n" +
private final static String code = "import org.pentaho.di.core.row.ValueMetaInterface;\n" +
"import java.util.List;\n" +
"import java.io.File;\n" +
"import java.security.MessageDigest;\n" +

37
frontend/src/views/system/datasource/form.vue
View File

@ -46,13 +46,13 @@
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.port')" prop="configuration.port">
<el-input v-model="form.configuration.port" autocomplete="off" />
</el-form-item>
<el-form-item v-if="form.type=='oracle'">
<el-form-item v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg'">
<el-button icon="el-icon-plus" size="mini" @click="getSchema()">
{{ $t('datasource.get_schema') }}
</el-button>
</el-form-item>
<el-form-item v-if="form.type=='oracle'" :label="$t('datasource.schema')">
<el-form-item v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg'" :label="$t('datasource.schema')">
<el-select filterable v-model="form.configuration.schema" :placeholder="$t('datasource.please_choose_schema')" class="select-width">
<el-option
v-for="item in schemas"
@ -64,21 +64,21 @@
</el-form-item>
<el-form-item v-if="form.type=='sqlServer'">
<el-button icon="el-icon-plus" size="mini" @click="getSchema()">
{{ $t('datasource.get_schema') }}
</el-button>
</el-form-item>
<el-form-item v-if="form.type=='sqlServer'" :label="$t('datasource.schema')">
<el-select filterable v-model="form.configuration.schema" :placeholder="$t('datasource.please_choose_schema')" class="select-width">
<el-option
v-for="item in schemas"
:key="item"
:label="item"
:value="item"
/>
</el-select>
</el-form-item>
<!-- <el-form-item v-if="form.type=='sqlServer'">-->
<!-- <el-button icon="el-icon-plus" size="mini" @click="getSchema()">-->
<!-- {{ $t('datasource.get_schema') }}-->
<!-- </el-button>-->
<!-- </el-form-item>-->
<!-- <el-form-item v-if="form.type=='sqlServer'" :label="$t('datasource.schema')">-->
<!-- <el-select filterable v-model="form.configuration.schema" :placeholder="$t('datasource.please_choose_schema')" class="select-width">-->
<!-- <el-option-->
<!-- v-for="item in schemas"-->
<!-- :key="item"-->
<!-- :label="item"-->
<!-- :value="item"-->
<!-- />-->
<!-- </el-select>-->
<!-- </el-form-item>-->
</el-form>
<div v-if="canEdit" slot="footer" class="dialog-footer">
@ -124,7 +124,8 @@ export default {
},
allTypes: [{ name: 'mysql', label: 'MySQL', type: 'jdbc' },
{ name: 'oracle', label: 'Oracle', type: 'jdbc' },
{ name: 'sqlServer', label: 'SQLSERVER', type: 'jdbc' }],
{ name: 'sqlServer', label: 'SQLSERVER', type: 'jdbc' },
{ name: 'pg', label: 'PostgreSQL', type: 'jdbc' }],
schemas: [],
canEdit: false,
originConfiguration: {}