center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge pull request #789 from dataease/pr@dev@es

Browse Source 
feat: 支持直连模式（elasticsearch）
This commit is contained in:
taojinlong 2021-09-08 15:51:01 +08:00 committed by GitHub
commit 43d0f227e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 1290 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java
View File

@ -6,7 +6,8 @@ public enum DatasourceTypes {
pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""),
doris("doris", "doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\"");
oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""),
es("es", "es", "", "\"", "\"", "\"", "\"");
private String feature;
private String desc;

14
backend/src/main/java/io/dataease/datasource/dto/EsConfigDTO.java Normal file
View File

@ -0,0 +1,14 @@
package io.dataease.datasource.dto;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class EsConfigDTO {
private String url;
private String username;
private String password;
private String dataSourceType = "es";
}

28
backend/src/main/java/io/dataease/datasource/dto/es/EsReponse.java Normal file
View File

@ -0,0 +1,28 @@
package io.dataease.datasource.dto.es;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class EsReponse {
private List<Column>columns = new ArrayList<>();
private List<String[]>rows = new ArrayList<>();
private String cursor;
private Integer status;
private Error error;
@Data
public class Error{
private String type;
private String reason;
}
@Data
public class Column {
private String name;
private String type;
}
}

9
backend/src/main/java/io/dataease/datasource/dto/es/Requst.java Normal file
View File

@ -0,0 +1,9 @@
package io.dataease.datasource.dto.es;
import lombok.Data;
@Data
public class Requst {
private String query;
private Integer fetch_size = 10000;
}

8
backend/src/main/java/io/dataease/datasource/dto/es/RequstWithCursor.java Normal file
View File

@ -0,0 +1,8 @@
package io.dataease.datasource.dto.es;
import lombok.Data;
@Data
public class RequstWithCursor extends Requst{
private String cursor;
}

221
backend/src/main/java/io/dataease/datasource/provider/EsProvider.java Normal file
View File

@ -0,0 +1,221 @@
package io.dataease.datasource.provider;
import com.google.gson.Gson;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import io.dataease.commons.utils.HttpClientConfig;
import io.dataease.commons.utils.HttpClientUtil;
import io.dataease.datasource.dto.*;
import io.dataease.datasource.dto.es.EsReponse;
import io.dataease.datasource.dto.es.Requst;
import io.dataease.datasource.dto.es.RequstWithCursor;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.exception.DataEaseException;
import io.dataease.provider.es.EsQueryProvider;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHeaders;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
import java.util.*;
@Service("es")
public class EsProvider extends DatasourceProvider {
private static Map<String, ComboPooledDataSource> jdbcConnection = new HashMap<>();
private static int initPoolSize = 5;
private static int maxConnections = 200;
/**
* 增加缓存机制 key 由 'provider_sql_' dsr.datasource.id dsr.table dsr.query共4部分组成，命中则使用缓存直接返回不再执行sql逻辑
* @param dsr
* @return
* @throws Exception
*/
/**
* 这里使用声明式缓存不是很妥当
* 改为chartViewService中使用编程式缓存
@Cacheable(
value = JdbcConstants.JDBC_PROVIDER_KEY,
key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query",
condition = "#dsr.pageSize == null || #dsr.pageSize == 0L"
)
*/
@Override
public List<String[]> getData(DatasourceRequest dsr) throws Exception {
List<String[]> list = new LinkedList<>();
try {
EsConfigDTO esConfigDTO = new Gson().fromJson(dsr.getDatasource().getConfiguration(), EsConfigDTO.class);
HttpClientConfig httpClientConfig = new HttpClientConfig();
String auth = esConfigDTO.getUsername() + ":" + esConfigDTO.getPassword();
byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
Requst requst = new Requst();
requst.setQuery(dsr.getQuery());
requst.setFetch_size(dsr.getFetchSize());
String url = esConfigDTO.getUrl().endsWith("/") ? esConfigDTO.getUrl() + "_sql?format=json" : esConfigDTO.getUrl() + "/" + "_sql?format=json";
String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig);
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
list.addAll(fetchResult(esReponse));
if(dsr.isPageable()){
Integer realSize = dsr.getPage() * dsr.getPageSize() < list.size() ? dsr.getPage() * dsr.getPageSize(): list.size();
list = list.subList((dsr.getPage() - 1) * dsr.getPageSize(), realSize);
}
if(!dsr.isPreviewData()){
while (StringUtils.isNotEmpty(esReponse.getCursor())) {
RequstWithCursor requstWithCursor = new RequstWithCursor();
requstWithCursor.setQuery(dsr.getQuery());
requstWithCursor.setFetch_size(dsr.getFetchSize());
requstWithCursor.setCursor(esReponse.getCursor());
response = HttpClientUtil.post(url, new Gson().toJson(requstWithCursor), httpClientConfig);
esReponse = new Gson().fromJson(response, EsReponse.class);
list.addAll(fetchResult(esReponse));
}
}
} catch (Exception e) {
DataEaseException.throwException(e);
}
return list;
}
@Override
public List<String[]> fetchResult(DatasourceRequest datasourceRequest) throws Exception {
List<String[]> list = new LinkedList<>();
try {
String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json");
list = fetchResult(response);
} catch (Exception e) {
DataEaseException.throwException(e);
}
return list;
}
private List<String[]> fetchResult(String response) throws Exception {
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
return fetchResult(esReponse);
}
private List<String[]> fetchResult(EsReponse esReponse) throws Exception {
List<String[]> list = new LinkedList<>();
if(esReponse.getError() != null){
throw new Exception(esReponse.getError().getReason());
}
list.addAll(esReponse.getRows());
return list;
}
@Override
public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
List<TableFiled> tableFileds = new ArrayList<>();
try {
String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json");
tableFileds = fetchResultField(response);
} catch (Exception e) {
DataEaseException.throwException(e);
}
return tableFileds;
}
private List<TableFiled> fetchResultField(String response) throws Exception {
List<TableFiled> fieldList = new ArrayList<>();
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
if(esReponse.getError() != null){
throw new Exception(esReponse.getError().getReason());
}
for (String[] row : esReponse.getRows()) {
TableFiled field = new TableFiled();
field.setFieldName(row[0]);
field.setRemarks(row[0]);
field.setFieldType(row[2]);
field.setFieldSize(EsQueryProvider.transFieldTypeSize(row[2]));
fieldList.add(field);
}
return fieldList;
}
@Override
public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception {
Map<String, List> result = new HashMap<>();
try {
String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json");
result.put("dataList", fetchResult(response));
result.put("fieldList", fetchResultField(response));
} catch (Exception e) {
DataEaseException.throwException(e);
}
return result;
}
@Override
public void handleDatasource(DatasourceRequest datasourceRequest, String type) throws Exception {
}
@Override
public List<String> getTables(DatasourceRequest datasourceRequest) throws Exception {
List<String> tables = new ArrayList<>();
try {
String response = exexQuery(datasourceRequest, "show tables", "_sql?format=json");
tables = fetchTables(response);
} catch (Exception e) {
DataEaseException.throwException(e);
}
return tables;
}
private List<String> fetchTables(String response) throws Exception {
List<String> tables = new ArrayList<>();
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
if(esReponse.getError() != null){
throw new Exception(esReponse.getError().getReason());
}
for (String[] row : esReponse.getRows()) {
if(row.length == 3 && row[1].equalsIgnoreCase("TABLE") && row[2].equalsIgnoreCase("INDEX")){
tables.add(row[0]);
}
}
return tables;
}
@Override
public List<String> getSchema(DatasourceRequest datasourceRequest) throws Exception {
return new ArrayList<>();
}
@Override
public List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
List<TableFiled> tableFileds = new ArrayList<>();
try {
String response = exexQuery(datasourceRequest, "desc " + datasourceRequest.getTable(), "_sql?format=json");
tableFileds = fetchResultField(response);
} catch (Exception e) {
DataEaseException.throwException(e);
}
return tableFileds;
}
@Override
public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
getTables(datasourceRequest);
}
private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri){
EsConfigDTO esConfigDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfigDTO.class);
HttpClientConfig httpClientConfig = new HttpClientConfig();
String auth = esConfigDTO.getUsername() + ":" + esConfigDTO.getPassword();
byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
Requst requst = new Requst();
requst.setQuery(sql);
requst.setFetch_size(datasourceRequest.getFetchSize());
String url = esConfigDTO.getUrl().endsWith("/") ? esConfigDTO.getUrl() + uri : esConfigDTO.getUrl() + "/" + uri;
String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig);
return response;
}
}

4
backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java
View File

@ -30,6 +30,8 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("jdbc", DatasourceProvider.class);
case pg:
return context.getBean("jdbc", DatasourceProvider.class);
case es:
return context.getBean("es", DatasourceProvider.class);
default:
return context.getBean("jdbc", DatasourceProvider.class);
}
@ -48,6 +50,8 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("pgQuery", QueryProvider.class);
case oracle:
return context.getBean("oracleQuery", QueryProvider.class);
case es:
return context.getBean("esQuery", QueryProvider.class);
default:
return context.getBean("mysqlQuery", QueryProvider.class);
}

8
backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java
View File

@ -11,8 +11,12 @@ public class DatasourceRequest {
protected String query;
protected String table;
protected Datasource datasource;
private Long pageSize;
private Long startPage;
private Integer pageSize;
private Integer page;
private Integer realSize;
private Integer fetchSize = 10000;
private boolean pageable = false;
private boolean previewData = false;
}

933
backend/src/main/java/io/dataease/provider/es/EsQueryProvider.java Normal file
View File

@ -0,0 +1,933 @@
package io.dataease.provider.es;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableFieldExample;
import io.dataease.base.domain.Datasource;
import io.dataease.base.mapper.DatasetTableFieldMapper;
import io.dataease.commons.constants.DeTypeConstants;
import io.dataease.controller.request.chart.ChartExtFilterRequest;
import io.dataease.dto.chart.ChartCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupFile;
import javax.annotation.Resource;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("esQuery")
public class EsQueryProvider extends QueryProvider {
@Resource
private DatasetTableFieldMapper datasetTableFieldMapper;
@Override
public Integer transFieldType(String field) {
switch (field) {
case "keyword":
case "text":
case "binary":
case "object":
case "nested":
case "null":
case "ip":
case "unsupported":
return DeTypeConstants.DE_STRING;
case "boolean":
case "byte":
case "short":
case "integer":
case "long":
return DeTypeConstants.DE_INT;
case "double":
case "float":
case "half_float":
case "scaled_float":
return DeTypeConstants.DE_FLOAT;
case "datetime":
return DeTypeConstants.DE_TIME;
default:
return 0;
}
}
public static Integer transFieldTypeSize(String field) {
switch (field) {
case "null":
return 1;
case "boolean":
return 1;
case "byte":
return 3;
case "short":
return 5;
case "integer":
return 10;
case "long":
return 19;
case "double":
case "float":
case "half_float":
case "scaled_float":
return 15;
case "keyword":
case "text":
case "binary":
case "object":
case "nested":
return 65532;
case "datetime":
return 50;
case "ip":
return 50;
case "unsupported":
return 50;
default:
return 0;
}
}
@Override
public String createSQLPreview(String sql, String orderBy) {
return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp ORDER BY null " + " LIMIT 1000";
}
@Override
public String createQuerySQL(String table, List<DatasetTableField> fields, boolean isGroup, Datasource ds) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
if (CollectionUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
DatasetTableField f = fields.get(i);
String originField;
if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == DeTypeConstants.DE_INT) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(f.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == DeTypeConstants.DE_TIME) {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
} else {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
String fieldName = "";
// 处理横轴字段
if (f.getDeExtractType() == DeTypeConstants.DE_TIME) {
if (f.getDeType() == DeTypeConstants.DE_INT || f.getDeType() == DeTypeConstants.DE_FLOAT) {
fieldName = String.format(EsSqlLConstants.CAST, originField, "bigint");
} else {
fieldName = originField;
}
} else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) {
if (f.getDeType() == DeTypeConstants.DE_INT) {
fieldName = String.format(EsSqlLConstants.CAST, originField, "bigint");
} else if (f.getDeType() == DeTypeConstants.DE_FLOAT) {
fieldName = String.format(EsSqlLConstants.CAST, originField, "double");
} else if (f.getDeType() == DeTypeConstants.DE_TIME) {
fieldName = String.format(EsSqlLConstants.CAST, originField, "timestamp");
} else {
fieldName = originField;
}
} else {
if (f.getDeType() == DeTypeConstants.DE_TIME) {
String cast = String.format(EsSqlLConstants.CAST, originField, "timestamp");
fieldName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, EsSqlLConstants.DEFAULT_DATE_FORMAT);
} else if (f.getDeType() == DeTypeConstants.DE_INT) {
fieldName = String.format(EsSqlLConstants.CAST, originField, "timestamp");
} else {
fieldName = originField;
}
}
xFields.add(SQLObj.builder()
.fieldName(fieldName)
.fieldAlias(fieldAlias)
.build());
}
}
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("previewSql");
st_sql.add("isGroup", isGroup);
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
return st_sql.render();
}
@Override
public String createQuerySQLAsTmp(String sql, List<DatasetTableField> fields, boolean isGroup) {
return createQuerySQL("(" + sqlFix(sql) + ")", fields, isGroup, null);
}
@Override
public String createQuerySQLWithPage(String table, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds) {
return createQuerySQL(table, fields, isGroup, null);
}
@Override
public String createQueryTableWithLimit(String table, List<DatasetTableField> fields, Integer limit, boolean isGroup, Datasource ds) {
return createQuerySQL(table, fields, isGroup, null);
}
@Override
public String createQuerySqlWithLimit(String sql, List<DatasetTableField> fields, Integer limit, boolean isGroup) {
return createQuerySQLAsTmp(sql, fields, isGroup);
}
@Override
public String createQuerySQLAsTmpWithPage(String sql, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup) {
return createQuerySQLAsTmp(sql, fields, isGroup);
}
@Override
public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xWheres = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(xAxis)) {
for (int i = 0; i < xAxis.size(); i++) {
ChartViewFieldDTO x = xAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_INT) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_TIME) {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴过滤
// xWheres.addAll(getXWheres(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
List<SQLObj> yFields = new ArrayList<>();
List<SQLObj> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(yAxis)) {
for (int i = 0; i < yAxis.size(); i++) {
ChartViewFieldDTO y = yAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_INT) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_TIME) {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.addAll(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
List<SQLObj> customWheres = transCustomFilterList(tableObj, customFilter);
// 处理仪表板字段过滤
List<SQLObj> extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
fields.addAll(yFields);
List<SQLObj> wheres = new ArrayList<>();
wheres.addAll(xWheres);
if (customWheres != null) wheres.addAll(customWheres);
if (extWheres != null) wheres.addAll(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
orders.addAll(yOrders);
List<SQLObj> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres);
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(EsSqlLConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return st.render();
}
@Override
public String getSQLAsTmp(String sql, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) {
return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList, null);
}
@Override
public String getSQLStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, Datasource ds) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xWheres = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
List<ChartViewFieldDTO> xList = new ArrayList<>();
xList.addAll(xAxis);
xList.addAll(extStack);
if (CollectionUtils.isNotEmpty(xList)) {
for (int i = 0; i < xList.size(); i++) {
ChartViewFieldDTO x = xList.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_INT) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_TIME) {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴过滤
// xWheres.addAll(getXWheres(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
List<SQLObj> yFields = new ArrayList<>();
List<SQLObj> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(yAxis)) {
for (int i = 0; i < yAxis.size(); i++) {
ChartViewFieldDTO y = yAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_INT) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_TIME) {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.addAll(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
List<SQLObj> customWheres = transCustomFilterList(tableObj, customFilter);
// 处理仪表板字段过滤
List<SQLObj> extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
fields.addAll(yFields);
List<SQLObj> wheres = new ArrayList<>();
wheres.addAll(xWheres);
if (customWheres != null) wheres.addAll(customWheres);
if (extWheres != null) wheres.addAll(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
orders.addAll(yOrders);
List<SQLObj> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres);
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(EsSqlLConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return st.render();
}
@Override
public String getSQLAsTmpStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack) {
return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack, null);
}
@Override
public String getSQLScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, Datasource ds) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xWheres = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(xAxis)) {
for (int i = 0; i < xAxis.size(); i++) {
ChartViewFieldDTO x = xAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴过滤
// xWheres.addAll(getXWheres(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
List<SQLObj> yFields = new ArrayList<>();
List<SQLObj> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
List<ChartViewFieldDTO> yList = new ArrayList<>();
yList.addAll(yAxis);
yList.addAll(extBubble);
if (CollectionUtils.isNotEmpty(yList)) {
for (int i = 0; i < yList.size(); i++) {
ChartViewFieldDTO y = yList.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.addAll(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
List<SQLObj> customWheres = transCustomFilterList(tableObj, customFilter);
// 处理仪表板字段过滤
List<SQLObj> extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
fields.addAll(yFields);
List<SQLObj> wheres = new ArrayList<>();
wheres.addAll(xWheres);
if (customWheres != null) wheres.addAll(customWheres);
if (extWheres != null) wheres.addAll(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
orders.addAll(yOrders);
List<SQLObj> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres);
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(EsSqlLConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return st.render();
}
@Override
public String getSQLAsTmpScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble) {
return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble, null);
}
@Override
public String searchTable(String table) {
return "";
}
@Override
public String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) {
// 字段汇总 排序等
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> yFields = new ArrayList<>();
List<SQLObj> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(yAxis)) {
for (int i = 0; i < yAxis.size(); i++) {
ChartViewFieldDTO y = yAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.addAll(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
List<SQLObj> customWheres = transCustomFilterList(tableObj, customFilter);
// 处理仪表板字段过滤
List<SQLObj> extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(yFields);
List<SQLObj> wheres = new ArrayList<>();
if (customWheres != null) wheres.addAll(customWheres);
if (extWheres != null) wheres.addAll(extWheres);
List<SQLObj> groups = new ArrayList<>();
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(yOrders);
List<SQLObj> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres);
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(EsSqlLConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return st.render();
}
@Override
public String getSQLSummaryAsTmp(String sql, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) {
return getSQLSummary("(" + sqlFix(sql) + ")", yAxis, customFilter, extFilterRequestList);
}
@Override
public String wrapSql(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0";
return tmpSql;
}
@Override
public String createRawQuerySQL(String table, List<DatasetTableField> fields, Datasource ds) {
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("\"").append(f.getOriginName()).append("\" AS ").append(f.getDataeaseName());
return stringBuilder.toString();
}).toArray(String[]::new);
return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
}
@Override
public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null);
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":
return " = ";
case "not_eq":
return " <> ";
case "lt":
return " < ";
case "le":
return " <= ";
case "gt":
return " > ";
case "ge":
return " >= ";
case "in":
return " IN ";
case "not in":
return " NOT IN ";
case "like":
return " LIKE ";
case "not like":
return " NOT LIKE ";
case "null":
return " IN ";
case "not_null":
return " IS NOT NULL AND %s <> ''";
case "between":
return " BETWEEN ";
default:
return "";
}
}
public List<SQLObj> transCustomFilterList(SQLObj tableObj, List<ChartCustomFilterDTO> requestList) {
if (CollectionUtils.isEmpty(requestList)) {
return null;
}
List<SQLObj> list = new ArrayList<>();
for (ChartCustomFilterDTO request : requestList) {
DatasetTableField field = request.getField();
if (ObjectUtils.isEmpty(field)) {
continue;
}
String value = request.getValue();
String whereName = "";
String whereTerm = transMysqlFilterTerm(request.getTerm());
String whereValue = "";
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originName = calcFieldRegex(field.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
}
if (field.getDeType() == DeTypeConstants.DE_TIME) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
whereName = String.format(EsSqlLConstants.CAST, originName, "timestamp");
}
if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == DeTypeConstants.DE_BOOL) {
String cast = String.format(EsSqlLConstants.CAST, originName, "timestamp");
whereName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, EsSqlLConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == 1) {
whereName = originName;
}
} else {
whereName = originName;
}
if (StringUtils.equalsIgnoreCase(request.getTerm(), "null")) {
whereValue = EsSqlLConstants.WHERE_VALUE_NULL;
} else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_null")) {
whereTerm = String.format(whereTerm, originName);
} else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {
whereValue = String.format(EsSqlLConstants.WHERE_VALUE_VALUE, value);
}
list.add(SQLObj.builder()
.whereField(whereName)
.whereTermAndValue(whereTerm + whereValue)
.build());
}
return list;
}
public List<SQLObj> transExtFilterList(SQLObj tableObj, List<ChartExtFilterRequest> requestList) {
if (CollectionUtils.isEmpty(requestList)) {
return null;
}
List<SQLObj> list = new ArrayList<>();
for (ChartExtFilterRequest request : requestList) {
List<String> value = request.getValue();
DatasetTableField field = request.getDatasetTableField();
if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) {
continue;
}
String whereName = "";
String whereTerm = transMysqlFilterTerm(request.getOperator());
String whereValue = "";
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originName = calcFieldRegex(field.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
}
if (field.getDeType() == 1) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
whereName = String.format(EsSqlLConstants.CAST, originName, "timestamp");
}
if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
String cast = String.format(EsSqlLConstants.CAST, originName, "timestamp");
whereName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, EsSqlLConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == 1) {
whereName = originName;
}
} else {
whereName = originName;
}
if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) {
whereValue = "'%" + value.get(0) + "%'";
} else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) {
if (request.getDatasetTableField().getDeType() == 1) {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0))));
String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1))));
whereValue = String.format(EsSqlLConstants.WHERE_BETWEEN, startTime, endTime);
} else {
whereValue = String.format(EsSqlLConstants.WHERE_BETWEEN, value.get(0), value.get(1));
}
} else {
whereValue = String.format(EsSqlLConstants.WHERE_VALUE_VALUE, value.get(0));
}
list.add(SQLObj.builder()
.whereField(whereName)
.whereTermAndValue(whereTerm + whereValue)
.build());
}
return list;
}
private String sqlFix(String sql) {
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
return sql;
}
private String transDateFormat(String dateStyle, String datePattern) {
String split = "-";
if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) {
split = "-";
} else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) {
split = "/";
}
switch (dateStyle) {
case "y":
return "YYYY";
case "y_M":
return "YYYY" + split + "MM";
case "y_M_d":
return "YYYY" + split + "MM" + split + "dd";
case "H_m_s":
return "HH:mm:ss";
case "y_M_d_H_m":
return "YYYY" + split + "MM" + split + "dd" + " HH:mm";
case "y_M_d_H_m_s":
return "YYYY" + split + "MM" + split + "dd" + " HH:mm:ss";
default:
return "YYYY-MM-dd HH:mm:ss";
}
}
private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) {
String fieldName = "";
if (x.getDeExtractType() == DeTypeConstants.DE_TIME) {
if (x.getDeType() == DeTypeConstants.DE_INT || x.getDeType() == DeTypeConstants.DE_FLOAT) {
fieldName = String.format(EsSqlLConstants.CAST, originField, "bigint");
} else if (x.getDeType() == DeTypeConstants.DE_TIME) {
String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
fieldName = String.format(EsSqlLConstants.DATETIME_FORMAT, originField, format);
} else {
fieldName = originField;
}
} else {
if (x.getDeType() == DeTypeConstants.DE_TIME) {
String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
if (x.getDeExtractType() == DeTypeConstants.DE_STRING) {
String cast = String.format(EsSqlLConstants.CAST, originField, "timestamp");
fieldName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, format);
} else {
String cast = String.format(EsSqlLConstants.CAST, originField, "timestamp");
fieldName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, format);
}
} else {
fieldName = originField;
}
}
return SQLObj.builder()
.fieldName(fieldName)
.fieldAlias(fieldAlias)
.build();
}
private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) {
String fieldName = "";
if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) {
fieldName = EsSqlLConstants.AGG_COUNT;
} else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) {
fieldName = String.format(EsSqlLConstants.AGG_FIELD, y.getSummary(), originField);
} else {
if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) {
String cast = String.format(EsSqlLConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? "bigint" : "double");
String agg = String.format(EsSqlLConstants.AGG_FIELD, y.getSummary(), cast);
fieldName = String.format(EsSqlLConstants.CAST, agg, EsSqlLConstants.DEFAULT_FLOAT_FORMAT);
} else {
String cast = String.format(EsSqlLConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? "bigint" : "double");
fieldName = String.format(EsSqlLConstants.AGG_FIELD, y.getSummary(), cast);
}
}
return SQLObj.builder()
.fieldName(fieldName)
.fieldAlias(fieldAlias)
.build();
}
private List<SQLObj> getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) {
List<SQLObj> list = new ArrayList<>();
if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) {
y.getFilter().forEach(f -> {
String whereTerm = transMysqlFilterTerm(f.getTerm());
String whereValue = "";
// 原始类型不是时间，在de中被转成时间的字段做处理
if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) {
whereValue = EsSqlLConstants.WHERE_VALUE_NULL;
} else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) {
whereTerm = String.format(whereTerm, originField);
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')";
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
whereValue = "'%" + f.getValue() + "%'";
} else {
whereValue = String.format(EsSqlLConstants.WHERE_VALUE_VALUE, f.getValue());
}
list.add(SQLObj.builder()
.whereField(fieldAlias)
.whereAlias(fieldAlias)
.whereTermAndValue(whereTerm + whereValue)
.build());
});
}
return list;
}
private String calcFieldRegex(String originField, SQLObj tableObj) {
originField = originField.replaceAll("[\\t\\n\\r]]", "");
// 正则提取[xxx]
String regex = "\\[(.*?)]";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(originField);
Set<String> ids = new HashSet<>();
while (matcher.find()) {
String id = matcher.group(1);
ids.add(id);
}
if (CollectionUtils.isEmpty(ids)) {
return originField;
}
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids));
List<DatasetTableField> calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample);
for (DatasetTableField ele : calcFields) {
originField = originField.replaceAll("\\[" + ele.getId() + "]",
String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName()));
}
return originField;
}
}

39
backend/src/main/java/io/dataease/provider/es/EsSqlLConstants.java Normal file
View File

@ -0,0 +1,39 @@
package io.dataease.provider.es;
import io.dataease.provider.SQLConstants;
import static io.dataease.datasource.constants.DatasourceTypes.es;
/**
* @Author gin
* @Date 2021/7/8 7:22 下午
*/
public class EsSqlLConstants extends SQLConstants {
public static final String KEYWORD_TABLE = es.getKeywordPrefix() + "%s" + es.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + es.getKeywordPrefix() + "%s" + es.getKeywordSuffix();
public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)";
public static final String DATETIME_FORMAT = "DATETIME_FORMAT(%s,'%s')";
public static final String CAST = "CAST(%s AS %s)";
public static final String DEFAULT_DATE_FORMAT = "YYYY-MM-dd HH:mm:ss";
public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)";
public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)";
public static final String WHERE_VALUE_NULL = "(NULL,'')";
public static final String WHERE_VALUE_VALUE = "'%s'";
public static final String AGG_COUNT = "COUNT(*)";
public static final String AGG_FIELD = "%s(%s)";
public static final String WHERE_BETWEEN = "'%s' AND '%s'";
public static final String BRACKETS = "(%s)";
}

8
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -469,14 +469,22 @@ public class DataSetTableService {
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
map.put("sql", datasourceRequest.getQuery());
datasourceRequest.setPage(page);
datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow()));
datasourceRequest.setPageSize(pageSize);
datasourceRequest.setRealSize(realSize);
datasourceRequest.setPreviewData(true);
try {
datasourceRequest.setPageable(true);
data.addAll(datasourceProvider.getData(datasourceRequest));
} catch (Exception e) {
e.printStackTrace();
DEException.throwException(e.getMessage());
}
try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds));
datasourceRequest.setPageable(false);
dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size());
} catch (Exception e) {
e.printStackTrace();

1
frontend/src/lang/en.js
View File

@ -1052,6 +1052,7 @@ export default {
please_input_user_name: 'Please enter user name',
please_input_password: 'Please enter Password',
please_input_host: 'Please enter host',
please_input_url: 'Please enter url adress',
please_input_port: 'Please enter port',
modify: 'Edit data Source',
validate_success: 'Verification successful',

1
frontend/src/lang/tw.js
View File

@ -1051,6 +1051,7 @@ export default {
please_input_user_name: '請輸入用戶名',
please_input_password: '請輸入密碼',
please_input_host: '請輸入主機',
please_input_url: '請輸入URL地址',
please_input_port: '請輸入',
modify: '編輯數據源',
validate_success: '校驗成功',

1
frontend/src/lang/zh.js
View File

@ -1084,6 +1084,7 @@ export default {
please_input_user_name: '请输入用户名',
please_input_password: '请输入密码',
please_input_host: '请输入主机',
please_input_url: '请输入URL地址',
please_input_port: '请输入端口',
modify: '编辑数据源',
validate_success: '校验成功',

25
frontend/src/views/system/datasource/form.vue
View File

@ -27,6 +27,9 @@
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.host')" prop="configuration.host">
<el-input v-model="form.configuration.host" autocomplete="off" />
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='es'" :label="$t('datasource.url')" prop="configuration.url">
<el-input v-model="form.configuration.url" placeholder="请输入 Elasticsearch 地址，如: http://es_host:es_port" autocomplete="off" />
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.data_base')" prop="configuration.dataBase">
<el-input v-model="form.configuration.dataBase" autocomplete="off" />
</el-form-item>
@ -36,10 +39,10 @@
<el-radio v-model="form.configuration.connectionType" label="serviceName">{{ $t('datasource.oracle_service_name') }}</el-radio>
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.user_name')" prop="configuration.username">
<el-form-item :label="$t('datasource.user_name')" prop="configuration.username">
<el-input v-model="form.configuration.username" autocomplete="off" />
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.password')" prop="configuration.password">
<el-form-item :label="$t('datasource.password')" prop="configuration.password">
<el-input v-model="form.configuration.password" autocomplete="off" show-password />
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.port')" prop="configuration.port">
@ -61,25 +64,25 @@
/>
</el-select>
</el-form-item>
<el-collapse>
<el-collapse v-if="form.configuration.dataSourceType=='jdbc'">
<el-collapse-item :title="$t('datasource.priority')" name="1">
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.initial_pool_size')" prop="configuration.initialPoolSize">
<el-form-item :label="$t('datasource.initial_pool_size')" prop="configuration.initialPoolSize">
<el-input v-model="form.configuration.initialPoolSize" autocomplete="off" type="number" min="0" size="small" />
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.min_pool_size')" prop="configuration.minPoolSize">
<el-form-item :label="$t('datasource.min_pool_size')" prop="configuration.minPoolSize">
<el-input v-model="form.configuration.minPoolSize" autocomplete="off" type="number" min="0" />
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.max_pool_size')" prop="configuration.maxPoolSize">
<el-form-item :label="$t('datasource.max_pool_size')" prop="configuration.maxPoolSize">
<el-input v-model="form.configuration.maxPoolSize" autocomplete="off" type="number" min="0" />
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.max_idle_time')" prop="configuration.maxIdleTime">
<el-form-item :label="$t('datasource.max_idle_time')" prop="configuration.maxIdleTime">
<el-input v-model="form.configuration.maxIdleTime" autocomplete="off" type="number" min="0" />
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.acquire_increment')" prop="configuration.acquireIncrement">
<el-form-item :label="$t('datasource.acquire_increment')" prop="configuration.acquireIncrement">
<el-input v-model="form.configuration.acquireIncrement" autocomplete="off" type="number" min="0" />
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.connect_timeout')" prop="configuration.connectTimeout">
<el-form-item :label="$t('datasource.connect_timeout')" prop="configuration.connectTimeout">
<el-input v-model="form.configuration.connectTimeout" autocomplete="off" type="number" min="0" />
</el-form-item>
@ -135,6 +138,7 @@ export default {
'configuration.username': [{ required: true, message: this.$t('datasource.please_input_user_name'), trigger: 'blur' }],
'configuration.password': [{ required: true, message: this.$t('datasource.please_input_password'), trigger: 'change' }],
'configuration.host': [{ required: true, message: this.$t('datasource.please_input_host'), trigger: 'change' }],
'configuration.url': [{ required: true, message: this.$t('datasource.please_input_url'), trigger: 'change' }],
'configuration.port': [{ required: true, message: this.$t('datasource.please_input_port'), trigger: 'change' }],
'configuration.initialPoolSize': [{ required: true, message: this.$t('datasource.please_input_initial_pool_size'), trigger: 'change' }],
'configuration.minPoolSize': [{ required: true, message: this.$t('datasource.please_input_min_pool_size'), trigger: 'change' }],
@ -146,7 +150,8 @@ export default {
allTypes: [{ name: 'mysql', label: 'MySQL', type: 'jdbc' },
{ name: 'oracle', label: 'Oracle', type: 'jdbc' },
{ name: 'sqlServer', label: 'SQL Server', type: 'jdbc' },
{ name: 'pg', label: 'PostgreSQL', type: 'jdbc' }],
{ name: 'pg', label: 'PostgreSQL', type: 'jdbc' },
{ name: 'es', label: 'Elasticsearch', type: 'es' }],
schemas: [],
canEdit: false,
originConfiguration: {}