diff --git a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java index eb849fff12..150aa7dc39 100644 --- a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java @@ -110,7 +110,6 @@ public class ChartDataManage { } var dillAxis = new ArrayList(); - DatasetGroupInfoDTO table = datasetGroupManage.getDatasetGroupInfoDTO(view.getTableId(), null); if (table == null) { DEException.throwException(ResultCode.DATA_IS_WRONG.code(), Translator.get("i18n_no_ds")); diff --git a/core/core-backend/src/main/java/io/dataease/dataset/constant/DatasetTableType.java b/core/core-backend/src/main/java/io/dataease/dataset/constant/DatasetTableType.java index 1975fc8d6c..047df2ccb1 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/constant/DatasetTableType.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/constant/DatasetTableType.java @@ -6,4 +6,5 @@ package io.dataease.dataset.constant; public class DatasetTableType { public static String DB = "db"; public static String SQL = "sql"; + public static String Es = "es"; } diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java index 8c88efdf99..771ea7f91f 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java @@ -129,6 +129,15 @@ public class DatasetDataManage { datasourceRequest.setTable(tableInfoDTO.getTable()); } + tableFields = provider.fetchTableField(datasourceRequest); + } else if (StringUtils.equalsIgnoreCase(type, DatasetTableType.Es)) { + CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasetTableDTO.getDatasourceId()); + Provider provider = ProviderFactory.getProvider(type); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); + BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource); + datasourceRequest.setDatasource(datasourceSchemaDTO); + datasourceRequest.setTable(datasetTableDTO.getTableName()); tableFields = provider.fetchTableField(datasourceRequest); } else { // excel,api @@ -185,9 +194,7 @@ public class DatasetDataManage { DEException.throwException(Translator.get("i18n_no_column_permission")); } } - buildFieldName(sqlMap, fields); - Map dsMap = (Map) sqlMap.get("dsMap"); DatasourceUtils.checkDsStatus(dsMap); List dsList = new ArrayList<>(); @@ -202,13 +209,11 @@ public class DatasetDataManage { } sql = Utils.replaceSchemaAlias(sql, dsMap); } - List rowPermissionsTree = new ArrayList<>(); TokenUserBO user = AuthUtils.getUser(); if (user != null && checkPermission) { rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId()); } - Provider provider; if (crossDs) { provider = ProviderFactory.getDefaultProvider(); @@ -236,7 +241,6 @@ public class DatasetDataManage { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setQuery(querySQL); datasourceRequest.setDsList(dsMap); - Map data = provider.fetchResultField(datasourceRequest); Map map = new LinkedHashMap<>(); diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java index 70549c206e..0de143bce4 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java @@ -494,7 +494,16 @@ public class DatasetSQLManage { datasourceSchemaDTO.setSchemaAlias(schemaAlias); dsMap.put(coreDatasource.getId(), datasourceSchemaDTO); } - } else { + } else if (StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.Es)){ + CoreDatasource coreDatasource = coreDatasourceMapper.selectById(ds.getDatasourceId()); + schemaAlias = String.format(SQLConstants.SCHEMA, coreDatasource.getId()); + if (!dsMap.containsKey(coreDatasource.getId())) { + DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); + BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource); + datasourceSchemaDTO.setSchemaAlias(schemaAlias); + dsMap.put(coreDatasource.getId(), datasourceSchemaDTO); + } + }else { CoreDatasource coreDatasource = engineManage.getDeEngine(); schemaAlias = String.format(SQLConstants.SCHEMA, coreDatasource.getId()); if (!dsMap.containsKey(coreDatasource.getId())) { diff --git a/core/core-backend/src/main/java/io/dataease/datasource/dto/es/EsResponse.java b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/EsResponse.java new file mode 100644 index 0000000000..45e4a42c04 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/EsResponse.java @@ -0,0 +1,29 @@ +package io.dataease.datasource.dto.es; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class EsResponse { + private List columns = new ArrayList<>(); + private List rows = new ArrayList<>(); + private String cursor; + private Integer status; + private Error error; + private String version; + + @Data + public class Error { + private String type; + private String reason; + } + + @Data + public class Column { + private String name; + private String type; + } + +} diff --git a/core/core-backend/src/main/java/io/dataease/datasource/dto/es/Request.java b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/Request.java new file mode 100644 index 0000000000..6a79b1a827 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/Request.java @@ -0,0 +1,10 @@ +package io.dataease.datasource.dto.es; + +import lombok.Data; + +@Data +public class Request { + private String query; + private Integer fetch_size = 10000; + private boolean field_multi_value_leniency = true; +} diff --git a/core/core-backend/src/main/java/io/dataease/datasource/dto/es/RequestWithCursor.java b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/RequestWithCursor.java new file mode 100644 index 0000000000..d67519aa8f --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/RequestWithCursor.java @@ -0,0 +1,8 @@ +package io.dataease.datasource.dto.es; + +import lombok.Data; + +@Data +public class RequestWithCursor extends Request { + private String cursor; +} diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java index 9599cf20e3..ef4d3f9a49 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java @@ -1271,7 +1271,6 @@ public class CalciteProvider extends Provider { try { connection = initConnection(dsMap); } catch (Exception e) { - e.printStackTrace(); } }); diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/EsProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/EsProvider.java new file mode 100644 index 0000000000..dea1744934 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/EsProvider.java @@ -0,0 +1,211 @@ +package io.dataease.datasource.provider; + +import com.google.gson.Gson; +import com.google.gson.JsonParser; + +import io.dataease.dataset.utils.FieldUtils; +import io.dataease.datasource.dto.es.EsResponse; +import io.dataease.datasource.dto.es.Request; +import io.dataease.datasource.type.Es; +import io.dataease.exception.DEException; +import io.dataease.extensions.datasource.dto.*; +import io.dataease.extensions.datasource.provider.Provider; +import io.dataease.i18n.Translator; + +import io.dataease.utils.HttpClientConfig; +import io.dataease.utils.HttpClientUtil; +import io.dataease.utils.JsonUtil; +import org.apache.commons.codec.binary.Base64; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHeaders; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; + +@Service("esProvider") +public class EsProvider extends Provider { + + @Override + public List getSchema(DatasourceRequest datasourceRequest) { + return new ArrayList<>(); + } + + @Override + public List getTables(DatasourceRequest datasourceRequest) { + List tables = new ArrayList<>(); + try { + String response = execQuery(datasourceRequest, "show tables", "?format=json"); + tables = fetchTables(response); + tables = tables.stream().filter(table -> StringUtils.isNotEmpty(table.getTableName()) && !table.getTableName().startsWith(".")).collect(Collectors.toList()); + tables.forEach(table -> { + table.setDatasourceId(datasourceRequest.getDatasource().getId()); + }); + } catch (Exception e) { + e.getMessage(); + DEException.throwException(e); + } + return tables; + } + + @Override + public ConnectionObj getConnection(DatasourceDTO coreDatasource) throws Exception { + return null; + } + + @Override + public String checkStatus(DatasourceRequest datasourceRequest) throws Exception { + Es es = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Es.class); + String response = execGetQuery(datasourceRequest); + if (JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error") != null) { + throw new Exception(JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error").get("reason").getAsString()); + } + String version = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("version").get("number").getAsString(); + String[] versionList = version.split("\\."); + if (Integer.valueOf(versionList[0]) < 7 && Integer.valueOf(versionList[1]) < 3) { + throw new Exception(Translator.get("i18n_es_limit")); + } + if (Integer.valueOf(versionList[0]) == 6) { + es.setUri("_xpack/sql"); + } + if (Integer.valueOf(versionList[0]) > 6) { + es.setUri("_sql"); + } + datasourceRequest.getDatasource().setConfiguration(JsonUtil.toJSONString(es).toString()); + getTables(datasourceRequest); + return "Success"; + } + + @Override + public Map fetchResultField(DatasourceRequest datasourceRequest) { + Map result = new HashMap<>(); + try { + String response = execQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json"); + result.put("dataList", fetchResultData(response)); + result.put("fieldList", fetchResultField4Sql(response)); + } catch (Exception e) { + e.printStackTrace(); + DEException.throwException(e); + } + return result; + } + + @Override + public List fetchTableField(DatasourceRequest datasourceRequest) { + List tableFields = new ArrayList<>(); + try { + String response = execQuery(datasourceRequest, "select * from " + datasourceRequest.getTable() + " limit 0", "?format=json"); + tableFields = fetchResultField4Sql(response); + } catch (Exception e) { + DEException.throwException(e); + } + return tableFields; + } + + + @Override + public void hidePW(DatasourceDTO datasourceDTO) { + } + + + private List fetchResultData(String response) throws Exception { + EsResponse esResponse = new Gson().fromJson(response, EsResponse.class); + return fetchResultData(esResponse); + } + + private List fetchResultData(EsResponse esResponse) throws Exception { + List list = new LinkedList<>(); + if (esResponse.getError() != null) { + throw new Exception(esResponse.getError().getReason()); + } + list.addAll(esResponse.getRows()); + return list; + } + + private List fetchResultField4Sql(String response) throws Exception { + List fieldList = new ArrayList<>(); + EsResponse esResponse = new Gson().fromJson(response, EsResponse.class); + if (esResponse.getError() != null) { + throw new Exception(esResponse.getError().getReason()); + } + + for (EsResponse.Column column : esResponse.getColumns()) { + TableField field = new TableField(); + field.setOriginName(column.getName()); + field.setOriginName(column.getName()); + field.setFieldType(column.getType()); + field.setType(column.getType().toUpperCase()); + field.setFieldType(field.getType()); + int deType = FieldUtils.transType2DeType(field.getType()); + field.setDeExtractType(deType); + field.setDeType(deType); + fieldList.add(field); + } + return fieldList; + } + + private List fetchTables(String response) throws Exception { + List tables = new ArrayList<>(); + EsResponse esResponse = new Gson().fromJson(response, EsResponse.class); + if (esResponse.getError() != null) { + throw new Exception(esResponse.getError().getReason()); + } + + for (String[] row : esResponse.getRows()) { + + DatasetTableDTO tableDesc = new DatasetTableDTO(); + if (row.length == 3 && row[1].contains("TABLE") && row[2].equalsIgnoreCase("INDEX")) { + tableDesc.setTableName(row[0]); + } + if (row.length == 2 && row[1].contains("TABLE")) { + tableDesc.setTableName(row[0]); + } + if (row.length == 4 && row[2].contains("TABLE") && row[3].equalsIgnoreCase("INDEX")) { + tableDesc.setTableName(row[1]); + } + tableDesc.setType("es"); + tables.add(tableDesc); + } + return tables; + } + + + private String execQuery(DatasourceRequest datasourceRequest, String sql, String uri) { + Es es = null; + if (datasourceRequest.getDatasource() == null) { + Collection datasourceSchemaDTOS = datasourceRequest.getDsList().values(); + es = JsonUtil.parseObject(datasourceSchemaDTOS.stream().findFirst().get().getConfiguration(), Es.class); + } else { + es = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Es.class); + } + + uri = es.getUri() + uri; + HttpClientConfig httpClientConfig = new HttpClientConfig(); + if (StringUtils.isNotEmpty(es.getUsername()) && StringUtils.isNotEmpty(es.getPassword())) { + String auth = es.getUsername() + ":" + es.getPassword(); + byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); + httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); + } + Request request = new Request(); + request.setQuery(sql); + request.setFetch_size(datasourceRequest.getFetchSize()); + String url = es.getUrl().endsWith("/") ? es.getUrl() + uri : es.getUrl() + "/" + uri; + return HttpClientUtil.post(url, new Gson().toJson(request), httpClientConfig); + + } + + private String execGetQuery(DatasourceRequest datasourceRequest) { + Es es = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Es.class); + HttpClientConfig httpClientConfig = new HttpClientConfig(); + if (StringUtils.isNotEmpty(es.getUsername()) && StringUtils.isNotEmpty(es.getPassword())) { + String auth = es.getUsername() + ":" + es.getPassword(); + byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); + httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); + } + return HttpClientUtil.get(es.getUrl(), httpClientConfig); + } + + +} diff --git a/core/core-backend/src/main/java/io/dataease/datasource/type/Es.java b/core/core-backend/src/main/java/io/dataease/datasource/type/Es.java new file mode 100644 index 0000000000..fdc1bc2990 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/type/Es.java @@ -0,0 +1,15 @@ +package io.dataease.datasource.type; + + +import lombok.Data; +import org.springframework.stereotype.Component; + +@Data +public class Es { + private String url; + private String username; + private String password; + private String version; + private String uri; + +} diff --git a/core/core-frontend/src/assets/svg/es-ds.svg b/core/core-frontend/src/assets/svg/es-ds.svg new file mode 100644 index 0000000000..60e557758f --- /dev/null +++ b/core/core-frontend/src/assets/svg/es-ds.svg @@ -0,0 +1 @@ + diff --git a/core/core-frontend/src/components/icon-group/datasource-list.ts b/core/core-frontend/src/components/icon-group/datasource-list.ts index 44167b92e7..2c86934665 100644 --- a/core/core-frontend/src/components/icon-group/datasource-list.ts +++ b/core/core-frontend/src/components/icon-group/datasource-list.ts @@ -13,6 +13,7 @@ import redshiftDs from '@/assets/svg/redshift-ds.svg' import APIDs from '@/assets/svg/API-ds.svg' import ExcelDs from '@/assets/svg/Excel-ds.svg' import dorisDs from '@/assets/svg/doris-ds.svg' +import esDs from '@/assets/svg/es-ds.svg' const iconDatasourceMap = { mysql: mysqlDs, oracle: oracleDs, @@ -28,7 +29,8 @@ const iconDatasourceMap = { redshift: redshiftDs, API: APIDs, Excel: ExcelDs, - doris: dorisDs + doris: dorisDs, + es: esDs } export { iconDatasourceMap } diff --git a/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue b/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue index d758aef0a0..fc6f8b1566 100644 --- a/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue +++ b/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue @@ -928,7 +928,7 @@ defineExpose({ list = Arrays.stream(DatasourceConfiguration.DatasourceType.values()).map(DatasourceConfiguration.DatasourceType::getType).toList(); if (list.contains(type)) { return SpringContextUtil.getApplicationContext().getBean("calciteProvider", Provider.class); diff --git a/sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/vo/DatasourceConfiguration.java b/sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/vo/DatasourceConfiguration.java index 20448b46c0..a57b335e82 100644 --- a/sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/vo/DatasourceConfiguration.java +++ b/sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/vo/DatasourceConfiguration.java @@ -18,6 +18,7 @@ public class DatasourceConfiguration extends Configuration { impala("impala", "Apache Impala", "OLAP", "`", "`"), mariadb("mariadb", "Mariadb", "OLTP", "`", "`"), StarRocks("StarRocks", "StarRocks", "OLAP", "`", "`"), + es("es", "Elasticsearch", "OLAP", "\"", "\""), doris("doris", "Apache Doris", "OLAP", "`", "`"), TiDB("TiDB", "TiDB", "OLTP", "`", "`"), oracle("oracle", "ORACLE", "OLTP", "\"", "\""),