From 93a94cccfe376f2d0e4542a51626579c8aa4f7a3 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Fri, 20 Sep 2024 15:36:56 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81ES?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chart/manage/ChartDataManage.java | 1 - .../dataset/constant/DatasetTableType.java | 1 + .../dataset/manage/DatasetDataManage.java | 14 ++++++++----- .../dataset/manage/DatasetSQLManage.java | 11 +++++++++- .../datasource/provider/CalciteProvider.java | 1 - .../data/datasource/form/EditorDetail.vue | 21 ++++++++++++++----- .../visualized/data/datasource/form/option.ts | 6 ++++++ .../visualized/data/datasource/index.vue | 2 ++ .../datasource/factory/ProviderFactory.java | 3 +++ .../vo/DatasourceConfiguration.java | 1 + 10 files changed, 48 insertions(+), 13 deletions(-) diff --git a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java index 863f3061db..6257e24d31 100644 --- a/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java @@ -110,7 +110,6 @@ public class ChartDataManage { } var dillAxis = new ArrayList(); - DatasetGroupInfoDTO table = datasetGroupManage.getDatasetGroupInfoDTO(view.getTableId(), null); if (table == null) { DEException.throwException(ResultCode.DATA_IS_WRONG.code(), Translator.get("i18n_no_ds")); diff --git a/core/core-backend/src/main/java/io/dataease/dataset/constant/DatasetTableType.java b/core/core-backend/src/main/java/io/dataease/dataset/constant/DatasetTableType.java index 1975fc8d6c..047df2ccb1 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/constant/DatasetTableType.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/constant/DatasetTableType.java @@ -6,4 +6,5 @@ package io.dataease.dataset.constant; public class DatasetTableType { public static String DB = "db"; public static String SQL = "sql"; + public static String Es = "es"; } diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java index 3715efd527..5a8cad814e 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java @@ -129,6 +129,15 @@ public class DatasetDataManage { datasourceRequest.setTable(tableInfoDTO.getTable()); } + tableFields = provider.fetchTableField(datasourceRequest); + } else if (StringUtils.equalsIgnoreCase(type, DatasetTableType.Es)) { + CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasetTableDTO.getDatasourceId()); + Provider provider = ProviderFactory.getProvider(type); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); + BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource); + datasourceRequest.setDatasource(datasourceSchemaDTO); + datasourceRequest.setTable(datasetTableDTO.getTableName()); tableFields = provider.fetchTableField(datasourceRequest); } else { // excel,api @@ -185,9 +194,7 @@ public class DatasetDataManage { DEException.throwException(Translator.get("i18n_no_column_permission")); } } - buildFieldName(sqlMap, fields); - Map dsMap = (Map) sqlMap.get("dsMap"); DatasourceUtils.checkDsStatus(dsMap); List dsList = new ArrayList<>(); @@ -202,13 +209,11 @@ public class DatasetDataManage { } sql = Utils.replaceSchemaAlias(sql, dsMap); } - List rowPermissionsTree = new ArrayList<>(); TokenUserBO user = AuthUtils.getUser(); if (user != null && checkPermission) { rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId()); } - Provider provider; if (crossDs) { provider = ProviderFactory.getDefaultProvider(); @@ -236,7 +241,6 @@ public class DatasetDataManage { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setQuery(querySQL); datasourceRequest.setDsList(dsMap); - Map data = provider.fetchResultField(datasourceRequest); Map map = new LinkedHashMap<>(); diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java index 70549c206e..0de143bce4 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java @@ -494,7 +494,16 @@ public class DatasetSQLManage { datasourceSchemaDTO.setSchemaAlias(schemaAlias); dsMap.put(coreDatasource.getId(), datasourceSchemaDTO); } - } else { + } else if (StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.Es)){ + CoreDatasource coreDatasource = coreDatasourceMapper.selectById(ds.getDatasourceId()); + schemaAlias = String.format(SQLConstants.SCHEMA, coreDatasource.getId()); + if (!dsMap.containsKey(coreDatasource.getId())) { + DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); + BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource); + datasourceSchemaDTO.setSchemaAlias(schemaAlias); + dsMap.put(coreDatasource.getId(), datasourceSchemaDTO); + } + }else { CoreDatasource coreDatasource = engineManage.getDeEngine(); schemaAlias = String.format(SQLConstants.SCHEMA, coreDatasource.getId()); if (!dsMap.containsKey(coreDatasource.getId())) { diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java index 7718872e2f..e5c6067937 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java @@ -1274,7 +1274,6 @@ public class CalciteProvider extends Provider { try { connection = initConnection(dsMap); } catch (Exception e) { - e.printStackTrace(); } }); diff --git a/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue b/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue index 26338d78d8..e85e2098ad 100644 --- a/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue +++ b/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue @@ -911,7 +911,7 @@ defineExpose({ Date: Wed, 25 Sep 2024 16:23:39 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=20=E6=94=AF=E6=8C=81es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasource/dto/es/EsResponse.java | 29 +++ .../dataease/datasource/dto/es/Request.java | 10 + .../datasource/dto/es/RequestWithCursor.java | 8 + .../datasource/provider/EsProvider.java | 211 ++++++++++++++++++ .../java/io/dataease/datasource/type/Es.java | 15 ++ 5 files changed, 273 insertions(+) create mode 100644 core/core-backend/src/main/java/io/dataease/datasource/dto/es/EsResponse.java create mode 100644 core/core-backend/src/main/java/io/dataease/datasource/dto/es/Request.java create mode 100644 core/core-backend/src/main/java/io/dataease/datasource/dto/es/RequestWithCursor.java create mode 100644 core/core-backend/src/main/java/io/dataease/datasource/provider/EsProvider.java create mode 100644 core/core-backend/src/main/java/io/dataease/datasource/type/Es.java diff --git a/core/core-backend/src/main/java/io/dataease/datasource/dto/es/EsResponse.java b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/EsResponse.java new file mode 100644 index 0000000000..45e4a42c04 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/EsResponse.java @@ -0,0 +1,29 @@ +package io.dataease.datasource.dto.es; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class EsResponse { + private List columns = new ArrayList<>(); + private List rows = new ArrayList<>(); + private String cursor; + private Integer status; + private Error error; + private String version; + + @Data + public class Error { + private String type; + private String reason; + } + + @Data + public class Column { + private String name; + private String type; + } + +} diff --git a/core/core-backend/src/main/java/io/dataease/datasource/dto/es/Request.java b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/Request.java new file mode 100644 index 0000000000..6a79b1a827 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/Request.java @@ -0,0 +1,10 @@ +package io.dataease.datasource.dto.es; + +import lombok.Data; + +@Data +public class Request { + private String query; + private Integer fetch_size = 10000; + private boolean field_multi_value_leniency = true; +} diff --git a/core/core-backend/src/main/java/io/dataease/datasource/dto/es/RequestWithCursor.java b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/RequestWithCursor.java new file mode 100644 index 0000000000..d67519aa8f --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/dto/es/RequestWithCursor.java @@ -0,0 +1,8 @@ +package io.dataease.datasource.dto.es; + +import lombok.Data; + +@Data +public class RequestWithCursor extends Request { + private String cursor; +} diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/EsProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/EsProvider.java new file mode 100644 index 0000000000..dea1744934 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/EsProvider.java @@ -0,0 +1,211 @@ +package io.dataease.datasource.provider; + +import com.google.gson.Gson; +import com.google.gson.JsonParser; + +import io.dataease.dataset.utils.FieldUtils; +import io.dataease.datasource.dto.es.EsResponse; +import io.dataease.datasource.dto.es.Request; +import io.dataease.datasource.type.Es; +import io.dataease.exception.DEException; +import io.dataease.extensions.datasource.dto.*; +import io.dataease.extensions.datasource.provider.Provider; +import io.dataease.i18n.Translator; + +import io.dataease.utils.HttpClientConfig; +import io.dataease.utils.HttpClientUtil; +import io.dataease.utils.JsonUtil; +import org.apache.commons.codec.binary.Base64; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHeaders; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; + +@Service("esProvider") +public class EsProvider extends Provider { + + @Override + public List getSchema(DatasourceRequest datasourceRequest) { + return new ArrayList<>(); + } + + @Override + public List getTables(DatasourceRequest datasourceRequest) { + List tables = new ArrayList<>(); + try { + String response = execQuery(datasourceRequest, "show tables", "?format=json"); + tables = fetchTables(response); + tables = tables.stream().filter(table -> StringUtils.isNotEmpty(table.getTableName()) && !table.getTableName().startsWith(".")).collect(Collectors.toList()); + tables.forEach(table -> { + table.setDatasourceId(datasourceRequest.getDatasource().getId()); + }); + } catch (Exception e) { + e.getMessage(); + DEException.throwException(e); + } + return tables; + } + + @Override + public ConnectionObj getConnection(DatasourceDTO coreDatasource) throws Exception { + return null; + } + + @Override + public String checkStatus(DatasourceRequest datasourceRequest) throws Exception { + Es es = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Es.class); + String response = execGetQuery(datasourceRequest); + if (JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error") != null) { + throw new Exception(JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error").get("reason").getAsString()); + } + String version = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("version").get("number").getAsString(); + String[] versionList = version.split("\\."); + if (Integer.valueOf(versionList[0]) < 7 && Integer.valueOf(versionList[1]) < 3) { + throw new Exception(Translator.get("i18n_es_limit")); + } + if (Integer.valueOf(versionList[0]) == 6) { + es.setUri("_xpack/sql"); + } + if (Integer.valueOf(versionList[0]) > 6) { + es.setUri("_sql"); + } + datasourceRequest.getDatasource().setConfiguration(JsonUtil.toJSONString(es).toString()); + getTables(datasourceRequest); + return "Success"; + } + + @Override + public Map fetchResultField(DatasourceRequest datasourceRequest) { + Map result = new HashMap<>(); + try { + String response = execQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json"); + result.put("dataList", fetchResultData(response)); + result.put("fieldList", fetchResultField4Sql(response)); + } catch (Exception e) { + e.printStackTrace(); + DEException.throwException(e); + } + return result; + } + + @Override + public List fetchTableField(DatasourceRequest datasourceRequest) { + List tableFields = new ArrayList<>(); + try { + String response = execQuery(datasourceRequest, "select * from " + datasourceRequest.getTable() + " limit 0", "?format=json"); + tableFields = fetchResultField4Sql(response); + } catch (Exception e) { + DEException.throwException(e); + } + return tableFields; + } + + + @Override + public void hidePW(DatasourceDTO datasourceDTO) { + } + + + private List fetchResultData(String response) throws Exception { + EsResponse esResponse = new Gson().fromJson(response, EsResponse.class); + return fetchResultData(esResponse); + } + + private List fetchResultData(EsResponse esResponse) throws Exception { + List list = new LinkedList<>(); + if (esResponse.getError() != null) { + throw new Exception(esResponse.getError().getReason()); + } + list.addAll(esResponse.getRows()); + return list; + } + + private List fetchResultField4Sql(String response) throws Exception { + List fieldList = new ArrayList<>(); + EsResponse esResponse = new Gson().fromJson(response, EsResponse.class); + if (esResponse.getError() != null) { + throw new Exception(esResponse.getError().getReason()); + } + + for (EsResponse.Column column : esResponse.getColumns()) { + TableField field = new TableField(); + field.setOriginName(column.getName()); + field.setOriginName(column.getName()); + field.setFieldType(column.getType()); + field.setType(column.getType().toUpperCase()); + field.setFieldType(field.getType()); + int deType = FieldUtils.transType2DeType(field.getType()); + field.setDeExtractType(deType); + field.setDeType(deType); + fieldList.add(field); + } + return fieldList; + } + + private List fetchTables(String response) throws Exception { + List tables = new ArrayList<>(); + EsResponse esResponse = new Gson().fromJson(response, EsResponse.class); + if (esResponse.getError() != null) { + throw new Exception(esResponse.getError().getReason()); + } + + for (String[] row : esResponse.getRows()) { + + DatasetTableDTO tableDesc = new DatasetTableDTO(); + if (row.length == 3 && row[1].contains("TABLE") && row[2].equalsIgnoreCase("INDEX")) { + tableDesc.setTableName(row[0]); + } + if (row.length == 2 && row[1].contains("TABLE")) { + tableDesc.setTableName(row[0]); + } + if (row.length == 4 && row[2].contains("TABLE") && row[3].equalsIgnoreCase("INDEX")) { + tableDesc.setTableName(row[1]); + } + tableDesc.setType("es"); + tables.add(tableDesc); + } + return tables; + } + + + private String execQuery(DatasourceRequest datasourceRequest, String sql, String uri) { + Es es = null; + if (datasourceRequest.getDatasource() == null) { + Collection datasourceSchemaDTOS = datasourceRequest.getDsList().values(); + es = JsonUtil.parseObject(datasourceSchemaDTOS.stream().findFirst().get().getConfiguration(), Es.class); + } else { + es = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Es.class); + } + + uri = es.getUri() + uri; + HttpClientConfig httpClientConfig = new HttpClientConfig(); + if (StringUtils.isNotEmpty(es.getUsername()) && StringUtils.isNotEmpty(es.getPassword())) { + String auth = es.getUsername() + ":" + es.getPassword(); + byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); + httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); + } + Request request = new Request(); + request.setQuery(sql); + request.setFetch_size(datasourceRequest.getFetchSize()); + String url = es.getUrl().endsWith("/") ? es.getUrl() + uri : es.getUrl() + "/" + uri; + return HttpClientUtil.post(url, new Gson().toJson(request), httpClientConfig); + + } + + private String execGetQuery(DatasourceRequest datasourceRequest) { + Es es = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Es.class); + HttpClientConfig httpClientConfig = new HttpClientConfig(); + if (StringUtils.isNotEmpty(es.getUsername()) && StringUtils.isNotEmpty(es.getPassword())) { + String auth = es.getUsername() + ":" + es.getPassword(); + byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); + httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); + } + return HttpClientUtil.get(es.getUrl(), httpClientConfig); + } + + +} diff --git a/core/core-backend/src/main/java/io/dataease/datasource/type/Es.java b/core/core-backend/src/main/java/io/dataease/datasource/type/Es.java new file mode 100644 index 0000000000..fdc1bc2990 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/datasource/type/Es.java @@ -0,0 +1,15 @@ +package io.dataease.datasource.type; + + +import lombok.Data; +import org.springframework.stereotype.Component; + +@Data +public class Es { + private String url; + private String username; + private String password; + private String version; + private String uri; + +}