From 00650ce9f2ad15e758aa8f21aec5c9b92897e912 Mon Sep 17 00:00:00 2001 From: rc452860 <87719349@qq.com> Date: Tue, 11 Jan 2022 11:11:54 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E6=95=B0=E6=8D=AE=E9=9B=86=E6=97=B6=E6=98=BE=E7=A4=BA?= =?UTF-8?q?=E8=A1=A8=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataease/dto/datasource/DBTableDTO.java | 2 + .../io/dataease/dto/datasource/TableDesc.java | 15 +++ .../datasource/DatasourceProvider.java | 3 +- .../provider/datasource/EsProvider.java | 96 ++++++++++--------- .../provider/datasource/JdbcProvider.java | 27 ++++-- .../service/datasource/DatasourceService.java | 9 +- frontend/src/views/dataset/add/AddDB.vue | 9 +- 7 files changed, 103 insertions(+), 58 deletions(-) create mode 100644 backend/src/main/java/io/dataease/dto/datasource/TableDesc.java diff --git a/backend/src/main/java/io/dataease/dto/datasource/DBTableDTO.java b/backend/src/main/java/io/dataease/dto/datasource/DBTableDTO.java index 3c42edf228..efa12809b8 100644 --- a/backend/src/main/java/io/dataease/dto/datasource/DBTableDTO.java +++ b/backend/src/main/java/io/dataease/dto/datasource/DBTableDTO.java @@ -15,6 +15,8 @@ public class DBTableDTO { private String datasourceId; @ApiModelProperty("数据源名称") private String name; + @ApiModelProperty("表注释") + private String remark; @ApiModelProperty("启用检测") private boolean enableCheck; @ApiModelProperty("数据集路径") diff --git a/backend/src/main/java/io/dataease/dto/datasource/TableDesc.java b/backend/src/main/java/io/dataease/dto/datasource/TableDesc.java new file mode 100644 index 0000000000..e53066e9d7 --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/datasource/TableDesc.java @@ -0,0 +1,15 @@ +package io.dataease.dto.datasource; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class TableDesc { + @ApiModelProperty("表名称") + private String name; + @ApiModelProperty("表备注") + private String remark; +} diff --git a/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java b/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java index a09e5576cc..d99d7a80b0 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java @@ -1,5 +1,6 @@ package io.dataease.provider.datasource; +import io.dataease.dto.datasource.TableDesc; import io.dataease.dto.datasource.TableFiled; import io.dataease.controller.request.datasource.DatasourceRequest; @@ -12,7 +13,7 @@ public abstract class DatasourceProvider { abstract public List getData(DatasourceRequest datasourceRequest) throws Exception; - abstract public List getTables(DatasourceRequest datasourceRequest) throws Exception; + abstract public List getTables(DatasourceRequest datasourceRequest) throws Exception; public void checkStatus(DatasourceRequest datasourceRequest) throws Exception { getData(datasourceRequest); diff --git a/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java b/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java index 8943c498c5..3b1e9c7c61 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java @@ -9,6 +9,7 @@ import io.dataease.controller.request.datasource.es.Requst; import io.dataease.controller.request.datasource.es.RequstWithCursor; import io.dataease.controller.request.datasource.DatasourceRequest; import io.dataease.dto.datasource.EsConfiguration; +import io.dataease.dto.datasource.TableDesc; import io.dataease.dto.datasource.TableFiled; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; @@ -37,11 +38,11 @@ public class EsProvider extends DatasourceProvider { /** * 这里使用声明式缓存不是很妥当 * 改为chartViewService中使用编程式缓存 - @Cacheable( - value = JdbcConstants.JDBC_PROVIDER_KEY, - key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query", - condition = "#dsr.pageSize == null || #dsr.pageSize == 0L" - ) + * + * @Cacheable( value = JdbcConstants.JDBC_PROVIDER_KEY, + * key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query", + * condition = "#dsr.pageSize == null || #dsr.pageSize == 0L" + * ) */ @Override public List getData(DatasourceRequest dsr) throws Exception { @@ -49,7 +50,7 @@ public class EsProvider extends DatasourceProvider { try { EsConfiguration esConfiguration = new Gson().fromJson(dsr.getDatasource().getConfiguration(), EsConfiguration.class); HttpClientConfig httpClientConfig = new HttpClientConfig(); - if(StringUtils.isNotEmpty(esConfiguration.getEsUsername())){ + if (StringUtils.isNotEmpty(esConfiguration.getEsUsername())) { String auth = esConfiguration.getEsUsername() + ":" + esConfiguration.getEsPassword(); byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); @@ -58,24 +59,24 @@ public class EsProvider extends DatasourceProvider { requst.setQuery(dsr.getQuery()); requst.setFetch_size(dsr.getFetchSize()); String url = esConfiguration.getUrl().endsWith("/") ? esConfiguration.getUrl() + esConfiguration.getUri() + "?format=json" : esConfiguration.getUrl() + "/" + esConfiguration.getUri() + "?format=json"; - String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); + String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); list.addAll(fetchResult(esReponse)); - if(dsr.isPageable()){ - Integer realSize = dsr.getPage() * dsr.getPageSize() < list.size() ? dsr.getPage() * dsr.getPageSize(): list.size(); + if (dsr.isPageable()) { + Integer realSize = dsr.getPage() * dsr.getPageSize() < list.size() ? dsr.getPage() * dsr.getPageSize() : list.size(); list = list.subList((dsr.getPage() - 1) * dsr.getPageSize(), realSize); } - if(!dsr.isPreviewData()){ - while (StringUtils.isNotEmpty(esReponse.getCursor())) { - RequstWithCursor requstWithCursor = new RequstWithCursor(); - requstWithCursor.setQuery(dsr.getQuery()); - requstWithCursor.setFetch_size(dsr.getFetchSize()); - requstWithCursor.setCursor(esReponse.getCursor()); - response = HttpClientUtil.post(url, new Gson().toJson(requstWithCursor), httpClientConfig); - esReponse = new Gson().fromJson(response, EsReponse.class); - list.addAll(fetchResult(esReponse)); - } + if (!dsr.isPreviewData()) { + while (StringUtils.isNotEmpty(esReponse.getCursor())) { + RequstWithCursor requstWithCursor = new RequstWithCursor(); + requstWithCursor.setQuery(dsr.getQuery()); + requstWithCursor.setFetch_size(dsr.getFetchSize()); + requstWithCursor.setCursor(esReponse.getCursor()); + response = HttpClientUtil.post(url, new Gson().toJson(requstWithCursor), httpClientConfig); + esReponse = new Gson().fromJson(response, EsReponse.class); + list.addAll(fetchResult(esReponse)); + } } } catch (Exception e) { e.printStackTrace(); @@ -110,7 +111,7 @@ public class EsProvider extends DatasourceProvider { private List fetchResult(EsReponse esReponse) throws Exception { List list = new LinkedList<>(); - if(esReponse.getError() != null){ + if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } list.addAll(esReponse.getRows()); @@ -132,7 +133,7 @@ public class EsProvider extends DatasourceProvider { private List fetchResultField(String response) throws Exception { List fieldList = new ArrayList<>(); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); - if(esReponse.getError() != null){ + if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } @@ -150,7 +151,7 @@ public class EsProvider extends DatasourceProvider { private List fetchResultField4Sql(String response) throws Exception { List fieldList = new ArrayList<>(); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); - if(esReponse.getError() != null){ + if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } @@ -183,31 +184,35 @@ public class EsProvider extends DatasourceProvider { } @Override - public List getTables(DatasourceRequest datasourceRequest) throws Exception { - List tables = new ArrayList<>(); + public List getTables(DatasourceRequest datasourceRequest) throws Exception { + List tables = new ArrayList<>(); try { String response = exexQuery(datasourceRequest, "show tables", "?format=json"); tables = fetchTables(response); - tables = tables.stream().filter(table -> !table.startsWith(".")).collect(Collectors.toList()); + tables = tables.stream().filter(table -> !table.getName().startsWith(".")).collect(Collectors.toList()); } catch (Exception e) { DataEaseException.throwException(e); } return tables; } - private List fetchTables(String response) throws Exception { - List tables = new ArrayList<>(); + private List fetchTables(String response) throws Exception { + List tables = new ArrayList<>(); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); - if(esReponse.getError() != null){ + if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } for (String[] row : esReponse.getRows()) { - if(row.length == 3 && row[1].contains("TABLE") && row[2].equalsIgnoreCase("INDEX")){ - tables.add(row[0]); - } - if(row.length == 2 && row[1].contains("TABLE")){ - tables.add(row[0]); + if (row.length == 3 && row[1].contains("TABLE") && row[2].equalsIgnoreCase("INDEX")) { + TableDesc tableDesc = new TableDesc(); + tableDesc.setName(row[0]); + tables.add(tableDesc); + } + if (row.length == 2 && row[1].contains("TABLE")) { + TableDesc tableDesc = new TableDesc(); + tableDesc.setName(row[0]); + tables.add(tableDesc); } } return tables; @@ -219,36 +224,35 @@ public class EsProvider extends DatasourceProvider { } - @Override public void checkStatus(DatasourceRequest datasourceRequest) throws Exception { EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class); String response = exexGetQuery(datasourceRequest); - if(JSONObject.parseObject(response).getJSONObject("error") != null){ + if (JSONObject.parseObject(response).getJSONObject("error") != null) { throw new Exception(JSONObject.parseObject(response).getJSONObject("error").getString("reason")); } - String version = JSONObject.parseObject(response).getJSONObject("version").getString("number"); + String version = JSONObject.parseObject(response).getJSONObject("version").getString("number"); String[] versionList = version.split("\\."); - if(Integer.valueOf(versionList[0]) < 7 && Integer.valueOf(versionList[1]) < 3){ + if (Integer.valueOf(versionList[0]) < 7 && Integer.valueOf(versionList[1]) < 3) { throw new Exception(Translator.get("i18n_es_limit")); } - if(Integer.valueOf(versionList[0]) == 6 ) { + if (Integer.valueOf(versionList[0]) == 6) { esConfiguration.setUri("_xpack/sql"); } - if(Integer.valueOf(versionList[0]) == 7 ) { + if (Integer.valueOf(versionList[0]) == 7) { esConfiguration.setUri("_sql"); } datasourceRequest.getDatasource().setConfiguration(new Gson().toJson(esConfiguration)); getTables(datasourceRequest); } - private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri){ + private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri) { EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class); - uri = esConfiguration.getUri()+uri; + uri = esConfiguration.getUri() + uri; HttpClientConfig httpClientConfig = new HttpClientConfig(); - if(StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())){ + if (StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())) { String auth = esConfiguration.getEsUsername() + ":" + esConfiguration.getEsPassword(); byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); @@ -258,20 +262,20 @@ public class EsProvider extends DatasourceProvider { requst.setQuery(sql); requst.setFetch_size(datasourceRequest.getFetchSize()); String url = esConfiguration.getUrl().endsWith("/") ? esConfiguration.getUrl() + uri : esConfiguration.getUrl() + "/" + uri; - String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); + String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); return response; } - private String exexGetQuery(DatasourceRequest datasourceRequest){ + private String exexGetQuery(DatasourceRequest datasourceRequest) { EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class); HttpClientConfig httpClientConfig = new HttpClientConfig(); - if(StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())){ + if (StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())) { String auth = esConfiguration.getEsUsername() + ":" + esConfiguration.getEsPassword(); byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); } - String response = HttpClientUtil.get(esConfiguration.getUrl(), httpClientConfig); + String response = HttpClientUtil.get(esConfiguration.getUrl(), httpClientConfig); return response; } diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java index 48a2af205f..5e558dd700 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java @@ -302,12 +302,12 @@ public class JdbcProvider extends DatasourceProvider { } @Override - public List getTables(DatasourceRequest datasourceRequest) throws Exception { - List tables = new ArrayList<>(); + public List getTables(DatasourceRequest datasourceRequest) throws Exception { + List tables = new ArrayList<>(); String queryStr = getTablesSql(datasourceRequest); try (Connection con = getConnectionFromPool(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryStr)) { while (resultSet.next()) { - tables.add(resultSet.getString(1)); + tables.add(getTableDesc(datasourceRequest, resultSet)); } } catch (Exception e) { DataEaseException.throwException(e); @@ -317,7 +317,7 @@ public class JdbcProvider extends DatasourceProvider { if (queryView != null) { try (Connection con = getConnectionFromPool(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryView)) { while (resultSet.next()) { - tables.add(resultSet.getString(1)); + tables.add(getTableDesc(datasourceRequest, resultSet)); } } catch (Exception e) { DataEaseException.throwException(e); @@ -327,6 +327,19 @@ public class JdbcProvider extends DatasourceProvider { return tables; } + private TableDesc getTableDesc(DatasourceRequest datasourceRequest, ResultSet resultSet) throws SQLException { + TableDesc tableDesc = new TableDesc(); + DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); + if (datasourceType == DatasourceTypes.oracle) { + tableDesc.setRemark(resultSet.getString(3)); + } + if (datasourceType == DatasourceTypes.mysql) { + tableDesc.setRemark(resultSet.getString(2)); + } + tableDesc.setName(resultSet.getString(1)); + return tableDesc; + } + @Override public List getSchema(DatasourceRequest datasourceRequest) throws Exception { List schemas = new ArrayList<>(); @@ -583,6 +596,8 @@ public class JdbcProvider extends DatasourceProvider { switch (datasourceType) { case mysql: case mariadb: + JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class); + return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase()); case de_doris: case ds_doris: case hive: @@ -600,7 +615,7 @@ public class JdbcProvider extends DatasourceProvider { if (StringUtils.isEmpty(oracleConfiguration.getSchema())) { throw new Exception(Translator.get("i18n_schema_is_empty")); } - return "select table_name, owner from all_tables where owner='" + oracleConfiguration.getSchema() + "'"; + return "select table_name, owner, comments from all_tab_comments where owner='" + oracleConfiguration.getSchema() + "' AND table_type = 'TABLE'"; case pg: PgConfiguration pgConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfiguration.class); if (StringUtils.isEmpty(pgConfiguration.getSchema())) { @@ -649,7 +664,7 @@ public class JdbcProvider extends DatasourceProvider { if (StringUtils.isEmpty(oracleConfiguration.getSchema())) { throw new Exception(Translator.get("i18n_schema_is_empty")); } - return "select VIEW_NAME from all_views where owner='" + oracleConfiguration.getSchema() + "'"; + return "select table_name, owner, comments from all_tab_comments where owner='" + oracleConfiguration.getSchema() + "' AND table_type = 'VIEW'"; case pg: PgConfiguration pgConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfiguration.class); if (StringUtils.isEmpty(pgConfiguration.getSchema())) { diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index e900d106bd..c709074ebf 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -193,22 +193,23 @@ public class DatasourceService { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); datasourceProvider.checkStatus(datasourceRequest); - List tables = datasourceProvider.getTables(datasourceRequest); + List tables = datasourceProvider.getTables(datasourceRequest); // 获取当前数据源下的db类型数据集 DatasetTableExample datasetTableExample = new DatasetTableExample(); datasetTableExample.createCriteria().andTypeEqualTo("db").andDataSourceIdEqualTo(datasource.getId()); List datasetTables = datasetTableMapper.selectByExampleWithBLOBs(datasetTableExample); List list = new ArrayList<>(); - for (String name : tables) { + for (TableDesc tableDesc : tables) { DBTableDTO dbTableDTO = new DBTableDTO(); dbTableDTO.setDatasourceId(datasource.getId()); - dbTableDTO.setName(name); + dbTableDTO.setName(tableDesc.getName()); + dbTableDTO.setRemark(tableDesc.getRemark()); dbTableDTO.setEnableCheck(true); dbTableDTO.setDatasetPath(null); for (DatasetTable datasetTable : datasetTables) { DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class); - if (StringUtils.equals(name, dataTableInfoDTO.getTable())) { + if (StringUtils.equals(tableDesc.getName(), dataTableInfoDTO.getTable())) { dbTableDTO.setEnableCheck(false); List parents = dataSetGroupService.getParents(datasetTable.getSceneId()); StringBuilder stringBuilder = new StringBuilder(); diff --git a/frontend/src/views/dataset/add/AddDB.vue b/frontend/src/views/dataset/add/AddDB.vue index ef82b31e8c..1a773e0d79 100644 --- a/frontend/src/views/dataset/add/AddDB.vue +++ b/frontend/src/views/dataset/add/AddDB.vue @@ -58,7 +58,7 @@ border :label="t.name" :disabled="!t.enableCheck" - /> + >{{ showTableNameWithComment(t) }} @@ -133,6 +133,13 @@ export default { this.kettleRunning = res.data }) }, + showTableNameWithComment(t) { + if (t.remark) { + return `${t.name}(${t.remark})` + } else { + return `${t.name}` + } + }, save() { let ds = {} this.options.forEach(ele => {