center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge pull request #1615 from rc452860/dev

Browse Source 
feat: 添加数据库数据集时显示表注释
This commit is contained in:
taojinlong 2022-01-17 18:16:45 +08:00 committed by GitHub
commit d64f0050a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 103 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/dataease/dto/datasource/DBTableDTO.java
View File

@ -15,6 +15,8 @@ public class DBTableDTO {
private String datasourceId;
@ApiModelProperty("数据源名称")
private String name;
@ApiModelProperty("表注释")
private String remark;
@ApiModelProperty("启用检测")
private boolean enableCheck;
@ApiModelProperty("数据集路径")

15
backend/src/main/java/io/dataease/dto/datasource/TableDesc.java Normal file
View File

@ -0,0 +1,15 @@
package io.dataease.dto.datasource;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class TableDesc {
@ApiModelProperty("表名称")
private String name;
@ApiModelProperty("表备注")
private String remark;
}

3
backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java
View File

@ -1,5 +1,6 @@
package io.dataease.provider.datasource;
import io.dataease.dto.datasource.TableDesc;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.controller.request.datasource.DatasourceRequest;
@ -12,7 +13,7 @@ public abstract class DatasourceProvider {
abstract public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception;
abstract public List<String> getTables(DatasourceRequest datasourceRequest) throws Exception;
abstract public List<TableDesc> getTables(DatasourceRequest datasourceRequest) throws Exception;
public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
getData(datasourceRequest);

96
backend/src/main/java/io/dataease/provider/datasource/EsProvider.java
View File

@ -9,6 +9,7 @@ import io.dataease.controller.request.datasource.es.Requst;
import io.dataease.controller.request.datasource.es.RequstWithCursor;
import io.dataease.controller.request.datasource.DatasourceRequest;
import io.dataease.dto.datasource.EsConfiguration;
import io.dataease.dto.datasource.TableDesc;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator;
@ -37,11 +38,11 @@ public class EsProvider extends DatasourceProvider {
/**
* 这里使用声明式缓存不是很妥当
* 改为chartViewService中使用编程式缓存
@Cacheable(
value = JdbcConstants.JDBC_PROVIDER_KEY,
key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query",
condition = "#dsr.pageSize == null || #dsr.pageSize == 0L"
)
*
* @Cacheable( value = JdbcConstants.JDBC_PROVIDER_KEY,
* key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query",
* condition = "#dsr.pageSize == null || #dsr.pageSize == 0L"
* )
*/
@Override
public List<String[]> getData(DatasourceRequest dsr) throws Exception {
@ -49,7 +50,7 @@ public class EsProvider extends DatasourceProvider {
try {
EsConfiguration esConfiguration = new Gson().fromJson(dsr.getDatasource().getConfiguration(), EsConfiguration.class);
HttpClientConfig httpClientConfig = new HttpClientConfig();
if(StringUtils.isNotEmpty(esConfiguration.getEsUsername())){
if (StringUtils.isNotEmpty(esConfiguration.getEsUsername())) {
String auth = esConfiguration.getEsUsername() + ":" + esConfiguration.getEsPassword();
byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
@ -58,24 +59,24 @@ public class EsProvider extends DatasourceProvider {
requst.setQuery(dsr.getQuery());
requst.setFetch_size(dsr.getFetchSize());
String url = esConfiguration.getUrl().endsWith("/") ? esConfiguration.getUrl() + esConfiguration.getUri() + "?format=json" : esConfiguration.getUrl() + "/" + esConfiguration.getUri() + "?format=json";
String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig);
String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig);
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
list.addAll(fetchResult(esReponse));
if(dsr.isPageable()){
Integer realSize = dsr.getPage() * dsr.getPageSize() < list.size() ? dsr.getPage() * dsr.getPageSize(): list.size();
if (dsr.isPageable()) {
Integer realSize = dsr.getPage() * dsr.getPageSize() < list.size() ? dsr.getPage() * dsr.getPageSize() : list.size();
list = list.subList((dsr.getPage() - 1) * dsr.getPageSize(), realSize);
}
if(!dsr.isPreviewData()){
while (StringUtils.isNotEmpty(esReponse.getCursor())) {
RequstWithCursor requstWithCursor = new RequstWithCursor();
requstWithCursor.setQuery(dsr.getQuery());
requstWithCursor.setFetch_size(dsr.getFetchSize());
requstWithCursor.setCursor(esReponse.getCursor());
response = HttpClientUtil.post(url, new Gson().toJson(requstWithCursor), httpClientConfig);
esReponse = new Gson().fromJson(response, EsReponse.class);
list.addAll(fetchResult(esReponse));
}
if (!dsr.isPreviewData()) {
while (StringUtils.isNotEmpty(esReponse.getCursor())) {
RequstWithCursor requstWithCursor = new RequstWithCursor();
requstWithCursor.setQuery(dsr.getQuery());
requstWithCursor.setFetch_size(dsr.getFetchSize());
requstWithCursor.setCursor(esReponse.getCursor());
response = HttpClientUtil.post(url, new Gson().toJson(requstWithCursor), httpClientConfig);
esReponse = new Gson().fromJson(response, EsReponse.class);
list.addAll(fetchResult(esReponse));
}
}
} catch (Exception e) {
e.printStackTrace();
@ -110,7 +111,7 @@ public class EsProvider extends DatasourceProvider {
private List<String[]> fetchResult(EsReponse esReponse) throws Exception {
List<String[]> list = new LinkedList<>();
if(esReponse.getError() != null){
if (esReponse.getError() != null) {
throw new Exception(esReponse.getError().getReason());
}
list.addAll(esReponse.getRows());
@ -132,7 +133,7 @@ public class EsProvider extends DatasourceProvider {
private List<TableFiled> fetchResultField(String response) throws Exception {
List<TableFiled> fieldList = new ArrayList<>();
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
if(esReponse.getError() != null){
if (esReponse.getError() != null) {
throw new Exception(esReponse.getError().getReason());
}
@ -150,7 +151,7 @@ public class EsProvider extends DatasourceProvider {
private List<TableFiled> fetchResultField4Sql(String response) throws Exception {
List<TableFiled> fieldList = new ArrayList<>();
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
if(esReponse.getError() != null){
if (esReponse.getError() != null) {
throw new Exception(esReponse.getError().getReason());
}
@ -183,31 +184,35 @@ public class EsProvider extends DatasourceProvider {
}
@Override
public List<String> getTables(DatasourceRequest datasourceRequest) throws Exception {
List<String> tables = new ArrayList<>();
public List<TableDesc> getTables(DatasourceRequest datasourceRequest) throws Exception {
List<TableDesc> tables = new ArrayList<>();
try {
String response = exexQuery(datasourceRequest, "show tables", "?format=json");
tables = fetchTables(response);
tables = tables.stream().filter(table -> !table.startsWith(".")).collect(Collectors.toList());
tables = tables.stream().filter(table -> !table.getName().startsWith(".")).collect(Collectors.toList());
} catch (Exception e) {
DataEaseException.throwException(e);
}
return tables;
}
private List<String> fetchTables(String response) throws Exception {
List<String> tables = new ArrayList<>();
private List<TableDesc> fetchTables(String response) throws Exception {
List<TableDesc> tables = new ArrayList<>();
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
if(esReponse.getError() != null){
if (esReponse.getError() != null) {
throw new Exception(esReponse.getError().getReason());
}
for (String[] row : esReponse.getRows()) {
if(row.length == 3 && row[1].contains("TABLE") && row[2].equalsIgnoreCase("INDEX")){
tables.add(row[0]);
}
if(row.length == 2 && row[1].contains("TABLE")){
tables.add(row[0]);
if (row.length == 3 && row[1].contains("TABLE") && row[2].equalsIgnoreCase("INDEX")) {
TableDesc tableDesc = new TableDesc();
tableDesc.setName(row[0]);
tables.add(tableDesc);
}
if (row.length == 2 && row[1].contains("TABLE")) {
TableDesc tableDesc = new TableDesc();
tableDesc.setName(row[0]);
tables.add(tableDesc);
}
}
return tables;
@ -219,36 +224,35 @@ public class EsProvider extends DatasourceProvider {
}
@Override
public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class);
String response = exexGetQuery(datasourceRequest);
if(JSONObject.parseObject(response).getJSONObject("error") != null){
if (JSONObject.parseObject(response).getJSONObject("error") != null) {
throw new Exception(JSONObject.parseObject(response).getJSONObject("error").getString("reason"));
}
String version = JSONObject.parseObject(response).getJSONObject("version").getString("number");
String version = JSONObject.parseObject(response).getJSONObject("version").getString("number");
String[] versionList = version.split("\\.");
if(Integer.valueOf(versionList[0]) < 7 && Integer.valueOf(versionList[1]) < 3){
if (Integer.valueOf(versionList[0]) < 7 && Integer.valueOf(versionList[1]) < 3) {
throw new Exception(Translator.get("i18n_es_limit"));
}
if(Integer.valueOf(versionList[0]) == 6 ) {
if (Integer.valueOf(versionList[0]) == 6) {
esConfiguration.setUri("_xpack/sql");
}
if(Integer.valueOf(versionList[0]) == 7 ) {
if (Integer.valueOf(versionList[0]) == 7) {
esConfiguration.setUri("_sql");
}
datasourceRequest.getDatasource().setConfiguration(new Gson().toJson(esConfiguration));
getTables(datasourceRequest);
}
private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri){
private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri) {
EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class);
uri = esConfiguration.getUri()+uri;
uri = esConfiguration.getUri() + uri;
HttpClientConfig httpClientConfig = new HttpClientConfig();
if(StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())){
if (StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())) {
String auth = esConfiguration.getEsUsername() + ":" + esConfiguration.getEsPassword();
byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
@ -258,20 +262,20 @@ public class EsProvider extends DatasourceProvider {
requst.setQuery(sql);
requst.setFetch_size(datasourceRequest.getFetchSize());
String url = esConfiguration.getUrl().endsWith("/") ? esConfiguration.getUrl() + uri : esConfiguration.getUrl() + "/" + uri;
String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig);
String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig);
return response;
}
private String exexGetQuery(DatasourceRequest datasourceRequest){
private String exexGetQuery(DatasourceRequest datasourceRequest) {
EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class);
HttpClientConfig httpClientConfig = new HttpClientConfig();
if(StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())){
if (StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())) {
String auth = esConfiguration.getEsUsername() + ":" + esConfiguration.getEsPassword();
byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
}
String response = HttpClientUtil.get(esConfiguration.getUrl(), httpClientConfig);
String response = HttpClientUtil.get(esConfiguration.getUrl(), httpClientConfig);
return response;
}

27
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@ -302,12 +302,12 @@ public class JdbcProvider extends DatasourceProvider {
}
@Override
public List<String> getTables(DatasourceRequest datasourceRequest) throws Exception {
List<String> tables = new ArrayList<>();
public List<TableDesc> getTables(DatasourceRequest datasourceRequest) throws Exception {
List<TableDesc> tables = new ArrayList<>();
String queryStr = getTablesSql(datasourceRequest);
try (Connection con = getConnectionFromPool(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryStr)) {
while (resultSet.next()) {
tables.add(resultSet.getString(1));
tables.add(getTableDesc(datasourceRequest, resultSet));
}
} catch (Exception e) {
DataEaseException.throwException(e);
@ -317,7 +317,7 @@ public class JdbcProvider extends DatasourceProvider {
if (queryView != null) {
try (Connection con = getConnectionFromPool(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryView)) {
while (resultSet.next()) {
tables.add(resultSet.getString(1));
tables.add(getTableDesc(datasourceRequest, resultSet));
}
} catch (Exception e) {
DataEaseException.throwException(e);
@ -327,6 +327,19 @@ public class JdbcProvider extends DatasourceProvider {
return tables;
}
private TableDesc getTableDesc(DatasourceRequest datasourceRequest, ResultSet resultSet) throws SQLException {
TableDesc tableDesc = new TableDesc();
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType());
if (datasourceType == DatasourceTypes.oracle) {
tableDesc.setRemark(resultSet.getString(3));
}
if (datasourceType == DatasourceTypes.mysql) {
tableDesc.setRemark(resultSet.getString(2));
}
tableDesc.setName(resultSet.getString(1));
return tableDesc;
}
@Override
public List<String> getSchema(DatasourceRequest datasourceRequest) throws Exception {
List<String> schemas = new ArrayList<>();
@ -583,6 +596,8 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class);
return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase());
case de_doris:
case ds_doris:
case hive:
@ -600,7 +615,7 @@ public class JdbcProvider extends DatasourceProvider {
if (StringUtils.isEmpty(oracleConfiguration.getSchema())) {
throw new Exception(Translator.get("i18n_schema_is_empty"));
}
return "select table_name, owner from all_tables where owner='" + oracleConfiguration.getSchema() + "'";
return "select table_name, owner, comments from all_tab_comments where owner='" + oracleConfiguration.getSchema() + "' AND table_type = 'TABLE'";
case pg:
PgConfiguration pgConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfiguration.class);
if (StringUtils.isEmpty(pgConfiguration.getSchema())) {
@ -649,7 +664,7 @@ public class JdbcProvider extends DatasourceProvider {
if (StringUtils.isEmpty(oracleConfiguration.getSchema())) {
throw new Exception(Translator.get("i18n_schema_is_empty"));
}
return "select VIEW_NAME from all_views where owner='" + oracleConfiguration.getSchema() + "'";
return "select table_name, owner, comments from all_tab_comments where owner='" + oracleConfiguration.getSchema() + "' AND table_type = 'VIEW'";
case pg:
PgConfiguration pgConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfiguration.class);
if (StringUtils.isEmpty(pgConfiguration.getSchema())) {

9
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -193,22 +193,23 @@ public class DatasourceService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
datasourceProvider.checkStatus(datasourceRequest);
List<String> tables = datasourceProvider.getTables(datasourceRequest);
List<TableDesc> tables = datasourceProvider.getTables(datasourceRequest);
// 获取当前数据源下的db类型数据集
DatasetTableExample datasetTableExample = new DatasetTableExample();
datasetTableExample.createCriteria().andTypeEqualTo("db").andDataSourceIdEqualTo(datasource.getId());
List<DatasetTable> datasetTables = datasetTableMapper.selectByExampleWithBLOBs(datasetTableExample);
List<DBTableDTO> list = new ArrayList<>();
for (String name : tables) {
for (TableDesc tableDesc : tables) {
DBTableDTO dbTableDTO = new DBTableDTO();
dbTableDTO.setDatasourceId(datasource.getId());
dbTableDTO.setName(name);
dbTableDTO.setName(tableDesc.getName());
dbTableDTO.setRemark(tableDesc.getRemark());
dbTableDTO.setEnableCheck(true);
dbTableDTO.setDatasetPath(null);
for (DatasetTable datasetTable : datasetTables) {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
if (StringUtils.equals(name, dataTableInfoDTO.getTable())) {
if (StringUtils.equals(tableDesc.getName(), dataTableInfoDTO.getTable())) {
dbTableDTO.setEnableCheck(false);
List<DatasetGroup> parents = dataSetGroupService.getParents(datasetTable.getSceneId());
StringBuilder stringBuilder = new StringBuilder();

9
frontend/src/views/dataset/add/AddDB.vue
View File

@ -58,7 +58,7 @@
border
:label="t.name"
:disabled="!t.enableCheck"
/>
>{{ showTableNameWithComment(t) }}</el-checkbox>
</el-tooltip>
</el-checkbox-group>
</el-col>
@ -133,6 +133,13 @@ export default {
this.kettleRunning = res.data
})
},
showTableNameWithComment(t) {
if (t.remark) {
return `${t.name}(${t.remark})`
} else {
return `${t.name}`
}
},
save() {
let ds = {}
this.options.forEach(ele => {