Merge pull request #12448 from dataease/pr@dev2@fixds

Pr@dev2@fixds
This commit is contained in:
taojinlong 2024-09-25 16:25:32 +08:00 committed by GitHub
commit d521e1ce32
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 336 additions and 16 deletions

View File

@ -110,7 +110,6 @@ public class ChartDataManage {
}
var dillAxis = new ArrayList<ChartViewFieldDTO>();
DatasetGroupInfoDTO table = datasetGroupManage.getDatasetGroupInfoDTO(view.getTableId(), null);
if (table == null) {
DEException.throwException(ResultCode.DATA_IS_WRONG.code(), Translator.get("i18n_no_ds"));

View File

@ -6,4 +6,5 @@ package io.dataease.dataset.constant;
public class DatasetTableType {
public static String DB = "db";
public static String SQL = "sql";
public static String Es = "es";
}

View File

@ -129,6 +129,15 @@ public class DatasetDataManage {
datasourceRequest.setTable(tableInfoDTO.getTable());
}
tableFields = provider.fetchTableField(datasourceRequest);
} else if (StringUtils.equalsIgnoreCase(type, DatasetTableType.Es)) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasetTableDTO.getDatasourceId());
Provider provider = ProviderFactory.getProvider(type);
DatasourceRequest datasourceRequest = new DatasourceRequest();
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);
datasourceRequest.setDatasource(datasourceSchemaDTO);
datasourceRequest.setTable(datasetTableDTO.getTableName());
tableFields = provider.fetchTableField(datasourceRequest);
} else {
// excel,api
@ -185,9 +194,7 @@ public class DatasetDataManage {
DEException.throwException(Translator.get("i18n_no_column_permission"));
}
}
buildFieldName(sqlMap, fields);
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
DatasourceUtils.checkDsStatus(dsMap);
List<String> dsList = new ArrayList<>();
@ -202,13 +209,11 @@ public class DatasetDataManage {
}
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>();
TokenUserBO user = AuthUtils.getUser();
if (user != null && checkPermission) {
rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId());
}
Provider provider;
if (crossDs) {
provider = ProviderFactory.getDefaultProvider();
@ -236,7 +241,6 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setQuery(querySQL);
datasourceRequest.setDsList(dsMap);
Map<String, Object> data = provider.fetchResultField(datasourceRequest);
Map<String, Object> map = new LinkedHashMap<>();

View File

@ -494,7 +494,16 @@ public class DatasetSQLManage {
datasourceSchemaDTO.setSchemaAlias(schemaAlias);
dsMap.put(coreDatasource.getId(), datasourceSchemaDTO);
}
} else {
} else if (StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.Es)){
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(ds.getDatasourceId());
schemaAlias = String.format(SQLConstants.SCHEMA, coreDatasource.getId());
if (!dsMap.containsKey(coreDatasource.getId())) {
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);
datasourceSchemaDTO.setSchemaAlias(schemaAlias);
dsMap.put(coreDatasource.getId(), datasourceSchemaDTO);
}
}else {
CoreDatasource coreDatasource = engineManage.getDeEngine();
schemaAlias = String.format(SQLConstants.SCHEMA, coreDatasource.getId());
if (!dsMap.containsKey(coreDatasource.getId())) {

View File

@ -0,0 +1,29 @@
package io.dataease.datasource.dto.es;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class EsResponse {
private List<Column> columns = new ArrayList<>();
private List<String[]> rows = new ArrayList<>();
private String cursor;
private Integer status;
private Error error;
private String version;
@Data
public class Error {
private String type;
private String reason;
}
@Data
public class Column {
private String name;
private String type;
}
}

View File

@ -0,0 +1,10 @@
package io.dataease.datasource.dto.es;
import lombok.Data;
@Data
public class Request {
private String query;
private Integer fetch_size = 10000;
private boolean field_multi_value_leniency = true;
}

View File

@ -0,0 +1,8 @@
package io.dataease.datasource.dto.es;
import lombok.Data;
@Data
public class RequestWithCursor extends Request {
private String cursor;
}

View File

@ -1271,7 +1271,6 @@ public class CalciteProvider extends Provider {
try {
connection = initConnection(dsMap);
} catch (Exception e) {
e.printStackTrace();
}
});

View File

@ -0,0 +1,211 @@
package io.dataease.datasource.provider;
import com.google.gson.Gson;
import com.google.gson.JsonParser;
import io.dataease.dataset.utils.FieldUtils;
import io.dataease.datasource.dto.es.EsResponse;
import io.dataease.datasource.dto.es.Request;
import io.dataease.datasource.type.Es;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.*;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.i18n.Translator;
import io.dataease.utils.HttpClientConfig;
import io.dataease.utils.HttpClientUtil;
import io.dataease.utils.JsonUtil;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Collectors;
@Service("esProvider")
public class EsProvider extends Provider {
@Override
public List<String> getSchema(DatasourceRequest datasourceRequest) {
return new ArrayList<>();
}
@Override
public List<DatasetTableDTO> getTables(DatasourceRequest datasourceRequest) {
List<DatasetTableDTO> tables = new ArrayList<>();
try {
String response = execQuery(datasourceRequest, "show tables", "?format=json");
tables = fetchTables(response);
tables = tables.stream().filter(table -> StringUtils.isNotEmpty(table.getTableName()) && !table.getTableName().startsWith(".")).collect(Collectors.toList());
tables.forEach(table -> {
table.setDatasourceId(datasourceRequest.getDatasource().getId());
});
} catch (Exception e) {
e.getMessage();
DEException.throwException(e);
}
return tables;
}
@Override
public ConnectionObj getConnection(DatasourceDTO coreDatasource) throws Exception {
return null;
}
@Override
public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
Es es = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Es.class);
String response = execGetQuery(datasourceRequest);
if (JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error") != null) {
throw new Exception(JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error").get("reason").getAsString());
}
String version = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("version").get("number").getAsString();
String[] versionList = version.split("\\.");
if (Integer.valueOf(versionList[0]) < 7 && Integer.valueOf(versionList[1]) < 3) {
throw new Exception(Translator.get("i18n_es_limit"));
}
if (Integer.valueOf(versionList[0]) == 6) {
es.setUri("_xpack/sql");
}
if (Integer.valueOf(versionList[0]) > 6) {
es.setUri("_sql");
}
datasourceRequest.getDatasource().setConfiguration(JsonUtil.toJSONString(es).toString());
getTables(datasourceRequest);
return "Success";
}
@Override
public Map<String, Object> fetchResultField(DatasourceRequest datasourceRequest) {
Map<String, Object> result = new HashMap<>();
try {
String response = execQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
result.put("dataList", fetchResultData(response));
result.put("fieldList", fetchResultField4Sql(response));
} catch (Exception e) {
e.printStackTrace();
DEException.throwException(e);
}
return result;
}
@Override
public List<TableField> fetchTableField(DatasourceRequest datasourceRequest) {
List<TableField> tableFields = new ArrayList<>();
try {
String response = execQuery(datasourceRequest, "select * from " + datasourceRequest.getTable() + " limit 0", "?format=json");
tableFields = fetchResultField4Sql(response);
} catch (Exception e) {
DEException.throwException(e);
}
return tableFields;
}
@Override
public void hidePW(DatasourceDTO datasourceDTO) {
}
private List<String[]> fetchResultData(String response) throws Exception {
EsResponse esResponse = new Gson().fromJson(response, EsResponse.class);
return fetchResultData(esResponse);
}
private List<String[]> fetchResultData(EsResponse esResponse) throws Exception {
List<String[]> list = new LinkedList<>();
if (esResponse.getError() != null) {
throw new Exception(esResponse.getError().getReason());
}
list.addAll(esResponse.getRows());
return list;
}
private List<TableField> fetchResultField4Sql(String response) throws Exception {
List<TableField> fieldList = new ArrayList<>();
EsResponse esResponse = new Gson().fromJson(response, EsResponse.class);
if (esResponse.getError() != null) {
throw new Exception(esResponse.getError().getReason());
}
for (EsResponse.Column column : esResponse.getColumns()) {
TableField field = new TableField();
field.setOriginName(column.getName());
field.setOriginName(column.getName());
field.setFieldType(column.getType());
field.setType(column.getType().toUpperCase());
field.setFieldType(field.getType());
int deType = FieldUtils.transType2DeType(field.getType());
field.setDeExtractType(deType);
field.setDeType(deType);
fieldList.add(field);
}
return fieldList;
}
private List<DatasetTableDTO> fetchTables(String response) throws Exception {
List<DatasetTableDTO> tables = new ArrayList<>();
EsResponse esResponse = new Gson().fromJson(response, EsResponse.class);
if (esResponse.getError() != null) {
throw new Exception(esResponse.getError().getReason());
}
for (String[] row : esResponse.getRows()) {
DatasetTableDTO tableDesc = new DatasetTableDTO();
if (row.length == 3 && row[1].contains("TABLE") && row[2].equalsIgnoreCase("INDEX")) {
tableDesc.setTableName(row[0]);
}
if (row.length == 2 && row[1].contains("TABLE")) {
tableDesc.setTableName(row[0]);
}
if (row.length == 4 && row[2].contains("TABLE") && row[3].equalsIgnoreCase("INDEX")) {
tableDesc.setTableName(row[1]);
}
tableDesc.setType("es");
tables.add(tableDesc);
}
return tables;
}
private String execQuery(DatasourceRequest datasourceRequest, String sql, String uri) {
Es es = null;
if (datasourceRequest.getDatasource() == null) {
Collection<DatasourceSchemaDTO> datasourceSchemaDTOS = datasourceRequest.getDsList().values();
es = JsonUtil.parseObject(datasourceSchemaDTOS.stream().findFirst().get().getConfiguration(), Es.class);
} else {
es = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Es.class);
}
uri = es.getUri() + uri;
HttpClientConfig httpClientConfig = new HttpClientConfig();
if (StringUtils.isNotEmpty(es.getUsername()) && StringUtils.isNotEmpty(es.getPassword())) {
String auth = es.getUsername() + ":" + es.getPassword();
byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
}
Request request = new Request();
request.setQuery(sql);
request.setFetch_size(datasourceRequest.getFetchSize());
String url = es.getUrl().endsWith("/") ? es.getUrl() + uri : es.getUrl() + "/" + uri;
return HttpClientUtil.post(url, new Gson().toJson(request), httpClientConfig);
}
private String execGetQuery(DatasourceRequest datasourceRequest) {
Es es = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Es.class);
HttpClientConfig httpClientConfig = new HttpClientConfig();
if (StringUtils.isNotEmpty(es.getUsername()) && StringUtils.isNotEmpty(es.getPassword())) {
String auth = es.getUsername() + ":" + es.getPassword();
byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
}
return HttpClientUtil.get(es.getUrl(), httpClientConfig);
}
}

View File

@ -0,0 +1,15 @@
package io.dataease.datasource.type;
import lombok.Data;
import org.springframework.stereotype.Component;
@Data
public class Es {
private String url;
private String username;
private String password;
private String version;
private String uri;
}

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 6.2 KiB

View File

@ -13,6 +13,7 @@ import redshiftDs from '@/assets/svg/redshift-ds.svg'
import APIDs from '@/assets/svg/API-ds.svg'
import ExcelDs from '@/assets/svg/Excel-ds.svg'
import dorisDs from '@/assets/svg/doris-ds.svg'
import esDs from '@/assets/svg/es-ds.svg'
const iconDatasourceMap = {
mysql: mysqlDs,
oracle: oracleDs,
@ -28,7 +29,8 @@ const iconDatasourceMap = {
redshift: redshiftDs,
API: APIDs,
Excel: ExcelDs,
doris: dorisDs
doris: dorisDs,
es: esDs
}
export { iconDatasourceMap }

View File

@ -928,7 +928,7 @@ defineExpose({
</div>
</template>
<template v-if="notapiexcelconfig">
<el-form-item label="连接方式" prop="type">
<el-form-item label="连接方式" prop="type" v-if="form.type !== 'es'">
<el-radio-group v-model="form.configuration.urlType">
<el-radio label="hostName">主机名</el-radio>
<el-radio label="jdbcUrl">JDBC 连接</el-radio>
@ -950,7 +950,7 @@ defineExpose({
<el-form-item
:label="t('datasource.host')"
prop="configuration.host"
v-if="form.configuration.urlType !== 'jdbcUrl'"
v-if="form.configuration.urlType !== 'jdbcUrl' && form.type !== 'es'"
>
<el-input
v-model="form.configuration.host"
@ -961,7 +961,7 @@ defineExpose({
<el-form-item
:label="t('datasource.port')"
prop="configuration.port"
v-if="form.configuration.urlType !== 'jdbcUrl'"
v-if="form.configuration.urlType !== 'jdbcUrl' && form.type !== 'es'"
>
<el-input-number
v-model="form.configuration.port"
@ -977,7 +977,7 @@ defineExpose({
<el-form-item
:label="t('datasource.data_base')"
prop="configuration.dataBase"
v-if="form.configuration.urlType !== 'jdbcUrl'"
v-if="form.configuration.urlType !== 'jdbcUrl' && form.type !== 'es'"
>
<el-input
v-model="form.configuration.dataBase"
@ -1029,6 +1029,17 @@ defineExpose({
{{ t('datasource.kerbers_info') }}
</p>
</el-form-item>
<el-form-item
v-if="form.type == 'es'"
:label="$t('datasource.datasource_url')"
prop="configuration.url"
>
<el-input
v-model="form.configuration.url"
:placeholder="$t('datasource.please_input_datasource_url')"
autocomplete="off"
/>
</el-form-item>
<el-form-item :label="t('datasource.user_name')" v-if="form.type !== 'presto'">
<el-input
:placeholder="t('common.inputText') + t('datasource.user_name')"
@ -1083,7 +1094,7 @@ defineExpose({
</el-form-item>
<el-form-item
:label="t('datasource.extra_params')"
v-if="form.configuration.urlType !== 'jdbcUrl'"
v-if="form.configuration.urlType !== 'jdbcUrl' && form.type !== 'es'"
>
<el-input
:placeholder="t('common.inputText') + t('datasource.extra_params')"

View File

@ -42,6 +42,12 @@ export const dsTypes = [
extraParams:
'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
},
{
type: 'es',
name: 'Elasticsearch',
catalog: 'OLAP',
extraParams: ''
},
{
type: 'StarRocks',
name: 'StarRocks',

View File

@ -142,7 +142,7 @@ const nickName = ref('')
const dsName = ref('')
const userDrawer = ref(false)
const rawDatasourceList = ref([])
const showPriority = ref(true)
const showPriority = ref(false)
const showSSH = ref(true)
const datasourceEditor = ref()
const activeTab = ref('')
@ -378,6 +378,7 @@ const initSearch = () => {
state.filterTable = tableData.value.filter(ele =>
ele.tableName.toLowerCase().includes(nickName.value.toLowerCase())
)
console.log(tableData.value)
state.paginationConfig.total = state.filterTable.length
}
@ -862,6 +863,7 @@ const operation = (cmd: string, data: Tree, nodeType: string) => {
}
const handleClick = (tabName: TabPaneName) => {
console.log(tabName)
switch (tabName) {
case 'config':
listDatasourceTables({ datasourceId: nodeInfo.id }).then(res => {
@ -1372,7 +1374,7 @@ const getMenuList = (val: boolean) => {
}}</BaseInfoItem>
</el-col>
</el-row>
<template v-if="!['Excel', 'API'].includes(nodeInfo.type)">
<template v-if="!['Excel', 'API', 'es'].includes(nodeInfo.type)">
<el-row :gutter="24" v-show="nodeInfo.configuration.urlType !== 'jdbcUrl'">
<el-col :span="12">
<BaseInfoItem :label="t('datasource.host')">{{
@ -1499,6 +1501,15 @@ const getMenuList = (val: boolean) => {
</el-row>
</template>
</template>
<template v-if="['es'].includes(nodeInfo.type)">
<el-row :gutter="24">
<el-col :span="12">
<BaseInfoItem :label="t('datasource.datasource_url')">{{
nodeInfo.configuration.url
}}</BaseInfoItem>
</el-col>
</el-row>
</template>
</template>
</BaseInfoContent>
<BaseInfoContent

View File

@ -21,6 +21,9 @@ import java.util.concurrent.ConcurrentHashMap;
public class ProviderFactory {
public static Provider getProvider(String type) throws DEException {
if (type.equalsIgnoreCase("es")) {
return SpringContextUtil.getApplicationContext().getBean("esProvider", Provider.class);
}
List<String> list = Arrays.stream(DatasourceConfiguration.DatasourceType.values()).map(DatasourceConfiguration.DatasourceType::getType).toList();
if (list.contains(type)) {
return SpringContextUtil.getApplicationContext().getBean("calciteProvider", Provider.class);

View File

@ -18,6 +18,7 @@ public class DatasourceConfiguration extends Configuration {
impala("impala", "Apache Impala", "OLAP", "`", "`"),
mariadb("mariadb", "Mariadb", "OLTP", "`", "`"),
StarRocks("StarRocks", "StarRocks", "OLAP", "`", "`"),
es("es", "Elasticsearch", "OLAP", "\"", "\""),
doris("doris", "Apache Doris", "OLAP", "`", "`"),
TiDB("TiDB", "TiDB", "OLTP", "`", "`"),
oracle("oracle", "ORACLE", "OLTP", "\"", "\""),