center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat: 支持集群（doris）

Browse Source
This commit is contained in:
taojinlong 2022-03-22 19:33:09 +08:00
parent 6a883eb6b2
commit a045f51515
6 changed files with 356 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/dataease/dto/datasource/DorisConfiguration.java
View File

@ -7,7 +7,7 @@ import lombok.Setter;
@Setter
public class DorisConfiguration extends MysqlConfiguration {
private Integer httpPort;
private Integer httpPort = 8030;
private Integer replicationNum = 1;
private Integer bucketNum = 10;

1
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@ -487,7 +487,6 @@ public class JdbcProvider extends DatasourceProvider {
break;
case impala:
ImpalaConfiguration impalaConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), ImpalaConfiguration.class);
System.out.println(new Gson().toJson(impalaConfiguration));
username = impalaConfiguration.getUsername();
password = impalaConfiguration.getPassword();
driver = impalaConfiguration.getDriver();

1
backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java
View File

@ -43,7 +43,6 @@ public class MongoQueryProvider extends QueryProvider {
@Override
public Integer transFieldType(String field) {
System.out.println(field);
field = field.toUpperCase();
switch (field) {
case "CHAR":

82
backend/src/main/java/io/dataease/service/engine/EngineService.java
View File

@ -1,14 +1,20 @@
package io.dataease.service.engine;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import io.dataease.base.domain.Datasource;
import io.dataease.base.domain.DeEngine;
import io.dataease.base.domain.DeEngineExample;
import io.dataease.base.mapper.DeEngineMapper;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.HttpClientConfig;
import io.dataease.commons.utils.HttpClientUtil;
import io.dataease.controller.ResultHolder;
import io.dataease.controller.request.datasource.DatasourceRequest;
import io.dataease.dto.DatasourceDTO;
import io.dataease.dto.datasource.DorisConfiguration;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
@ -20,7 +26,9 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.Base64;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
@Service
@ -33,38 +41,38 @@ public class EngineService {
@Resource
private DatasourceService datasource;
public Boolean isLocalMode(){
public Boolean isLocalMode() {
return env.getProperty("engine_mode", "local").equalsIgnoreCase("local");
}
public Boolean isSimpleMode(){
public Boolean isSimpleMode() {
return env.getProperty("engine_mode", "local").equalsIgnoreCase("simple");
}
public Boolean isClusterMode(){
public Boolean isClusterMode() {
return env.getProperty("engine_mode", "local").equalsIgnoreCase("cluster");
}
public String mode(){
public String mode() {
return env.getProperty("engine_mode", "local");
}
public DeEngine info(){
public DeEngine info() {
DeEngineExample deEngineExample = new DeEngineExample();
if(isClusterMode()){
if (isClusterMode()) {
deEngineExample.createCriteria().andTypeEqualTo("engine_doris");
}else {
} else {
deEngineExample.createCriteria().andTypeEqualTo("engine_mysql");
}
List<DeEngine> deEngines = deEngineMapper.selectByExampleWithBLOBs(deEngineExample);
if(CollectionUtils.isEmpty(deEngines)){
if (CollectionUtils.isEmpty(deEngines)) {
return new DeEngine();
}
return deEngines.get(0);
}
public ResultHolder validate(DatasourceDTO datasource) throws Exception {
if(StringUtils.isEmpty(datasource.getType()) || StringUtils.isEmpty(datasource.getConfiguration())){
if (StringUtils.isEmpty(datasource.getType()) || StringUtils.isEmpty(datasource.getConfiguration())) {
throw new Exception("未完整设置数据引擎");
}
try {
@ -72,17 +80,49 @@ public class EngineService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceProvider.checkStatus(datasourceRequest);
return ResultHolder.success(datasource);
}catch (Exception e){
} catch (Exception e) {
return ResultHolder.error("Engine is invalid: " + e.getMessage());
}
if (datasource.getType().equalsIgnoreCase("engine_doris")) {
DorisConfiguration dorisConfiguration = new Gson().fromJson(datasource.getConfiguration(), DorisConfiguration.class);
HttpClientConfig httpClientConfig = new HttpClientConfig();
String authValue = "Basic " + Base64.getUrlEncoder().encodeToString((dorisConfiguration.getUsername()
+ ":" + dorisConfiguration.getPassword()).getBytes());
httpClientConfig.addHeader("Authorization", authValue);
String response;
try {
response = HttpClientUtil.get("http://" + dorisConfiguration.getHost() + ":" + dorisConfiguration.getHttpPort() + "/api/backends", httpClientConfig);
}catch (Exception e){
return ResultHolder.error("Engine is invalid: " + e.getMessage());
}
JSONArray backends = Optional.ofNullable(JSONObject.parseObject(response).getJSONObject("data")).orElse(new JSONObject()).getJSONArray("backends");
if(CollectionUtils.isEmpty(backends)){
return ResultHolder.error("Engine is invalid: no backends found.");
}
Integer alives = 0;
for (int i = 0; i < backends.size(); i++) {
JSONObject kv = backends.getJSONObject(i);
if (kv.getBoolean("is_alive")) {
alives ++;
}
}
if(alives < dorisConfiguration.getReplicationNum()){
return ResultHolder.error("Engine params is invalid: 副本数量不能大于节点数量.");
}
}
return ResultHolder.success(datasource);
}
public ResultHolder save(DeEngine engine) throws Exception {
if(StringUtils.isEmpty(engine.getId())){
if (StringUtils.isEmpty(engine.getId())) {
engine.setId(UUID.randomUUID().toString());
deEngineMapper.insert(engine);
}else {
} else {
deEngineMapper.updateByPrimaryKeyWithBLOBs(engine);
datasource.handleConnectionPool(getDeEngine(), "delete");
}
@ -91,17 +131,19 @@ public class EngineService {
return ResultHolder.success(engine);
}
private void setDs(DeEngine engine){
CacheUtils.put("ENGINE", "engine", engine, null, null);
private void setDs(DeEngine engine) {
Datasource datasource = new Datasource();
BeanUtils.copyBean(datasource, engine);
CacheUtils.put("ENGINE", "engine", datasource, null, null);
}
public Datasource getDeEngine() throws Exception{
public Datasource getDeEngine() throws Exception {
Object catcheEngine = CacheUtils.get("ENGINE", "engine");
if(catcheEngine != null){
if (catcheEngine != null) {
return (Datasource) catcheEngine;
}
if(isLocalMode()){
if (isLocalMode()) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("dataSourceType", "jdbc");
jsonObject.put("dataBase", env.getProperty("doris.db", "doris"));
@ -118,9 +160,9 @@ public class EngineService {
engine.setType("engine_doris");
engine.setConfiguration(jsonObject.toJSONString());
setDs(engine);
}else {
} else {
List<DeEngine> deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample());
if(CollectionUtils.isEmpty(deEngines)){
if (CollectionUtils.isEmpty(deEngines)) {
throw new Exception("未设置数据引擎");
}
setDs(deEngines.get(0));

2
frontend/src/lang/zh.js
View File

@ -1303,6 +1303,8 @@ export default {
host: '主机名/IP地址',
doris_host: 'Doris 地址',
port: '端口',
query_port: 'Query Port',
http_port: 'Http Port',
datasource_url: '地址',
please_input_datasource_url: '请输入 Elasticsearch 地址，如: http://es_host:es_port',
please_input_data_base: '请输入数据库名称',

291
frontend/src/views/system/SysParam/ClusterModeSetting.vue Normal file
View File

@ -0,0 +1,291 @@
<template>
<div>
<el-form ref="form" v-loading="loading"
:model="form"
:rules="rules"
class="demo-form-inline"
:disabled="show"
label-width="180px"
label-position="top"
size="small"
>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.doris_host')" prop="configuration.host">
<el-input v-model="form.configuration.host"/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.data_base')" prop="configuration.dataBase">
<el-input v-model="form.configuration.dataBase"/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.user_name')">
<el-input v-model="form.configuration.username"/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.password')">
<el-input v-model="form.configuration.password" show-password/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.query_port')" prop="configuration.port">
<el-input v-model="form.configuration.port" autocomplete="off" type="number" min="0"/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.http_port')" prop="configuration.port">
<el-input v-model="form.configuration.httpPort" autocomplete="off" type="number" min="0"/>
</el-form-item>
</el-col>
</el-row>
<el-collapse>
<el-collapse-item :title="$t('datasource.priority')" name="1">
<el-form-item :label="$t('datasource.replication_num')" prop="configuration.replicationNum">
<el-input v-model="form.configuration.replicationNum" autocomplete="off" type="number" min="1"/>
</el-form-item>
<el-form-item :label="$t('datasource.bucket_num')" prop="configuration.bucketNum">
<el-input v-model="form.configuration.bucketNum" autocomplete="off" type="number" min="1"/>
</el-form-item>
<el-form-item :label="$t('datasource.initial_pool_size')" prop="configuration.initialPoolSize">
<el-input v-model="form.configuration.initialPoolSize" autocomplete="off" type="number" min="0" size="small"/>
</el-form-item>
<el-form-item :label="$t('datasource.min_pool_size')" prop="configuration.minPoolSize">
<el-input v-model="form.configuration.minPoolSize" autocomplete="off" type="number" min="0"/>
</el-form-item>
<el-form-item :label="$t('datasource.max_pool_size')" prop="configuration.maxPoolSize">
<el-input v-model="form.configuration.maxPoolSize" autocomplete="off" type="number" min="0"/>
</el-form-item>
</el-collapse-item>
</el-collapse>
</el-form>
<div>
<el-button type="primary" size="small" @click="validaDatasource">
{{ $t('commons.validate') }}
</el-button>
<el-button v-if="showEdit" size="small" @click="edit">
{{ $t('commons.edit') }}
</el-button>
<el-button v-if="showSave" type="success" size="small" @click="save">
{{ $t('commons.save') }}
</el-button>
<el-button v-if="showCancel" type="info" size="small" @click="cancel">
{{ $t('commons.cancel') }}
</el-button>
</div>
</div>
</template>
<script>
import {engineInfo, validate, save} from '@/api/system/engine'
import i18n from "@/lang";
export default {
name: 'ClusterMode',
data() {
return {
form:
{
type: 'engine_doris',
configuration: {
host: '',
dataBase: '',
username: '',
password: '',
port: '',
httpPort: 8030,
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true',
replicationNum: 1,
bucketNum: 10,
minPoolSize: 5,
maxPoolSize: 50,
initialPoolSize: 5
}
},
originConfiguration: {
host: '',
dataBase: '',
username: '',
password: '',
port: '',
httpPort: 8030,
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true',
replicationNum: 1,
bucketNum: 10,
minPoolSize: 5,
maxPoolSize: 50,
initialPoolSize: 5
},
input: '',
visible: true,
showEdit: true,
showSave: false,
showCancel: false,
show: true,
disabledConnection: false,
disabledSave: false,
loading: false,
rules: {
'configuration.host': [
{
required: true,
message: this.$t('datasource.please_input_host'),
trigger: ['change', 'blur']
}
],
'configuration.port': [
{
required: true,
message: this.$t('datasource.please_input_port'),
trigger: ['change', 'blur']
}
],
'configuration.dataBase': [
{
required: true,
message: this.$t('datasource.please_input_data_base'),
trigger: ['change', 'blur']
}
],
'configuration.replicationNum': [
{
required: true,
message: this.$t('datasource.please_input_replication_num'),
trigger: ['change', 'blur']
}
],
'configuration.bucketNum': [
{
required: true,
message: this.$t('datasource.please_input_bucket_num'),
trigger: ['change', 'blur']
}
]
},
allTypes: [
{
name: 'engine_mysql',
label: 'MySQL',
type: 'jdbc',
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
}
]
}
},
created() {
this.query()
},
methods: {
query() {
engineInfo().then(response => {
if (response.data.id) {
this.form = JSON.parse(JSON.stringify(response.data))
this.form.configuration = JSON.parse(this.form.configuration)
this.originConfiguration = JSON.parse(JSON.stringify(this.form.configuration))
}
this.$nextTick(() => {
this.$refs.form.clearValidate()
})
})
},
edit() {
this.showEdit = false
this.showSave = true
this.showCancel = true
this.show = false
},
save() {
if (this.form.configuration.dataSourceType === 'jdbc' && this.form.configuration.port <= 0) {
this.$message.error(i18n.t('datasource.port_no_less_then_0'))
return
}
if (this.form.configuration.initialPoolSize < 0 || this.form.configuration.minPoolSize < 0 || this.form.configuration.maxPoolSize < 0) {
this.$message.error(i18n.t('datasource.no_less_then_0'))
return
}
this.$refs.form.validate(valid => {
if (!valid) {
return false
}
const form = JSON.parse(JSON.stringify(this.form))
form.configuration = JSON.stringify(form.configuration)
save(form).then(res => {
this.showEdit = true
this.showCancel = false
this.showSave = false
this.show = true
this.originConfiguration = JSON.parse(JSON.stringify(this.form.configuration))
this.$success(i18n.t('commons.save_success'))
})
})
},
cancel() {
this.showEdit = true
this.showCancel = false
this.showSave = false
this.show = true
this.form.configuration = JSON.parse(JSON.stringify(this.originConfiguration))
},
changeType() {
for (let i = 0; i < this.allTypes.length; i++) {
if (this.allTypes[i].name === this.form.type) {
this.form.configuration.dataSourceType = this.allTypes[i].type
this.form.configuration.extraParams = this.allTypes[i].extraParams
}
}
},
validaDatasource() {
if (!this.form.configuration.schema && this.form.type === 'oracle') {
this.$message.error(i18n.t('datasource.please_choose_schema'))
return
}
if (this.form.configuration.dataSourceType === 'jdbc' && this.form.configuration.port <= 0) {
this.$message.error(i18n.t('datasource.port_no_less_then_0'))
return
}
this.$refs.form.validate(valid => {
if (valid) {
const data = JSON.parse(JSON.stringify(this.form))
data.configuration = JSON.stringify(data.configuration)
validate(data).then(res => {
if (res.success) {
this.$success(i18n.t('datasource.validate_success'))
} else {
if (res.message.length < 2500) {
this.$error(res.message)
} else {
this.$error(res.message.substring(0, 2500) + '......')
}
}
}).catch(res => {
this.$error(res.message)
})
} else {
return false
}
})
},
}
}
</script>
<style scoped>
</style>