center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat: 支持集群（doris）

Browse Source
This commit is contained in:
taojinlong 2022-03-22 17:46:54 +08:00
parent 301b5155c0
commit 1fa8fae237
11 changed files with 55 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/main/java/io/dataease/dto/datasource/DorisConfiguration.java
View File

@ -8,4 +8,7 @@ import lombok.Setter;
public class DorisConfiguration extends MysqlConfiguration {
private Integer httpPort;
private Integer replicationNum = 1;
private Integer bucketNum = 10;
}

3
backend/src/main/java/io/dataease/provider/DDLProvider.java
View File

@ -1,6 +1,7 @@
package io.dataease.provider;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import java.util.List;
@ -17,7 +18,7 @@ public abstract class DDLProvider {
public abstract String replaceTable(String name);
public abstract String createTableSql(String name, List<DatasetTableField> datasetTableFields);
public abstract String createTableSql(String name, List<DatasetTableField> datasetTableFields, Datasource engine);
public abstract String insertSql(String name, List<String[]> dataList, int page, int pageNumber);
}

3
backend/src/main/java/io/dataease/provider/DDLProviderImpl.java
View File

@ -1,6 +1,7 @@
package io.dataease.provider;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import io.dataease.commons.utils.Md5Utils;
import java.util.Arrays;
@ -28,7 +29,7 @@ public class DDLProviderImpl extends DDLProvider {
}
@Override
public String createTableSql(String name, List<DatasetTableField> datasetTableFields) {
public String createTableSql(String name, List<DatasetTableField> datasetTableFields, Datasource engine) {
return null;
}

16
backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java
View File

@ -1,7 +1,12 @@
package io.dataease.provider.engine.doris;
import com.google.gson.Gson;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import io.dataease.commons.utils.TableUtils;
import io.dataease.dto.datasource.DorisConfiguration;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.datasource.MysqlConfiguration;
import io.dataease.provider.DDLProviderImpl;
import org.springframework.stereotype.Service;
@ -16,8 +21,8 @@ public class DorisDDLProvider extends DDLProviderImpl {
private static final String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
"Column_Fields" +
"UNIQUE KEY(dataease_uuid)\n" +
"DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
"PROPERTIES(\"replication_num\" = \"1\");";
"DISTRIBUTED BY HASH(dataease_uuid) BUCKETS BUCKETS_NUM\n" +
"PROPERTIES(\"replication_num\" = \"ReplicationNum\");";
@Override
public String createView(String name, String viewSQL) {
@ -41,9 +46,12 @@ public class DorisDDLProvider extends DDLProviderImpl {
}
@Override
public String createTableSql(String tableName, List<DatasetTableField> datasetTableFields) {
public String createTableSql(String tableName, List<DatasetTableField> datasetTableFields, Datasource engine) {
DorisConfiguration dorisConfiguration = new Gson().fromJson(engine.getConfiguration(), DorisConfiguration.class);
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql);
return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql)
.replace("BUCKETS_NUM", dorisConfiguration.getBucketNum().toString())
.replace("ReplicationNum", dorisConfiguration.getReplicationNum().toString());
}
private String createDorisTableColumnSql(final List<DatasetTableField> datasetTableFields) {

3
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java
View File

@ -1,6 +1,7 @@
package io.dataease.provider.engine.mysql;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import io.dataease.commons.utils.TableUtils;
import io.dataease.provider.DDLProviderImpl;
import org.springframework.stereotype.Service;
@ -43,7 +44,7 @@ public class MysqlDDLProvider extends DDLProviderImpl {
}
@Override
public String createTableSql(String tableName, List<DatasetTableField> datasetTableFields) {
public String createTableSql(String tableName, List<DatasetTableField> datasetTableFields, Datasource engine) {
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql);
}

2
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -605,7 +605,7 @@ public class ExtractDataService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(engine);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
datasourceRequest.setQuery(ddlProvider.createTableSql(tableName, datasetTableFields));
datasourceRequest.setQuery(ddlProvider.createTableSql(tableName, datasetTableFields, engine));
jdbcProvider.exec(datasourceRequest);
}

5
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -59,6 +59,11 @@ public class DatasourceService {
@DeCleaner(DePermissionType.DATASOURCE)
public Datasource addDatasource(Datasource datasource) throws Exception{
try{
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasource.getType());
}catch (Exception e){
throw e;
}
checkName(datasource);
long currentTimeMillis = System.currentTimeMillis();
datasource.setId(UUID.randomUUID().toString());

37
backend/src/main/java/io/dataease/service/engine/EngineService.java
View File

@ -9,6 +9,7 @@ import io.dataease.commons.utils.BeanUtils;
import io.dataease.controller.ResultHolder;
import io.dataease.controller.request.datasource.DatasourceRequest;
import io.dataease.dto.DatasourceDTO;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.service.datasource.DatasourceService;
@ -31,8 +32,6 @@ public class EngineService {
private DeEngineMapper deEngineMapper;
@Resource
private DatasourceService datasource;
static private Datasource ds = null;
public Boolean isLocalMode(){
return env.getProperty("engine_mode", "local").equalsIgnoreCase("local");
@ -51,7 +50,13 @@ public class EngineService {
}
public DeEngine info(){
List<DeEngine> deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample());
DeEngineExample deEngineExample = new DeEngineExample();
if(isClusterMode()){
deEngineExample.createCriteria().andTypeEqualTo("engine_doris");
}else {
deEngineExample.createCriteria().andTypeEqualTo("engine_mysql");
}
List<DeEngine> deEngines = deEngineMapper.selectByExampleWithBLOBs(deEngineExample);
if(CollectionUtils.isEmpty(deEngines)){
return new DeEngine();
}
@ -69,7 +74,7 @@ public class EngineService {
datasourceProvider.checkStatus(datasourceRequest);
return ResultHolder.success(datasource);
}catch (Exception e){
return ResultHolder.error("Datasource is invalid: " + e.getMessage());
return ResultHolder.error("Engine is invalid: " + e.getMessage());
}
}
@ -79,26 +84,23 @@ public class EngineService {
deEngineMapper.insert(engine);
}else {
deEngineMapper.updateByPrimaryKeyWithBLOBs(engine);
datasource.handleConnectionPool(getDeEngine(), "delete");
}
datasource.handleConnectionPool(this.ds, "delete");
setDs(engine);
datasource.handleConnectionPool(this.ds, "add");
datasource.handleConnectionPool(getDeEngine(), "add");
return ResultHolder.success(engine);
}
private void setDs(DeEngine engine){
if(this.ds == null){
this.ds = new Datasource();
BeanUtils.copyBean(this.ds, engine);
}else {
BeanUtils.copyBean(this.ds, engine);
}
CacheUtils.put("ENGINE", "engine", engine, null, null);
}
public Datasource getDeEngine() throws Exception{
if (this.ds != null) {
return this.ds;
Object catcheEngine = CacheUtils.get("ENGINE", "engine");
if(catcheEngine != null){
return (Datasource) catcheEngine;
}
if(isLocalMode()){
JSONObject jsonObject = new JSONObject();
jsonObject.put("dataSourceType", "jdbc");
@ -123,12 +125,7 @@ public class EngineService {
}
setDs(deEngines.get(0));
}
// if(isSimpleMode()){
//
// }
//TODO cluster mode
return this.ds;
return getDeEngine();
}

5
frontend/src/lang/zh.js
View File

@ -1301,6 +1301,7 @@ export default {
user_name: '用户名',
password: '密码',
host: '主机名/IP地址',
doris_host: 'Doris 地址',
port: '端口',
datasource_url: '地址',
please_input_datasource_url: '请输入 Elasticsearch 地址，如: http://es_host:es_port',
@ -1332,6 +1333,10 @@ export default {
initial_pool_size: '初始连接数',
min_pool_size: '最小连接数',
max_pool_size: '最大连接数',
bucket_num: 'Bucket 数量',
replication_num: '副本数量',
please_input_bucket_num: '请输入 Bucket 数量',
please_input_replication_num: '请输入副本数量',
max_idle_time: '最大空闲(秒)',
acquire_increment: '增长数',
connect_timeout: '连接超时(秒)',

1
frontend/src/views/system/SysParam/SimpleModeSetting.vue
View File

@ -1,6 +1,5 @@
<template>
<div>
<!--邮件表单-->
<el-form ref="form" v-loading="loading"
:model="form"
:rules="rules"

7
frontend/src/views/system/SysParam/index.vue
View File

@ -30,6 +30,10 @@
<simple-mode />
</el-tab-pane>
<el-tab-pane v-if="engineMode==='cluster'" :lazy="true" :label="$t('system_parameter_setting.engine_mode_setting')" name="six">
<cluster-mode />
</el-tab-pane>
</el-tabs>
</layout-content>
</template>
@ -37,13 +41,14 @@
import BasicSetting from './BasicSetting'
import EmailSetting from './EmailSetting'
import SimpleMode from './SimpleModeSetting'
import ClusterMode from './ClusterModeSetting'
import LayoutContent from '@/components/business/LayoutContent'
import PluginCom from '@/views/system/plugin/PluginCom'
import { pluginLoaded } from '@/api/user'
import { engineMode } from '@/api/system/engine'
export default {
components: { BasicSetting, EmailSetting, LayoutContent, PluginCom, SimpleMode},
components: { BasicSetting, EmailSetting, LayoutContent, PluginCom, SimpleMode, ClusterMode},
data() {
return {
activeName: 'zero',