center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge remote-tracking branch 'origin/main' into main

Browse Source
This commit is contained in:
wangjiahao 2021-06-21 17:17:51 +08:00
commit f155f3d799
8 changed files with 400 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -5,7 +5,7 @@
# DataEase - 人人可用的开源数据可视化分析工具
DataEase 是开源的数据可视化分析工具，帮助用户分析数据、改善业务。DataEase 支持丰富的数据源连接，能够通过拖拉拽方式快速制作图表，并可以方便的与他人分享。
DataEase 是开源的数据可视化分析工具，帮助用户快速分析数据并洞察业务趋势，从而实现业务的改进与优化。DataEase 支持丰富的数据源连接，能够通过拖拉拽方式快速制作图表，并可以方便的与他人分享。
- 图表展示: 支持 PC 端、移动端及大屏;
- 图表制作: 支持丰富的图表类型(基于 Apache ECharts 实现)、支持拖拉拽方式快速制作仪表板;

2
backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java
View File

@ -32,5 +32,5 @@ public abstract class DatasourceProvider {
abstract public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception;
abstract public void initDataSource(DatasourceRequest datasourceRequest) throws Exception;
abstract public void initDataSource(DatasourceRequest datasourceRequest, String type) throws Exception;
}

62
backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java
View File

@ -271,7 +271,7 @@ public class JdbcProvider extends DatasourceProvider {
private Connection getConnectionFromPool(DatasourceRequest datasourceRequest) throws Exception {
ComboPooledDataSource dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId());
if (dataSource == null) {
initDataSource(datasourceRequest);
initDataSource(datasourceRequest, "add");
}
dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId());
Connection co = dataSource.getConnection();
@ -279,30 +279,48 @@ public class JdbcProvider extends DatasourceProvider {
}
@Override
public void initDataSource(DatasourceRequest datasourceRequest) throws Exception {
ComboPooledDataSource dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId());
if (dataSource == null) {
dataSource = new ComboPooledDataSource();
setCredential(datasourceRequest, dataSource);
dataSource.setMaxIdleTime(30); // 最大空闲时间
dataSource.setAcquireIncrement(5);// 增长数
dataSource.setInitialPoolSize(initPoolSize);// 初始连接数
dataSource.setMinPoolSize(initPoolSize); // 最小连接数
dataSource.setMaxPoolSize(maxConnections); // 最大连接数
dataSource.setAcquireRetryAttempts(30);// 获取连接重试次数
dataSource.setIdleConnectionTestPeriod(60); // 每60s检查数据库空闲连接
dataSource.setMaxStatements(0); // c3p0全局的PreparedStatements缓存的大小
dataSource.setBreakAfterAcquireFailure(false); // 获取连接失败将会引起所有等待连接池来获取连接的线程抛出异常。但是数据源仍有效保留，并在下次调用getConnection()的时候继续尝试获取连接。如果设为true，那么在尝试获取连接失败后该数据源将申明已断开并永久关闭。Default: false
dataSource.setTestConnectionOnCheckout(false); // 在每个connection 提交是校验有效性
dataSource.setTestConnectionOnCheckin(true); // 取得连接的同时将校验连接的有效性
dataSource.setCheckoutTimeout(60000); // 从连接池获取连接的超时时间，如设为0则无限期等待。单位毫秒，默认为0
dataSource.setPreferredTestQuery("SELECT 1");
dataSource.setDebugUnreturnedConnectionStackTraces(true);
dataSource.setUnreturnedConnectionTimeout(3600);
jdbcConnection.put(datasourceRequest.getDatasource().getId(), dataSource);
public void initDataSource(DatasourceRequest datasourceRequest, String type) throws Exception {
switch (type){
case "add":
ComboPooledDataSource dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId());
if (dataSource == null) {
extracted(datasourceRequest);
}
break;
case "edit":
jdbcConnection.remove(datasourceRequest.getDatasource().getId());
extracted(datasourceRequest);
break;
case "delete":
jdbcConnection.remove(datasourceRequest.getDatasource().getId());
break;
default:
break;
}
}
private void extracted(DatasourceRequest datasourceRequest) throws PropertyVetoException {
ComboPooledDataSource dataSource;
dataSource = new ComboPooledDataSource();
setCredential(datasourceRequest, dataSource);
dataSource.setMaxIdleTime(30); // 最大空闲时间
dataSource.setAcquireIncrement(5);// 增长数
dataSource.setInitialPoolSize(initPoolSize);// 初始连接数
dataSource.setMinPoolSize(initPoolSize); // 最小连接数
dataSource.setMaxPoolSize(maxConnections); // 最大连接数
dataSource.setAcquireRetryAttempts(30);// 获取连接重试次数
dataSource.setIdleConnectionTestPeriod(60); // 每60s检查数据库空闲连接
dataSource.setMaxStatements(0); // c3p0全局的PreparedStatements缓存的大小
dataSource.setBreakAfterAcquireFailure(false); // 获取连接失败将会引起所有等待连接池来获取连接的线程抛出异常。但是数据源仍有效保留，并在下次调用getConnection()的时候继续尝试获取连接。如果设为true，那么在尝试获取连接失败后该数据源将申明已断开并永久关闭。Default: false
dataSource.setTestConnectionOnCheckout(false); // 在每个connection 提交是校验有效性
dataSource.setTestConnectionOnCheckin(true); // 取得连接的同时将校验连接的有效性
dataSource.setCheckoutTimeout(60000); // 从连接池获取连接的超时时间，如设为0则无限期等待。单位毫秒，默认为0
dataSource.setPreferredTestQuery("SELECT 1");
dataSource.setDebugUnreturnedConnectionStackTraces(true);
dataSource.setUnreturnedConnectionTimeout(3600);
jdbcConnection.put(datasourceRequest.getDatasource().getId(), dataSource);
}
private static Connection getConnection(DatasourceRequest datasourceRequest) throws Exception {
String username = null;
String password = null;

28
backend/src/main/java/io/dataease/datasource/service/DatasourceService.java
View File

@ -55,9 +55,24 @@ public class DatasourceService {
datasource.setCreateTime(currentTimeMillis);
datasource.setCreateBy(String.valueOf(AuthUtils.getUser().getUsername()));
datasourceMapper.insertSelective(datasource);
initConnectionPool(datasource, "add");
return datasource;
}
private void initConnectionPool(Datasource datasource, String type) {
commonThreadPool.addTask(() -> {
try {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceProvider.initDataSource(datasourceRequest, type);
LogUtil.info("Succsss to init datasource connection pool: " + datasource.getName());
} catch (Exception e) {
LogUtil.error("Failed to init datasource connection pool: " + datasource.getName(), e);
}
});
}
public List<DatasourceDTO> getDatasourceList(DatasourceUnionRequest request) throws Exception {
request.setSort("update_time desc");
return extDataSourceMapper.queryUnion(request);
@ -92,6 +107,7 @@ public class DatasourceService {
datasource.setCreateTime(null);
datasource.setUpdateTime(System.currentTimeMillis());
datasourceMapper.updateByPrimaryKeySelective(datasource);
initConnectionPool(datasource, "edit");
}
public void validate(Datasource datasource) throws Exception {
@ -148,17 +164,7 @@ public class DatasourceService {
List<Datasource> datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample());
datasources.forEach(datasource -> {
try {
commonThreadPool.addTask(() -> {
try {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceProvider.initDataSource(datasourceRequest);
LogUtil.info("Succsss to init datasource connection pool: " + datasource.getName());
} catch (Exception e) {
LogUtil.error("Failed to init datasource connection pool: " + datasource.getName(), e);
}
});
initConnectionPool(datasource, "add");
} catch (Exception e) {
e.printStackTrace();
}

36
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
View File

@ -1,13 +1,15 @@
package io.dataease.service.dataset;
import io.dataease.base.domain.*;
import io.dataease.base.domain.DatasetTable;
import io.dataease.base.domain.DatasetTableTask;
import io.dataease.base.domain.DatasetTableTaskExample;
import io.dataease.base.domain.DatasetTableTaskLog;
import io.dataease.base.mapper.DatasetTableTaskMapper;
import io.dataease.commons.constants.JobStatus;
import io.dataease.commons.constants.ScheduleType;
import io.dataease.controller.request.dataset.DataSetTaskRequest;
import io.dataease.i18n.Translator;
import io.dataease.service.ScheduleService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.quartz.CronExpression;
@ -44,25 +46,29 @@ public class DataSetTableTaskService {
dataSetTableService.saveIncrementalConfig(dataSetTaskRequest.getDatasetTableIncrementalConfig());
// check
if (StringUtils.isNotEmpty(datasetTableTask.getCron())) {
if (!CronExpression.isValidExpression(datasetTableTask.getCron())) {
throw new RuntimeException(Translator.get("i18n_cron_expression_error"));
if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(),"CRON")){
if (StringUtils.isNotEmpty(datasetTableTask.getCron())) {
if (!CronExpression.isValidExpression(datasetTableTask.getCron())) {
throw new RuntimeException(Translator.get("i18n_cron_expression_error"));
}
}
// check start time and end time
if (StringUtils.equalsIgnoreCase(datasetTableTask.getEnd(), "1")
&& ObjectUtils.isNotEmpty(datasetTableTask.getStartTime())
&& ObjectUtils.isNotEmpty(datasetTableTask.getEndTime())
&& datasetTableTask.getStartTime() != 0
&& datasetTableTask.getEndTime() != 0
&& datasetTableTask.getStartTime() > datasetTableTask.getEndTime()) {
throw new RuntimeException(Translator.get("i18n_cron_time_error"));
}
}
// check start time and end time
if (ObjectUtils.isNotEmpty(datasetTableTask.getStartTime())
&& ObjectUtils.isNotEmpty(datasetTableTask.getEndTime())
&& datasetTableTask.getStartTime() != 0
&& datasetTableTask.getEndTime() != 0
&& datasetTableTask.getStartTime() > datasetTableTask.getEndTime()) {
throw new RuntimeException(Translator.get("i18n_cron_time_error"));
}
if (StringUtils.isEmpty(datasetTableTask.getId())) {
datasetTableTask.setId(UUID.randomUUID().toString());
datasetTableTask.setCreateTime(System.currentTimeMillis());
// SIMPLE 类型，提前占位
if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
if(datasetTableTask.getType().equalsIgnoreCase("add_scope")){
if (datasetTableTask.getType().equalsIgnoreCase("add_scope")) {
DatasetTable datasetTable = dataSetTableService.get(datasetTableTask.getTableId());
if (datasetTable.getLastUpdateTime() == 0 || datasetTable.getLastUpdateTime() == null) {
throw new Exception(Translator.get("i18n_not_exec_add_sync"));
@ -70,7 +76,7 @@ public class DataSetTableTaskService {
}
if (extractDataService.updateSyncStatusIsNone(dataSetTableService.get(datasetTableTask.getTableId()))) {
throw new Exception(Translator.get("i18n_sync_job_exists"));
}else {
} else {
//write log
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableTask.getTableId());

139
backend/src/main/resources/db/migration/V8__demo.sql Normal file
View File

File diff suppressed because one or more lines are too long

167
frontend/src/views/chart/group/Group.vue
View File

@ -21,26 +21,26 @@
<!-- </el-button>-->
<!-- </el-row>-->
<!-- <el-row>-->
<!-- <el-form>-->
<!-- <el-form-item class="form-item">-->
<!-- <el-input-->
<!-- v-model="search"-->
<!-- size="mini"-->
<!-- :placeholder="$t('chart.search')"-->
<!-- prefix-icon="el-icon-search"-->
<!-- clearable-->
<!-- />-->
<!-- </el-form-item>-->
<!-- </el-form>-->
<!-- </el-row>-->
<el-row>
<el-form>
<el-form-item class="form-item">
<el-input
v-model="search"
size="mini"
:placeholder="$t('chart.search')"
prefix-icon="el-icon-search"
clearable
/>
</el-form-item>
</el-form>
</el-row>
<el-col class="custom-tree-container">
<div class="block">
<el-tree
ref="asyncTree"
:default-expanded-keys="expandedArray"
:data="data"
:data="tData"
node-key="id"
:expand-on-click-node="true"
:load="loadNode"
@ -328,6 +328,7 @@
<script>
import { post } from '@/api/chart/chart'
import { authModel } from '@/api/system/sysAuth'
import TableSelector from '../view/TableSelector'
import GroupMoveSelector from '../components/TreeSelector/GroupMoveSelector'
import ChartMoveSelector from '../components/TreeSelector/ChartMoveSelector'
@ -360,7 +361,7 @@ export default {
search: '',
editGroup: false,
editTable: false,
data: [],
tData: [],
chartData: [],
currGroup: {},
expandedArray: [],
@ -395,7 +396,9 @@ export default {
treeProps: {
label: 'name',
children: 'children',
isLeaf: 'isLeaf'
isLeaf: 'isLeaf',
id: 'id',
parentId: 'pid'
},
dsForm: {
name: '',
@ -411,7 +414,8 @@ export default {
tDs: {},
groupMoveConfirmDisabled: true,
dsMoveConfirmDisabled: true,
moveDialogTitle: ''
moveDialogTitle: '',
isTreeSearch: false
}
},
computed: {
@ -422,11 +426,20 @@ export default {
},
watch: {
search(val) {
if (val && val !== '') {
this.chartData = JSON.parse(JSON.stringify(this.tables.filter(ele => { return ele.name.includes(val) })))
} else {
this.chartData = JSON.parse(JSON.stringify(this.tables))
// if (val && val !== '') {
// this.chartData = JSON.parse(JSON.stringify(this.tables.filter(ele => { return ele.name.includes(val) })))
// } else {
// this.chartData = JSON.parse(JSON.stringify(this.tables))
// }
this.$emit('switchComponent', { name: '' })
this.tData = []
this.expandedArray = []
if (this.timer) {
clearTimeout(this.timer)
}
this.timer = setTimeout(() => {
this.getTreeData(val)
}, 500)
},
saveStatus() {
this.refreshNodeBy(this.saveStatus.sceneId)
@ -617,13 +630,13 @@ export default {
groupTree(group) {
post('/chart/group/tree', group).then(response => {
this.data = response.data
this.tData = response.data
})
},
treeNode(group) {
post('/chart/group/treeNode', group).then(res => {
this.data = res.data
this.tData = res.data
})
},
@ -778,31 +791,39 @@ export default {
},
loadNode(node, resolve) {
// if (!this.isTreeSearch) {
if (node.level > 0) {
this.tables = []
this.chartData = []
if (node.data.id) {
post('/chart/view/listAndGroup', {
sort: 'name asc,create_time desc',
sceneId: node.data.id
}).then(response => {
this.tables = response.data
this.chartData = JSON.parse(JSON.stringify(this.tables))
resolve(this.chartData)
})
if (!this.isTreeSearch) {
if (node.level > 0) {
this.tables = []
this.chartData = []
if (node.data.id) {
post('/chart/view/listAndGroup', {
sort: 'name asc,create_time desc',
sceneId: node.data.id
}).then(response => {
this.tables = response.data
this.chartData = JSON.parse(JSON.stringify(this.tables))
resolve(this.chartData)
})
}
}
} else {
resolve(node.data.children)
}
// }
},
refreshNodeBy(id) {
if (!id || id === '0') {
this.treeNode(this.groupForm)
if (this.isTreeSearch) {
this.tData = []
this.expandedArray = []
this.searchTree(this.search)
} else {
const node = this.$refs.asyncTree.getNode(id) // 通过节点id找到对应树节点对象
node.loaded = false
node.expand() // 主动调用展开节点方法，重新查询该节点下的所有子节点
if (!id || id === '0') {
this.treeNode(this.groupForm)
} else {
const node = this.$refs.asyncTree.getNode(id) // 通过节点id找到对应树节点对象
node.loaded = false
node.expand() // 主动调用展开节点方法，重新查询该节点下的所有子节点
}
}
},
@ -867,6 +888,68 @@ export default {
} else {
this.dsMoveConfirmDisabled = false
}
},
searchTree(val) {
const queryCondition = {
withExtend: 'parent',
modelType: 'chart',
name: val
}
authModel(queryCondition).then(res => {
// this.highlights(res.data)
this.tData = this.buildTree(res.data)
console.log(this.tData)
})
},
buildTree(arrs) {
const idMapping = arrs.reduce((acc, el, i) => {
acc[el[this.treeProps.id]] = i
return acc
}, {})
const roots = []
arrs.forEach(el => {
// 判断根节点 ###
el.type = el.modelInnerType
el.isLeaf = el.leaf
if (el[this.treeProps.parentId] === null || el[this.treeProps.parentId] === 0 || el[this.treeProps.parentId] === '0') {
roots.push(el)
return
}
// 用映射表找到父元素
const parentEl = arrs[idMapping[el[this.treeProps.parentId]]]
// 把当前元素添加到父元素的`children`数组中
parentEl.children = [...(parentEl.children || []), el]
// 设置展开节点 如果没有子节点则不进行展开
if (parentEl.children.length > 0) {
this.expandedArray.push(parentEl[this.treeProps.id])
}
})
return roots
},
// 高亮显示搜索内容
highlights(data) {
if (data && this.search && this.search.length > 0) {
const replaceReg = new RegExp(this.search, 'g')// 匹配关键字正则
const replaceString = '<span style="color: #0a7be0">' + this.search + '</span>' // 高亮替换v-html值
data.forEach(item => {
item.name = item.name.replace(replaceReg, replaceString) // 开始替换
item.label = item.label.replace(replaceReg, replaceString) // 开始替换
})
}
},
getTreeData(val) {
if (val) {
this.isTreeSearch = true
this.searchTree(val)
} else {
this.isTreeSearch = false
this.treeNode(this.groupForm)
}
}
}
}

81
frontend/src/views/dataset/group/Group.vue
View File

@ -21,19 +21,19 @@
<!-- </el-button>-->
<!-- </el-row>-->
<!-- <el-row>-->
<!-- <el-form>-->
<!-- <el-form-item class="form-item">-->
<!-- <el-input-->
<!-- v-model="search"-->
<!-- size="mini"-->
<!-- :placeholder="$t('dataset.search')"-->
<!-- prefix-icon="el-icon-search"-->
<!-- clearable-->
<!-- />-->
<!-- </el-form-item>-->
<!-- </el-form>-->
<!-- </el-row>-->
<el-row>
<el-form>
<el-form-item class="form-item">
<el-input
v-model="search"
size="mini"
:placeholder="$t('dataset.search')"
prefix-icon="el-icon-search"
clearable
/>
</el-form-item>
</el-form>
</el-row>
<el-col class="custom-tree-container">
<div class="block">
@ -435,17 +435,14 @@ export default {
// } else {
// this.tableData = JSON.parse(JSON.stringify(this.tables))
// }
this.$emit('switchComponent', { name: '' })
this.tData = []
this.expandedArray = []
if (this.timer) {
clearTimeout(this.timer)
}
this.timer = setTimeout(() => {
if (val) {
this.searchTree(val)
this.isTreeSearch = true
} else {
this.treeNode(this.groupForm)
this.isTreeSearch = false
}
this.getTreeData(val)
}, 500)
},
saveStatus() {
@ -830,16 +827,24 @@ export default {
})
}
}
} else {
resolve(node.data.children)
}
},
refreshNodeBy(id) {
if (!id || id === '0') {
this.treeNode(this.groupForm)
if (this.isTreeSearch) {
this.tData = []
this.expandedArray = []
this.searchTree(this.search)
} else {
const node = this.$refs.asyncTree.getNode(id) // 通过节点id找到对应树节点对象
node.loaded = false
node.expand() // 主动调用展开节点方法，重新查询该节点下的所有子节点
if (!id || id === '0') {
this.treeNode(this.groupForm)
} else {
const node = this.$refs.asyncTree.getNode(id) // 通过节点id找到对应树节点对象
node.loaded = false
node.expand() // 主动调用展开节点方法，重新查询该节点下的所有子节点
}
}
},
@ -850,7 +855,9 @@ export default {
name: val
}
authModel(queryCondition).then(res => {
// this.highlights(res.data)
this.tData = this.buildTree(res.data)
console.log(this.tData)
})
},
@ -862,6 +869,8 @@ export default {
const roots = []
arrs.forEach(el => {
// 判断根节点 ###
el.type = el.modelInnerType
el.isLeaf = el.leaf
if (el[this.treeProps.parentId] === null || el[this.treeProps.parentId] === 0 || el[this.treeProps.parentId] === '0') {
roots.push(el)
return
@ -877,6 +886,28 @@ export default {
}
})
return roots
},
// 高亮显示搜索内容
highlights(data) {
if (data && this.search && this.search.length > 0) {
const replaceReg = new RegExp(this.search, 'g')// 匹配关键字正则
const replaceString = '<span style="color: #0a7be0">' + this.search + '</span>' // 高亮替换v-html值
data.forEach(item => {
item.name = item.name.replace(replaceReg, replaceString) // 开始替换
item.label = item.label.replace(replaceReg, replaceString) // 开始替换
})
}
},
getTreeData(val) {
if (val) {
this.isTreeSearch = true
this.searchTree(val)
} else {
this.isTreeSearch = false
this.treeNode(this.groupForm)
}
}
}
}