center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge remote-tracking branch 'origin/main' into main

Browse Source
This commit is contained in:
wangjiahao 2021-06-15 18:18:27 +08:00
commit 1e03bd347a
16 changed files with 153 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
View File

@ -180,6 +180,9 @@ public class ChartViewService {
}
data = datasourceProvider.getData(datasourceRequest);
}
if (StringUtils.containsIgnoreCase(view.getType(), "pie") && data.size() > 1000) {
data = data.subList(0, 1000);
}
// 图表组件可再扩展
List<String> x = new ArrayList<>();

63
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -81,6 +81,9 @@ public class DataSetTableService {
private QrtzSchedulerStateMapper qrtzSchedulerStateMapper;
@Resource
private DatasetTableTaskLogMapper datasetTableTaskLogMapper;
private static String lastUpdateTime = "${__last_update_time__}";
private static String currentUpdateTime = "${__current_update_time__}";
@Value("${upload.file.path}")
private String path;
@ -652,6 +655,7 @@ public class DataSetTableService {
} else {
return new DatasetTableIncrementalConfig();
}
}
public DatasetTableIncrementalConfig incrementalConfig(String datasetTableId) {
@ -661,7 +665,7 @@ public class DataSetTableService {
}
public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) {
public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception{
if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) {
return;
}
@ -671,8 +675,65 @@ public class DataSetTableService {
} else {
datasetTableIncrementalConfigMapper.updateByPrimaryKey(datasetTableIncrementalConfig);
}
checkColumes(datasetTableIncrementalConfig);
}
private void checkColumes(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception {
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(datasetTableIncrementalConfig.getTableId());
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(datasetTable.getId());
datasetTableFields.sort((o1, o2) -> {
if (o1.getOriginName() == null) {
return -1;
}
if (o2.getOriginName() == null) {
return 1;
}
return o1.getOriginName().compareTo(o2.getOriginName());
});
List<String> originNameFileds = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList());
Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
datasourceRequest.setQuery(sql);
List<String> sqlFileds = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{
sqlFileds.add(filed);
});
sort(sqlFileds);
if(!originNameFileds.equals(sqlFileds)){
throw new Exception(Translator.get("i18n_sql_add_not_matching") + sqlFileds.toString());
}
}
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
datasourceRequest.setQuery(sql);
List<String> sqlFileds = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{
sqlFileds.add(filed);
});
sort(sqlFileds);
if(!originNameFileds.equals(sqlFileds)){
throw new Exception(Translator.get("i18n_sql_delete_not_matching") + sqlFileds.toString());
}
}
}
private void sort(List<String> sqlFileds){
sqlFileds.sort((o1, o2) -> {
if (o1 == null) {
return -1;
}
if (o2 == null) {
return 1;
}
return o1.compareTo(o2);
});
}
private void checkName(DatasetTable datasetTable) {
// if (StringUtils.isEmpty(datasetTable.getId()) && StringUtils.equalsIgnoreCase("db", datasetTable.getType())) {
// return;

2
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
View File

@ -24,7 +24,7 @@ import java.util.UUID;
* @Date 2021/3/4 1:26 下午
*/
@Service
@Transactional
@Transactional(rollbackFor = Exception.class)
public class DataSetTableTaskService {
@Resource
private DatasetTableTaskMapper datasetTableTaskMapper;

21
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -245,7 +245,7 @@ public class ExtractDataService {
datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
}
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql);
@ -253,7 +253,7 @@ public class ExtractDataService {
extractData(datasetTable, "incremental_add");
}
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql);
@ -563,23 +563,26 @@ public class ExtractDataService {
}
private String fetchSqlField(String sql, Datasource ds) throws Exception {
String tmpSql = sql;
String tmpSql = "SELECT * FROM (" + sqlFix(sql) + ") AS tmp " + " LIMIT 0";
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
if (tmpSql.trim().endsWith(";")) {
tmpSql = tmpSql.substring(0, tmpSql.length() - 1) + " limit 0";
} else {
tmpSql = tmpSql + " limit 0";
}
datasourceRequest.setQuery(tmpSql);
List<String>dorisFileds = new ArrayList<>();
List<String> dorisFileds = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{
dorisFileds.add(DorisTableUtils.columnName(filed));
});
return String.join(",", dorisFileds);
}
private String sqlFix(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
return sql;
}
private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String selectSQL) throws Exception {
TransMeta transMeta = new TransMeta();

4
backend/src/main/resources/i18n/messages_en_US.properties
View File

@ -254,4 +254,6 @@ i18n_dataset_delete=Data set is delete
i18n_chart_delete=Chart is delete
i18n_not_exec_add_sync=There is no completed synchronization task. Incremental synchronization cannot be performed
i18n_excel_header_empty=Excel first row can not empty
i18n_custom_ds_delete=Custom dataset union data is deleted,can not display
i18n_custom_ds_delete=Custom dataset union data is deleted,can not display
i18n_sql_add_not_matching=The data column of incremental SQL does not match the dataset,
i18n_sql_delete_not_matching=The data column of incremental delete SQL does not match the dataset,

3
backend/src/main/resources/i18n/messages_zh_CN.properties
View File

@ -255,3 +255,6 @@ i18n_chart_delete=当前用到的视图已被删除
i18n_not_exec_add_sync=没有已完成的同步任务，无法进行增量同步
i18n_excel_header_empty=Excel第一行为空
i18n_custom_ds_delete=自定义数据集所关联数据被删除，无法正常显示
i18n_sql_add_not_matching=增量添加 sql 的数据列与数据集不匹配,
i18n_sql_delete_not_matching=增量删除 sql 的数据列与数据集不匹配,

4
backend/src/main/resources/i18n/messages_zh_TW.properties
View File

@ -256,4 +256,6 @@ i18n_dataset_delete=當前用到的數據集已被刪除
i18n_chart_delete=當前用到的視圖已被刪除
i18n_not_exec_add_sync=沒有已經完成的同步任務，無法進行增量同步
i18n_excel_header_empty=Excel第一行為空
i18n_custom_ds_delete=自定義數據集所關聯數據被刪除，無法正常顯示
i18n_custom_ds_delete=自定義數據集所關聯數據被刪除，無法正常顯示
i18n_sql_add_not_matching=增量添加 sql 的數據列與數據集不匹配,
i18n_sql_delete_not_matching=增量刪除 sql 的數據列與數據集不匹配,

12
frontend/src/layout/components/AppMain.vue
View File

@ -2,9 +2,7 @@
<section class="app-main">
<transition name="fade-transform" mode="out-in">
<el-main class="ms-main-container">
<!-- <keep-alive> -->
<router-view :key="key" />
<!-- </keep-alive> -->
</el-main>
</transition>
</section>
@ -26,11 +24,11 @@ export default {
.app-main {
/* topbar 56 */
min-height: calc(100vh - 56px);
width: 100%;
height: 100%;
position: relative;
overflow: hidden;
// min-height: calc(100vh - 56px);
// width: 100%;
// height: 100%;
// position: relative;
// overflow: hidden;
}
.fixed-header + .app-main {

25
frontend/src/layout/components/Sidebar/index.vue
View File

@ -1,20 +1,18 @@
<template>
<div :class="{'has-logo':showLogo}">
<logo v-if="showLogo" :collapse="isCollapse" />
<el-scrollbar wrap-class="scrollbar-wrapper">
<el-menu
:default-active="activeMenu"
:collapse="isCollapse"
:background-color="variables.menuBg"
<el-menu
:default-active="activeMenu"
:collapse="isCollapse"
:background-color="variables.menuBg"
:unique-opened="false"
:active-text-color="variables.menuActiveText"
:collapse-transition="false"
mode="vertical"
>
<sidebar-item v-for="route in routes" :key="route.path" :item="route" :base-path="route.path" />
</el-menu>
</el-scrollbar>
:unique-opened="false"
:active-text-color="variables.menuActiveText"
:collapse-transition="false"
mode="vertical"
>
<sidebar-item v-for="route in routes" :key="route.path" :item="route" :base-path="route.path" />
</el-menu>
</div>
</template>
@ -23,7 +21,6 @@ import { mapGetters } from 'vuex'
import Logo from './Logo'
import SidebarItem from './SidebarItem'
import variables from '@/styles/variables.scss'
export default {
components: { SidebarItem, Logo },
computed: {

50
frontend/src/layout/index.vue
View File

@ -2,35 +2,43 @@
<div :class="classObj" class="app-wrapper">
<licbar />
<topbar />
<div v-if="device==='mobile'&&sidebar.opened" class="drawer-bg" @click="handleClickOutside" />
<sidebar v-if="!sidebar.hide" class="sidebar-container" />
<div :class="{sidebarHide: sidebar.hide}" class="main-container">
<div :class="{'fixed-header':fixedHeader}">
<!-- <navbar /> -->
</div>
<de-container style="padding-top: 56px;">
<de-aside-container v-if="!sidebar.hide">
<sidebar class="sidebar-container" />
</de-aside-container>
<de-main-container class="la-main-container">
<app-main />
</de-main-container>
</de-container>
<!-- <de-main-container>
<app-main />
<right-panel v-if="showSettings">
<settings />
</right-panel>
</div>
</de-main-container> -->
<!-- <div :class="{sidebarHide: sidebar.hide}" class="main-container">
<app-main />
</div> -->
</div>
</template>
<script>
import RightPanel from '@/components/RightPanel'
import { Sidebar, Settings, AppMain, Topbar, Licbar } from './components'
// import { Sidebar, Settings, AppMain, Topbar } from './components'
import { Sidebar, AppMain, Topbar, Licbar } from './components'
import ResizeMixin from './mixin/ResizeHandler'
import DeMainContainer from '@/components/dataease/DeMainContainer'
import DeContainer from '@/components/dataease/DeContainer'
import DeAsideContainer from '@/components/dataease/DeAsideContainer'
export default {
name: 'Layout',
components: {
RightPanel,
Sidebar,
Settings,
AppMain,
Topbar,
Licbar
Licbar,
DeMainContainer,
DeContainer,
DeAsideContainer
},
mixins: [ResizeMixin],
computed: {
@ -70,8 +78,8 @@ export default {
.app-wrapper {
@include clearfix;
position: relative;
// height: 100%;
height: $contentHeight;
height: 100%;
// height: $contentHeight;
width: 100%;
&.mobile.openSidebar{
position: fixed;
@ -104,4 +112,8 @@ export default {
.mobile .fixed-header {
width: 100%;
}
.la-main-container {
padding: 0px !important;
}
</style>

8
frontend/src/styles/sidebar.scss
View File

@ -13,9 +13,9 @@
}
.sidebar-container {
transition: width 0.28s;
width: $sideBarWidth !important;
// width: $sideBarWidth !important;
background-color: $menuBg;
// height: 100%;
// width: 260px;
height: $contentHeight;
position: fixed;
font-size: 0px;
@ -25,7 +25,9 @@
left: 0;
z-index: 999;
overflow: hidden;
border-right: 1px solid rgba(0, 0, 0, 0.12);
// min-width: 260px;
// max-width: 400px;
// border-right: 1px solid rgba(0, 0, 0, 0.12);
// reset element-ui css
.horizontal-collapse-transition {

2
frontend/src/views/chart/components/filter/DimensionFilterEditor.vue
View File

@ -23,7 +23,7 @@
</el-select>
</el-col>
<el-col :span="6">
<el-input v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
<el-input v-show="!f.term.includes('null')" v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
</el-col>
<el-col :span="6">
<el-button type="text" icon="el-icon-delete" circle style="float: right" @click="removeFilter(index)" />

2
frontend/src/views/chart/components/filter/QuotaFilterEditor.vue
View File

@ -23,7 +23,7 @@
</el-select>
</el-col>
<el-col :span="6">
<el-input v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
<el-input v-show="!f.term.includes('null')" v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
</el-col>
<el-col :span="6">
<el-button type="text" icon="el-icon-delete" circle style="float: right" @click="removeFilter(index)" />

2
frontend/src/views/chart/components/filter/ResultFilterEditor.vue
View File

@ -37,7 +37,7 @@
</el-select>
</el-col>
<el-col :span="6">
<el-input v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
<el-input v-show="!f.term.includes('null')" v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
</el-col>
<el-col :span="6">
<el-button type="text" icon="el-icon-delete" circle style="float: right" @click="removeFilter(index)" />

18
frontend/src/views/chart/view/ChartEdit.vue
View File

@ -334,7 +334,7 @@
</template>
<script>
import { post, ajaxGetData } from '@/api/chart/chart'
import { ajaxGetData, post } from '@/api/chart/chart'
import draggable from 'vuedraggable'
import DimensionItem from '../components/drag-item/DimensionItem'
import QuotaItem from '../components/drag-item/QuotaItem'
@ -342,18 +342,17 @@ import ResultFilterEditor from '../components/filter/ResultFilterEditor'
import ChartComponent from '../components/ChartComponent'
import bus from '@/utils/bus'
import DatasetChartDetail from '../../dataset/common/DatasetChartDetail'
// shape attr,component style
import {
DEFAULT_BACKGROUND_COLOR,
DEFAULT_COLOR_CASE,
DEFAULT_LABEL,
DEFAULT_LEGEND_STYLE,
DEFAULT_SIZE,
DEFAULT_TITLE_STYLE,
DEFAULT_LEGEND_STYLE,
DEFAULT_LABEL,
DEFAULT_TOOLTIP,
DEFAULT_XAXIS_STYLE,
DEFAULT_YAXIS_STYLE,
DEFAULT_BACKGROUND_COLOR
DEFAULT_YAXIS_STYLE
} from '../chart/chart'
import ColorSelector from '../components/shape-attr/ColorSelector'
import SizeSelector from '../components/shape-attr/SizeSelector'
@ -646,6 +645,9 @@ export default {
this.resetView()
this.httpRequest.status = err.response.data.success
this.httpRequest.msg = err.response.data.message
this.$nextTick(() => {
this.getChart(id)
})
return true
})
} else {
@ -666,9 +668,9 @@ export default {
response.data.data = this.data
this.chart = response.data
this.httpRequest.status = true
// this.httpRequest.status = true
}).catch(err => {
this.resetView()
// this.resetView()
this.httpRequest.status = err.response.data.success
this.httpRequest.msg = err.response.data.message
return true

2
frontend/src/views/dataset/data/UpdateInfo.vue
View File

@ -472,7 +472,6 @@ export default {
})
},
saveIncrementalConfig() {
this.update_setting = false
if (this.incrementalUpdateType === 'incrementalAdd') {
this.incrementalConfig.incrementalAdd = this.sql
} else {
@ -485,6 +484,7 @@ export default {
type: 'success',
showClose: true
})
this.update_setting = false
})
},
saveTask(task) {