diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index 020157435f..56d2f90035 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -180,6 +180,9 @@ public class ChartViewService { } data = datasourceProvider.getData(datasourceRequest); } + if (StringUtils.containsIgnoreCase(view.getType(), "pie") && data.size() > 1000) { + data = data.subList(0, 1000); + } // 图表组件可再扩展 List x = new ArrayList<>(); diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index fc4bf662fa..56d212fcc9 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -81,6 +81,9 @@ public class DataSetTableService { private QrtzSchedulerStateMapper qrtzSchedulerStateMapper; @Resource private DatasetTableTaskLogMapper datasetTableTaskLogMapper; + private static String lastUpdateTime = "${__last_update_time__}"; + private static String currentUpdateTime = "${__current_update_time__}"; + @Value("${upload.file.path}") private String path; @@ -652,6 +655,7 @@ public class DataSetTableService { } else { return new DatasetTableIncrementalConfig(); } + } public DatasetTableIncrementalConfig incrementalConfig(String datasetTableId) { @@ -661,7 +665,7 @@ public class DataSetTableService { } - public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) { + public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception{ if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) { return; } @@ -671,8 +675,65 @@ public class DataSetTableService { } else { datasetTableIncrementalConfigMapper.updateByPrimaryKey(datasetTableIncrementalConfig); } + checkColumes(datasetTableIncrementalConfig); } + private void checkColumes(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception { + DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(datasetTableIncrementalConfig.getTableId()); + List datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(datasetTable.getId()); + datasetTableFields.sort((o1, o2) -> { + if (o1.getOriginName() == null) { + return -1; + } + if (o2.getOriginName() == null) { + return 1; + } + return o1.getOriginName().compareTo(o2.getOriginName()); + }); + List originNameFileds = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList()); + Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 + String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()) + .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); + datasourceRequest.setQuery(sql); + List sqlFileds = new ArrayList<>(); + datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{ + sqlFileds.add(filed); + }); + sort(sqlFileds); + if(!originNameFileds.equals(sqlFileds)){ + throw new Exception(Translator.get("i18n_sql_add_not_matching") + sqlFileds.toString()); + } + } + if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除 + String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()) + .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); + datasourceRequest.setQuery(sql); + List sqlFileds = new ArrayList<>(); + datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{ + sqlFileds.add(filed); + }); + sort(sqlFileds); + if(!originNameFileds.equals(sqlFileds)){ + throw new Exception(Translator.get("i18n_sql_delete_not_matching") + sqlFileds.toString()); + } + } + } + + private void sort(List sqlFileds){ + sqlFileds.sort((o1, o2) -> { + if (o1 == null) { + return -1; + } + if (o2 == null) { + return 1; + } + return o1.compareTo(o2); + }); + } private void checkName(DatasetTable datasetTable) { // if (StringUtils.isEmpty(datasetTable.getId()) && StringUtils.equalsIgnoreCase("db", datasetTable.getType())) { // return; diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java index e696aaaeb4..a67bc741a4 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java @@ -24,7 +24,7 @@ import java.util.UUID; * @Date 2021/3/4 1:26 下午 */ @Service -@Transactional +@Transactional(rollbackFor = Exception.class) public class DataSetTableTaskService { @Resource private DatasetTableTaskMapper datasetTableTaskMapper; diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 5e37ea24a9..351a7a4870 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -245,7 +245,7 @@ public class ExtractDataService { datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); } - if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 + if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString()) .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql); @@ -253,7 +253,7 @@ public class ExtractDataService { extractData(datasetTable, "incremental_add"); } - if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除 + if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除 String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString()) .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql); @@ -563,23 +563,26 @@ public class ExtractDataService { } private String fetchSqlField(String sql, Datasource ds) throws Exception { - String tmpSql = sql; + String tmpSql = "SELECT * FROM (" + sqlFix(sql) + ") AS tmp " + " LIMIT 0"; DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - if (tmpSql.trim().endsWith(";")) { - tmpSql = tmpSql.substring(0, tmpSql.length() - 1) + " limit 0"; - } else { - tmpSql = tmpSql + " limit 0"; - } datasourceRequest.setQuery(tmpSql); - ListdorisFileds = new ArrayList<>(); + List dorisFileds = new ArrayList<>(); datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{ dorisFileds.add(DorisTableUtils.columnName(filed)); }); return String.join(",", dorisFileds); } + private String sqlFix(String sql) { + sql = sql.trim(); + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } + private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String selectSQL) throws Exception { TransMeta transMeta = new TransMeta(); diff --git a/backend/src/main/resources/i18n/messages_en_US.properties b/backend/src/main/resources/i18n/messages_en_US.properties index 30e941a221..bcd8d9a172 100644 --- a/backend/src/main/resources/i18n/messages_en_US.properties +++ b/backend/src/main/resources/i18n/messages_en_US.properties @@ -254,4 +254,6 @@ i18n_dataset_delete=Data set is delete i18n_chart_delete=Chart is delete i18n_not_exec_add_sync=There is no completed synchronization task. Incremental synchronization cannot be performed i18n_excel_header_empty=Excel first row can not empty -i18n_custom_ds_delete=Custom dataset union data is deleted,can not display \ No newline at end of file +i18n_custom_ds_delete=Custom dataset union data is deleted,can not display +i18n_sql_add_not_matching=The data column of incremental SQL does not match the dataset, +i18n_sql_delete_not_matching=The data column of incremental delete SQL does not match the dataset, \ No newline at end of file diff --git a/backend/src/main/resources/i18n/messages_zh_CN.properties b/backend/src/main/resources/i18n/messages_zh_CN.properties index 4b0fd4d364..0d7141b769 100644 --- a/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -255,3 +255,6 @@ i18n_chart_delete=当前用到的视图已被删除 i18n_not_exec_add_sync=没有已完成的同步任务,无法进行增量同步 i18n_excel_header_empty=Excel第一行为空 i18n_custom_ds_delete=自定义数据集所关联数据被删除,无法正常显示 +i18n_sql_add_not_matching=增量添加 sql 的数据列与数据集不匹配, +i18n_sql_delete_not_matching=增量删除 sql 的数据列与数据集不匹配, + diff --git a/backend/src/main/resources/i18n/messages_zh_TW.properties b/backend/src/main/resources/i18n/messages_zh_TW.properties index d8f81a16a3..42a79e08ed 100644 --- a/backend/src/main/resources/i18n/messages_zh_TW.properties +++ b/backend/src/main/resources/i18n/messages_zh_TW.properties @@ -256,4 +256,6 @@ i18n_dataset_delete=當前用到的數據集已被刪除 i18n_chart_delete=當前用到的視圖已被刪除 i18n_not_exec_add_sync=沒有已經完成的同步任務,無法進行增量同步 i18n_excel_header_empty=Excel第一行為空 -i18n_custom_ds_delete=自定義數據集所關聯數據被刪除,無法正常顯示 \ No newline at end of file +i18n_custom_ds_delete=自定義數據集所關聯數據被刪除,無法正常顯示 +i18n_sql_add_not_matching=增量添加 sql 的數據列與數據集不匹配, +i18n_sql_delete_not_matching=增量刪除 sql 的數據列與數據集不匹配, \ No newline at end of file diff --git a/frontend/src/layout/components/AppMain.vue b/frontend/src/layout/components/AppMain.vue index 0111801d9d..68235f7d8f 100644 --- a/frontend/src/layout/components/AppMain.vue +++ b/frontend/src/layout/components/AppMain.vue @@ -2,9 +2,7 @@
- -
@@ -26,11 +24,11 @@ export default { .app-main { /* topbar 56 */ - min-height: calc(100vh - 56px); - width: 100%; - height: 100%; - position: relative; - overflow: hidden; + // min-height: calc(100vh - 56px); + // width: 100%; + // height: 100%; + // position: relative; + // overflow: hidden; } .fixed-header + .app-main { diff --git a/frontend/src/layout/components/Sidebar/index.vue b/frontend/src/layout/components/Sidebar/index.vue index d937418561..e2cd53868a 100644 --- a/frontend/src/layout/components/Sidebar/index.vue +++ b/frontend/src/layout/components/Sidebar/index.vue @@ -1,20 +1,18 @@ @@ -23,7 +21,6 @@ import { mapGetters } from 'vuex' import Logo from './Logo' import SidebarItem from './SidebarItem' import variables from '@/styles/variables.scss' - export default { components: { SidebarItem, Logo }, computed: { diff --git a/frontend/src/layout/index.vue b/frontend/src/layout/index.vue index ce02660731..87884310d7 100644 --- a/frontend/src/layout/index.vue +++ b/frontend/src/layout/index.vue @@ -2,35 +2,43 @@
-
- -
-
- -
+ + + + + + + + + + + + + +