Merge remote-tracking branch 'origin/main' into main

backend/src/main/java/io/dataease/service/chart/ChartViewService.java

@@ -180,6 +180,9 @@ public class ChartViewService {
             }
             data = datasourceProvider.getData(datasourceRequest);
         }
+        if (StringUtils.containsIgnoreCase(view.getType(), "pie") && data.size() > 1000) {
+            data = data.subList(0, 1000);
+        }
 
         // 图表组件可再扩展
         List<String> x = new ArrayList<>();

backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java

@@ -81,6 +81,9 @@ public class DataSetTableService {
     private QrtzSchedulerStateMapper qrtzSchedulerStateMapper;
     @Resource
     private DatasetTableTaskLogMapper datasetTableTaskLogMapper;
+    private static String lastUpdateTime = "${__last_update_time__}";
+    private static String currentUpdateTime = "${__current_update_time__}";
+
     @Value("${upload.file.path}")
     private String path;
 
@@ -652,6 +655,7 @@ public class DataSetTableService {
         } else {
             return new DatasetTableIncrementalConfig();
         }
+
     }
 
     public DatasetTableIncrementalConfig incrementalConfig(String datasetTableId) {
@@ -661,7 +665,7 @@ public class DataSetTableService {
     }
 
 
-    public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) {
+    public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception{
         if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) {
             return;
         }
@@ -671,8 +675,65 @@ public class DataSetTableService {
         } else {
             datasetTableIncrementalConfigMapper.updateByPrimaryKey(datasetTableIncrementalConfig);
         }
+        checkColumes(datasetTableIncrementalConfig);
     }
 
+    private void checkColumes(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception {
+        DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(datasetTableIncrementalConfig.getTableId());
+        List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(datasetTable.getId());
+        datasetTableFields.sort((o1, o2) -> {
+            if (o1.getOriginName() == null) {
+                return -1;
+            }
+            if (o2.getOriginName() == null) {
+                return 1;
+            }
+            return o1.getOriginName().compareTo(o2.getOriginName());
+        });
+        List<String> originNameFileds = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList());
+        Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
+        DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
+        DatasourceRequest datasourceRequest = new DatasourceRequest();
+        datasourceRequest.setDatasource(ds);
+        if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
+            String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())
+                    .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
+            datasourceRequest.setQuery(sql);
+            List<String> sqlFileds = new ArrayList<>();
+            datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{
+                sqlFileds.add(filed);
+            });
+            sort(sqlFileds);
+            if(!originNameFileds.equals(sqlFileds)){
+                throw new Exception(Translator.get("i18n_sql_add_not_matching") + sqlFileds.toString());
+            }
+        }
+        if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
+            String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())
+                    .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
+            datasourceRequest.setQuery(sql);
+            List<String> sqlFileds = new ArrayList<>();
+            datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{
+                sqlFileds.add(filed);
+            });
+            sort(sqlFileds);
+            if(!originNameFileds.equals(sqlFileds)){
+                throw new Exception(Translator.get("i18n_sql_delete_not_matching") + sqlFileds.toString());
+            }
+        }
+    }
+
+    private void sort(List<String> sqlFileds){
+        sqlFileds.sort((o1, o2) -> {
+            if (o1 == null) {
+                return -1;
+            }
+            if (o2 == null) {
+                return 1;
+            }
+            return o1.compareTo(o2);
+        });
+    }
     private void checkName(DatasetTable datasetTable) {
 //        if (StringUtils.isEmpty(datasetTable.getId()) && StringUtils.equalsIgnoreCase("db", datasetTable.getType())) {
 //            return;

backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java

@@ -24,7 +24,7 @@ import java.util.UUID;
  * @Date 2021/3/4 1:26 下午
  */
 @Service
-@Transactional
+@Transactional(rollbackFor = Exception.class)
 public class DataSetTableTaskService {
     @Resource
     private DatasetTableTaskMapper datasetTableTaskMapper;

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -245,7 +245,7 @@ public class ExtractDataService {
                             datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
                         }
 
-                        if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
+                        if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
                             String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString())
                                     .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
                             generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql);
@@ -253,7 +253,7 @@ public class ExtractDataService {
                             extractData(datasetTable, "incremental_add");
                         }
 
-                        if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
+                        if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
                             String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString())
                                     .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
                             generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql);
@@ -563,23 +563,26 @@ public class ExtractDataService {
     }
 
     private String fetchSqlField(String sql, Datasource ds) throws Exception {
-        String tmpSql = sql;
+        String tmpSql = "SELECT * FROM (" + sqlFix(sql) + ") AS tmp " + " LIMIT 0";
         DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
         DatasourceRequest datasourceRequest = new DatasourceRequest();
         datasourceRequest.setDatasource(ds);
-        if (tmpSql.trim().endsWith(";")) {
-            tmpSql = tmpSql.substring(0, tmpSql.length() - 1) + " limit 0";
-        } else {
-            tmpSql = tmpSql + " limit 0";
-        }
         datasourceRequest.setQuery(tmpSql);
-        List<String>dorisFileds = new ArrayList<>();
+        List<String> dorisFileds = new ArrayList<>();
         datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{
             dorisFileds.add(DorisTableUtils.columnName(filed));
         });
         return String.join(",", dorisFileds);
     }
 
+    private String sqlFix(String sql) {
+        sql = sql.trim();
+        if (sql.lastIndexOf(";") == (sql.length() - 1)) {
+            sql = sql.substring(0, sql.length() - 1);
+        }
+        return sql;
+    }
+
 
     private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String selectSQL) throws Exception {
         TransMeta transMeta = new TransMeta();

backend/src/main/resources/i18n/messages_en_US.properties

@@ -254,4 +254,6 @@ i18n_dataset_delete=Data set is delete
 i18n_chart_delete=Chart is delete
 i18n_not_exec_add_sync=There is no completed synchronization task. Incremental synchronization cannot be performed
 i18n_excel_header_empty=Excel first row can not empty
-i18n_custom_ds_delete=Custom dataset union data is deleted,can not display
+i18n_custom_ds_delete=Custom dataset union data is deleted,can not display
+i18n_sql_add_not_matching=The data column of incremental SQL does not match the dataset,
+i18n_sql_delete_not_matching=The data column of incremental delete SQL does not match the dataset,

backend/src/main/resources/i18n/messages_zh_CN.properties

@@ -255,3 +255,6 @@ i18n_chart_delete=当前用到的视图已被删除
 i18n_not_exec_add_sync=没有已完成的同步任务，无法进行增量同步
 i18n_excel_header_empty=Excel第一行为空
 i18n_custom_ds_delete=自定义数据集所关联数据被删除，无法正常显示
+i18n_sql_add_not_matching=增量添加 sql 的数据列与数据集不匹配, 
+i18n_sql_delete_not_matching=增量删除 sql 的数据列与数据集不匹配, 
+

backend/src/main/resources/i18n/messages_zh_TW.properties

@@ -256,4 +256,6 @@ i18n_dataset_delete=當前用到的數據集已被刪除
 i18n_chart_delete=當前用到的視圖已被刪除
 i18n_not_exec_add_sync=沒有已經完成的同步任務，無法進行增量同步
 i18n_excel_header_empty=Excel第一行為空
-i18n_custom_ds_delete=自定義數據集所關聯數據被刪除，無法正常顯示
+i18n_custom_ds_delete=自定義數據集所關聯數據被刪除，無法正常顯示
+i18n_sql_add_not_matching=增量添加 sql 的數據列與數據集不匹配, 
+i18n_sql_delete_not_matching=增量刪除 sql 的數據列與數據集不匹配, 

frontend/src/layout/components/AppMain.vue

@@ -2,9 +2,7 @@
   <section class="app-main">
     <transition name="fade-transform" mode="out-in">
       <el-main class="ms-main-container">
-        <!-- <keep-alive> -->
         <router-view :key="key" />
-        <!-- </keep-alive> -->
       </el-main>
     </transition>
   </section>
@@ -26,11 +24,11 @@ export default {
 
     .app-main {
         /* topbar 56  */
-        min-height: calc(100vh - 56px);
+        // min-height: calc(100vh - 56px);
-        width: 100%;
+        // width: 100%;
-        height: 100%;
+        // height: 100%;
-        position: relative;
+         // position: relative;
-        overflow: hidden;
+        // overflow: hidden;
     }
 
     .fixed-header + .app-main {

frontend/src/layout/components/Sidebar/index.vue

@@ -1,20 +1,18 @@
 <template>
   <div :class="{'has-logo':showLogo}">
     <logo v-if="showLogo" :collapse="isCollapse" />
-    <el-scrollbar wrap-class="scrollbar-wrapper">
+    <el-menu
-      <el-menu
+      :default-active="activeMenu"
-        :default-active="activeMenu"
+      :collapse="isCollapse"
-        :collapse="isCollapse"
+      :background-color="variables.menuBg"
-        :background-color="variables.menuBg"
 
-        :unique-opened="false"
+      :unique-opened="false"
-        :active-text-color="variables.menuActiveText"
+      :active-text-color="variables.menuActiveText"
-        :collapse-transition="false"
+      :collapse-transition="false"
-        mode="vertical"
+      mode="vertical"
-      >
+    >
-        <sidebar-item v-for="route in routes" :key="route.path" :item="route" :base-path="route.path" />
+      <sidebar-item v-for="route in routes" :key="route.path" :item="route" :base-path="route.path" />
-      </el-menu>
+    </el-menu>
-    </el-scrollbar>
   </div>
 </template>
 
@@ -23,7 +21,6 @@ import { mapGetters } from 'vuex'
 import Logo from './Logo'
 import SidebarItem from './SidebarItem'
 import variables from '@/styles/variables.scss'
-
 export default {
   components: { SidebarItem, Logo },
   computed: {

frontend/src/layout/index.vue

@@ -2,35 +2,43 @@
   <div :class="classObj" class="app-wrapper">
     <licbar />
     <topbar />
-    <div v-if="device==='mobile'&&sidebar.opened" class="drawer-bg" @click="handleClickOutside" />
+
-    <sidebar v-if="!sidebar.hide" class="sidebar-container" />
+    <de-container style="padding-top: 56px;">
-    <div :class="{sidebarHide: sidebar.hide}" class="main-container">
+      <de-aside-container v-if="!sidebar.hide">
-      <div :class="{'fixed-header':fixedHeader}">
+        <sidebar class="sidebar-container" />
-        <!-- <navbar /> -->
+      </de-aside-container>
-      </div>
+
+      <de-main-container class="la-main-container">
+        <app-main />
+      </de-main-container>
+    </de-container>
+
+    <!-- <de-main-container>
       <app-main />
-      <right-panel v-if="showSettings">
+    </de-main-container> -->
-        <settings />
+
-      </right-panel>
+    <!-- <div :class="{sidebarHide: sidebar.hide}" class="main-container">
-    </div>
+      <app-main />
+    </div> -->
   </div>
 </template>
 
 <script>
-import RightPanel from '@/components/RightPanel'
+import { Sidebar, AppMain, Topbar, Licbar } from './components'
-import { Sidebar, Settings, AppMain, Topbar, Licbar } from './components'
-// import { Sidebar, Settings, AppMain, Topbar } from './components'
 import ResizeMixin from './mixin/ResizeHandler'
-
+import DeMainContainer from '@/components/dataease/DeMainContainer'
+import DeContainer from '@/components/dataease/DeContainer'
+import DeAsideContainer from '@/components/dataease/DeAsideContainer'
 export default {
   name: 'Layout',
   components: {
-    RightPanel,
     Sidebar,
-    Settings,
     AppMain,
     Topbar,
-    Licbar
+    Licbar,
+    DeMainContainer,
+    DeContainer,
+    DeAsideContainer
   },
   mixins: [ResizeMixin],
   computed: {
@@ -70,8 +78,8 @@ export default {
   .app-wrapper {
     @include clearfix;
     position: relative;
-    // height: 100%;
+    height: 100%;
-    height: $contentHeight;
+    // height: $contentHeight;
     width: 100%;
     &.mobile.openSidebar{
       position: fixed;
@@ -104,4 +112,8 @@ export default {
   .mobile .fixed-header {
     width: 100%;
   }
+
+  .la-main-container {
+      padding: 0px !important;
+  }
 </style>

frontend/src/styles/sidebar.scss

@@ -13,9 +13,9 @@
   }
   .sidebar-container {
     transition: width 0.28s;
-    width: $sideBarWidth !important;
+    // width: $sideBarWidth !important;
     background-color: $menuBg;
-    // height: 100%;
+     // width: 260px;
     height: $contentHeight;
     position: fixed;
     font-size: 0px;
@@ -25,7 +25,9 @@
     left: 0;
     z-index: 999;
     overflow: hidden;
-    border-right: 1px solid rgba(0, 0, 0, 0.12);
+    // min-width: 260px;
+    // max-width: 400px;
+    // border-right: 1px solid rgba(0, 0, 0, 0.12);
 
     // reset element-ui css
     .horizontal-collapse-transition {

frontend/src/views/chart/components/filter/DimensionFilterEditor.vue

@@ -23,7 +23,7 @@
           </el-select>
         </el-col>
         <el-col :span="6">
-          <el-input v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
+          <el-input v-show="!f.term.includes('null')" v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
         </el-col>
         <el-col :span="6">
           <el-button type="text" icon="el-icon-delete" circle style="float: right" @click="removeFilter(index)" />

frontend/src/views/chart/components/filter/QuotaFilterEditor.vue

@@ -23,7 +23,7 @@
           </el-select>
         </el-col>
         <el-col :span="6">
-          <el-input v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
+          <el-input v-show="!f.term.includes('null')" v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
         </el-col>
         <el-col :span="6">
           <el-button type="text" icon="el-icon-delete" circle style="float: right" @click="removeFilter(index)" />

frontend/src/views/chart/components/filter/ResultFilterEditor.vue

@@ -37,7 +37,7 @@
           </el-select>
         </el-col>
         <el-col :span="6">
-          <el-input v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
+          <el-input v-show="!f.term.includes('null')" v-model="f.value" class="value-item" :placeholder="$t('chart.no_limit')" size="mini" clearable />
         </el-col>
         <el-col :span="6">
           <el-button type="text" icon="el-icon-delete" circle style="float: right" @click="removeFilter(index)" />

frontend/src/views/chart/view/ChartEdit.vue

@@ -334,7 +334,7 @@
 </template>
 
 <script>
-import { post, ajaxGetData } from '@/api/chart/chart'
+import { ajaxGetData, post } from '@/api/chart/chart'
 import draggable from 'vuedraggable'
 import DimensionItem from '../components/drag-item/DimensionItem'
 import QuotaItem from '../components/drag-item/QuotaItem'
@@ -342,18 +342,17 @@ import ResultFilterEditor from '../components/filter/ResultFilterEditor'
 import ChartComponent from '../components/ChartComponent'
 import bus from '@/utils/bus'
 import DatasetChartDetail from '../../dataset/common/DatasetChartDetail'
-
 // shape attr,component style
 import {
+  DEFAULT_BACKGROUND_COLOR,
   DEFAULT_COLOR_CASE,
+  DEFAULT_LABEL,
+  DEFAULT_LEGEND_STYLE,
   DEFAULT_SIZE,
   DEFAULT_TITLE_STYLE,
-  DEFAULT_LEGEND_STYLE,
-  DEFAULT_LABEL,
   DEFAULT_TOOLTIP,
   DEFAULT_XAXIS_STYLE,
-  DEFAULT_YAXIS_STYLE,
+  DEFAULT_YAXIS_STYLE
-  DEFAULT_BACKGROUND_COLOR
 } from '../chart/chart'
 import ColorSelector from '../components/shape-attr/ColorSelector'
 import SizeSelector from '../components/shape-attr/SizeSelector'
@@ -646,6 +645,9 @@ export default {
           this.resetView()
           this.httpRequest.status = err.response.data.success
           this.httpRequest.msg = err.response.data.message
+          this.$nextTick(() => {
+            this.getChart(id)
+          })
           return true
         })
       } else {
@@ -666,9 +668,9 @@ export default {
           response.data.data = this.data
           this.chart = response.data
 
-          this.httpRequest.status = true
+          // this.httpRequest.status = true
         }).catch(err => {
-          this.resetView()
+          // this.resetView()
           this.httpRequest.status = err.response.data.success
           this.httpRequest.msg = err.response.data.message
           return true

frontend/src/views/dataset/data/UpdateInfo.vue

@@ -472,7 +472,6 @@ export default {
       })
     },
     saveIncrementalConfig() {
-      this.update_setting = false
       if (this.incrementalUpdateType === 'incrementalAdd') {
         this.incrementalConfig.incrementalAdd = this.sql
       } else {
@@ -485,6 +484,7 @@ export default {
           type: 'success',
           showClose: true
         })
+        this.update_setting = false
       })
     },
     saveTask(task) {