Merge pull request #11711 from dataease/pr@dev-v2@fixDS

Pr@dev v2@fix ds

core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java

@@ -59,6 +59,14 @@ public class SqlparserUtils {
         Statement statement = CCJSqlParserUtil.parse(tmpSql);
         Select select = (Select) statement;
 
+        if(CollectionUtils.isNotEmpty(select.getWithItemsList())){
+            for (Iterator<WithItem> iter = select.getWithItemsList().iterator(); iter.hasNext(); ) {
+                WithItem withItem = iter.next();
+                ParenthesedSelect parenthesedSelect = (ParenthesedSelect) withItem.getSelect();
+                parenthesedSelect.setSelect((Select) CCJSqlParserUtil.parse(removeVariables(parenthesedSelect.getSelect().toString(), dsType)));
+            }
+        }
+
         if (select.getSelectBody() instanceof PlainSelect) {
             return handlePlainSelect((PlainSelect) select.getSelectBody(), select, dsType);
         } else {
@@ -70,7 +78,7 @@ public class SqlparserUtils {
                     result.append(" ").append(setOperationList.getOperations().get(i).toString()).append(" ");
                 }
             }
-            return result.toString();
+            return select.toString();
         }
     }
 
@@ -412,35 +420,6 @@ public class SqlparserUtils {
     }
 
 
-    public static String removeVariables(final String sql) {
-        String tmpSql = sql;
-        Pattern pattern = Pattern.compile(regex);
-        Matcher matcher = pattern.matcher(sql);
-        boolean hasVariables = false;
-        while (matcher.find()) {
-            hasVariables = true;
-            tmpSql = tmpSql.replace(matcher.group(), SubstitutedParams);
-        }
-        if (!hasVariables && !tmpSql.contains(SubstitutedParams)) {
-            return tmpSql;
-        }
-
-        SqlParser.Config config =
-                SqlParser.config()
-                        .withLex(Lex.JAVA)
-                        .withIdentifierMaxLength(256);
-        SqlParser sqlParser = SqlParser.create(tmpSql, config);
-        SqlNode sqlNode;
-        try {
-            sqlNode = sqlParser.parseStmt();
-        } catch (SqlParseException e) {
-            throw new RuntimeException("使用 Calcite 进行语法分析发生了异常", e);
-        }
-        // 递归遍历语法树
-        getDependencies(sqlNode, false);
-        return sqlNode.toString();
-    }
-
     private static void getDependencies(SqlNode sqlNode, Boolean fromOrJoin) {
         if (sqlNode == null) {
             return;

core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java

@@ -37,8 +37,9 @@ public class ExcelUtils {
     private static String path = "/opt/dataease2.0/data/excel/";
     private static ObjectMapper objectMapper = new ObjectMapper();
 
-    private static  TypeReference<List<TableField>> TableFieldListTypeReference = new TypeReference<List<TableField>>() {
+    private static TypeReference<List<TableField>> TableFieldListTypeReference = new TypeReference<List<TableField>>() {
     };
+
     public static List<DatasetTableDTO> getTables(DatasourceRequest datasourceRequest) throws DEException {
         List<DatasetTableDTO> tableDescs = new ArrayList<>();
         try {
@@ -48,7 +49,7 @@ public class ExcelUtils {
                 datasetTableDTO.setTableName(rootNode.get(i).get("deTableName").asText());
                 datasetTableDTO.setName(rootNode.get(i).get("deTableName").asText());
                 datasetTableDTO.setDatasourceId(datasourceRequest.getDatasource().getId());
-                datasetTableDTO.setLastUpdateTime(rootNode.get(i).get("lastUpdateTime") == null? datasourceRequest.getDatasource().getCreateTime(): rootNode.get(i).get("lastUpdateTime").asLong(0L));
+                datasetTableDTO.setLastUpdateTime(rootNode.get(i).get("lastUpdateTime") == null ? datasourceRequest.getDatasource().getCreateTime() : rootNode.get(i).get("lastUpdateTime").asLong(0L));
                 tableDescs.add(datasetTableDTO);
             }
         } catch (Exception e) {
@@ -58,12 +59,12 @@ public class ExcelUtils {
         return tableDescs;
     }
 
-    public static Map<String,String> getTableNamesMap(String configration) throws DEException {
+    public static Map<String, String> getTableNamesMap(String configration) throws DEException {
-        Map<String,String> result = new HashMap<>();
+        Map<String, String> result = new HashMap<>();
         try {
             JsonNode rootNode = objectMapper.readTree(configration);
             for (int i = 0; i < rootNode.size(); i++) {
-                result.put(rootNode.get(i).get("tableName").asText(),rootNode.get(i).get("deTableName").asText());
+                result.put(rootNode.get(i).get("tableName").asText(), rootNode.get(i).get("deTableName").asText());
             }
         } catch (Exception e) {
             DEException.throwException(e);
@@ -293,7 +294,7 @@ public class ExcelUtils {
                     cells.add(str);
                 }
                 if (!isEmpty(cells)) {
-                    if(cells.size() > size){
+                    if (cells.size() > size) {
                         cells = cells.subList(0, size);
                     }
                     data.add(cells.toArray(new String[]{}));
@@ -307,7 +308,14 @@ public class ExcelUtils {
     }
 
     private String cellType(String value) {
-        if(value.length()> 19){
+        if (StringUtils.isEmpty(value) || value.length() > 19) {
+            return "TEXT";
+        }
+        if (value.length() > 1 && value.startsWith("0")) {
+            return "TEXT";
+        }
+        String regex = "^\\d+(\\.\\d+)?$";
+        if (!value.matches(regex)) {
             return "TEXT";
         }
         try {
@@ -331,9 +339,9 @@ public class ExcelUtils {
             tableFiled.setFieldType(cellType(value));
         } else {
             String type = cellType(value);
-            if(tableFiled.getFieldType() == null){
+            if (tableFiled.getFieldType() == null) {
                 tableFiled.setFieldType(type);
-            }else {
+            } else {
                 if (type.equalsIgnoreCase("TEXT")) {
                     tableFiled.setFieldType(type);
                 }
@@ -454,7 +462,7 @@ public class ExcelUtils {
             String[] split = s.split(",");
             for (int i = 0; i < split.length; i++) {
                 String filedName = split[i];
-                if(StringUtils.isEmpty(filedName)){
+                if (StringUtils.isEmpty(filedName)) {
                     DEException.throwException("首行行中不允许有空单元格！");
                 }
                 if (filedName.startsWith(UFEFF)) {

core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java

@@ -142,6 +142,10 @@ public class ExportCenterManage {
         }
     }
 
+    public String exportLimit() {
+        return String.valueOf(limit);
+    }
+
     public void download(String id, HttpServletResponse response) throws Exception {
         CoreExportTask exportTask = exportTaskMapper.selectById(id);
         OutputStream outputStream = response.getOutputStream();
@@ -267,6 +271,9 @@ public class ExportCenterManage {
         if (exportTaskDTO.getExportFromType().equalsIgnoreCase("chart")) {
             exportTaskDTO.setExportFromName(coreChartViewMapper.selectById(exportTaskDTO.getExportFrom()).getTitle());
         }
+        if (exportTaskDTO.getExportFromType().equalsIgnoreCase("dataset")) {
+            exportTaskDTO.setExportFromName(coreDatasetGroupMapper.selectById(exportTaskDTO.getExportFrom()).getName());
+        }
     }
 
     private String hostName() {

core/core-backend/src/main/java/io/dataease/exportCenter/server/ExportCenterServer.java

@@ -47,4 +47,8 @@ public class ExportCenterServer implements ExportCenterApi {
     public void retry(String id) {
         exportCenterManage.retry(id);
     }
+
+    public String exportLimit() {
+        return exportCenterManage.exportLimit();
+    }
 }

core/core-frontend/src/api/dataset.ts

@@ -130,6 +130,12 @@ export const exportDatasetData = async (data): Promise<IResponse> => {
   })
 }
 
+export const exportLimit = async (): Promise<boolean> => {
+  return request.post({ url: `/exportCenter/exportLimit`, data: {} }).then(res => {
+    return res?.data
+  })
+}
+
 export const perDelete = async (id): Promise<boolean> => {
   return request.post({ url: `/datasetTree/perDelete/${id}`, data: {} }).then(res => {
     return res?.data

core/core-frontend/src/views/visualized/data/dataset/index.vue

@@ -37,7 +37,8 @@ import {
   getDatasetPreview,
   barInfoApi,
   perDelete,
-  exportDatasetData
+  exportDatasetData,
+  exportLimit
 } from '@/api/dataset'
 import EmptyBackground from '@/components/empty-background/src/EmptyBackground.vue'
 import DeResourceGroupOpt from '@/views/common/DeResourceGroupOpt.vue'
@@ -85,6 +86,7 @@ const rootManage = ref(false)
 const showExport = ref(false)
 const rowAuth = ref()
 const exportDatasetLoading = ref(false)
+const limit = ref('10万')
 const exportForm = ref({})
 const table = ref({})
 const exportFormRef = ref()
@@ -313,6 +315,7 @@ onBeforeMount(() => {
   nodeInfo.id = (route.params.id as string) || (route.query.id as string) || ''
   loadInit()
   getData()
+  getLimit()
 })
 
 const columns = shallowRef([])
@@ -678,6 +681,12 @@ const loadInit = () => {
   }
 }
 
+const getLimit = () => {
+  exportLimit().then(res => {
+    limit.value = res
+  })
+}
+
 const sortTypeTip = computed(() => {
   return sortList.find(ele => ele.value === state.curSortType).name
 })
@@ -916,18 +925,18 @@ const getMenuList = (val: boolean) => {
                 <template #icon> <Icon name="icon_operation-analysis_outlined"></Icon> </template
                 >{{ t('data_set.new_data_screen') }}
               </el-button>
+              <el-button secondary @click="exportDataset">
+                <template #icon>
+                  <Icon name="icon_download_outlined"></Icon>
+                </template>
+                数据集导出
+              </el-button>
               <el-button type="primary" @click="editorDataset" v-if="nodeInfo.weight >= 7">
                 <template #icon>
                   <Icon name="icon_edit_outlined"></Icon>
                 </template>
                 {{ t('visualization.edit') }}
               </el-button>
-              <el-button type="primary" @click="exportDataset">
-                <template #icon>
-                  <Icon name="dv-preview-download"></Icon>
-                </template>
-                数据集导出
-              </el-button>
             </div>
           </div>
           <div class="tab-border">
@@ -1064,7 +1073,7 @@ const getMenuList = (val: boolean) => {
         </div>
       </el-form-item>
     </el-form>
-    <span class="tip">提示：最多支持导出10万条数据</span>
+    <span class="tip">提示：最多支持导出{{ limit }}条数据</span>
     <template v-slot:footer>
       <div class="dialog-footer">
         <el-button secondary @click="closeExport">{{ $t('dataset.cancel') }} </el-button>

core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue

@@ -648,8 +648,8 @@ defineExpose({
                   <template #prefix>
                     <el-icon>
                       <Icon
-                        :name="`field_${fieldType[scope.row.deType]}`"
+                        :name="`field_${fieldType[scope.row.deExtractType]}`"
-                        :className="`field-icon-${fieldType[scope.row.deType]}`"
+                        :className="`field-icon-${fieldType[scope.row.deExtractType]}`"
                       ></Icon>
                     </el-icon>
                   </template>
@@ -662,8 +662,8 @@ defineExpose({
                     <span style="float: left">
                       <el-icon>
                         <Icon
-                          :name="`field_${fieldType[scope.row.deType]}`"
+                          :name="`field_${fieldType[item.value]}`"
-                          :className="`field-icon-${fieldType[scope.row.deType]}`"
+                          :className="`field-icon-${fieldType[item.value]}`"
                         ></Icon>
                       </el-icon>
                     </span>

sdk/api/api-base/src/main/java/io/dataease/api/exportCenter/ExportCenterApi.java

@@ -38,4 +38,7 @@ public interface ExportCenterApi {
     @PostMapping("/retry/{id}")
     public void  retry(@PathVariable String id);
 
+    @PostMapping("/exportLimit")
+    public String  exportLimit();
+
 }