feat: 【API数据源】API数据源支持设置主键，增量同步可根据主键更新已同步的数据

core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java

@@ -394,7 +394,11 @@ public class ChartDataServer implements ChartDataApi {
                             detailsSheet.setColumnWidth(j, 255 * 20);
                         } else if (cellValObj != null) {
                             try {
-                                cell.setCellValue(cellValObj.toString());
+                                if ((excelTypes[j].equals(DeTypeConstants.DE_INT) || excelTypes[j].equals(DeTypeConstants.DE_FLOAT)) && StringUtils.isNotEmpty(cellValObj.toString())) {
+                                    cell.setCellValue(Double.valueOf(cellValObj.toString()));
+                                } else if (cellValObj != null) {
+                                    cell.setCellValue(cellValObj.toString());
+                                }
                             } catch (Exception e) {
                                 LogUtil.warn("export excel data transform error");
                             }

core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java

@@ -70,7 +70,7 @@ public class DatasourceSyncManage {
                 if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
                     createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields);
                 }
-                extractExcelData(datasourceRequest, updateType);
+                extractExcelData(datasourceRequest, updateType, tableFields);
                 if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
                     replaceTable(datasourceRequest.getTable());
                 }
@@ -145,7 +145,7 @@ public class DatasourceSyncManage {
                 if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
                     createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields);
                 }
-                extractApiData(datasourceRequest, updateType);
+                extractApiData(datasourceRequest, updateType, tableFields);
                 if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
                     replaceTable(datasourceRequest.getTable());
                 }
@@ -198,7 +198,7 @@ public class DatasourceSyncManage {
                     if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
                         createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields);
                     }
-                    extractApiData(datasourceRequest, updateType);
+                    extractApiData(datasourceRequest, updateType, tableFields);
                     if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
                         replaceTable(datasourceRequest.getTable());
                     }
@@ -222,20 +222,10 @@ public class DatasourceSyncManage {
         }
     }
 
-    private void extractApiData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType) throws Exception {
+    private void extractApiData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType, List<TableField> tableFields) throws Exception {
         Map<String, Object> result = ApiUtils.fetchResultField(datasourceRequest);
         List<String[]> dataList = (List<String[]>) result.get("dataList");
-        String engineTableName;
-        switch (extractType) {
-            case all_scope:
-                engineTableName = TableUtils.tmpName(TableUtils.tableName(datasourceRequest.getTable()));
-                break;
-            default:
-                engineTableName = TableUtils.tableName(datasourceRequest.getTable());
-                break;
-        }
         CoreDeEngine engine = engineManage.info();
-
         EngineRequest engineRequest = new EngineRequest();
         engineRequest.setEngine(engine);
         EngineProvider engineProvider = ProviderUtil.getEngineProvider(engine.getType());
@@ -246,14 +236,13 @@ public class DatasourceSyncManage {
         } else {
             totalPage = dataList.size() / pageNumber;
         }
-
         for (int page = 1; page <= totalPage; page++) {
-            engineRequest.setQuery(engineProvider.insertSql(engineTableName, dataList, page, pageNumber));
+            engineRequest.setQuery(engineProvider.insertSql(datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields));
             calciteProvider.exec(engineRequest);
         }
     }
 
-    private void extractExcelData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType) throws Exception {
+    private void extractExcelData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType, List<TableField> tableFields) throws Exception {
         ExcelUtils excelUtils = new ExcelUtils();
         List<String[]> dataList = excelUtils.fetchDataList(datasourceRequest);
         String engineTableName;
@@ -278,7 +267,7 @@ public class DatasourceSyncManage {
             totalPage = dataList.size() / pageNumber;
         }
         for (int page = 1; page <= totalPage; page++) {
-            engineRequest.setQuery(engineProvider.insertSql(engineTableName, dataList, page, pageNumber));
+            engineRequest.setQuery(engineProvider.insertSql(engineTableName, extractType, dataList, page, pageNumber, tableFields));
             calciteProvider.exec(engineRequest);
         }
     }

core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java

@@ -580,6 +580,7 @@ public class ApiUtils {
                     o.put("checked", true);
                     o.put("name", field.getName());
                     o.put("primaryKey", field.isPrimaryKey());
+                    o.put("length", field.getLength());
                     o.put("deExtractType", field.getDeExtractType());
                 }
             }

core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java

@@ -2,6 +2,7 @@ package io.dataease.datasource.provider;
 
 import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
 import io.dataease.datasource.request.EngineRequest;
+import io.dataease.datasource.server.DatasourceServer;
 import io.dataease.extensions.datasource.dto.TableField;
 
 import java.util.List;
@@ -21,7 +22,7 @@ public abstract class EngineProvider {
 
     public abstract String createTableSql(String name, List<TableField> tableFields, CoreDeEngine engine);
 
-    public abstract String insertSql(String name, List<String[]> dataList, int page, int pageNumber);
+    public abstract String insertSql(String tableName, DatasourceServer.UpdateType extractType, List<String[]> dataList, int page, int pageNumber, List<TableField> tableFields);
 
 
 }

core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java

@@ -3,20 +3,11 @@ package io.dataease.datasource.provider;
 
 import io.dataease.dataset.utils.TableUtils;
 import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
-import io.dataease.datasource.request.EngineRequest;
+import io.dataease.datasource.server.DatasourceServer;
-import io.dataease.datasource.type.H2;
-import io.dataease.datasource.type.Mysql;
-import io.dataease.extensions.datasource.dto.ConnectionObj;
-import io.dataease.extensions.datasource.dto.DatasourceDTO;
 import io.dataease.extensions.datasource.dto.TableField;
-import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
-import io.dataease.utils.BeanUtils;
-import io.dataease.utils.JsonUtil;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 
-import java.sql.PreparedStatement;
-import java.sql.Statement;
 import java.util.Arrays;
 import java.util.List;
 
@@ -34,8 +25,17 @@ public class H2EngineProvider extends EngineProvider {
     }
 
     @Override
-    public String insertSql(String name, List<String[]> dataList, int page, int pageNumber) {
+    public String insertSql(String tableName, DatasourceServer.UpdateType extractType, List<String[]> dataList, int page, int pageNumber,List<TableField> tableFields) {
-        String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", name);
+        String engineTableName;
+        switch (extractType) {
+            case all_scope:
+                engineTableName = TableUtils.tmpName(TableUtils.tableName(tableName));
+                break;
+            default:
+                engineTableName = TableUtils.tableName(tableName);
+                break;
+        }
+        String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", engineTableName);
         StringBuffer values = new StringBuffer();
 
         Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
@@ -88,7 +88,11 @@ public class H2EngineProvider extends EngineProvider {
             int size = tableField.getPrecision() * 4;
             switch (tableField.getDeType()) {
                 case 0:
-                    columnFields.append("varchar(2048)").append(",`");
+                    if (StringUtils.isNotEmpty(tableField.getLength())) {
+                        columnFields.append("varchar(length)".replace("length", tableField.getLength())).append(",`");
+                    } else {
+                        columnFields.append("longtext").append(",`");
+                    }
                     break;
                 case 1:
                     columnFields.append("varchar(2048)").append(",`");
@@ -103,7 +107,7 @@ public class H2EngineProvider extends EngineProvider {
                     columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
                     break;
                 default:
-                    columnFields.append("varchar(2048)").append(",`");
+                    columnFields.append("longtext").append(",`");
                     break;
             }
         }

core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java

@@ -3,21 +3,15 @@ package io.dataease.datasource.provider;
 
 import io.dataease.dataset.utils.TableUtils;
 import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
-import io.dataease.datasource.request.EngineRequest;
+import io.dataease.datasource.server.DatasourceServer;
-import io.dataease.datasource.type.Mysql;
-import io.dataease.extensions.datasource.dto.ConnectionObj;
-import io.dataease.extensions.datasource.dto.DatasourceDTO;
 import io.dataease.extensions.datasource.dto.TableField;
-import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
+import org.apache.commons.collections4.CollectionUtils;
-import io.dataease.utils.BeanUtils;
-import io.dataease.utils.JsonUtil;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
-
+import java.util.ArrayList;
-import java.sql.PreparedStatement;
-import java.sql.Statement;
 import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * @Author gin
@@ -37,8 +31,18 @@ public class MysqlEngineProvider extends EngineProvider {
     }
 
     @Override
-    public String insertSql(String name, List<String[]> dataList, int page, int pageNumber) {
+    public String insertSql(String tableName, DatasourceServer.UpdateType extractType, List<String[]> dataList, int page, int pageNumber, List<TableField> tableFields) {
-        String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", name);
+        String engineTableName;
+        switch (extractType) {
+            case all_scope:
+                engineTableName = TableUtils.tmpName(TableUtils.tableName(tableName));
+                break;
+            default:
+                engineTableName = TableUtils.tableName(tableName);
+                break;
+        }
+
+        String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", engineTableName);
         StringBuffer values = new StringBuffer();
 
         Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
@@ -54,7 +58,18 @@ public class MysqlEngineProvider extends EngineProvider {
             values.append("('").append(String.join("','", Arrays.asList(strings1)))
                     .append("'),");
         }
-        return (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null");
+        List<TableField> keys = tableFields.stream().filter(TableField::isPrimaryKey).toList();
+        List<TableField> notKeys = tableFields.stream().filter(tableField -> !tableField.isPrimaryKey()).toList();
+        String insetSql = (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null");
+        if (CollectionUtils.isNotEmpty(keys) && extractType.equals(DatasourceServer.UpdateType.add_scope)) {
+            insetSql = insetSql + " ON DUPLICATE KEY UPDATE ";
+            List<String> updateColumes = new ArrayList<>();
+            for (TableField notKey : notKeys) {
+                updateColumes.add("column = VALUES(column)".replace("column", notKey.getName()));
+            }
+            insetSql = insetSql + updateColumes.stream().collect(Collectors.joining(","));
+        }
+        return insetSql;
     }
 
 
@@ -93,7 +108,11 @@ public class MysqlEngineProvider extends EngineProvider {
             int size = tableField.getPrecision() * 4;
             switch (tableField.getDeExtractType()) {
                 case 0:
-                    columnFields.append("varchar(1024)").append(",`");
+                    if (StringUtils.isNotEmpty(tableField.getLength())) {
+                        columnFields.append("varchar(length)".replace("length", tableField.getLength())).append(",`");
+                    } else {
+                        columnFields.append("longtext").append(",`");
+                    }
                     break;
                 case 1:
                     columnFields.append("datetime").append(",`");
@@ -108,7 +127,7 @@ public class MysqlEngineProvider extends EngineProvider {
                     columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
                     break;
                 default:
-                    columnFields.append("varchar(1024)").append(",`");
+                    columnFields.append("longtext").append(",`");
                     break;
             }
         }

core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java

@@ -21,6 +21,7 @@ import io.dataease.dataset.dao.auto.entity.CoreDatasetGroup;
 import io.dataease.dataset.dao.auto.mapper.CoreDatasetGroupMapper;
 import io.dataease.dataset.manage.*;
 import io.dataease.datasource.utils.DatasourceUtils;
+import io.dataease.engine.constant.DeTypeConstants;
 import io.dataease.engine.sql.SQLProvider;
 import io.dataease.engine.trans.Field2SQLObj;
 import io.dataease.engine.trans.Order2SQLObj;
@@ -349,7 +350,7 @@ public class ExportCenterManage implements BaseExportApi {
         startViewTask(exportTask, request);
     }
 
-    public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request)throws Exception{
+    public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request) throws Exception {
         datasetGroupManage.getDatasetGroupInfoDTO(exportFrom, null);
         CoreExportTask exportTask = new CoreExportTask();
         exportTask.setId(UUID.randomUUID().toString());
@@ -571,7 +572,15 @@ public class ExportCenterManage implements BaseExportApi {
                                                 cell.setCellStyle(cellStyle);
                                                 detailsSheet.setColumnWidth(j, 255 * 20);
                                             } else {
-                                                cell.setCellValue(rowData.get(j));
+                                                if ((allFields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || allFields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) {
+                                                    try {
+                                                        cell.setCellValue(Double.valueOf(rowData.get(j)));
+                                                    } catch (Exception e) {
+                                                        cell.setCellValue(rowData.get(j));
+                                                    }
+                                                } else {
+                                                    cell.setCellValue(rowData.get(j));
+                                                }
                                             }
                                         }
                                     }
@@ -668,7 +677,6 @@ public class ExportCenterManage implements BaseExportApi {
                         Sheet detailsSheet = wb.createSheet("数据");
                         ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes);
                     }
-
                 } else {
                     //多个sheet
                     for (int i = 0; i < request.getMultiInfo().size(); i++) {

core/core-frontend/src/locales/tw.ts

@@ -1073,6 +1073,7 @@ export default {
     has_repeat_name: 'API 數據表名稱重復',
     has_repeat_field_name: '字段名重復，請修改后再選擇',
     primary_key_change: '主鍵不能改變:',
+    primary_key_length: '主键必须设置长度: ',
     api_field_not_empty: '字段不能為空',
     success_copy: '復制成功',
     valid: '有效',
@@ -1125,6 +1126,7 @@ export default {
     parse_filed: '解析字段',
     field_rename: '重命名',
     set_key: '設為主鍵',
+    length: '字段长度',
     select_type: '選擇數據源類型',
     sync_table: '同步指定表',
     req_completed: '請求成功',

core/core-frontend/src/locales/zh-CN.ts

@@ -1085,6 +1085,7 @@ export default {
     has_repeat_name: 'API 数据表名称重复',
     has_repeat_field_name: '字段名重复，请修改后再选择',
     primary_key_change: '主键不能改变:',
+    primary_key_length: '主键必须设置长度: ',
     api_field_not_empty: '字段不能为空',
     success_copy: '复制成功',
     valid: '有效',
@@ -1136,6 +1137,7 @@ export default {
     end_time: '结束时间',
     parse_filed: '解析字段',
     set_key: '设为主键',
+    length: '字段长度',
     field_rename: '重命名',
     select_type: '选择数据源类型',
     sync_table: '同步指定表',

core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue

@@ -18,6 +18,7 @@ import { iconFieldMap } from '@/components/icon-group/field-list'
 
 export interface Field {
   name: string
+  length: number
   value: Array<{}>
   checked: boolean
   primaryKey: boolean
@@ -48,6 +49,7 @@ export interface JsonField {
   name: string
   checked: false
   primaryKey: false
+  length: string
   extField: number
   jsonPath: string
   type: string
@@ -258,7 +260,7 @@ const saveItem = () => {
     for (let i = 0; i < apiItem.fields.length; i++) {
       if (apiItem.fields[i].primaryKey) {
         let find = false
-        for (let j = 0; j < fields.length - 1; j++) {
+        for (let j = 0; j < fields.length; j++) {
           if (fields[j].name === apiItem.fields[i].name && fields[j].primaryKey) {
             find = true
           }
@@ -268,10 +270,10 @@ const saveItem = () => {
         }
       }
     }
-    for (let i = 0; i < fields.length - 1; i++) {
+    for (let i = 0; i < fields.length; i++) {
       if (fields[i].primaryKey) {
         let find = false
-        for (let j = i + 1; j < apiItem.fields.length; j++) {
+        for (let j = 0; j < apiItem.fields.length; j++) {
           if (fields[i].name === apiItem.fields[j].name && apiItem.fields[j].primaryKey) {
             find = true
           }
@@ -285,6 +287,12 @@ const saveItem = () => {
       ElMessage.error(t('datasource.primary_key_change') + msg)
       return
     }
+  } else {
+    for (let i = 0; i < apiItem.fields.length; i++) {
+      if (apiItem.fields[i].primaryKey && !apiItem.fields[i].length) {
+        ElMessage.error(t('datasource.primary_key_length') + apiItem.fields[i].name)
+      }
+    }
   }
   returnAPIItem('returnItem', cloneDeep(apiItem))
   edit_api_item.value = false
@@ -381,6 +389,14 @@ const disabledByChildren = item => {
   }
 }
 
+const disabledFieldLength = item => {
+  if (item.hasOwnProperty('children') && item.children.length > 0) {
+    return true
+  } else {
+    return item.deExtractType !== 0
+  }
+}
+
 const disabledChangeFieldByChildren = item => {
   if (apiItem.type == 'params') {
     return true
@@ -391,6 +407,12 @@ const disabledChangeFieldByChildren = item => {
     return false
   }
 }
+
+const deExtractTypeChange = item => {
+  if (item.deExtractType !== 0) {
+    item.length = ''
+  }
+}
 const previewData = () => {
   showEmpty.value = false
   const data = []
@@ -491,7 +513,7 @@ defineExpose({
     "
     v-model="edit_api_item"
     custom-class="api-datasource-drawer"
-    size="840px"
+    size="1000px"
     :before-close="closeEditItem"
     direction="rtl"
   >
@@ -661,7 +683,7 @@ defineExpose({
               prop="originName"
               :label="t('datasource.parse_filed')"
               :show-overflow-tooltip="true"
-              width="255"
+              width="200"
             >
               <template #default="scope">
                 <el-checkbox
@@ -696,6 +718,7 @@ defineExpose({
                   :disabled="disabledChangeFieldByChildren(scope.row)"
                   class="select-type"
                   style="display: inline-block; width: 120px"
+                  @change="deExtractTypeChange(scope.row)"
                 >
                   <template #prefix>
                     <el-icon>
@@ -733,12 +756,33 @@ defineExpose({
               </template>
             </el-table-column>
 
+            <el-table-column
+              prop="length"
+              :label="t('datasource.length')"
+              v-if="apiItem.type !== 'params'"
+            >
+              <template #default="scope">
+                <el-input-number
+                  :disabled="disabledFieldLength(scope.row)"
+                  v-model="scope.row.length"
+                  autocomplete="off"
+                  step-strictly
+                  class="text-left edit-all-line"
+                  :min="1"
+                  :max="4096"
+                  :placeholder="t('common.inputText')"
+                  controls-position="right"
+                  type="number"
+                />
+              </template>
+            </el-table-column>
+
             <el-table-column
               prop="primaryKey"
               class-name="checkbox-table"
               :label="t('datasource.set_key')"
               v-if="apiItem.type !== 'params'"
-              width="155"
+              width="100"
             >
               <template #default="scope">
                 <el-checkbox

sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/dto/TableField.java

@@ -14,6 +14,7 @@ public class TableField implements Serializable {
     private int precision;
     private long size;
     private int scale;
+    private String length;
     private boolean checked = false;
     private boolean primaryKey = false;
     private String fieldType;