feat(数据源): API数据源支持设置主键，增量同步可根据主键更新已同步的数据

core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java

@@ -44,6 +44,9 @@ public class ApiUtils {
         };
         List<ApiDefinition> apiDefinitionList = JsonUtil.parseList(datasourceRequest.getDatasource().getConfiguration(), listTypeReference);
         for (ApiDefinition apiDefinition : apiDefinitionList) {
+            if (apiDefinition == null) {
+                continue;
+            }
             if (StringUtils.isNotEmpty(apiDefinition.getType()) && apiDefinition.getType().equalsIgnoreCase("params")) {
                 continue;
             }
@@ -116,6 +119,9 @@ public class ApiUtils {
         List<ApiDefinition> apiDefinitionList = JsonUtil.parseList(datasourceRequest.getDatasource().getConfiguration(), listTypeReference);
         List<ObjectNode> status = new ArrayList();
         for (ApiDefinition apiDefinition : apiDefinitionList) {
+            if (apiDefinition == null) {
+                continue;
+            }
             datasourceRequest.setTable(apiDefinition.getName());
             ObjectNode apiItemStatuses = objectMapper.createObjectNode();
             try {
@@ -573,6 +579,7 @@ public class ApiUtils {
                 if (!ObjectUtils.isEmpty(o.get("jsonPath")) && StringUtils.isNotEmpty(field.getJsonPath()) && field.getJsonPath().equals(o.get("jsonPath").toString())) {
                     o.put("checked", true);
                     o.put("name", field.getName());
+                    o.put("primaryKey", field.isPrimaryKey());
                     o.put("deExtractType", field.getDeExtractType());
                 }
             }
@@ -715,7 +722,7 @@ public class ApiUtils {
         } catch (Exception e) {
             DEException.throwException(e);
         }
-        return apiDefinitionListTemp.stream().filter(apiDefinition -> apiDefinition.getType() != null && apiDefinition.getType().equalsIgnoreCase("params")).collect(Collectors.toList());
+        return apiDefinitionListTemp.stream().filter(apiDefinition -> apiDefinition != null && apiDefinition.getType() != null && apiDefinition.getType().equalsIgnoreCase("params")).collect(Collectors.toList());
     }
 
     private static ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest) throws DEException {
@@ -730,6 +737,9 @@ public class ApiUtils {
         }
         if (!CollectionUtils.isEmpty(apiDefinitionListTemp)) {
             for (ApiDefinition apiDefinition : apiDefinitionListTemp) {
+                if (apiDefinition == null) {
+                    continue;
+                }
                 if (apiDefinition.getDeTableName().equalsIgnoreCase(datasourceRequest.getTable()) || apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable())) {
                     apiDefinitionList.add(apiDefinition);
                 }
@@ -744,6 +754,9 @@ public class ApiUtils {
         }
         ApiDefinition find = null;
         for (ApiDefinition apiDefinition : apiDefinitionList) {
+            if (apiDefinition == null) {
+                continue;
+            }
             if (apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable()) || apiDefinition.getDeTableName().equalsIgnoreCase(datasourceRequest.getTable())) {
                 find = apiDefinition;
             }

core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java

@@ -78,34 +78,43 @@ public class H2EngineProvider extends EngineProvider {
     }
 
     private String createTableSql(final List<TableField> tableFields) {
-        StringBuilder Column_Fields = new StringBuilder("`");
+        StringBuilder columnFields = new StringBuilder("`");
+        StringBuilder key = new StringBuilder();
         for (TableField tableField : tableFields) {
-            Column_Fields.append(tableField.getName()).append("` ");
+            if (tableField.isPrimaryKey()) {
+                key.append("`").append(tableField.getName()).append("`, ");
+            }
+            columnFields.append(tableField.getName()).append("` ");
             int size = tableField.getPrecision() * 4;
             switch (tableField.getDeType()) {
                 case 0:
-                    Column_Fields.append("varchar(2048)").append(",`");
+                    columnFields.append("varchar(2048)").append(",`");
                     break;
                 case 1:
-                    Column_Fields.append("varchar(2048)").append(",`");
+                    columnFields.append("varchar(2048)").append(",`");
                     break;
                 case 2:
-                    Column_Fields.append("bigint(20)").append(",`");
+                    columnFields.append("bigint(20)").append(",`");
                     break;
                 case 3:
-                    Column_Fields.append("varchar(100)").append(",`");
+                    columnFields.append("varchar(100)").append(",`");
                     break;
                 case 4:
-                    Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
+                    columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
                     break;
                 default:
-                    Column_Fields.append("varchar(2048)").append(",`");
+                    columnFields.append("varchar(2048)").append(",`");
                     break;
             }
         }
-
+        if (StringUtils.isEmpty(key.toString())) {
-        Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
+            columnFields = new StringBuilder(columnFields.substring(0, columnFields.length() - 2));
-        Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
+        } else {
-        return Column_Fields.toString();
+            key = new StringBuilder(key.substring(0, key.length() - 2));
+            columnFields = new StringBuilder(columnFields.substring(0, columnFields.length() - 1));
+            columnFields.append("PRIMARY KEY (PRIMARYKEY)".replace("PRIMARYKEY", key.toString()));
+        }
+        columnFields = new StringBuilder("(" + columnFields + ")");
+        return columnFields.toString();
     }
 }

core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java

@@ -83,34 +83,44 @@ public class MysqlEngineProvider extends EngineProvider {
     }
 
     private String createTableSql(final List<TableField> tableFields) {
-        StringBuilder Column_Fields = new StringBuilder("`");
+        StringBuilder columnFields = new StringBuilder("`");
+        StringBuilder key = new StringBuilder();
         for (TableField tableField : tableFields) {
-            Column_Fields.append(tableField.getName()).append("` ");
+            if (tableField.isPrimaryKey()) {
+                key.append("`").append(tableField.getName()).append("`, ");
+            }
+            columnFields.append(tableField.getName()).append("` ");
             int size = tableField.getPrecision() * 4;
             switch (tableField.getDeExtractType()) {
                 case 0:
-                    Column_Fields.append("varchar(1024)").append(",`");
+                    columnFields.append("varchar(1024)").append(",`");
                     break;
                 case 1:
-                    Column_Fields.append("datetime").append(",`");
+                    columnFields.append("datetime").append(",`");
                     break;
                 case 2:
-                    Column_Fields.append("bigint(20)").append(",`");
+                    columnFields.append("bigint(20)").append(",`");
                     break;
                 case 3:
-                    Column_Fields.append("decimal(27,8)").append(",`");
+                    columnFields.append("decimal(27,8)").append(",`");
                     break;
                 case 4:
-                    Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
+                    columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
                     break;
                 default:
-                    Column_Fields.append("varchar(1024)").append(",`");
+                    columnFields.append("varchar(1024)").append(",`");
                     break;
             }
         }
+        if (StringUtils.isEmpty(key.toString())) {
+            columnFields = new StringBuilder(columnFields.substring(0, columnFields.length() - 2));
+        } else {
+            key = new StringBuilder(key.substring(0, key.length() - 2));
+            columnFields = new StringBuilder(columnFields.substring(0, columnFields.length() - 1));
+            columnFields.append("PRIMARY KEY (PRIMARYKEY)".replace("PRIMARYKEY", key.toString()));
+        }
 
-        Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
+        columnFields = new StringBuilder("(" + columnFields + ")");
-        Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
+        return columnFields.toString();
-        return Column_Fields.toString();
     }
 }

core/core-frontend/src/locales/tw.ts

@@ -1072,6 +1072,7 @@ export default {
     api_table_not_empty: 'API 數據表不能為空',
     has_repeat_name: 'API 數據表名稱重復',
     has_repeat_field_name: '字段名重復，請修改后再選擇',
+    primary_key_change: '主鍵不能改變:',
     api_field_not_empty: '字段不能為空',
     success_copy: '復制成功',
     valid: '有效',

core/core-frontend/src/locales/zh-CN.ts

@@ -1072,6 +1072,7 @@ export default {
     api_table_not_empty: 'API 数据表不能为空',
     has_repeat_name: 'API 数据表名称重复',
     has_repeat_field_name: '字段名重复，请修改后再选择',
+    primary_key_change: '主键不能改变:',
     api_field_not_empty: '字段不能为空',
     success_copy: '复制成功',
     valid: '有效',
@@ -1122,6 +1123,7 @@ export default {
     start_time: '开始时间',
     end_time: '结束时间',
     parse_filed: '解析字段',
+    set_key: '设为主键',
     field_rename: '重命名',
     select_type: '选择数据源类型',
     sync_table: '同步指定表',

core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue

@@ -20,6 +20,7 @@ export interface Field {
   name: string
   value: Array<{}>
   checked: boolean
+  primaryKey: boolean
   children?: Array<{}>
 }
 
@@ -46,6 +47,7 @@ export interface JsonField {
   children: null
   name: string
   checked: false
+  primaryKey: false
   extField: number
   jsonPath: string
   type: string
@@ -61,6 +63,7 @@ const originFieldItem = reactive({
 
 let apiItemList = reactive<ApiConfiguration[]>([])
 let paramsList = reactive<ApiConfiguration[]>([])
+let fields = reactive<Field[]>([])
 
 let apiItem = reactive<ApiItem>({
   status: '',
@@ -157,13 +160,18 @@ const rule = reactive<FormRules>({
   ]
 })
 const activeName = ref('table')
+const editItem = ref(false)
 provide('api-active-name', activeName)
-const initApiItem = (val: ApiItem, from, name) => {
+const initApiItem = (val: ApiItem, from, name, edit) => {
   activeName.value = name
+  editItem.value = edit
   apiItemList = from.apiConfiguration
-  paramsList = from.paramsConfiguration
+  fields = val.fields
-  if (val.type !== 'params') {
+  if (from.paramsConfiguration) {
-    valueList.value = []
+    paramsList = from.paramsConfiguration
+  }
+  valueList.value = []
+  if (val.type !== 'params' && paramsList) {
     for (let i = 0; i < paramsList.length; i++) {
       valueList.value = valueList.value.concat(paramsList[i].fields)
     }
@@ -236,6 +244,7 @@ const saveItem = () => {
       }
     }
   }
+
   for (let i = 0; i < apiItem.fields.length - 1; i++) {
     for (let j = i + 1; j < apiItem.fields.length; j++) {
       if (apiItem.fields[i].name === apiItem.fields[j].name) {
@@ -244,6 +253,39 @@ const saveItem = () => {
       }
     }
   }
+  if (editItem.value) {
+    let msg = ''
+    for (let i = 0; i < apiItem.fields.length; i++) {
+      if (apiItem.fields[i].primaryKey) {
+        let find = false
+        for (let j = 0; j < fields.length - 1; j++) {
+          if (fields[j].name === apiItem.fields[i].name && fields[j].primaryKey) {
+            find = true
+          }
+        }
+        if (!find) {
+          msg = msg + ' ' + apiItem.fields[i].name
+        }
+      }
+    }
+    for (let i = 0; i < fields.length - 1; i++) {
+      if (fields[i].primaryKey) {
+        let find = false
+        for (let j = i + 1; j < apiItem.fields.length; j++) {
+          if (fields[i].name === apiItem.fields[j].name && apiItem.fields[j].primaryKey) {
+            find = true
+          }
+        }
+        if (!find) {
+          msg = msg + ' ' + fields[i].name
+        }
+      }
+    }
+    if (msg !== '') {
+      ElMessage.error(t('datasource.primary_key_change') + msg)
+      return
+    }
+  }
   returnAPIItem('returnItem', cloneDeep(apiItem))
   edit_api_item.value = false
 }
@@ -646,7 +688,7 @@ defineExpose({
             <el-table-column
               prop="deExtractType"
               :label="t('datasource.field_type')"
-              :disabled="apiItem.type == 'params'"
+              :disabled="apiItem.type === 'params'"
             >
               <template #default="scope">
                 <el-select
@@ -690,6 +732,23 @@ defineExpose({
                 </el-select>
               </template>
             </el-table-column>
+
+            <el-table-column
+              prop="primaryKey"
+              class-name="checkbox-table"
+              :label="t('datasource.set_key')"
+              v-if="apiItem.type !== 'params'"
+              width="155"
+            >
+              <template #default="scope">
+                <el-checkbox
+                  :key="scope.row.jsonPath"
+                  v-model="scope.row.primaryKey"
+                  :disabled="editItem || !scope.row.checked"
+                >
+                </el-checkbox>
+              </template>
+            </el-table-column>
           </el-table>
         </div>
         <p

core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue

@@ -389,9 +389,11 @@ const copyItem = (item?: ApiConfiguration) => {
 }
 const addApiItem = item => {
   let apiItem = null
+  let editItem = false
   api_table_title.value = t('datasource.data_table')
   if (item) {
     apiItem = cloneDeep(item)
+    editItem = true
   } else {
     apiItem = cloneDeep(defaultApiItem)
     apiItem.type = activeName.value
@@ -400,14 +402,13 @@ const addApiItem = item => {
         ? form.value.apiConfiguration[form.value.apiConfiguration.length - 1].serialNumber + 1
         : 0
     let serialNumber2 =
-      form.value.paramsConfiguration.length > 0
+      form.value.paramsConfiguration && form.value.paramsConfiguration.length > 0
         ? form.value.paramsConfiguration[form.value.paramsConfiguration.length - 1].serialNumber + 1
         : 0
-
     apiItem.serialNumber = serialNumber1 + serialNumber2
   }
   nextTick(() => {
-    editApiItem.value.initApiItem(apiItem, form.value, activeName.value)
+    editApiItem.value.initApiItem(apiItem, form.value, activeName.value, editItem)
   })
 }
 
@@ -780,7 +781,10 @@ defineExpose({
           <div class="title-form_primary flex-space table-info-mr" v-show="activeStep !== 2">
             <el-tabs v-model="activeName" class="api-tabs">
               <el-tab-pane :label="t('datasource.data_table')" name="table"></el-tab-pane>
-              <el-tab-pane :label="t('data_source.connection_method')" name="params"></el-tab-pane>
+              <el-tab-pane
+                :label="t('data_source.interface_parameters')"
+                name="params"
+              ></el-tab-pane>
             </el-tabs>
             <el-button type="primary" style="margin-left: auto" @click="() => addApiItem(null)">
               <template #icon>

core/core-frontend/src/views/visualized/data/datasource/form/index.vue

@@ -341,7 +341,9 @@ const validateDS = () => {
     }
     let apiItems = []
     apiItems = apiItems.concat(request.apiConfiguration)
-    apiItems = apiItems.concat(request.paramsConfiguration)
+    if (request.paramsConfiguration) {
+      apiItems = apiItems.concat(request.paramsConfiguration)
+    }
     request.configuration = Base64.encode(JSON.stringify(apiItems))
     request.syncSetting.startTime = new Date(request.syncSetting.startTime).getTime()
     request.syncSetting.endTime = new Date(request.syncSetting.endTime).getTime()
@@ -459,7 +461,9 @@ const saveDS = () => {
     }
     let apiItems = []
     apiItems = apiItems.concat(request.apiConfiguration)
-    apiItems = apiItems.concat(request.paramsConfiguration)
+    if (request.paramsConfiguration) {
+      apiItems = apiItems.concat(request.paramsConfiguration)
+    }
     request.configuration = Base64.encode(JSON.stringify(apiItems))
     request.syncSetting.startTime = new Date(request.syncSetting.startTime).getTime()
     request.syncSetting.endTime = new Date(request.syncSetting.endTime).getTime()

sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/dto/TableField.java

@@ -15,6 +15,7 @@ public class TableField implements Serializable {
     private long size;
     private int scale;
     private boolean checked = false;
+    private boolean primaryKey = false;
     private String fieldType;
     private Integer deType;
     private Integer deExtractType;