github/dataease
github/dataease
3
0
Fork 1
mirror of https://github.com/dataease/dataease.git synced 2025-02-24 11:32:57 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #13289 from dataease/pr@dev-v2@fixds

Browse Source 
feat(数据源): API数据源支持设置主键，增量同步可根据主键更新已同步的数据
This commit is contained in:
taojinlong 2024-11-13 18:19:45 +08:00 committed by GitHub
commit c043e210c3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 138 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java
View File

@ -44,6 +44,9 @@ public class ApiUtils {
};
List<ApiDefinition> apiDefinitionList = JsonUtil.parseList(datasourceRequest.getDatasource().getConfiguration(), listTypeReference);
for (ApiDefinition apiDefinition : apiDefinitionList) {
if (apiDefinition == null) {
continue;
}
if (StringUtils.isNotEmpty(apiDefinition.getType()) && apiDefinition.getType().equalsIgnoreCase("params")) {
continue;
}
@ -116,6 +119,9 @@ public class ApiUtils {
List<ApiDefinition> apiDefinitionList = JsonUtil.parseList(datasourceRequest.getDatasource().getConfiguration(), listTypeReference);
List<ObjectNode> status = new ArrayList();
for (ApiDefinition apiDefinition : apiDefinitionList) {
if (apiDefinition == null) {
continue;
}
datasourceRequest.setTable(apiDefinition.getName());
ObjectNode apiItemStatuses = objectMapper.createObjectNode();
try {
@ -573,6 +579,7 @@ public class ApiUtils {
if (!ObjectUtils.isEmpty(o.get("jsonPath")) && StringUtils.isNotEmpty(field.getJsonPath()) && field.getJsonPath().equals(o.get("jsonPath").toString())) {
o.put("checked", true);
o.put("name", field.getName());
o.put("primaryKey", field.isPrimaryKey());
o.put("deExtractType", field.getDeExtractType());
}
}
@ -715,7 +722,7 @@ public class ApiUtils {
} catch (Exception e) {
DEException.throwException(e);
}
return apiDefinitionListTemp.stream().filter(apiDefinition -> apiDefinition.getType() != null && apiDefinition.getType().equalsIgnoreCase("params")).collect(Collectors.toList());
return apiDefinitionListTemp.stream().filter(apiDefinition -> apiDefinition != null && apiDefinition.getType() != null && apiDefinition.getType().equalsIgnoreCase("params")).collect(Collectors.toList());
}
private static ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest) throws DEException {
@ -730,6 +737,9 @@ public class ApiUtils {
}
if (!CollectionUtils.isEmpty(apiDefinitionListTemp)) {
for (ApiDefinition apiDefinition : apiDefinitionListTemp) {
if (apiDefinition == null) {
continue;
}
if (apiDefinition.getDeTableName().equalsIgnoreCase(datasourceRequest.getTable()) || apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable())) {
apiDefinitionList.add(apiDefinition);
}
@ -744,6 +754,9 @@ public class ApiUtils {
}
ApiDefinition find = null;
for (ApiDefinition apiDefinition : apiDefinitionList) {
if (apiDefinition == null) {
continue;
}
if (apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable()) || apiDefinition.getDeTableName().equalsIgnoreCase(datasourceRequest.getTable())) {
find = apiDefinition;
}

33
core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java
View File

@ -78,34 +78,43 @@ public class H2EngineProvider extends EngineProvider {
}
private String createTableSql(final List<TableField> tableFields) {
StringBuilder Column_Fields = new StringBuilder("`");
StringBuilder columnFields = new StringBuilder("`");
StringBuilder key = new StringBuilder();
for (TableField tableField : tableFields) {
Column_Fields.append(tableField.getName()).append("` ");
if (tableField.isPrimaryKey()) {
key.append("`").append(tableField.getName()).append("`, ");
}
columnFields.append(tableField.getName()).append("` ");
int size = tableField.getPrecision() * 4;
switch (tableField.getDeType()) {
case 0:
Column_Fields.append("varchar(2048)").append(",`");
columnFields.append("varchar(2048)").append(",`");
break;
case 1:
Column_Fields.append("varchar(2048)").append(",`");
columnFields.append("varchar(2048)").append(",`");
break;
case 2:
Column_Fields.append("bigint(20)").append(",`");
columnFields.append("bigint(20)").append(",`");
break;
case 3:
Column_Fields.append("varchar(100)").append(",`");
columnFields.append("varchar(100)").append(",`");
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
break;
default:
Column_Fields.append("varchar(2048)").append(",`");
columnFields.append("varchar(2048)").append(",`");
break;
}
}
Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
return Column_Fields.toString();
if (StringUtils.isEmpty(key.toString())) {
columnFields = new StringBuilder(columnFields.substring(0, columnFields.length() - 2));
} else {
key = new StringBuilder(key.substring(0, key.length() - 2));
columnFields = new StringBuilder(columnFields.substring(0, columnFields.length() - 1));
columnFields.append("PRIMARY KEY (PRIMARYKEY)".replace("PRIMARYKEY", key.toString()));
}
columnFields = new StringBuilder("(" + columnFields + ")");
return columnFields.toString();
}
}

32
core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java
View File

@ -83,34 +83,44 @@ public class MysqlEngineProvider extends EngineProvider {
}
private String createTableSql(final List<TableField> tableFields) {
StringBuilder Column_Fields = new StringBuilder("`");
StringBuilder columnFields = new StringBuilder("`");
StringBuilder key = new StringBuilder();
for (TableField tableField : tableFields) {
Column_Fields.append(tableField.getName()).append("` ");
if (tableField.isPrimaryKey()) {
key.append("`").append(tableField.getName()).append("`, ");
}
columnFields.append(tableField.getName()).append("` ");
int size = tableField.getPrecision() * 4;
switch (tableField.getDeExtractType()) {
case 0:
Column_Fields.append("varchar(1024)").append(",`");
columnFields.append("varchar(1024)").append(",`");
break;
case 1:
Column_Fields.append("datetime").append(",`");
columnFields.append("datetime").append(",`");
break;
case 2:
Column_Fields.append("bigint(20)").append(",`");
columnFields.append("bigint(20)").append(",`");
break;
case 3:
Column_Fields.append("decimal(27,8)").append(",`");
columnFields.append("decimal(27,8)").append(",`");
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
break;
default:
Column_Fields.append("varchar(1024)").append(",`");
columnFields.append("varchar(1024)").append(",`");
break;
}
}
if (StringUtils.isEmpty(key.toString())) {
columnFields = new StringBuilder(columnFields.substring(0, columnFields.length() - 2));
} else {
key = new StringBuilder(key.substring(0, key.length() - 2));
columnFields = new StringBuilder(columnFields.substring(0, columnFields.length() - 1));
columnFields.append("PRIMARY KEY (PRIMARYKEY)".replace("PRIMARYKEY", key.toString()));
}
Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
return Column_Fields.toString();
columnFields = new StringBuilder("(" + columnFields + ")");
return columnFields.toString();
}
}

1
core/core-frontend/src/locales/tw.ts
View File

@ -1072,6 +1072,7 @@ export default {
api_table_not_empty: 'API 數據表不能為空',
has_repeat_name: 'API 數據表名稱重復',
has_repeat_field_name: '字段名重復，請修改后再選擇',
primary_key_change: '主鍵不能改變:',
api_field_not_empty: '字段不能為空',
success_copy: '復制成功',
valid: '有效',

2
core/core-frontend/src/locales/zh-CN.ts
View File

@ -1072,6 +1072,7 @@ export default {
api_table_not_empty: 'API 数据表不能为空',
has_repeat_name: 'API 数据表名称重复',
has_repeat_field_name: '字段名重复，请修改后再选择',
primary_key_change: '主键不能改变:',
api_field_not_empty: '字段不能为空',
success_copy: '复制成功',
valid: '有效',
@ -1122,6 +1123,7 @@ export default {
start_time: '开始时间',
end_time: '结束时间',
parse_filed: '解析字段',
set_key: '设为主键',
field_rename: '重命名',
select_type: '选择数据源类型',
sync_table: '同步指定表',

69
core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue
View File

@ -20,6 +20,7 @@ export interface Field {
name: string
value: Array<{}>
checked: boolean
primaryKey: boolean
children?: Array<{}>
}
@ -46,6 +47,7 @@ export interface JsonField {
children: null
name: string
checked: false
primaryKey: false
extField: number
jsonPath: string
type: string
@ -61,6 +63,7 @@ const originFieldItem = reactive({
let apiItemList = reactive<ApiConfiguration[]>([])
let paramsList = reactive<ApiConfiguration[]>([])
let fields = reactive<Field[]>([])
let apiItem = reactive<ApiItem>({
status: '',
@ -157,13 +160,18 @@ const rule = reactive<FormRules>({
]
})
const activeName = ref('table')
const editItem = ref(false)
provide('api-active-name', activeName)
const initApiItem = (val: ApiItem, from, name) => {
const initApiItem = (val: ApiItem, from, name, edit) => {
activeName.value = name
editItem.value = edit
apiItemList = from.apiConfiguration
paramsList = from.paramsConfiguration
if (val.type !== 'params') {
valueList.value = []
fields = val.fields
if (from.paramsConfiguration) {
paramsList = from.paramsConfiguration
}
valueList.value = []
if (val.type !== 'params' && paramsList) {
for (let i = 0; i < paramsList.length; i++) {
valueList.value = valueList.value.concat(paramsList[i].fields)
}
@ -236,6 +244,7 @@ const saveItem = () => {
}
}
}
for (let i = 0; i < apiItem.fields.length - 1; i++) {
for (let j = i + 1; j < apiItem.fields.length; j++) {
if (apiItem.fields[i].name === apiItem.fields[j].name) {
@ -244,6 +253,39 @@ const saveItem = () => {
}
}
}
if (editItem.value) {
let msg = ''
for (let i = 0; i < apiItem.fields.length; i++) {
if (apiItem.fields[i].primaryKey) {
let find = false
for (let j = 0; j < fields.length - 1; j++) {
if (fields[j].name === apiItem.fields[i].name && fields[j].primaryKey) {
find = true
}
}
if (!find) {
msg = msg + ' ' + apiItem.fields[i].name
}
}
}
for (let i = 0; i < fields.length - 1; i++) {
if (fields[i].primaryKey) {
let find = false
for (let j = i + 1; j < apiItem.fields.length; j++) {
if (fields[i].name === apiItem.fields[j].name && apiItem.fields[j].primaryKey) {
find = true
}
}
if (!find) {
msg = msg + ' ' + fields[i].name
}
}
}
if (msg !== '') {
ElMessage.error(t('datasource.primary_key_change') + msg)
return
}
}
returnAPIItem('returnItem', cloneDeep(apiItem))
edit_api_item.value = false
}
@ -646,7 +688,7 @@ defineExpose({
<el-table-column
prop="deExtractType"
:label="t('datasource.field_type')"
:disabled="apiItem.type == 'params'"
:disabled="apiItem.type === 'params'"
>
<template #default="scope">
<el-select
@ -690,6 +732,23 @@ defineExpose({
</el-select>
</template>
</el-table-column>
<el-table-column
prop="primaryKey"
class-name="checkbox-table"
:label="t('datasource.set_key')"
v-if="apiItem.type !== 'params'"
width="155"
>
<template #default="scope">
<el-checkbox
:key="scope.row.jsonPath"
v-model="scope.row.primaryKey"
:disabled="editItem || !scope.row.checked"
>
</el-checkbox>
</template>
</el-table-column>
</el-table>
</div>
<p

12
core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue
View File

@ -389,9 +389,11 @@ const copyItem = (item?: ApiConfiguration) => {
}
const addApiItem = item => {
let apiItem = null
let editItem = false
api_table_title.value = t('datasource.data_table')
if (item) {
apiItem = cloneDeep(item)
editItem = true
} else {
apiItem = cloneDeep(defaultApiItem)
apiItem.type = activeName.value
@ -400,14 +402,13 @@ const addApiItem = item => {
? form.value.apiConfiguration[form.value.apiConfiguration.length - 1].serialNumber + 1
: 0
let serialNumber2 =
form.value.paramsConfiguration.length > 0
form.value.paramsConfiguration && form.value.paramsConfiguration.length > 0
? form.value.paramsConfiguration[form.value.paramsConfiguration.length - 1].serialNumber + 1
: 0
apiItem.serialNumber = serialNumber1 + serialNumber2
}
nextTick(() => {
editApiItem.value.initApiItem(apiItem, form.value, activeName.value)
editApiItem.value.initApiItem(apiItem, form.value, activeName.value, editItem)
})
}
@ -780,7 +781,10 @@ defineExpose({
<div class="title-form_primary flex-space table-info-mr" v-show="activeStep !== 2">
<el-tabs v-model="activeName" class="api-tabs">
<el-tab-pane :label="t('datasource.data_table')" name="table"></el-tab-pane>
<el-tab-pane :label="t('data_source.connection_method')" name="params"></el-tab-pane>
<el-tab-pane
:label="t('data_source.interface_parameters')"
name="params"
></el-tab-pane>
</el-tabs>
<el-button type="primary" style="margin-left: auto" @click="() => addApiItem(null)">
<template #icon>

8
core/core-frontend/src/views/visualized/data/datasource/form/index.vue
View File

@ -341,7 +341,9 @@ const validateDS = () => {
}
let apiItems = []
apiItems = apiItems.concat(request.apiConfiguration)
apiItems = apiItems.concat(request.paramsConfiguration)
if (request.paramsConfiguration) {
apiItems = apiItems.concat(request.paramsConfiguration)
}
request.configuration = Base64.encode(JSON.stringify(apiItems))
request.syncSetting.startTime = new Date(request.syncSetting.startTime).getTime()
request.syncSetting.endTime = new Date(request.syncSetting.endTime).getTime()
@ -459,7 +461,9 @@ const saveDS = () => {
}
let apiItems = []
apiItems = apiItems.concat(request.apiConfiguration)
apiItems = apiItems.concat(request.paramsConfiguration)
if (request.paramsConfiguration) {
apiItems = apiItems.concat(request.paramsConfiguration)
}
request.configuration = Base64.encode(JSON.stringify(apiItems))
request.syncSetting.startTime = new Date(request.syncSetting.startTime).getTime()
request.syncSetting.endTime = new Date(request.syncSetting.endTime).getTime()

1
sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/dto/TableField.java
View File

@ -15,6 +15,7 @@ public class TableField implements Serializable {
private long size;
private int scale;
private boolean checked = false;
private boolean primaryKey = false;
private String fieldType;
private Integer deType;
private Integer deExtractType;