github/dataease
github/dataease
3
0
Fork 1
mirror of https://github.com/dataease/dataease.git synced 2025-02-24 11:32:57 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: 【API数据源】API数据源支持设置主键，增量同步可根据主键更新已同步的数据

Browse Source
This commit is contained in:
taojinlong 2024-11-22 17:32:37 +08:00
parent e0ef4e4027
commit b972458406
11 changed files with 133 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java
View File

@ -394,7 +394,11 @@ public class ChartDataServer implements ChartDataApi {
detailsSheet.setColumnWidth(j, 255 * 20);
} else if (cellValObj != null) {
try {
if ((excelTypes[j].equals(DeTypeConstants.DE_INT) || excelTypes[j].equals(DeTypeConstants.DE_FLOAT)) && StringUtils.isNotEmpty(cellValObj.toString())) {
cell.setCellValue(Double.valueOf(cellValObj.toString()));
} else if (cellValObj != null) {
cell.setCellValue(cellValObj.toString());
}
} catch (Exception e) {
LogUtil.warn("export excel data transform error");
}

25
core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java
View File

@ -70,7 +70,7 @@ public class DatasourceSyncManage {
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields);
}
extractExcelData(datasourceRequest, updateType);
extractExcelData(datasourceRequest, updateType, tableFields);
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
replaceTable(datasourceRequest.getTable());
}
@ -145,7 +145,7 @@ public class DatasourceSyncManage {
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields);
}
extractApiData(datasourceRequest, updateType);
extractApiData(datasourceRequest, updateType, tableFields);
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
replaceTable(datasourceRequest.getTable());
}
@ -198,7 +198,7 @@ public class DatasourceSyncManage {
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields);
}
extractApiData(datasourceRequest, updateType);
extractApiData(datasourceRequest, updateType, tableFields);
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
replaceTable(datasourceRequest.getTable());
}
@ -222,20 +222,10 @@ public class DatasourceSyncManage {
}
}
private void extractApiData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType) throws Exception {
private void extractApiData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType, List<TableField> tableFields) throws Exception {
Map<String, Object> result = ApiUtils.fetchResultField(datasourceRequest);
List<String[]> dataList = (List<String[]>) result.get("dataList");
String engineTableName;
switch (extractType) {
case all_scope:
engineTableName = TableUtils.tmpName(TableUtils.tableName(datasourceRequest.getTable()));
break;
default:
engineTableName = TableUtils.tableName(datasourceRequest.getTable());
break;
}
CoreDeEngine engine = engineManage.info();
EngineRequest engineRequest = new EngineRequest();
engineRequest.setEngine(engine);
EngineProvider engineProvider = ProviderUtil.getEngineProvider(engine.getType());
@ -246,14 +236,13 @@ public class DatasourceSyncManage {
} else {
totalPage = dataList.size() / pageNumber;
}
for (int page = 1; page <= totalPage; page++) {
engineRequest.setQuery(engineProvider.insertSql(engineTableName, dataList, page, pageNumber));
engineRequest.setQuery(engineProvider.insertSql(datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields));
calciteProvider.exec(engineRequest);
}
}
private void extractExcelData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType) throws Exception {
private void extractExcelData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType, List<TableField> tableFields) throws Exception {
ExcelUtils excelUtils = new ExcelUtils();
List<String[]> dataList = excelUtils.fetchDataList(datasourceRequest);
String engineTableName;
@ -278,7 +267,7 @@ public class DatasourceSyncManage {
totalPage = dataList.size() / pageNumber;
}
for (int page = 1; page <= totalPage; page++) {
engineRequest.setQuery(engineProvider.insertSql(engineTableName, dataList, page, pageNumber));
engineRequest.setQuery(engineProvider.insertSql(engineTableName, extractType, dataList, page, pageNumber, tableFields));
calciteProvider.exec(engineRequest);
}
}

1
core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java
View File

@ -580,6 +580,7 @@ public class ApiUtils {
o.put("checked", true);
o.put("name", field.getName());
o.put("primaryKey", field.isPrimaryKey());
o.put("length", field.getLength());
o.put("deExtractType", field.getDeExtractType());
}
}

3
core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java
View File

@ -2,6 +2,7 @@ package io.dataease.datasource.provider;
import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
import io.dataease.datasource.request.EngineRequest;
import io.dataease.datasource.server.DatasourceServer;
import io.dataease.extensions.datasource.dto.TableField;
import java.util.List;
@ -21,7 +22,7 @@ public abstract class EngineProvider {
public abstract String createTableSql(String name, List<TableField> tableFields, CoreDeEngine engine);
public abstract String insertSql(String name, List<String[]> dataList, int page, int pageNumber);
public abstract String insertSql(String tableName, DatasourceServer.UpdateType extractType, List<String[]> dataList, int page, int pageNumber, List<TableField> tableFields);
}

32
core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java
View File

@ -3,20 +3,11 @@ package io.dataease.datasource.provider;
import io.dataease.dataset.utils.TableUtils;
import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
import io.dataease.datasource.request.EngineRequest;
import io.dataease.datasource.type.H2;
import io.dataease.datasource.type.Mysql;
import io.dataease.extensions.datasource.dto.ConnectionObj;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.datasource.server.DatasourceServer;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.JsonUtil;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.sql.PreparedStatement;
import java.sql.Statement;
import java.util.Arrays;
import java.util.List;
@ -34,8 +25,17 @@ public class H2EngineProvider extends EngineProvider {
}
@Override
public String insertSql(String name, List<String[]> dataList, int page, int pageNumber) {
String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", name);
public String insertSql(String tableName, DatasourceServer.UpdateType extractType, List<String[]> dataList, int page, int pageNumber,List<TableField> tableFields) {
String engineTableName;
switch (extractType) {
case all_scope:
engineTableName = TableUtils.tmpName(TableUtils.tableName(tableName));
break;
default:
engineTableName = TableUtils.tableName(tableName);
break;
}
String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", engineTableName);
StringBuffer values = new StringBuffer();
Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
@ -88,7 +88,11 @@ public class H2EngineProvider extends EngineProvider {
int size = tableField.getPrecision() * 4;
switch (tableField.getDeType()) {
case 0:
columnFields.append("varchar(2048)").append(",`");
if (StringUtils.isNotEmpty(tableField.getLength())) {
columnFields.append("varchar(length)".replace("length", tableField.getLength())).append(",`");
} else {
columnFields.append("longtext").append(",`");
}
break;
case 1:
columnFields.append("varchar(2048)").append(",`");
@ -103,7 +107,7 @@ public class H2EngineProvider extends EngineProvider {
columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
break;
default:
columnFields.append("varchar(2048)").append(",`");
columnFields.append("longtext").append(",`");
break;
}
}

49
core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java
View File

@ -3,21 +3,15 @@ package io.dataease.datasource.provider;
import io.dataease.dataset.utils.TableUtils;
import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
import io.dataease.datasource.request.EngineRequest;
import io.dataease.datasource.type.Mysql;
import io.dataease.extensions.datasource.dto.ConnectionObj;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.datasource.server.DatasourceServer;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.JsonUtil;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.sql.PreparedStatement;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
/**
* @Author gin
@ -37,8 +31,18 @@ public class MysqlEngineProvider extends EngineProvider {
}
@Override
public String insertSql(String name, List<String[]> dataList, int page, int pageNumber) {
String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", name);
public String insertSql(String tableName, DatasourceServer.UpdateType extractType, List<String[]> dataList, int page, int pageNumber, List<TableField> tableFields) {
String engineTableName;
switch (extractType) {
case all_scope:
engineTableName = TableUtils.tmpName(TableUtils.tableName(tableName));
break;
default:
engineTableName = TableUtils.tableName(tableName);
break;
}
String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", engineTableName);
StringBuffer values = new StringBuffer();
Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
@ -54,7 +58,18 @@ public class MysqlEngineProvider extends EngineProvider {
values.append("('").append(String.join("','", Arrays.asList(strings1)))
.append("'),");
}
return (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null");
List<TableField> keys = tableFields.stream().filter(TableField::isPrimaryKey).toList();
List<TableField> notKeys = tableFields.stream().filter(tableField -> !tableField.isPrimaryKey()).toList();
String insetSql = (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null");
if (CollectionUtils.isNotEmpty(keys) && extractType.equals(DatasourceServer.UpdateType.add_scope)) {
insetSql = insetSql + " ON DUPLICATE KEY UPDATE ";
List<String> updateColumes = new ArrayList<>();
for (TableField notKey : notKeys) {
updateColumes.add("column = VALUES(column)".replace("column", notKey.getName()));
}
insetSql = insetSql + updateColumes.stream().collect(Collectors.joining(","));
}
return insetSql;
}
@ -93,7 +108,11 @@ public class MysqlEngineProvider extends EngineProvider {
int size = tableField.getPrecision() * 4;
switch (tableField.getDeExtractType()) {
case 0:
columnFields.append("varchar(1024)").append(",`");
if (StringUtils.isNotEmpty(tableField.getLength())) {
columnFields.append("varchar(length)".replace("length", tableField.getLength())).append(",`");
} else {
columnFields.append("longtext").append(",`");
}
break;
case 1:
columnFields.append("datetime").append(",`");
@ -108,7 +127,7 @@ public class MysqlEngineProvider extends EngineProvider {
columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`");
break;
default:
columnFields.append("varchar(1024)").append(",`");
columnFields.append("longtext").append(",`");
break;
}
}

12
core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java
View File

@ -21,6 +21,7 @@ import io.dataease.dataset.dao.auto.entity.CoreDatasetGroup;
import io.dataease.dataset.dao.auto.mapper.CoreDatasetGroupMapper;
import io.dataease.dataset.manage.*;
import io.dataease.datasource.utils.DatasourceUtils;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Field2SQLObj;
import io.dataease.engine.trans.Order2SQLObj;
@ -349,7 +350,7 @@ public class ExportCenterManage implements BaseExportApi {
startViewTask(exportTask, request);
}
public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request)throws Exception{
public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request) throws Exception {
datasetGroupManage.getDatasetGroupInfoDTO(exportFrom, null);
CoreExportTask exportTask = new CoreExportTask();
exportTask.setId(UUID.randomUUID().toString());
@ -571,8 +572,16 @@ public class ExportCenterManage implements BaseExportApi {
cell.setCellStyle(cellStyle);
detailsSheet.setColumnWidth(j, 255 * 20);
} else {
if ((allFields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || allFields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) {
try {
cell.setCellValue(Double.valueOf(rowData.get(j)));
} catch (Exception e) {
cell.setCellValue(rowData.get(j));
}
} else {
cell.setCellValue(rowData.get(j));
}
}
}
}
}
@ -668,7 +677,6 @@ public class ExportCenterManage implements BaseExportApi {
Sheet detailsSheet = wb.createSheet("数据");
ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes);
}
} else {
//多个sheet
for (int i = 0; i < request.getMultiInfo().size(); i++) {

2
core/core-frontend/src/locales/tw.ts
View File

@ -1073,6 +1073,7 @@ export default {
has_repeat_name: 'API 數據表名稱重復',
has_repeat_field_name: '字段名重復，請修改后再選擇',
primary_key_change: '主鍵不能改變:',
primary_key_length: '主键必须设置长度: ',
api_field_not_empty: '字段不能為空',
success_copy: '復制成功',
valid: '有效',
@ -1125,6 +1126,7 @@ export default {
parse_filed: '解析字段',
field_rename: '重命名',
set_key: '設為主鍵',
length: '字段长度',
select_type: '選擇數據源類型',
sync_table: '同步指定表',
req_completed: '請求成功',

2
core/core-frontend/src/locales/zh-CN.ts
View File

@ -1085,6 +1085,7 @@ export default {
has_repeat_name: 'API 数据表名称重复',
has_repeat_field_name: '字段名重复，请修改后再选择',
primary_key_change: '主键不能改变:',
primary_key_length: '主键必须设置长度: ',
api_field_not_empty: '字段不能为空',
success_copy: '复制成功',
valid: '有效',
@ -1136,6 +1137,7 @@ export default {
end_time: '结束时间',
parse_filed: '解析字段',
set_key: '设为主键',
length: '字段长度',
field_rename: '重命名',
select_type: '选择数据源类型',
sync_table: '同步指定表',

56
core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue
View File

@ -18,6 +18,7 @@ import { iconFieldMap } from '@/components/icon-group/field-list'
export interface Field {
name: string
length: number
value: Array<{}>
checked: boolean
primaryKey: boolean
@ -48,6 +49,7 @@ export interface JsonField {
name: string
checked: false
primaryKey: false
length: string
extField: number
jsonPath: string
type: string
@ -258,7 +260,7 @@ const saveItem = () => {
for (let i = 0; i < apiItem.fields.length; i++) {
if (apiItem.fields[i].primaryKey) {
let find = false
for (let j = 0; j < fields.length - 1; j++) {
for (let j = 0; j < fields.length; j++) {
if (fields[j].name === apiItem.fields[i].name && fields[j].primaryKey) {
find = true
}
@ -268,10 +270,10 @@ const saveItem = () => {
}
}
}
for (let i = 0; i < fields.length - 1; i++) {
for (let i = 0; i < fields.length; i++) {
if (fields[i].primaryKey) {
let find = false
for (let j = i + 1; j < apiItem.fields.length; j++) {
for (let j = 0; j < apiItem.fields.length; j++) {
if (fields[i].name === apiItem.fields[j].name && apiItem.fields[j].primaryKey) {
find = true
}
@ -285,6 +287,12 @@ const saveItem = () => {
ElMessage.error(t('datasource.primary_key_change') + msg)
return
}
} else {
for (let i = 0; i < apiItem.fields.length; i++) {
if (apiItem.fields[i].primaryKey && !apiItem.fields[i].length) {
ElMessage.error(t('datasource.primary_key_length') + apiItem.fields[i].name)
}
}
}
returnAPIItem('returnItem', cloneDeep(apiItem))
edit_api_item.value = false
@ -381,6 +389,14 @@ const disabledByChildren = item => {
}
}
const disabledFieldLength = item => {
if (item.hasOwnProperty('children') && item.children.length > 0) {
return true
} else {
return item.deExtractType !== 0
}
}
const disabledChangeFieldByChildren = item => {
if (apiItem.type == 'params') {
return true
@ -391,6 +407,12 @@ const disabledChangeFieldByChildren = item => {
return false
}
}
const deExtractTypeChange = item => {
if (item.deExtractType !== 0) {
item.length = ''
}
}
const previewData = () => {
showEmpty.value = false
const data = []
@ -491,7 +513,7 @@ defineExpose({
"
v-model="edit_api_item"
custom-class="api-datasource-drawer"
size="840px"
size="1000px"
:before-close="closeEditItem"
direction="rtl"
>
@ -661,7 +683,7 @@ defineExpose({
prop="originName"
:label="t('datasource.parse_filed')"
:show-overflow-tooltip="true"
width="255"
width="200"
>
<template #default="scope">
<el-checkbox
@ -696,6 +718,7 @@ defineExpose({
:disabled="disabledChangeFieldByChildren(scope.row)"
class="select-type"
style="display: inline-block; width: 120px"
@change="deExtractTypeChange(scope.row)"
>
<template #prefix>
<el-icon>
@ -733,12 +756,33 @@ defineExpose({
</template>
</el-table-column>
<el-table-column
prop="length"
:label="t('datasource.length')"
v-if="apiItem.type !== 'params'"
>
<template #default="scope">
<el-input-number
:disabled="disabledFieldLength(scope.row)"
v-model="scope.row.length"
autocomplete="off"
step-strictly
class="text-left edit-all-line"
:min="1"
:max="4096"
:placeholder="t('common.inputText')"
controls-position="right"
type="number"
/>
</template>
</el-table-column>
<el-table-column
prop="primaryKey"
class-name="checkbox-table"
:label="t('datasource.set_key')"
v-if="apiItem.type !== 'params'"
width="155"
width="100"
>
<template #default="scope">
<el-checkbox

1
sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/dto/TableField.java
View File

@ -14,6 +14,7 @@ public class TableField implements Serializable {
private int precision;
private long size;
private int scale;
private String length;
private boolean checked = false;
private boolean primaryKey = false;
private String fieldType;