From b972458406a9eff00556c57c50b5e4c03a7403c3 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Fri, 22 Nov 2024 17:32:37 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E3=80=90API=E6=95=B0=E6=8D=AE=E6=BA=90?= =?UTF-8?q?=E3=80=91API=E6=95=B0=E6=8D=AE=E6=BA=90=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E4=B8=BB=E9=94=AE=EF=BC=8C=E5=A2=9E=E9=87=8F?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=8F=AF=E6=A0=B9=E6=8D=AE=E4=B8=BB=E9=94=AE?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=B7=B2=E5=90=8C=E6=AD=A5=E7=9A=84=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chart/server/ChartDataServer.java | 6 +- .../manage/DatasourceSyncManage.java | 25 +++------ .../datasource/provider/ApiUtils.java | 1 + .../datasource/provider/EngineProvider.java | 3 +- .../datasource/provider/H2EngineProvider.java | 32 ++++++----- .../provider/MysqlEngineProvider.java | 49 +++++++++++----- .../manage/ExportCenterManage.java | 14 ++++- core/core-frontend/src/locales/tw.ts | 2 + core/core-frontend/src/locales/zh-CN.ts | 2 + .../datasource/form/ApiHttpRequestDraw.vue | 56 +++++++++++++++++-- .../extensions/datasource/dto/TableField.java | 1 + 11 files changed, 133 insertions(+), 58 deletions(-) diff --git a/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java b/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java index c7b36bb416..119dd15667 100644 --- a/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java +++ b/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java @@ -394,7 +394,11 @@ public class ChartDataServer implements ChartDataApi { detailsSheet.setColumnWidth(j, 255 * 20); } else if (cellValObj != null) { try { - cell.setCellValue(cellValObj.toString()); + if ((excelTypes[j].equals(DeTypeConstants.DE_INT) || excelTypes[j].equals(DeTypeConstants.DE_FLOAT)) && StringUtils.isNotEmpty(cellValObj.toString())) { + cell.setCellValue(Double.valueOf(cellValObj.toString())); + } else if (cellValObj != null) { + cell.setCellValue(cellValObj.toString()); + } } catch (Exception e) { LogUtil.warn("export excel data transform error"); } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java b/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java index a9f22716b9..9807a444bd 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java @@ -70,7 +70,7 @@ public class DatasourceSyncManage { if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields); } - extractExcelData(datasourceRequest, updateType); + extractExcelData(datasourceRequest, updateType, tableFields); if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { replaceTable(datasourceRequest.getTable()); } @@ -145,7 +145,7 @@ public class DatasourceSyncManage { if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields); } - extractApiData(datasourceRequest, updateType); + extractApiData(datasourceRequest, updateType, tableFields); if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { replaceTable(datasourceRequest.getTable()); } @@ -198,7 +198,7 @@ public class DatasourceSyncManage { if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields); } - extractApiData(datasourceRequest, updateType); + extractApiData(datasourceRequest, updateType, tableFields); if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { replaceTable(datasourceRequest.getTable()); } @@ -222,20 +222,10 @@ public class DatasourceSyncManage { } } - private void extractApiData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType) throws Exception { + private void extractApiData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType, List tableFields) throws Exception { Map result = ApiUtils.fetchResultField(datasourceRequest); List dataList = (List) result.get("dataList"); - String engineTableName; - switch (extractType) { - case all_scope: - engineTableName = TableUtils.tmpName(TableUtils.tableName(datasourceRequest.getTable())); - break; - default: - engineTableName = TableUtils.tableName(datasourceRequest.getTable()); - break; - } CoreDeEngine engine = engineManage.info(); - EngineRequest engineRequest = new EngineRequest(); engineRequest.setEngine(engine); EngineProvider engineProvider = ProviderUtil.getEngineProvider(engine.getType()); @@ -246,14 +236,13 @@ public class DatasourceSyncManage { } else { totalPage = dataList.size() / pageNumber; } - for (int page = 1; page <= totalPage; page++) { - engineRequest.setQuery(engineProvider.insertSql(engineTableName, dataList, page, pageNumber)); + engineRequest.setQuery(engineProvider.insertSql(datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields)); calciteProvider.exec(engineRequest); } } - private void extractExcelData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType) throws Exception { + private void extractExcelData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType, List tableFields) throws Exception { ExcelUtils excelUtils = new ExcelUtils(); List dataList = excelUtils.fetchDataList(datasourceRequest); String engineTableName; @@ -278,7 +267,7 @@ public class DatasourceSyncManage { totalPage = dataList.size() / pageNumber; } for (int page = 1; page <= totalPage; page++) { - engineRequest.setQuery(engineProvider.insertSql(engineTableName, dataList, page, pageNumber)); + engineRequest.setQuery(engineProvider.insertSql(engineTableName, extractType, dataList, page, pageNumber, tableFields)); calciteProvider.exec(engineRequest); } } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java index b55cb447d1..9a6ca85974 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java @@ -580,6 +580,7 @@ public class ApiUtils { o.put("checked", true); o.put("name", field.getName()); o.put("primaryKey", field.isPrimaryKey()); + o.put("length", field.getLength()); o.put("deExtractType", field.getDeExtractType()); } } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java index 695c9decc7..1618a273ac 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java @@ -2,6 +2,7 @@ package io.dataease.datasource.provider; import io.dataease.datasource.dao.auto.entity.CoreDeEngine; import io.dataease.datasource.request.EngineRequest; +import io.dataease.datasource.server.DatasourceServer; import io.dataease.extensions.datasource.dto.TableField; import java.util.List; @@ -21,7 +22,7 @@ public abstract class EngineProvider { public abstract String createTableSql(String name, List tableFields, CoreDeEngine engine); - public abstract String insertSql(String name, List dataList, int page, int pageNumber); + public abstract String insertSql(String tableName, DatasourceServer.UpdateType extractType, List dataList, int page, int pageNumber, List tableFields); } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java index f63f418631..b0c9adedd1 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java @@ -3,20 +3,11 @@ package io.dataease.datasource.provider; import io.dataease.dataset.utils.TableUtils; import io.dataease.datasource.dao.auto.entity.CoreDeEngine; -import io.dataease.datasource.request.EngineRequest; -import io.dataease.datasource.type.H2; -import io.dataease.datasource.type.Mysql; -import io.dataease.extensions.datasource.dto.ConnectionObj; -import io.dataease.extensions.datasource.dto.DatasourceDTO; +import io.dataease.datasource.server.DatasourceServer; import io.dataease.extensions.datasource.dto.TableField; -import io.dataease.extensions.datasource.vo.DatasourceConfiguration; -import io.dataease.utils.BeanUtils; -import io.dataease.utils.JsonUtil; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; -import java.sql.PreparedStatement; -import java.sql.Statement; import java.util.Arrays; import java.util.List; @@ -34,8 +25,17 @@ public class H2EngineProvider extends EngineProvider { } @Override - public String insertSql(String name, List dataList, int page, int pageNumber) { - String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", name); + public String insertSql(String tableName, DatasourceServer.UpdateType extractType, List dataList, int page, int pageNumber,List tableFields) { + String engineTableName; + switch (extractType) { + case all_scope: + engineTableName = TableUtils.tmpName(TableUtils.tableName(tableName)); + break; + default: + engineTableName = TableUtils.tableName(tableName); + break; + } + String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", engineTableName); StringBuffer values = new StringBuffer(); Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size(); @@ -88,7 +88,11 @@ public class H2EngineProvider extends EngineProvider { int size = tableField.getPrecision() * 4; switch (tableField.getDeType()) { case 0: - columnFields.append("varchar(2048)").append(",`"); + if (StringUtils.isNotEmpty(tableField.getLength())) { + columnFields.append("varchar(length)".replace("length", tableField.getLength())).append(",`"); + } else { + columnFields.append("longtext").append(",`"); + } break; case 1: columnFields.append("varchar(2048)").append(",`"); @@ -103,7 +107,7 @@ public class H2EngineProvider extends EngineProvider { columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`"); break; default: - columnFields.append("varchar(2048)").append(",`"); + columnFields.append("longtext").append(",`"); break; } } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java index b45e28c5fd..5edc98e7d3 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java @@ -3,21 +3,15 @@ package io.dataease.datasource.provider; import io.dataease.dataset.utils.TableUtils; import io.dataease.datasource.dao.auto.entity.CoreDeEngine; -import io.dataease.datasource.request.EngineRequest; -import io.dataease.datasource.type.Mysql; -import io.dataease.extensions.datasource.dto.ConnectionObj; -import io.dataease.extensions.datasource.dto.DatasourceDTO; +import io.dataease.datasource.server.DatasourceServer; import io.dataease.extensions.datasource.dto.TableField; -import io.dataease.extensions.datasource.vo.DatasourceConfiguration; -import io.dataease.utils.BeanUtils; -import io.dataease.utils.JsonUtil; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; - -import java.sql.PreparedStatement; -import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; /** * @Author gin @@ -37,8 +31,18 @@ public class MysqlEngineProvider extends EngineProvider { } @Override - public String insertSql(String name, List dataList, int page, int pageNumber) { - String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", name); + public String insertSql(String tableName, DatasourceServer.UpdateType extractType, List dataList, int page, int pageNumber, List tableFields) { + String engineTableName; + switch (extractType) { + case all_scope: + engineTableName = TableUtils.tmpName(TableUtils.tableName(tableName)); + break; + default: + engineTableName = TableUtils.tableName(tableName); + break; + } + + String insertSql = "INSERT INTO `TABLE_NAME` VALUES ".replace("TABLE_NAME", engineTableName); StringBuffer values = new StringBuffer(); Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size(); @@ -54,7 +58,18 @@ public class MysqlEngineProvider extends EngineProvider { values.append("('").append(String.join("','", Arrays.asList(strings1))) .append("'),"); } - return (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null"); + List keys = tableFields.stream().filter(TableField::isPrimaryKey).toList(); + List notKeys = tableFields.stream().filter(tableField -> !tableField.isPrimaryKey()).toList(); + String insetSql = (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null"); + if (CollectionUtils.isNotEmpty(keys) && extractType.equals(DatasourceServer.UpdateType.add_scope)) { + insetSql = insetSql + " ON DUPLICATE KEY UPDATE "; + List updateColumes = new ArrayList<>(); + for (TableField notKey : notKeys) { + updateColumes.add("column = VALUES(column)".replace("column", notKey.getName())); + } + insetSql = insetSql + updateColumes.stream().collect(Collectors.joining(",")); + } + return insetSql; } @@ -93,7 +108,11 @@ public class MysqlEngineProvider extends EngineProvider { int size = tableField.getPrecision() * 4; switch (tableField.getDeExtractType()) { case 0: - columnFields.append("varchar(1024)").append(",`"); + if (StringUtils.isNotEmpty(tableField.getLength())) { + columnFields.append("varchar(length)".replace("length", tableField.getLength())).append(",`"); + } else { + columnFields.append("longtext").append(",`"); + } break; case 1: columnFields.append("datetime").append(",`"); @@ -108,7 +127,7 @@ public class MysqlEngineProvider extends EngineProvider { columnFields.append("TINYINT(length)".replace("length", String.valueOf(tableField.getPrecision()))).append(",`"); break; default: - columnFields.append("varchar(1024)").append(",`"); + columnFields.append("longtext").append(",`"); break; } } diff --git a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java index 8a33f1c07f..4770e47bec 100644 --- a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java +++ b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java @@ -21,6 +21,7 @@ import io.dataease.dataset.dao.auto.entity.CoreDatasetGroup; import io.dataease.dataset.dao.auto.mapper.CoreDatasetGroupMapper; import io.dataease.dataset.manage.*; import io.dataease.datasource.utils.DatasourceUtils; +import io.dataease.engine.constant.DeTypeConstants; import io.dataease.engine.sql.SQLProvider; import io.dataease.engine.trans.Field2SQLObj; import io.dataease.engine.trans.Order2SQLObj; @@ -349,7 +350,7 @@ public class ExportCenterManage implements BaseExportApi { startViewTask(exportTask, request); } - public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request)throws Exception{ + public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request) throws Exception { datasetGroupManage.getDatasetGroupInfoDTO(exportFrom, null); CoreExportTask exportTask = new CoreExportTask(); exportTask.setId(UUID.randomUUID().toString()); @@ -571,7 +572,15 @@ public class ExportCenterManage implements BaseExportApi { cell.setCellStyle(cellStyle); detailsSheet.setColumnWidth(j, 255 * 20); } else { - cell.setCellValue(rowData.get(j)); + if ((allFields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || allFields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) { + try { + cell.setCellValue(Double.valueOf(rowData.get(j))); + } catch (Exception e) { + cell.setCellValue(rowData.get(j)); + } + } else { + cell.setCellValue(rowData.get(j)); + } } } } @@ -668,7 +677,6 @@ public class ExportCenterManage implements BaseExportApi { Sheet detailsSheet = wb.createSheet("数据"); ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); } - } else { //多个sheet for (int i = 0; i < request.getMultiInfo().size(); i++) { diff --git a/core/core-frontend/src/locales/tw.ts b/core/core-frontend/src/locales/tw.ts index 589fe44d3b..e2b50cacf8 100644 --- a/core/core-frontend/src/locales/tw.ts +++ b/core/core-frontend/src/locales/tw.ts @@ -1073,6 +1073,7 @@ export default { has_repeat_name: 'API 數據表名稱重復', has_repeat_field_name: '字段名重復，請修改后再選擇', primary_key_change: '主鍵不能改變:', + primary_key_length: '主键必须设置长度: ', api_field_not_empty: '字段不能為空', success_copy: '復制成功', valid: '有效', @@ -1125,6 +1126,7 @@ export default { parse_filed: '解析字段', field_rename: '重命名', set_key: '設為主鍵', + length: '字段长度', select_type: '選擇數據源類型', sync_table: '同步指定表', req_completed: '請求成功', diff --git a/core/core-frontend/src/locales/zh-CN.ts b/core/core-frontend/src/locales/zh-CN.ts index 8edf5cef03..9f8905d1b3 100644 --- a/core/core-frontend/src/locales/zh-CN.ts +++ b/core/core-frontend/src/locales/zh-CN.ts @@ -1085,6 +1085,7 @@ export default { has_repeat_name: 'API 数据表名称重复', has_repeat_field_name: '字段名重复，请修改后再选择', primary_key_change: '主键不能改变:', + primary_key_length: '主键必须设置长度: ', api_field_not_empty: '字段不能为空', success_copy: '复制成功', valid: '有效', @@ -1136,6 +1137,7 @@ export default { end_time: '结束时间', parse_filed: '解析字段', set_key: '设为主键', + length: '字段长度', field_rename: '重命名', select_type: '选择数据源类型', sync_table: '同步指定表', diff --git a/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue b/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue index 122e56e8e3..f43363b95c 100644 --- a/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue +++ b/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue @@ -18,6 +18,7 @@ import { iconFieldMap } from '@/components/icon-group/field-list' export interface Field { name: string + length: number value: Array<{}> checked: boolean primaryKey: boolean @@ -48,6 +49,7 @@ export interface JsonField { name: string checked: false primaryKey: false + length: string extField: number jsonPath: string type: string @@ -258,7 +260,7 @@ const saveItem = () => { for (let i = 0; i < apiItem.fields.length; i++) { if (apiItem.fields[i].primaryKey) { let find = false - for (let j = 0; j < fields.length - 1; j++) { + for (let j = 0; j < fields.length; j++) { if (fields[j].name === apiItem.fields[i].name && fields[j].primaryKey) { find = true } @@ -268,10 +270,10 @@ const saveItem = () => { } } } - for (let i = 0; i < fields.length - 1; i++) { + for (let i = 0; i < fields.length; i++) { if (fields[i].primaryKey) { let find = false - for (let j = i + 1; j < apiItem.fields.length; j++) { + for (let j = 0; j < apiItem.fields.length; j++) { if (fields[i].name === apiItem.fields[j].name && apiItem.fields[j].primaryKey) { find = true } @@ -285,6 +287,12 @@ const saveItem = () => { ElMessage.error(t('datasource.primary_key_change') + msg) return } + } else { + for (let i = 0; i < apiItem.fields.length; i++) { + if (apiItem.fields[i].primaryKey && !apiItem.fields[i].length) { + ElMessage.error(t('datasource.primary_key_length') + apiItem.fields[i].name) + } + } } returnAPIItem('returnItem', cloneDeep(apiItem)) edit_api_item.value = false @@ -381,6 +389,14 @@ const disabledByChildren = item => { } } +const disabledFieldLength = item => { + if (item.hasOwnProperty('children') && item.children.length > 0) { + return true + } else { + return item.deExtractType !== 0 + } +} + const disabledChangeFieldByChildren = item => { if (apiItem.type == 'params') { return true @@ -391,6 +407,12 @@ const disabledChangeFieldByChildren = item => { return false } } + +const deExtractTypeChange = item => { + if (item.deExtractType !== 0) { + item.length = '' + } +} const previewData = () => { showEmpty.value = false const data = [] @@ -491,7 +513,7 @@ defineExpose({ " v-model="edit_api_item" custom-class="api-datasource-drawer" - size="840px" + size="1000px" :before-close="closeEditItem" direction="rtl" > @@ -661,7 +683,7 @@ defineExpose({ prop="originName" :label="t('datasource.parse_filed')" :show-overflow-tooltip="true" - width="255" + width="200" >