diff --git a/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java b/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java index 856622cbc1..e30b2a6339 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java @@ -17,6 +17,7 @@ import io.dataease.extensions.datasource.dto.DatasetTableDTO; import io.dataease.extensions.datasource.dto.DatasourceDTO; import io.dataease.extensions.datasource.dto.DatasourceRequest; import io.dataease.extensions.datasource.dto.TableField; +import io.dataease.extensions.datasource.vo.DatasourceConfiguration; import io.dataease.job.schedule.ExtractDataJob; import io.dataease.job.schedule.ScheduleManager; import io.dataease.utils.BeanUtils; @@ -26,7 +27,6 @@ import org.apache.commons.lang3.StringUtils; import org.quartz.JobExecutionContext; import org.quartz.JobKey; import org.quartz.TriggerKey; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import java.util.Date; @@ -50,7 +50,6 @@ public class DatasourceSyncManage { @Resource private CalciteProvider calciteProvider; - @Async public void extractExcelData(CoreDatasource coreDatasource, String type) { if (coreDatasource == null) { LogUtil.error("Can not find CoreDatasource: " + coreDatasource.getName()); @@ -85,7 +84,12 @@ public class DatasourceSyncManage { } datasetTableTaskLog.setTaskStatus(TaskStatus.Error.toString()); datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n Failed to sync datatable: " + datasourceRequest.getTable() + ", " + e.getMessage()); - DEException.throwException(e); + if (e.getMessage().contains("Duplicate entry")) { + DEException.throwException("不能追加主键相同的数据, " + e.getMessage()); + } else { + DEException.throwException(e); + } + } finally { datasourceTaskServer.saveLog(datasetTableTaskLog); } @@ -237,7 +241,7 @@ public class DatasourceSyncManage { totalPage = dataList.size() / pageNumber; } for (int page = 1; page <= totalPage; page++) { - engineRequest.setQuery(engineProvider.insertSql(datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields)); + engineRequest.setQuery(engineProvider.insertSql(DatasourceConfiguration.DatasourceType.API.name(), datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields)); calciteProvider.exec(engineRequest); } } @@ -257,7 +261,7 @@ public class DatasourceSyncManage { totalPage = dataList.size() / pageNumber; } for (int page = 1; page <= totalPage; page++) { - engineRequest.setQuery(engineProvider.insertSql(datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields)); + engineRequest.setQuery(engineProvider.insertSql(DatasourceConfiguration.DatasourceType.Excel.name(), datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields)); calciteProvider.exec(engineRequest); } } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java index 1618a273ac..2b21ca4b1d 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/EngineProvider.java @@ -22,7 +22,7 @@ public abstract class EngineProvider { public abstract String createTableSql(String name, List tableFields, CoreDeEngine engine); - public abstract String insertSql(String tableName, DatasourceServer.UpdateType extractType, List dataList, int page, int pageNumber, List tableFields); + public abstract String insertSql(String dsType, String tableName, DatasourceServer.UpdateType extractType, List dataList, int page, int pageNumber, List tableFields); } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java index ef7624ddca..d3c87940d1 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java @@ -25,7 +25,7 @@ public class H2EngineProvider extends EngineProvider { } @Override - public String insertSql(String tableName, DatasourceServer.UpdateType extractType, List dataList, int page, int pageNumber,List tableFields) { + public String insertSql(String dsType, String tableName, DatasourceServer.UpdateType extractType, List dataList, int page, int pageNumber,List tableFields) { String engineTableName; switch (extractType) { case all_scope: diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java index 588b29a84f..f03536bbbf 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java @@ -31,7 +31,7 @@ public class MysqlEngineProvider extends EngineProvider { } @Override - public String insertSql(String tableName, DatasourceServer.UpdateType extractType, List dataList, int page, int pageNumber, List tableFields) { + public String insertSql(String dsType, String tableName, DatasourceServer.UpdateType extractType, List dataList, int page, int pageNumber, List tableFields) { String engineTableName; switch (extractType) { case all_scope: @@ -62,17 +62,20 @@ public class MysqlEngineProvider extends EngineProvider { values.append("('").append(String.join("','", Arrays.asList(strings1))) .append("'),"); } - List keys = tableFields.stream().filter(tableField -> tableField.isPrimaryKey() && tableField.isChecked()).toList(); - List notKeys = tableFields.stream().filter(tableField -> tableField.isChecked() && !tableField.isPrimaryKey()).toList(); String insetSql = (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null"); - if (CollectionUtils.isNotEmpty(keys) && extractType.equals(DatasourceServer.UpdateType.add_scope)) { - insetSql = insetSql + " ON DUPLICATE KEY UPDATE "; - List updateColumes = new ArrayList<>(); - for (TableField notKey : notKeys) { - updateColumes.add("column = VALUES(column)".replace("column", notKey.getName())); + if (dsType.equalsIgnoreCase("api")) { + List keys = tableFields.stream().filter(tableField -> tableField.isPrimaryKey() && tableField.isChecked()).toList(); + List notKeys = tableFields.stream().filter(tableField -> tableField.isChecked() && !tableField.isPrimaryKey()).toList(); + if (CollectionUtils.isNotEmpty(keys) && extractType.equals(DatasourceServer.UpdateType.add_scope)) { + insetSql = insetSql + " ON DUPLICATE KEY UPDATE "; + List updateColumes = new ArrayList<>(); + for (TableField notKey : notKeys) { + updateColumes.add("column = VALUES(column)".replace("column", notKey.getName())); + } + insetSql = insetSql + updateColumes.stream().collect(Collectors.joining(",")); } - insetSql = insetSql + updateColumes.stream().collect(Collectors.joining(",")); } + return insetSql; }