center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: 定时同步任务支持在历史数据变动后增量更新到doris中

Browse Source
This commit is contained in:
taojinlong 2024-01-24 16:51:45 +08:00
parent 42c85bdc82
commit 3df5e3e2e9
3 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java
View File

@ -370,6 +370,9 @@ public class ExcelXlsReader implements HSSFListener {
private String checkType(String str, int thisColumn) {
if (str.length() > 19) {
return "TEXT";
}
String type = null;
try {
double d = Double.valueOf(str);

3
core/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -2949,6 +2949,9 @@ public class DataSetTableService {
}
private String cellType(String value) {
if (value.length() > 19) {
return "TEXT";
}
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sdf.parse(value);

16
core/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -479,9 +479,10 @@ public class ExtractDataService {
}
private void extractApiData(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType) throws Exception {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
List<ApiDefinition> lists = new Gson().fromJson(datasource.getConfiguration(), new TypeToken<ArrayList<ApiDefinition>>() {
}.getType());
lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable())).collect(Collectors.toList());
lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(dataTableInfoDTO.getTable())).collect(Collectors.toList());
if (CollectionUtils.isEmpty(lists)) {
throw new Exception("未找到API数据表");
}
@ -521,15 +522,16 @@ public class ExtractDataService {
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), dataFile);
break;
}
BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile));
for (String[] strings : dataList) {
String content = "";
for (int i = 0; i < strings.length; i++) {
content = content + strings[i] + separator;
content = i != strings.length - 1 ? content + strings[i] + separator : content + strings[i];
}
boolean isSetKey = dataTableInfoDTO.isSetKey() && CollectionUtils.isNotEmpty(dataTableInfoDTO.getKeys());
if (!isSetKey) {
content = Md5Utils.md5(content) + separator + content;
}
content = content + Md5Utils.md5(content);
bw.write(content);
bw.newLine();
}
@ -565,8 +567,8 @@ public class ExtractDataService {
} catch (Exception e) {
throw e;
} finally {
File deleteFile = new File(root_path + datasetTable.getId() + ".sh");
FileUtils.forceDelete(deleteFile);
// File deleteFile = new File(root_path + datasetTable.getId() + ".sh");
// FileUtils.forceDelete(deleteFile);
}
}