center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge branch 'dev' into pr@dev_st_fix

Browse Source
This commit is contained in:
dataeaseShu 2023-05-10 11:09:45 +08:00
commit b238d64fcb
8 changed files with 47 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java
View File

@ -2,6 +2,7 @@ package io.dataease.commons.utils;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.i18n.Translator;
import io.dataease.plugins.common.base.domain.DatasetTableField;
import io.dataease.plugins.common.dto.datasource.TableField;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.poi.openxml4j.opc.OPCPackage;
@ -109,6 +110,16 @@ public class ExcelXlsxReader extends DefaultHandler {
//定义该文档一行最大的单元格数，用来补全一行最后可能缺失的单元格
private String maxRef = null;
public List<DatasetTableField> getDatasetTableFields() {
return datasetTableFields;
}
public void setDatasetTableFields(List<DatasetTableField> datasetTableFields) {
this.datasetTableFields = datasetTableFields;
}
private List<DatasetTableField> datasetTableFields = null;
/**
* 单元格
*/
@ -418,10 +429,10 @@ public class ExcelXlsxReader extends DefaultHandler {
break;
case NUMBER: //数字
if (formatString != null && isDateFormat) {
if (obtainedNum != null) {
thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, formatString).trim();
} else {
if (getDatasetTableFields() != null && getDatasetTableFields().get(curCol).getDeExtractType() == 1) {
thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, "yyyy-mm-dd hh:mm:ss").trim();
} else {
thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, formatString).trim();
}
} else {
thisStr = value;

29
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -197,7 +197,7 @@ public class ExtractDataService {
generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
extractData(datasetTable, "all_scope");
} else {
extractExcelDataForSimpleMode(datasetTable, "all_scope");
extractExcelDataForSimpleMode(datasetTable, "all_scope", datasetTableFields);
}
replaceTable(TableUtils.tableName(datasetTableId));
saveSuccessLog(datasetTableTaskLog, false);
@ -250,7 +250,7 @@ public class ExtractDataService {
generateJobFile("incremental_add", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
extractData(datasetTable, "incremental_add");
} else {
extractExcelDataForSimpleMode(datasetTable, "incremental_add");
extractExcelDataForSimpleMode(datasetTable, "incremental_add", datasetTableFields);
}
saveSuccessLog(datasetTableTaskLog, false);
updateTableStatus(datasetTableId, JobStatus.Completed, execTime);
@ -724,7 +724,7 @@ public class ExtractDataService {
return datasetTableTaskLog;
}
private void extractExcelDataForSimpleMode(DatasetTable datasetTable, String extractType) throws Exception {
private void extractExcelDataForSimpleMode(DatasetTable datasetTable, String extractType, List<DatasetTableField> datasetTableFields) throws Exception {
List<String[]> data = new ArrayList<>();
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
List<ExcelSheetData> excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList();
@ -738,6 +738,7 @@ public class ExtractDataService {
}
if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) {
ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader();
excelXlsxReader.setDatasetTableFields(datasetTableFields);
excelXlsxReader.process(new FileInputStream(excelSheetData.getPath()));
totalSheets = excelXlsxReader.totalSheets;
}
@ -1310,6 +1311,17 @@ public class ExtractDataService {
.replace("ExcelCompletion", "");
}
String handleMysqlBIGINTUNSIGNEDStr = "";
if (datasourceType.equals(DatasourceTypes.mysql)) {
for (DatasetTableField datasetTableField : datasetTableFields) {
if(datasetTableField.getType().equalsIgnoreCase("BIGINT UNSIGNED")){
handleMysqlBIGINTUNSIGNEDStr = handleMysqlBIGINTUNSIGNEDStr + handleMysqlBIGINTUNSIGNED.replace("BIGINTUNSIGNEDFIELD", datasetTableField.getDataeaseName()) + "; \n";
}
}
}
tmp_code = tmp_code.replace("handleMysqlBIGINTUNSIGNED", handleMysqlBIGINTUNSIGNEDStr);
UserDefinedJavaClassDef userDefinedJavaClassDef = new UserDefinedJavaClassDef(UserDefinedJavaClassDef.ClassType.TRANSFORM_CLASS, "Processor", tmp_code);
userDefinedJavaClassDef.setActive(true);
@ -1403,6 +1415,16 @@ public class ExtractDataService {
" }catch (Exception e){}\n" +
" }";
private final static String handleMysqlBIGINTUNSIGNED = "if(filed.equalsIgnoreCase(\"BIGINTUNSIGNEDFIELD\")){\n" +
"\t\t\tif(tmp != null && tmp.endsWith(\".0\")){\n" +
" \ttry {\n" +
" Long.valueOf(tmp.substring(0, tmp.length()-2));\n" +
" get(Fields.Out, filed).setValue(r, tmp.substring(0, tmp.length()-2));\n" +
" get(Fields.Out, filed).getValueMeta().setType(2);\n" +
" }catch (Exception e){}\n" +
" \t\t}\n" +
"\t\t}";
private final static String handleWraps = " if(tmp != null && ( tmp.contains(\"\\r\") || tmp.contains(\"\\n\"))){\n" +
"\t\t\ttmp = tmp.trim();\n" +
" tmp = tmp.replaceAll(\"\\r\",\" \");\n" +
@ -1445,6 +1467,7 @@ public class ExtractDataService {
" List<String> fields = Arrays.asList(\"Column_Fields\".split(\",\"));\n" +
" for (String filed : fields) {\n" +
" String tmp = get(Fields.In, filed).getString(r);\n" +
"handleMysqlBIGINTUNSIGNED \n" +
"handleCharset \n" +
"handleWraps \n" +
"ExcelCompletion \n" +

2
frontend/src/api/panel/enshrine.js
View File

@ -29,6 +29,6 @@ export function starStatus(panelId) {
return request({
url: '/api/store/status/' + panelId,
method: 'post',
loading: true
loading: false
})
}

3
frontend/src/api/panel/linkJump.js
View File

@ -33,6 +33,7 @@ export function queryTargetPanelJumpInfo(requestInfo) {
export function queryPanelJumpInfo(panelId) {
return request({
url: '/linkJump/queryPanelJumpInfo/' + panelId,
method: 'get'
method: 'get',
loading: false
})
}

3
frontend/src/api/panel/linkage.js
View File

@ -21,6 +21,7 @@ export function saveLinkage(requestInfo) {
export function getPanelAllLinkageInfo(panelId) {
return request({
url: '/linkage/getPanelAllLinkageInfo/' + panelId,
method: 'get'
method: 'get',
loading: false
})
}

2
frontend/src/api/panel/panel.js
View File

@ -97,7 +97,7 @@ export function viewPanelLog(data) {
return request({
url: 'panel/group/viewLog',
method: 'post',
loading: true,
loading: false,
data
})
}

2
frontend/src/api/panel/pdfTemplate.js
View File

@ -4,6 +4,6 @@ export function queryAll() {
return request({
url: '/pdf-template/queryAll',
method: 'get',
loading: true
loading: false
})
}

1
frontend/src/views/dataset/data/UpdateInfo.vue
View File

@ -909,7 +909,6 @@ export default {
}
this.incrementalConfig.tableId = this.table.id
task.startTime = new Date(task.startTime).getTime()
console.log(task.endTime)
task.endTime = new Date(task.endTime).getTime()
task.tableId = this.table.id
const form = JSON.parse(JSON.stringify(task))