github/dataease
github/dataease
3
0
Fork 1
mirror of https://github.com/dataease/dataease.git synced 2025-02-24 19:42:56 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: excel 数据源识别错误问题

Browse Source
This commit is contained in:
taojinlong 2024-08-22 16:53:23 +08:00
parent 5772df1142
commit a1b1b78629
2 changed files with 27 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java
View File

@ -59,6 +59,14 @@ public class SqlparserUtils {
Statement statement = CCJSqlParserUtil.parse(tmpSql);
Select select = (Select) statement;
if(CollectionUtils.isNotEmpty(select.getWithItemsList())){
for (Iterator<WithItem> iter = select.getWithItemsList().iterator(); iter.hasNext(); ) {
WithItem withItem = iter.next();
ParenthesedSelect parenthesedSelect = (ParenthesedSelect) withItem.getSelect();
parenthesedSelect.setSelect((Select) CCJSqlParserUtil.parse(removeVariables(parenthesedSelect.getSelect().toString(), dsType)));
}
}
if (select.getSelectBody() instanceof PlainSelect) {
return handlePlainSelect((PlainSelect) select.getSelectBody(), select, dsType);
} else {
@ -70,7 +78,7 @@ public class SqlparserUtils {
result.append(" ").append(setOperationList.getOperations().get(i).toString()).append(" ");
}
}
return result.toString();
return select.toString();
}
}
@ -412,35 +420,6 @@ public class SqlparserUtils {
}
public static String removeVariables(final String sql) {
String tmpSql = sql;
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(sql);
boolean hasVariables = false;
while (matcher.find()) {
hasVariables = true;
tmpSql = tmpSql.replace(matcher.group(), SubstitutedParams);
}
if (!hasVariables && !tmpSql.contains(SubstitutedParams)) {
return tmpSql;
}
SqlParser.Config config =
SqlParser.config()
.withLex(Lex.JAVA)
.withIdentifierMaxLength(256);
SqlParser sqlParser = SqlParser.create(tmpSql, config);
SqlNode sqlNode;
try {
sqlNode = sqlParser.parseStmt();
} catch (SqlParseException e) {
throw new RuntimeException("使用 Calcite 进行语法分析发生了异常", e);
}
// 递归遍历语法树
getDependencies(sqlNode, false);
return sqlNode.toString();
}
private static void getDependencies(SqlNode sqlNode, Boolean fromOrJoin) {
if (sqlNode == null) {
return;

28
core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java
View File

@ -37,8 +37,9 @@ public class ExcelUtils {
private static String path = "/opt/dataease2.0/data/excel/";
private static ObjectMapper objectMapper = new ObjectMapper();
private static TypeReference<List<TableField>> TableFieldListTypeReference = new TypeReference<List<TableField>>() {
private static TypeReference<List<TableField>> TableFieldListTypeReference = new TypeReference<List<TableField>>() {
};
public static List<DatasetTableDTO> getTables(DatasourceRequest datasourceRequest) throws DEException {
List<DatasetTableDTO> tableDescs = new ArrayList<>();
try {
@ -48,7 +49,7 @@ public class ExcelUtils {
datasetTableDTO.setTableName(rootNode.get(i).get("deTableName").asText());
datasetTableDTO.setName(rootNode.get(i).get("deTableName").asText());
datasetTableDTO.setDatasourceId(datasourceRequest.getDatasource().getId());
datasetTableDTO.setLastUpdateTime(rootNode.get(i).get("lastUpdateTime") == null? datasourceRequest.getDatasource().getCreateTime(): rootNode.get(i).get("lastUpdateTime").asLong(0L));
datasetTableDTO.setLastUpdateTime(rootNode.get(i).get("lastUpdateTime") == null ? datasourceRequest.getDatasource().getCreateTime() : rootNode.get(i).get("lastUpdateTime").asLong(0L));
tableDescs.add(datasetTableDTO);
}
} catch (Exception e) {
@ -58,12 +59,12 @@ public class ExcelUtils {
return tableDescs;
}
public static Map<String,String> getTableNamesMap(String configration) throws DEException {
Map<String,String> result = new HashMap<>();
public static Map<String, String> getTableNamesMap(String configration) throws DEException {
Map<String, String> result = new HashMap<>();
try {
JsonNode rootNode = objectMapper.readTree(configration);
for (int i = 0; i < rootNode.size(); i++) {
result.put(rootNode.get(i).get("tableName").asText(),rootNode.get(i).get("deTableName").asText());
result.put(rootNode.get(i).get("tableName").asText(), rootNode.get(i).get("deTableName").asText());
}
} catch (Exception e) {
DEException.throwException(e);
@ -293,7 +294,7 @@ public class ExcelUtils {
cells.add(str);
}
if (!isEmpty(cells)) {
if(cells.size() > size){
if (cells.size() > size) {
cells = cells.subList(0, size);
}
data.add(cells.toArray(new String[]{}));
@ -307,7 +308,14 @@ public class ExcelUtils {
}
private String cellType(String value) {
if(value.length()> 19){
if (StringUtils.isEmpty(value) || value.length() > 19) {
return "TEXT";
}
if (value.length() > 1 && value.startsWith("0")) {
return "TEXT";
}
String regex = "^\\d+(\\.\\d+)?$";
if (!value.matches(regex)) {
return "TEXT";
}
try {
@ -331,9 +339,9 @@ public class ExcelUtils {
tableFiled.setFieldType(cellType(value));
} else {
String type = cellType(value);
if(tableFiled.getFieldType() == null){
if (tableFiled.getFieldType() == null) {
tableFiled.setFieldType(type);
}else {
} else {
if (type.equalsIgnoreCase("TEXT")) {
tableFiled.setFieldType(type);
}
@ -454,7 +462,7 @@ public class ExcelUtils {
String[] split = s.split(",");
for (int i = 0; i < split.length; i++) {
String filedName = split[i];
if(StringUtils.isEmpty(filedName)){
if (StringUtils.isEmpty(filedName)) {
DEException.throwException("首行行中不允许有空单元格！");
}
if (filedName.startsWith(UFEFF)) {