center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: 精简模式抽取csv数据

Browse Source
This commit is contained in:
taojinlong 2023-03-01 16:43:49 +08:00
parent 2f47c2d6dc
commit 80c4cedb7f
2 changed files with 28 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java
View File

@ -56,11 +56,7 @@ public class MysqlDDLProvider extends DDLProviderImpl {
Integer size = datasetTableField.getSize() * 4; Integer size = datasetTableField.getSize() * 4;
switch (datasetTableField.getDeExtractType()) { switch (datasetTableField.getDeExtractType()) {
case 0: case 0:
if (size < 65533) { Column_Fields.append("longtext").append(",`");
Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
}else {
Column_Fields.append("longtext").append(",`");
}
break; break;
case 1: case 1:
size = size < 50? 50 : size; size = size < 50? 50 : size;

27
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -741,6 +741,33 @@ public class ExtractDataService {
totalSheets = excelXlsxReader.totalSheets; totalSheets = excelXlsxReader.totalSheets;
} }
if (StringUtils.equalsIgnoreCase(suffix, "csv")) {
List<TableField> fields = new ArrayList<>();
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(excelSheetData.getPath()), StandardCharsets.UTF_8));
String s = reader.readLine();// first line
String[] split = s.split(",");
for (String s1 : split) {
TableField tableFiled = new TableField();
tableFiled.setFieldName(s1);
tableFiled.setRemarks(s1);
tableFiled.setFieldType("TEXT");
fields.add(tableFiled);
}
List<List<String>> csvData = new ArrayList<>();
String line;
while ((line = reader.readLine()) != null) {
csvData.add(Arrays.asList(line.split(",")));
}
ExcelSheetData csvSheetData = new ExcelSheetData();
String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);
csvSheetData.setFields(fields);
csvSheetData.setData(csvData);
csvSheetData.setExcelLabel(excelSheetData.getExcelLabel());
csvSheetData.setFieldsMd5(Md5Utils.md5(StringUtils.join(fieldArray, ",")));
totalSheets = Arrays.asList(csvSheetData);
}
for (ExcelSheetData sheet : totalSheets) { for (ExcelSheetData sheet : totalSheets) {
if (sheet.getExcelLabel().equalsIgnoreCase(excelSheetData.getExcelLabel())) { if (sheet.getExcelLabel().equalsIgnoreCase(excelSheetData.getExcelLabel())) {
for (List<String> dataItem : sheet.getData()) { for (List<String> dataItem : sheet.getData()) {