center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge pull request #3157 from dataease/pr@dev@decimal

Browse Source 
fix(数据集): 抽取数据精度丢失
This commit is contained in:
taojinlong 2022-09-26 10:45:38 +08:00 committed by GitHub
commit 667be5411e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 56 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/main/java/io/dataease/commons/utils/TableUtils.java
View File

@ -26,7 +26,7 @@ public class TableUtils {
return "f_" + Md5Utils.md5(dorisName).substring(8, 24);
}
public static String columnName(String filedName) {
return "C_" + Md5Utils.md5(filedName);
public static String columnName(String fieldName) {
return "C_" + Md5Utils.md5(fieldName);
}
}

3
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@ -173,6 +173,9 @@ public class JdbcProvider extends DefaultJdbcProvider {
}
}
}
if(StringUtils.isNotEmpty(tableField.getFieldType()) && tableField.getFieldType().equalsIgnoreCase("DECIMAL")){
tableField.setAccuracy(Integer.valueOf(resultSet.getString("DECIMAL_DIGITS")));
}
return tableField;
}

4
backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java
View File

@ -72,7 +72,11 @@ public class DorisDDLProvider extends DDLProviderImpl {
Column_Fields.append("bigint").append(",`");
break;
case 3:
if(datasetTableField.getType().equalsIgnoreCase("DECIMAL") && datasetTableField.getAccuracy() != 0){
Column_Fields.append("DECIMAL(" + datasetTableField.getSize() + "," + datasetTableField.getAccuracy() + ")").append(",`");
}else {
Column_Fields.append("DOUBLE").append(",`");
}
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(size))).append(",`");

58
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -1768,16 +1768,16 @@ public class DataSetTableService {
List<DatasetTableField> datasetTableFields = new ArrayList<>();
if (CollectionUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
TableField filed = fields.get(i);
TableField field = fields.get(i);
DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(datasetTableId);
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setName(filed.getRemarks());
datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName()));
datasetTableField.setType(filed.getFieldType());
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(filed.getFieldType()));
datasetTableField.setSize(filed.getFieldSize());
datasetTableField.setOriginName(field.getFieldName());
datasetTableField.setName(field.getRemarks());
datasetTableField.setDataeaseName(TableUtils.columnName(field.getFieldName()));
datasetTableField.setType(field.getFieldType());
datasetTableField.setDeType(transFieldType(field.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(field.getFieldType()));
datasetTableField.setSize(field.getFieldSize());
datasetTableField.setChecked(true);
datasetTableField.setColumnIndex(i);
datasetTableField.setLastSyncTime(System.currentTimeMillis());
@ -1939,44 +1939,46 @@ public class DataSetTableService {
if (CollectionUtils.isNotEmpty(fields)) {
List<String> originNameList = new ArrayList<>();
for (int i = 0; i < fields.size(); i++) {
TableField filed = fields.get(i);
originNameList.add(filed.getFieldName());
TableField field = fields.get(i);
originNameList.add(field.getFieldName());
DatasetTableField datasetTableField = DatasetTableField.builder().build();
// 物理字段名设定为唯一，查询当前数据集下是否已存在该字段，存在则update，不存在则insert
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
// 字段名一致，认为字段没有改变
datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andOriginNameEqualTo(filed.getFieldName());
datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andOriginNameEqualTo(field.getFieldName());
List<DatasetTableField> datasetTableFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample);
if (CollectionUtils.isNotEmpty(datasetTableFields)) {
datasetTableField.setId(datasetTableFields.get(0).getId());
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setType(filed.getFieldType());
datasetTableField.setSize(filed.getFieldSize());
datasetTableField.setOriginName(field.getFieldName());
datasetTableField.setType(field.getFieldType());
datasetTableField.setSize(field.getFieldSize());
datasetTableField.setAccuracy(field.getAccuracy());
if (ObjectUtils.isEmpty(ds)) {
datasetTableField.setDeExtractType(transFieldType(filed.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(field.getFieldType()));
} else {
Integer fieldType = qp.transFieldType(filed.getFieldType());
Integer fieldType = qp.transFieldType(field.getFieldType());
datasetTableField.setDeExtractType(fieldType);
}
} else {
datasetTableField.setTableId(datasetTable.getId());
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setName(filed.getRemarks());
datasetTableField.setOriginName(field.getFieldName());
datasetTableField.setName(field.getRemarks());
if (datasetTable.getMode() == 1 && StringUtils.equalsIgnoreCase("union", datasetTable.getType())) {
datasetTableField.setDataeaseName(filed.getFieldName());
datasetTableField.setDataeaseName(field.getFieldName());
} else {
datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName()));
datasetTableField.setDataeaseName(TableUtils.columnName(field.getFieldName()));
}
datasetTableField.setType(filed.getFieldType());
datasetTableField.setType(field.getFieldType());
if (ObjectUtils.isEmpty(ds)) {
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(filed.getFieldType()));
datasetTableField.setDeType(transFieldType(field.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(field.getFieldType()));
} else {
Integer fieldType = qp.transFieldType(filed.getFieldType());
Integer fieldType = qp.transFieldType(field.getFieldType());
datasetTableField.setDeType(fieldType == 4 ? 2 : (fieldType == 6 ? 0 : fieldType));
datasetTableField.setDeExtractType(fieldType);
}
datasetTableField.setSize(filed.getFieldSize());
datasetTableField.setSize(field.getFieldSize());
datasetTableField.setAccuracy(field.getAccuracy());
datasetTableField.setChecked(true);
datasetTableField.setLastSyncTime(syncTime);
datasetTableField.setExtField(0);
@ -2094,8 +2096,8 @@ public class DataSetTableService {
List<String> sqlFileds = new ArrayList<>();
try {
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName)
.forEach(filed -> {
sqlFileds.add(filed);
.forEach(field -> {
sqlFileds.add(field);
});
} catch (Exception e) {
DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage());
@ -2114,7 +2116,7 @@ public class DataSetTableService {
List<String> sqlFileds = new ArrayList<>();
try {
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName)
.forEach(filed -> sqlFileds.add(filed));
.forEach(field -> sqlFileds.add(field));
} catch (Exception e) {
DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage());
}

16
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -198,25 +198,25 @@ public class ExtractDataService {
saveSuccessLog(datasetTableTaskLog, false);
updateTableStatus(datasetTableId, JobStatus.Completed, execTime);
if (ops.equalsIgnoreCase("替换")) {
List<DatasetTableField> oldFileds = getDatasetTableFields(datasetTable.getId());
List<DatasetTableField> oldFields = getDatasetTableFields(datasetTable.getId());
List<DatasetTableField> toAdd = new ArrayList<>();
List<DatasetTableField> toDelete = new ArrayList<>();
for (DatasetTableField oldFiled : oldFileds) {
for (DatasetTableField oldField : oldFields) {
boolean delete = true;
for (DatasetTableField datasetTableField : datasetTableFields) {
if (oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())) {
if (oldField.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldField.getDeExtractType().equals(datasetTableField.getDeExtractType())) {
delete = false;
}
}
if (delete) {
toDelete.add(oldFiled);
toDelete.add(oldField);
}
}
for (DatasetTableField datasetTableField : datasetTableFields) {
boolean add = true;
for (DatasetTableField oldFiled : oldFileds) {
if (oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())) {
for (DatasetTableField oldField : oldFields) {
if (oldField.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldField.getDeExtractType().equals(datasetTableField.getDeExtractType())) {
add = false;
}
}
@ -887,8 +887,8 @@ public class ExtractDataService {
datasourceRequest.setDatasource(ds);
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> dorisFields = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName).forEach(filed -> {
dorisFields.add(TableUtils.columnName(filed));
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName).forEach(field -> {
dorisFields.add(TableUtils.columnName(field));
});
return String.join(",", dorisFields);
}

12
backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java
View File

@ -54,16 +54,16 @@ public class DirectFieldService implements DataSetFieldService {
@Override
public List<Object> fieldValues(String fieldId, Long userId, Boolean userPermissions, Boolean rowAndColumnMgm) throws Exception {
List<String> filedIds = new ArrayList<>();
filedIds.add(fieldId);
return fieldValues(filedIds, null, userId, userPermissions, false, rowAndColumnMgm);
List<String> fieldIds = new ArrayList<>();
fieldIds.add(fieldId);
return fieldValues(fieldIds, null, userId, userPermissions, false, rowAndColumnMgm);
}
@Override
public List<Object> fieldValues(String fieldId, DeSortDTO sortDTO, Long userId, Boolean userPermissions, Boolean rowAndColumnMgm) throws Exception {
List<String> filedIds = new ArrayList<>();
filedIds.add(fieldId);
return fieldValues(filedIds, sortDTO, userId, userPermissions, false, rowAndColumnMgm);
List<String> fieldIds = new ArrayList<>();
fieldIds.add(fieldId);
return fieldValues(fieldIds, sortDTO, userId, userPermissions, false, rowAndColumnMgm);
}
public List<DeSortField> buildSorts(List<DatasetTableField> allFields, DeSortDTO sortDTO) {

2
backend/src/main/resources/db/migration/V40__1.15.sql
View File

@ -119,6 +119,8 @@ ALTER TABLE `sys_user_assist`
ADD COLUMN `dingtalk_id` varchar(255) NULL COMMENT '钉钉账号' AFTER `wecom_id`,
ADD COLUMN `lark_id` varchar(255) NULL COMMENT '飞书账号' AFTER `dingtalk_id`;
ALTER TABLE `dataset_table_field` ADD COLUMN `accuracy` INT(11) NULL DEFAULT 0 COMMENT '精度' AFTER `last_sync_time`;
INSERT INTO `sys_auth_detail` (`id`, `auth_id`, `privilege_name`, `privilege_type`, `privilege_value`, `privilege_extend`, `remark`, `create_user`, `create_time`, `update_time`, `copy_from`, `copy_id`) VALUES ('d55c4585-3b0a-11ed-8009-0242ac130005', '379c3124-7a30-4c1b-8ae4-de23eaf34b71', 'i18n_auth_grant', 15, 0, 'grant', '基础权限-授权', 'admin', 1663915323000, NULL, NULL, NULL);
INSERT INTO `sys_auth_detail` (`id`, `auth_id`, `privilege_name`, `privilege_type`, `privilege_value`, `privilege_extend`, `remark`, `create_user`, `create_time`, `update_time`, `copy_from`, `copy_id`) VALUES ('d55c4c0d-3b0a-11ed-8009-0242ac130005', '379c3124-7a30-4c1b-8ae4-de23eaf34b71', 'i18n_auth_manage', 3, 0, 'manage', '基础权限-管理', 'admin', 1663915323000, NULL, NULL, NULL);
INSERT INTO `sys_auth_detail` (`id`, `auth_id`, `privilege_name`, `privilege_type`, `privilege_value`, `privilege_extend`, `remark`, `create_user`, `create_time`, `update_time`, `copy_from`, `copy_id`) VALUES ('d55c4d88-3b0a-11ed-8009-0242ac130005', '379c3124-7a30-4c1b-8ae4-de23eaf34b71', 'i18n_auth_use', 1, 1, 'use', '基础权限-使用', 'admin', 1663915323000, NULL, NULL, NULL);