center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge branch 'dev' into pr@dev_st_fix

Browse Source
This commit is contained in:
dataeaseShu 2023-05-26 15:41:26 +08:00
commit 889884b1d2
14 changed files with 213 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@ -781,6 +781,9 @@ public class JdbcProvider extends DefaultJdbcProvider {
case StarRocks:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfiguration.class);
mysqlConfiguration.getJdbc();
if(!mysqlConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
throw new Exception("Invalid database name");
}
break;
case redshift:
RedshiftConfiguration redshiftConfiguration = new Gson().fromJson(datasource.getConfiguration(), RedshiftConfiguration.class);
@ -791,6 +794,48 @@ public class JdbcProvider extends DefaultJdbcProvider {
throw new Exception("Invalid database name");
}
break;
case sqlServer:
SqlServerConfiguration sqlServerConfiguration = new Gson().fromJson(datasource.getConfiguration(), SqlServerConfiguration.class);
if(!sqlServerConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
throw new Exception("Invalid database name");
}
break;
case pg:
PgConfiguration pgConfiguration = new Gson().fromJson(datasource.getConfiguration(), PgConfiguration.class);
if(!pgConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
throw new Exception("Invalid database name");
}
break;
case oracle:
OracleConfiguration oracleConfiguration = new Gson().fromJson(datasource.getConfiguration(), OracleConfiguration.class);
if(!oracleConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$") && !oracleConfiguration.getConnectionType().equalsIgnoreCase("serviceName")){
throw new Exception("Invalid database name");
}
break;
case mongo:
MongodbConfiguration mongodbConfiguration = new Gson().fromJson(datasource.getConfiguration(), MongodbConfiguration.class);
if(!mongodbConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
throw new Exception("Invalid database name");
}
break;
case impala:
ImpalaConfiguration impalaConfiguration = new Gson().fromJson(datasource.getConfiguration(), ImpalaConfiguration.class);
if(!impalaConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
throw new Exception("Invalid database name");
}
break;
case hive:
HiveConfiguration hiveConfiguration = new Gson().fromJson(datasource.getConfiguration(), HiveConfiguration.class);
if(!hiveConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
throw new Exception("Invalid database name");
}
break;
case db2:
Db2Configuration db2Configuration = new Gson().fromJson(datasource.getConfiguration(), Db2Configuration.class);
if(!db2Configuration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
throw new Exception("Invalid database name");
}
break;
default:
break;
}

19
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java
View File

@ -41,10 +41,7 @@ public class MysqlDDLProvider extends DDLProviderImpl {
strings1[i] = null;
continue;
}
strings1[i] = strings[i].replace("'", "\\'");
if(strings1[i].equals("/")){
strings1[i] = "//";
}
strings1[i] = strings[i].replace("\\", "\\\\").replace("'", "\\'");
}
values.append("('").append(UUID.randomUUID())
.append("','").append(String.join("','", Arrays.asList(strings1)))
@ -90,27 +87,19 @@ public class MysqlDDLProvider extends DDLProviderImpl {
break;
case 1:
size = size < 50? 50 : size;
if (size < 65533) {
Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
}else {
Column_Fields.append("longtext").append(",`");
}
Column_Fields.append("longtext").append(",`");
break;
case 2:
Column_Fields.append("bigint(20)").append(",`");
break;
case 3:
Column_Fields.append("varchar(100)").append(",`");
Column_Fields.append("longtext").append(",`");
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
break;
default:
if (size < 65533) {
Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
}else {
Column_Fields.append("longtext").append(",`");
}
Column_Fields.append("longtext").append(",`");
break;
}
}

14
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -2454,7 +2454,19 @@ public class DataSetTableService {
if (num > 1000) {
break;
}
data.add(Arrays.asList(line.split(",")));
String str;
line += ",";
Pattern pCells = Pattern.compile("(\"[^\"]*(\"{2})*[^\"]*\")*[^,]*,");
Matcher mCells = pCells.matcher(line);
List<String> cells = new ArrayList();//每行记录一个list
//读取每个单元格
while (mCells.find()) {
str = mCells.group();
str = str.replaceAll("(?sm)\"?([^\"]*(\"{2})*[^\"]*)\"?.*,", "$1");
str = str.replaceAll("(?sm)(\"(\"))", "$2");
cells.add(str);
}
data.add(cells);
num++;
}
ExcelSheetData excelSheetData = new ExcelSheetData();

21
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -72,6 +72,8 @@ import javax.annotation.Resource;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Service
@ -758,13 +760,20 @@ public class ExtractDataService {
List<List<String>> csvData = new ArrayList<>();
String line;
while ((line = reader.readLine()) != null) {
if(line.endsWith(",")){
List<String> list = new ArrayList<>(Arrays.asList(line.split(",")));
list.add("");
csvData.add(list);
}else {
csvData.add(Arrays.asList(line.split(",")));
String str;
line += ",";
Pattern pCells = Pattern.compile("(\"[^\"]*(\"{2})*[^\"]*\")*[^,]*,");
Matcher mCells = pCells.matcher(line);
List<String> cells = new ArrayList();//每行记录一个list
//读取每个单元格
while (mCells.find()) {
str = mCells.group();
str = str.replaceAll("(?sm)\"?([^\"]*(\"{2})*[^\"]*)\"?.*,", "$1");
str = str.replaceAll("(?sm)(\"(\"))", "$2");
cells.add(str);
}
csvData.add(cells);
}
ExcelSheetData csvSheetData = new ExcelSheetData();
String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);

1
frontend/src/lang/en.js
View File

@ -1489,6 +1489,7 @@ export default {
total_sort_desc: 'DESC',
total_sort_field: 'Sort Field',
empty_data_strategy: 'Empty Data Strategy',
empty_data_field_ctrl: 'Field Control',
break_line: 'Keep',
set_zero: 'Set Zero',
ignore_data: 'Hide Data',

1
frontend/src/lang/tw.js
View File

@ -1491,6 +1491,7 @@ export default {
break_line: '保持為空',
set_zero: '置為0',
ignore_data: '隱藏空值',
empty_data_field_ctrl: '字段設置',
sub_dimension_tip: '該字段為必填項，且不應使用類別軸中的字段，若無需該字段，請選擇基礎圖表進行展示，否則展示效果不理想',
drill_dimension_tip: '鑽取字段僅支持數據集中的字段',
table_scroll_tip: '明細表僅在分頁模式為"下拉"時生效。',

1
frontend/src/lang/zh.js
View File

@ -1487,6 +1487,7 @@ export default {
total_sort_desc: '降序',
total_sort_field: '排序字段',
empty_data_strategy: '空值处理',
empty_data_field_ctrl: '字段设置',
break_line: '保持为空',
set_zero: '置为0',
ignore_data: '隐藏空值',

3
frontend/src/views/chart/chart/chart.js
View File

@ -467,7 +467,8 @@ export const DEFAULT_FUNCTION_CFG = {
sliderBg: '#FFFFFF',
sliderFillBg: '#BCD6F1',
sliderTextClolor: '#999999',
emptyDataStrategy: 'breakLine'
emptyDataStrategy: 'breakLine',
emptyDataFieldCtrl: []
}
export const DEFAULT_THRESHOLD = {
gaugeThreshold: '',

18
frontend/src/views/chart/chart/map/map.js
View File

@ -38,7 +38,7 @@ const fillGradientColor = (data, colors) => {
})
return data
}
export function baseMapOption(chart_option, chart, themeStyle, curAreaCode, seriesId) {
export function baseMapOption(chart_option, geoJson, chart, themeStyle, curAreaCode, seriesId) {
// 处理shape attr
let customAttr = {}
let isGradient = false
@ -151,18 +151,16 @@ export function baseMapOption(chart_option, chart, themeStyle, curAreaCode, seri
if (senior) {
senior = JSON.parse(senior)
}
// 空值处理，echarts 对于值为 null 的默认策略是不展示，也就是保持为空，所以只需要处理忽略数据和置为 0 就行
// 隐藏和不展示的区别是隐藏不会参与颜色分布的计算，而不展示会参与颜色计算
// 空值处理，echarts 对于值为 null 的默认策略是不展示，也就是保持为空，所以只需要处理置为 0 就行
let emptyDataStrategy = senior?.functionCfg?.emptyDataStrategy
if (!emptyDataStrategy) {
emptyDataStrategy = 'breakLine'
}
const subArea = new Set(geoJson.features.map(item => item.properties.name))
for (let i = 0; i < valueArr.length; i++) {
const y = valueArr[i]
if (y.value === null && emptyDataStrategy === 'ignoreData') {
continue
}
y.name = chart.data.x[i]
subArea.delete(y.name)
if (y.value === null && emptyDataStrategy === 'setZero') {
const tmp = _.clone(y)
tmp.value = 0
@ -171,6 +169,14 @@ export function baseMapOption(chart_option, chart, themeStyle, curAreaCode, seri
}
chart_option.series[0].data.push(y)
}
if (emptyDataStrategy === 'setZero' && subArea.size > 0) {
subArea.forEach(item => {
chart_option.series[0].data.push({
name: item,
value: 0
})
})
}
if (isGradient) {
chart_option.series[0].data = fillGradientColor(chart_option.series[0].data, customAttr.color.colors)

16
frontend/src/views/chart/chart/table/table-info.js
View File

@ -2,7 +2,7 @@ import { TableSheet, S2Event, PivotSheet, DataCell, EXTRA_FIELD, TOTAL_VALUE } f
import { getCustomTheme, getSize } from '@/views/chart/chart/common/common_table'
import { DEFAULT_COLOR_CASE, DEFAULT_TOTAL } from '@/views/chart/chart/chart'
import { formatterItem, valueFormatter } from '@/views/chart/chart/formatter'
import { hexColorToRGBA } from '@/views/chart/chart/util'
import { handleTableEmptyStrategy, hexColorToRGBA } from '@/views/chart/chart/util'
export function baseTableInfo(s2, container, chart, action, tableData, pageInfo) {
const containerDom = document.getElementById(container)
@ -127,14 +127,15 @@ export function baseTableInfo(s2, container, chart, action, tableData, pageInfo)
})
})
}
// 空值处理
const newData = handleTableEmptyStrategy(tableData, chart)
// data config
const s2DataConfig = {
fields: {
columns: columns
},
meta: meta,
data: tableData
data: newData
}
const customAttr = JSON.parse(chart.customAttr)
@ -300,13 +301,15 @@ export function baseTableNormal(s2, container, chart, action, tableData) {
})
}
// 空值处理
const newData = handleTableEmptyStrategy(tableData, chart)
// data config
const s2DataConfig = {
fields: {
columns: columns
},
meta: meta,
data: tableData
data: newData
}
const customAttr = JSON.parse(chart.customAttr)
@ -492,7 +495,8 @@ export function baseTablePivot(s2, container, chart, action, headerAction, table
}
sortParams.push(sort)
}
// 空值处理
const newData = handleTableEmptyStrategy(tableData, chart)
// data config
const s2DataConfig = {
fields: {
@ -501,7 +505,7 @@ export function baseTablePivot(s2, container, chart, action, headerAction, table
values: v
},
meta: meta,
data: tableData,
data: newData,
sortParams: sortParams
}

35
frontend/src/views/chart/chart/util.js
View File

@ -3690,3 +3690,38 @@ export function resetRgbOpacity(sourceColor, times) {
}
return sourceColor
}
export function handleTableEmptyStrategy(tableData, chart) {
let newData = tableData
let intersection = []
let senior = chart.senior
if (senior) {
senior = JSON.parse(senior)
}
let emptyDataStrategy = senior?.functionCfg?.emptyDataStrategy
if (!emptyDataStrategy) {
emptyDataStrategy = 'breakLine'
}
const emptyDataFieldCtrl = senior?.functionCfg?.emptyDataFieldCtrl
if (emptyDataStrategy !== 'breakLine' && emptyDataFieldCtrl?.length && tableData?.length) {
const deNames = _.keys(tableData[0])
intersection = _.intersection(deNames, emptyDataFieldCtrl)
}
if (intersection.length) {
newData = _.clone(tableData)
for (let i = 0; i < newData.length; i++) {
for (let j = 0, tmp = intersection.length; j < tmp; j++) {
const deName = intersection[j]
if (newData[i][deName] === null) {
if (emptyDataStrategy === 'setZero') {
newData[i][deName] = 0
}
if (emptyDataStrategy === 'ignoreData') {
newData = _.filter(newData, (_, index) => index !== i)
}
}
}
}
}
return newData
}

2
frontend/src/views/chart/components/ChartComponent.vue
View File

@ -439,7 +439,7 @@ export default {
this.buttonTextColor = null
}
}
const chart_option = baseMapOption(base_json, chart, this.buttonTextColor, curAreaCode, this.currentSeriesId)
const chart_option = baseMapOption(base_json, geoJson, chart, this.buttonTextColor, curAreaCode, this.currentSeriesId)
this.myEcharts(chart_option)
const opt = this.myChart.getOption()
if (opt && opt.series) {

71
frontend/src/views/chart/components/senior/FunctionCfg.vue
View File

@ -80,9 +80,32 @@
>
<el-radio :label="'breakLine'">{{ $t('chart.break_line') }}</el-radio>
<el-radio :label="'setZero'">{{ $t('chart.set_zero') }}</el-radio>
<el-radio :label="'ignoreData'">{{ $t('chart.ignore_data') }}</el-radio>
<el-radio
v-show="showIgnoreOption"
:label="'ignoreData'"
>
{{ $t('chart.ignore_data') }}
</el-radio>
</el-radio-group>
</el-form-item>
<el-form-item
v-show="showEmptyDataFieldCtrl"
:label="$t('chart.empty_data_field_ctrl')"
class="form-item"
>
<el-select
v-model="functionForm.emptyDataFieldCtrl"
multiple
@change="changeFunctionCfg"
>
<el-option
v-for="option in fieldOptions"
:key="option.value"
:label="option.label"
:value="option.value"
/>
</el-select>
</el-form-item>
</el-form>
</el-col>
</div>
@ -103,16 +126,27 @@ export default {
data() {
return {
functionForm: JSON.parse(JSON.stringify(DEFAULT_FUNCTION_CFG)),
predefineColors: COLOR_PANEL
predefineColors: COLOR_PANEL,
fieldOptions: []
}
},
computed: {
showSlider() {
return this.chart.type !== 'bidirectional-bar' && !equalsAny(this.chart.type, 'map')
return this.chart.type !== 'bidirectional-bar' &&
!equalsAny(this.chart.type, 'map') &&
!includesAny(this.chart.type, 'table')
},
showEmptyStrategy() {
return (this.chart.render === 'antv' && includesAny(this.chart.type, 'line', 'bar', 'area')) ||
this.chart.render === 'echarts' && equalsAny(this.chart.type, 'map')
return (this.chart.render === 'antv' &&
includesAny(this.chart.type, 'line', 'bar', 'area', 'table')) ||
(this.chart.render === 'echarts' && equalsAny(this.chart.type, 'map'))
},
showIgnoreOption() {
return !equalsAny(this.chart.type, 'map', 'table-pivot', 'table-info')
},
showEmptyDataFieldCtrl() {
return this.showEmptyStrategy &&
this.functionForm.emptyDataStrategy !== 'breakLine'
}
},
watch: {
@ -140,6 +174,33 @@ export default {
} else {
this.functionForm = JSON.parse(JSON.stringify(DEFAULT_FUNCTION_CFG))
}
this.initFieldCtrl()
}
},
initFieldCtrl() {
if (this.showEmptyDataFieldCtrl) {
this.fieldOptions = []
let axis
if (equalsAny(this.chart.type, 'table-normal', 'table-pivot')) {
axis = this.chart.yaxis
}
if (this.chart.type === 'table-info') {
axis = this.chart.xaxis
}
let axisArr = []
if (Object.prototype.toString.call(axis) === '[object Array]') {
axisArr = axisArr.concat(axis)
} else {
axisArr = axisArr.concat(JSON.parse(axis))
}
axisArr.forEach(item => {
if (item.groupType === 'q') {
this.fieldOptions.push({
label: item.name,
value: item.dataeaseName
})
}
})
}
},
changeFunctionCfg() {

9
frontend/src/views/chart/view/ChartEdit.vue
View File

@ -1955,11 +1955,12 @@ export default {
equalsAny(this.view.type, 'text', 'label', 'map', 'buddle-map')
},
showSeniorCfg() {
return includesAny(this.view.type, 'bar', 'line', 'area', 'mix') ||
return includesAny(this.view.type, 'bar', 'line', 'area', 'mix', 'table') ||
equalsAny(this.view.type, 'table-normal', 'table-info', 'map')
},
showFunctionCfg() {
return includesAny(this.view.type, 'bar', 'line', 'area', 'mix', 'map')
return includesAny(this.view.type, 'bar', 'line', 'area', 'mix', 'table') ||
equalsAny(this.view.type, 'map')
},
showScrollCfg() {
return equalsAny(this.view.type, 'table-normal', 'table-info')
@ -3362,6 +3363,10 @@ export default {
this.view.senior.functionCfg.emptyDataStrategy = 'ignoreData'
} else if (type.includes('line')) {
this.view.customAttr.label.position = 'top'
} else if (equalsAny(type, 'table-info', 'table-pivot')) {
if (this.view?.senior?.functionCfg?.emptyDataStrategy === 'ignoreData') {
this.view.senior.functionCfg.emptyDataStrategy = 'breakLine'
}
}
// reset custom colors
this.view.customAttr.color.seriesColors = []