diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java index 2581ceb3d9..c0e1cb202e 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java @@ -781,6 +781,9 @@ public class JdbcProvider extends DefaultJdbcProvider { case StarRocks: MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfiguration.class); mysqlConfiguration.getJdbc(); + if(!mysqlConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){ + throw new Exception("Invalid database name"); + } break; case redshift: RedshiftConfiguration redshiftConfiguration = new Gson().fromJson(datasource.getConfiguration(), RedshiftConfiguration.class); @@ -791,6 +794,48 @@ public class JdbcProvider extends DefaultJdbcProvider { throw new Exception("Invalid database name"); } break; + case sqlServer: + SqlServerConfiguration sqlServerConfiguration = new Gson().fromJson(datasource.getConfiguration(), SqlServerConfiguration.class); + if(!sqlServerConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){ + throw new Exception("Invalid database name"); + } + break; + case pg: + PgConfiguration pgConfiguration = new Gson().fromJson(datasource.getConfiguration(), PgConfiguration.class); + if(!pgConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){ + throw new Exception("Invalid database name"); + } + break; + case oracle: + OracleConfiguration oracleConfiguration = new Gson().fromJson(datasource.getConfiguration(), OracleConfiguration.class); + if(!oracleConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$") && !oracleConfiguration.getConnectionType().equalsIgnoreCase("serviceName")){ + throw new Exception("Invalid database name"); + } + break; + case mongo: + MongodbConfiguration mongodbConfiguration = new Gson().fromJson(datasource.getConfiguration(), MongodbConfiguration.class); + if(!mongodbConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){ + throw new Exception("Invalid database name"); + } + break; + case impala: + ImpalaConfiguration impalaConfiguration = new Gson().fromJson(datasource.getConfiguration(), ImpalaConfiguration.class); + if(!impalaConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){ + throw new Exception("Invalid database name"); + } + break; + case hive: + HiveConfiguration hiveConfiguration = new Gson().fromJson(datasource.getConfiguration(), HiveConfiguration.class); + if(!hiveConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){ + throw new Exception("Invalid database name"); + } + break; + case db2: + Db2Configuration db2Configuration = new Gson().fromJson(datasource.getConfiguration(), Db2Configuration.class); + if(!db2Configuration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){ + throw new Exception("Invalid database name"); + } + break; default: break; } diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java index 258c4254ca..7ed47d27ef 100644 --- a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java +++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java @@ -41,10 +41,7 @@ public class MysqlDDLProvider extends DDLProviderImpl { strings1[i] = null; continue; } - strings1[i] = strings[i].replace("'", "\\'"); - if(strings1[i].equals("/")){ - strings1[i] = "//"; - } + strings1[i] = strings[i].replace("\\", "\\\\").replace("'", "\\'"); } values.append("('").append(UUID.randomUUID()) .append("','").append(String.join("','", Arrays.asList(strings1))) @@ -90,27 +87,19 @@ public class MysqlDDLProvider extends DDLProviderImpl { break; case 1: size = size < 50? 50 : size; - if (size < 65533) { - Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`"); - }else { - Column_Fields.append("longtext").append(",`"); - } + Column_Fields.append("longtext").append(",`"); break; case 2: Column_Fields.append("bigint(20)").append(",`"); break; case 3: - Column_Fields.append("varchar(100)").append(",`"); + Column_Fields.append("longtext").append(",`"); break; case 4: Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`"); break; default: - if (size < 65533) { - Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`"); - }else { - Column_Fields.append("longtext").append(",`"); - } + Column_Fields.append("longtext").append(",`"); break; } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 41098d1a72..e62a5088c3 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -2454,7 +2454,19 @@ public class DataSetTableService { if (num > 1000) { break; } - data.add(Arrays.asList(line.split(","))); + String str; + line += ","; + Pattern pCells = Pattern.compile("(\"[^\"]*(\"{2})*[^\"]*\")*[^,]*,"); + Matcher mCells = pCells.matcher(line); + List cells = new ArrayList();//每行记录一个list + //读取每个单元格 + while (mCells.find()) { + str = mCells.group(); + str = str.replaceAll("(?sm)\"?([^\"]*(\"{2})*[^\"]*)\"?.*,", "$1"); + str = str.replaceAll("(?sm)(\"(\"))", "$2"); + cells.add(str); + } + data.add(cells); num++; } ExcelSheetData excelSheetData = new ExcelSheetData(); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 71ff655b11..a10a0cd1ac 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -72,6 +72,8 @@ import javax.annotation.Resource; import java.io.*; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Service @@ -758,13 +760,20 @@ public class ExtractDataService { List> csvData = new ArrayList<>(); String line; while ((line = reader.readLine()) != null) { - if(line.endsWith(",")){ - List list = new ArrayList<>(Arrays.asList(line.split(","))); - list.add(""); - csvData.add(list); - }else { - csvData.add(Arrays.asList(line.split(","))); + + String str; + line += ","; + Pattern pCells = Pattern.compile("(\"[^\"]*(\"{2})*[^\"]*\")*[^,]*,"); + Matcher mCells = pCells.matcher(line); + List cells = new ArrayList();//每行记录一个list + //读取每个单元格 + while (mCells.find()) { + str = mCells.group(); + str = str.replaceAll("(?sm)\"?([^\"]*(\"{2})*[^\"]*)\"?.*,", "$1"); + str = str.replaceAll("(?sm)(\"(\"))", "$2"); + cells.add(str); } + csvData.add(cells); } ExcelSheetData csvSheetData = new ExcelSheetData(); String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new); diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 533bf9b5ee..c6b4eb561e 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -1489,6 +1489,7 @@ export default { total_sort_desc: 'DESC', total_sort_field: 'Sort Field', empty_data_strategy: 'Empty Data Strategy', + empty_data_field_ctrl: 'Field Control', break_line: 'Keep', set_zero: 'Set Zero', ignore_data: 'Hide Data', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index 209d78504a..d3a421ef11 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -1491,6 +1491,7 @@ export default { break_line: '保持為空', set_zero: '置為0', ignore_data: '隱藏空值', + empty_data_field_ctrl: '字段設置', sub_dimension_tip: '該字段為必填項,且不應使用類別軸中的字段,若無需該字段,請選擇基礎圖表進行展示,否則展示效果不理想', drill_dimension_tip: '鑽取字段僅支持數據集中的字段', table_scroll_tip: '明細表僅在分頁模式為"下拉"時生效。', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 6916b2234a..69525742ad 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -1487,6 +1487,7 @@ export default { total_sort_desc: '降序', total_sort_field: '排序字段', empty_data_strategy: '空值处理', + empty_data_field_ctrl: '字段设置', break_line: '保持为空', set_zero: '置为0', ignore_data: '隐藏空值', diff --git a/frontend/src/views/chart/chart/chart.js b/frontend/src/views/chart/chart/chart.js index 118cb83554..83f6f1e9f6 100644 --- a/frontend/src/views/chart/chart/chart.js +++ b/frontend/src/views/chart/chart/chart.js @@ -467,7 +467,8 @@ export const DEFAULT_FUNCTION_CFG = { sliderBg: '#FFFFFF', sliderFillBg: '#BCD6F1', sliderTextClolor: '#999999', - emptyDataStrategy: 'breakLine' + emptyDataStrategy: 'breakLine', + emptyDataFieldCtrl: [] } export const DEFAULT_THRESHOLD = { gaugeThreshold: '', diff --git a/frontend/src/views/chart/chart/map/map.js b/frontend/src/views/chart/chart/map/map.js index 4355229fb5..1d534f9a9e 100644 --- a/frontend/src/views/chart/chart/map/map.js +++ b/frontend/src/views/chart/chart/map/map.js @@ -38,7 +38,7 @@ const fillGradientColor = (data, colors) => { }) return data } -export function baseMapOption(chart_option, chart, themeStyle, curAreaCode, seriesId) { +export function baseMapOption(chart_option, geoJson, chart, themeStyle, curAreaCode, seriesId) { // 处理shape attr let customAttr = {} let isGradient = false @@ -151,18 +151,16 @@ export function baseMapOption(chart_option, chart, themeStyle, curAreaCode, seri if (senior) { senior = JSON.parse(senior) } - // 空值处理,echarts 对于值为 null 的默认策略是不展示,也就是保持为空,所以只需要处理忽略数据和置为 0 就行 - // 隐藏和不展示的区别是隐藏不会参与颜色分布的计算,而不展示会参与颜色计算 + // 空值处理,echarts 对于值为 null 的默认策略是不展示,也就是保持为空,所以只需要处理置为 0 就行 let emptyDataStrategy = senior?.functionCfg?.emptyDataStrategy if (!emptyDataStrategy) { emptyDataStrategy = 'breakLine' } + const subArea = new Set(geoJson.features.map(item => item.properties.name)) for (let i = 0; i < valueArr.length; i++) { const y = valueArr[i] - if (y.value === null && emptyDataStrategy === 'ignoreData') { - continue - } y.name = chart.data.x[i] + subArea.delete(y.name) if (y.value === null && emptyDataStrategy === 'setZero') { const tmp = _.clone(y) tmp.value = 0 @@ -171,6 +169,14 @@ export function baseMapOption(chart_option, chart, themeStyle, curAreaCode, seri } chart_option.series[0].data.push(y) } + if (emptyDataStrategy === 'setZero' && subArea.size > 0) { + subArea.forEach(item => { + chart_option.series[0].data.push({ + name: item, + value: 0 + }) + }) + } if (isGradient) { chart_option.series[0].data = fillGradientColor(chart_option.series[0].data, customAttr.color.colors) diff --git a/frontend/src/views/chart/chart/table/table-info.js b/frontend/src/views/chart/chart/table/table-info.js index 321f6398d0..4f262bf81e 100644 --- a/frontend/src/views/chart/chart/table/table-info.js +++ b/frontend/src/views/chart/chart/table/table-info.js @@ -2,7 +2,7 @@ import { TableSheet, S2Event, PivotSheet, DataCell, EXTRA_FIELD, TOTAL_VALUE } f import { getCustomTheme, getSize } from '@/views/chart/chart/common/common_table' import { DEFAULT_COLOR_CASE, DEFAULT_TOTAL } from '@/views/chart/chart/chart' import { formatterItem, valueFormatter } from '@/views/chart/chart/formatter' -import { hexColorToRGBA } from '@/views/chart/chart/util' +import { handleTableEmptyStrategy, hexColorToRGBA } from '@/views/chart/chart/util' export function baseTableInfo(s2, container, chart, action, tableData, pageInfo) { const containerDom = document.getElementById(container) @@ -127,14 +127,15 @@ export function baseTableInfo(s2, container, chart, action, tableData, pageInfo) }) }) } - + // 空值处理 + const newData = handleTableEmptyStrategy(tableData, chart) // data config const s2DataConfig = { fields: { columns: columns }, meta: meta, - data: tableData + data: newData } const customAttr = JSON.parse(chart.customAttr) @@ -300,13 +301,15 @@ export function baseTableNormal(s2, container, chart, action, tableData) { }) } + // 空值处理 + const newData = handleTableEmptyStrategy(tableData, chart) // data config const s2DataConfig = { fields: { columns: columns }, meta: meta, - data: tableData + data: newData } const customAttr = JSON.parse(chart.customAttr) @@ -492,7 +495,8 @@ export function baseTablePivot(s2, container, chart, action, headerAction, table } sortParams.push(sort) } - + // 空值处理 + const newData = handleTableEmptyStrategy(tableData, chart) // data config const s2DataConfig = { fields: { @@ -501,7 +505,7 @@ export function baseTablePivot(s2, container, chart, action, headerAction, table values: v }, meta: meta, - data: tableData, + data: newData, sortParams: sortParams } diff --git a/frontend/src/views/chart/chart/util.js b/frontend/src/views/chart/chart/util.js index dfd14592a0..52ad0e780b 100644 --- a/frontend/src/views/chart/chart/util.js +++ b/frontend/src/views/chart/chart/util.js @@ -3690,3 +3690,38 @@ export function resetRgbOpacity(sourceColor, times) { } return sourceColor } + +export function handleTableEmptyStrategy(tableData, chart) { + let newData = tableData + let intersection = [] + let senior = chart.senior + if (senior) { + senior = JSON.parse(senior) + } + let emptyDataStrategy = senior?.functionCfg?.emptyDataStrategy + if (!emptyDataStrategy) { + emptyDataStrategy = 'breakLine' + } + const emptyDataFieldCtrl = senior?.functionCfg?.emptyDataFieldCtrl + if (emptyDataStrategy !== 'breakLine' && emptyDataFieldCtrl?.length && tableData?.length) { + const deNames = _.keys(tableData[0]) + intersection = _.intersection(deNames, emptyDataFieldCtrl) + } + if (intersection.length) { + newData = _.clone(tableData) + for (let i = 0; i < newData.length; i++) { + for (let j = 0, tmp = intersection.length; j < tmp; j++) { + const deName = intersection[j] + if (newData[i][deName] === null) { + if (emptyDataStrategy === 'setZero') { + newData[i][deName] = 0 + } + if (emptyDataStrategy === 'ignoreData') { + newData = _.filter(newData, (_, index) => index !== i) + } + } + } + } + } + return newData +} diff --git a/frontend/src/views/chart/components/ChartComponent.vue b/frontend/src/views/chart/components/ChartComponent.vue index 9d7bbbb614..a591aad8c0 100644 --- a/frontend/src/views/chart/components/ChartComponent.vue +++ b/frontend/src/views/chart/components/ChartComponent.vue @@ -439,7 +439,7 @@ export default { this.buttonTextColor = null } } - const chart_option = baseMapOption(base_json, chart, this.buttonTextColor, curAreaCode, this.currentSeriesId) + const chart_option = baseMapOption(base_json, geoJson, chart, this.buttonTextColor, curAreaCode, this.currentSeriesId) this.myEcharts(chart_option) const opt = this.myChart.getOption() if (opt && opt.series) { diff --git a/frontend/src/views/chart/components/senior/FunctionCfg.vue b/frontend/src/views/chart/components/senior/FunctionCfg.vue index d026f3018d..632fac468b 100644 --- a/frontend/src/views/chart/components/senior/FunctionCfg.vue +++ b/frontend/src/views/chart/components/senior/FunctionCfg.vue @@ -80,9 +80,32 @@ > {{ $t('chart.break_line') }} {{ $t('chart.set_zero') }} - {{ $t('chart.ignore_data') }} + + {{ $t('chart.ignore_data') }} + + + + + + @@ -103,16 +126,27 @@ export default { data() { return { functionForm: JSON.parse(JSON.stringify(DEFAULT_FUNCTION_CFG)), - predefineColors: COLOR_PANEL + predefineColors: COLOR_PANEL, + fieldOptions: [] } }, computed: { showSlider() { - return this.chart.type !== 'bidirectional-bar' && !equalsAny(this.chart.type, 'map') + return this.chart.type !== 'bidirectional-bar' && + !equalsAny(this.chart.type, 'map') && + !includesAny(this.chart.type, 'table') }, showEmptyStrategy() { - return (this.chart.render === 'antv' && includesAny(this.chart.type, 'line', 'bar', 'area')) || - this.chart.render === 'echarts' && equalsAny(this.chart.type, 'map') + return (this.chart.render === 'antv' && + includesAny(this.chart.type, 'line', 'bar', 'area', 'table')) || + (this.chart.render === 'echarts' && equalsAny(this.chart.type, 'map')) + }, + showIgnoreOption() { + return !equalsAny(this.chart.type, 'map', 'table-pivot', 'table-info') + }, + showEmptyDataFieldCtrl() { + return this.showEmptyStrategy && + this.functionForm.emptyDataStrategy !== 'breakLine' } }, watch: { @@ -140,6 +174,33 @@ export default { } else { this.functionForm = JSON.parse(JSON.stringify(DEFAULT_FUNCTION_CFG)) } + this.initFieldCtrl() + } + }, + initFieldCtrl() { + if (this.showEmptyDataFieldCtrl) { + this.fieldOptions = [] + let axis + if (equalsAny(this.chart.type, 'table-normal', 'table-pivot')) { + axis = this.chart.yaxis + } + if (this.chart.type === 'table-info') { + axis = this.chart.xaxis + } + let axisArr = [] + if (Object.prototype.toString.call(axis) === '[object Array]') { + axisArr = axisArr.concat(axis) + } else { + axisArr = axisArr.concat(JSON.parse(axis)) + } + axisArr.forEach(item => { + if (item.groupType === 'q') { + this.fieldOptions.push({ + label: item.name, + value: item.dataeaseName + }) + } + }) } }, changeFunctionCfg() { diff --git a/frontend/src/views/chart/view/ChartEdit.vue b/frontend/src/views/chart/view/ChartEdit.vue index 41f23cefa9..4a7626ee4d 100644 --- a/frontend/src/views/chart/view/ChartEdit.vue +++ b/frontend/src/views/chart/view/ChartEdit.vue @@ -1955,11 +1955,12 @@ export default { equalsAny(this.view.type, 'text', 'label', 'map', 'buddle-map') }, showSeniorCfg() { - return includesAny(this.view.type, 'bar', 'line', 'area', 'mix') || + return includesAny(this.view.type, 'bar', 'line', 'area', 'mix', 'table') || equalsAny(this.view.type, 'table-normal', 'table-info', 'map') }, showFunctionCfg() { - return includesAny(this.view.type, 'bar', 'line', 'area', 'mix', 'map') + return includesAny(this.view.type, 'bar', 'line', 'area', 'mix', 'table') || + equalsAny(this.view.type, 'map') }, showScrollCfg() { return equalsAny(this.view.type, 'table-normal', 'table-info') @@ -3362,6 +3363,10 @@ export default { this.view.senior.functionCfg.emptyDataStrategy = 'ignoreData' } else if (type.includes('line')) { this.view.customAttr.label.position = 'top' + } else if (equalsAny(type, 'table-info', 'table-pivot')) { + if (this.view?.senior?.functionCfg?.emptyDataStrategy === 'ignoreData') { + this.view.senior.functionCfg.emptyDataStrategy = 'breakLine' + } } // reset custom colors this.view.customAttr.color.seriesColors = []