Merge branch 'dev' into pr@dev_st_fix

backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java

@@ -781,6 +781,9 @@ public class JdbcProvider extends DefaultJdbcProvider {
             case StarRocks:
                 MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfiguration.class);
                 mysqlConfiguration.getJdbc();
+                if(!mysqlConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
+                    throw new Exception("Invalid database name");
+                }
                 break;
             case redshift:
                 RedshiftConfiguration redshiftConfiguration = new Gson().fromJson(datasource.getConfiguration(), RedshiftConfiguration.class);
@@ -791,6 +794,48 @@ public class JdbcProvider extends DefaultJdbcProvider {
                     throw new Exception("Invalid database name");
                 }
                 break;
+            case sqlServer:
+                SqlServerConfiguration sqlServerConfiguration = new Gson().fromJson(datasource.getConfiguration(), SqlServerConfiguration.class);
+                if(!sqlServerConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
+                    throw new Exception("Invalid database name");
+                }
+                break;
+            case pg:
+                PgConfiguration pgConfiguration = new Gson().fromJson(datasource.getConfiguration(), PgConfiguration.class);
+                if(!pgConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
+                    throw new Exception("Invalid database name");
+                }
+                break;
+            case oracle:
+                OracleConfiguration oracleConfiguration = new Gson().fromJson(datasource.getConfiguration(), OracleConfiguration.class);
+                if(!oracleConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$") && !oracleConfiguration.getConnectionType().equalsIgnoreCase("serviceName")){
+                    throw new Exception("Invalid database name");
+                }
+                break;
+            case mongo:
+                MongodbConfiguration mongodbConfiguration = new Gson().fromJson(datasource.getConfiguration(), MongodbConfiguration.class);
+                if(!mongodbConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
+                    throw new Exception("Invalid database name");
+                }
+                break;
+            case impala:
+                ImpalaConfiguration impalaConfiguration = new Gson().fromJson(datasource.getConfiguration(), ImpalaConfiguration.class);
+                if(!impalaConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
+                    throw new Exception("Invalid database name");
+                }
+                break;
+            case hive:
+                HiveConfiguration hiveConfiguration = new Gson().fromJson(datasource.getConfiguration(), HiveConfiguration.class);
+                if(!hiveConfiguration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
+                    throw new Exception("Invalid database name");
+                }
+                break;
+            case db2:
+                Db2Configuration db2Configuration = new Gson().fromJson(datasource.getConfiguration(), Db2Configuration.class);
+                if(!db2Configuration.getDataBase().matches("^[0-9a-zA-Z_]{1,}$")){
+                    throw new Exception("Invalid database name");
+                }
+                break;
             default:
                 break;
         }

backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java

@@ -41,10 +41,7 @@ public class MysqlDDLProvider extends DDLProviderImpl {
                     strings1[i] = null;
                     continue;
                 }
-                strings1[i] = strings[i].replace("'", "\\'");
+                strings1[i] = strings[i].replace("\\", "\\\\").replace("'", "\\'");
-                if(strings1[i].equals("/")){
-                    strings1[i] = "//";
-                }
             }
             values.append("('").append(UUID.randomUUID())
                     .append("','").append(String.join("','", Arrays.asList(strings1)))
@@ -90,27 +87,19 @@ public class MysqlDDLProvider extends DDLProviderImpl {
                     break;
                 case 1:
                     size  = size < 50? 50 : size;
-                    if (size < 65533) {
+                    Column_Fields.append("longtext").append(",`");
-                        Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
-                    }else {
-                        Column_Fields.append("longtext").append(",`");
-                    }
                     break;
                 case 2:
                     Column_Fields.append("bigint(20)").append(",`");
                     break;
                 case 3:
-                    Column_Fields.append("varchar(100)").append(",`");
+                    Column_Fields.append("longtext").append(",`");
                     break;
                 case 4:
                     Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
                     break;
                 default:
-                    if (size < 65533) {
+                    Column_Fields.append("longtext").append(",`");
-                        Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
-                    }else {
-                        Column_Fields.append("longtext").append(",`");
-                    }
                     break;
             }
         }

backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java

@@ -2454,7 +2454,19 @@ public class DataSetTableService {
                 if (num > 1000) {
                     break;
                 }
-                data.add(Arrays.asList(line.split(",")));
+                String str;
+                line += ",";
+                Pattern pCells = Pattern.compile("(\"[^\"]*(\"{2})*[^\"]*\")*[^,]*,");
+                Matcher mCells = pCells.matcher(line);
+                List<String> cells = new ArrayList();//每行记录一个list
+                //读取每个单元格
+                while (mCells.find()) {
+                    str = mCells.group();
+                    str = str.replaceAll("(?sm)\"?([^\"]*(\"{2})*[^\"]*)\"?.*,", "$1");
+                    str = str.replaceAll("(?sm)(\"(\"))", "$2");
+                    cells.add(str);
+                }
+                data.add(cells);
                 num++;
             }
             ExcelSheetData excelSheetData = new ExcelSheetData();

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -72,6 +72,8 @@ import javax.annotation.Resource;
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 @Service
@@ -758,13 +760,20 @@ public class ExtractDataService {
                 List<List<String>> csvData = new ArrayList<>();
                 String line;
                 while ((line = reader.readLine()) != null) {
-                    if(line.endsWith(",")){
+
-                        List<String> list = new ArrayList<>(Arrays.asList(line.split(",")));
+                    String str;
-                        list.add("");
+                    line += ",";
-                        csvData.add(list);
+                    Pattern pCells = Pattern.compile("(\"[^\"]*(\"{2})*[^\"]*\")*[^,]*,");
-                    }else {
+                    Matcher mCells = pCells.matcher(line);
-                        csvData.add(Arrays.asList(line.split(",")));
+                    List<String> cells = new ArrayList();//每行记录一个list
+                    //读取每个单元格
+                    while (mCells.find()) {
+                        str = mCells.group();
+                        str = str.replaceAll("(?sm)\"?([^\"]*(\"{2})*[^\"]*)\"?.*,", "$1");
+                        str = str.replaceAll("(?sm)(\"(\"))", "$2");
+                        cells.add(str);
                     }
+                    csvData.add(cells);
                 }
                 ExcelSheetData csvSheetData = new ExcelSheetData();
                 String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);

frontend/src/lang/en.js

@@ -1489,6 +1489,7 @@ export default {
     total_sort_desc: 'DESC',
     total_sort_field: 'Sort Field',
     empty_data_strategy: 'Empty Data Strategy',
+    empty_data_field_ctrl: 'Field Control',
     break_line: 'Keep',
     set_zero: 'Set Zero',
     ignore_data: 'Hide Data',

frontend/src/lang/tw.js

@@ -1491,6 +1491,7 @@ export default {
     break_line: '保持為空',
     set_zero: '置為0',
     ignore_data: '隱藏空值',
+    empty_data_field_ctrl: '字段設置',
     sub_dimension_tip: '該字段為必填項，且不應使用類別軸中的字段，若無需該字段，請選擇基礎圖表進行展示，否則展示效果不理想',
     drill_dimension_tip: '鑽取字段僅支持數據集中的字段',
     table_scroll_tip: '明細表僅在分頁模式為"下拉"時生效。',

frontend/src/lang/zh.js

@@ -1487,6 +1487,7 @@ export default {
     total_sort_desc: '降序',
     total_sort_field: '排序字段',
     empty_data_strategy: '空值处理',
+    empty_data_field_ctrl: '字段设置',
     break_line: '保持为空',
     set_zero: '置为0',
     ignore_data: '隐藏空值',

frontend/src/views/chart/chart/chart.js

@@ -467,7 +467,8 @@ export const DEFAULT_FUNCTION_CFG = {
   sliderBg: '#FFFFFF',
   sliderFillBg: '#BCD6F1',
   sliderTextClolor: '#999999',
-  emptyDataStrategy: 'breakLine'
+  emptyDataStrategy: 'breakLine',
+  emptyDataFieldCtrl: []
 }
 export const DEFAULT_THRESHOLD = {
   gaugeThreshold: '',

frontend/src/views/chart/chart/map/map.js

@@ -38,7 +38,7 @@ const fillGradientColor = (data, colors) => {
   })
   return data
 }
-export function baseMapOption(chart_option, chart, themeStyle, curAreaCode, seriesId) {
+export function baseMapOption(chart_option, geoJson, chart, themeStyle, curAreaCode, seriesId) {
   // 处理shape attr
   let customAttr = {}
   let isGradient = false
@@ -151,18 +151,16 @@ export function baseMapOption(chart_option, chart, themeStyle, curAreaCode, seri
       if (senior) {
         senior = JSON.parse(senior)
       }
-      // 空值处理，echarts 对于值为 null 的默认策略是不展示，也就是保持为空，所以只需要处理忽略数据和置为 0 就行
+      // 空值处理，echarts 对于值为 null 的默认策略是不展示，也就是保持为空，所以只需要处理置为 0 就行
-      // 隐藏和不展示的区别是隐藏不会参与颜色分布的计算，而不展示会参与颜色计算
       let emptyDataStrategy = senior?.functionCfg?.emptyDataStrategy
       if (!emptyDataStrategy) {
         emptyDataStrategy = 'breakLine'
       }
+      const subArea = new Set(geoJson.features.map(item => item.properties.name))
       for (let i = 0; i < valueArr.length; i++) {
         const y = valueArr[i]
-        if (y.value === null && emptyDataStrategy === 'ignoreData') {
-          continue
-        }
         y.name = chart.data.x[i]
+        subArea.delete(y.name)
         if (y.value === null && emptyDataStrategy === 'setZero') {
           const tmp = _.clone(y)
           tmp.value = 0
@@ -171,6 +169,14 @@ export function baseMapOption(chart_option, chart, themeStyle, curAreaCode, seri
         }
         chart_option.series[0].data.push(y)
       }
+      if (emptyDataStrategy === 'setZero' && subArea.size > 0) {
+        subArea.forEach(item => {
+          chart_option.series[0].data.push({
+            name: item,
+            value: 0
+          })
+        })
+      }
 
       if (isGradient) {
         chart_option.series[0].data = fillGradientColor(chart_option.series[0].data, customAttr.color.colors)

frontend/src/views/chart/chart/table/table-info.js

@@ -2,7 +2,7 @@ import { TableSheet, S2Event, PivotSheet, DataCell, EXTRA_FIELD, TOTAL_VALUE } f
 import { getCustomTheme, getSize } from '@/views/chart/chart/common/common_table'
 import { DEFAULT_COLOR_CASE, DEFAULT_TOTAL } from '@/views/chart/chart/chart'
 import { formatterItem, valueFormatter } from '@/views/chart/chart/formatter'
-import { hexColorToRGBA } from '@/views/chart/chart/util'
+import { handleTableEmptyStrategy, hexColorToRGBA } from '@/views/chart/chart/util'
 export function baseTableInfo(s2, container, chart, action, tableData, pageInfo) {
   const containerDom = document.getElementById(container)
 
@@ -127,14 +127,15 @@ export function baseTableInfo(s2, container, chart, action, tableData, pageInfo)
       })
     })
   }
-
+  // 空值处理
+  const newData = handleTableEmptyStrategy(tableData, chart)
   // data config
   const s2DataConfig = {
     fields: {
       columns: columns
     },
     meta: meta,
-    data: tableData
+    data: newData
   }
 
   const customAttr = JSON.parse(chart.customAttr)
@@ -300,13 +301,15 @@ export function baseTableNormal(s2, container, chart, action, tableData) {
     })
   }
 
+  // 空值处理
+  const newData = handleTableEmptyStrategy(tableData, chart)
   // data config
   const s2DataConfig = {
     fields: {
       columns: columns
     },
     meta: meta,
-    data: tableData
+    data: newData
   }
 
   const customAttr = JSON.parse(chart.customAttr)
@@ -492,7 +495,8 @@ export function baseTablePivot(s2, container, chart, action, headerAction, table
     }
     sortParams.push(sort)
   }
-
+  // 空值处理
+  const newData = handleTableEmptyStrategy(tableData, chart)
   // data config
   const s2DataConfig = {
     fields: {
@@ -501,7 +505,7 @@ export function baseTablePivot(s2, container, chart, action, headerAction, table
       values: v
     },
     meta: meta,
-    data: tableData,
+    data: newData,
     sortParams: sortParams
   }
 

frontend/src/views/chart/chart/util.js

@@ -3690,3 +3690,38 @@ export function resetRgbOpacity(sourceColor, times) {
   }
   return sourceColor
 }
+
+export function handleTableEmptyStrategy(tableData, chart) {
+  let newData = tableData
+  let intersection = []
+  let senior = chart.senior
+  if (senior) {
+    senior = JSON.parse(senior)
+  }
+  let emptyDataStrategy = senior?.functionCfg?.emptyDataStrategy
+  if (!emptyDataStrategy) {
+    emptyDataStrategy = 'breakLine'
+  }
+  const emptyDataFieldCtrl = senior?.functionCfg?.emptyDataFieldCtrl
+  if (emptyDataStrategy !== 'breakLine' && emptyDataFieldCtrl?.length && tableData?.length) {
+    const deNames = _.keys(tableData[0])
+    intersection = _.intersection(deNames, emptyDataFieldCtrl)
+  }
+  if (intersection.length) {
+    newData = _.clone(tableData)
+    for (let i = 0; i < newData.length; i++) {
+      for (let j = 0, tmp = intersection.length; j < tmp; j++) {
+        const deName = intersection[j]
+        if (newData[i][deName] === null) {
+          if (emptyDataStrategy === 'setZero') {
+            newData[i][deName] = 0
+          }
+          if (emptyDataStrategy === 'ignoreData') {
+            newData = _.filter(newData, (_, index) => index !== i)
+          }
+        }
+      }
+    }
+  }
+  return newData
+}

frontend/src/views/chart/components/ChartComponent.vue

@@ -439,7 +439,7 @@ export default {
           this.buttonTextColor = null
         }
       }
-      const chart_option = baseMapOption(base_json, chart, this.buttonTextColor, curAreaCode, this.currentSeriesId)
+      const chart_option = baseMapOption(base_json, geoJson, chart, this.buttonTextColor, curAreaCode, this.currentSeriesId)
       this.myEcharts(chart_option)
       const opt = this.myChart.getOption()
       if (opt && opt.series) {

frontend/src/views/chart/components/senior/FunctionCfg.vue

@@ -80,9 +80,32 @@
           >
             <el-radio :label="'breakLine'">{{ $t('chart.break_line') }}</el-radio>
             <el-radio :label="'setZero'">{{ $t('chart.set_zero') }}</el-radio>
-            <el-radio :label="'ignoreData'">{{ $t('chart.ignore_data') }}</el-radio>
+            <el-radio
+              v-show="showIgnoreOption"
+              :label="'ignoreData'"
+            >
+              {{ $t('chart.ignore_data') }}
+            </el-radio>
           </el-radio-group>
         </el-form-item>
+        <el-form-item
+          v-show="showEmptyDataFieldCtrl"
+          :label="$t('chart.empty_data_field_ctrl')"
+          class="form-item"
+        >
+          <el-select
+            v-model="functionForm.emptyDataFieldCtrl"
+            multiple
+            @change="changeFunctionCfg"
+          >
+            <el-option
+              v-for="option in fieldOptions"
+              :key="option.value"
+              :label="option.label"
+              :value="option.value"
+            />
+          </el-select>
+        </el-form-item>
       </el-form>
     </el-col>
   </div>
@@ -103,16 +126,27 @@ export default {
   data() {
     return {
       functionForm: JSON.parse(JSON.stringify(DEFAULT_FUNCTION_CFG)),
-      predefineColors: COLOR_PANEL
+      predefineColors: COLOR_PANEL,
+      fieldOptions: []
     }
   },
   computed: {
     showSlider() {
-      return this.chart.type !== 'bidirectional-bar' && !equalsAny(this.chart.type, 'map')
+      return this.chart.type !== 'bidirectional-bar' &&
+        !equalsAny(this.chart.type, 'map') &&
+        !includesAny(this.chart.type, 'table')
     },
     showEmptyStrategy() {
-      return (this.chart.render === 'antv' && includesAny(this.chart.type, 'line', 'bar', 'area')) ||
+      return (this.chart.render === 'antv' &&
-      this.chart.render === 'echarts' && equalsAny(this.chart.type, 'map')
+        includesAny(this.chart.type, 'line', 'bar', 'area', 'table')) ||
+        (this.chart.render === 'echarts' && equalsAny(this.chart.type, 'map'))
+    },
+    showIgnoreOption() {
+      return !equalsAny(this.chart.type, 'map', 'table-pivot', 'table-info')
+    },
+    showEmptyDataFieldCtrl() {
+      return this.showEmptyStrategy &&
+        this.functionForm.emptyDataStrategy !== 'breakLine'
     }
   },
   watch: {
@@ -140,6 +174,33 @@ export default {
         } else {
           this.functionForm = JSON.parse(JSON.stringify(DEFAULT_FUNCTION_CFG))
         }
+        this.initFieldCtrl()
+      }
+    },
+    initFieldCtrl() {
+      if (this.showEmptyDataFieldCtrl) {
+        this.fieldOptions = []
+        let axis
+        if (equalsAny(this.chart.type, 'table-normal', 'table-pivot')) {
+          axis = this.chart.yaxis
+        }
+        if (this.chart.type === 'table-info') {
+          axis = this.chart.xaxis
+        }
+        let axisArr = []
+        if (Object.prototype.toString.call(axis) === '[object Array]') {
+          axisArr = axisArr.concat(axis)
+        } else {
+          axisArr = axisArr.concat(JSON.parse(axis))
+        }
+        axisArr.forEach(item => {
+          if (item.groupType === 'q') {
+            this.fieldOptions.push({
+              label: item.name,
+              value: item.dataeaseName
+            })
+          }
+        })
       }
     },
     changeFunctionCfg() {

frontend/src/views/chart/view/ChartEdit.vue

@@ -1955,11 +1955,12 @@ export default {
         equalsAny(this.view.type, 'text', 'label', 'map', 'buddle-map')
     },
     showSeniorCfg() {
-      return includesAny(this.view.type, 'bar', 'line', 'area', 'mix') ||
+      return includesAny(this.view.type, 'bar', 'line', 'area', 'mix', 'table') ||
         equalsAny(this.view.type, 'table-normal', 'table-info', 'map')
     },
     showFunctionCfg() {
-      return includesAny(this.view.type, 'bar', 'line', 'area', 'mix', 'map')
+      return includesAny(this.view.type, 'bar', 'line', 'area', 'mix', 'table') ||
+        equalsAny(this.view.type, 'map')
     },
     showScrollCfg() {
       return equalsAny(this.view.type, 'table-normal', 'table-info')
@@ -3362,6 +3363,10 @@ export default {
         this.view.senior.functionCfg.emptyDataStrategy = 'ignoreData'
       } else if (type.includes('line')) {
         this.view.customAttr.label.position = 'top'
+      } else if (equalsAny(type, 'table-info', 'table-pivot')) {
+        if (this.view?.senior?.functionCfg?.emptyDataStrategy === 'ignoreData') {
+          this.view.senior.functionCfg.emptyDataStrategy = 'breakLine'
+        }
       }
       // reset custom colors
       this.view.customAttr.color.seriesColors = []