@ -57,7 +57,7 @@ import java.text.SimpleDateFormat;
import java.util.* ;
import java.util.stream.Collectors ;
; import static io.dataease.commons.constants.ColumnPermissionConstants.Desensitization_desc ;
import static io.dataease.commons.constants.ColumnPermissionConstants.Desensitization_desc ;
/ * *
* @Author gin
@ -110,7 +110,8 @@ public class DataSetTableService {
}
private void extractData ( DataSetTableRequest datasetTable ) throws Exception {
if ( datasetTable . getMode ( ) = = 1 & & StringUtils . isNotEmpty ( datasetTable . getSyncType ( ) ) & & datasetTable . getSyncType ( ) . equalsIgnoreCase ( " sync_now " ) ) {
if ( datasetTable . getMode ( ) = = 1 & & StringUtils . isNotEmpty ( datasetTable . getSyncType ( ) )
& & datasetTable . getSyncType ( ) . equalsIgnoreCase ( " sync_now " ) ) {
DataSetTaskRequest dataSetTaskRequest = new DataSetTaskRequest ( ) ;
DatasetTableTask datasetTableTask = new DatasetTableTask ( ) ;
datasetTableTask . setTableId ( datasetTable . getId ( ) ) ;
@ -130,7 +131,8 @@ public class DataSetTableService {
if ( StringUtils . isEmpty ( datasetTable . getId ( ) ) ) {
if ( datasetTable . isMergeSheet ( ) ) {
Map < String , List < ExcelSheetData > > map = datasetTable . getSheets ( ) . stream ( ) . collect ( Collectors . groupingBy ( ExcelSheetData : : getFieldsMd5 ) ) ;
Map < String , List < ExcelSheetData > > map = datasetTable . getSheets ( ) . stream ( )
. collect ( Collectors . groupingBy ( ExcelSheetData : : getFieldsMd5 ) ) ;
for ( String s : map . keySet ( ) ) {
DataSetTableRequest sheetTable = new DataSetTableRequest ( ) ;
BeanUtils . copyBean ( sheetTable , datasetTable ) ;
@ -141,7 +143,8 @@ public class DataSetTableService {
sheetTable . setName ( excelSheetDataList . get ( 0 ) . getDatasetName ( ) ) ;
checkName ( sheetTable ) ;
excelSheetDataList . forEach ( excelSheetData - > {
String [ ] fieldArray = excelSheetData . getFields ( ) . stream ( ) . map ( TableFiled : : getFieldName ) . toArray ( String [ ] : : new ) ;
String [ ] fieldArray = excelSheetData . getFields ( ) . stream ( ) . map ( TableFiled : : getFieldName )
. toArray ( String [ ] : : new ) ;
if ( checkIsRepeat ( fieldArray ) ) {
DataEaseException . throwException ( Translator . get ( " i18n_excel_field_repeat " ) ) ;
}
@ -156,11 +159,13 @@ public class DataSetTableService {
datasetIdList . add ( sheetTable . getId ( ) ) ;
}
datasetIdList . forEach ( datasetId - > {
commonThreadPool . addTask ( ( ) - > extractDataService . extractExcelData ( datasetId , " all_scope " , " 初始导入 " , null , datasetIdList ) ) ;
commonThreadPool . addTask ( ( ) - > extractDataService . extractExcelData ( datasetId , " all_scope " , " 初始导入 " ,
null , datasetIdList ) ) ;
} ) ;
} else {
for ( ExcelSheetData sheet : datasetTable . getSheets ( ) ) {
String [ ] fieldArray = sheet . getFields ( ) . stream ( ) . map ( TableFiled : : getFieldName ) . toArray ( String [ ] : : new ) ;
String [ ] fieldArray = sheet . getFields ( ) . stream ( ) . map ( TableFiled : : getFieldName )
. toArray ( String [ ] : : new ) ;
if ( checkIsRepeat ( fieldArray ) ) {
DataEaseException . throwException ( Translator . get ( " i18n_excel_field_repeat " ) ) ;
}
@ -183,7 +188,8 @@ public class DataSetTableService {
datasetIdList . add ( sheetTable . getId ( ) ) ;
}
datasetIdList . forEach ( datasetId - > {
commonThreadPool . addTask ( ( ) - > extractDataService . extractExcelData ( datasetId , " all_scope " , " 初始导入 " , null , datasetIdList ) ) ;
commonThreadPool . addTask ( ( ) - > extractDataService . extractExcelData ( datasetId , " all_scope " , " 初始导入 " ,
null , datasetIdList ) ) ;
} ) ;
}
@ -191,11 +197,13 @@ public class DataSetTableService {
}
List < ExcelSheetData > excelSheetDataList = new ArrayList < > ( ) ;
List < String > oldFields = datasetTable . getSheets ( ) . get ( 0 ) . getFields ( ) . stream ( ) . map ( TableFiled : : getRemarks ) . collect ( Collectors . toList ( ) ) ;
List < String > oldFields = datasetTable . getSheets ( ) . get ( 0 ) . getFields ( ) . stream ( ) . map ( TableFiled : : getRemarks )
. collect ( Collectors . toList ( ) ) ;
for ( ExcelSheetData sheet : datasetTable . getSheets ( ) ) {
/ / 替换时 ,
/ / 替换时 ,
if ( datasetTable . getEditType ( ) = = 0 ) {
List < String > newFields = sheet . getFields ( ) . stream ( ) . map ( TableFiled : : getRemarks ) . collect ( Collectors . toList ( ) ) ;
List < String > newFields = sheet . getFields ( ) . stream ( ) . map ( TableFiled : : getRemarks )
. collect ( Collectors . toList ( ) ) ;
if ( ! oldFields . equals ( newFields ) ) {
DataEaseException . throwException ( Translator . get ( " i18n_excel_column_inconsistent " ) ) ;
}
@ -216,9 +224,12 @@ public class DataSetTableService {
int update = datasetTableMapper . updateByPrimaryKeySelective ( datasetTable ) ;
/ / 替換時 , 先不刪除旧字段 ; 同步成功后再删除
if ( datasetTable . getEditType ( ) = = 0 ) {
commonThreadPool . addTask ( ( ) - > extractDataService . extractExcelData ( datasetTable . getId ( ) , " all_scope " , " 替换 " , saveExcelTableField ( datasetTable . getId ( ) , datasetTable . getSheets ( ) . get ( 0 ) . getFields ( ) , false ) , Arrays . asList ( datasetTable . getId ( ) ) ) ) ;
commonThreadPool . addTask ( ( ) - > extractDataService . extractExcelData ( datasetTable . getId ( ) , " all_scope " , " 替换 " ,
saveExcelTableField ( datasetTable . getId ( ) , datasetTable . getSheets ( ) . get ( 0 ) . getFields ( ) , false ) ,
Arrays . asList ( datasetTable . getId ( ) ) ) ) ;
} else if ( datasetTable . getEditType ( ) = = 1 ) {
commonThreadPool . addTask ( ( ) - > extractDataService . extractExcelData ( datasetTable . getId ( ) , " add_scope " , " 追加 " , null , Arrays . asList ( datasetTable . getId ( ) ) ) ) ;
commonThreadPool . addTask ( ( ) - > extractDataService . extractExcelData ( datasetTable . getId ( ) , " add_scope " , " 追加 " ,
null , Arrays . asList ( datasetTable . getId ( ) ) ) ) ;
}
}
@ -285,7 +296,8 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest ( ) ;
datasourceRequest . setDatasource ( dorisDatasource ) ;
DDLProvider ddlProvider = ProviderFactory . getDDLProvider ( dorisDatasource . getType ( ) ) ;
if ( StringUtils . equalsIgnoreCase ( " custom " , table . getType ( ) ) | | StringUtils . equalsIgnoreCase ( " union " , table . getType ( ) ) ) {
if ( StringUtils . equalsIgnoreCase ( " custom " , table . getType ( ) )
| | StringUtils . equalsIgnoreCase ( " union " , table . getType ( ) ) ) {
datasourceRequest . setQuery ( ddlProvider . dropView ( dorisTableName ) ) ;
jdbcProvider . exec ( datasourceRequest ) ;
datasourceRequest . setQuery ( ddlProvider . dropView ( DorisTableUtils . dorisTmpName ( dorisTableName ) ) ) ;
@ -402,7 +414,8 @@ public class DataSetTableService {
return datasourceProvider . getTableFileds ( datasourceRequest ) ;
}
public Map < String , List < DatasetTableField > > getFieldsFromDE ( DataSetTableRequest dataSetTableRequest ) throws Exception {
public Map < String , List < DatasetTableField > > getFieldsFromDE ( DataSetTableRequest dataSetTableRequest )
throws Exception {
DatasetTableField datasetTableField = DatasetTableField . builder ( ) . build ( ) ;
datasetTableField . setTableId ( dataSetTableRequest . getId ( ) ) ;
datasetTableField . setChecked ( Boolean . TRUE ) ;
@ -441,10 +454,11 @@ public class DataSetTableService {
return map ;
}
public Map < String , Object > getPreviewData ( DataSetTableRequest dataSetTableRequest , Integer page , Integer pageSize , List < DatasetTableField > extFields ) throws Exception {
public Map < String , Object > getPreviewData ( DataSetTableRequest dataSetTableRequest , Integer page , Integer pageSize ,
List < DatasetTableField > extFields ) throws Exception {
Map < String , Object > map = new HashMap < > ( ) ;
DatasetTableField datasetTableField = DatasetTableField . builder ( ) . tableId ( dataSetTableRequest . getId ( ) ) . checked ( Boolean . TRUE ) . build ( ) ;
DatasetTableField datasetTableField = DatasetTableField . builder ( ) . tableId ( dataSetTableRequest . getId ( ) )
. checked ( Boolean . TRUE ) . build ( ) ;
List < DatasetTableField > fields = dataSetTableFieldsService . list ( datasetTableField ) ;
if ( CollectionUtils . isNotEmpty ( extFields ) ) {
fields . addAll ( extFields ) ;
@ -456,7 +470,7 @@ public class DataSetTableService {
return map ;
}
DatasetTable datasetTable = datasetTableMapper . selectByPrimaryKey ( dataSetTableRequest . getId ( ) ) ;
/ / 列权限
/ / 列权限
List < String > desensitizationList = new ArrayList < > ( ) ;
fields = permissionService . filterColumnPermissons ( fields , desensitizationList , datasetTable . getId ( ) , null ) ;
if ( CollectionUtils . isEmpty ( fields ) ) {
@ -465,19 +479,20 @@ public class DataSetTableService {
map . put ( " page " , new DataSetPreviewPage ( ) ) ;
return map ;
}
/ / 行权限
/ / 行权限
List < ChartFieldCustomFilterDTO > customFilter = permissionService . getCustomFilters ( fields , datasetTable , null ) ;
String [ ] fieldArray = fields . stream ( ) . map ( DatasetTableField : : getDataeaseName ) . toArray ( String [ ] : : new ) ;
DataTableInfoDTO dataTableInfoDTO = new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) , DataTableInfoDTO . class ) ;
List < String [ ] > data = new ArrayList < > ( ) ;
DataSetPreviewPage dataSetPreviewPage = new DataSetPreviewPage ( ) ;
dataSetPreviewPage . setShow ( Integer . valueOf ( dataSetTableRequest . getRow ( ) ) ) ;
dataSetPreviewPage . setPage ( page ) ;
dataSetPreviewPage . setPageSize ( pageSize ) ;
int realSize = Integer . parseInt ( dataSetTableRequest . getRow ( ) ) < pageSize ? Integer . parseInt ( dataSetTableRequest . getRow ( ) ) : pageSize ;
int realSize = Integer . parseInt ( dataSetTableRequest . getRow ( ) ) < pageSize
? Integer . parseInt ( dataSetTableRequest . getRow ( ) )
: pageSize ;
if ( page = = Integer . parseInt ( dataSetTableRequest . getRow ( ) ) / pageSize + 1 ) {
realSize = Integer . parseInt ( dataSetTableRequest . getRow ( ) ) % pageSize ;
}
@ -496,8 +511,8 @@ public class DataSetTableService {
String table = dataTableInfoDTO . getTable ( ) ;
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
datasourceRequest . setQuery (
qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
map . put ( " sql " , datasourceRequest . getQuery ( ) ) ;
datasourceRequest . setPage ( page ) ;
@ -514,7 +529,8 @@ public class DataSetTableService {
}
try {
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields , Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields ,
Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
datasourceRequest . setPageable ( false ) ;
dataSetPreviewPage . setTotal ( datasourceProvider . getData ( datasourceRequest ) . size ( ) ) ;
} catch ( Exception e ) {
@ -532,7 +548,8 @@ public class DataSetTableService {
datasourceRequest . setDatasource ( ds ) ;
String table = DorisTableUtils . dorisName ( dataSetTableRequest . getId ( ) ) ;
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
datasourceRequest . setQuery (
qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
map . put ( " sql " , datasourceRequest . getQuery ( ) ) ;
try {
data . addAll ( jdbcProvider . getData ( datasourceRequest ) ) ;
@ -541,7 +558,8 @@ public class DataSetTableService {
DEException . throwException ( Translator . get ( " i18n_ds_error " ) + " -> " + e . getMessage ( ) ) ;
}
try {
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields , Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields ,
Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
dataSetPreviewPage . setTotal ( jdbcProvider . getData ( datasourceRequest ) . size ( ) ) ;
} catch ( Exception e ) {
logger . error ( e . getMessage ( ) ) ;
@ -564,7 +582,8 @@ public class DataSetTableService {
String sql = dataTableInfoDTO . getSql ( ) ;
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
datasourceRequest . setQuery ( qp . createQuerySQLWithPage ( sql , fields , page , pageSize , realSize , false , customFilter ) ) ;
datasourceRequest . setQuery (
qp . createQuerySQLWithPage ( sql , fields , page , pageSize , realSize , false , customFilter ) ) ;
map . put ( " sql " , datasourceRequest . getQuery ( ) ) ;
datasourceRequest . setPage ( page ) ;
datasourceRequest . setFetchSize ( Integer . parseInt ( dataSetTableRequest . getRow ( ) ) ) ;
@ -580,7 +599,8 @@ public class DataSetTableService {
}
try {
datasourceRequest . setPageable ( false ) ;
datasourceRequest . setQuery ( qp . createQuerySqlWithLimit ( sql , fields , Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , customFilter ) ) ;
datasourceRequest . setQuery ( qp . createQuerySqlWithLimit ( sql , fields ,
Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , customFilter ) ) ;
dataSetPreviewPage . setTotal ( datasourceProvider . getData ( datasourceRequest ) . size ( ) ) ;
} catch ( Exception e ) {
logger . error ( e . getMessage ( ) ) ;
@ -597,7 +617,8 @@ public class DataSetTableService {
datasourceRequest . setDatasource ( ds ) ;
String table = DorisTableUtils . dorisName ( dataSetTableRequest . getId ( ) ) ;
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
datasourceRequest . setQuery (
qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
map . put ( " sql " , datasourceRequest . getQuery ( ) ) ;
try {
data . addAll ( jdbcProvider . getData ( datasourceRequest ) ) ;
@ -606,7 +627,8 @@ public class DataSetTableService {
DEException . throwException ( Translator . get ( " i18n_ds_error " ) + " -> " + e . getMessage ( ) ) ;
}
try {
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields , Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields ,
Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
dataSetPreviewPage . setTotal ( jdbcProvider . getData ( datasourceRequest ) . size ( ) ) ;
} catch ( Exception e ) {
logger . error ( e . getMessage ( ) ) ;
@ -624,7 +646,8 @@ public class DataSetTableService {
datasourceRequest . setDatasource ( ds ) ;
String table = DorisTableUtils . dorisName ( dataSetTableRequest . getId ( ) ) ;
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
datasourceRequest . setQuery (
qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
map . put ( " sql " , datasourceRequest . getQuery ( ) ) ;
try {
data . addAll ( jdbcProvider . getData ( datasourceRequest ) ) ;
@ -633,7 +656,8 @@ public class DataSetTableService {
DEException . throwException ( Translator . get ( " i18n_ds_error " ) + " -> " + e . getMessage ( ) ) ;
}
try {
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields , Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields ,
Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
dataSetPreviewPage . setTotal ( jdbcProvider . getData ( datasourceRequest ) . size ( ) ) ;
} catch ( Exception e ) {
logger . error ( e . getMessage ( ) ) ;
@ -650,7 +674,8 @@ public class DataSetTableService {
datasourceRequest . setDatasource ( ds ) ;
DataTableInfoDTO dt = new Gson ( ) . fromJson ( datasetTable . getInfo ( ) , DataTableInfoDTO . class ) ;
List < DataSetTableUnionDTO > list = dataSetTableUnionService . listByTableId ( dt . getList ( ) . get ( 0 ) . getTableId ( ) ) ;
List < DataSetTableUnionDTO > list = dataSetTableUnionService
. listByTableId ( dt . getList ( ) . get ( 0 ) . getTableId ( ) ) ;
String sql = " " ;
try {
@ -660,7 +685,8 @@ public class DataSetTableService {
DEException . throwException ( Translator . get ( " i18n_ds_error " ) + " -> " + e . getMessage ( ) ) ;
}
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
datasourceRequest . setQuery ( qp . createQuerySQLWithPage ( sql , fields , page , pageSize , realSize , false , customFilter ) ) ;
datasourceRequest . setQuery (
qp . createQuerySQLWithPage ( sql , fields , page , pageSize , realSize , false , customFilter ) ) ;
map . put ( " sql " , datasourceRequest . getQuery ( ) ) ;
datasourceRequest . setPage ( page ) ;
datasourceRequest . setFetchSize ( Integer . parseInt ( dataSetTableRequest . getRow ( ) ) ) ;
@ -676,7 +702,8 @@ public class DataSetTableService {
}
try {
datasourceRequest . setPageable ( false ) ;
datasourceRequest . setQuery ( qp . createQuerySqlWithLimit ( sql , fields , Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , customFilter ) ) ;
datasourceRequest . setQuery ( qp . createQuerySqlWithLimit ( sql , fields ,
Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , customFilter ) ) ;
dataSetPreviewPage . setTotal ( datasourceProvider . getData ( datasourceRequest ) . size ( ) ) ;
} catch ( Exception e ) {
logger . error ( e . getMessage ( ) ) ;
@ -689,7 +716,8 @@ public class DataSetTableService {
datasourceRequest . setDatasource ( ds ) ;
String table = DorisTableUtils . dorisName ( dataSetTableRequest . getId ( ) ) ;
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
datasourceRequest . setQuery (
qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
map . put ( " sql " , datasourceRequest . getQuery ( ) ) ;
try {
data . addAll ( jdbcProvider . getData ( datasourceRequest ) ) ;
@ -699,7 +727,8 @@ public class DataSetTableService {
}
try {
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields , Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields ,
Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
dataSetPreviewPage . setTotal ( jdbcProvider . getData ( datasourceRequest ) . size ( ) ) ;
} catch ( Exception e ) {
logger . error ( e . getMessage ( ) ) ;
@ -726,7 +755,8 @@ public class DataSetTableService {
DEException . throwException ( Translator . get ( " i18n_ds_error " ) + " -> " + e . getMessage ( ) ) ;
}
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
datasourceRequest . setQuery ( qp . createQuerySQLWithPage ( sql , fields , page , pageSize , realSize , false , customFilter ) ) ;
datasourceRequest . setQuery (
qp . createQuerySQLWithPage ( sql , fields , page , pageSize , realSize , false , customFilter ) ) ;
map . put ( " sql " , datasourceRequest . getQuery ( ) ) ;
datasourceRequest . setPage ( page ) ;
datasourceRequest . setFetchSize ( Integer . parseInt ( dataSetTableRequest . getRow ( ) ) ) ;
@ -742,7 +772,8 @@ public class DataSetTableService {
}
try {
datasourceRequest . setPageable ( false ) ;
datasourceRequest . setQuery ( qp . createQuerySqlWithLimit ( sql , fields , Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , customFilter ) ) ;
datasourceRequest . setQuery ( qp . createQuerySqlWithLimit ( sql , fields ,
Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , customFilter ) ) ;
dataSetPreviewPage . setTotal ( datasourceProvider . getData ( datasourceRequest ) . size ( ) ) ;
} catch ( Exception e ) {
logger . error ( e . getMessage ( ) ) ;
@ -755,7 +786,8 @@ public class DataSetTableService {
datasourceRequest . setDatasource ( ds ) ;
String table = DorisTableUtils . dorisName ( dataSetTableRequest . getId ( ) ) ;
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
datasourceRequest . setQuery (
qp . createQueryTableWithPage ( table , fields , page , pageSize , realSize , false , ds , customFilter ) ) ;
map . put ( " sql " , datasourceRequest . getQuery ( ) ) ;
try {
data . addAll ( jdbcProvider . getData ( datasourceRequest ) ) ;
@ -765,7 +797,8 @@ public class DataSetTableService {
}
try {
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields , Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
datasourceRequest . setQuery ( qp . createQueryTableWithLimit ( table , fields ,
Integer . valueOf ( dataSetTableRequest . getRow ( ) ) , false , ds , customFilter ) ) ;
dataSetPreviewPage . setTotal ( jdbcProvider . getData ( datasourceRequest ) . size ( ) ) ;
} catch ( Exception e ) {
logger . error ( e . getMessage ( ) ) ;
@ -880,8 +913,12 @@ public class DataSetTableService {
/ / 获取每个字段在当前de数据库中的name , 作为sql查询后的remarks返回前端展示
for ( DatasetTableField datasetTableField : fieldList ) {
for ( TableFiled tableFiled : fields ) {
if ( StringUtils . equalsIgnoreCase ( tableFiled . getFieldName ( ) , DorisTableUtils . dorisFieldName ( datasetTableField . getTableId ( ) + " _ " + datasetTableField . getDataeaseName ( ) ) )
| | StringUtils . equalsIgnoreCase ( tableFiled . getFieldName ( ) , DorisTableUtils . dorisFieldNameShort ( datasetTableField . getTableId ( ) + " _ " + datasetTableField . getOriginName ( ) ) ) ) {
if ( StringUtils . equalsIgnoreCase ( tableFiled . getFieldName ( ) ,
DorisTableUtils . dorisFieldName (
datasetTableField . getTableId ( ) + " _ " + datasetTableField . getDataeaseName ( ) ) )
| | StringUtils . equalsIgnoreCase ( tableFiled . getFieldName ( ) ,
DorisTableUtils . dorisFieldNameShort ( datasetTableField . getTableId ( ) + " _ "
+ datasetTableField . getOriginName ( ) ) ) ) {
tableFiled . setRemarks ( datasetTableField . getName ( ) ) ;
break ;
}
@ -898,7 +935,8 @@ public class DataSetTableService {
public Map < String , Object > getCustomPreview ( DataSetTableRequest dataSetTableRequest ) throws Exception {
DataTableInfoDTO dataTableInfoDTO = new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) , DataTableInfoDTO . class ) ;
List < DataSetTableUnionDTO > list = dataSetTableUnionService . listByTableId ( dataTableInfoDTO . getList ( ) . get ( 0 ) . getTableId ( ) ) ;
List < DataSetTableUnionDTO > list = dataSetTableUnionService
. listByTableId ( dataTableInfoDTO . getList ( ) . get ( 0 ) . getTableId ( ) ) ;
String sql ;
DatasourceRequest datasourceRequest = new DatasourceRequest ( ) ;
@ -935,11 +973,15 @@ public class DataSetTableService {
/ / 获取每个字段在当前de数据库中的name , 作为sql查询后的remarks返回前端展示
List < DatasetTableField > checkedFieldList = new ArrayList < > ( ) ;
dataTableInfoDTO . getList ( ) . forEach ( ele - > checkedFieldList . addAll ( dataSetTableFieldsService . getListByIds ( ele . getCheckedFields ( ) ) ) ) ;
dataTableInfoDTO . getList ( ) . forEach (
ele - > checkedFieldList . addAll ( dataSetTableFieldsService . getListByIds ( ele . getCheckedFields ( ) ) ) ) ;
for ( DatasetTableField datasetTableField : checkedFieldList ) {
for ( TableFiled tableFiled : fields ) {
if ( StringUtils . equalsIgnoreCase ( tableFiled . getFieldName ( ) , DorisTableUtils . dorisFieldName ( datasetTableField . getTableId ( ) + " _ " + datasetTableField . getDataeaseName ( ) ) )
| | StringUtils . equalsIgnoreCase ( tableFiled . getFieldName ( ) , DorisTableUtils . dorisFieldName ( datasetTableField . getTableId ( ) + " _ " + datasetTableField . getOriginName ( ) ) ) ) {
if ( StringUtils . equalsIgnoreCase ( tableFiled . getFieldName ( ) ,
DorisTableUtils . dorisFieldName (
datasetTableField . getTableId ( ) + " _ " + datasetTableField . getDataeaseName ( ) ) )
| | StringUtils . equalsIgnoreCase ( tableFiled . getFieldName ( ) , DorisTableUtils . dorisFieldName (
datasetTableField . getTableId ( ) + " _ " + datasetTableField . getOriginName ( ) ) ) ) {
tableFiled . setRemarks ( datasetTableField . getName ( ) ) ;
break ;
}
@ -967,7 +1009,10 @@ public class DataSetTableService {
if ( CollectionUtils . isEmpty ( fields ) ) {
throw new RuntimeException ( Translator . get ( " i18n_cst_ds_tb_or_field_deleted " ) ) ;
}
String [ ] array = fields . stream ( ) . map ( f - > table + " . " + f . getDataeaseName ( ) + " AS " + DorisTableUtils . dorisFieldName ( ele . getTableId ( ) + " _ " + f . getDataeaseName ( ) ) ) . toArray ( String [ ] : : new ) ;
String [ ] array = fields . stream ( )
. map ( f - > table + " . " + f . getDataeaseName ( ) + " AS "
+ DorisTableUtils . dorisFieldName ( ele . getTableId ( ) + " _ " + f . getDataeaseName ( ) ) )
. toArray ( String [ ] : : new ) ;
customInfo . put ( table , array ) ;
} ) ;
DataTableInfoCustomUnion first = dataTableInfoDTO . getList ( ) . get ( 0 ) ;
@ -994,25 +1039,32 @@ public class DataSetTableService {
join . append ( convertUnionTypeToSQL ( dto . getSourceUnionRelation ( ) ) )
. append ( DorisTableUtils . dorisName ( dto . getTargetTableId ( ) ) )
. append ( " ON " )
. append ( DorisTableUtils . dorisName ( dto . getSourceTableId ( ) ) ) . append ( " . " ) . append ( sourceField . getDataeaseName ( ) )
. append ( DorisTableUtils . dorisName ( dto . getSourceTableId ( ) ) ) . append ( " . " )
. append ( sourceField . getDataeaseName ( ) )
. append ( " = " )
. append ( DorisTableUtils . dorisName ( dto . getTargetTableId ( ) ) ) . append ( " . " ) . append ( targetField . getDataeaseName ( ) ) ;
. append ( DorisTableUtils . dorisName ( dto . getTargetTableId ( ) ) ) . append ( " . " )
. append ( targetField . getDataeaseName ( ) ) ;
}
}
}
if ( StringUtils . isEmpty ( f ) ) {
throw new RuntimeException ( Translator . get ( " i18n_custom_ds_delete " ) ) ;
}
return MessageFormat . format ( " SELECT {0} FROM {1} " , f , DorisTableUtils . dorisName ( first . getTableId ( ) ) ) + join . toString ( ) ;
return MessageFormat . format ( " SELECT {0} FROM {1} " , f , DorisTableUtils . dorisName ( first . getTableId ( ) ) )
+ join . toString ( ) ;
} else {
if ( StringUtils . isEmpty ( StringUtils . join ( customInfo . get ( DorisTableUtils . dorisName ( first . getTableId ( ) ) ) , " , " ) ) ) {
if ( StringUtils
. isEmpty ( StringUtils . join ( customInfo . get ( DorisTableUtils . dorisName ( first . getTableId ( ) ) ) , " , " ) ) ) {
throw new RuntimeException ( Translator . get ( " i18n_custom_ds_delete " ) ) ;
}
return MessageFormat . format ( " SELECT {0} FROM {1} " , StringUtils . join ( customInfo . get ( DorisTableUtils . dorisName ( first . getTableId ( ) ) ) , " , " ) , DorisTableUtils . dorisName ( first . getTableId ( ) ) ) ;
return MessageFormat . format ( " SELECT {0} FROM {1} " ,
StringUtils . join ( customInfo . get ( DorisTableUtils . dorisName ( first . getTableId ( ) ) ) , " , " ) ,
DorisTableUtils . dorisName ( first . getTableId ( ) ) ) ;
}
}
public String getCustomSQLDatasource ( DataTableInfoDTO dataTableInfoDTO , List < DataSetTableUnionDTO > list , Datasource ds ) {
public String getCustomSQLDatasource ( DataTableInfoDTO dataTableInfoDTO , List < DataSetTableUnionDTO > list ,
Datasource ds ) {
DatasourceTypes datasourceTypes = DatasourceTypes . valueOf ( ds . getType ( ) ) ;
String keyword = datasourceTypes . getKeywordPrefix ( ) + " %s " + datasourceTypes . getKeywordSuffix ( ) ;
Map < String , String [ ] > customInfo = new TreeMap < > ( ) ;
@ -1026,7 +1078,10 @@ public class DataSetTableService {
if ( CollectionUtils . isEmpty ( fields ) ) {
throw new RuntimeException ( Translator . get ( " i18n_cst_ds_tb_or_field_deleted " ) ) ;
}
String [ ] array = fields . stream ( ) . map ( f - > String . format ( keyword , table ) + " . " + String . format ( keyword , f . getOriginName ( ) ) + " AS " + DorisTableUtils . dorisFieldName ( ele . getTableId ( ) + " _ " + f . getOriginName ( ) ) ) . toArray ( String [ ] : : new ) ;
String [ ] array = fields . stream ( )
. map ( f - > String . format ( keyword , table ) + " . " + String . format ( keyword , f . getOriginName ( ) ) + " AS "
+ DorisTableUtils . dorisFieldName ( ele . getTableId ( ) + " _ " + f . getOriginName ( ) ) )
. toArray ( String [ ] : : new ) ;
customInfo . put ( table , array ) ;
}
DataTableInfoCustomUnion first = dataTableInfoDTO . getList ( ) . get ( 0 ) ;
@ -1052,15 +1107,19 @@ public class DataSetTableService {
DEException . throwException ( Translator . get ( " i18n_dataset_field_delete " ) ) ;
}
DatasetTable sourceTable = datasetTableMapper . selectByPrimaryKey ( dto . getSourceTableId ( ) ) ;
String sourceTableName = new Gson ( ) . fromJson ( sourceTable . getInfo ( ) , DataTableInfoDTO . class ) . getTable ( ) ;
String sourceTableName = new Gson ( ) . fromJson ( sourceTable . getInfo ( ) , DataTableInfoDTO . class )
. getTable ( ) ;
DatasetTable targetTable = datasetTableMapper . selectByPrimaryKey ( dto . getTargetTableId ( ) ) ;
String targetTableName = new Gson ( ) . fromJson ( targetTable . getInfo ( ) , DataTableInfoDTO . class ) . getTable ( ) ;
String targetTableName = new Gson ( ) . fromJson ( targetTable . getInfo ( ) , DataTableInfoDTO . class )
. getTable ( ) ;
join . append ( convertUnionTypeToSQL ( dto . getSourceUnionRelation ( ) ) )
. append ( String . format ( keyword , targetTableName ) )
. append ( " ON " )
. append ( String . format ( keyword , sourceTableName ) ) . append ( " . " ) . append ( String . format ( keyword , sourceField . getOriginName ( ) ) )
. append ( String . format ( keyword , sourceTableName ) ) . append ( " . " )
. append ( String . format ( keyword , sourceField . getOriginName ( ) ) )
. append ( " = " )
. append ( String . format ( keyword , targetTableName ) ) . append ( " . " ) . append ( String . format ( keyword , targetField . getOriginName ( ) ) ) ;
. append ( String . format ( keyword , targetTableName ) ) . append ( " . " )
. append ( String . format ( keyword , targetField . getOriginName ( ) ) ) ;
}
}
}
@ -1072,7 +1131,8 @@ public class DataSetTableService {
if ( StringUtils . isEmpty ( StringUtils . join ( customInfo . get ( tableName ) , " , " ) ) ) {
throw new RuntimeException ( Translator . get ( " i18n_custom_ds_delete " ) ) ;
}
return MessageFormat . format ( " SELECT {0} FROM {1} " , StringUtils . join ( customInfo . get ( tableName ) , " , " ) , String . format ( keyword , tableName ) ) ;
return MessageFormat . format ( " SELECT {0} FROM {1} " , StringUtils . join ( customInfo . get ( tableName ) , " , " ) ,
String . format ( keyword , tableName ) ) ;
}
}
@ -1109,11 +1169,15 @@ public class DataSetTableService {
String table = DorisTableUtils . dorisName ( tableId ) ;
DatasetTable datasetTable = datasetTableMapper . selectByPrimaryKey ( tableId ) ;
if ( ObjectUtils . isEmpty ( datasetTable ) ) {
DEException . throwException ( Translator . get ( " i18n_custom_ds_delete " ) + String . format ( " :table id [%s] " , tableId ) ) ;
DEException . throwException (
Translator . get ( " i18n_custom_ds_delete " ) + String . format ( " :table id [%s] " , tableId ) ) ;
}
List < DatasetTableField > fields = dataSetTableFieldsService . getListByIdsEach ( unionDTO . getCurrentDsField ( ) ) ;
String [ ] array = fields . stream ( ) . map ( f - > table + " . " + f . getDataeaseName ( ) + " AS " + DorisTableUtils . dorisFieldName ( tableId + " _ " + f . getDataeaseName ( ) ) ) . toArray ( String [ ] : : new ) ;
String [ ] array = fields . stream ( )
. map ( f - > table + " . " + f . getDataeaseName ( ) + " AS "
+ DorisTableUtils . dorisFieldName ( tableId + " _ " + f . getDataeaseName ( ) ) )
. toArray ( String [ ] : : new ) ;
checkedInfo . put ( table , array ) ;
checkedFields . addAll ( fields ) ;
/ / 获取child的fields和union
@ -1149,12 +1213,16 @@ public class DataSetTableService {
for ( int i = 0 ; i < unionParamDTO . getUnionFields ( ) . size ( ) ; i + + ) {
UnionItemDTO unionItemDTO = unionParamDTO . getUnionFields ( ) . get ( i ) ;
/ / 通过field id取得field详情 , 并且以第一组为准 , 寻找dataset table
DatasetTableField parentField = dataSetTableFieldsService . get ( unionItemDTO . getParentField ( ) . getId ( ) ) ;
DatasetTableField currentField = dataSetTableFieldsService . get ( unionItemDTO . getCurrentField ( ) . getId ( ) ) ;
DatasetTableField parentField = dataSetTableFieldsService
. get ( unionItemDTO . getParentField ( ) . getId ( ) ) ;
DatasetTableField currentField = dataSetTableFieldsService
. get ( unionItemDTO . getCurrentField ( ) . getId ( ) ) ;
join . append ( DorisTableUtils . dorisName ( parentTable . getId ( ) ) ) . append ( " . " ) . append ( parentField . getDataeaseName ( ) )
join . append ( DorisTableUtils . dorisName ( parentTable . getId ( ) ) ) . append ( " . " )
. append ( parentField . getDataeaseName ( ) )
. append ( " = " )
. append ( DorisTableUtils . dorisName ( currentTable . getId ( ) ) ) . append ( " . " ) . append ( currentField . getDataeaseName ( ) ) ;
. append ( DorisTableUtils . dorisName ( currentTable . getId ( ) ) ) . append ( " . " )
. append ( currentField . getDataeaseName ( ) ) ;
if ( i < unionParamDTO . getUnionFields ( ) . size ( ) - 1 ) {
join . append ( " AND " ) ;
}
@ -1163,13 +1231,16 @@ public class DataSetTableService {
if ( StringUtils . isEmpty ( f ) ) {
DEException . throwException ( Translator . get ( " i18n_union_ds_no_checked " ) ) ;
}
sql = MessageFormat . format ( " SELECT {0} FROM {1} " , f , DorisTableUtils . dorisName ( union . get ( 0 ) . getCurrentDs ( ) . getId ( ) ) ) + join . toString ( ) ;
sql = MessageFormat . format ( " SELECT {0} FROM {1} " , f ,
DorisTableUtils . dorisName ( union . get ( 0 ) . getCurrentDs ( ) . getId ( ) ) ) + join . toString ( ) ;
} else {
String f = StringUtils . join ( checkedInfo . get ( DorisTableUtils . dorisName ( union . get ( 0 ) . getCurrentDs ( ) . getId ( ) ) ) , " , " ) ;
String f = StringUtils . join ( checkedInfo . get ( DorisTableUtils . dorisName ( union . get ( 0 ) . getCurrentDs ( ) . getId ( ) ) ) ,
" , " ) ;
if ( StringUtils . isEmpty ( f ) ) {
throw new RuntimeException ( Translator . get ( " i18n_union_ds_no_checked " ) ) ;
}
sql = MessageFormat . format ( " SELECT {0} FROM {1} " , f , DorisTableUtils . dorisName ( union . get ( 0 ) . getCurrentDs ( ) . getId ( ) ) ) ;
sql = MessageFormat . format ( " SELECT {0} FROM {1} " , f ,
DorisTableUtils . dorisName ( union . get ( 0 ) . getCurrentDs ( ) . getId ( ) ) ) ;
}
Map < String , Object > map = new HashMap < > ( ) ;
map . put ( " sql " , sql ) ;
@ -1179,18 +1250,23 @@ public class DataSetTableService {
}
/ / 递归计算出所有子级的checkedFields和unionParam
private void getUnionSQLDorisJoin ( List < UnionDTO > childrenDs , Map < String , String [ ] > checkedInfo , List < UnionParamDTO > unionList , List < DatasetTableField > checkedFields ) {
private void getUnionSQLDorisJoin ( List < UnionDTO > childrenDs , Map < String , String [ ] > checkedInfo ,
List < UnionParamDTO > unionList , List < DatasetTableField > checkedFields ) {
for ( int i = 0 ; i < childrenDs . size ( ) ; i + + ) {
UnionDTO unionDTO = childrenDs . get ( i ) ;
String tableId = unionDTO . getCurrentDs ( ) . getId ( ) ;
String table = DorisTableUtils . dorisName ( tableId ) ;
DatasetTable datasetTable = datasetTableMapper . selectByPrimaryKey ( tableId ) ;
if ( ObjectUtils . isEmpty ( datasetTable ) ) {
DEException . throwException ( Translator . get ( " i18n_custom_ds_delete " ) + String . format ( " :table id [%s] " , tableId ) ) ;
DEException . throwException (
Translator . get ( " i18n_custom_ds_delete " ) + String . format ( " :table id [%s] " , tableId ) ) ;
}
List < DatasetTableField > fields = dataSetTableFieldsService . getListByIdsEach ( unionDTO . getCurrentDsField ( ) ) ;
String [ ] array = fields . stream ( ) . map ( f - > table + " . " + f . getDataeaseName ( ) + " AS " + DorisTableUtils . dorisFieldName ( tableId + " _ " + f . getDataeaseName ( ) ) ) . toArray ( String [ ] : : new ) ;
String [ ] array = fields . stream ( )
. map ( f - > table + " . " + f . getDataeaseName ( ) + " AS "
+ DorisTableUtils . dorisFieldName ( tableId + " _ " + f . getDataeaseName ( ) ) )
. toArray ( String [ ] : : new ) ;
checkedInfo . put ( table , array ) ;
checkedFields . addAll ( fields ) ;
@ -1212,17 +1288,24 @@ public class DataSetTableService {
List < UnionParamDTO > unionList = new ArrayList < > ( ) ;
List < DatasetTableField > checkedFields = new ArrayList < > ( ) ;
String sql = " " ;
String tableName = new Gson ( ) . fromJson ( datasetTableMapper . selectByPrimaryKey ( union . get ( 0 ) . getCurrentDs ( ) . getId ( ) ) . getInfo ( ) , DataTableInfoDTO . class ) . getTable ( ) ;
String tableName = new Gson ( )
. fromJson ( datasetTableMapper . selectByPrimaryKey ( union . get ( 0 ) . getCurrentDs ( ) . getId ( ) ) . getInfo ( ) ,
DataTableInfoDTO . class )
. getTable ( ) ;
for ( UnionDTO unionDTO : union ) {
DatasetTable datasetTable = datasetTableMapper . selectByPrimaryKey ( unionDTO . getCurrentDs ( ) . getId ( ) ) ;
String table = new Gson ( ) . fromJson ( datasetTable . getInfo ( ) , DataTableInfoDTO . class ) . getTable ( ) ;
String tableId = unionDTO . getCurrentDs ( ) . getId ( ) ;
if ( ObjectUtils . isEmpty ( datasetTable ) ) {
DEException . throwException ( Translator . get ( " i18n_custom_ds_delete " ) + String . format ( " :table id [%s] " , tableId ) ) ;
DEException . throwException (
Translator . get ( " i18n_custom_ds_delete " ) + String . format ( " :table id [%s] " , tableId ) ) ;
}
List < DatasetTableField > fields = dataSetTableFieldsService . getListByIdsEach ( unionDTO . getCurrentDsField ( ) ) ;
String [ ] array = fields . stream ( ) . map ( f - > String . format ( keyword , table ) + " . " + String . format ( keyword , f . getOriginName ( ) ) + " AS " + DorisTableUtils . dorisFieldNameShort ( tableId + " _ " + f . getOriginName ( ) ) ) . toArray ( String [ ] : : new ) ;
String [ ] array = fields . stream ( )
. map ( f - > String . format ( keyword , table ) + " . " + String . format ( keyword , f . getOriginName ( ) ) + " AS "
+ DorisTableUtils . dorisFieldNameShort ( tableId + " _ " + f . getOriginName ( ) ) )
. toArray ( String [ ] : : new ) ;
checkedInfo . put ( table , array ) ;
checkedFields . addAll ( fields ) ;
/ / 获取child的fields和union
@ -1253,19 +1336,24 @@ public class DataSetTableService {
DatasetTable parentTable = datasetTableMapper . selectByPrimaryKey ( pField . getTableId ( ) ) ;
String parentTableName = new Gson ( ) . fromJson ( parentTable . getInfo ( ) , DataTableInfoDTO . class ) . getTable ( ) ;
DatasetTable currentTable = datasetTableMapper . selectByPrimaryKey ( cField . getTableId ( ) ) ;
String currentTableName = new Gson ( ) . fromJson ( currentTable . getInfo ( ) , DataTableInfoDTO . class ) . getTable ( ) ;
String currentTableName = new Gson ( ) . fromJson ( currentTable . getInfo ( ) , DataTableInfoDTO . class )
. getTable ( ) ;
join . append ( " " ) . append ( joinType ) . append ( " " ) . append ( String . format ( keyword , currentTableName ) )
. append ( " ON " ) ;
for ( int i = 0 ; i < unionParamDTO . getUnionFields ( ) . size ( ) ; i + + ) {
UnionItemDTO unionItemDTO = unionParamDTO . getUnionFields ( ) . get ( i ) ;
/ / 通过field id取得field详情 , 并且以第一组为准 , 寻找dataset table
DatasetTableField parentField = dataSetTableFieldsService . get ( unionItemDTO . getParentField ( ) . getId ( ) ) ;
DatasetTableField currentField = dataSetTableFieldsService . get ( unionItemDTO . getCurrentField ( ) . getId ( ) ) ;
DatasetTableField parentField = dataSetTableFieldsService
. get ( unionItemDTO . getParentField ( ) . getId ( ) ) ;
DatasetTableField currentField = dataSetTableFieldsService
. get ( unionItemDTO . getCurrentField ( ) . getId ( ) ) ;
join . append ( String . format ( keyword , parentTableName ) ) . append ( " . " ) . append ( String . format ( keyword , parentField . getOriginName ( ) ) )
join . append ( String . format ( keyword , parentTableName ) ) . append ( " . " )
. append ( String . format ( keyword , parentField . getOriginName ( ) ) )
. append ( " = " )
. append ( String . format ( keyword , currentTableName ) ) . append ( " . " ) . append ( String . format ( keyword , currentField . getOriginName ( ) ) ) ;
. append ( String . format ( keyword , currentTableName ) ) . append ( " . " )
. append ( String . format ( keyword , currentField . getOriginName ( ) ) ) ;
if ( i < unionParamDTO . getUnionFields ( ) . size ( ) - 1 ) {
join . append ( " AND " ) ;
}
@ -1300,20 +1388,25 @@ public class DataSetTableService {
}
/ / 递归计算出所有子级的checkedFields和unionParam
private void getUnionSQLDatasourceJoin ( List < UnionDTO > childrenDs , Map < String , String [ ] > checkedInfo , List < UnionParamDTO > unionList , String keyword , List < DatasetTableField > checkedFields ) {
private void getUnionSQLDatasourceJoin ( List < UnionDTO > childrenDs , Map < String , String [ ] > checkedInfo ,
List < UnionParamDTO > unionList , String keyword , List < DatasetTableField > checkedFields ) {
for ( int i = 0 ; i < childrenDs . size ( ) ; i + + ) {
UnionDTO unionDTO = childrenDs . get ( i ) ;
DatasetTable datasetTable = datasetTableMapper . selectByPrimaryKey ( unionDTO . getCurrentDs ( ) . getId ( ) ) ;
String tableId = unionDTO . getCurrentDs ( ) . getId ( ) ;
if ( ObjectUtils . isEmpty ( datasetTable ) ) {
DEException . throwException ( Translator . get ( " i18n_custom_ds_delete " ) + String . format ( " :table id [%s] " , tableId ) ) ;
DEException . throwException (
Translator . get ( " i18n_custom_ds_delete " ) + String . format ( " :table id [%s] " , tableId ) ) ;
}
String table = new Gson ( ) . fromJson ( datasetTable . getInfo ( ) , DataTableInfoDTO . class ) . getTable ( ) ;
List < DatasetTableField > fields = dataSetTableFieldsService . getListByIdsEach ( unionDTO . getCurrentDsField ( ) ) ;
String [ ] array = fields . stream ( ) . map ( f - > String . format ( keyword , table ) + " . " + String . format ( keyword , f . getOriginName ( ) ) + " AS " + DorisTableUtils . dorisFieldNameShort ( tableId + " _ " + f . getOriginName ( ) ) ) . toArray ( String [ ] : : new ) ;
String [ ] array = fields . stream ( )
. map ( f - > String . format ( keyword , table ) + " . " + String . format ( keyword , f . getOriginName ( ) ) + " AS "
+ DorisTableUtils . dorisFieldNameShort ( tableId + " _ " + f . getOriginName ( ) ) )
. toArray ( String [ ] : : new ) ;
checkedInfo . put ( table , array ) ;
checkedFields . addAll ( fields ) ;
@ -1366,7 +1459,8 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest ( ) ;
datasourceRequest . setDatasource ( ds ) ;
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
String sqlAsTable = qp . createSQLPreview ( new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) , DataTableInfoDTO . class ) . getSql ( ) , null ) ;
String sqlAsTable = qp . createSQLPreview (
new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) , DataTableInfoDTO . class ) . getSql ( ) , null ) ;
datasourceRequest . setQuery ( sqlAsTable ) ;
fields = datasourceProvider . fetchResultField ( datasourceRequest ) ;
} else if ( StringUtils . equalsIgnoreCase ( datasetTable . getType ( ) , " excel " ) ) {
@ -1374,12 +1468,15 @@ public class DataSetTableService {
} else if ( StringUtils . equalsIgnoreCase ( datasetTable . getType ( ) , " custom " ) ) {
if ( datasetTable . getMode ( ) = = 1 ) {
/ / save field
DataTableInfoDTO dataTableInfoDTO = new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) , DataTableInfoDTO . class ) ;
DataTableInfoDTO dataTableInfoDTO = new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) ,
DataTableInfoDTO . class ) ;
List < DataTableInfoCustomUnion > list = dataTableInfoDTO . getList ( ) ;
List < DatasetTableField > fieldList = new ArrayList < > ( ) ;
list . forEach ( ele - > {
List < DatasetTableField > listByIds = dataSetTableFieldsService . getListByIdsEach ( ele . getCheckedFields ( ) ) ;
listByIds . forEach ( f - > f . setDataeaseName ( DorisTableUtils . dorisFieldName ( ele . getTableId ( ) + " _ " + f . getDataeaseName ( ) ) ) ) ;
List < DatasetTableField > listByIds = dataSetTableFieldsService
. getListByIdsEach ( ele . getCheckedFields ( ) ) ;
listByIds . forEach ( f - > f . setDataeaseName (
DorisTableUtils . dorisFieldName ( ele . getTableId ( ) + " _ " + f . getDataeaseName ( ) ) ) ) ;
fieldList . addAll ( listByIds ) ;
} ) ;
for ( int i = 0 ; i < fieldList . size ( ) ; i + + ) {
@ -1392,7 +1489,8 @@ public class DataSetTableService {
dataSetTableFieldsService . batchEdit ( fieldList ) ;
/ / custom 创建doris视图
if ( datasetTable . getMode ( ) = = 1 ) {
createDorisView ( DorisTableUtils . dorisName ( datasetTable . getId ( ) ) , getCustomSQLDoris ( dataTableInfoDTO , dataSetTableUnionService . listByTableId ( dataTableInfoDTO . getList ( ) . get ( 0 ) . getTableId ( ) ) ) ) ;
createDorisView ( DorisTableUtils . dorisName ( datasetTable . getId ( ) ) , getCustomSQLDoris ( dataTableInfoDTO ,
dataSetTableUnionService . listByTableId ( dataTableInfoDTO . getList ( ) . get ( 0 ) . getTableId ( ) ) ) ) ;
}
return ;
} else {
@ -1400,21 +1498,26 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest ( ) ;
datasourceRequest . setDatasource ( ds ) ;
DataTableInfoDTO dt = new Gson ( ) . fromJson ( datasetTable . getInfo ( ) , DataTableInfoDTO . class ) ;
List < DataSetTableUnionDTO > list = dataSetTableUnionService . listByTableId ( dt . getList ( ) . get ( 0 ) . getTableId ( ) ) ;
List < DataSetTableUnionDTO > list = dataSetTableUnionService
. listByTableId ( dt . getList ( ) . get ( 0 ) . getTableId ( ) ) ;
String sqlAsTable = getCustomSQLDatasource ( dt , list , ds ) ;
datasourceRequest . setQuery ( sqlAsTable ) ;
fields = datasourceProvider . fetchResultField ( datasourceRequest ) ;
DataTableInfoDTO dataTableInfoDTO = new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) , DataTableInfoDTO . class ) ;
DataTableInfoDTO dataTableInfoDTO = new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) ,
DataTableInfoDTO . class ) ;
List < DataTableInfoCustomUnion > listField = dataTableInfoDTO . getList ( ) ;
List < DatasetTableField > fieldList = new ArrayList < > ( ) ;
listField . forEach ( ele - > {
List < DatasetTableField > listByIds = dataSetTableFieldsService . getListByIdsEach ( ele . getCheckedFields ( ) ) ;
List < DatasetTableField > listByIds = dataSetTableFieldsService
. getListByIdsEach ( ele . getCheckedFields ( ) ) ;
fieldList . addAll ( listByIds ) ;
} ) ;
for ( DatasetTableField field : fieldList ) {
for ( TableFiled tableFiled : fields ) {
if ( StringUtils . equalsIgnoreCase ( DorisTableUtils . dorisFieldName ( field . getTableId ( ) + " _ " + field . getOriginName ( ) ) , tableFiled . getFieldName ( ) ) ) {
if ( StringUtils . equalsIgnoreCase (
DorisTableUtils . dorisFieldName ( field . getTableId ( ) + " _ " + field . getOriginName ( ) ) ,
tableFiled . getFieldName ( ) ) ) {
tableFiled . setRemarks ( field . getName ( ) ) ;
break ;
}
@ -1428,7 +1531,8 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest ( ) ;
datasourceRequest . setDatasource ( ds ) ;
/ / save field
DataTableInfoDTO dataTableInfoDTO = new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) , DataTableInfoDTO . class ) ;
DataTableInfoDTO dataTableInfoDTO = new Gson ( ) . fromJson ( dataSetTableRequest . getInfo ( ) ,
DataTableInfoDTO . class ) ;
Map < String , Object > sqlMap = getUnionSQLDoris ( dataTableInfoDTO ) ;
String sql = ( String ) sqlMap . get ( " sql " ) ;
List < DatasetTableField > fieldList = ( List < DatasetTableField > ) sqlMap . get ( " field " ) ;
@ -1441,7 +1545,9 @@ public class DataSetTableService {
fields = datasourceProvider . fetchResultField ( datasourceRequest ) ;
for ( DatasetTableField field : fieldList ) {
for ( TableFiled tableFiled : fields ) {
if ( StringUtils . equalsIgnoreCase ( DorisTableUtils . dorisFieldName ( field . getTableId ( ) + " _ " + field . getDataeaseName ( ) ) , tableFiled . getFieldName ( ) ) ) {
if ( StringUtils . equalsIgnoreCase (
DorisTableUtils . dorisFieldName ( field . getTableId ( ) + " _ " + field . getDataeaseName ( ) ) ,
tableFiled . getFieldName ( ) ) ) {
tableFiled . setRemarks ( field . getName ( ) ) ;
break ;
}
@ -1463,7 +1569,9 @@ public class DataSetTableService {
for ( DatasetTableField field : fieldList ) {
for ( TableFiled tableFiled : fields ) {
if ( StringUtils . equalsIgnoreCase ( DorisTableUtils . dorisFieldNameShort ( field . getTableId ( ) + " _ " + field . getOriginName ( ) ) , tableFiled . getFieldName ( ) ) ) {
if ( StringUtils . equalsIgnoreCase (
DorisTableUtils . dorisFieldNameShort ( field . getTableId ( ) + " _ " + field . getOriginName ( ) ) ,
tableFiled . getFieldName ( ) ) ) {
tableFiled . setRemarks ( field . getName ( ) ) ;
break ;
}
@ -1484,8 +1592,10 @@ public class DataSetTableService {
/ / 物理字段名设定为唯一 , 查询当前数据集下是否已存在该字段 , 存在则update , 不存在则insert
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample ( ) ;
/ / 字段名一致 , 认为字段没有改变
datasetTableFieldExample . createCriteria ( ) . andTableIdEqualTo ( datasetTable . getId ( ) ) . andOriginNameEqualTo ( filed . getFieldName ( ) ) ;
List < DatasetTableField > datasetTableFields = datasetTableFieldMapper . selectByExample ( datasetTableFieldExample ) ;
datasetTableFieldExample . createCriteria ( ) . andTableIdEqualTo ( datasetTable . getId ( ) )
. andOriginNameEqualTo ( filed . getFieldName ( ) ) ;
List < DatasetTableField > datasetTableFields = datasetTableFieldMapper
. selectByExample ( datasetTableFieldExample ) ;
if ( CollectionUtils . isNotEmpty ( datasetTableFields ) ) {
datasetTableField . setId ( datasetTableFields . get ( 0 ) . getId ( ) ) ;
datasetTableField . setOriginName ( filed . getFieldName ( ) ) ;
@ -1526,7 +1636,8 @@ public class DataSetTableService {
}
/ / delete 数据库中多余的字段
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample ( ) ;
datasetTableFieldExample . createCriteria ( ) . andTableIdEqualTo ( datasetTable . getId ( ) ) . andExtFieldEqualTo ( 0 ) . andOriginNameNotIn ( originNameList ) ;
datasetTableFieldExample . createCriteria ( ) . andTableIdEqualTo ( datasetTable . getId ( ) ) . andExtFieldEqualTo ( 0 )
. andOriginNameNotIn ( originNameList ) ;
datasetTableFieldMapper . deleteByExample ( datasetTableFieldExample ) ;
}
}
@ -1560,7 +1671,8 @@ public class DataSetTableService {
}
}
public DatasetTableIncrementalConfig incrementalConfig ( DatasetTableIncrementalConfig datasetTableIncrementalConfig ) {
public DatasetTableIncrementalConfig incrementalConfig (
DatasetTableIncrementalConfig datasetTableIncrementalConfig ) {
if ( StringUtils . isEmpty ( datasetTableIncrementalConfig . getTableId ( ) ) ) {
return new DatasetTableIncrementalConfig ( ) ;
}
@ -1581,7 +1693,6 @@ public class DataSetTableService {
return incrementalConfig ( datasetTableIncrementalConfig ) ;
}
public void saveIncrementalConfig ( DatasetTableIncrementalConfig datasetTableIncrementalConfig ) throws Exception {
if ( datasetTableIncrementalConfig = = null | | StringUtils . isEmpty ( datasetTableIncrementalConfig . getTableId ( ) ) ) {
return ;
@ -1608,21 +1719,25 @@ public class DataSetTableService {
return o1 . getColumnIndex ( ) . compareTo ( o2 . getColumnIndex ( ) ) ;
} ) ;
List < String > originNameFileds = datasetTableFields . stream ( ) . map ( DatasetTableField : : getOriginName ) . collect ( Collectors . toList ( ) ) ;
List < String > originNameFileds = datasetTableFields . stream ( ) . map ( DatasetTableField : : getOriginName )
. collect ( Collectors . toList ( ) ) ;
Datasource ds = datasourceMapper . selectByPrimaryKey ( datasetTable . getDataSourceId ( ) ) ;
QueryProvider qp = ProviderFactory . getQueryProvider ( ds . getType ( ) ) ;
DatasourceProvider datasourceProvider = ProviderFactory . getProvider ( ds . getType ( ) ) ;
DatasourceRequest datasourceRequest = new DatasourceRequest ( ) ;
datasourceRequest . setDatasource ( ds ) ;
if ( StringUtils . isNotEmpty ( datasetTableIncrementalConfig . getIncrementalAdd ( ) ) & & StringUtils . isNotEmpty ( datasetTableIncrementalConfig . getIncrementalAdd ( ) . replace ( " " , " " ) ) ) { / / 增量添加
String sql = datasetTableIncrementalConfig . getIncrementalAdd ( ) . replace ( lastUpdateTime , Long . valueOf ( System . currentTimeMillis ( ) ) . toString ( ) )
if ( StringUtils . isNotEmpty ( datasetTableIncrementalConfig . getIncrementalAdd ( ) )
& & StringUtils . isNotEmpty ( datasetTableIncrementalConfig . getIncrementalAdd ( ) . replace ( " " , " " ) ) ) { / / 增量添加
String sql = datasetTableIncrementalConfig . getIncrementalAdd ( )
. replace ( lastUpdateTime , Long . valueOf ( System . currentTimeMillis ( ) ) . toString ( ) )
. replace ( currentUpdateTime , Long . valueOf ( System . currentTimeMillis ( ) ) . toString ( ) ) ;
datasourceRequest . setQuery ( qp . wrapSql ( sql ) ) ;
List < String > sqlFileds = new ArrayList < > ( ) ;
try {
datasourceProvider . fetchResultField ( datasourceRequest ) . stream ( ) . map ( TableFiled : : getFieldName ) . forEach ( filed - > {
sqlFileds . add ( filed ) ;
} ) ;
datasourceProvider . fetchResultField ( datasourceRequest ) . stream ( ) . map ( TableFiled : : getFieldName )
. forEach ( filed - > {
sqlFileds . add ( filed ) ;
} ) ;
} catch ( Exception e ) {
DataEaseException . throwException ( Translator . get ( " i18n_check_sql_error " ) + e . getMessage ( ) ) ;
}
@ -1631,13 +1746,16 @@ public class DataSetTableService {
DataEaseException . throwException ( Translator . get ( " i18n_sql_add_not_matching " ) + sqlFileds . toString ( ) ) ;
}
}
if ( StringUtils . isNotEmpty ( datasetTableIncrementalConfig . getIncrementalDelete ( ) ) & & StringUtils . isNotEmpty ( datasetTableIncrementalConfig . getIncrementalDelete ( ) . replace ( " " , " " ) ) ) { / / 增量删除
String sql = datasetTableIncrementalConfig . getIncrementalDelete ( ) . replace ( lastUpdateTime , Long . valueOf ( System . currentTimeMillis ( ) ) . toString ( ) )
if ( StringUtils . isNotEmpty ( datasetTableIncrementalConfig . getIncrementalDelete ( ) )
& & StringUtils . isNotEmpty ( datasetTableIncrementalConfig . getIncrementalDelete ( ) . replace ( " " , " " ) ) ) { / / 增量删除
String sql = datasetTableIncrementalConfig . getIncrementalDelete ( )
. replace ( lastUpdateTime , Long . valueOf ( System . currentTimeMillis ( ) ) . toString ( ) )
. replace ( currentUpdateTime , Long . valueOf ( System . currentTimeMillis ( ) ) . toString ( ) ) ;
datasourceRequest . setQuery ( qp . wrapSql ( sql ) ) ;
List < String > sqlFileds = new ArrayList < > ( ) ;
try {
datasourceProvider . fetchResultField ( datasourceRequest ) . stream ( ) . map ( TableFiled : : getFieldName ) . forEach ( filed - > sqlFileds . add ( filed ) ) ;
datasourceProvider . fetchResultField ( datasourceRequest ) . stream ( ) . map ( TableFiled : : getFieldName )
. forEach ( filed - > sqlFileds . add ( filed ) ) ;
} catch ( Exception e ) {
DataEaseException . throwException ( Translator . get ( " i18n_check_sql_error " ) + e . getMessage ( ) ) ;
}
@ -1686,7 +1804,8 @@ public class DataSetTableService {
if ( StringUtils . isNotEmpty ( tableId ) & & editType = = 1 ) {
List < DatasetTableField > datasetTableFields = dataSetTableFieldsService . getFieldsByTableId ( tableId ) ;
datasetTableFields . stream ( ) . filter ( datasetTableField - > datasetTableField . getExtField ( ) = = 0 ) . collect ( Collectors . toList ( ) ) ;
datasetTableFields . stream ( ) . filter ( datasetTableField - > datasetTableField . getExtField ( ) = = 0 )
. collect ( Collectors . toList ( ) ) ;
datasetTableFields . sort ( ( o1 , o2 ) - > {
if ( o1 . getColumnIndex ( ) = = null ) {
return - 1 ;
@ -1696,7 +1815,8 @@ public class DataSetTableService {
}
return o1 . getColumnIndex ( ) . compareTo ( o2 . getColumnIndex ( ) ) ;
} ) ;
List < String > oldFields = datasetTableFields . stream ( ) . map ( DatasetTableField : : getOriginName ) . collect ( Collectors . toList ( ) ) ;
List < String > oldFields = datasetTableFields . stream ( ) . map ( DatasetTableField : : getOriginName )
. collect ( Collectors . toList ( ) ) ;
for ( ExcelSheetData excelSheetData : excelSheetDataList ) {
List < TableFiled > fields = excelSheetData . getFields ( ) ;
List < String > newFields = fields . stream ( ) . map ( TableFiled : : getRemarks ) . collect ( Collectors . toList ( ) ) ;
@ -1711,7 +1831,9 @@ public class DataSetTableService {
} else {
retrunSheetDataList = excelSheetDataList ;
}
retrunSheetDataList = retrunSheetDataList . stream ( ) . filter ( excelSheetData - > CollectionUtils . isNotEmpty ( excelSheetData . getFields ( ) ) ) . collect ( Collectors . toList ( ) ) ;
retrunSheetDataList = retrunSheetDataList . stream ( )
. filter ( excelSheetData - > CollectionUtils . isNotEmpty ( excelSheetData . getFields ( ) ) )
. collect ( Collectors . toList ( ) ) ;
/ / save file
String excelId = UUID . randomUUID ( ) . toString ( ) ;
String filePath = saveFile ( file , excelId ) ;
@ -1738,7 +1860,8 @@ public class DataSetTableService {
return excelFileData ;
}
private List < ExcelSheetData > parseExcel2 ( String filename , InputStream inputStream , boolean isPreview ) throws Exception {
private List < ExcelSheetData > parseExcel2 ( String filename , InputStream inputStream , boolean isPreview )
throws Exception {
List < ExcelSheetData > excelSheetDataList = new ArrayList < > ( ) ;
String suffix = filename . substring ( filename . lastIndexOf ( " . " ) + 1 ) ;
if ( StringUtils . equalsIgnoreCase ( suffix , " xls " ) ) {
@ -1754,7 +1877,8 @@ public class DataSetTableService {
inputStream . close ( ) ;
excelSheetDataList . forEach ( excelSheetData - > {
List < List < String > > data = excelSheetData . getData ( ) ;
String [ ] fieldArray = excelSheetData . getFields ( ) . stream ( ) . map ( TableFiled : : getFieldName ) . toArray ( String [ ] : : new ) ;
String [ ] fieldArray = excelSheetData . getFields ( ) . stream ( ) . map ( TableFiled : : getFieldName )
. toArray ( String [ ] : : new ) ;
List < Map < String , Object > > jsonArray = new ArrayList < > ( ) ;
if ( CollectionUtils . isNotEmpty ( data ) ) {
jsonArray = data . stream ( ) . map ( ele - > {
@ -1772,7 +1896,8 @@ public class DataSetTableService {
return excelSheetDataList ;
}
private Map < String , Object > parseExcel ( String filename , InputStream inputStream , boolean isPreview ) throws Exception {
private Map < String , Object > parseExcel ( String filename , InputStream inputStream , boolean isPreview )
throws Exception {
String suffix = filename . substring ( filename . lastIndexOf ( " . " ) + 1 ) ;
List < TableFiled > fields = new ArrayList < > ( ) ;
List < String [ ] > data = new ArrayList < > ( ) ;
@ -1925,7 +2050,6 @@ public class DataSetTableService {
return map ;
}
private String readCell ( Cell cell , boolean cellType , TableFiled tableFiled ) {
if ( cell = = null ) {
return " " ;
@ -1939,7 +2063,8 @@ public class DataSetTableService {
double eps = 1e - 10 ;
if ( value - Math . floor ( value ) < eps ) {
if ( cellType ) {
if ( StringUtils . isEmpty ( tableFiled . getFieldType ( ) ) | | tableFiled . getFieldType ( ) . equalsIgnoreCase ( " TEXT " ) ) {
if ( StringUtils . isEmpty ( tableFiled . getFieldType ( ) )
| | tableFiled . getFieldType ( ) . equalsIgnoreCase ( " TEXT " ) ) {
tableFiled . setFieldType ( " LONG " ) ;
}
}
@ -1990,7 +2115,8 @@ public class DataSetTableService {
double eps = 1e - 10 ;
if ( value - Math . floor ( value ) < eps ) {
if ( cellType ) {
if ( StringUtils . isEmpty ( tableFiled . getFieldType ( ) ) | | tableFiled . getFieldType ( ) . equalsIgnoreCase ( " TEXT " ) ) {
if ( StringUtils . isEmpty ( tableFiled . getFieldType ( ) )
| | tableFiled . getFieldType ( ) . equalsIgnoreCase ( " TEXT " ) ) {
tableFiled . setFieldType ( " LONG " ) ;
}
}
@ -2048,13 +2174,17 @@ public class DataSetTableService {
public void updateDatasetTableStatus ( ) {
List < QrtzSchedulerState > qrtzSchedulerStates = qrtzSchedulerStateMapper . selectByExample ( null ) ;
List < String > activeQrtzInstances = qrtzSchedulerStates . stream ( ) . filter ( qrtzSchedulerState - > qrtzSchedulerState . getLastCheckinTime ( ) + qrtzSchedulerState . getCheckinInterval ( ) + 1000 > utilMapper . currentTimestamp ( ) ) . map ( QrtzSchedulerStateKey : : getInstanceName ) . collect ( Collectors . toList ( ) ) ;
List < String > activeQrtzInstances = qrtzSchedulerStates . stream ( )
. filter ( qrtzSchedulerState - > qrtzSchedulerState . getLastCheckinTime ( )
+ qrtzSchedulerState . getCheckinInterval ( ) + 1000 > utilMapper . currentTimestamp ( ) )
. map ( QrtzSchedulerStateKey : : getInstanceName ) . collect ( Collectors . toList ( ) ) ;
List < DatasetTable > jobStoppeddDatasetTables = new ArrayList < > ( ) ;
DatasetTableExample example = new DatasetTableExample ( ) ;
example . createCriteria ( ) . andSyncStatusEqualTo ( JobStatus . Underway . name ( ) ) ;
datasetTableMapper . selectByExample ( example ) . forEach ( datasetTable - > {
if ( StringUtils . isEmpty ( datasetTable . getQrtzInstance ( ) ) | | ! activeQrtzInstances . contains ( datasetTable . getQrtzInstance ( ) . substring ( 0 , datasetTable . getQrtzInstance ( ) . length ( ) - 13 ) ) ) {
if ( StringUtils . isEmpty ( datasetTable . getQrtzInstance ( ) ) | | ! activeQrtzInstances . contains (
datasetTable . getQrtzInstance ( ) . substring ( 0 , datasetTable . getQrtzInstance ( ) . length ( ) - 13 ) ) ) {
jobStoppeddDatasetTables . add ( datasetTable ) ;
}
} ) ;
@ -2066,7 +2196,8 @@ public class DataSetTableService {
DatasetTable record = new DatasetTable ( ) ;
record . setSyncStatus ( JobStatus . Error . name ( ) ) ;
example . clear ( ) ;
example . createCriteria ( ) . andSyncStatusEqualTo ( JobStatus . Underway . name ( ) ) . andIdIn ( jobStoppeddDatasetTables . stream ( ) . map ( DatasetTable : : getId ) . collect ( Collectors . toList ( ) ) ) ;
example . createCriteria ( ) . andSyncStatusEqualTo ( JobStatus . Underway . name ( ) )
. andIdIn ( jobStoppeddDatasetTables . stream ( ) . map ( DatasetTable : : getId ) . collect ( Collectors . toList ( ) ) ) ;
datasetTableMapper . updateByExampleSelective ( record , example ) ;
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog ( ) ;
@ -2074,8 +2205,10 @@ public class DataSetTableService {
datasetTableTaskLog . setInfo ( " Job stopped due to system error. " ) ;
DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample ( ) ;
datasetTableTaskLogExample . createCriteria ( ) . andStatusEqualTo ( JobStatus . Underway . name ( ) ) . andTableIdIn ( jobStoppeddDatasetTables . stream ( ) . map ( DatasetTable : : getId ) . collect ( Collectors . toList ( ) ) ) ;
List < String > taskIds = datasetTableTaskLogMapper . selectByExample ( datasetTableTaskLogExample ) . stream ( ) . map ( DatasetTableTaskLog : : getTaskId ) . collect ( Collectors . toList ( ) ) ;
datasetTableTaskLogExample . createCriteria ( ) . andStatusEqualTo ( JobStatus . Underway . name ( ) )
. andTableIdIn ( jobStoppeddDatasetTables . stream ( ) . map ( DatasetTable : : getId ) . collect ( Collectors . toList ( ) ) ) ;
List < String > taskIds = datasetTableTaskLogMapper . selectByExample ( datasetTableTaskLogExample ) . stream ( )
. map ( DatasetTableTaskLog : : getTaskId ) . collect ( Collectors . toList ( ) ) ;
datasetTableTaskLogMapper . updateByExampleSelective ( datasetTableTaskLog , datasetTableTaskLogExample ) ;
dataSetTableTaskService . updateTaskStatus ( taskIds , JobStatus . Error ) ;