center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat: 新增数据导出中心功能

Browse Source
This commit is contained in:
taojinlong 2024-04-07 17:54:55 +08:00
parent a87e999e6f
commit 8050583f66
10 changed files with 232 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/backend/src/main/java/io/dataease/listener/AppStartListener.java
View File

@ -7,6 +7,7 @@ import io.dataease.plugins.common.base.domain.DatasetTableTask;
import io.dataease.commons.constants.ScheduleType;
import io.dataease.service.ScheduleService;
import io.dataease.service.dataset.DataSetTableTaskService;
import io.dataease.service.exportCenter.ExportCenterService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationListener;
@ -22,6 +23,8 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
@Resource
private ScheduleService scheduleService;
@Resource
private ExportCenterService exportCenterService;
@Resource
private DataSetTableTaskService dataSetTableTaskService;
@Override
@ -51,5 +54,6 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
e.printStackTrace();
}
}
exportCenterService.reInitExportTask();
}
}

52
core/backend/src/main/java/io/dataease/service/exportCenter/ExportCenterService.java
View File

@ -55,6 +55,7 @@ import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.InetAddress;
import java.text.DecimalFormat;
import java.util.*;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@ -222,6 +223,39 @@ public class ExportCenterService {
}
public void reInitExportTask() {
ExportTaskExample exportTaskExample = new ExportTaskExample();
ExportTaskExample.Criteria criteria = exportTaskExample.createCriteria();
criteria.andExportMachineNameEqualTo(hostName()).andExportStatusEqualTo("IN_PROGRESS");
ExportTask record = new ExportTask();
record.setExportStatus("FAILED");
exportTaskMapper.updateByExampleSelective(record, exportTaskExample);
exportTaskExample.clear();
criteria = exportTaskExample.createCriteria();
criteria.andExportMachineNameEqualTo(hostName()).andExportStatusEqualTo("PENDING");
exportTaskMapper.selectByExampleWithBLOBs(exportTaskExample).parallelStream().forEach(exportTask -> {
if(exportTask.getExportFromType().equalsIgnoreCase("dataset")){
DataSetExportRequest request = new Gson().fromJson(exportTask.getParams(), DataSetExportRequest.class);
startDatasetTask(exportTask, request);
}
if(exportTask.getExportFromType().equalsIgnoreCase("chart")){
PanelViewDetailsRequest request = new Gson().fromJson(exportTask.getParams(), PanelViewDetailsRequest.class);
startViewTask(exportTask, request);
}
});
}
private String hostName() {
String hostname = null;
try {
InetAddress localMachine = InetAddress.getLocalHost();
hostname = localMachine.getHostName();
} catch (Exception e) {
DataEaseException.throwException("请设置主机名！");
}
return hostname;
}
public void addTask(String exportFrom, String exportFromType, PanelViewDetailsRequest request) {
ExportTask exportTask = new ExportTask();
exportTask.setId(UUID.randomUUID().toString());
@ -232,8 +266,13 @@ public class ExportCenterService {
exportTask.setFileName(request.getViewName() + ".xlsx");
exportTask.setExportPogress("0");
exportTask.setExportTime(System.currentTimeMillis());
exportTask.setParams(new Gson().toJson(request));
exportTask.setExportMachineName(hostName());
exportTaskMapper.insert(exportTask);
startViewTask(exportTask, request);
}
private void startViewTask(ExportTask exportTask, PanelViewDetailsRequest request){
String dataPath = exportData_path + exportTask.getId();
File directory = new File(dataPath);
boolean isCreated = directory.mkdir();
@ -408,10 +447,7 @@ public class ExportCenterService {
exportTaskMapper.updateByPrimaryKey(exportTask);
}
});
}
public void addTask(String exportFrom, String exportFromType, DataSetExportRequest request) {
ExportTask exportTask = new ExportTask();
exportTask.setId(UUID.randomUUID().toString());
@ -422,13 +458,17 @@ public class ExportCenterService {
exportTask.setFileName(request.getFilename() + ".xlsx");
exportTask.setExportPogress("0");
exportTask.setExportTime(System.currentTimeMillis());
exportTask.setParams(new Gson().toJson(request));
exportTask.setExportMachineName(hostName());
exportTaskMapper.insert(exportTask);
startDatasetTask(exportTask, request);
}
private void startDatasetTask(ExportTask exportTask, DataSetExportRequest request){
String dataPath = exportData_path + exportTask.getId();
File directory = new File(dataPath);
boolean isCreated = directory.mkdir();
scheduledThreadPoolExecutor.execute(() -> {
try {
exportTask.setExportStatus("IN_PROGRESS");
@ -613,8 +653,6 @@ public class ExportCenterService {
exportTaskMapper.updateByPrimaryKey(exportTask);
}
});
}
public Boolean checkEngineTableIsExists(String id) throws Exception {

2
core/backend/src/main/resources/db/migration/V64__1.18.19_export_task.sql
View File

@ -9,5 +9,7 @@ CREATE TABLE `export_task` (
`export_from_type` varchar(255) DEFAULT NULL,
`export_time` bigint(20) DEFAULT NULL,
`export_pogress` varchar(255) DEFAULT NULL,
`export_machine_name` varchar(512) DEFAULT NULL,
`params` longtext NOT NULL COMMENT '过滤参数',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_general_ci;

9
core/frontend/src/views/dataset/exportExcel/index.vue
View File

@ -52,7 +52,7 @@
</el-table-column>
<el-table-column prop="operate" width="80" label="操作">
<template slot-scope="scope">
<el-button v-if="activeName === 'SUCCESS'" type="text" size="mini" @click="downloadClick(scope.row)">
<el-button v-if="scope.row.exportStatus === 'SUCCESS'" type="text" size="mini" @click="downloadClick(scope.row)">
<div class="download-export">
<svg-icon
icon-class="icon_download_outlined"
@ -104,6 +104,10 @@ export default {
label: "失败(0)",
name: "FAILED",
},
{
label: "等待中(0)",
name: "PENDING",
},
{
label: "全部(0)",
name: "ALL",
@ -143,6 +147,9 @@ export default {
if(item.name === 'FAILED'){
item.label = '失败' + '(' + res.data.filter(task => task.exportStatus === 'FAILED').length + ')'
}
if(item.name === 'PENDING'){
item.label = '等待中' + '(' + res.data.filter(task => task.exportStatus === 'PENDING').length + ')'
}
})
if(this.activeName === 'ALL'){
this.tableData = res.data

1
installer/dataease/docker-compose.yml
View File

@ -15,6 +15,7 @@ services:
- ${DE_BASE}/dataease/data/static-resource:/opt/dataease/data/static-resource
- ${DE_BASE}/dataease/custom-drivers:/opt/dataease/custom-drivers
- ${DE_BASE}/dataease/data/custom:/opt/dataease/data/custom
- ${DE_BASE}/dataease/data/exportData:/opt/dataease/data/exportData
- ${DE_BASE}/dataease/data/business:/opt/dataease/data/business
- ${DE_BASE}/dataease/data/ehcache:/opt/dataease/data/ehcache
shm_size: 1gb

2
installer/dataease/install.sh
View File

@ -162,8 +162,10 @@ mkdir -p ${DE_RUN_BASE}/data/kettle
mkdir -p ${DE_RUN_BASE}/data/fe
mkdir -p ${DE_RUN_BASE}/data/be
mkdir -p ${DE_RUN_BASE}/data/mysql
mkdir -p ${DE_RUN_BASE}/data/exportData
mkdir -p ${DE_RUN_BASE}/data/static-resource
mkdir -p ${DE_RUN_BASE}/custom-drivers
mkdir -p ${DE_RUN_BASE}/custom-drivers
mkdir -p ${DE_RUN_BASE}/data/business

4
sdk/dataease-plugin-common/src/main/java/io/dataease/plugins/common/base/domain/ExportTask.java
View File

@ -23,5 +23,9 @@ public class ExportTask implements Serializable {
private String exportPogress;
private String exportMachineName;
private String params;
private static final long serialVersionUID = 1L;
}

70
sdk/dataease-plugin-common/src/main/java/io/dataease/plugins/common/base/domain/ExportTaskExample.java
View File

@ -703,6 +703,76 @@ public class ExportTaskExample {
addCriterion("export_pogress not between", value1, value2, "exportPogress");
return (Criteria) this;
}
public Criteria andExportMachineNameIsNull() {
addCriterion("export_machine_name is null");
return (Criteria) this;
}
public Criteria andExportMachineNameIsNotNull() {
addCriterion("export_machine_name is not null");
return (Criteria) this;
}
public Criteria andExportMachineNameEqualTo(String value) {
addCriterion("export_machine_name =", value, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameNotEqualTo(String value) {
addCriterion("export_machine_name <>", value, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameGreaterThan(String value) {
addCriterion("export_machine_name >", value, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameGreaterThanOrEqualTo(String value) {
addCriterion("export_machine_name >=", value, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameLessThan(String value) {
addCriterion("export_machine_name <", value, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameLessThanOrEqualTo(String value) {
addCriterion("export_machine_name <=", value, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameLike(String value) {
addCriterion("export_machine_name like", value, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameNotLike(String value) {
addCriterion("export_machine_name not like", value, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameIn(List<String> values) {
addCriterion("export_machine_name in", values, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameNotIn(List<String> values) {
addCriterion("export_machine_name not in", values, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameBetween(String value1, String value2) {
addCriterion("export_machine_name between", value1, value2, "exportMachineName");
return (Criteria) this;
}
public Criteria andExportMachineNameNotBetween(String value1, String value2) {
addCriterion("export_machine_name not between", value1, value2, "exportMachineName");
return (Criteria) this;
}
}
public static class Criteria extends GeneratedCriteria {

6
sdk/dataease-plugin-common/src/main/java/io/dataease/plugins/common/base/mapper/ExportTaskMapper.java
View File

@ -16,15 +16,21 @@ public interface ExportTaskMapper {
int insertSelective(ExportTask record);
List<ExportTask> selectByExampleWithBLOBs(ExportTaskExample example);
List<ExportTask> selectByExample(ExportTaskExample example);
ExportTask selectByPrimaryKey(String id);
int updateByExampleSelective(@Param("record") ExportTask record, @Param("example") ExportTaskExample example);
int updateByExampleWithBLOBs(@Param("record") ExportTask record, @Param("example") ExportTaskExample example);
int updateByExample(@Param("record") ExportTask record, @Param("example") ExportTaskExample example);
int updateByPrimaryKeySelective(ExportTask record);
int updateByPrimaryKeyWithBLOBs(ExportTask record);
int updateByPrimaryKey(ExportTask record);
}

98
sdk/dataease-plugin-common/src/main/java/io/dataease/plugins/common/base/mapper/ExportTaskMapper.xml
View File

@ -11,6 +11,10 @@
<result column="export_from_type" jdbcType="VARCHAR" property="exportFromType" />
<result column="export_time" jdbcType="BIGINT" property="exportTime" />
<result column="export_pogress" jdbcType="VARCHAR" property="exportPogress" />
<result column="export_machine_name" jdbcType="VARCHAR" property="exportMachineName" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="io.dataease.plugins.common.base.domain.ExportTask">
<result column="params" jdbcType="LONGVARCHAR" property="params" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
@ -72,8 +76,27 @@
</sql>
<sql id="Base_Column_List">
id, user_id, file_name, file_size, export_from, export_status, export_from_type,
export_time, export_pogress
export_time, export_pogress, export_machine_name
</sql>
<sql id="Blob_Column_List">
params
</sql>
<select id="selectByExampleWithBLOBs" parameterType="io.dataease.plugins.common.base.domain.ExportTaskExample" resultMap="ResultMapWithBLOBs">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from export_task
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByExample" parameterType="io.dataease.plugins.common.base.domain.ExportTaskExample" resultMap="BaseResultMap">
select
<if test="distinct">
@ -88,9 +111,11 @@
order by ${orderByClause}
</if>
</select>
<select id="selectByPrimaryKey" parameterType="java.lang.String" resultMap="BaseResultMap">
<select id="selectByPrimaryKey" parameterType="java.lang.String" resultMap="ResultMapWithBLOBs">
select
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from export_task
where id = #{id,jdbcType=VARCHAR}
</select>
@ -107,12 +132,12 @@
<insert id="insert" parameterType="io.dataease.plugins.common.base.domain.ExportTask">
insert into export_task (id, user_id, file_name,
file_size, export_from, export_status,
export_from_type, export_time, export_pogress
)
export_from_type, export_time, export_pogress,
export_machine_name, params)
values (#{id,jdbcType=VARCHAR}, #{userId,jdbcType=BIGINT}, #{fileName,jdbcType=VARCHAR},
#{fileSize,jdbcType=DOUBLE}, #{exportFrom,jdbcType=VARCHAR}, #{exportStatus,jdbcType=VARCHAR},
#{exportFromType,jdbcType=VARCHAR}, #{exportTime,jdbcType=BIGINT}, #{exportPogress,jdbcType=VARCHAR}
)
#{exportFromType,jdbcType=VARCHAR}, #{exportTime,jdbcType=BIGINT}, #{exportPogress,jdbcType=VARCHAR},
#{exportMachineName,jdbcType=VARCHAR}, #{params,jdbcType=LONGVARCHAR})
</insert>
<insert id="insertSelective" parameterType="io.dataease.plugins.common.base.domain.ExportTask">
insert into export_task
@ -144,6 +169,12 @@
<if test="exportPogress != null">
export_pogress,
</if>
<if test="exportMachineName != null">
export_machine_name,
</if>
<if test="params != null">
params,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
@ -173,6 +204,12 @@
<if test="exportPogress != null">
#{exportPogress,jdbcType=VARCHAR},
</if>
<if test="exportMachineName != null">
#{exportMachineName,jdbcType=VARCHAR},
</if>
<if test="params != null">
#{params,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="io.dataease.plugins.common.base.domain.ExportTaskExample" resultType="java.lang.Long">
@ -211,11 +248,34 @@
<if test="record.exportPogress != null">
export_pogress = #{record.exportPogress,jdbcType=VARCHAR},
</if>
<if test="record.exportMachineName != null">
export_machine_name = #{record.exportMachineName,jdbcType=VARCHAR},
</if>
<if test="record.params != null">
params = #{record.params,jdbcType=LONGVARCHAR},
</if>
</set>
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByExampleWithBLOBs" parameterType="map">
update export_task
set id = #{record.id,jdbcType=VARCHAR},
user_id = #{record.userId,jdbcType=BIGINT},
file_name = #{record.fileName,jdbcType=VARCHAR},
file_size = #{record.fileSize,jdbcType=DOUBLE},
export_from = #{record.exportFrom,jdbcType=VARCHAR},
export_status = #{record.exportStatus,jdbcType=VARCHAR},
export_from_type = #{record.exportFromType,jdbcType=VARCHAR},
export_time = #{record.exportTime,jdbcType=BIGINT},
export_pogress = #{record.exportPogress,jdbcType=VARCHAR},
export_machine_name = #{record.exportMachineName,jdbcType=VARCHAR},
params = #{record.params,jdbcType=LONGVARCHAR}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByExample" parameterType="map">
update export_task
set id = #{record.id,jdbcType=VARCHAR},
@ -226,7 +286,8 @@
export_status = #{record.exportStatus,jdbcType=VARCHAR},
export_from_type = #{record.exportFromType,jdbcType=VARCHAR},
export_time = #{record.exportTime,jdbcType=BIGINT},
export_pogress = #{record.exportPogress,jdbcType=VARCHAR}
export_pogress = #{record.exportPogress,jdbcType=VARCHAR},
export_machine_name = #{record.exportMachineName,jdbcType=VARCHAR}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
@ -258,9 +319,29 @@
<if test="exportPogress != null">
export_pogress = #{exportPogress,jdbcType=VARCHAR},
</if>
<if test="exportMachineName != null">
export_machine_name = #{exportMachineName,jdbcType=VARCHAR},
</if>
<if test="params != null">
params = #{params,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=VARCHAR}
</update>
<update id="updateByPrimaryKeyWithBLOBs" parameterType="io.dataease.plugins.common.base.domain.ExportTask">
update export_task
set user_id = #{userId,jdbcType=BIGINT},
file_name = #{fileName,jdbcType=VARCHAR},
file_size = #{fileSize,jdbcType=DOUBLE},
export_from = #{exportFrom,jdbcType=VARCHAR},
export_status = #{exportStatus,jdbcType=VARCHAR},
export_from_type = #{exportFromType,jdbcType=VARCHAR},
export_time = #{exportTime,jdbcType=BIGINT},
export_pogress = #{exportPogress,jdbcType=VARCHAR},
export_machine_name = #{exportMachineName,jdbcType=VARCHAR},
params = #{params,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=VARCHAR}
</update>
<update id="updateByPrimaryKey" parameterType="io.dataease.plugins.common.base.domain.ExportTask">
update export_task
set user_id = #{userId,jdbcType=BIGINT},
@ -270,7 +351,8 @@
export_status = #{exportStatus,jdbcType=VARCHAR},
export_from_type = #{exportFromType,jdbcType=VARCHAR},
export_time = #{exportTime,jdbcType=BIGINT},
export_pogress = #{exportPogress,jdbcType=VARCHAR}
export_pogress = #{exportPogress,jdbcType=VARCHAR},
export_machine_name = #{exportMachineName,jdbcType=VARCHAR}
where id = #{id,jdbcType=VARCHAR}
</update>
</mapper>