center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat: 数据导出中心

Browse Source
This commit is contained in:
taojinlong 2024-05-24 17:25:52 +08:00
parent 539d18581b
commit 59b97fdee2
12 changed files with 213 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -1,6 +1,6 @@
FROM registry.cn-qingdao.aliyuncs.com/dataease/alpine-openjdk17-jre
RUN mkdir -p /opt/apps/config /opt/dataease2.0/drivers/ /opt/dataease2.0/cache/ /opt/dataease2.0/data/map /opt/dataease2.0/data/static-resource/ /opt/dataease2.0/data/appearance/
RUN mkdir -p /opt/apps/config /opt/dataease2.0/drivers/ /opt/dataease2.0/cache/ /opt/dataease2.0/data/map /opt/dataease2.0/data/static-resource/ /opt/dataease2.0/data/appearance/ /opt/dataease2.0/data/exportData/
ADD drivers/* /opt/dataease2.0/drivers/
ADD mapFiles/ /opt/dataease2.0/data/map/

15
core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java
View File

@ -5,13 +5,16 @@ import io.dataease.api.chart.dto.ChartViewDTO;
import io.dataease.api.chart.dto.ViewDetailField;
import io.dataease.api.chart.request.ChartExcelRequest;
import io.dataease.chart.manage.ChartDataManage;
import io.dataease.constant.AuthConstant;
import io.dataease.constant.CommonConstants;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.exception.DEException;
import io.dataease.exportCenter.manage.ExportCenterManage;
import io.dataease.result.ResultCode;
import io.dataease.utils.LogUtil;
import io.dataease.visualization.manage.VisualizationTemplateExtendDataManage;
import jakarta.annotation.Resource;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ArrayUtils;
@ -22,12 +25,13 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
@ -38,10 +42,11 @@ import java.util.stream.Collectors;
public class ChartDataServer implements ChartDataApi {
@Resource
private ChartDataManage chartDataManage;
@Resource
private ExportCenterManage exportCenterManage;
@Resource
private VisualizationTemplateExtendDataManage extendDataManage;
@Value("${export.views.limit:500000}")
private Integer limit;
@ -76,6 +81,12 @@ public class ChartDataServer implements ChartDataApi {
@Override
public void innerExportDetails(ChartExcelRequest request, HttpServletResponse response) throws Exception {
HttpServletRequest httpServletRequest = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest();
String linkToken = httpServletRequest.getHeader(AuthConstant.LINK_TOKEN_KEY);
if (StringUtils.isEmpty(linkToken)) {
exportCenterManage.addTask(request.getViewId(), "chart", request);
return;
}
OutputStream outputStream = response.getOutputStream();
try {
findExcelData(request);

47
core/core-backend/src/main/java/io/dataease/visualization/server/DataVisualizationServer.java
View File

@ -13,6 +13,7 @@ import io.dataease.api.visualization.vo.DataVisualizationVO;
import io.dataease.api.visualization.vo.VisualizationResourceVO;
import io.dataease.api.visualization.vo.VisualizationWatermarkVO;
import io.dataease.chart.dao.auto.entity.CoreChartView;
import io.dataease.chart.dao.auto.mapper.CoreChartViewMapper;
import io.dataease.chart.manage.ChartDataManage;
import io.dataease.chart.manage.ChartViewManege;
import io.dataease.commons.constants.DataVisualizationConstants;
@ -41,6 +42,7 @@ import io.dataease.visualization.dao.auto.mapper.VisualizationWatermarkMapper;
import io.dataease.visualization.dao.ext.mapper.ExtDataVisualizationMapper;
import io.dataease.visualization.manage.CoreVisualizationManage;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.ObjectUtils;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.RequestBody;
@ -48,10 +50,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
@ -64,6 +63,8 @@ public class DataVisualizationServer implements DataVisualizationApi {
@Resource
private ChartViewManege chartViewManege;
@Resource
private CoreChartViewMapper coreChartViewMapper;
@Resource
private ExtDataVisualizationMapper extDataVisualizationMapper;
@ -391,4 +392,42 @@ public class DataVisualizationServer implements DataVisualizationApi {
}
}
public String getAbsPath(String id) {
CoreChartView coreChartView = coreChartViewMapper.selectById(id);
if (coreChartView == null) {
return null;
}
if (coreChartView.getSceneId() == null) {
return coreChartView.getTitle();
}
List<DataVisualizationInfo> parents = getParents(coreChartView.getSceneId());
StringBuilder stringBuilder = new StringBuilder();
parents.forEach(ele -> {
if (ObjectUtils.isNotEmpty(ele)) {
stringBuilder.append(ele.getName()).append("/");
}
});
stringBuilder.append(coreChartView.getTitle());
return stringBuilder.toString();
}
public List<DataVisualizationInfo> getParents(Long id) {
List<DataVisualizationInfo> list = new ArrayList<>();
DataVisualizationInfo dataVisualizationInfo = visualizationInfoMapper.selectById(id);
list.add(dataVisualizationInfo);
getParent(list, dataVisualizationInfo);
Collections.reverse(list);
return list;
}
public void getParent(List<DataVisualizationInfo> list, DataVisualizationInfo dataVisualizationInfo) {
if (ObjectUtils.isNotEmpty(dataVisualizationInfo)) {
if (dataVisualizationInfo.getPid() != null) {
DataVisualizationInfo d = visualizationInfoMapper.selectById(dataVisualizationInfo.getPid());
list.add(d);
getParent(list, d);
}
}
}
}

21
core/core-backend/src/main/resources/db/migration/V2.7__ddl.sql
View File

@ -21,4 +21,23 @@ ALTER TABLE `xpack_setting_authentication`
ALTER TABLE `xpack_setting_authentication`
ADD COLUMN `synced` tinyint(1) NOT NULL DEFAULT 0 COMMENT '已同步' AFTER `plugin_json`;
ALTER TABLE `xpack_setting_authentication`
ADD COLUMN `valid` tinyint(1) NOT NULL DEFAULT 0 COMMENT '有效' AFTER `synced`;
ADD COLUMN `valid` tinyint(1) NOT NULL DEFAULT 0 COMMENT '有效' AFTER `synced`;
DROP TABLE IF EXISTS `core_export_task`;
CREATE TABLE `core_export_task`
(
`id` varchar(255) NOT NULL,
`user_id` bigint(20) NOT NULL,
`file_name` varchar(2048) DEFAULT NULL,
`file_size` DOUBLE DEFAULT NULL,
`file_size_unit` varchar(255) DEFAULT NULL,
`export_from` varchar(255) DEFAULT NULL,
`export_status` varchar(255) DEFAULT NULL,
`export_from_type` varchar(255) DEFAULT NULL,
`export_time` bigint(20) DEFAULT NULL,
`export_progress` varchar(255) DEFAULT NULL,
`export_machine_name` varchar(512) DEFAULT NULL,
`params` longtext NOT NULL COMMENT '过滤参数',
PRIMARY KEY (`id`)
) COMMENT='导出任务表';

2
core/core-frontend/src/api/dataset.ts
View File

@ -250,7 +250,7 @@ export const getFunction = async (): Promise<DatasetDetail[]> => {
export const exportTasks = async (type): Promise<IResponse> => {
return request.post({ url: '/exportCenter/exportTasks/' + type, data: {} }).then(res => {
return res?.data
return res
})
}

44
core/core-frontend/src/components/visualization/ComponentEditBar.vue
View File

@ -146,7 +146,7 @@
</template>
<script lang="ts" setup>
import { computed, onBeforeUnmount, onMounted, reactive, toRefs, watch } from 'vue'
import { computed, h, onBeforeUnmount, onMounted, reactive, toRefs, watch } from 'vue'
import { dvMainStoreWithOut } from '@/store/modules/data-visualization/dvMain'
import { storeToRefs } from 'pinia'
import { useI18n } from '@/hooks/web/useI18n'
@ -156,7 +156,8 @@ import { useEmitt } from '@/hooks/web/useEmitt'
import { copyStoreWithOut } from '@/store/modules/data-visualization/copy'
import { exportExcelDownload } from '@/views/chart/components/js/util'
import FieldsList from '@/custom-component/rich-text/FieldsList.vue'
import { ElTooltip } from 'element-plus-secondary'
import { ElMessage, ElTooltip } from 'element-plus-secondary'
import { Button } from 'vant'
const dvMainStore = dvMainStoreWithOut()
const snapshotStore = snapshotStoreWithOut()
const copyStore = copyStoreWithOut()
@ -312,12 +313,49 @@ const showBarTooltipPosition = computed(() => {
}
})
const exportData = () => {
// bus.$emit('data-export-center')
}
const openMessageLoading = cb => {
console.log('bb')
const iconClass = `el-icon-loading`
const customClass = `de-message-loading de-message-export`
ElMessage({
message: h('p', null, [
'后台导出中,可前往',
h(
Button,
{
props: {
type: 'text',
size: 'mini'
},
class: 'btn-text',
on: {
click: () => {
cb()
}
}
},
'数据导出中心'
),
'查看进度，进行下载'
]),
iconClass,
showClose: true,
customClass
})
}
const exportAsExcel = () => {
const viewDataInfo = dvMainStore.getViewDataDetails(element.value.id)
const chartExtRequest = dvMainStore.getLastViewRequestInfo(element.value.id)
const viewInfo = dvMainStore.getViewDetails(element.value.id)
const chart = { ...viewInfo, chartExtRequest, data: viewDataInfo }
exportExcelDownload(chart)
exportExcelDownload(chart, () => {
openMessageLoading(exportData)
})
}
const exportAsImage = () => {
// do export

40
core/core-frontend/src/components/visualization/UserViewEnlarge.vue
View File

@ -71,7 +71,7 @@
<script setup lang="ts">
import ComponentWrapper from '@/components/data-visualization/canvas/ComponentWrapper.vue'
import { computed, nextTick, ref } from 'vue'
import { computed, h, nextTick, ref } from 'vue'
import { toPng } from 'html-to-image'
import { useI18n } from '@/hooks/web/useI18n'
import { deepCopy } from '@/utils/utils'
@ -81,6 +81,8 @@ import { exportExcelDownload } from '@/views/chart/components/js/util'
import { storeToRefs } from 'pinia'
import { assign } from 'lodash-es'
import { useEmitt } from '@/hooks/web/useEmitt'
import { Button } from 'vant'
import { ElMessage } from 'element-plus-secondary'
const downLoading = ref(false)
const dvMainStore = dvMainStoreWithOut()
const dialogShow = ref(false)
@ -194,10 +196,46 @@ const downloadViewDetails = () => {
const chart = { ...viewInfo.value, chartExtRequest, data: viewDataInfo }
exportLoading.value = true
exportExcelDownload(chart, () => {
console.log('aa')
openMessageLoading(exportData)
exportLoading.value = false
})
}
const exportData = () => {
// bus.$emit('data-export-center')
}
const openMessageLoading = cb => {
console.log('bb')
const iconClass = `el-icon-loading`
const customClass = `de-message-loading de-message-export`
ElMessage({
message: h('p', null, [
'后台导出中,可前往',
h(
Button,
{
props: {
type: 'text',
size: 'mini'
},
class: 'btn-text',
on: {
click: () => {
cb()
}
}
},
'数据导出中心'
),
'查看进度，进行下载'
]),
iconClass,
showClose: true,
customClass
})
}
const htmlToImage = () => {
downLoading.value = true
useEmitt().emitter.emit('renderChart-' + viewInfo.value.id)

25
core/core-frontend/src/views/chart/components/js/util.ts
View File

@ -10,6 +10,7 @@ import { PickOptions } from '@antv/g2plot/esm/core/plot'
import { innerExportDetails } from '@/api/chart'
import { ElMessage } from 'element-plus-secondary'
import { useI18n } from '@/hooks/web/useI18n'
import { useLinkStoreWithOut } from '@/store/modules/link'
const { t } = useI18n()
// 同时支持将hex和rgb，转换成rgba
export function hexColorToRGBA(hex, alpha) {
@ -460,18 +461,24 @@ export const exportExcelDownload = (chart, callBack?) => {
viewInfo: chart,
detailFields
}
const linkStore = useLinkStoreWithOut()
const method = innerExportDetails
method(request)
.then(res => {
const blob = new Blob([res.data], { type: 'application/vnd.ms-excel' })
const link = document.createElement('a')
link.style.display = 'none'
link.href = URL.createObjectURL(blob)
link.download = excelName + '.xlsx' // 下载的文件名
document.body.appendChild(link)
link.click()
document.body.removeChild(link)
callBack('success')
if (linkStore.getLinkToken) {
const blob = new Blob([res.data], { type: 'application/vnd.ms-excel' })
const link = document.createElement('a')
link.style.display = 'none'
link.href = URL.createObjectURL(blob)
link.download = excelName + '.xlsx' // 下载的文件名
document.body.appendChild(link)
link.click()
document.body.removeChild(link)
callBack('success')
} else {
callBack && callBack(res)
}
})
.catch(() => {
console.error('Excel download error')

25
core/core-frontend/src/views/visualized/data/dataset/ExportExcel.vue
View File

@ -1,7 +1,7 @@
<script lang="ts" setup>
import { ref, h, onUnmounted } from 'vue'
import { EmptyBackground } from '@/components/empty-background'
import { ElButton, ElMessage, ElMessageBox } from 'element-plus-secondary'
import { ElButton, ElMessage, ElMessageBox, ElTabPane, ElTabs } from 'element-plus-secondary'
import { RefreshLeft } from '@element-plus/icons-vue'
import {
exportTasks,
@ -54,7 +54,10 @@ const handleClose = () => {
onUnmounted(() => {
clearInterval(timer)
})
const handleClick = () => {
const handleClick = tab => {
if (tab) {
activeName.value = tab.paneName
}
if (activeName.value === 'ALL') {
description.value = t('data_export.no_file')
} else if (activeName.value === 'FAILED') {
@ -305,7 +308,7 @@ const delAll = () => {
})
.then(() => {
exportDeleteAll(
activeName,
activeName.value,
multipleSelection.value.map(ele => ele.id)
).then(() => {
ElMessage.success(t('commons.delete_success'))
@ -410,8 +413,8 @@ defineExpose({
<el-table-column prop="exportFromName" :label="$t('data_export.export_obj')" width="200" />
<el-table-column prop="exportFromType" width="120" :label="$t('data_export.export_from')">
<template #default="scope">
<span v-if="scope.row.exportFromType === 'dataset'">{{ $t('dataset.datalist') }}</span>
<span v-if="scope.row.exportFromType === 'chart'">{{ $t('panel.view') }}</span>
<span v-if="scope.row.exportFromType === 'dataset'">数据集</span>
<span v-if="scope.row.exportFromType === 'chart'">视图</span>
</template>
</el-table-column>
<el-table-column prop="exportTime" width="180" :label="$t('data_export.export_time')">
@ -419,7 +422,7 @@ defineExpose({
<span>{{ timestampFormatDate(scope.row.exportTime) }}</span>
</template>
</el-table-column>
<el-table-column fixed="right" prop="operate" width="80" :label="$t('commons.operating')">
<el-table-column fixed="right" prop="operate" width="150" :label="$t('commons.operating')">
<template #default="scope">
<el-button
v-if="scope.row.exportStatus === 'SUCCESS'"
@ -432,18 +435,14 @@ defineExpose({
</el-icon>
</div>
</el-button>
<el-button
v-if="scope.row.exportStatus === 'FAILED'"
type="text"
@click="retry(scope.row)"
>
<el-button type="text" @click="retry(scope.row)">
<template #icon>
<Icon name="dv-preview-download"></Icon>
<Icon name="de-refresh"></Icon>
</template>
</el-button>
<el-button type="text" @click="deleteField(scope.row)">
<template #icon>
<Icon name="dv-preview-download"></Icon>
<Icon name="de-delete"></Icon>
</template>
</el-button>
</template>

1
installer/dataease/docker-compose.yml
View File

@ -13,6 +13,7 @@ services:
- ${DE_BASE}/dataease2.0/cache:/opt/dataease2.0/cache
- ${DE_BASE}/dataease2.0/data/geo:/opt/dataease2.0/data/geo
- ${DE_BASE}/dataease2.0/data/appearance:/opt/dataease2.0/data/appearance
- ${DE_BASE}/dataease2.0/data/exportData:/opt/dataease2.0/data/exportData
depends_on:
DE_MYSQL_HOST:
condition: service_healthy

12
installer/install.sh
View File

@ -27,7 +27,7 @@ function prop {
function check_and_prepare_env_params() {
log "当前时间 : $(date)"
log_title "检查安装环境并初始化环境变量"
log_title "检查安装环境并初始化环境变量"
cd ${CURRENT_DIR}
if [ -f /usr/bin/dectl ]; then
@ -39,10 +39,10 @@ function check_and_prepare_env_params() {
log_content "停止 DataEase 服务"
if [[ -f /etc/systemd/system/dataease.service ]];then
systemctl stop dataease
else
else
dectl stop
fi
INSTALL_TYPE='upgrade'
v2_version=$(dectl version | head -n 2 | grep "v2.")
@ -99,7 +99,7 @@ function prepare_de_run_base() {
env | grep DE_ >.env
mkdir -p ${DE_RUN_BASE}/{cache,logs,conf}
mkdir -p ${DE_RUN_BASE}/data/{mysql,static-resource,map,etcd_data,geo,appearance}
mkdir -p ${DE_RUN_BASE}/data/{mysql,static-resource,map,etcd_data,geo,appearance,exportData}
mkdir -p ${DE_RUN_BASE}/apisix/logs
mkdir -p ${DE_RUN_BASE}/task/logs
chmod 777 ${DE_RUN_BASE}/apisix/logs ${DE_RUN_BASE}/data/etcd_data ${DE_RUN_BASE}/task/logs
@ -132,7 +132,7 @@ function update_dectl() {
\cp dectl /usr/local/bin && chmod +x /usr/local/bin/dectl
if [ ! -f /usr/bin/dectl ]; then
ln -s /usr/local/bin/dectl /usr/bin/dectl 2>/dev/null
fi
fi
}
function prepare_system_settings() {
@ -312,4 +312,4 @@ function main() {
start_de_service
}
main
main

20
sdk/common/src/main/java/io/dataease/utils/FileUtils.java
View File

@ -269,4 +269,24 @@ public class FileUtils {
return bytes;
}
public static boolean deleteDirectoryRecursively(String directoryPath) {
File directory = new File(directoryPath);
if (!directory.exists()) {
return true;
}
File[] files = directory.listFiles();
if (files != null) {
for (File file : files) {
if (file.isDirectory()) {
deleteDirectoryRecursively(file.getAbsolutePath());
} else {
boolean deletionSuccess = file.delete();
}
}
}
return directory.delete();
}
}