github/dataease
github/dataease
3
0
Fork 1
mirror of https://github.com/dataease/dataease.git synced 2025-02-25 12:03:05 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into dev

Browse Source 
# Conflicts:
#	backend/src/main/resources/db/migration/V33__1.9.sql
#	backend/src/main/resources/generatorConfig.xml
This commit is contained in:
wangjiahao 2022-03-14 21:16:14 +08:00
commit 5b363b985b
65 changed files with 3450 additions and 442 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
backend/src/main/java/io/dataease/base/domain/DeEngine.java Normal file
View File

@ -0,0 +1,27 @@
package io.dataease.base.domain;
import java.io.Serializable;
import lombok.Data;
@Data
public class DeEngine implements Serializable {
private String id;
private String name;
private String desc;
private String type;
private Long createTime;
private Long updateTime;
private String createBy;
private String status;
private String configuration;
private static final long serialVersionUID = 1L;
}

740
backend/src/main/java/io/dataease/base/domain/DeEngineExample.java Normal file
View File

@ -0,0 +1,740 @@
package io.dataease.base.domain;
import java.util.ArrayList;
import java.util.List;
public class DeEngineExample {
protected String orderByClause;
protected boolean distinct;
protected List<Criteria> oredCriteria;
public DeEngineExample() {
oredCriteria = new ArrayList<Criteria>();
}
public void setOrderByClause(String orderByClause) {
this.orderByClause = orderByClause;
}
public String getOrderByClause() {
return orderByClause;
}
public void setDistinct(boolean distinct) {
this.distinct = distinct;
}
public boolean isDistinct() {
return distinct;
}
public List<Criteria> getOredCriteria() {
return oredCriteria;
}
public void or(Criteria criteria) {
oredCriteria.add(criteria);
}
public Criteria or() {
Criteria criteria = createCriteriaInternal();
oredCriteria.add(criteria);
return criteria;
}
public Criteria createCriteria() {
Criteria criteria = createCriteriaInternal();
if (oredCriteria.size() == 0) {
oredCriteria.add(criteria);
}
return criteria;
}
protected Criteria createCriteriaInternal() {
Criteria criteria = new Criteria();
return criteria;
}
public void clear() {
oredCriteria.clear();
orderByClause = null;
distinct = false;
}
protected abstract static class GeneratedCriteria {
protected List<Criterion> criteria;
protected GeneratedCriteria() {
super();
criteria = new ArrayList<Criterion>();
}
public boolean isValid() {
return criteria.size() > 0;
}
public List<Criterion> getAllCriteria() {
return criteria;
}
public List<Criterion> getCriteria() {
return criteria;
}
protected void addCriterion(String condition) {
if (condition == null) {
throw new RuntimeException("Value for condition cannot be null");
}
criteria.add(new Criterion(condition));
}
protected void addCriterion(String condition, Object value, String property) {
if (value == null) {
throw new RuntimeException("Value for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(String condition, Object value1, Object value2, String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value1, value2));
}
public Criteria andIdIsNull() {
addCriterion("id is null");
return (Criteria) this;
}
public Criteria andIdIsNotNull() {
addCriterion("id is not null");
return (Criteria) this;
}
public Criteria andIdEqualTo(String value) {
addCriterion("id =", value, "id");
return (Criteria) this;
}
public Criteria andIdNotEqualTo(String value) {
addCriterion("id <>", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThan(String value) {
addCriterion("id >", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThanOrEqualTo(String value) {
addCriterion("id >=", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThan(String value) {
addCriterion("id <", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThanOrEqualTo(String value) {
addCriterion("id <=", value, "id");
return (Criteria) this;
}
public Criteria andIdLike(String value) {
addCriterion("id like", value, "id");
return (Criteria) this;
}
public Criteria andIdNotLike(String value) {
addCriterion("id not like", value, "id");
return (Criteria) this;
}
public Criteria andIdIn(List<String> values) {
addCriterion("id in", values, "id");
return (Criteria) this;
}
public Criteria andIdNotIn(List<String> values) {
addCriterion("id not in", values, "id");
return (Criteria) this;
}
public Criteria andIdBetween(String value1, String value2) {
addCriterion("id between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andIdNotBetween(String value1, String value2) {
addCriterion("id not between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andNameIsNull() {
addCriterion("`name` is null");
return (Criteria) this;
}
public Criteria andNameIsNotNull() {
addCriterion("`name` is not null");
return (Criteria) this;
}
public Criteria andNameEqualTo(String value) {
addCriterion("`name` =", value, "name");
return (Criteria) this;
}
public Criteria andNameNotEqualTo(String value) {
addCriterion("`name` <>", value, "name");
return (Criteria) this;
}
public Criteria andNameGreaterThan(String value) {
addCriterion("`name` >", value, "name");
return (Criteria) this;
}
public Criteria andNameGreaterThanOrEqualTo(String value) {
addCriterion("`name` >=", value, "name");
return (Criteria) this;
}
public Criteria andNameLessThan(String value) {
addCriterion("`name` <", value, "name");
return (Criteria) this;
}
public Criteria andNameLessThanOrEqualTo(String value) {
addCriterion("`name` <=", value, "name");
return (Criteria) this;
}
public Criteria andNameLike(String value) {
addCriterion("`name` like", value, "name");
return (Criteria) this;
}
public Criteria andNameNotLike(String value) {
addCriterion("`name` not like", value, "name");
return (Criteria) this;
}
public Criteria andNameIn(List<String> values) {
addCriterion("`name` in", values, "name");
return (Criteria) this;
}
public Criteria andNameNotIn(List<String> values) {
addCriterion("`name` not in", values, "name");
return (Criteria) this;
}
public Criteria andNameBetween(String value1, String value2) {
addCriterion("`name` between", value1, value2, "name");
return (Criteria) this;
}
public Criteria andNameNotBetween(String value1, String value2) {
addCriterion("`name` not between", value1, value2, "name");
return (Criteria) this;
}
public Criteria andDescIsNull() {
addCriterion("`desc` is null");
return (Criteria) this;
}
public Criteria andDescIsNotNull() {
addCriterion("`desc` is not null");
return (Criteria) this;
}
public Criteria andDescEqualTo(String value) {
addCriterion("`desc` =", value, "desc");
return (Criteria) this;
}
public Criteria andDescNotEqualTo(String value) {
addCriterion("`desc` <>", value, "desc");
return (Criteria) this;
}
public Criteria andDescGreaterThan(String value) {
addCriterion("`desc` >", value, "desc");
return (Criteria) this;
}
public Criteria andDescGreaterThanOrEqualTo(String value) {
addCriterion("`desc` >=", value, "desc");
return (Criteria) this;
}
public Criteria andDescLessThan(String value) {
addCriterion("`desc` <", value, "desc");
return (Criteria) this;
}
public Criteria andDescLessThanOrEqualTo(String value) {
addCriterion("`desc` <=", value, "desc");
return (Criteria) this;
}
public Criteria andDescLike(String value) {
addCriterion("`desc` like", value, "desc");
return (Criteria) this;
}
public Criteria andDescNotLike(String value) {
addCriterion("`desc` not like", value, "desc");
return (Criteria) this;
}
public Criteria andDescIn(List<String> values) {
addCriterion("`desc` in", values, "desc");
return (Criteria) this;
}
public Criteria andDescNotIn(List<String> values) {
addCriterion("`desc` not in", values, "desc");
return (Criteria) this;
}
public Criteria andDescBetween(String value1, String value2) {
addCriterion("`desc` between", value1, value2, "desc");
return (Criteria) this;
}
public Criteria andDescNotBetween(String value1, String value2) {
addCriterion("`desc` not between", value1, value2, "desc");
return (Criteria) this;
}
public Criteria andTypeIsNull() {
addCriterion("`type` is null");
return (Criteria) this;
}
public Criteria andTypeIsNotNull() {
addCriterion("`type` is not null");
return (Criteria) this;
}
public Criteria andTypeEqualTo(String value) {
addCriterion("`type` =", value, "type");
return (Criteria) this;
}
public Criteria andTypeNotEqualTo(String value) {
addCriterion("`type` <>", value, "type");
return (Criteria) this;
}
public Criteria andTypeGreaterThan(String value) {
addCriterion("`type` >", value, "type");
return (Criteria) this;
}
public Criteria andTypeGreaterThanOrEqualTo(String value) {
addCriterion("`type` >=", value, "type");
return (Criteria) this;
}
public Criteria andTypeLessThan(String value) {
addCriterion("`type` <", value, "type");
return (Criteria) this;
}
public Criteria andTypeLessThanOrEqualTo(String value) {
addCriterion("`type` <=", value, "type");
return (Criteria) this;
}
public Criteria andTypeLike(String value) {
addCriterion("`type` like", value, "type");
return (Criteria) this;
}
public Criteria andTypeNotLike(String value) {
addCriterion("`type` not like", value, "type");
return (Criteria) this;
}
public Criteria andTypeIn(List<String> values) {
addCriterion("`type` in", values, "type");
return (Criteria) this;
}
public Criteria andTypeNotIn(List<String> values) {
addCriterion("`type` not in", values, "type");
return (Criteria) this;
}
public Criteria andTypeBetween(String value1, String value2) {
addCriterion("`type` between", value1, value2, "type");
return (Criteria) this;
}
public Criteria andTypeNotBetween(String value1, String value2) {
addCriterion("`type` not between", value1, value2, "type");
return (Criteria) this;
}
public Criteria andCreateTimeIsNull() {
addCriterion("create_time is null");
return (Criteria) this;
}
public Criteria andCreateTimeIsNotNull() {
addCriterion("create_time is not null");
return (Criteria) this;
}
public Criteria andCreateTimeEqualTo(Long value) {
addCriterion("create_time =", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeNotEqualTo(Long value) {
addCriterion("create_time <>", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeGreaterThan(Long value) {
addCriterion("create_time >", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeGreaterThanOrEqualTo(Long value) {
addCriterion("create_time >=", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeLessThan(Long value) {
addCriterion("create_time <", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeLessThanOrEqualTo(Long value) {
addCriterion("create_time <=", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeIn(List<Long> values) {
addCriterion("create_time in", values, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeNotIn(List<Long> values) {
addCriterion("create_time not in", values, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeBetween(Long value1, Long value2) {
addCriterion("create_time between", value1, value2, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeNotBetween(Long value1, Long value2) {
addCriterion("create_time not between", value1, value2, "createTime");
return (Criteria) this;
}
public Criteria andUpdateTimeIsNull() {
addCriterion("update_time is null");
return (Criteria) this;
}
public Criteria andUpdateTimeIsNotNull() {
addCriterion("update_time is not null");
return (Criteria) this;
}
public Criteria andUpdateTimeEqualTo(Long value) {
addCriterion("update_time =", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeNotEqualTo(Long value) {
addCriterion("update_time <>", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeGreaterThan(Long value) {
addCriterion("update_time >", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeGreaterThanOrEqualTo(Long value) {
addCriterion("update_time >=", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeLessThan(Long value) {
addCriterion("update_time <", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeLessThanOrEqualTo(Long value) {
addCriterion("update_time <=", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeIn(List<Long> values) {
addCriterion("update_time in", values, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeNotIn(List<Long> values) {
addCriterion("update_time not in", values, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeBetween(Long value1, Long value2) {
addCriterion("update_time between", value1, value2, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeNotBetween(Long value1, Long value2) {
addCriterion("update_time not between", value1, value2, "updateTime");
return (Criteria) this;
}
public Criteria andCreateByIsNull() {
addCriterion("create_by is null");
return (Criteria) this;
}
public Criteria andCreateByIsNotNull() {
addCriterion("create_by is not null");
return (Criteria) this;
}
public Criteria andCreateByEqualTo(String value) {
addCriterion("create_by =", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByNotEqualTo(String value) {
addCriterion("create_by <>", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByGreaterThan(String value) {
addCriterion("create_by >", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByGreaterThanOrEqualTo(String value) {
addCriterion("create_by >=", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByLessThan(String value) {
addCriterion("create_by <", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByLessThanOrEqualTo(String value) {
addCriterion("create_by <=", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByLike(String value) {
addCriterion("create_by like", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByNotLike(String value) {
addCriterion("create_by not like", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByIn(List<String> values) {
addCriterion("create_by in", values, "createBy");
return (Criteria) this;
}
public Criteria andCreateByNotIn(List<String> values) {
addCriterion("create_by not in", values, "createBy");
return (Criteria) this;
}
public Criteria andCreateByBetween(String value1, String value2) {
addCriterion("create_by between", value1, value2, "createBy");
return (Criteria) this;
}
public Criteria andCreateByNotBetween(String value1, String value2) {
addCriterion("create_by not between", value1, value2, "createBy");
return (Criteria) this;
}
public Criteria andStatusIsNull() {
addCriterion("`status` is null");
return (Criteria) this;
}
public Criteria andStatusIsNotNull() {
addCriterion("`status` is not null");
return (Criteria) this;
}
public Criteria andStatusEqualTo(String value) {
addCriterion("`status` =", value, "status");
return (Criteria) this;
}
public Criteria andStatusNotEqualTo(String value) {
addCriterion("`status` <>", value, "status");
return (Criteria) this;
}
public Criteria andStatusGreaterThan(String value) {
addCriterion("`status` >", value, "status");
return (Criteria) this;
}
public Criteria andStatusGreaterThanOrEqualTo(String value) {
addCriterion("`status` >=", value, "status");
return (Criteria) this;
}
public Criteria andStatusLessThan(String value) {
addCriterion("`status` <", value, "status");
return (Criteria) this;
}
public Criteria andStatusLessThanOrEqualTo(String value) {
addCriterion("`status` <=", value, "status");
return (Criteria) this;
}
public Criteria andStatusLike(String value) {
addCriterion("`status` like", value, "status");
return (Criteria) this;
}
public Criteria andStatusNotLike(String value) {
addCriterion("`status` not like", value, "status");
return (Criteria) this;
}
public Criteria andStatusIn(List<String> values) {
addCriterion("`status` in", values, "status");
return (Criteria) this;
}
public Criteria andStatusNotIn(List<String> values) {
addCriterion("`status` not in", values, "status");
return (Criteria) this;
}
public Criteria andStatusBetween(String value1, String value2) {
addCriterion("`status` between", value1, value2, "status");
return (Criteria) this;
}
public Criteria andStatusNotBetween(String value1, String value2) {
addCriterion("`status` not between", value1, value2, "status");
return (Criteria) this;
}
}
public static class Criteria extends GeneratedCriteria {
protected Criteria() {
super();
}
}
public static class Criterion {
private String condition;
private Object value;
private Object secondValue;
private boolean noValue;
private boolean singleValue;
private boolean betweenValue;
private boolean listValue;
private String typeHandler;
public String getCondition() {
return condition;
}
public Object getValue() {
return value;
}
public Object getSecondValue() {
return secondValue;
}
public boolean isNoValue() {
return noValue;
}
public boolean isSingleValue() {
return singleValue;
}
public boolean isBetweenValue() {
return betweenValue;
}
public boolean isListValue() {
return listValue;
}
public String getTypeHandler() {
return typeHandler;
}
protected Criterion(String condition) {
super();
this.condition = condition;
this.typeHandler = null;
this.noValue = true;
}
protected Criterion(String condition, Object value, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.typeHandler = typeHandler;
if (value instanceof List<?>) {
this.listValue = true;
} else {
this.singleValue = true;
}
}
protected Criterion(String condition, Object value) {
this(condition, value, null);
}
protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.secondValue = secondValue;
this.typeHandler = typeHandler;
this.betweenValue = true;
}
protected Criterion(String condition, Object value, Object secondValue) {
this(condition, value, secondValue, null);
}
}
}

36
backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.java Normal file
View File

@ -0,0 +1,36 @@
package io.dataease.base.mapper;
import io.dataease.base.domain.DeEngine;
import io.dataease.base.domain.DeEngineExample;
import java.util.List;
import org.apache.ibatis.annotations.Param;
public interface DeEngineMapper {
long countByExample(DeEngineExample example);
int deleteByExample(DeEngineExample example);
int deleteByPrimaryKey(String id);
int insert(DeEngine record);
int insertSelective(DeEngine record);
List<DeEngine> selectByExampleWithBLOBs(DeEngineExample example);
List<DeEngine> selectByExample(DeEngineExample example);
DeEngine selectByPrimaryKey(String id);
int updateByExampleSelective(@Param("record") DeEngine record, @Param("example") DeEngineExample example);
int updateByExampleWithBLOBs(@Param("record") DeEngine record, @Param("example") DeEngineExample example);
int updateByExample(@Param("record") DeEngine record, @Param("example") DeEngineExample example);
int updateByPrimaryKeySelective(DeEngine record);
int updateByPrimaryKeyWithBLOBs(DeEngine record);
int updateByPrimaryKey(DeEngine record);
}

323
backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.xml Normal file
View File

@ -0,0 +1,323 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="io.dataease.base.mapper.DeEngineMapper">
<resultMap id="BaseResultMap" type="io.dataease.base.domain.DeEngine">
<id column="id" jdbcType="VARCHAR" property="id" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="desc" jdbcType="VARCHAR" property="desc" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="create_time" jdbcType="BIGINT" property="createTime" />
<result column="update_time" jdbcType="BIGINT" property="updateTime" />
<result column="create_by" jdbcType="VARCHAR" property="createBy" />
<result column="status" jdbcType="VARCHAR" property="status" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="io.dataease.base.domain.DeEngine">
<result column="configuration" jdbcType="LONGVARCHAR" property="configuration" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Update_By_Example_Where_Clause">
<where>
<foreach collection="example.oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id, `name`, `desc`, `type`, create_time, update_time, create_by, `status`
</sql>
<sql id="Blob_Column_List">
configuration
</sql>
<select id="selectByExampleWithBLOBs" parameterType="io.dataease.base.domain.DeEngineExample" resultMap="ResultMapWithBLOBs">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from de_engine
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByExample" parameterType="io.dataease.base.domain.DeEngineExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
from de_engine
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByPrimaryKey" parameterType="java.lang.String" resultMap="ResultMapWithBLOBs">
select
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from de_engine
where id = #{id,jdbcType=VARCHAR}
</select>
<delete id="deleteByPrimaryKey" parameterType="java.lang.String">
delete from de_engine
where id = #{id,jdbcType=VARCHAR}
</delete>
<delete id="deleteByExample" parameterType="io.dataease.base.domain.DeEngineExample">
delete from de_engine
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</delete>
<insert id="insert" parameterType="io.dataease.base.domain.DeEngine">
insert into de_engine (id, `name`, `desc`,
`type`, create_time, update_time,
create_by, `status`, configuration
)
values (#{id,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{desc,jdbcType=VARCHAR},
#{type,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{updateTime,jdbcType=BIGINT},
#{createBy,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{configuration,jdbcType=LONGVARCHAR}
)
</insert>
<insert id="insertSelective" parameterType="io.dataease.base.domain.DeEngine">
insert into de_engine
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null">
id,
</if>
<if test="name != null">
`name`,
</if>
<if test="desc != null">
`desc`,
</if>
<if test="type != null">
`type`,
</if>
<if test="createTime != null">
create_time,
</if>
<if test="updateTime != null">
update_time,
</if>
<if test="createBy != null">
create_by,
</if>
<if test="status != null">
`status`,
</if>
<if test="configuration != null">
configuration,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
#{id,jdbcType=VARCHAR},
</if>
<if test="name != null">
#{name,jdbcType=VARCHAR},
</if>
<if test="desc != null">
#{desc,jdbcType=VARCHAR},
</if>
<if test="type != null">
#{type,jdbcType=VARCHAR},
</if>
<if test="createTime != null">
#{createTime,jdbcType=BIGINT},
</if>
<if test="updateTime != null">
#{updateTime,jdbcType=BIGINT},
</if>
<if test="createBy != null">
#{createBy,jdbcType=VARCHAR},
</if>
<if test="status != null">
#{status,jdbcType=VARCHAR},
</if>
<if test="configuration != null">
#{configuration,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="io.dataease.base.domain.DeEngineExample" resultType="java.lang.Long">
select count(*) from de_engine
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</select>
<update id="updateByExampleSelective" parameterType="map">
update de_engine
<set>
<if test="record.id != null">
id = #{record.id,jdbcType=VARCHAR},
</if>
<if test="record.name != null">
`name` = #{record.name,jdbcType=VARCHAR},
</if>
<if test="record.desc != null">
`desc` = #{record.desc,jdbcType=VARCHAR},
</if>
<if test="record.type != null">
`type` = #{record.type,jdbcType=VARCHAR},
</if>
<if test="record.createTime != null">
create_time = #{record.createTime,jdbcType=BIGINT},
</if>
<if test="record.updateTime != null">
update_time = #{record.updateTime,jdbcType=BIGINT},
</if>
<if test="record.createBy != null">
create_by = #{record.createBy,jdbcType=VARCHAR},
</if>
<if test="record.status != null">
`status` = #{record.status,jdbcType=VARCHAR},
</if>
<if test="record.configuration != null">
configuration = #{record.configuration,jdbcType=LONGVARCHAR},
</if>
</set>
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByExampleWithBLOBs" parameterType="map">
update de_engine
set id = #{record.id,jdbcType=VARCHAR},
`name` = #{record.name,jdbcType=VARCHAR},
`desc` = #{record.desc,jdbcType=VARCHAR},
`type` = #{record.type,jdbcType=VARCHAR},
create_time = #{record.createTime,jdbcType=BIGINT},
update_time = #{record.updateTime,jdbcType=BIGINT},
create_by = #{record.createBy,jdbcType=VARCHAR},
`status` = #{record.status,jdbcType=VARCHAR},
configuration = #{record.configuration,jdbcType=LONGVARCHAR}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByExample" parameterType="map">
update de_engine
set id = #{record.id,jdbcType=VARCHAR},
`name` = #{record.name,jdbcType=VARCHAR},
`desc` = #{record.desc,jdbcType=VARCHAR},
`type` = #{record.type,jdbcType=VARCHAR},
create_time = #{record.createTime,jdbcType=BIGINT},
update_time = #{record.updateTime,jdbcType=BIGINT},
create_by = #{record.createBy,jdbcType=VARCHAR},
`status` = #{record.status,jdbcType=VARCHAR}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByPrimaryKeySelective" parameterType="io.dataease.base.domain.DeEngine">
update de_engine
<set>
<if test="name != null">
`name` = #{name,jdbcType=VARCHAR},
</if>
<if test="desc != null">
`desc` = #{desc,jdbcType=VARCHAR},
</if>
<if test="type != null">
`type` = #{type,jdbcType=VARCHAR},
</if>
<if test="createTime != null">
create_time = #{createTime,jdbcType=BIGINT},
</if>
<if test="updateTime != null">
update_time = #{updateTime,jdbcType=BIGINT},
</if>
<if test="createBy != null">
create_by = #{createBy,jdbcType=VARCHAR},
</if>
<if test="status != null">
`status` = #{status,jdbcType=VARCHAR},
</if>
<if test="configuration != null">
configuration = #{configuration,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=VARCHAR}
</update>
<update id="updateByPrimaryKeyWithBLOBs" parameterType="io.dataease.base.domain.DeEngine">
update de_engine
set `name` = #{name,jdbcType=VARCHAR},
`desc` = #{desc,jdbcType=VARCHAR},
`type` = #{type,jdbcType=VARCHAR},
create_time = #{createTime,jdbcType=BIGINT},
update_time = #{updateTime,jdbcType=BIGINT},
create_by = #{createBy,jdbcType=VARCHAR},
`status` = #{status,jdbcType=VARCHAR},
configuration = #{configuration,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=VARCHAR}
</update>
<update id="updateByPrimaryKey" parameterType="io.dataease.base.domain.DeEngine">
update de_engine
set `name` = #{name,jdbcType=VARCHAR},
`desc` = #{desc,jdbcType=VARCHAR},
`type` = #{type,jdbcType=VARCHAR},
create_time = #{createTime,jdbcType=BIGINT},
update_time = #{updateTime,jdbcType=BIGINT},
create_by = #{createBy,jdbcType=VARCHAR},
`status` = #{status,jdbcType=VARCHAR}
where id = #{id,jdbcType=VARCHAR}
</update>
</mapper>

5
backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java
View File

@ -8,14 +8,15 @@ public enum DatasourceTypes {
ds_doris("ds_doris", "ds_doris", "com.mysql.jdbc.Driver", "`", "`", "'", "'"),
pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""),
de_doris("de_doris", "de_doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""),
mongo("mongo", "mongodb", "com.mongodb.jdbc.MongoDriver", "`", "`", "\"", "\""),
ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"),
db2("db2", "db2", "com.ibm.db2.jcc.DB2Driver", "\"", "\"", "\"", "\""),
es("es", "es", "", "\"", "\"", "\"", "\""),
redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
api("api", "api", "", "\"", "\"", "\"", "\"");
api("api", "api", "", "\"", "\"", "\"", "\""),
engine_doris("engine_doris", "engine_doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
engine_mysql("mysql", "mysql", "com.mysql.jdbc.Driver", "`", "`", "'", "'");
private String feature;

14
backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java → backend/src/main/java/io/dataease/commons/utils/TableUtils.java
View File

@ -1,28 +1,28 @@
package io.dataease.commons.utils;
public class DorisTableUtils {
public class TableUtils {
public static String dorisName(String datasetId) {
public static String tableName(String datasetId) {
return "ds_" + datasetId.replace("-", "_");
}
public static String dorisTmpName(String dorisName) {
public static String tmpName(String dorisName) {
return "tmp_" + dorisName;
}
public static String dorisDeleteName(String dorisName) {
public static String deleteName(String dorisName) {
return "delete_" + dorisName;
}
public static String dorisAddName(String dorisName) {
public static String addName(String dorisName) {
return "add_" + dorisName;
}
public static String dorisFieldName(String dorisName) {
public static String fieldName(String dorisName) {
return "f_" + Md5Utils.md5(dorisName);
}
public static String dorisFieldNameShort(String dorisName) {
public static String fieldNameShort(String dorisName) {
return "f_" + Md5Utils.md5(dorisName).substring(8, 24);
}

22
backend/src/main/java/io/dataease/config/CommonConfig.java
View File

@ -23,28 +23,6 @@ public class CommonConfig {
private Environment env; // 保存了配置文件的信息
private static String root_path = "/opt/dataease/data/kettle/";
@Bean(name = "DorisDatasource")
@ConditionalOnMissingBean
public Datasource configuration() {
JSONObject jsonObject = new JSONObject();
jsonObject.put("dataSourceType", "jdbc");
jsonObject.put("dataBase", env.getProperty("doris.db", "doris"));
jsonObject.put("username", env.getProperty("doris.user", "root"));
jsonObject.put("password", env.getProperty("doris.password", "dataease"));
jsonObject.put("host", env.getProperty("doris.host", "doris"));
jsonObject.put("port", env.getProperty("doris.port", "9030"));
jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030"));
Datasource datasource = new Datasource();
datasource.setId("doris");
datasource.setName("doris");
datasource.setDesc("doris");
datasource.setType("de_doris");
datasource.setConfiguration(jsonObject.toJSONString());
return datasource;
}
@Bean
@ConditionalOnMissingBean
public KettleFileRepository kettleFileRepository() throws Exception {

45
backend/src/main/java/io/dataease/controller/engine/EngineController.java Normal file
View File

@ -0,0 +1,45 @@
package io.dataease.controller.engine;
import io.dataease.base.domain.DeEngine;
import io.dataease.controller.ResultHolder;
import io.dataease.dto.DatasourceDTO;
import io.dataease.service.engine.EngineService;
import org.springframework.web.bind.annotation.*;
import springfox.documentation.annotations.ApiIgnore;
import javax.annotation.Resource;
@ApiIgnore
@RequestMapping("engine")
@RestController
public class EngineController {
@Resource
private EngineService engineService;
@ApiIgnore
@GetMapping("/mode")
public String runMode() throws Exception{
return engineService.mode();
}
@ApiIgnore
@GetMapping("/info")
public DeEngine info() throws Exception{
return engineService.info();
}
@ApiIgnore
@PostMapping("/validate")
public ResultHolder validate(@RequestBody DatasourceDTO datasource) throws Exception {
return engineService.validate(datasource);
}
@ApiIgnore
@PostMapping("/save")
public ResultHolder save(@RequestBody DeEngine engine) throws Exception {
return engineService.save(engine);
}
}

3
backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java
View File

@ -20,5 +20,8 @@ public class DataSourceInitStartListener implements ApplicationListener<Applicat
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
datasourceService.initAllDataSourceConnectionPool();
dataSetTableService.updateDatasetTableStatus();
}
}

23
backend/src/main/java/io/dataease/provider/DDLProvider.java Normal file
View File

@ -0,0 +1,23 @@
package io.dataease.provider;
import io.dataease.base.domain.DatasetTableField;
import java.util.List;
/**
* @Author gin
* @Date 2021/5/17 4:19 下午
*/
public abstract class DDLProvider {
public abstract String createView(String name, String viewSQL);
public abstract String dropTable(String name);
public abstract String dropView(String name);
public abstract String replaceTable(String name);
public abstract String createTableSql(String name, List<DatasetTableField> datasetTableFields);
public abstract String insertSql(String name, List<String[]> dataList, int page, int pageNumber);
}

50
backend/src/main/java/io/dataease/provider/DDLProviderImpl.java Normal file
View File

@ -0,0 +1,50 @@
package io.dataease.provider;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.commons.utils.Md5Utils;
import java.util.Arrays;
import java.util.List;
public class DDLProviderImpl extends DDLProvider {
@Override
public String createView(String name, String viewSQL) {
return null;
}
@Override
public String dropTable(String name) {
return null;
}
@Override
public String dropView(String name) {
return null;
}
@Override
public String replaceTable(String name) {
return null;
}
@Override
public String createTableSql(String name, List<DatasetTableField> datasetTableFields) {
return null;
}
@Override
public String insertSql(String name, List<String[]> dataList, int page, int pageNumber) {
String insertSql = "INSERT INTO TABLE_NAME VALUES ".replace("TABLE_NAME", name);
StringBuffer values = new StringBuffer();
Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) {
values.append("(").append(Md5Utils.md5(String.join(",", Arrays.asList(strings))))
.append("," ).append(String.join(",", Arrays.asList(strings)))
.append("),");
}
return insertSql + values.substring(0, values.length() - 1);
}
}

16
backend/src/main/java/io/dataease/provider/ProviderFactory.java
View File

@ -2,8 +2,6 @@ package io.dataease.provider;
import io.dataease.commons.constants.DatasourceTypes;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.query.DDLProvider;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.api.ApiProvider;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
@ -40,8 +38,6 @@ public class ProviderFactory implements ApplicationContextAware {
case mariadb:
case ds_doris:
return context.getBean("mysqlQuery", QueryProvider.class);
case de_doris:
return context.getBean("dorisQuery", QueryProvider.class);
case sqlServer:
return context.getBean("sqlserverQuery", QueryProvider.class);
case pg:
@ -62,6 +58,10 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("db2Query", QueryProvider.class);
case api:
return context.getBean("apiQuery", ApiProvider.class);
case engine_doris:
return context.getBean("dorisEngineQuery", QueryProvider.class);
case engine_mysql:
return context.getBean("mysqlEngineQuery", QueryProvider.class);
default:
return context.getBean("mysqlQuery", QueryProvider.class);
}
@ -70,10 +70,12 @@ public class ProviderFactory implements ApplicationContextAware {
public static DDLProvider getDDLProvider(String type) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case de_doris:
return context.getBean("dorisDDL", DDLProvider.class);
case engine_doris:
return context.getBean("dorisEngineDDL", DDLProvider.class);
case engine_mysql:
return context.getBean("mysqlEngineDDL", DDLProvider.class);
default:
return context.getBean("dorisDDL", DDLProvider.class);
return context.getBean("dorisEngineDDL", DDLProvider.class);
}
}

2
backend/src/main/java/io/dataease/provider/query/QueryProvider.java → backend/src/main/java/io/dataease/provider/QueryProvider.java
View File

@ -1,4 +1,4 @@
package io.dataease.provider.query;
package io.dataease.provider;
import com.google.gson.Gson;
import io.dataease.base.domain.ChartViewWithBLOBs;

2
backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java → backend/src/main/java/io/dataease/provider/QueryProviderImpl.java
View File

@ -1,4 +1,4 @@
package io.dataease.provider.query;
package io.dataease.provider;
import io.dataease.base.domain.ChartViewWithBLOBs;
import io.dataease.base.domain.DatasetTableField;

2
backend/src/main/java/io/dataease/provider/query/SQLConstants.java → backend/src/main/java/io/dataease/provider/SQLConstants.java
View File

@ -1,4 +1,4 @@
package io.dataease.provider.query;
package io.dataease.provider;
import java.util.ArrayList;
import java.util.List;

12
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@ -11,7 +11,7 @@ import io.dataease.dto.datasource.*;
import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.QueryProvider;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@ -226,7 +226,7 @@ public class JdbcProvider extends DatasourceProvider {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType());
switch (datasourceType) {
case mysql:
case de_doris:
case engine_doris:
case ds_doris:
case mariadb:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
@ -425,7 +425,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
case de_doris:
case engine_doris:
case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
username = mysqlConfiguration.getUsername();
@ -529,7 +529,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
case de_doris:
case engine_doris:
case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
dataSource.setUrl(mysqlConfiguration.getJdbc());
@ -607,7 +607,7 @@ public class JdbcProvider extends DatasourceProvider {
case mariadb:
JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class);
return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase());
case de_doris:
case engine_doris:
case ds_doris:
case hive:
return "show tables";
@ -656,7 +656,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
case de_doris:
case engine_doris:
case ds_doris:
case ck:
return null;

10
backend/src/main/java/io/dataease/provider/query/doris/DorisConstants.java → backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java
View File

@ -1,17 +1,17 @@
package io.dataease.provider.query.doris;
package io.dataease.provider.engine.doris;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.de_doris;
import static io.dataease.commons.constants.DatasourceTypes.engine_doris;
/**
* @Author gin
* @Date 2021/7/8 7:22 下午
*/
public class DorisConstants extends SQLConstants {
public static final String KEYWORD_TABLE = de_doris.getKeywordPrefix() + "%s" + de_doris.getKeywordSuffix();
public static final String KEYWORD_TABLE = engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + de_doris.getKeywordPrefix() + "%s" + de_doris.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix();
public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)";

85
backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java Normal file
View File

@ -0,0 +1,85 @@
package io.dataease.provider.engine.doris;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.commons.utils.TableUtils;
import io.dataease.provider.DDLProviderImpl;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* @Author gin
* @Date 2021/5/17 4:27 下午
*/
@Service("dorisEngineDDL")
public class DorisDDLProvider extends DDLProviderImpl {
private static final String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
"Column_Fields" +
"UNIQUE KEY(dataease_uuid)\n" +
"DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
"PROPERTIES(\"replication_num\" = \"1\");";
@Override
public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
}
@Override
public String dropTable(String name) {
return "DROP TABLE IF EXISTS " + name;
}
@Override
public String dropView(String name) {
return "DROP VIEW IF EXISTS " + name;
}
@Override
public String replaceTable(String name){
return "ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false')"
.replace("DORIS_TABLE", name).replace("DORIS_TMP_TABLE", TableUtils.tmpName(name));
}
@Override
public String createTableSql(String tableName, List<DatasetTableField> datasetTableFields) {
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql);
}
private String createDorisTableColumnSql(final List<DatasetTableField> datasetTableFields) {
StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `");
for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields.append(datasetTableField.getDataeaseName()).append("` ");
Integer size = datasetTableField.getSize() * 3;
if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
size = 65533;
}
switch (datasetTableField.getDeExtractType()) {
case 0:
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
case 1:
size = size < 50? 50 : size;
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
case 2:
Column_Fields.append("bigint").append(",`");
break;
case 3:
Column_Fields.append("DOUBLE").append(",`");
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(size))).append(",`");
break;
default:
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
}
}
Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
return Column_Fields.toString();
}
}

10
backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java → backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java
View File

@ -1,4 +1,4 @@
package io.dataease.provider.query.doris;
package io.dataease.provider.engine.doris;
import io.dataease.base.domain.ChartViewWithBLOBs;
import io.dataease.base.domain.DatasetTableField;
@ -10,8 +10,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -28,13 +28,13 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin
* @Date 2021/5/17 4:11 下午
*/
@Service("dorisQuery")
@Service("dorisEngineQuery")
public class DorisQueryProvider extends QueryProvider {
@Resource
private DatasetTableFieldMapper datasetTableFieldMapper;

49
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java Normal file
View File

@ -0,0 +1,49 @@
package io.dataease.provider.engine.mysql;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.engine_mysql;
/**
* @Author gin
* @Date 2021/7/8 7:22 下午
*/
public class MysqlConstants extends SQLConstants {
public static final String KEYWORD_TABLE = engine_mysql.getKeywordPrefix() + "%s" + engine_mysql.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + engine_mysql.getKeywordPrefix() + "%s" + engine_mysql.getKeywordSuffix();
public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)";
public static final String DATE_FORMAT = "DATE_FORMAT(%s,'%s')";
public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')";
public static final String STR_TO_DATE = "STR_TO_DATE(%s,'%s')";
public static final String CAST = "CAST(%s AS %s)";
public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S";
public static final String DEFAULT_INT_FORMAT = "BIGINT";
public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)";
public static final String WHERE_VALUE_NULL = "(NULL,'')";
public static final String WHERE_VALUE_VALUE = "'%s'";
public static final String WHERE_NUMBER_VALUE = "%s";
public static final String AGG_COUNT = "COUNT(*)";
public static final String AGG_FIELD = "%s(%s)";
public static final String WHERE_BETWEEN = "'%s' AND '%s'";
public static final String BRACKETS = "(%s)";
public static final String ROUND = "ROUND(%s,%s)";
public static final String VARCHAR = "VARCHAR";
}

35
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java Normal file
View File

@ -0,0 +1,35 @@
package io.dataease.provider.engine.mysql;
import io.dataease.commons.utils.TableUtils;
import io.dataease.provider.DDLProviderImpl;
import org.springframework.stereotype.Service;
/**
* @Author gin
* @Date 2021/5/17 4:27 下午
*/
@Service("mysqlEngineDDL")
public class MysqlDDLProvider extends DDLProviderImpl {
@Override
public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
}
@Override
public String dropTable(String name) {
return "DROP TABLE IF EXISTS " + name;
}
@Override
public String dropView(String name) {
return "DROP VIEW IF EXISTS " + name;
}
@Override
public String replaceTable(String name){
String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE; "
.replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name));
String dropTableSql = "DROP TABLE IF EXISTS " + TableUtils.tmpName(name);
return replaceTableSql + ";" + dropTableSql;
}
}

1107
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java Normal file
View File

File diff suppressed because it is too large Load Diff

13
backend/src/main/java/io/dataease/provider/query/DDLProvider.java
View File

@ -1,13 +0,0 @@
package io.dataease.provider.query;
/**
* @Author gin
* @Date 2021/5/17 4:19 下午
*/
public abstract class DDLProvider {
public abstract String createView(String name, String viewSQL);
public abstract String dropTable(String name);
public abstract String dropView(String name);
}

2
backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.api;
import io.dataease.provider.query.QueryProviderImpl;
import io.dataease.provider.QueryProviderImpl;
import org.springframework.stereotype.Service;
@Service("apiQuery")

2
backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.ck;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.ck;

6
backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java
View File

@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.db2;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.db2;

6
backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.Db2Configuration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -31,7 +31,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("db2Query")
public class Db2QueryProvider extends QueryProvider {

26
backend/src/main/java/io/dataease/provider/query/doris/DorisDDLProvider.java
View File

@ -1,26 +0,0 @@
package io.dataease.provider.query.doris;
import io.dataease.provider.query.DDLProvider;
import org.springframework.stereotype.Service;
/**
* @Author gin
* @Date 2021/5/17 4:27 下午
*/
@Service("dorisDDL")
public class DorisDDLProvider extends DDLProvider {
@Override
public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
}
@Override
public String dropTable(String name) {
return "DROP TABLE IF EXISTS " + name;
}
@Override
public String dropView(String name) {
return "DROP VIEW IF EXISTS " + name;
}
}

6
backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java
View File

@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("esQuery")
public class EsQueryProvider extends QueryProvider {

2
backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.es;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.es;

2
backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.hive;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mysql;

6
backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java
View File

@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.mongodb;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mongo;

6
backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java
View File

@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.mysql;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mysql;

6
backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java
View File

@ -10,8 +10,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -28,7 +28,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.oracle;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.oracle;

6
backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.datasource.OracleConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -31,7 +31,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.pg;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.pg;

6
backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import io.dataease.provider.query.sqlserver.SqlServerSQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
@ -32,7 +32,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("pgQuery")

2
backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.redshift;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.pg;

6
backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import io.dataease.provider.query.pg.PgConstants;
import io.dataease.provider.query.sqlserver.SqlServerSQLConstants;
import org.apache.commons.collections4.CollectionUtils;
@ -33,7 +33,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**

2
backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.sqlserver;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.sqlServer;

6
backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -31,7 +31,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("sqlserverQuery")
public class SqlserverQueryProvider extends QueryProvider {

9
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
View File

@ -2,7 +2,6 @@ package io.dataease.service.chart;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import io.dataease.auth.api.dto.CurrentUserDto;
import io.dataease.auth.entity.SysUserEntity;
import io.dataease.auth.service.AuthUserService;
import io.dataease.base.domain.*;
@ -15,7 +14,6 @@ import io.dataease.commons.constants.CommonConstants;
import io.dataease.commons.constants.JdbcConstants;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.commons.utils.LogUtil;
import io.dataease.controller.request.chart.*;
import io.dataease.controller.request.datasource.DatasourceRequest;
@ -29,12 +27,13 @@ import io.dataease.i18n.Translator;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.QueryProvider;
import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService;
import io.dataease.service.dataset.DataSetTableUnionService;
import io.dataease.service.dataset.PermissionService;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.service.engine.EngineService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -74,6 +73,8 @@ public class ChartViewService {
@Resource
private AuthUserService authUserService;
@Resource
private EngineService engineService;
@Resource
private ChartViewCacheMapper chartViewCacheMapper;
//默认使用非公平
@ -530,7 +531,7 @@ public class ChartViewService {
data = datasourceProvider.getData(datasourceRequest);
} else if (table.getMode() == 1) {// 抽取
// 连接doris，构建doris数据源查询
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
datasourceRequest.setDatasource(ds);
String tableName = "ds_" + table.getId().replaceAll("-", "_");

4
backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java
View File

@ -4,7 +4,7 @@ import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableFieldExample;
import io.dataease.base.mapper.DatasetTableFieldMapper;
import io.dataease.commons.exception.DEException;
import io.dataease.commons.utils.DorisTableUtils;
import io.dataease.commons.utils.TableUtils;
import io.dataease.i18n.Translator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
@ -36,7 +36,7 @@ public class DataSetTableFieldsService {
datasetTableField.setId(UUID.randomUUID().toString());
// 若dataeasename为空，则用MD5(id)作为dataeasename
if (StringUtils.isEmpty(datasetTableField.getDataeaseName())) {
datasetTableField.setDataeaseName(DorisTableUtils.columnName(datasetTableField.getId()));
datasetTableField.setDataeaseName(TableUtils.columnName(datasetTableField.getId()));
}
if (ObjectUtils.isEmpty(datasetTableField.getLastSyncTime())) {
datasetTableField.setLastSyncTime(System.currentTimeMillis());

115
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -28,8 +28,9 @@ import io.dataease.plugins.loader.ClassloaderResponsity;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.provider.query.DDLProvider;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.DDLProvider;
import io.dataease.provider.QueryProvider;
import io.dataease.service.engine.EngineService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -97,6 +98,8 @@ public class DataSetTableService {
private DatasetTableFieldMapper datasetTableFieldMapper;
@Resource
private PermissionService permissionService;
@Resource
private EngineService engineService;
private static final String lastUpdateTime = "${__last_update_time__}";
private static final String currentUpdateTime = "${__current_update_time__}";
@ -296,8 +299,8 @@ public class DataSetTableService {
}
private void deleteDorisTable(String datasetId, DatasetTable table) throws Exception {
String dorisTableName = DorisTableUtils.dorisName(datasetId);
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
String dorisTableName = TableUtils.tableName(datasetId);
Datasource dorisDatasource = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
@ -306,12 +309,12 @@ public class DataSetTableService {
|| StringUtils.equalsIgnoreCase("union", table.getType())) {
datasourceRequest.setQuery(ddlProvider.dropView(dorisTableName));
jdbcProvider.exec(datasourceRequest);
datasourceRequest.setQuery(ddlProvider.dropView(DorisTableUtils.dorisTmpName(dorisTableName)));
datasourceRequest.setQuery(ddlProvider.dropView(TableUtils.tmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
} else {
datasourceRequest.setQuery(ddlProvider.dropTable(dorisTableName));
jdbcProvider.exec(datasourceRequest);
datasourceRequest.setQuery(ddlProvider.dropTable(DorisTableUtils.dorisTmpName(dorisTableName)));
datasourceRequest.setQuery(ddlProvider.dropTable(TableUtils.tmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
}
}
@ -551,11 +554,11 @@ public class DataSetTableService {
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -620,11 +623,11 @@ public class DataSetTableService {
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -649,11 +652,11 @@ public class DataSetTableService {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -719,11 +722,11 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
}
} else {
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -789,11 +792,11 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
}
} else {
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -889,7 +892,7 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
sqlMap = getUnionSQLDatasource(dataTableInfoDTO, ds);
} else {
ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
ds = engineService.getDeEngine();
datasourceRequest.setDatasource(ds);
sqlMap = getUnionSQLDoris(dataTableInfoDTO);
}
@ -923,10 +926,10 @@ public class DataSetTableService {
for (DatasetTableField datasetTableField : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
DorisTableUtils.dorisFieldName(
TableUtils.fieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableField.getFieldName(),
DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_"
TableUtils.fieldNameShort(datasetTableField.getTableId() + "_"
+ datasetTableField.getOriginName()))) {
tableField.setRemarks(datasetTableField.getName());
break;
@ -955,7 +958,7 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
sql = getCustomSQLDatasource(dataTableInfoDTO, list, ds);
} else {
ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
ds = engineService.getDeEngine();
datasourceRequest.setDatasource(ds);
sql = getCustomSQLDoris(dataTableInfoDTO, list);
}
@ -987,9 +990,9 @@ public class DataSetTableService {
for (DatasetTableField datasetTableField : checkedFieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
DorisTableUtils.dorisFieldName(
TableUtils.fieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldName(
|| StringUtils.equalsIgnoreCase(tableField.getFieldName(), TableUtils.fieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) {
tableField.setRemarks(datasetTableField.getName());
break;
@ -1009,7 +1012,7 @@ public class DataSetTableService {
private String getCustomSQLDoris(DataTableInfoDTO dataTableInfoDTO, List<DataSetTableUnionDTO> list) {
Map<String, String[]> customInfo = new TreeMap<>();
dataTableInfoDTO.getList().forEach(ele -> {
String table = DorisTableUtils.dorisName(ele.getTableId());
String table = TableUtils.tableName(ele.getTableId());
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(ele.getTableId());
if (ObjectUtils.isEmpty(datasetTable)) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
@ -1020,7 +1023,7 @@ public class DataSetTableService {
}
String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
+ DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName()))
+ TableUtils.fieldName(ele.getTableId() + "_" + f.getDataeaseName()))
.toArray(String[]::new);
customInfo.put(table, array);
});
@ -1046,12 +1049,12 @@ public class DataSetTableService {
}
join.append(convertUnionTypeToSQL(dto.getSourceUnionRelation()))
.append(DorisTableUtils.dorisName(dto.getTargetTableId()))
.append(TableUtils.tableName(dto.getTargetTableId()))
.append(" ON ")
.append(DorisTableUtils.dorisName(dto.getSourceTableId())).append(".")
.append(TableUtils.tableName(dto.getSourceTableId())).append(".")
.append(sourceField.getDataeaseName())
.append(" = ")
.append(DorisTableUtils.dorisName(dto.getTargetTableId())).append(".")
.append(TableUtils.tableName(dto.getTargetTableId())).append(".")
.append(targetField.getDataeaseName());
}
}
@ -1059,16 +1062,16 @@ public class DataSetTableService {
if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
}
return MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(first.getTableId()))
return MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.tableName(first.getTableId()))
+ join.toString();
} else {
if (StringUtils
.isEmpty(StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","))) {
.isEmpty(StringUtils.join(customInfo.get(TableUtils.tableName(first.getTableId())), ","))) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
}
return MessageFormat.format("SELECT {0} FROM {1}",
StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","),
DorisTableUtils.dorisName(first.getTableId()));
StringUtils.join(customInfo.get(TableUtils.tableName(first.getTableId())), ","),
TableUtils.tableName(first.getTableId()));
}
}
@ -1089,7 +1092,7 @@ public class DataSetTableService {
}
String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
+ DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getOriginName()))
+ TableUtils.fieldName(ele.getTableId() + "_" + f.getOriginName()))
.toArray(String[]::new);
customInfo.put(table, array);
}
@ -1175,7 +1178,7 @@ public class DataSetTableService {
for (UnionDTO unionDTO : union) {
// doris 使用数据集id做表名，拼sql将用到该名称
String tableId = unionDTO.getCurrentDs().getId();
String table = DorisTableUtils.dorisName(tableId);
String table = TableUtils.tableName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(
@ -1193,7 +1196,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
+ DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName()))
+ TableUtils.fieldName(tableId + "_" + f.getDataeaseName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@ -1225,7 +1228,7 @@ public class DataSetTableService {
DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId());
DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId());
join.append(" ").append(joinType).append(" ").append(DorisTableUtils.dorisName(currentTable.getId()))
join.append(" ").append(joinType).append(" ").append(TableUtils.tableName(currentTable.getId()))
.append(" ON ");
for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) {
UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i);
@ -1235,10 +1238,10 @@ public class DataSetTableService {
DatasetTableField currentField = dataSetTableFieldsService
.get(unionItemDTO.getCurrentField().getId());
join.append(DorisTableUtils.dorisName(parentTable.getId())).append(".")
join.append(TableUtils.tableName(parentTable.getId())).append(".")
.append(parentField.getDataeaseName())
.append(" = ")
.append(DorisTableUtils.dorisName(currentTable.getId())).append(".")
.append(TableUtils.tableName(currentTable.getId())).append(".")
.append(currentField.getDataeaseName());
if (i < unionParamDTO.getUnionFields().size() - 1) {
join.append(" AND ");
@ -1249,15 +1252,15 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_union_ds_no_checked"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f,
DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())) + join.toString();
TableUtils.tableName(union.get(0).getCurrentDs().getId())) + join.toString();
} else {
String f = StringUtils.join(checkedInfo.get(DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())),
String f = StringUtils.join(checkedInfo.get(TableUtils.tableName(union.get(0).getCurrentDs().getId())),
",");
if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_union_ds_no_checked"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f,
DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId()));
TableUtils.tableName(union.get(0).getCurrentDs().getId()));
}
Map<String, Object> map = new HashMap<>();
map.put("sql", sql);
@ -1272,7 +1275,7 @@ public class DataSetTableService {
for (int i = 0; i < childrenDs.size(); i++) {
UnionDTO unionDTO = childrenDs.get(i);
String tableId = unionDTO.getCurrentDs().getId();
String table = DorisTableUtils.dorisName(tableId);
String table = TableUtils.tableName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(
@ -1282,7 +1285,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
+ DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName()))
+ TableUtils.fieldName(tableId + "_" + f.getDataeaseName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@ -1329,7 +1332,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
+ DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName()))
+ TableUtils.fieldNameShort(tableId + "_" + f.getOriginName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@ -1430,7 +1433,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
+ DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName()))
+ TableUtils.fieldNameShort(tableId + "_" + f.getOriginName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@ -1451,7 +1454,7 @@ public class DataSetTableService {
datasetTableField.setTableId(datasetTableId);
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setName(filed.getRemarks());
datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName()));
datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName()));
datasetTableField.setType(filed.getFieldType());
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(filed.getFieldType()));
@ -1501,7 +1504,7 @@ public class DataSetTableService {
List<DatasetTableField> listByIds = dataSetTableFieldsService
.getListByIdsEach(ele.getCheckedFields());
listByIds.forEach(f -> f.setDataeaseName(
DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName())));
TableUtils.fieldName(ele.getTableId() + "_" + f.getDataeaseName())));
fieldList.addAll(listByIds);
});
for (int i = 0; i < fieldList.size(); i++) {
@ -1514,7 +1517,7 @@ public class DataSetTableService {
dataSetTableFieldsService.batchEdit(fieldList);
// custom 创建doris视图
if (datasetTable.getMode() == 1) {
createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO,
createDorisView(TableUtils.tableName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO,
dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId())));
}
return;
@ -1541,7 +1544,7 @@ public class DataSetTableService {
for (DatasetTableField field : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getOriginName()),
TableUtils.fieldName(field.getTableId() + "_" + field.getOriginName()),
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
@ -1551,7 +1554,7 @@ public class DataSetTableService {
}
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) {
if (datasetTable.getMode() == 1) {
ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
ds = engineService.getDeEngine();
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
@ -1564,14 +1567,14 @@ public class DataSetTableService {
List<UnionParamDTO> join = (List<UnionParamDTO>) sqlMap.get("join");
// custom 创建doris视图
createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), sql);
createDorisView(TableUtils.tableName(datasetTable.getId()), sql);
datasourceRequest.setQuery(sql);
fields = datasourceProvider.fetchResultField(datasourceRequest);
for (DatasetTableField field : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()),
TableUtils.fieldName(field.getTableId() + "_" + field.getDataeaseName()),
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
@ -1595,7 +1598,7 @@ public class DataSetTableService {
for (DatasetTableField field : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()),
TableUtils.fieldNameShort(field.getTableId() + "_" + field.getOriginName()),
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
@ -1637,7 +1640,7 @@ public class DataSetTableService {
if (datasetTable.getMode() == 1 && StringUtils.equalsIgnoreCase("union", datasetTable.getType())) {
datasetTableField.setDataeaseName(filed.getFieldName());
} else {
datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName()));
datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName()));
}
datasetTableField.setType(filed.getFieldType());
if (ObjectUtils.isEmpty(ds)) {
@ -1665,7 +1668,7 @@ public class DataSetTableService {
}
private void createDorisView(String dorisTableName, String customSql) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource dorisDatasource = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
@ -2165,12 +2168,12 @@ public class DataSetTableService {
}
public Boolean checkDorisTableIsExists(String id) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource dorisDatasource = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType());
datasourceRequest.setQuery(qp.searchTable(DorisTableUtils.dorisName(id)));
datasourceRequest.setQuery(qp.searchTable(TableUtils.tableName(id)));
List<String[]> data = jdbcProvider.getData(datasourceRequest);
return CollectionUtils.isNotEmpty(data);
}

422
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -12,6 +12,7 @@ import io.dataease.commons.model.AuthURD;
import io.dataease.commons.utils.*;
import io.dataease.commons.constants.DatasourceTypes;
import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.provider.DDLProvider;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.provider.ProviderFactory;
@ -22,7 +23,8 @@ import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.exception.DataEaseException;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.QueryProvider;
import io.dataease.service.engine.EngineService;
import io.dataease.service.message.DeMsgutil;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
@ -94,6 +96,8 @@ public class ExtractDataService {
private DatasourceService datasourceService;
@Resource
private ExtChartViewMapper extChartViewMapper;
@Resource
private EngineService engineService;
private static final String lastUpdateTime = "${__last_update_time__}";
private static final String currentUpdateTime = "${__current_update_time__}";
@ -111,13 +115,7 @@ public class ExtractDataService {
private String user;
@Value("${carte.passwd:cluster}")
private String passwd;
private static final String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
"Column_Fields" +
"UNIQUE KEY(dataease_uuid)\n" +
"DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
"PROPERTIES(\"replication_num\" = \"1\");";
private static final String dropTableSql = "DROP TABLE IF EXISTS TABLE_NAME;";
private static final String shellScript = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" +
"if [ $? -eq 0 ] ; then\n" +
" failstatus=$(echo $result | grep '\"Status\": \"Fail\"')\n" +
@ -162,7 +160,7 @@ public class ExtractDataService {
}
UpdateType updateType = UpdateType.valueOf(type);
DatasetTableTaskLog datasetTableTaskLog;
if(datasetTableFields == null){
if (datasetTableFields == null) {
datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
}
datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
@ -175,32 +173,36 @@ public class ExtractDataService {
}
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
switch (updateType) {
case all_scope: // 全量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql);
createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql);
generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields);
createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields);
Long execTime = System.currentTimeMillis();
extractData(datasetTable, "all_scope");
replaceTable(DorisTableUtils.dorisName(datasetTableId));
if (!engineService.isSimpleMode()) {
generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
extractData(datasetTable, "all_scope");
} else {
extractExcelDataForSimpleMode(datasetTable, "all_scope");
}
replaceTable(TableUtils.tableName(datasetTableId));
saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
if(ops.equalsIgnoreCase("替换")){
if (ops.equalsIgnoreCase("替换")) {
List<DatasetTableField> oldFileds = getDatasetTableFields(datasetTable.getId());
List<DatasetTableField> toAdd = new ArrayList<>();
List<DatasetTableField> toDelete = new ArrayList<>();
for (DatasetTableField oldFiled : oldFileds) {
boolean delete = true;
for (DatasetTableField datasetTableField : datasetTableFields) {
if(oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())){
if (oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())) {
delete = false;
}
}
if(delete){
if (delete) {
toDelete.add(oldFiled);
}
}
@ -208,11 +210,11 @@ public class ExtractDataService {
for (DatasetTableField datasetTableField : datasetTableFields) {
boolean add = true;
for (DatasetTableField oldFiled : oldFileds) {
if(oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())){
if (oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())) {
add = false;
}
}
if(add){
if (add) {
toAdd.add(datasetTableField);
}
}
@ -222,7 +224,7 @@ public class ExtractDataService {
} catch (Exception e) {
saveErrorLog(datasetTableId, null, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)));
} finally {
deleteFile("all_scope", datasetTableId);
deleteExcelFile(datasetTable, datasetTableIds);
@ -232,10 +234,14 @@ public class ExtractDataService {
case add_scope: // 增量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
generateJobFile("incremental_add", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
Long execTime = System.currentTimeMillis();
extractData(datasetTable, "incremental_add");
if (!engineService.isSimpleMode()) {
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
generateJobFile("incremental_add", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
extractData(datasetTable, "incremental_add");
} else {
extractExcelDataForSimpleMode(datasetTable, "incremental_add");
}
saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
} catch (Exception e) {
@ -257,21 +263,6 @@ public class ExtractDataService {
}
}
private List<DatasetTableField> getDatasetTableFields(String datasetTableId){
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build());
datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
datasetTableFields.sort((o1, o2) -> {
if (o1.getColumnIndex() == null) {
return -1;
}
if (o2.getColumnIndex() == null) {
return 1;
}
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
return datasetTableFields;
}
public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) {
DatasetTable datasetTable = getDatasetTable(datasetTableId);
if (datasetTable == null) {
@ -306,19 +297,17 @@ public class ExtractDataService {
datasource.setType(datasetTable.getType());
}
List<DatasetTableField> datasetTableFields = getDatasetTableFields(datasetTable.getId());
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
boolean msg = false;
JobStatus lastExecStatus = JobStatus.Completed;
Long execTime = null;
switch (updateType) {
case all_scope: // 全量更新
try {
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql);
createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql);
createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields);
createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields);
execTime = System.currentTimeMillis();
extractData(datasetTable, datasource, datasetTableFields, "all_scope", null);
replaceTable(DorisTableUtils.dorisName(datasetTableId));
replaceTable(TableUtils.tableName(datasetTableId));
saveSuccessLog(datasetTableTaskLog);
msg = true;
lastExecStatus = JobStatus.Completed;
@ -328,21 +317,39 @@ public class ExtractDataService {
lastExecStatus = JobStatus.Error;
execTime = null;
} finally {
try { deleteFile("all_scope", datasetTableId); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
try { sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
try { dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus); }catch (Exception ignore){
try {
deleteFile("all_scope", datasetTableId);
} catch (Exception ignore) {
System.out.println(ignore.getMessage());
}
try {
sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg);
} catch (Exception ignore) {
System.out.println(ignore.getMessage());
}
try {
dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus);
} catch (Exception ignore) {
System.out.println(ignore.getMessage());
}
try {
updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime);
} catch (Exception ignore) {
System.out.println(ignore.getMessage());
}
try {
dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)));
} catch (Exception ignore) {
System.out.println(ignore.getMessage());
}
try { updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
try { dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
}
break;
case add_scope: // 增量更新
try {
if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){
if (datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())) {
extractData(datasetTable, datasource, datasetTableFields, "incremental_add", null);
}else{
} else {
DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId);
if (datasetTable.getLastUpdateTime() == null || datasetTable.getLastUpdateTime() == 0) {
throw new Exception("未进行全量同步");
@ -370,10 +377,23 @@ public class ExtractDataService {
lastExecStatus = JobStatus.Error;
execTime = null;
} finally {
try { deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); }catch (Exception ignore){}
try { sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg); }catch (Exception ignore){}
try { dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus); }catch (Exception ignore){}
try { updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); }catch (Exception ignore){}
try {
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
} catch (Exception ignore) {
}
try {
sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg);
} catch (Exception ignore) {
}
try {
dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus);
} catch (Exception ignore) {
}
try {
updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime);
} catch (Exception ignore) {
}
}
break;
}
@ -385,43 +405,62 @@ public class ExtractDataService {
}
private void extractData(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType, String selectSQL) throws Exception{
if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){
extractDataByDE(datasetTable, datasource, datasetTableFields, extractType);
private List<DatasetTableField> getDatasetTableFields(String datasetTableId) {
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build());
datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
datasetTableFields.sort((o1, o2) -> {
if (o1.getColumnIndex() == null) {
return -1;
}
if (o2.getColumnIndex() == null) {
return 1;
}
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
return datasetTableFields;
}
private void extractData(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType, String selectSQL) throws Exception {
if (datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())) {
extractApiData(datasetTable, datasource, datasetTableFields, extractType);
return;
}
extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, selectSQL);
}
private void extractDataByDE(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType)throws Exception{
private void extractApiData(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType) throws Exception {
List<ApiDefinition> lists = JSONObject.parseArray(datasource.getConfiguration(), ApiDefinition.class);
lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable())).collect(Collectors.toList());
if(CollectionUtils.isEmpty(lists)){
if (CollectionUtils.isEmpty(lists)) {
throw new Exception("未找到API数据表");
}
if(lists.size() > 1 ){
if (lists.size() > 1) {
throw new Exception("存在重名的API数据表");
}
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceRequest.setTable(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable());
Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
List<String[]> dataList = result.get("dataList");
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
if (engineService.isSimpleMode()) {
extractDataForSimpleMode(extractType, datasetTable.getId(), dataList);
}
Datasource engine = engineService.getDeEngine();
DorisConfiguration dorisConfiguration = new Gson().fromJson(engine.getConfiguration(), DorisConfiguration.class);
String columns = datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")) + ",dataease_uuid";
String dataFile = null;
String script = null;
switch (extractType) {
case "all_scope":
dataFile = root_path + DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())));
dataFile = root_path + TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())));
break;
default:
dataFile = root_path + DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()));
dataFile = root_path + TableUtils.addName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()));
break;
}
@ -429,8 +468,8 @@ public class ExtractDataService {
BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile));
for (String[] strings : dataList) {
String content = "";
for (int i=0;i< strings.length;i++){
content = content + strings[i] + separator;
for (int i = 0; i < strings.length; i++) {
content = content + strings[i] + separator;
}
content = content + Md5Utils.md5(content);
bw.write(content);
@ -452,7 +491,7 @@ public class ExtractDataService {
try {
Process process = Runtime.getRuntime().exec(root_path + datasetTable.getId() + ".sh");
process.waitFor();
if(process.waitFor() != 0){
if (process.waitFor() != 0) {
BufferedReader input = new BufferedReader(new InputStreamReader(process.getErrorStream()));
String errMsg = "";
String line = "";
@ -465,16 +504,45 @@ public class ExtractDataService {
}
throw new Exception(errMsg);
}
}catch (Exception e){
} catch (Exception e) {
throw e;
}finally {
} finally {
File deleteFile = new File(root_path + datasetTable.getId() + ".sh");
FileUtils.forceDelete(deleteFile);
}
}
private void extractDataByKettle(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType, String selectSQL)throws Exception{
private void extractDataForSimpleMode(String extractType, String datasetId, List<String[]> dataList) throws Exception {
String tableName;
switch (extractType) {
case "all_scope":
tableName = TableUtils.tmpName(TableUtils.tableName(datasetId));
break;
default:
tableName = TableUtils.tableName(datasetId);
break;
}
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(engine);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
int pageNumber = 100; //一次插入 100条
int totalPage;
if (dataList.size() % pageNumber > 0) {
totalPage = dataList.size() / pageNumber + 1;
} else {
totalPage = dataList.size() / pageNumber;
}
for (int page = 1; page <= totalPage; page++) {
datasourceRequest.setQuery(ddlProvider.insertSql(tableName, dataList, page, pageNumber));
jdbcProvider.exec(datasourceRequest);
}
}
private void extractDataByKettle(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType, String selectSQL) throws Exception {
generateTransFile(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
generateJobFile(extractType, datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
extractData(datasetTable, extractType);
@ -542,70 +610,43 @@ public class ExtractDataService {
}
private String createDorisTableColumnSql(final List<DatasetTableField> datasetTableFields) {
StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `");
for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields.append(datasetTableField.getDataeaseName()).append("` ");
Integer size = datasetTableField.getSize() * 3;
if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
size = 65533;
}
switch (datasetTableField.getDeExtractType()) {
case 0:
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
case 1:
size = size < 50? 50 : size;
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
case 2:
Column_Fields.append("bigint").append(",`");
break;
case 3:
Column_Fields.append("DOUBLE").append(",`");
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(size))).append(",`");
break;
default:
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
}
}
Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
return Column_Fields.toString();
}
private void createDorisTable(String dorisTableName, String dorisTableColumnSql) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
private void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTableColumnSql));
datasourceRequest.setDatasource(engine);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
datasourceRequest.setQuery(ddlProvider.createTableSql(tableName, datasetTableFields));
jdbcProvider.exec(datasourceRequest);
}
private void dropDorisTable(String dorisTableName) {
private void dropDorisTable(String tableName) {
try {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(dropTableSql.replace("TABLE_NAME", dorisTableName));
datasourceRequest.setDatasource(engine);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
datasourceRequest.setQuery(ddlProvider.dropTable(tableName));
jdbcProvider.exec(datasourceRequest);
} catch (Exception ignore) {
}
}
private void replaceTable(String dorisTableName) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery("ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false');".replace("DORIS_TABLE", dorisTableName).replace("DORIS_TMP_TABLE", DorisTableUtils.dorisTmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
datasourceRequest.setDatasource(engine);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
String[] replaceTableSql = ddlProvider.replaceTable(dorisTableName).split(";");
for (int i = 0; i < replaceTableSql.length; i++) {
if (StringUtils.isNotEmpty(replaceTableSql[i])) {
datasourceRequest.setQuery(replaceTableSql[i]);
jdbcProvider.exec(datasourceRequest);
}
}
}
private DatasetTable getDatasetTable(String datasetTableId) {
@ -661,8 +702,26 @@ public class ExtractDataService {
return datasetTableTaskLog;
}
private void extractExcelDataForSimpleMode(DatasetTable datasetTable, String extractType) throws Exception {
List<String[]> data = new ArrayList<>();
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
List<ExcelSheetData> excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList();
ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader();
for (ExcelSheetData excelSheetData : excelSheetDataList) {
excelXlsxReader.process(new FileInputStream(excelSheetData.getPath()));
for (ExcelSheetData sheet : excelXlsxReader.totalSheets) {
if (sheet.getExcelLable().equalsIgnoreCase(excelSheetData.getExcelLable())) {
for (List<String> dataItem : sheet.getData()) {
data.add(dataItem.toArray(new String[dataItem.size()]));
}
}
}
}
extractDataForSimpleMode(extractType, datasetTable.getId(), data);
}
private void extractData(DatasetTable datasetTable, String extractType) throws Exception {
if(StringUtils.isNotEmpty(datasetTable.getDataSourceId())){
if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) {
datasourceService.validate(datasetTable.getDataSourceId());
}
KettleFileRepository repository = CommonBeanFactory.getBean(KettleFileRepository.class);
@ -671,16 +730,16 @@ public class ExtractDataService {
JobMeta jobMeta = null;
switch (extractType) {
case "all_scope":
jobMeta = repository.loadJob("job_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
jobMeta = repository.loadJob("job_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
break;
case "incremental_add":
jobMeta = repository.loadJob("job_add_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_add_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
jobMeta = repository.loadJob("job_add_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_add_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
break;
case "incremental_delete":
jobMeta = repository.loadJob("job_delete_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_delete_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
jobMeta = repository.loadJob("job_delete_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_delete_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
break;
default:
break;
@ -738,27 +797,30 @@ public class ExtractDataService {
}
private void generateJobFile(String extractType, DatasetTable datasetTable, String columnFields) throws Exception {
if (engineService.isSimpleMode()) {
return;
}
String outFile;
String jobName = null;
String script = null;
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource dorisDatasource = engineService.getDeEngine();
DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
String columns = columnFields + ",dataease_uuid";
switch (extractType) {
case "all_scope":
outFile = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()));
jobName = "job_" + DorisTableUtils.dorisName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())), root_path + outFile + "." + extention);
outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()));
jobName = "job_" + TableUtils.tableName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention);
break;
case "incremental_add":
outFile = DorisTableUtils.dorisAddName(datasetTable.getId());
jobName = "job_add_" + DorisTableUtils.dorisName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + outFile + "." + extention);
outFile = TableUtils.addName(datasetTable.getId());
jobName = "job_add_" + TableUtils.tableName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
break;
case "incremental_delete":
outFile = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId()));
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + outFile + "." + extention);
jobName = "job_delete_" + DorisTableUtils.dorisName(datasetTable.getId());
outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId()));
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
jobName = "job_delete_" + TableUtils.tableName(datasetTable.getId());
break;
default:
break;
@ -816,12 +878,15 @@ public class ExtractDataService {
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> dorisFields = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName).forEach(filed -> {
dorisFields.add(DorisTableUtils.columnName(filed));
dorisFields.add(TableUtils.columnName(filed));
});
return String.join(",", dorisFields);
}
private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String selectSQL) throws Exception {
if (engineService.isSimpleMode()) {
return;
}
TransMeta transMeta = new TransMeta();
String outFile = null;
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasource.getType());
@ -902,18 +967,18 @@ public class ExtractDataService {
switch (extractType) {
case "all_scope":
transName = "trans_" + DorisTableUtils.dorisName(datasetTable.getId());
outFile = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()));
transName = "trans_" + TableUtils.tableName(datasetTable.getId());
outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()));
transMeta.setName(transName);
break;
case "incremental_add":
transName = "trans_add_" + DorisTableUtils.dorisName(datasetTable.getId());
outFile = DorisTableUtils.dorisAddName(datasetTable.getId());
transName = "trans_add_" + TableUtils.tableName(datasetTable.getId());
outFile = TableUtils.addName(datasetTable.getId());
transMeta.setName(transName);
break;
case "incremental_delete":
transName = "trans_delete_" + DorisTableUtils.dorisName(datasetTable.getId());
outFile = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId()));
transName = "trans_delete_" + TableUtils.tableName(datasetTable.getId());
outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId()));
transMeta.setName(transName);
break;
default:
@ -965,7 +1030,7 @@ public class ExtractDataService {
return fromStep;
}
private StepMeta excelInputStep(String Info, List<DatasetTableField> datasetTableFields){
private StepMeta excelInputStep(String Info, List<DatasetTableField> datasetTableFields) {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class);
List<ExcelSheetData> excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList();
String suffix = excelSheetDataList.get(0).getPath().substring(excelSheetDataList.get(0).getPath().lastIndexOf(".") + 1);
@ -975,10 +1040,10 @@ public class ExtractDataService {
List<String> files = new ArrayList<>();
List<String> filesRequired = new ArrayList<>();
for (ExcelSheetData excelSheetData : excelSheetDataList) {
if(!sheetNames.contains(excelSheetData.getExcelLable())){
if (!sheetNames.contains(excelSheetData.getExcelLable())) {
sheetNames.add(excelSheetData.getExcelLable());
}
if(!files.contains(excelSheetData.getPath())){
if (!files.contains(excelSheetData.getPath())) {
files.add(excelSheetData.getPath());
filesRequired.add("Y");
}
@ -992,7 +1057,7 @@ public class ExtractDataService {
excelInputMeta.setSheetName(sheetNames.toArray(new String[sheetNames.size()]));
}
excelInputMeta.setPassword("Encrypted");
excelInputMeta.setFileName( files.toArray(new String[files.size()]));
excelInputMeta.setFileName(files.toArray(new String[files.size()]));
excelInputMeta.setFileRequired(filesRequired.toArray(new String[filesRequired.size()]));
excelInputMeta.setStartsWithHeader(true);
excelInputMeta.setIgnoreEmptyRows(true);
@ -1024,9 +1089,9 @@ public class ExtractDataService {
textFileOutputMeta.setSeparator(separator);
textFileOutputMeta.setExtension(extention);
if (datasource.getType().equalsIgnoreCase(DatasourceTypes.oracle.name()) ) {
if (datasource.getType().equalsIgnoreCase(DatasourceTypes.oracle.name())) {
TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1];
for(int i=0;i< datasetTableFields.size();i++){
for (int i = 0; i < datasetTableFields.size(); i++) {
TextFileField textFileField = new TextFileField();
textFileField.setName(datasetTableFields.get(i).getOriginName());
textFileField.setType("String");
@ -1038,9 +1103,9 @@ public class ExtractDataService {
outputFields[datasetTableFields.size()] = textFileField;
textFileOutputMeta.setOutputFields(outputFields);
}else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.sqlServer.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.pg.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.mysql.name())){
} else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.sqlServer.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.pg.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.mysql.name())) {
TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1];
for(int i=0;i< datasetTableFields.size();i++){
for (int i = 0; i < datasetTableFields.size(); i++) {
TextFileField textFileField = new TextFileField();
textFileField.setName(datasetTableFields.get(i).getDataeaseName());
if (datasetTableFields.get(i).getDeExtractType().equals(DeTypeConstants.DE_TIME)) {
@ -1058,9 +1123,9 @@ public class ExtractDataService {
outputFields[datasetTableFields.size()] = textFileField;
textFileOutputMeta.setOutputFields(outputFields);
}else if(datasource.getType().equalsIgnoreCase(DatasourceTypes.excel.name())) {
} else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.excel.name())) {
TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1];
for(int i=0;i< datasetTableFields.size();i++){
for (int i = 0; i < datasetTableFields.size(); i++) {
TextFileField textFileField = new TextFileField();
textFileField.setName(datasetTableFields.get(i).getDataeaseName());
if (datasetTableFields.get(i).getDeExtractType().equals(DeTypeConstants.DE_INT)) {
@ -1078,7 +1143,7 @@ public class ExtractDataService {
outputFields[datasetTableFields.size()] = textFileField;
textFileOutputMeta.setOutputFields(outputFields);
}else {
} else {
textFileOutputMeta.setOutputFields(new TextFileField[0]);
}
@ -1093,7 +1158,7 @@ public class ExtractDataService {
String excelCompletion = "";
for (DatasetTableField datasetTableField : datasetTableFields) {
if(datasetTableField.getDeExtractType().equals(DeTypeConstants.DE_BINARY)){
if (datasetTableField.getDeExtractType().equals(DeTypeConstants.DE_BINARY)) {
handleBinaryTypeCode.append("\n").append(handleBinaryType.replace("FIELD", datasetTableField.getDataeaseName()));
}
}
@ -1134,7 +1199,7 @@ public class ExtractDataService {
}
public void deleteFile(String type, String dataSetTableId) {
if(kettleFilesKeep){
if (kettleFilesKeep) {
return;
}
String transName = null;
@ -1143,19 +1208,19 @@ public class ExtractDataService {
switch (type) {
case "all_scope":
transName = "trans_" + DorisTableUtils.dorisName(dataSetTableId);
jobName = "job_" + DorisTableUtils.dorisName(dataSetTableId);
fileName = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(dataSetTableId));
transName = "trans_" + TableUtils.tableName(dataSetTableId);
jobName = "job_" + TableUtils.tableName(dataSetTableId);
fileName = TableUtils.tmpName(TableUtils.tableName(dataSetTableId));
break;
case "incremental_add":
transName = "trans_add_" + DorisTableUtils.dorisName(dataSetTableId);
jobName = "job_add_" + DorisTableUtils.dorisName(dataSetTableId);
fileName = DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(dataSetTableId));
transName = "trans_add_" + TableUtils.tableName(dataSetTableId);
jobName = "job_add_" + TableUtils.tableName(dataSetTableId);
fileName = TableUtils.addName(TableUtils.tableName(dataSetTableId));
break;
case "incremental_delete":
transName = "trans_delete_" + DorisTableUtils.dorisName(dataSetTableId);
jobName = "job_delete_" + DorisTableUtils.dorisName(dataSetTableId);
fileName = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(dataSetTableId));
transName = "trans_delete_" + TableUtils.tableName(dataSetTableId);
jobName = "job_delete_" + TableUtils.tableName(dataSetTableId);
fileName = TableUtils.deleteName(TableUtils.tableName(dataSetTableId));
break;
default:
break;
@ -1165,29 +1230,33 @@ public class ExtractDataService {
deleteFile(root_path + transName + ".ktr");
}
private void deleteExcelFile(DatasetTable datasetTable, List<String>datasetTableIds){
private void deleteExcelFile(DatasetTable datasetTable, List<String> datasetTableIds) {
List<DatasetTable> datasetTables = dataSetTableService.list(datasetTableIds);
for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
Boolean allIsFinished = true;
for (DatasetTable table : datasetTables) {
for(ExcelSheetData data : new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()){
if(data.getPath().equalsIgnoreCase(excelSheetData.getPath())){
if(StringUtils.isEmpty(table.getSyncStatus()) || table.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())){
for (ExcelSheetData data : new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
if (data.getPath().equalsIgnoreCase(excelSheetData.getPath())) {
if (StringUtils.isEmpty(table.getSyncStatus()) || table.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())) {
allIsFinished = false;
}
}
}
}
if(allIsFinished){
if (allIsFinished) {
deleteFile(excelSheetData.getPath());
}else {
try { Thread.sleep(5000); }catch (Exception ignore){}
} else {
try {
Thread.sleep(5000);
} catch (Exception ignore) {
}
deleteExcelFile(datasetTable, datasetTableIds);
}
}
}
private void deleteFile(String filePath){
if(StringUtils.isEmpty(filePath)){
private void deleteFile(String filePath) {
if (StringUtils.isEmpty(filePath)) {
return;
}
try {
@ -1196,6 +1265,7 @@ public class ExtractDataService {
} catch (Exception e) {
}
}
public boolean isKettleRunning() {
try {
if (!InetAddress.getByName(carte).isReachable(1000)) {

8
backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java
View File

@ -5,7 +5,6 @@ import io.dataease.base.domain.DatasetTable;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import io.dataease.commons.constants.ColumnPermissionConstants;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.i18n.Translator;
import io.dataease.provider.datasource.DatasourceProvider;
@ -15,7 +14,8 @@ import io.dataease.service.dataset.*;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.dto.dataset.DataSetTableUnionDTO;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.QueryProvider;
import io.dataease.service.engine.EngineService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -40,6 +40,8 @@ public class DirectFieldService implements DataSetFieldService {
private DataSetTableUnionService dataSetTableUnionService;
@Resource
private PermissionService permissionService;
@Resource
private EngineService engineService;
@Override
public List<Object> fieldValues(String fieldId, Long userId, Boolean userPermissions) throws Exception {
@ -100,7 +102,7 @@ public class DirectFieldService implements DataSetFieldService {
}
} else if (datasetTable.getMode() == 1) {// 抽取
// 连接doris，构建doris数据源查询
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
datasourceProvider = ProviderFactory.getProvider(ds.getType());
datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);

5
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -71,7 +71,7 @@ public class DatasourceService {
return datasource;
}
private void handleConnectionPool(Datasource datasource, String type) {
public void handleConnectionPool(Datasource datasource, String type) {
commonThreadPool.addTask(() -> {
try {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
@ -94,7 +94,7 @@ public class DatasourceService {
switch (datasourceType) {
case mysql:
case mariadb:
case de_doris:
case engine_doris:
case ds_doris:
datasourceDTO.setConfiguration(JSONObject.toJSONString(new Gson().fromJson(datasourceDTO.getConfiguration(), MysqlConfiguration.class)) );
break;
@ -221,7 +221,6 @@ public class DatasourceService {
}catch (Exception e){
return ResultHolder.error("Datasource is invalid: " + e.getMessage());
}
}
public ResultHolder validate(String datasourceId) {

121
backend/src/main/java/io/dataease/service/engine/EngineService.java Normal file
View File

@ -0,0 +1,121 @@
package io.dataease.service.engine;
import com.alibaba.fastjson.JSONObject;
import io.dataease.base.domain.Datasource;
import io.dataease.base.domain.DeEngine;
import io.dataease.base.domain.DeEngineExample;
import io.dataease.base.mapper.DeEngineMapper;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.controller.ResultHolder;
import io.dataease.controller.request.datasource.DatasourceRequest;
import io.dataease.dto.DatasourceDTO;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.service.datasource.DatasourceService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.List;
import java.util.UUID;
@Service
@Transactional(rollbackFor = Exception.class)
public class EngineService {
@Resource
private Environment env;
@Resource
private DeEngineMapper deEngineMapper;
@Resource
private DatasourceService datasource;
static private Datasource ds = new Datasource();
public Boolean isLocalMode(){
return env.getProperty("engine_mode", "local").equalsIgnoreCase("local");
}
public Boolean isSimpleMode(){
return env.getProperty("engine_mode", "local").equalsIgnoreCase("simple");
}
public Boolean isClusterMode(){
return env.getProperty("engine_mode", "local").equalsIgnoreCase("cluster");
}
public String mode(){
return env.getProperty("engine_mode", "local");
}
public DeEngine info(){
List<DeEngine> deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample());
if(CollectionUtils.isEmpty(deEngines)){
return new DeEngine();
}
return deEngines.get(0);
}
public ResultHolder validate(DatasourceDTO datasource) throws Exception {
try {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceProvider.checkStatus(datasourceRequest);
return ResultHolder.success(datasource);
}catch (Exception e){
return ResultHolder.error("Datasource is invalid: " + e.getMessage());
}
}
public ResultHolder save(DeEngine engine) throws Exception {
if(StringUtils.isEmpty(engine.getId())){
engine.setId(UUID.randomUUID().toString());
deEngineMapper.insert(engine);
}else {
deEngineMapper.updateByPrimaryKeyWithBLOBs(engine);
}
datasource.handleConnectionPool(this.ds, "delete");
BeanUtils.copyBean(this.ds, engine);
datasource.handleConnectionPool(this.ds, "add");
return ResultHolder.success(engine);
}
public Datasource getDeEngine() throws Exception{
if (this.ds != null || StringUtils.isNotEmpty(ds.getType())) {
return this.ds;
}
if(isLocalMode()){
JSONObject jsonObject = new JSONObject();
jsonObject.put("dataSourceType", "jdbc");
jsonObject.put("dataBase", env.getProperty("doris.db", "doris"));
jsonObject.put("username", env.getProperty("doris.user", "root"));
jsonObject.put("password", env.getProperty("doris.password", "dataease"));
jsonObject.put("host", env.getProperty("doris.host", "doris"));
jsonObject.put("port", env.getProperty("doris.port", "9030"));
jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030"));
Datasource datasource = new Datasource();
datasource.setId("doris");
datasource.setName("doris");
datasource.setDesc("doris");
datasource.setType("engine_doris");
datasource.setConfiguration(jsonObject.toJSONString());
this.ds = datasource;
}
if(isSimpleMode()){
List<DeEngine> deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample());
if(CollectionUtils.isEmpty(deEngines)){
throw new Exception("未设置数据引擎");
}
BeanUtils.copyBean(this.ds, deEngines.get(0));
}
//TODO cluster mode
return this.ds;
}
}

13
backend/src/main/resources/db/migration/V33__1.9.sql
View File

@ -68,3 +68,16 @@ CREATE TABLE `panel_group_extend_data` (
SET FOREIGN_KEY_CHECKS = 1;
CREATE TABLE `de_engine` (
`id` varchar(50) NOT NULL DEFAULT '' COMMENT 'ID',
`name` varchar(50) DEFAULT NULL COMMENT '名称',
`desc` varchar(50) DEFAULT NULL COMMENT '描述',
`type` varchar(50) NOT NULL COMMENT '类型',
`configuration` longtext NOT NULL COMMENT '详细信息',
`create_time` bigint(13) DEFAULT NULL COMMENT 'Create timestamp',
`update_time` bigint(13) DEFAULT NULL COMMENT 'Update timestamp',
`create_by` varchar(50) DEFAULT NULL COMMENT '创建人ID',
`status` varchar(45) DEFAULT NULL COMMENT '状态',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

16
backend/src/main/resources/generatorConfig.xml
View File

@ -60,17 +60,9 @@
</javaClientGenerator>
<!--要生成的数据库表 -->
<table tableName="panel_group_extend"/>
<table tableName="panel_group_extend_data"/>
<!-- <table tableName="dataease_code_version"/>-->
<!-- <table tableName="chart_view"/>-->
<!-- <table tableName="chart_view_cache"/>-->
<!-- <table tableName="chart_view"/>-->
<table tableName="panel_view"/>
<!-- <table tableName="panel_link_jump"/>-->
<!-- <table tableName="panel_link_jump_info"/>-->
<!-- <table tableName="panel_link_jump_target_view_info"/>-->
<!-- <table tableName="panel_view_linkage"/>-->
<!-- <table tableName="panel_view_linkage_field"/>-->
<table tableName="de_engine">
<columnOverride column="configuration" property="configuration" javaType="java.lang.String"/>
</table>
<table tableName="chart_view_cache"/>
</context>
</generatorConfiguration>

36
frontend/src/api/system/engine.js Normal file
View File

@ -0,0 +1,36 @@
import request from '@/utils/request'
import {validateDs} from "@/api/system/datasource";
export function engineMode() {
return request({
url: '/engine/mode',
method: 'get',
loading: true
})
}
export function engineInfo() {
return request({
url: '/engine/info',
method: 'get',
loading: true
})
}
export function validate(data) {
return request({
url: '/engine/validate',
method: 'post',
loading: true,
data
})
}
export function save(data) {
return request({
url: '/engine/save',
method: 'post',
loading: true,
data
})
}

3
frontend/src/lang/en.js
View File

@ -647,7 +647,8 @@ export default {
port: 'Port number cannot be empty',
account: 'Account cannot be empty',
test_recipients: 'Test recipients',
tip: 'Tip: use as test mail recipient only'
tip: 'Tip: use as test mail recipient only',
engine_mode_setting: 'Engine Setting'
},
chart: {
save_snapshot: 'Save Snapshot',

3
frontend/src/lang/tw.js
View File

@ -649,7 +649,8 @@ export default {
port: '端口號不能爲空',
account: '賬戶不能爲空',
test_recipients: '測試收件人',
tip: '提示：僅用來作爲測試郵件收件人'
tip: '提示：僅用來作爲測試郵件收件人',
engine_mode_setting: '引擎設置'
},
chart: {
save_snapshot: '保存縮略圖',

3
frontend/src/lang/zh.js
View File

@ -650,7 +650,8 @@ export default {
port: '端口号不能为空',
account: '账户不能为空',
test_recipients: '测试收件人',
tip: '提示：仅用来作为测试邮件收件人'
tip: '提示：仅用来作为测试邮件收件人',
engine_mode_setting: '引擎设置'
},
chart: {
save_snapshot: '保存缩略图',

7
frontend/src/views/dataset/add/AddApi.vue
View File

@ -28,7 +28,7 @@
</el-form-item>
<el-form-item class="form-item">
<el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
<el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.type==='es' || selectedDatasource.type==='ck' || selectedDatasource.type==='mongo' || selectedDatasource.type==='redshift' || selectedDatasource.type==='hive'" />
<el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning && engineMode!=='simple'" />
</el-select>
</el-form-item>
@ -66,6 +66,7 @@
<script>
import { listApiDatasource, post, isKettleRunning } from '@/api/dataset/dataset'
import {engineMode} from "@/api/system/engine";
export default {
name: 'AddApi',
@ -86,6 +87,7 @@ export default {
syncType: 'sync_now',
tableData: [],
kettleRunning: false,
engineMode: 'local',
selectedDatasource: {}
}
},
@ -119,6 +121,9 @@ export default {
},
created() {
this.kettleState()
engineMode().then(res => {
this.engineMode = res.data
})
},
methods: {
initDataSource() {

79
frontend/src/views/dataset/add/AddDB.vue
View File

@ -13,7 +13,7 @@
</el-button>
</el-row>
</el-row>
<el-divider />
<el-divider/>
<el-row>
<el-form :inline="true">
<el-form-item class="form-item">
@ -28,15 +28,16 @@
</el-form-item>
<el-form-item class="form-item">
<el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
<el-option :label="$t('dataset.direct_connect')" value="0" />
<el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.type==='es' || selectedDatasource.type==='ck' || selectedDatasource.type==='mongo' || selectedDatasource.type==='redshift' || selectedDatasource.type==='hive'" />
<el-option :label="$t('dataset.direct_connect')" value="0"/>
<el-option :label="$t('dataset.sync_data')" value="1"
:disabled="disabledSync"/>
</el-select>
</el-form-item>
<el-form-item v-if="mode === '1'" class="form-item">
<el-select v-model="syncType" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
<el-option :label="$t('dataset.sync_now')" value="sync_now" />
<el-option :label="$t('dataset.sync_latter')" value="sync_latter" />
<el-option :label="$t('dataset.sync_now')" value="sync_now"/>
<el-option :label="$t('dataset.sync_latter')" value="sync_latter"/>
</el-select>
</el-form-item>
@ -53,12 +54,14 @@
</el-row>
<el-col style="overflow-y: auto;">
<el-checkbox-group v-model="checkTableList" size="small">
<el-tooltip v-for="t in tableData" :key="t.name" :disabled="t.enableCheck" effect="dark" :content="$t('dataset.table_already_add_to')+': '+t.datasetPath" placement="bottom">
<el-tooltip v-for="t in tableData" :key="t.name" :disabled="t.enableCheck" effect="dark"
:content="$t('dataset.table_already_add_to')+': '+t.datasetPath" placement="bottom">
<el-checkbox
border
:label="t.name"
:disabled="!t.enableCheck"
>{{ showTableNameWithComment(t) }}</el-checkbox>
>{{ showTableNameWithComment(t) }}
</el-checkbox>
</el-tooltip>
</el-checkbox-group>
</el-col>
@ -66,7 +69,8 @@
</template>
<script>
import { listDatasource, post, isKettleRunning } from '@/api/dataset/dataset'
import {listDatasource, post, isKettleRunning} from '@/api/dataset/dataset'
import {engineMode} from "@/api/system/engine";
export default {
name: 'AddDB',
@ -87,13 +91,16 @@ export default {
syncType: 'sync_now',
tableData: [],
kettleRunning: false,
selectedDatasource: {}
selectedDatasource: {},
engineMode: 'local',
disabledSync: true,
disabledSyncDs: ['es', 'ck', 'mongo', 'redshift', 'hive']
}
},
watch: {
dataSource(val) {
if (val) {
post('/datasource/getTables', { id: val }).then(response => {
post('/datasource/getTables', {id: val}).then(response => {
this.tables = response.data
this.tableData = JSON.parse(JSON.stringify(this.tables))
})
@ -101,13 +108,20 @@ export default {
if (this.options[i].id === val) {
this.selectedDatasource = this.options[i]
this.mode = '0'
if (this.engineMode === 'simple' || (!this.kettleRunning || this.disabledSyncDs.indexOf(this.selectedDatasource.type) !== -1 )) {
this.disabledSync = true
} else {
this.disabledSync = false
}
}
}
}
},
searchTable(val) {
if (val && val !== '') {
this.tableData = JSON.parse(JSON.stringify(this.tables.filter(ele => { return ele.name.toLocaleLowerCase().includes(val.toLocaleLowerCase()) })))
this.tableData = JSON.parse(JSON.stringify(this.tables.filter(ele => {
return ele.name.toLocaleLowerCase().includes(val.toLocaleLowerCase())
})))
} else {
this.tableData = JSON.parse(JSON.stringify(this.tables))
}
@ -121,6 +135,9 @@ export default {
},
created() {
this.kettleState()
engineMode().then(res => {
this.engineMode = res.data
})
},
methods: {
initDataSource() {
@ -152,7 +169,7 @@ export default {
const tables = []
const mode = this.mode
const syncType = this.syncType
this.checkTableList.forEach(function(name) {
this.checkTableList.forEach(function (name) {
tables.push({
name: ds.name + '_' + name,
sceneId: sceneId,
@ -160,7 +177,7 @@ export default {
type: 'db',
syncType: syncType,
mode: parseInt(mode),
info: JSON.stringify({ table: name })
info: JSON.stringify({table: name})
})
})
post('/dataset/table/batchAdd', tables).then(response => {
@ -171,7 +188,7 @@ export default {
cancel() {
this.dataReset()
this.$emit('switchComponent', { name: '' })
this.$emit('switchComponent', {name: ''})
},
dataReset() {
@ -187,25 +204,25 @@ export default {
</script>
<style scoped>
.el-divider--horizontal {
margin: 12px 0;
}
.el-divider--horizontal {
margin: 12px 0;
}
.form-item {
margin-bottom: 6px;
}
.form-item {
margin-bottom: 6px;
}
.el-checkbox {
margin-bottom: 14px;
margin-left: 0;
margin-right: 14px;
}
.el-checkbox {
margin-bottom: 14px;
margin-left: 0;
margin-right: 14px;
}
.el-checkbox.is-bordered + .el-checkbox.is-bordered {
margin-left: 0;
}
.el-checkbox.is-bordered + .el-checkbox.is-bordered {
margin-left: 0;
}
span{
font-size: 14px;
}
span {
font-size: 14px;
}
</style>

15
frontend/src/views/dataset/add/AddSQL.vue
View File

@ -35,7 +35,7 @@
<el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
<el-option :label="$t('dataset.direct_connect')" value="0"/>
<el-option :label="$t('dataset.sync_data')" value="1"
:disabled="!kettleRunning || selectedDatasource.type==='es' || selectedDatasource.type==='ck'|| selectedDatasource.type==='mongo'|| selectedDatasource.type==='redshift' || selectedDatasource.type==='hive'"/>
:disabled="disabledSync"/>
</el-select>
</el-form-item>
@ -124,6 +124,7 @@ import 'codemirror/keymap/emacs.js'
import 'codemirror/addon/hint/show-hint.css'
import 'codemirror/addon/hint/sql-hint'
import 'codemirror/addon/hint/show-hint'
import {engineMode} from "@/api/system/engine";
export default {
name: 'AddSQL',
@ -157,7 +158,9 @@ export default {
syncType: 'sync_now',
height: 500,
kettleRunning: false,
selectedDatasource: {}
selectedDatasource: {},
engineMode: 'local',
disabledSync: true
}
},
computed: {
@ -187,6 +190,9 @@ export default {
},
created() {
this.kettleState()
engineMode().then(res => {
this.engineMode = res.data
})
},
methods: {
kettleState() {
@ -198,6 +204,11 @@ export default {
for (let i = 0; i < this.options.length; i++) {
if (this.options[i].id === this.dataSource) {
this.selectedDatasource = this.options[i]
if (this.engineMode === 'simple' || (!this.kettleRunning || this.disabledSyncDs.indexOf(this.selectedDatasource.type) !== -1 )) {
this.disabledSync = true
} else {
this.disabledSync = false
}
}
}
},

8
frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue
View File

@ -79,7 +79,6 @@
</template>
<script>
import { isKettleRunning, post } from '@/api/dataset/dataset'
import { queryAuthModel } from '@/api/authModel/authModel'
export default {
name: 'DatasetGroupSelectorTree',
@ -144,7 +143,6 @@ export default {
all: this.$t('commons.all'),
folder: this.$t('commons.folder')
},
kettleRunning: false,
sceneMode: false,
search: '',
data: [],
@ -200,14 +198,8 @@ export default {
this.treeNode()
},
created() {
this.kettleState()
},
methods: {
kettleState() {
isKettleRunning(false).then(res => {
this.kettleRunning = res.data
})
},
close() {
this.editGroup = false
this.groupForm = {

7
frontend/src/views/dataset/group/Group.vue
View File

@ -87,7 +87,7 @@
<svg-icon icon-class="ds-sql" class="ds-icon-sql" />
{{ $t('dataset.sql_data') }}
</el-dropdown-item>
<el-dropdown-item :command="beforeClickAddData('excel',data)" :disabled="!kettleRunning">
<el-dropdown-item :command="beforeClickAddData('excel',data)" :disabled="!kettleRunning && engineMode!=='simple'">
<svg-icon icon-class="ds-excel" class="ds-icon-excel" />
{{ $t('dataset.excel_data') }}
</el-dropdown-item>
@ -232,6 +232,7 @@ import { loadTable, getScene, addGroup, delGroup, delTable, post, isKettleRunnin
import GroupMoveSelector from './GroupMoveSelector'
import DsMoveSelector from './DsMoveSelector'
import { queryAuthModel } from '@/api/authModel/authModel'
import {engineMode} from "@/api/system/engine";
export default {
name: 'Group',
@ -304,6 +305,7 @@ export default {
},
isTreeSearch: false,
kettleRunning: false,
engineMode: 'local',
searchPids: [], // 查询命中的pid
filterText: '',
searchType: 'all',
@ -333,6 +335,9 @@ export default {
},
created() {
this.kettleState()
engineMode().then(res => {
this.engineMode = res.data
})
},
mounted() {
this.treeNode(true)

263
frontend/src/views/system/SysParam/SimpleModeSetting.vue Normal file
View File

@ -0,0 +1,263 @@
<template>
<div>
<!--邮件表单-->
<el-form ref="form" v-loading="loading"
:model="form"
:rules="rules"
class="demo-form-inline"
:disabled="show"
label-width="180px"
label-position="top"
size="small"
>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.type')" prop="type">
<el-select
v-model="form.type"
:placeholder="$t('datasource.please_choose_type')"
@change="changeType()"
filterable
>
<el-option
v-for="item in allTypes"
:key="item.name"
:label="item.label"
:value="item.name"
/>
</el-select>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.host')" prop="configuration.host">
<el-input v-model="form.configuration.host"/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.data_base')" prop="configuration.dataBase">
<el-input v-model="form.configuration.dataBase"/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.user_name')">
<el-input v-model="form.configuration.username"/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.password')">
<el-input v-model="form.configuration.password" show-password/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.port')" prop="configuration.port">
<el-input v-model="form.configuration.port" autocomplete="off" type="number" min="0"/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('datasource.extra_params')">
<el-input v-model="form.configuration.extraParams"/>
</el-form-item>
</el-col>
</el-row>
</el-form>
<div>
<el-button type="primary" size="small" @click="validaDatasource">
{{ $t('commons.validate') }}
</el-button>
<el-button v-if="showEdit" size="small" @click="edit">
{{ $t('commons.edit') }}
</el-button>
<el-button v-if="showSave" type="success" size="small" @click="save">
{{ $t('commons.save') }}
</el-button>
<el-button v-if="showCancel" type="info" size="small" @click="cancel">
{{ $t('commons.cancel') }}
</el-button>
</div>
</div>
</template>
<script>
import {engineInfo, validate, save} from '@/api/system/engine'
import i18n from "@/lang";
export default {
name: 'SimpleMode',
data() {
return {
form:
{
type: 'mysql',
configuration: {
host: '',
dataBase: '',
username: '',
password: '',
port: '',
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
}
},
originConfiguration: {
host: '',
dataBase: '',
username: '',
password: '',
port: '',
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
},
input: '',
visible: true,
showEdit: true,
showSave: false,
showCancel: false,
show: true,
disabledConnection: false,
disabledSave: false,
loading: false,
rules: {
host: [
{
required: true,
message: this.$t('system_parameter_setting.host'),
trigger: ['change', 'blur']
}
],
port: [
{
required: true,
message: this.$t('system_parameter_setting.port'),
trigger: ['change', 'blur']
}
],
account: [
{
required: true,
message: this.$t('system_parameter_setting.account'),
trigger: ['change', 'blur']
}]
},
allTypes: [
{
name: 'mysql',
label: 'MySQL',
type: 'jdbc',
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
}
]
}
},
created() {
this.query()
},
methods: {
query() {
engineInfo().then(response => {
if (response.data.id) {
this.form = JSON.parse(JSON.stringify(response.data))
this.form.configuration = JSON.parse(this.form.configuration)
this.originConfiguration = JSON.parse(JSON.stringify(this.form.configuration))
}
this.$nextTick(() => {
this.$refs.form.clearValidate()
})
})
},
edit() {
this.showEdit = false
this.showSave = true
this.showCancel = true
this.show = false
},
save() {
if (this.form.configuration.dataSourceType === 'jdbc' && this.form.configuration.port <= 0) {
this.$message.error(i18n.t('datasource.port_no_less_then_0'))
return
}
if (this.form.configuration.initialPoolSize < 0 || this.form.configuration.minPoolSize < 0 || this.form.configuration.maxPoolSize < 0) {
this.$message.error(i18n.t('datasource.no_less_then_0'))
return
}
this.$refs.form.validate(valid => {
if (!valid) {
return false
}
const form = JSON.parse(JSON.stringify(this.form))
form.configuration = JSON.stringify(form.configuration)
save(form).then(res => {
this.showEdit = true
this.showCancel = false
this.showSave = false
this.show = true
this.originConfiguration = JSON.parse(JSON.stringify(this.form.configuration))
this.$success(i18n.t('commons.save_success'))
})
})
},
cancel() {
this.showEdit = true
this.showCancel = false
this.showSave = false
this.show = true
this.form.configuration = JSON.parse(JSON.stringify(this.originConfiguration))
},
changeType() {
for (let i = 0; i < this.allTypes.length; i++) {
if (this.allTypes[i].name === this.form.type) {
this.form.configuration.dataSourceType = this.allTypes[i].type
this.form.configuration.extraParams = this.allTypes[i].extraParams
}
}
},
validaDatasource() {
if (!this.form.configuration.schema && this.form.type === 'oracle') {
this.$message.error(i18n.t('datasource.please_choose_schema'))
return
}
if (this.form.configuration.dataSourceType === 'jdbc' && this.form.configuration.port <= 0) {
this.$message.error(i18n.t('datasource.port_no_less_then_0'))
return
}
this.$refs.form.validate(valid => {
if (valid) {
const data = JSON.parse(JSON.stringify(this.form))
data.configuration = JSON.stringify(data.configuration)
validate(data).then(res => {
if (res.success) {
this.$success(i18n.t('datasource.validate_success'))
} else {
if (res.message.length < 2500) {
this.$error(res.message)
} else {
this.$error(res.message.substring(0, 2500) + '......')
}
}
}).catch(res => {
this.$error(res.message)
})
} else {
return false
}
})
},
}
}
</script>
<style scoped>
</style>

16
frontend/src/views/system/SysParam/index.vue
View File

@ -26,32 +26,42 @@
<plugin-com v-if="isPluginLoaded" ref="DisplaySetting" component-name="SsoSetting" />
</el-tab-pane>
<el-tab-pane v-if="engineMode==='simple'" :lazy="true" :label="$t('system_parameter_setting.engine_mode_setting')" name="six">
<simple-mode />
</el-tab-pane>
</el-tabs>
</layout-content>
</template>
<script>
import BasicSetting from './BasicSetting'
import EmailSetting from './EmailSetting'
import SimpleMode from './SimpleModeSetting'
import LayoutContent from '@/components/business/LayoutContent'
import PluginCom from '@/views/system/plugin/PluginCom'
import { pluginLoaded } from '@/api/user'
import { engineMode } from '@/api/system/engine'
export default {
components: { BasicSetting, EmailSetting, LayoutContent, PluginCom },
components: { BasicSetting, EmailSetting, LayoutContent, PluginCom, SimpleMode},
data() {
return {
activeName: 'zero',
isPluginLoaded: false
isPluginLoaded: false,
engineMode: 'local'
}
},
beforeCreate() {
pluginLoaded().then(res => {
this.isPluginLoaded = res.success && res.data
})
engineMode().then(res => {
this.engineMode = res.data
})
},
methods: {
handleClick(tab, event) {
console.log(tab, event)
// console.log(tab, event)
}
}
}