github/dataease
github/dataease
3
0
Fork 1
mirror of https://github.com/dataease/dataease.git synced 2025-02-25 20:42:55 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

feat: 支持精简模式

Browse Source
This commit is contained in:
taojinlong 2022-03-09 20:03:16 +08:00
parent 8177049f5f
commit 0609d24555
55 changed files with 3080 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
backend/pom.xml
View File

@ -427,48 +427,48 @@
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<!-- <plugins>-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<?m2e execute onConfiguration?>
<id>main-class-placement</id>
<phase>generate-resources</phase>
<configuration>
<target>
<move todir="src/main/resources/static">
<fileset dir="../frontend/dist">
<exclude name="*.html"/>
</fileset>
</move>
<move todir="src/main/resources/templates">
<fileset dir="../frontend/dist">
<include name="*.html"/>
</fileset>
</move>
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-antrun-plugin</artifactId>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <?m2e execute onConfiguration?>-->
<!-- <id>main-class-placement</id>-->
<!-- <phase>generate-resources</phase>-->
<!-- <configuration>-->
<!-- <target>-->
<!-- <move todir="src/main/resources/static">-->
<!-- <fileset dir="../frontend/dist">-->
<!-- <exclude name="*.html"/>-->
<!-- </fileset>-->
<!-- </move>-->
<!-- <move todir="src/main/resources/templates">-->
<!-- <fileset dir="../frontend/dist">-->
<!-- <include name="*.html"/>-->
<!-- </fileset>-->
<!-- </move>-->
<copy todir="src/main/resources/static/de-app">
<fileset dir="../mobile/dist">
<exclude name="*.html"/>
</fileset>
</copy>
<!-- <copy todir="src/main/resources/static/de-app">-->
<!-- <fileset dir="../mobile/dist">-->
<!-- <exclude name="*.html"/>-->
<!-- </fileset>-->
<!-- </copy>-->
<copy file="../mobile/dist/index.html" tofile="src/main/resources/templates/app.html" />
<!-- <copy file="../mobile/dist/index.html" tofile="src/main/resources/templates/app.html" />-->
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
<!-- </target>-->
<!-- </configuration>-->
<!-- <goals>-->
<!-- <goal>run</goal>-->
<!-- </goals>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
<!-- </plugins>-->
</build>
</profile>

27
backend/src/main/java/io/dataease/base/domain/DeEngine.java Normal file
View File

@ -0,0 +1,27 @@
package io.dataease.base.domain;
import java.io.Serializable;
import lombok.Data;
@Data
public class DeEngine implements Serializable {
private String id;
private String name;
private String desc;
private String type;
private Long createTime;
private Long updateTime;
private String createBy;
private String status;
private String configuration;
private static final long serialVersionUID = 1L;
}

740
backend/src/main/java/io/dataease/base/domain/DeEngineExample.java Normal file
View File

@ -0,0 +1,740 @@
package io.dataease.base.domain;
import java.util.ArrayList;
import java.util.List;
public class DeEngineExample {
protected String orderByClause;
protected boolean distinct;
protected List<Criteria> oredCriteria;
public DeEngineExample() {
oredCriteria = new ArrayList<Criteria>();
}
public void setOrderByClause(String orderByClause) {
this.orderByClause = orderByClause;
}
public String getOrderByClause() {
return orderByClause;
}
public void setDistinct(boolean distinct) {
this.distinct = distinct;
}
public boolean isDistinct() {
return distinct;
}
public List<Criteria> getOredCriteria() {
return oredCriteria;
}
public void or(Criteria criteria) {
oredCriteria.add(criteria);
}
public Criteria or() {
Criteria criteria = createCriteriaInternal();
oredCriteria.add(criteria);
return criteria;
}
public Criteria createCriteria() {
Criteria criteria = createCriteriaInternal();
if (oredCriteria.size() == 0) {
oredCriteria.add(criteria);
}
return criteria;
}
protected Criteria createCriteriaInternal() {
Criteria criteria = new Criteria();
return criteria;
}
public void clear() {
oredCriteria.clear();
orderByClause = null;
distinct = false;
}
protected abstract static class GeneratedCriteria {
protected List<Criterion> criteria;
protected GeneratedCriteria() {
super();
criteria = new ArrayList<Criterion>();
}
public boolean isValid() {
return criteria.size() > 0;
}
public List<Criterion> getAllCriteria() {
return criteria;
}
public List<Criterion> getCriteria() {
return criteria;
}
protected void addCriterion(String condition) {
if (condition == null) {
throw new RuntimeException("Value for condition cannot be null");
}
criteria.add(new Criterion(condition));
}
protected void addCriterion(String condition, Object value, String property) {
if (value == null) {
throw new RuntimeException("Value for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(String condition, Object value1, Object value2, String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value1, value2));
}
public Criteria andIdIsNull() {
addCriterion("id is null");
return (Criteria) this;
}
public Criteria andIdIsNotNull() {
addCriterion("id is not null");
return (Criteria) this;
}
public Criteria andIdEqualTo(String value) {
addCriterion("id =", value, "id");
return (Criteria) this;
}
public Criteria andIdNotEqualTo(String value) {
addCriterion("id <>", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThan(String value) {
addCriterion("id >", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThanOrEqualTo(String value) {
addCriterion("id >=", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThan(String value) {
addCriterion("id <", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThanOrEqualTo(String value) {
addCriterion("id <=", value, "id");
return (Criteria) this;
}
public Criteria andIdLike(String value) {
addCriterion("id like", value, "id");
return (Criteria) this;
}
public Criteria andIdNotLike(String value) {
addCriterion("id not like", value, "id");
return (Criteria) this;
}
public Criteria andIdIn(List<String> values) {
addCriterion("id in", values, "id");
return (Criteria) this;
}
public Criteria andIdNotIn(List<String> values) {
addCriterion("id not in", values, "id");
return (Criteria) this;
}
public Criteria andIdBetween(String value1, String value2) {
addCriterion("id between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andIdNotBetween(String value1, String value2) {
addCriterion("id not between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andNameIsNull() {
addCriterion("`name` is null");
return (Criteria) this;
}
public Criteria andNameIsNotNull() {
addCriterion("`name` is not null");
return (Criteria) this;
}
public Criteria andNameEqualTo(String value) {
addCriterion("`name` =", value, "name");
return (Criteria) this;
}
public Criteria andNameNotEqualTo(String value) {
addCriterion("`name` <>", value, "name");
return (Criteria) this;
}
public Criteria andNameGreaterThan(String value) {
addCriterion("`name` >", value, "name");
return (Criteria) this;
}
public Criteria andNameGreaterThanOrEqualTo(String value) {
addCriterion("`name` >=", value, "name");
return (Criteria) this;
}
public Criteria andNameLessThan(String value) {
addCriterion("`name` <", value, "name");
return (Criteria) this;
}
public Criteria andNameLessThanOrEqualTo(String value) {
addCriterion("`name` <=", value, "name");
return (Criteria) this;
}
public Criteria andNameLike(String value) {
addCriterion("`name` like", value, "name");
return (Criteria) this;
}
public Criteria andNameNotLike(String value) {
addCriterion("`name` not like", value, "name");
return (Criteria) this;
}
public Criteria andNameIn(List<String> values) {
addCriterion("`name` in", values, "name");
return (Criteria) this;
}
public Criteria andNameNotIn(List<String> values) {
addCriterion("`name` not in", values, "name");
return (Criteria) this;
}
public Criteria andNameBetween(String value1, String value2) {
addCriterion("`name` between", value1, value2, "name");
return (Criteria) this;
}
public Criteria andNameNotBetween(String value1, String value2) {
addCriterion("`name` not between", value1, value2, "name");
return (Criteria) this;
}
public Criteria andDescIsNull() {
addCriterion("`desc` is null");
return (Criteria) this;
}
public Criteria andDescIsNotNull() {
addCriterion("`desc` is not null");
return (Criteria) this;
}
public Criteria andDescEqualTo(String value) {
addCriterion("`desc` =", value, "desc");
return (Criteria) this;
}
public Criteria andDescNotEqualTo(String value) {
addCriterion("`desc` <>", value, "desc");
return (Criteria) this;
}
public Criteria andDescGreaterThan(String value) {
addCriterion("`desc` >", value, "desc");
return (Criteria) this;
}
public Criteria andDescGreaterThanOrEqualTo(String value) {
addCriterion("`desc` >=", value, "desc");
return (Criteria) this;
}
public Criteria andDescLessThan(String value) {
addCriterion("`desc` <", value, "desc");
return (Criteria) this;
}
public Criteria andDescLessThanOrEqualTo(String value) {
addCriterion("`desc` <=", value, "desc");
return (Criteria) this;
}
public Criteria andDescLike(String value) {
addCriterion("`desc` like", value, "desc");
return (Criteria) this;
}
public Criteria andDescNotLike(String value) {
addCriterion("`desc` not like", value, "desc");
return (Criteria) this;
}
public Criteria andDescIn(List<String> values) {
addCriterion("`desc` in", values, "desc");
return (Criteria) this;
}
public Criteria andDescNotIn(List<String> values) {
addCriterion("`desc` not in", values, "desc");
return (Criteria) this;
}
public Criteria andDescBetween(String value1, String value2) {
addCriterion("`desc` between", value1, value2, "desc");
return (Criteria) this;
}
public Criteria andDescNotBetween(String value1, String value2) {
addCriterion("`desc` not between", value1, value2, "desc");
return (Criteria) this;
}
public Criteria andTypeIsNull() {
addCriterion("`type` is null");
return (Criteria) this;
}
public Criteria andTypeIsNotNull() {
addCriterion("`type` is not null");
return (Criteria) this;
}
public Criteria andTypeEqualTo(String value) {
addCriterion("`type` =", value, "type");
return (Criteria) this;
}
public Criteria andTypeNotEqualTo(String value) {
addCriterion("`type` <>", value, "type");
return (Criteria) this;
}
public Criteria andTypeGreaterThan(String value) {
addCriterion("`type` >", value, "type");
return (Criteria) this;
}
public Criteria andTypeGreaterThanOrEqualTo(String value) {
addCriterion("`type` >=", value, "type");
return (Criteria) this;
}
public Criteria andTypeLessThan(String value) {
addCriterion("`type` <", value, "type");
return (Criteria) this;
}
public Criteria andTypeLessThanOrEqualTo(String value) {
addCriterion("`type` <=", value, "type");
return (Criteria) this;
}
public Criteria andTypeLike(String value) {
addCriterion("`type` like", value, "type");
return (Criteria) this;
}
public Criteria andTypeNotLike(String value) {
addCriterion("`type` not like", value, "type");
return (Criteria) this;
}
public Criteria andTypeIn(List<String> values) {
addCriterion("`type` in", values, "type");
return (Criteria) this;
}
public Criteria andTypeNotIn(List<String> values) {
addCriterion("`type` not in", values, "type");
return (Criteria) this;
}
public Criteria andTypeBetween(String value1, String value2) {
addCriterion("`type` between", value1, value2, "type");
return (Criteria) this;
}
public Criteria andTypeNotBetween(String value1, String value2) {
addCriterion("`type` not between", value1, value2, "type");
return (Criteria) this;
}
public Criteria andCreateTimeIsNull() {
addCriterion("create_time is null");
return (Criteria) this;
}
public Criteria andCreateTimeIsNotNull() {
addCriterion("create_time is not null");
return (Criteria) this;
}
public Criteria andCreateTimeEqualTo(Long value) {
addCriterion("create_time =", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeNotEqualTo(Long value) {
addCriterion("create_time <>", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeGreaterThan(Long value) {
addCriterion("create_time >", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeGreaterThanOrEqualTo(Long value) {
addCriterion("create_time >=", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeLessThan(Long value) {
addCriterion("create_time <", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeLessThanOrEqualTo(Long value) {
addCriterion("create_time <=", value, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeIn(List<Long> values) {
addCriterion("create_time in", values, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeNotIn(List<Long> values) {
addCriterion("create_time not in", values, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeBetween(Long value1, Long value2) {
addCriterion("create_time between", value1, value2, "createTime");
return (Criteria) this;
}
public Criteria andCreateTimeNotBetween(Long value1, Long value2) {
addCriterion("create_time not between", value1, value2, "createTime");
return (Criteria) this;
}
public Criteria andUpdateTimeIsNull() {
addCriterion("update_time is null");
return (Criteria) this;
}
public Criteria andUpdateTimeIsNotNull() {
addCriterion("update_time is not null");
return (Criteria) this;
}
public Criteria andUpdateTimeEqualTo(Long value) {
addCriterion("update_time =", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeNotEqualTo(Long value) {
addCriterion("update_time <>", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeGreaterThan(Long value) {
addCriterion("update_time >", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeGreaterThanOrEqualTo(Long value) {
addCriterion("update_time >=", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeLessThan(Long value) {
addCriterion("update_time <", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeLessThanOrEqualTo(Long value) {
addCriterion("update_time <=", value, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeIn(List<Long> values) {
addCriterion("update_time in", values, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeNotIn(List<Long> values) {
addCriterion("update_time not in", values, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeBetween(Long value1, Long value2) {
addCriterion("update_time between", value1, value2, "updateTime");
return (Criteria) this;
}
public Criteria andUpdateTimeNotBetween(Long value1, Long value2) {
addCriterion("update_time not between", value1, value2, "updateTime");
return (Criteria) this;
}
public Criteria andCreateByIsNull() {
addCriterion("create_by is null");
return (Criteria) this;
}
public Criteria andCreateByIsNotNull() {
addCriterion("create_by is not null");
return (Criteria) this;
}
public Criteria andCreateByEqualTo(String value) {
addCriterion("create_by =", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByNotEqualTo(String value) {
addCriterion("create_by <>", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByGreaterThan(String value) {
addCriterion("create_by >", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByGreaterThanOrEqualTo(String value) {
addCriterion("create_by >=", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByLessThan(String value) {
addCriterion("create_by <", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByLessThanOrEqualTo(String value) {
addCriterion("create_by <=", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByLike(String value) {
addCriterion("create_by like", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByNotLike(String value) {
addCriterion("create_by not like", value, "createBy");
return (Criteria) this;
}
public Criteria andCreateByIn(List<String> values) {
addCriterion("create_by in", values, "createBy");
return (Criteria) this;
}
public Criteria andCreateByNotIn(List<String> values) {
addCriterion("create_by not in", values, "createBy");
return (Criteria) this;
}
public Criteria andCreateByBetween(String value1, String value2) {
addCriterion("create_by between", value1, value2, "createBy");
return (Criteria) this;
}
public Criteria andCreateByNotBetween(String value1, String value2) {
addCriterion("create_by not between", value1, value2, "createBy");
return (Criteria) this;
}
public Criteria andStatusIsNull() {
addCriterion("`status` is null");
return (Criteria) this;
}
public Criteria andStatusIsNotNull() {
addCriterion("`status` is not null");
return (Criteria) this;
}
public Criteria andStatusEqualTo(String value) {
addCriterion("`status` =", value, "status");
return (Criteria) this;
}
public Criteria andStatusNotEqualTo(String value) {
addCriterion("`status` <>", value, "status");
return (Criteria) this;
}
public Criteria andStatusGreaterThan(String value) {
addCriterion("`status` >", value, "status");
return (Criteria) this;
}
public Criteria andStatusGreaterThanOrEqualTo(String value) {
addCriterion("`status` >=", value, "status");
return (Criteria) this;
}
public Criteria andStatusLessThan(String value) {
addCriterion("`status` <", value, "status");
return (Criteria) this;
}
public Criteria andStatusLessThanOrEqualTo(String value) {
addCriterion("`status` <=", value, "status");
return (Criteria) this;
}
public Criteria andStatusLike(String value) {
addCriterion("`status` like", value, "status");
return (Criteria) this;
}
public Criteria andStatusNotLike(String value) {
addCriterion("`status` not like", value, "status");
return (Criteria) this;
}
public Criteria andStatusIn(List<String> values) {
addCriterion("`status` in", values, "status");
return (Criteria) this;
}
public Criteria andStatusNotIn(List<String> values) {
addCriterion("`status` not in", values, "status");
return (Criteria) this;
}
public Criteria andStatusBetween(String value1, String value2) {
addCriterion("`status` between", value1, value2, "status");
return (Criteria) this;
}
public Criteria andStatusNotBetween(String value1, String value2) {
addCriterion("`status` not between", value1, value2, "status");
return (Criteria) this;
}
}
public static class Criteria extends GeneratedCriteria {
protected Criteria() {
super();
}
}
public static class Criterion {
private String condition;
private Object value;
private Object secondValue;
private boolean noValue;
private boolean singleValue;
private boolean betweenValue;
private boolean listValue;
private String typeHandler;
public String getCondition() {
return condition;
}
public Object getValue() {
return value;
}
public Object getSecondValue() {
return secondValue;
}
public boolean isNoValue() {
return noValue;
}
public boolean isSingleValue() {
return singleValue;
}
public boolean isBetweenValue() {
return betweenValue;
}
public boolean isListValue() {
return listValue;
}
public String getTypeHandler() {
return typeHandler;
}
protected Criterion(String condition) {
super();
this.condition = condition;
this.typeHandler = null;
this.noValue = true;
}
protected Criterion(String condition, Object value, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.typeHandler = typeHandler;
if (value instanceof List<?>) {
this.listValue = true;
} else {
this.singleValue = true;
}
}
protected Criterion(String condition, Object value) {
this(condition, value, null);
}
protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.secondValue = secondValue;
this.typeHandler = typeHandler;
this.betweenValue = true;
}
protected Criterion(String condition, Object value, Object secondValue) {
this(condition, value, secondValue, null);
}
}
}

36
backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.java Normal file
View File

@ -0,0 +1,36 @@
package io.dataease.base.mapper;
import io.dataease.base.domain.DeEngine;
import io.dataease.base.domain.DeEngineExample;
import java.util.List;
import org.apache.ibatis.annotations.Param;
public interface DeEngineMapper {
long countByExample(DeEngineExample example);
int deleteByExample(DeEngineExample example);
int deleteByPrimaryKey(String id);
int insert(DeEngine record);
int insertSelective(DeEngine record);
List<DeEngine> selectByExampleWithBLOBs(DeEngineExample example);
List<DeEngine> selectByExample(DeEngineExample example);
DeEngine selectByPrimaryKey(String id);
int updateByExampleSelective(@Param("record") DeEngine record, @Param("example") DeEngineExample example);
int updateByExampleWithBLOBs(@Param("record") DeEngine record, @Param("example") DeEngineExample example);
int updateByExample(@Param("record") DeEngine record, @Param("example") DeEngineExample example);
int updateByPrimaryKeySelective(DeEngine record);
int updateByPrimaryKeyWithBLOBs(DeEngine record);
int updateByPrimaryKey(DeEngine record);
}

323
backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.xml Normal file
View File

@ -0,0 +1,323 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="io.dataease.base.mapper.DeEngineMapper">
<resultMap id="BaseResultMap" type="io.dataease.base.domain.DeEngine">
<id column="id" jdbcType="VARCHAR" property="id" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="desc" jdbcType="VARCHAR" property="desc" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="create_time" jdbcType="BIGINT" property="createTime" />
<result column="update_time" jdbcType="BIGINT" property="updateTime" />
<result column="create_by" jdbcType="VARCHAR" property="createBy" />
<result column="status" jdbcType="VARCHAR" property="status" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="io.dataease.base.domain.DeEngine">
<result column="configuration" jdbcType="LONGVARCHAR" property="configuration" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Update_By_Example_Where_Clause">
<where>
<foreach collection="example.oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id, `name`, `desc`, `type`, create_time, update_time, create_by, `status`
</sql>
<sql id="Blob_Column_List">
configuration
</sql>
<select id="selectByExampleWithBLOBs" parameterType="io.dataease.base.domain.DeEngineExample" resultMap="ResultMapWithBLOBs">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from de_engine
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByExample" parameterType="io.dataease.base.domain.DeEngineExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
from de_engine
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByPrimaryKey" parameterType="java.lang.String" resultMap="ResultMapWithBLOBs">
select
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from de_engine
where id = #{id,jdbcType=VARCHAR}
</select>
<delete id="deleteByPrimaryKey" parameterType="java.lang.String">
delete from de_engine
where id = #{id,jdbcType=VARCHAR}
</delete>
<delete id="deleteByExample" parameterType="io.dataease.base.domain.DeEngineExample">
delete from de_engine
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</delete>
<insert id="insert" parameterType="io.dataease.base.domain.DeEngine">
insert into de_engine (id, `name`, `desc`,
`type`, create_time, update_time,
create_by, `status`, configuration
)
values (#{id,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{desc,jdbcType=VARCHAR},
#{type,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{updateTime,jdbcType=BIGINT},
#{createBy,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{configuration,jdbcType=LONGVARCHAR}
)
</insert>
<insert id="insertSelective" parameterType="io.dataease.base.domain.DeEngine">
insert into de_engine
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null">
id,
</if>
<if test="name != null">
`name`,
</if>
<if test="desc != null">
`desc`,
</if>
<if test="type != null">
`type`,
</if>
<if test="createTime != null">
create_time,
</if>
<if test="updateTime != null">
update_time,
</if>
<if test="createBy != null">
create_by,
</if>
<if test="status != null">
`status`,
</if>
<if test="configuration != null">
configuration,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
#{id,jdbcType=VARCHAR},
</if>
<if test="name != null">
#{name,jdbcType=VARCHAR},
</if>
<if test="desc != null">
#{desc,jdbcType=VARCHAR},
</if>
<if test="type != null">
#{type,jdbcType=VARCHAR},
</if>
<if test="createTime != null">
#{createTime,jdbcType=BIGINT},
</if>
<if test="updateTime != null">
#{updateTime,jdbcType=BIGINT},
</if>
<if test="createBy != null">
#{createBy,jdbcType=VARCHAR},
</if>
<if test="status != null">
#{status,jdbcType=VARCHAR},
</if>
<if test="configuration != null">
#{configuration,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="io.dataease.base.domain.DeEngineExample" resultType="java.lang.Long">
select count(*) from de_engine
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</select>
<update id="updateByExampleSelective" parameterType="map">
update de_engine
<set>
<if test="record.id != null">
id = #{record.id,jdbcType=VARCHAR},
</if>
<if test="record.name != null">
`name` = #{record.name,jdbcType=VARCHAR},
</if>
<if test="record.desc != null">
`desc` = #{record.desc,jdbcType=VARCHAR},
</if>
<if test="record.type != null">
`type` = #{record.type,jdbcType=VARCHAR},
</if>
<if test="record.createTime != null">
create_time = #{record.createTime,jdbcType=BIGINT},
</if>
<if test="record.updateTime != null">
update_time = #{record.updateTime,jdbcType=BIGINT},
</if>
<if test="record.createBy != null">
create_by = #{record.createBy,jdbcType=VARCHAR},
</if>
<if test="record.status != null">
`status` = #{record.status,jdbcType=VARCHAR},
</if>
<if test="record.configuration != null">
configuration = #{record.configuration,jdbcType=LONGVARCHAR},
</if>
</set>
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByExampleWithBLOBs" parameterType="map">
update de_engine
set id = #{record.id,jdbcType=VARCHAR},
`name` = #{record.name,jdbcType=VARCHAR},
`desc` = #{record.desc,jdbcType=VARCHAR},
`type` = #{record.type,jdbcType=VARCHAR},
create_time = #{record.createTime,jdbcType=BIGINT},
update_time = #{record.updateTime,jdbcType=BIGINT},
create_by = #{record.createBy,jdbcType=VARCHAR},
`status` = #{record.status,jdbcType=VARCHAR},
configuration = #{record.configuration,jdbcType=LONGVARCHAR}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByExample" parameterType="map">
update de_engine
set id = #{record.id,jdbcType=VARCHAR},
`name` = #{record.name,jdbcType=VARCHAR},
`desc` = #{record.desc,jdbcType=VARCHAR},
`type` = #{record.type,jdbcType=VARCHAR},
create_time = #{record.createTime,jdbcType=BIGINT},
update_time = #{record.updateTime,jdbcType=BIGINT},
create_by = #{record.createBy,jdbcType=VARCHAR},
`status` = #{record.status,jdbcType=VARCHAR}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByPrimaryKeySelective" parameterType="io.dataease.base.domain.DeEngine">
update de_engine
<set>
<if test="name != null">
`name` = #{name,jdbcType=VARCHAR},
</if>
<if test="desc != null">
`desc` = #{desc,jdbcType=VARCHAR},
</if>
<if test="type != null">
`type` = #{type,jdbcType=VARCHAR},
</if>
<if test="createTime != null">
create_time = #{createTime,jdbcType=BIGINT},
</if>
<if test="updateTime != null">
update_time = #{updateTime,jdbcType=BIGINT},
</if>
<if test="createBy != null">
create_by = #{createBy,jdbcType=VARCHAR},
</if>
<if test="status != null">
`status` = #{status,jdbcType=VARCHAR},
</if>
<if test="configuration != null">
configuration = #{configuration,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=VARCHAR}
</update>
<update id="updateByPrimaryKeyWithBLOBs" parameterType="io.dataease.base.domain.DeEngine">
update de_engine
set `name` = #{name,jdbcType=VARCHAR},
`desc` = #{desc,jdbcType=VARCHAR},
`type` = #{type,jdbcType=VARCHAR},
create_time = #{createTime,jdbcType=BIGINT},
update_time = #{updateTime,jdbcType=BIGINT},
create_by = #{createBy,jdbcType=VARCHAR},
`status` = #{status,jdbcType=VARCHAR},
configuration = #{configuration,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=VARCHAR}
</update>
<update id="updateByPrimaryKey" parameterType="io.dataease.base.domain.DeEngine">
update de_engine
set `name` = #{name,jdbcType=VARCHAR},
`desc` = #{desc,jdbcType=VARCHAR},
`type` = #{type,jdbcType=VARCHAR},
create_time = #{createTime,jdbcType=BIGINT},
update_time = #{updateTime,jdbcType=BIGINT},
create_by = #{createBy,jdbcType=VARCHAR},
`status` = #{status,jdbcType=VARCHAR}
where id = #{id,jdbcType=VARCHAR}
</update>
</mapper>

5
backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java
View File

@ -8,14 +8,15 @@ public enum DatasourceTypes {
ds_doris("ds_doris", "ds_doris", "com.mysql.jdbc.Driver", "`", "`", "'", "'"),
pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""),
de_doris("de_doris", "de_doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""),
mongo("mongo", "mongodb", "com.mongodb.jdbc.MongoDriver", "`", "`", "\"", "\""),
ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"),
db2("db2", "db2", "com.ibm.db2.jcc.DB2Driver", "\"", "\"", "\"", "\""),
es("es", "es", "", "\"", "\"", "\"", "\""),
redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
api("api", "api", "", "\"", "\"", "\"", "\"");
api("api", "api", "", "\"", "\"", "\"", "\""),
engine_doris("engine_doris", "engine_doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
engine_mysql("mysql", "mysql", "com.mysql.jdbc.Driver", "`", "`", "'", "'");
private String feature;

14
backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java → backend/src/main/java/io/dataease/commons/utils/TableUtils.java
View File

@ -1,28 +1,28 @@
package io.dataease.commons.utils;
public class DorisTableUtils {
public class TableUtils {
public static String dorisName(String datasetId) {
public static String tableName(String datasetId) {
return "ds_" + datasetId.replace("-", "_");
}
public static String dorisTmpName(String dorisName) {
public static String tmpName(String dorisName) {
return "tmp_" + dorisName;
}
public static String dorisDeleteName(String dorisName) {
public static String deleteName(String dorisName) {
return "delete_" + dorisName;
}
public static String dorisAddName(String dorisName) {
public static String addName(String dorisName) {
return "add_" + dorisName;
}
public static String dorisFieldName(String dorisName) {
public static String fieldName(String dorisName) {
return "f_" + Md5Utils.md5(dorisName);
}
public static String dorisFieldNameShort(String dorisName) {
public static String fieldNameShort(String dorisName) {
return "f_" + Md5Utils.md5(dorisName).substring(8, 24);
}

22
backend/src/main/java/io/dataease/config/CommonConfig.java
View File

@ -23,28 +23,6 @@ public class CommonConfig {
private Environment env; // 保存了配置文件的信息
private static String root_path = "/opt/dataease/data/kettle/";
@Bean(name = "DorisDatasource")
@ConditionalOnMissingBean
public Datasource configuration() {
JSONObject jsonObject = new JSONObject();
jsonObject.put("dataSourceType", "jdbc");
jsonObject.put("dataBase", env.getProperty("doris.db", "doris"));
jsonObject.put("username", env.getProperty("doris.user", "root"));
jsonObject.put("password", env.getProperty("doris.password", "dataease"));
jsonObject.put("host", env.getProperty("doris.host", "doris"));
jsonObject.put("port", env.getProperty("doris.port", "9030"));
jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030"));
Datasource datasource = new Datasource();
datasource.setId("doris");
datasource.setName("doris");
datasource.setDesc("doris");
datasource.setType("de_doris");
datasource.setConfiguration(jsonObject.toJSONString());
return datasource;
}
@Bean
@ConditionalOnMissingBean
public KettleFileRepository kettleFileRepository() throws Exception {

3
backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java
View File

@ -20,5 +20,8 @@ public class DataSourceInitStartListener implements ApplicationListener<Applicat
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
datasourceService.initAllDataSourceConnectionPool();
dataSetTableService.updateDatasetTableStatus();
}
}

10
backend/src/main/java/io/dataease/provider/query/DDLProvider.java → backend/src/main/java/io/dataease/provider/DDLProvider.java
View File

@ -1,4 +1,8 @@
package io.dataease.provider.query;
package io.dataease.provider;
import io.dataease.base.domain.DatasetTableField;
import java.util.List;
/**
* @Author gin
@ -10,4 +14,8 @@ public abstract class DDLProvider {
public abstract String dropTable(String name);
public abstract String dropView(String name);
public abstract String replaceTable(String name);
public abstract String createTableSql(String name, List<DatasetTableField> datasetTableFields);
}

34
backend/src/main/java/io/dataease/provider/DDLProviderImpl.java Normal file
View File

@ -0,0 +1,34 @@
package io.dataease.provider;
import io.dataease.base.domain.DatasetTableField;
import java.util.List;
public class DDLProviderImpl extends DDLProvider {
@Override
public String createView(String name, String viewSQL) {
return null;
}
@Override
public String dropTable(String name) {
return null;
}
@Override
public String dropView(String name) {
return null;
}
@Override
public String replaceTable(String name) {
return null;
}
@Override
public String createTableSql(String name, List<DatasetTableField> datasetTableFields) {
return null;
}
}

16
backend/src/main/java/io/dataease/provider/ProviderFactory.java
View File

@ -2,8 +2,6 @@ package io.dataease.provider;
import io.dataease.commons.constants.DatasourceTypes;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.query.DDLProvider;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.api.ApiProvider;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
@ -40,8 +38,6 @@ public class ProviderFactory implements ApplicationContextAware {
case mariadb:
case ds_doris:
return context.getBean("mysqlQuery", QueryProvider.class);
case de_doris:
return context.getBean("dorisQuery", QueryProvider.class);
case sqlServer:
return context.getBean("sqlserverQuery", QueryProvider.class);
case pg:
@ -62,6 +58,10 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("db2Query", QueryProvider.class);
case api:
return context.getBean("apiQuery", ApiProvider.class);
case engine_doris:
return context.getBean("dorisEngineQuery", QueryProvider.class);
case engine_mysql:
return context.getBean("mysqlEngineQuery", QueryProvider.class);
default:
return context.getBean("mysqlQuery", QueryProvider.class);
}
@ -70,10 +70,12 @@ public class ProviderFactory implements ApplicationContextAware {
public static DDLProvider getDDLProvider(String type) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case de_doris:
return context.getBean("dorisDDL", DDLProvider.class);
case engine_doris:
return context.getBean("dorisEngineDDL", DDLProvider.class);
case engine_mysql:
return context.getBean("mysqlEngineDDL", DDLProvider.class);
default:
return context.getBean("dorisDDL", DDLProvider.class);
return context.getBean("dorisEngineDDL", DDLProvider.class);
}
}

2
backend/src/main/java/io/dataease/provider/query/QueryProvider.java → backend/src/main/java/io/dataease/provider/QueryProvider.java
View File

@ -1,4 +1,4 @@
package io.dataease.provider.query;
package io.dataease.provider;
import com.google.gson.Gson;
import io.dataease.base.domain.ChartViewWithBLOBs;

2
backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java → backend/src/main/java/io/dataease/provider/QueryProviderImpl.java
View File

@ -1,4 +1,4 @@
package io.dataease.provider.query;
package io.dataease.provider;
import io.dataease.base.domain.ChartViewWithBLOBs;
import io.dataease.base.domain.DatasetTableField;

2
backend/src/main/java/io/dataease/provider/query/SQLConstants.java → backend/src/main/java/io/dataease/provider/SQLConstants.java
View File

@ -1,4 +1,4 @@
package io.dataease.provider.query;
package io.dataease.provider;
import java.util.ArrayList;
import java.util.List;

12
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@ -11,7 +11,7 @@ import io.dataease.dto.datasource.*;
import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.QueryProvider;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@ -226,7 +226,7 @@ public class JdbcProvider extends DatasourceProvider {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType());
switch (datasourceType) {
case mysql:
case de_doris:
case engine_doris:
case ds_doris:
case mariadb:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
@ -425,7 +425,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
case de_doris:
case engine_doris:
case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
username = mysqlConfiguration.getUsername();
@ -529,7 +529,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
case de_doris:
case engine_doris:
case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
dataSource.setUrl(mysqlConfiguration.getJdbc());
@ -607,7 +607,7 @@ public class JdbcProvider extends DatasourceProvider {
case mariadb:
JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class);
return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase());
case de_doris:
case engine_doris:
case ds_doris:
case hive:
return "show tables";
@ -656,7 +656,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
case de_doris:
case engine_doris:
case ds_doris:
case ck:
return null;

10
backend/src/main/java/io/dataease/provider/query/doris/DorisConstants.java → backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java
View File

@ -1,17 +1,17 @@
package io.dataease.provider.query.doris;
package io.dataease.provider.engine.doris;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.de_doris;
import static io.dataease.commons.constants.DatasourceTypes.engine_doris;
/**
* @Author gin
* @Date 2021/7/8 7:22 下午
*/
public class DorisConstants extends SQLConstants {
public static final String KEYWORD_TABLE = de_doris.getKeywordPrefix() + "%s" + de_doris.getKeywordSuffix();
public static final String KEYWORD_TABLE = engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + de_doris.getKeywordPrefix() + "%s" + de_doris.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix();
public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)";

85
backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java Normal file
View File

@ -0,0 +1,85 @@
package io.dataease.provider.engine.doris;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.commons.utils.TableUtils;
import io.dataease.provider.DDLProviderImpl;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* @Author gin
* @Date 2021/5/17 4:27 下午
*/
@Service("dorisEngineDDL")
public class DorisDDLProvider extends DDLProviderImpl {
private static final String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
"Column_Fields" +
"UNIQUE KEY(dataease_uuid)\n" +
"DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
"PROPERTIES(\"replication_num\" = \"1\");";
@Override
public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
}
@Override
public String dropTable(String name) {
return "DROP TABLE IF EXISTS " + name;
}
@Override
public String dropView(String name) {
return "DROP VIEW IF EXISTS " + name;
}
@Override
public String replaceTable(String name){
return "ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false')"
.replace("DORIS_TABLE", name).replace("DORIS_TMP_TABLE", TableUtils.tmpName(name));
}
@Override
public String createTableSql(String tableName, List<DatasetTableField> datasetTableFields) {
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql);
}
private String createDorisTableColumnSql(final List<DatasetTableField> datasetTableFields) {
StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `");
for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields.append(datasetTableField.getDataeaseName()).append("` ");
Integer size = datasetTableField.getSize() * 3;
if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
size = 65533;
}
switch (datasetTableField.getDeExtractType()) {
case 0:
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
case 1:
size = size < 50? 50 : size;
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
case 2:
Column_Fields.append("bigint").append(",`");
break;
case 3:
Column_Fields.append("DOUBLE").append(",`");
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(size))).append(",`");
break;
default:
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
}
}
Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
return Column_Fields.toString();
}
}

10
backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java → backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java
View File

@ -1,4 +1,4 @@
package io.dataease.provider.query.doris;
package io.dataease.provider.engine.doris;
import io.dataease.base.domain.ChartViewWithBLOBs;
import io.dataease.base.domain.DatasetTableField;
@ -10,8 +10,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -28,13 +28,13 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin
* @Date 2021/5/17 4:11 下午
*/
@Service("dorisQuery")
@Service("dorisEngineQuery")
public class DorisQueryProvider extends QueryProvider {
@Resource
private DatasetTableFieldMapper datasetTableFieldMapper;

49
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java Normal file
View File

@ -0,0 +1,49 @@
package io.dataease.provider.engine.mysql;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.engine_mysql;
/**
* @Author gin
* @Date 2021/7/8 7:22 下午
*/
public class MysqlConstants extends SQLConstants {
public static final String KEYWORD_TABLE = engine_mysql.getKeywordPrefix() + "%s" + engine_mysql.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + engine_mysql.getKeywordPrefix() + "%s" + engine_mysql.getKeywordSuffix();
public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)";
public static final String DATE_FORMAT = "DATE_FORMAT(%s,'%s')";
public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')";
public static final String STR_TO_DATE = "STR_TO_DATE(%s,'%s')";
public static final String CAST = "CAST(%s AS %s)";
public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S";
public static final String DEFAULT_INT_FORMAT = "BIGINT";
public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)";
public static final String WHERE_VALUE_NULL = "(NULL,'')";
public static final String WHERE_VALUE_VALUE = "'%s'";
public static final String WHERE_NUMBER_VALUE = "%s";
public static final String AGG_COUNT = "COUNT(*)";
public static final String AGG_FIELD = "%s(%s)";
public static final String WHERE_BETWEEN = "'%s' AND '%s'";
public static final String BRACKETS = "(%s)";
public static final String ROUND = "ROUND(%s,%s)";
public static final String VARCHAR = "VARCHAR";
}

35
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java Normal file
View File

@ -0,0 +1,35 @@
package io.dataease.provider.engine.mysql;
import io.dataease.commons.utils.TableUtils;
import io.dataease.provider.DDLProviderImpl;
import org.springframework.stereotype.Service;
/**
* @Author gin
* @Date 2021/5/17 4:27 下午
*/
@Service("mysqlEngineDDL")
public class MysqlDDLProvider extends DDLProviderImpl {
@Override
public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
}
@Override
public String dropTable(String name) {
return "DROP TABLE IF EXISTS " + name;
}
@Override
public String dropView(String name) {
return "DROP VIEW IF EXISTS " + name;
}
@Override
public String replaceTable(String name){
String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE; "
.replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name));
String dropTableSql = "DROP TABLE IF EXISTS " + TableUtils.tmpName(name);
return replaceTableSql + ";" + dropTableSql;
}
}

1107
backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java Normal file
View File

File diff suppressed because it is too large Load Diff

2
backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.api;
import io.dataease.provider.query.QueryProviderImpl;
import io.dataease.provider.QueryProviderImpl;
import org.springframework.stereotype.Service;
@Service("apiQuery")

2
backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.ck;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.ck;

6
backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java
View File

@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.db2;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.db2;

6
backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.Db2Configuration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -31,7 +31,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("db2Query")
public class Db2QueryProvider extends QueryProvider {

26
backend/src/main/java/io/dataease/provider/query/doris/DorisDDLProvider.java
View File

@ -1,26 +0,0 @@
package io.dataease.provider.query.doris;
import io.dataease.provider.query.DDLProvider;
import org.springframework.stereotype.Service;
/**
* @Author gin
* @Date 2021/5/17 4:27 下午
*/
@Service("dorisDDL")
public class DorisDDLProvider extends DDLProvider {
@Override
public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
}
@Override
public String dropTable(String name) {
return "DROP TABLE IF EXISTS " + name;
}
@Override
public String dropView(String name) {
return "DROP VIEW IF EXISTS " + name;
}
}

6
backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java
View File

@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("esQuery")
public class EsQueryProvider extends QueryProvider {

2
backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.es;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.es;

2
backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.hive;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mysql;

6
backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java
View File

@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.mongodb;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mongo;

6
backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java
View File

@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.mysql;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mysql;

6
backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java
View File

@ -10,8 +10,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -28,7 +28,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.oracle;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.oracle;

6
backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.datasource.OracleConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -31,7 +31,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin

2
backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.pg;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.pg;

6
backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import io.dataease.provider.query.sqlserver.SqlServerSQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
@ -32,7 +32,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("pgQuery")

2
backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.redshift;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.pg;

6
backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import io.dataease.provider.query.pg.PgConstants;
import io.dataease.provider.query.sqlserver.SqlServerSQLConstants;
import org.apache.commons.collections4.CollectionUtils;
@ -33,7 +33,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**

2
backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java
View File

@ -1,6 +1,6 @@
package io.dataease.provider.query.sqlserver;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.sqlServer;

6
backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java
View File

@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.SQLConstants;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -31,7 +31,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("sqlserverQuery")
public class SqlserverQueryProvider extends QueryProvider {

9
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
View File

@ -2,7 +2,6 @@ package io.dataease.service.chart;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import io.dataease.auth.api.dto.CurrentUserDto;
import io.dataease.auth.entity.SysUserEntity;
import io.dataease.auth.service.AuthUserService;
import io.dataease.base.domain.*;
@ -14,7 +13,6 @@ import io.dataease.commons.constants.CommonConstants;
import io.dataease.commons.constants.JdbcConstants;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.commons.utils.LogUtil;
import io.dataease.controller.request.chart.*;
import io.dataease.controller.request.datasource.DatasourceRequest;
@ -28,12 +26,13 @@ import io.dataease.i18n.Translator;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.QueryProvider;
import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService;
import io.dataease.service.dataset.DataSetTableUnionService;
import io.dataease.service.dataset.PermissionService;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.service.engine.EngineService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -71,6 +70,8 @@ public class ChartViewService {
private PermissionService permissionService;
@Resource
private AuthUserService authUserService;
@Resource
private EngineService engineService;
//默认使用非公平
private ReentrantLock lock = new ReentrantLock();
@ -471,7 +472,7 @@ public class ChartViewService {
data = datasourceProvider.getData(datasourceRequest);
} else if (table.getMode() == 1) {// 抽取
// 连接doris，构建doris数据源查询
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
datasourceRequest.setDatasource(ds);
String tableName = "ds_" + table.getId().replaceAll("-", "_");

4
backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java
View File

@ -4,7 +4,7 @@ import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableFieldExample;
import io.dataease.base.mapper.DatasetTableFieldMapper;
import io.dataease.commons.exception.DEException;
import io.dataease.commons.utils.DorisTableUtils;
import io.dataease.commons.utils.TableUtils;
import io.dataease.i18n.Translator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
@ -36,7 +36,7 @@ public class DataSetTableFieldsService {
datasetTableField.setId(UUID.randomUUID().toString());
// 若dataeasename为空，则用MD5(id)作为dataeasename
if (StringUtils.isEmpty(datasetTableField.getDataeaseName())) {
datasetTableField.setDataeaseName(DorisTableUtils.columnName(datasetTableField.getId()));
datasetTableField.setDataeaseName(TableUtils.columnName(datasetTableField.getId()));
}
if (ObjectUtils.isEmpty(datasetTableField.getLastSyncTime())) {
datasetTableField.setLastSyncTime(System.currentTimeMillis());

117
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -3,7 +3,6 @@ package io.dataease.service.dataset;
import com.google.gson.Gson;
import io.dataease.auth.annotation.DeCleaner;
import io.dataease.auth.api.dto.CurrentUserDto;
import io.dataease.auth.entity.SysUserEntity;
import io.dataease.base.domain.*;
import io.dataease.base.mapper.*;
import io.dataease.base.mapper.ext.ExtDataSetGroupMapper;
@ -29,8 +28,9 @@ import io.dataease.plugins.loader.ClassloaderResponsity;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.provider.query.DDLProvider;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.DDLProvider;
import io.dataease.provider.QueryProvider;
import io.dataease.service.engine.EngineService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -53,7 +53,6 @@ import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.*;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.text.MessageFormat;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
@ -97,6 +96,8 @@ public class DataSetTableService {
private DatasetTableFieldMapper datasetTableFieldMapper;
@Resource
private PermissionService permissionService;
@Resource
private EngineService engineService;
private static final String lastUpdateTime = "${__last_update_time__}";
private static final String currentUpdateTime = "${__current_update_time__}";
@ -296,8 +297,8 @@ public class DataSetTableService {
}
private void deleteDorisTable(String datasetId, DatasetTable table) throws Exception {
String dorisTableName = DorisTableUtils.dorisName(datasetId);
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
String dorisTableName = TableUtils.tableName(datasetId);
Datasource dorisDatasource = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
@ -306,12 +307,12 @@ public class DataSetTableService {
|| StringUtils.equalsIgnoreCase("union", table.getType())) {
datasourceRequest.setQuery(ddlProvider.dropView(dorisTableName));
jdbcProvider.exec(datasourceRequest);
datasourceRequest.setQuery(ddlProvider.dropView(DorisTableUtils.dorisTmpName(dorisTableName)));
datasourceRequest.setQuery(ddlProvider.dropView(TableUtils.tmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
} else {
datasourceRequest.setQuery(ddlProvider.dropTable(dorisTableName));
jdbcProvider.exec(datasourceRequest);
datasourceRequest.setQuery(ddlProvider.dropTable(DorisTableUtils.dorisTmpName(dorisTableName)));
datasourceRequest.setQuery(ddlProvider.dropTable(TableUtils.tmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
}
}
@ -551,11 +552,11 @@ public class DataSetTableService {
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -620,11 +621,11 @@ public class DataSetTableService {
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -649,11 +650,11 @@ public class DataSetTableService {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -719,11 +720,11 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
}
} else {
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -789,11 +790,11 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
}
} else {
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@ -889,7 +890,7 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
sqlMap = getUnionSQLDatasource(dataTableInfoDTO, ds);
} else {
ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
ds = engineService.getDeEngine();
datasourceRequest.setDatasource(ds);
sqlMap = getUnionSQLDoris(dataTableInfoDTO);
}
@ -923,10 +924,10 @@ public class DataSetTableService {
for (DatasetTableField datasetTableField : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
DorisTableUtils.dorisFieldName(
TableUtils.fieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableField.getFieldName(),
DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_"
TableUtils.fieldNameShort(datasetTableField.getTableId() + "_"
+ datasetTableField.getOriginName()))) {
tableField.setRemarks(datasetTableField.getName());
break;
@ -955,7 +956,7 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
sql = getCustomSQLDatasource(dataTableInfoDTO, list, ds);
} else {
ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
ds = engineService.getDeEngine();
datasourceRequest.setDatasource(ds);
sql = getCustomSQLDoris(dataTableInfoDTO, list);
}
@ -987,9 +988,9 @@ public class DataSetTableService {
for (DatasetTableField datasetTableField : checkedFieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
DorisTableUtils.dorisFieldName(
TableUtils.fieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldName(
|| StringUtils.equalsIgnoreCase(tableField.getFieldName(), TableUtils.fieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) {
tableField.setRemarks(datasetTableField.getName());
break;
@ -1009,7 +1010,7 @@ public class DataSetTableService {
private String getCustomSQLDoris(DataTableInfoDTO dataTableInfoDTO, List<DataSetTableUnionDTO> list) {
Map<String, String[]> customInfo = new TreeMap<>();
dataTableInfoDTO.getList().forEach(ele -> {
String table = DorisTableUtils.dorisName(ele.getTableId());
String table = TableUtils.tableName(ele.getTableId());
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(ele.getTableId());
if (ObjectUtils.isEmpty(datasetTable)) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
@ -1020,7 +1021,7 @@ public class DataSetTableService {
}
String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
+ DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName()))
+ TableUtils.fieldName(ele.getTableId() + "_" + f.getDataeaseName()))
.toArray(String[]::new);
customInfo.put(table, array);
});
@ -1046,12 +1047,12 @@ public class DataSetTableService {
}
join.append(convertUnionTypeToSQL(dto.getSourceUnionRelation()))
.append(DorisTableUtils.dorisName(dto.getTargetTableId()))
.append(TableUtils.tableName(dto.getTargetTableId()))
.append(" ON ")
.append(DorisTableUtils.dorisName(dto.getSourceTableId())).append(".")
.append(TableUtils.tableName(dto.getSourceTableId())).append(".")
.append(sourceField.getDataeaseName())
.append(" = ")
.append(DorisTableUtils.dorisName(dto.getTargetTableId())).append(".")
.append(TableUtils.tableName(dto.getTargetTableId())).append(".")
.append(targetField.getDataeaseName());
}
}
@ -1059,16 +1060,16 @@ public class DataSetTableService {
if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
}
return MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(first.getTableId()))
return MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.tableName(first.getTableId()))
+ join.toString();
} else {
if (StringUtils
.isEmpty(StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","))) {
.isEmpty(StringUtils.join(customInfo.get(TableUtils.tableName(first.getTableId())), ","))) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
}
return MessageFormat.format("SELECT {0} FROM {1}",
StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","),
DorisTableUtils.dorisName(first.getTableId()));
StringUtils.join(customInfo.get(TableUtils.tableName(first.getTableId())), ","),
TableUtils.tableName(first.getTableId()));
}
}
@ -1089,7 +1090,7 @@ public class DataSetTableService {
}
String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
+ DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getOriginName()))
+ TableUtils.fieldName(ele.getTableId() + "_" + f.getOriginName()))
.toArray(String[]::new);
customInfo.put(table, array);
}
@ -1175,7 +1176,7 @@ public class DataSetTableService {
for (UnionDTO unionDTO : union) {
// doris 使用数据集id做表名，拼sql将用到该名称
String tableId = unionDTO.getCurrentDs().getId();
String table = DorisTableUtils.dorisName(tableId);
String table = TableUtils.tableName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(
@ -1185,7 +1186,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
+ DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName()))
+ TableUtils.fieldName(tableId + "_" + f.getDataeaseName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@ -1217,7 +1218,7 @@ public class DataSetTableService {
DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId());
DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId());
join.append(" ").append(joinType).append(" ").append(DorisTableUtils.dorisName(currentTable.getId()))
join.append(" ").append(joinType).append(" ").append(TableUtils.tableName(currentTable.getId()))
.append(" ON ");
for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) {
UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i);
@ -1227,10 +1228,10 @@ public class DataSetTableService {
DatasetTableField currentField = dataSetTableFieldsService
.get(unionItemDTO.getCurrentField().getId());
join.append(DorisTableUtils.dorisName(parentTable.getId())).append(".")
join.append(TableUtils.tableName(parentTable.getId())).append(".")
.append(parentField.getDataeaseName())
.append(" = ")
.append(DorisTableUtils.dorisName(currentTable.getId())).append(".")
.append(TableUtils.tableName(currentTable.getId())).append(".")
.append(currentField.getDataeaseName());
if (i < unionParamDTO.getUnionFields().size() - 1) {
join.append(" AND ");
@ -1241,15 +1242,15 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_union_ds_no_checked"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f,
DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())) + join.toString();
TableUtils.tableName(union.get(0).getCurrentDs().getId())) + join.toString();
} else {
String f = StringUtils.join(checkedInfo.get(DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())),
String f = StringUtils.join(checkedInfo.get(TableUtils.tableName(union.get(0).getCurrentDs().getId())),
",");
if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_union_ds_no_checked"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f,
DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId()));
TableUtils.tableName(union.get(0).getCurrentDs().getId()));
}
Map<String, Object> map = new HashMap<>();
map.put("sql", sql);
@ -1264,7 +1265,7 @@ public class DataSetTableService {
for (int i = 0; i < childrenDs.size(); i++) {
UnionDTO unionDTO = childrenDs.get(i);
String tableId = unionDTO.getCurrentDs().getId();
String table = DorisTableUtils.dorisName(tableId);
String table = TableUtils.tableName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(
@ -1274,7 +1275,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
+ DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName()))
+ TableUtils.fieldName(tableId + "_" + f.getDataeaseName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@ -1313,7 +1314,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
+ DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName()))
+ TableUtils.fieldNameShort(tableId + "_" + f.getOriginName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@ -1414,7 +1415,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
+ DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName()))
+ TableUtils.fieldNameShort(tableId + "_" + f.getOriginName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@ -1435,7 +1436,7 @@ public class DataSetTableService {
datasetTableField.setTableId(datasetTableId);
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setName(filed.getRemarks());
datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName()));
datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName()));
datasetTableField.setType(filed.getFieldType());
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(filed.getFieldType()));
@ -1485,7 +1486,7 @@ public class DataSetTableService {
List<DatasetTableField> listByIds = dataSetTableFieldsService
.getListByIdsEach(ele.getCheckedFields());
listByIds.forEach(f -> f.setDataeaseName(
DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName())));
TableUtils.fieldName(ele.getTableId() + "_" + f.getDataeaseName())));
fieldList.addAll(listByIds);
});
for (int i = 0; i < fieldList.size(); i++) {
@ -1498,7 +1499,7 @@ public class DataSetTableService {
dataSetTableFieldsService.batchEdit(fieldList);
// custom 创建doris视图
if (datasetTable.getMode() == 1) {
createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO,
createDorisView(TableUtils.tableName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO,
dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId())));
}
return;
@ -1525,7 +1526,7 @@ public class DataSetTableService {
for (DatasetTableField field : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getOriginName()),
TableUtils.fieldName(field.getTableId() + "_" + field.getOriginName()),
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
@ -1535,7 +1536,7 @@ public class DataSetTableService {
}
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) {
if (datasetTable.getMode() == 1) {
ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
ds = engineService.getDeEngine();
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
@ -1548,14 +1549,14 @@ public class DataSetTableService {
List<UnionParamDTO> join = (List<UnionParamDTO>) sqlMap.get("join");
// custom 创建doris视图
createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), sql);
createDorisView(TableUtils.tableName(datasetTable.getId()), sql);
datasourceRequest.setQuery(sql);
fields = datasourceProvider.fetchResultField(datasourceRequest);
for (DatasetTableField field : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()),
TableUtils.fieldName(field.getTableId() + "_" + field.getDataeaseName()),
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
@ -1579,7 +1580,7 @@ public class DataSetTableService {
for (DatasetTableField field : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()),
TableUtils.fieldNameShort(field.getTableId() + "_" + field.getOriginName()),
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
@ -1621,7 +1622,7 @@ public class DataSetTableService {
if (datasetTable.getMode() == 1 && StringUtils.equalsIgnoreCase("union", datasetTable.getType())) {
datasetTableField.setDataeaseName(filed.getFieldName());
} else {
datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName()));
datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName()));
}
datasetTableField.setType(filed.getFieldType());
if (ObjectUtils.isEmpty(ds)) {
@ -1649,7 +1650,7 @@ public class DataSetTableService {
}
private void createDorisView(String dorisTableName, String customSql) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource dorisDatasource = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
@ -2149,12 +2150,12 @@ public class DataSetTableService {
}
public Boolean checkDorisTableIsExists(String id) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource dorisDatasource = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType());
datasourceRequest.setQuery(qp.searchTable(DorisTableUtils.dorisName(id)));
datasourceRequest.setQuery(qp.searchTable(TableUtils.tableName(id)));
List<String[]> data = jdbcProvider.getData(datasourceRequest);
return CollectionUtils.isNotEmpty(data);
}

249
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -12,6 +12,7 @@ import io.dataease.commons.model.AuthURD;
import io.dataease.commons.utils.*;
import io.dataease.commons.constants.DatasourceTypes;
import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.provider.DDLProvider;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.provider.ProviderFactory;
@ -22,7 +23,8 @@ import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.exception.DataEaseException;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.QueryProvider;
import io.dataease.service.engine.EngineService;
import io.dataease.service.message.DeMsgutil;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
@ -94,6 +96,8 @@ public class ExtractDataService {
private DatasourceService datasourceService;
@Resource
private ExtChartViewMapper extChartViewMapper;
@Resource
private EngineService engineService;
private static final String lastUpdateTime = "${__last_update_time__}";
private static final String currentUpdateTime = "${__current_update_time__}";
@ -111,17 +115,11 @@ public class ExtractDataService {
private String user;
@Value("${carte.passwd:cluster}")
private String passwd;
private static final String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
"Column_Fields" +
"UNIQUE KEY(dataease_uuid)\n" +
"DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
"PROPERTIES(\"replication_num\" = \"1\");";
private static final String dropTableSql = "DROP TABLE IF EXISTS TABLE_NAME;";
private static final String shellScript = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" +
"if [ $? == 0 ] ; then\n" +
"if [ $? -eq 0 ] ; then\n" +
" failstatus=$(echo $result | grep '\"Status\": \"Fail\"')\n" +
" if [[ \"$failstatus\" != \"\" ]]; then\n" +
" if [ \"x${failstatus}\" != \"x\" ];then" +
" echo $result\n" +
" exit 1\n" +
" fi\n" +
@ -175,18 +173,18 @@ public class ExtractDataService {
}
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
switch (updateType) {
case all_scope: // 全量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql);
createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql);
createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields);
createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields);
generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
Long execTime = System.currentTimeMillis();
extractData(datasetTable, "all_scope");
replaceTable(DorisTableUtils.dorisName(datasetTableId));
replaceTable(TableUtils.tableName(datasetTableId));
saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
if(ops.equalsIgnoreCase("替换")){
@ -222,7 +220,7 @@ public class ExtractDataService {
} catch (Exception e) {
saveErrorLog(datasetTableId, null, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)));
} finally {
deleteFile("all_scope", datasetTableId);
deleteExcelFile(datasetTable, datasetTableIds);
@ -257,21 +255,6 @@ public class ExtractDataService {
}
}
private List<DatasetTableField> getDatasetTableFields(String datasetTableId){
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build());
datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
datasetTableFields.sort((o1, o2) -> {
if (o1.getColumnIndex() == null) {
return -1;
}
if (o2.getColumnIndex() == null) {
return 1;
}
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
return datasetTableFields;
}
public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) {
DatasetTable datasetTable = getDatasetTable(datasetTableId);
if (datasetTable == null) {
@ -306,19 +289,17 @@ public class ExtractDataService {
datasource.setType(datasetTable.getType());
}
List<DatasetTableField> datasetTableFields = getDatasetTableFields(datasetTable.getId());
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
boolean msg = false;
JobStatus lastExecStatus = JobStatus.Completed;
Long execTime = null;
switch (updateType) {
case all_scope: // 全量更新
try {
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql);
createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql);
createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields);
createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields);
execTime = System.currentTimeMillis();
extractData(datasetTable, datasource, datasetTableFields, "all_scope", null);
replaceTable(DorisTableUtils.dorisName(datasetTableId));
replaceTable(TableUtils.tableName(datasetTableId));
saveSuccessLog(datasetTableTaskLog);
msg = true;
lastExecStatus = JobStatus.Completed;
@ -334,7 +315,7 @@ public class ExtractDataService {
System.out.println(ignore.getMessage());
}
try { updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
try { dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
try { dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId))); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
}
break;
@ -385,6 +366,21 @@ public class ExtractDataService {
}
private List<DatasetTableField> getDatasetTableFields(String datasetTableId){
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build());
datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
datasetTableFields.sort((o1, o2) -> {
if (o1.getColumnIndex() == null) {
return -1;
}
if (o2.getColumnIndex() == null) {
return 1;
}
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
return datasetTableFields;
}
private void extractData(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType, String selectSQL) throws Exception{
if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){
extractDataByDE(datasetTable, datasource, datasetTableFields, extractType);
@ -393,6 +389,30 @@ public class ExtractDataService {
extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, selectSQL);
}
private void extractDataByDEForSimpleMode(List<String[]> dataList)throws Exception{
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(engine);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
int i = 0;
for (String[] strings : dataList) {
String content = "";
for (int i=0;i< strings.length;i++){
content = content + strings[i] + separator;
}
datasourceRequest.setQuery(ddlProvider.createTableSql(tableName, datasetTableFields));
if(i==100 || ){
jdbcProvider.exec(datasourceRequest);
}
}
}
private void extractDataByDE(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType)throws Exception{
List<ApiDefinition> lists = JSONObject.parseArray(datasource.getConfiguration(), ApiDefinition.class);
lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable())).collect(Collectors.toList());
@ -408,20 +428,21 @@ public class ExtractDataService {
datasourceRequest.setTable(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable());
Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
List<String[]> dataList = result.get("dataList");
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
Datasource engine = engineService.getDeEngine();
DorisConfiguration dorisConfiguration = new Gson().fromJson(engine.getConfiguration(), DorisConfiguration.class);
String columns = datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")) + ",dataease_uuid";
String dataFile = null;
String script = null;
switch (extractType) {
case "all_scope":
dataFile = root_path + DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())));
dataFile = root_path + TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())));
break;
default:
dataFile = root_path + DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()));
dataFile = root_path + TableUtils.addName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()));
break;
}
@ -542,70 +563,44 @@ public class ExtractDataService {
}
private String createDorisTableColumnSql(final List<DatasetTableField> datasetTableFields) {
StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `");
for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields.append(datasetTableField.getDataeaseName()).append("` ");
Integer size = datasetTableField.getSize() * 3;
if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
size = 65533;
}
switch (datasetTableField.getDeExtractType()) {
case 0:
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
case 1:
size = size < 50? 50 : size;
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
case 2:
Column_Fields.append("bigint").append(",`");
break;
case 3:
Column_Fields.append("DOUBLE").append(",`");
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(size))).append(",`");
break;
default:
Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
break;
}
}
Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
return Column_Fields.toString();
}
private void createDorisTable(String dorisTableName, String dorisTableColumnSql) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
private void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTableColumnSql));
datasourceRequest.setDatasource(engine);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
datasourceRequest.setQuery(ddlProvider.createTableSql(tableName, datasetTableFields));
jdbcProvider.exec(datasourceRequest);
}
private void dropDorisTable(String dorisTableName) {
private void dropDorisTable(String tableName) {
try {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(dropTableSql.replace("TABLE_NAME", dorisTableName));
datasourceRequest.setDatasource(engine);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
datasourceRequest.setQuery(ddlProvider.dropTable(tableName));
jdbcProvider.exec(datasourceRequest);
} catch (Exception ignore) {
}
}
private void replaceTable(String dorisTableName) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery("ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false');".replace("DORIS_TABLE", dorisTableName).replace("DORIS_TMP_TABLE", DorisTableUtils.dorisTmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
datasourceRequest.setDatasource(engine);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
String[] replaceTableSql = ddlProvider.replaceTable(dorisTableName).split(";");
for(int i=0;i<replaceTableSql.length;i++){
if(StringUtils.isNotEmpty(replaceTableSql[i])){
datasourceRequest.setQuery(replaceTableSql[i]);
jdbcProvider.exec(datasourceRequest);
}
}
}
private DatasetTable getDatasetTable(String datasetTableId) {
@ -661,26 +656,36 @@ public class ExtractDataService {
return datasetTableTaskLog;
}
private void extractDataForSimpleMode(DatasetTable datasetTable, String extractType)throws Exception{
//API
//EXCEL
}
private void extractData(DatasetTable datasetTable, String extractType) throws Exception {
if(StringUtils.isNotEmpty(datasetTable.getDataSourceId())){
datasourceService.validate(datasetTable.getDataSourceId());
}
if(engineService.isSimpleMode()){
extractDataForSimpleMode(datasetTable, extractType);
}
KettleFileRepository repository = CommonBeanFactory.getBean(KettleFileRepository.class);
RepositoryDirectoryInterface repositoryDirectoryInterface = repository.loadRepositoryDirectoryTree();
TransMeta transMeta = null;
JobMeta jobMeta = null;
switch (extractType) {
case "all_scope":
jobMeta = repository.loadJob("job_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
jobMeta = repository.loadJob("job_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
break;
case "incremental_add":
jobMeta = repository.loadJob("job_add_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_add_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
jobMeta = repository.loadJob("job_add_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_add_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
break;
case "incremental_delete":
jobMeta = repository.loadJob("job_delete_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_delete_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
jobMeta = repository.loadJob("job_delete_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
transMeta = repository.loadTransformation("trans_delete_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
break;
default:
break;
@ -738,27 +743,30 @@ public class ExtractDataService {
}
private void generateJobFile(String extractType, DatasetTable datasetTable, String columnFields) throws Exception {
if(engineService.isSimpleMode()){
return;
}
String outFile;
String jobName = null;
String script = null;
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource dorisDatasource = engineService.getDeEngine();
DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
String columns = columnFields + ",dataease_uuid";
switch (extractType) {
case "all_scope":
outFile = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()));
jobName = "job_" + DorisTableUtils.dorisName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())), root_path + outFile + "." + extention);
outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()));
jobName = "job_" + TableUtils.tableName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention);
break;
case "incremental_add":
outFile = DorisTableUtils.dorisAddName(datasetTable.getId());
jobName = "job_add_" + DorisTableUtils.dorisName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + outFile + "." + extention);
outFile = TableUtils.addName(datasetTable.getId());
jobName = "job_add_" + TableUtils.tableName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
break;
case "incremental_delete":
outFile = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId()));
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + outFile + "." + extention);
jobName = "job_delete_" + DorisTableUtils.dorisName(datasetTable.getId());
outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId()));
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
jobName = "job_delete_" + TableUtils.tableName(datasetTable.getId());
break;
default:
break;
@ -816,12 +824,15 @@ public class ExtractDataService {
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> dorisFields = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName).forEach(filed -> {
dorisFields.add(DorisTableUtils.columnName(filed));
dorisFields.add(TableUtils.columnName(filed));
});
return String.join(",", dorisFields);
}
private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String selectSQL) throws Exception {
if(engineService.isSimpleMode()){
return;
}
TransMeta transMeta = new TransMeta();
String outFile = null;
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasource.getType());
@ -902,18 +913,18 @@ public class ExtractDataService {
switch (extractType) {
case "all_scope":
transName = "trans_" + DorisTableUtils.dorisName(datasetTable.getId());
outFile = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()));
transName = "trans_" + TableUtils.tableName(datasetTable.getId());
outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()));
transMeta.setName(transName);
break;
case "incremental_add":
transName = "trans_add_" + DorisTableUtils.dorisName(datasetTable.getId());
outFile = DorisTableUtils.dorisAddName(datasetTable.getId());
transName = "trans_add_" + TableUtils.tableName(datasetTable.getId());
outFile = TableUtils.addName(datasetTable.getId());
transMeta.setName(transName);
break;
case "incremental_delete":
transName = "trans_delete_" + DorisTableUtils.dorisName(datasetTable.getId());
outFile = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId()));
transName = "trans_delete_" + TableUtils.tableName(datasetTable.getId());
outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId()));
transMeta.setName(transName);
break;
default:
@ -1143,19 +1154,19 @@ public class ExtractDataService {
switch (type) {
case "all_scope":
transName = "trans_" + DorisTableUtils.dorisName(dataSetTableId);
jobName = "job_" + DorisTableUtils.dorisName(dataSetTableId);
fileName = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(dataSetTableId));
transName = "trans_" + TableUtils.tableName(dataSetTableId);
jobName = "job_" + TableUtils.tableName(dataSetTableId);
fileName = TableUtils.tmpName(TableUtils.tableName(dataSetTableId));
break;
case "incremental_add":
transName = "trans_add_" + DorisTableUtils.dorisName(dataSetTableId);
jobName = "job_add_" + DorisTableUtils.dorisName(dataSetTableId);
fileName = DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(dataSetTableId));
transName = "trans_add_" + TableUtils.tableName(dataSetTableId);
jobName = "job_add_" + TableUtils.tableName(dataSetTableId);
fileName = TableUtils.addName(TableUtils.tableName(dataSetTableId));
break;
case "incremental_delete":
transName = "trans_delete_" + DorisTableUtils.dorisName(dataSetTableId);
jobName = "job_delete_" + DorisTableUtils.dorisName(dataSetTableId);
fileName = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(dataSetTableId));
transName = "trans_delete_" + TableUtils.tableName(dataSetTableId);
jobName = "job_delete_" + TableUtils.tableName(dataSetTableId);
fileName = TableUtils.deleteName(TableUtils.tableName(dataSetTableId));
break;
default:
break;

8
backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java
View File

@ -5,7 +5,6 @@ import io.dataease.base.domain.DatasetTable;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import io.dataease.commons.constants.ColumnPermissionConstants;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.i18n.Translator;
import io.dataease.provider.datasource.DatasourceProvider;
@ -15,7 +14,8 @@ import io.dataease.service.dataset.*;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.dto.dataset.DataSetTableUnionDTO;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.QueryProvider;
import io.dataease.service.engine.EngineService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -40,6 +40,8 @@ public class DirectFieldService implements DataSetFieldService {
private DataSetTableUnionService dataSetTableUnionService;
@Resource
private PermissionService permissionService;
@Resource
private EngineService engineService;
@Override
public List<Object> fieldValues(String fieldId, Long userId, Boolean userPermissions) throws Exception {
@ -100,7 +102,7 @@ public class DirectFieldService implements DataSetFieldService {
}
} else if (datasetTable.getMode() == 1) {// 抽取
// 连接doris，构建doris数据源查询
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
Datasource ds = engineService.getDeEngine();
datasourceProvider = ProviderFactory.getProvider(ds.getType());
datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);

2
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -94,7 +94,7 @@ public class DatasourceService {
switch (datasourceType) {
case mysql:
case mariadb:
case de_doris:
case engine_doris:
case ds_doris:
datasourceDTO.setConfiguration(JSONObject.toJSONString(new Gson().fromJson(datasourceDTO.getConfiguration(), MysqlConfiguration.class)) );
break;

67
backend/src/main/java/io/dataease/service/engine/EngineService.java Normal file
View File

@ -0,0 +1,67 @@
package io.dataease.service.engine;
import com.alibaba.fastjson.JSONObject;
import io.dataease.base.domain.Datasource;
import io.dataease.base.domain.DeEngine;
import io.dataease.base.domain.DeEngineExample;
import io.dataease.base.mapper.DeEngineMapper;
import io.dataease.commons.utils.BeanUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
@Service
public class EngineService {
@Resource
private Environment env;
@Resource
private DeEngineMapper deEngineMapper;
static private Datasource ds = null;
public Boolean isLocalMode(){
return env.getProperty("engine_mode", "local").equalsIgnoreCase("local");
}
public Boolean isSimpleMode(){
return env.getProperty("engine_mode", "local").equalsIgnoreCase("simple");
}
public Datasource getDeEngine() throws Exception{
if (this.ds != null) {
return this.ds;
}
if(isLocalMode()){
JSONObject jsonObject = new JSONObject();
jsonObject.put("dataSourceType", "jdbc");
jsonObject.put("dataBase", env.getProperty("doris.db", "doris"));
jsonObject.put("username", env.getProperty("doris.user", "root"));
jsonObject.put("password", env.getProperty("doris.password", "dataease"));
jsonObject.put("host", env.getProperty("doris.host", "doris"));
jsonObject.put("port", env.getProperty("doris.port", "9030"));
jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030"));
Datasource datasource = new Datasource();
datasource.setId("doris");
datasource.setName("doris");
datasource.setDesc("doris");
datasource.setType("engine_doris");
datasource.setConfiguration(jsonObject.toJSONString());
this.ds = datasource;
}
if(isSimpleMode()){
List<DeEngine> deEngines = deEngineMapper.selectByExample(new DeEngineExample());
if(CollectionUtils.isEmpty(deEngines)){
throw new Exception("未设置数据引擎");
}
BeanUtils.copyBean(this.ds, deEngines.get(0));
}
//TODO cluster mode
return this.ds;
}
}

11
backend/src/main/resources/generatorConfig.xml
View File

@ -60,13 +60,8 @@
</javaClientGenerator>
<!--要生成的数据库表 -->
<!-- <table tableName="dataease_code_version"/>-->
<table tableName="chart_view"/>
<!-- <table tableName="panel_view"/>-->
<!-- <table tableName="panel_link_jump"/>-->
<!-- <table tableName="panel_link_jump_info"/>-->
<!-- <table tableName="panel_link_jump_target_view_info"/>-->
<!-- <table tableName="panel_view_linkage"/>-->
<!-- <table tableName="panel_view_linkage_field"/>-->
<table tableName="de_engine">
<columnOverride column="configuration" property="configuration" javaType="java.lang.String"/>
</table>
</context>
</generatorConfiguration>

3
frontend/src/lang/zh.js
View File

@ -650,7 +650,8 @@ export default {
port: '端口号不能为空',
account: '账户不能为空',
test_recipients: '测试收件人',
tip: '提示：仅用来作为测试邮件收件人'
tip: '提示：仅用来作为测试邮件收件人',
simple_mode_datasource: '数据设置'
},
chart: {
save_snapshot: '保存缩略图',

239
frontend/src/views/system/SysParam/SimplemodeDatasourceSetting.vue Normal file
View File

@ -0,0 +1,239 @@
<template>
<div>
<!--邮件表单-->
<el-form
ref="formInline"
v-loading="loading"
:model="formInline"
:rules="rules"
class="demo-form-inline"
:disabled="show"
size="small"
>
<el-row>
<el-col>
<el-form-item :label="$t('system_parameter_setting.SMTP_host')" prop="host">
<el-input
v-model="formInline.host"
:placeholder="$t('system_parameter_setting.SMTP_host')"
@input="change()"
/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('system_parameter_setting.SMTP_port')" prop="port">
<el-input
v-model="formInline.port"
:placeholder="$t('system_parameter_setting.SMTP_port')"
@input="change()"
/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('system_parameter_setting.SMTP_account')" prop="account">
<el-input
v-model="formInline.account"
:placeholder="$t('system_parameter_setting.SMTP_account')"
@input="change()"
/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('system_parameter_setting.SMTP_password')" prop="password">
<el-input
ref="input"
v-model="formInline.password"
:placeholder="$t('system_parameter_setting.SMTP_password')"
autocomplete="new-password"
show-password
type="text"
@focus="changeType"
/>
</el-form-item>
</el-col>
</el-row>
<el-row>
<el-col>
<el-form-item :label="$t('system_parameter_setting.test_recipients')">
<el-input
ref="input"
v-model="formInline.recipient"
:placeholder="$t('system_parameter_setting.test_recipients')"
autocomplete="new-password"
show-password
type="text"
/>
<p style="color: #8a8b8d">({{ $t('system_parameter_setting.tip') }})</p>
</el-form-item>
</el-col>
</el-row>
<!---->
<div style="border: 0px;margin-bottom: 20px">
<el-checkbox v-model="formInline.ssl" :label="$t('system_parameter_setting.SSL')" />
</div>
<div style="border: 0px;margin-bottom: 20px">
<el-checkbox v-model="formInline.tls" :label="$t('system_parameter_setting.TLS')" />
</div>
<template v-slot:footer />
</el-form>
<div>
<el-button type="primary" :disabled="disabledConnection" size="small" @click="testConnection('formInline')">
{{ $t('system_parameter_setting.test_connection') }}
</el-button>
<el-button v-if="showEdit" size="small" @click="edit">{{ $t('commons.edit') }}</el-button>
<el-button v-if="showSave" type="success" :disabled="disabledSave" size="small" @click="save('formInline')">
{{ $t('commons.save') }}
</el-button>
<el-button v-if="showCancel" type="info" size="small" @click="cancel">{{ $t('commons.cancel') }}</el-button>
</div>
</div>
</template>
<script>
import { emailInfo, updateInfo, validate } from '@/api/system/email'
export default {
name: 'EmailSetting',
data() {
return {
formInline: {},
input: '',
visible: true,
showEdit: true,
showSave: false,
showCancel: false,
show: true,
disabledConnection: false,
disabledSave: false,
loading: false,
rules: {
host: [
{
required: true,
message: this.$t('system_parameter_setting.host'),
trigger: ['change', 'blur']
}
],
port: [
{
required: true,
message: this.$t('system_parameter_setting.port'),
trigger: ['change', 'blur']
}
],
account: [
{
required: true,
message: this.$t('system_parameter_setting.account'),
trigger: ['change', 'blur']
}]
}
}
},
created() {
this.query()
},
methods: {
changeType() {
this.$refs.input = 'password'
},
query() {
emailInfo().then(response => {
this.formInline = response.data
this.formInline.ssl = this.formInline.ssl === 'true'
this.formInline.tls = this.formInline.tls === 'true'
this.$nextTick(() => {
this.$refs.formInline.clearValidate()
})
})
},
change() {
if (!this.formInline.host || !this.formInline.port || !this.formInline.account) {
this.disabledConnection = true
this.disabledSave = true
} else {
this.disabledConnection = false
this.disabledSave = false
}
},
testConnection(formInline) {
const param = {
'smtp.host': this.formInline.host,
'smtp.port': this.formInline.port,
'smtp.account': this.formInline.account,
'smtp.password': this.formInline.password,
'smtp.ssl': this.formInline.ssl,
'smtp.tls': this.formInline.tls,
'smtp.recipient': this.formInline.recipient
}
this.$refs[formInline].validate((valid) => {
if (valid) {
validate(param).then(response => {
this.$success(this.$t('commons.connection_successful'))
})
} else {
return false
}
})
},
edit() {
this.showEdit = false
this.showSave = true
this.showCancel = true
this.show = false
},
save(formInline) {
this.showEdit = true
this.showCancel = false
this.showSave = false
this.show = true
const param = [
{ paramKey: 'smtp.host', paramValue: this.formInline.host, type: 'text', sort: 1 },
{ paramKey: 'smtp.port', paramValue: this.formInline.port, type: 'text', sort: 2 },
{ paramKey: 'smtp.account', paramValue: this.formInline.account, type: 'text', sort: 3 },
{ paramKey: 'smtp.password', paramValue: this.formInline.password, type: 'password', sort: 4 },
{ paramKey: 'smtp.ssl', paramValue: this.formInline.ssl, type: 'text', sort: 5 },
{ paramKey: 'smtp.tls', paramValue: this.formInline.tls, type: 'text', sort: 6 },
{ paramKey: 'smtp.recipient', paramValue: this.formInline.recipient, type: 'text', sort: 8 }
]
this.$refs[formInline].validate(valid => {
if (valid) {
updateInfo(param).then(response => {
const flag = response.success
if (flag) {
this.$success(this.$t('commons.save_success'))
} else {
this.$message.error(this.$t('commons.save_failed'))
}
})
} else {
// this.result = false
}
})
},
cancel() {
this.showEdit = true
this.showCancel = false
this.showSave = false
this.show = true
this.query()
}
}
}
</script>
<style scoped>
</style>

4
frontend/src/views/system/SysParam/index.vue
View File

@ -6,6 +6,10 @@
<basic-setting />
</el-tab-pane>
<el-tab-pane :lazy="true" :label="$t('system_parameter_setting.simple_mode_datasource')" name="first">
<email-setting />
</el-tab-pane>
<el-tab-pane :lazy="true" :label="$t('system_parameter_setting.mailbox_service_settings')" name="first">
<email-setting />
</el-tab-pane>