Merge pull request #5969 from dataease/pr@dev_add_installer

chore: 合并installer
This commit is contained in:
xuwei-fit2cloud 2023-08-24 17:44:53 +08:00 committed by GitHub
commit 0bca32f358
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1251 additions and 0 deletions

4
installer/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
.idea/
*.iml
target/
.DS_Store

5
installer/README.md Normal file
View File

@ -0,0 +1,5 @@
# DataEase 离线安装包制作工程
本工程用来制作 DataEase 离线安装包。
如需下载 DataEase 安装包请移步https://community.fit2cloud.com/#/products/dataease/downloads

View File

@ -0,0 +1,43 @@
#!/bin/bash
# chkconfig: 2345 10 90
# description: DATAEASE service
function startDATAEASE
{
dectl reload
}
function stopDATAEASE
{
dectl uninstall
}
function restartDATAEASE
{
stopDATAEASE
startDATAEASE
}
function statusDATAEASE
{
dectl status
}
export HOSTNAME=$HOSTNAME
case "$1" in
start)
startDATAEASE
;;
stop)
stopDATAEASE
;;
restart)
restartDATAEASE
;;
status)
statusDATAEASE
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
;;
esac

View File

@ -0,0 +1,3 @@
CREATE DATABASE IF NOT EXISTS dataease;
SET PASSWORD FOR 'root' = PASSWORD('Password123@doris');
SET GLOBAL enable_spilling = true;

View File

@ -0,0 +1 @@
CREATE DATABASE `dataease` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;

317
installer/dataease/dectl Executable file
View File

@ -0,0 +1,317 @@
#!/bin/bash
action=$1
target=$2
args=$@
DE_BASE=/opt
source ${DE_BASE}/dataease/.env
export COMPOSE_HTTP_TIMEOUT=180
compose_files="-f docker-compose.yml"
compose_cmd="docker-compose"
docker-compose version >/dev/null 2>&1
if [ $? -ne 0 ]; then
docker compose version >/dev/null 2>&1
if [ $? -ne 0 ]; then
compose_cmd="docker compose"
fi
fi
if [ "${DE_ENGINE_MODE}" = "local" ]; then
if [[ ! ${DE_EXTERNAL_KETTLE} ]] || [ ${DE_EXTERNAL_KETTLE} = "false" ]; then
compose_files="${compose_files} -f docker-compose-kettle.yml"
fi
if [[ ! ${DE_EXTERNAL_DORIS} ]] || [ ${DE_EXTERNAL_DORIS} = "false" ]; then
compose_files="${compose_files} -f docker-compose-doris.yml"
fi
fi
if [[ ! ${DE_EXTERNAL_MYSQL} ]] || [ ${DE_EXTERNAL_MYSQL} = "false" ]; then
compose_files="${compose_files} -f docker-compose-mysql.yml"
fi
function usage() {
echo "DATAEASE 控制脚本"
echo
echo "Usage: "
echo " ./dectl [COMMAND] [ARGS...]"
echo " ./dectl --help"
echo
echo "Commands: "
echo " status 查看 DATAEASE 服务运行状态"
echo " start 启动 DATAEASE 服务"
echo " stop 停止 DATAEASE 服务"
echo " restart 重启 DATAEASE 服务"
echo " reload 重新加载 DATAEASE 服务"
echo " uninstall 卸载 DATAEASE 服务"
echo " upgrade 升级 DATAEASE 服务"
echo " version 查看 DATAEASE 版本信息"
echo " clear-images 清理 DATAEASE 旧版本的相关镜像"
echo " clear-logs 清理 DATAEASE 历史日志以及 Doris 临时日志"
}
function _healthcheck() {
echo
for b in {1..30}
do
sleep 3
http_code=$(curl -sILw "%{http_code}\n" http://localhost:${DE_PORT} -o /dev/null)
if [[ $http_code == 000 ]];then
echo "DataEase 服务状态 : 正在启动"
elif [[ $http_code == 200 ]];then
echo "DataEase 服务状态 : 正常运行"
break;
else
echo -e "DataEase 服务状态 : \033[31m无法访问\033[0m"
exit 1
fi
done
echo
}
function status() {
echo
echo "DataEase 容器运行状态"
cd ${DE_BASE}/dataease; ${compose_cmd} ${compose_files} ps
echo
docker_root_dir=$(docker info | grep "Docker Root Dir" | awk -F': ' '{print $2}')
echo "Docker 目录及文件大小 : "
du -sh $docker_root_dir
echo
echo "Docker 目录所在磁盘使用情况 : "
df -H $docker_root_dir
echo
echo "日志文件大小 : "
du -Sh ${DE_BASE}/dataease/logs/
echo
echo "DataEase 运行目录及文件大小 : "
du -sh ${DE_BASE}
echo
echo "DataEase 运行目录使用情况 : "
df -H ${DE_BASE}
echo
read available_disk percent_disk<<< $(df -H --output=avail,pcent ${DE_BASE} | tail -1)
available_disk=${available_disk%?}
percent_disk=${percent_disk%?}
if [[ $available_disk < 2.0 ]] && [[ $percent_disk > 85 ]];then
echo -e "\033[31m[警告] DataEase 运行目录所在磁盘剩余空间不足 2G 且磁盘使用率超过 85%Apache Doris 可能无法正常启动!\033[0m"
fi
_healthcheck
}
function start() {
echo
cd ${DE_BASE}/dataease; ${compose_cmd} ${compose_files} start ${target}
_healthcheck
}
function stop() {
echo
cd ${DE_BASE}/dataease; ${compose_cmd} ${compose_files} stop ${target}
}
function restart() {
echo
cd ${DE_BASE}/dataease; ${compose_cmd} ${compose_files} restart ${target}
_healthcheck
}
function reload() {
echo
cd ${DE_BASE}/dataease; ${compose_cmd} ${compose_files} up -d
_healthcheck
}
function uninstall() {
echo
cd ${DE_BASE}/dataease; ${compose_cmd} ${compose_files} down -v ${target}
}
function version() {
echo
de_version=$(grep "^ image:.*/dataease:" ${DE_BASE}/dataease/docker-compose.yml | awk -F'dataease:' '{print $2}')
echo $de_version
}
function _clear_component_docker_images() {
component=$1
compose_file=${DE_BASE}/dataease/docker-compose-${component}.yml
if [ $1 = "dataease" ];then
compose_file=${DE_BASE}/dataease/docker-compose.yml
fi
component_image_defined=$(grep "^ image:.*/$component:" ${compose_file} | uniq)
component_version=$(echo ${component_image_defined} | awk -F"${component}:" '{print $2}' | uniq)
component_image=$(echo ${component_image_defined} | awk -F'image: ' '{print $2}' | awk -F':' '{print $1}')
if test -z $(docker images | grep "dataease" | grep "$component_image" | grep -v " $component_version "); then
echo "$component_image 不存在旧版本镜像"
else
echo "清理${component}镜像"
docker rmi $(docker images | grep "dataease" | grep "$component_image" | grep -v " $component_version " | awk -F' ' '{print $1":"$2}')
echo "清理完毕"
fi
}
function clear_images() {
echo
_clear_component_docker_images 'dataease'
_clear_component_docker_images 'doris'
_clear_component_docker_images 'kettle'
_clear_component_docker_images 'mysql'
if test ! -z "$(docker images -f dangling=true -q)"; then
echo "清理虚悬镜像"
docker rmi $(docker images -f dangling=true -q)
echo "清理完毕"
fi
}
function clear_logs() {
echo "开始清理 DataEase 历史日志"
rm -rf ${DE_BASE}/dataease/logs/dataease/history/*
echo "清理完毕"
echo "开始清理 Apache Doris FE 日志"
rm -rf ${DE_BASE}/dataease/logs/fe/fe.gc.log.*
echo "" > ${DE_BASE}/dataease/logs/fe/fe.out
echo "清理完毕"
echo "开始清理 Apache Doris BE 日志"
echo "" > ${DE_BASE}/dataease/logs/be/be.INFO
echo "" > ${DE_BASE}/dataease/logs/be/be.WARNING
echo "" > ${DE_BASE}/dataease/logs/be/be.out
echo "清理完毕"
}
function upgrade() {
echo
git_urls=('gitee.com' 'github.com')
if [[ -x "$(command -v python)" ]];then
py_cmd='python'
elif [[ -x "$(command -v python3)" ]]; then
py_cmd='python3'
else
git_urls=('github.com')
fi
for git_url in ${git_urls[*]}
do
success="true"
for i in {1..3}
do
echo -ne "检测 ${git_url} ... ${i} "
curl -m 5 -kIs https://${git_url} >/dev/null
if [ $? != 0 ];then
echo "failed"
success="false"
break
else
echo "ok"
fi
done
if [[ ${success} == "true" ]];then
server_url=${git_url}
break
fi
done
if [[ "x${server_url}" == "x" ]];then
echo "没有找到稳定的下载服务器,请稍候重试"
exit 1
fi
if [[ "${server_url}" == "gitee.com" ]];then
owner='fit2cloud-feizhiyun'
repo='DataEase'
gitee_release_content=$(curl -s https://gitee.com/api/v5/repos/${owner}/${repo}/releases/latest)
# export LC_ALL="en_US.utf8"
latest_version=$($py_cmd -c "import json; obj=json.loads('$gitee_release_content', strict=False); print(obj['tag_name']);")
else
owner='dataease'
repo='dataease'
latest_version=$(curl -s https://api.github.com/repos/${owner}/${repo}/releases/latest | grep -e "\"tag_name\"" | sed -r 's/.*: "(.*)",/\1/')
fi
current_version=$(grep "^ image:.*/dataease:" ${DE_BASE}/dataease/docker-compose.yml | awk -F'dataease:' '{print $2}')
if [ "${latest_version}" = "" ];then
echo "未获取到最新版本"
exit 1
elif [ "${latest_version}" = "${current_version}" ];then
echo "最新版本与当前版本一致,退出升级过程"
exit 0
else
echo -e "\e[32m 检测到最新版本为\e[1;33m${latest_version}\e[0;32m 即将执行在线升级...\e[0m"
fi
sleep 2s
echo "使用下载服务器 ${server_url}"
cd /tmp
download_url="https://${server_url}/${owner}/${repo}/releases/download/${latest_version}/dataease-${latest_version}-online.tar.gz"
curl -LOk -m 60 -o dataease-${latest_version}-online.tar.gz $download_url
if [ $? -ne 0 ]; then
echo -e "\e[31m升级失败:连接下载服务器超时!\n可手动下载升级包然后执行\e[1;33m /bin/bash install.sh \e[0;31m离线升级也可以重新执行一次 dectl upgrade 命令。\e[0m"
return 2
fi
if [ ! -f dataease-${latest_version}-online.tar.gz ];then
echo "下载在线安装包失败,请试试重新执行一次安装命令。"
exit 1
fi
tar zxvf dataease-${latest_version}-online.tar.gz
if [ $? != 0 ];then
echo "下载在线安装包失败,请试试重新执行一次安装命令。"
rm -f dataease-${latest_version}-online.tar.gz
exit 1
fi
cd dataease-${latest_version}-online
/bin/bash install.sh
cd ..
rm -rf /tmp/dataease-${latest_version}-online
}
function main() {
case "${action}" in
status)
status
;;
start)
start
;;
stop)
stop
;;
restart)
restart
;;
reload)
reload
;;
uninstall)
uninstall
;;
upgrade)
upgrade
;;
version)
version
;;
clear-images)
clear_images
;;
clear-logs)
clear_logs
;;
help)
usage
;;
--help)
usage
;;
*)
echo "不支持的参数,请使用 help 或 --help 参数获取帮助"
;;
esac
}
main

View File

@ -0,0 +1,50 @@
version: '2.1'
services:
doris-fe:
image: registry.cn-qingdao.aliyuncs.com/dataease/doris:v1.2.4.1-0606
container_name: doris-fe
privileged: true
environment:
- DORIS_ROLE=fe-leader
- FE_SERVERS=fe1:${DE_DORIS_FE_IP}:9010
- FE_ID=1
volumes:
- ${DE_BASE}/dataease/data/fe:/opt/apache-doris/fe/doris-meta
- ${DE_BASE}/dataease/logs/fe:/opt/apache-doris/fe/log
- ${DE_BASE}/dataease/conf/fe.conf:/opt/apache-doris/fe/conf/fe.conf
networks:
dataease-network :
ipv4_address: ${DE_DORIS_FE_IP}
restart: always
healthcheck:
test: [ "CMD-SHELL", "curl -sS 127.0.0.1:8030 || exit 1" ]
interval: 10s
timeout: 5s
retries: 3
doris-be:
image: registry.cn-qingdao.aliyuncs.com/dataease/doris:v1.2.4.1-0606
container_name: doris-be
privileged: true
environment:
- DORIS_ROLE=be
- FE_SERVERS=fe1:${DE_DORIS_FE_IP}:9010
- BE_ADDR=${DE_DORIS_BE_IP}:9050
volumes:
- ${DE_BASE}/dataease/data/be:/opt/apache-doris/be/storage
- ${DE_BASE}/dataease/logs/be:/opt/apache-doris/be/log
- ${DE_BASE}/dataease/conf/be.conf:/opt/apache-doris/be/conf/be.conf
- ${DE_BASE}/dataease/bin/doris:/docker-entrypoint-initdb.d/
depends_on:
doris-fe:
condition: service_healthy
networks:
dataease-network :
ipv4_address: ${DE_DORIS_BE_IP}
restart: always
healthcheck:
test: [ "CMD-SHELL", "curl -sS 127.0.0.1:8040 || exit 1" ]
interval: 10s
timeout: 5s
retries: 3

View File

@ -0,0 +1,14 @@
version: '2.1'
services:
kettle:
image: registry.cn-qingdao.aliyuncs.com/dataease/kettle:v8.3-0318
container_name: kettle
environment:
- PENTAHO_DI_JAVA_OPTIONS=-Xms1024m -Xmx4096m -XX:MaxPermSize=256m -Dfile.encoding=utf-8
volumes:
- ${DE_BASE}/dataease/conf/:/opt/dataease/conf
- ${DE_BASE}/dataease/data/kettle:/opt/dataease/data/kettle
networks:
- dataease-network
restart: always

View File

@ -0,0 +1,21 @@
version: '2.1'
services:
mysql:
image: registry.cn-qingdao.aliyuncs.com/dataease/mysql:5.7.43
container_name: mysql
healthcheck:
test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost", "-u${DE_MYSQL_USER}", "-p${DE_MYSQL_PASSWORD}"]
interval: 5s
timeout: 3s
retries: 10
env_file:
- ${DE_BASE}/dataease/conf/mysql.env
ports:
- ${DE_MYSQL_PORT}:3306
volumes:
- ${DE_BASE}/dataease/conf/my.cnf:/etc/mysql/conf.d/my.cnf
- ${DE_BASE}/dataease/bin/mysql:/docker-entrypoint-initdb.d/
- ${DE_BASE}/dataease/data/mysql:/var/lib/mysql
networks:
- dataease-network

View File

@ -0,0 +1,34 @@
version: '2.1'
services:
dataease:
image: registry.cn-qingdao.aliyuncs.com/dataease/dataease:DE_TAG
container_name: dataease
ports:
- ${DE_PORT}:8081
mem_limit: 4096m
volumes:
- ${DE_BASE}/dataease/conf:/opt/dataease/conf
- ${DE_BASE}/dataease/logs:/opt/dataease/logs
- ${DE_BASE}/dataease/plugins/thirdpart:/opt/dataease/plugins/thirdpart
- ${DE_BASE}/dataease/data/kettle:/opt/dataease/data/kettle
- ${DE_BASE}/dataease/data/static-resource:/opt/dataease/data/static-resource
- ${DE_BASE}/dataease/custom-drivers:/opt/dataease/custom-drivers
- ${DE_BASE}/dataease/data/custom:/opt/dataease/data/custom
- ${DE_BASE}/dataease/data/business:/opt/dataease/data/business
- ${DE_BASE}/dataease/data/ehcache:/opt/dataease/data/ehcache
shm_size: 1gb
depends_on:
mysql:
condition: service_healthy
networks:
- dataease-network
networks:
dataease-network:
driver: bridge
ipam:
driver: default
config:
- subnet: ${DE_DOCKER_SUBNET}
gateway: ${DE_DOCKER_GATEWAY}

View File

@ -0,0 +1,61 @@
# 基础配置
## 安装目录
DE_BASE=/opt
## Service 端口
DE_PORT=80
## 部署及运行模式,可选值有 local、simple、cluster分别对应 本地模式、精简模式、集群模式
DE_ENGINE_MODE=simple
## docker 网段设置
DE_DOCKER_SUBNET=172.19.0.0/16
## docker 网关 IP
DE_DOCKER_GATEWAY=172.19.0.1
## Apache Doris FE IP (外部 Doris 此参数无效)
DE_DORIS_FE_IP=172.19.0.198
## Apache Doris BE IP (外部 Doris 此参数无效)
DE_DORIS_BE_IP=172.19.0.199
## 登录超时时间单位min。如果不设置则默认8小时也就是480
DE_LOGIN_TIMEOUT=480
# 数据库配置
## 是否使用外部数据库
DE_EXTERNAL_MYSQL=false
## 数据库地址
DE_MYSQL_HOST=mysql
## 数据库端口
DE_MYSQL_PORT=3306
## DataEase 数据库库名
DE_MYSQL_DB=dataease
## 数据库用户名
DE_MYSQL_USER=root
## 数据库密码
DE_MYSQL_PASSWORD=Password123@mysql
## 数据库参数
DE_MYSQL_PARAMS="autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false"
# Apache Doris 配置
## 是否使用外部 Apache Doris
DE_EXTERNAL_DORIS=false
## Doris 地址
DE_DORIS_HOST=doris-fe
## Doris 查询连接端口
DE_DORIS_PORT=9030
## Doris http端口
DE_DORIS_HTTPPORT=8030
## Doris 数据库名称
DE_DORIS_DB=dataease
## Doris 用户名
DE_DORIS_USER=root
## Doris 密码
DE_DORIS_PASSWORD=Password123@doris
# Kettle 配置
## 是否使用外部 Kettle - (目前还不支持外部Kettle除非不需运行Kettle否则请不要修改此参数)
DE_EXTERNAL_KETTLE=false
## Kettle 服务器地址
DE_CARTE_HOST=kettle
## Kettle 访问端口
DE_CARTE_PORT=18080
## Kettle 用户名
DE_CARTE_USER=cluster
## Kettle 密码
DE_CARTE_PASSWORD=cluster

375
installer/dataease/install.sh Executable file
View File

@ -0,0 +1,375 @@
#!/bin/bash
CURRENT_DIR=$(
cd "$(dirname "$0")"
pwd
)
function log() {
message="[DATAEASE Log]: $1 "
echo -e "${message}" 2>&1 | tee -a ${CURRENT_DIR}/install.log
}
args=$@
os=$(uname -a)
docker_config_folder="/etc/docker"
compose_files="-f docker-compose.yml"
INSTALL_TYPE='install'
if [ -f /usr/bin/dectl ]; then
# 获取已安装的 DataEase 的运行目录
DE_BASE=$(grep "^DE_BASE=" /usr/bin/dectl | cut -d'=' -f2)
dectl uninstall
INSTALL_TYPE='upgrade'
fi
set -a
if [[ $DE_BASE ]] && [[ -f $DE_BASE/dataease/.env ]]; then
source $DE_BASE/dataease/.env
INSTALL_TYPE='upgrade'
else
source ${CURRENT_DIR}/install.conf
INSTALL_TYPE='install'
fi
set +a
DE_RUN_BASE=$DE_BASE/dataease
conf_folder=${DE_RUN_BASE}/conf
templates_folder=${DE_RUN_BASE}/templates
mysql_container_name="mysql"
if [ -f ${DE_RUN_BASE}/docker-compose-mysql.yml ]; then
mysql_container_name=$(grep "container_name" ${DE_RUN_BASE}/docker-compose-mysql.yml | awk -F': ' '{print $2}')
fi
dataease_conf=${conf_folder}/dataease.properties
function prop {
[ -f "$1" ] | grep -P "^\s*[^#]?${2}=.*$" $1 | cut -d'=' -f2
}
if [ "x${DE_ENGINE_MODE}" = "x" ]; then
export DE_ENGINE_MODE="local"
fi
if [ "x${DE_DOCKER_SUBNET}" = "x" ]; then
export DE_DOCKER_SUBNET=$(grep "^[[:blank:]]*- subnet" ${DE_RUN_BASE}/docker-compose.yml | awk -F': ' '{print $2}')
fi
if [ "x${DE_DOCKER_GATEWAY}" = "x" ]; then
export DE_DOCKER_GATEWAY=$(grep "^[[:blank:]]*gateway" ${DE_RUN_BASE}/docker-compose.yml | awk -F': ' '{print $2}')
fi
if [ "x${DE_DORIS_FE_IP}" = "x" ]; then
DE_DORIS_FE_IP=$(grep "^[[:blank:]]*ipv4_address" ${DE_RUN_BASE}/docker-compose-doris.yml | awk -F': ' '{print $2}' | head -n 1)
export DE_DORIS_FE_IP
fi
if [ "x${DE_DORIS_BE_IP}" = "x" ]; then
DE_DORIS_BE_IP=$(grep "^[[:blank:]]*ipv4_address" ${DE_RUN_BASE}/docker-compose-doris.yml | awk -F': ' '{print $2}' | tail -n 1)
export DE_DORIS_BE_IP
fi
echo -e "*******************************************************\n" 2>&1 | tee -a ${CURRENT_DIR}/install.log
echo -e " 当前部署模式为 ${DE_ENGINE_MODE},如需切换模式,\n 请修改 $DE_BASE/dataease/.env 中的 DE_ENGINE_MODE 变量后,\n 重新执行 bash install.sh 即可\n" 2>&1 | tee -a ${CURRENT_DIR}/install.log
echo -e "*******************************************************\n" 2>&1 | tee -a ${CURRENT_DIR}/install.log
if [[ -f $dataease_conf ]]; then
DE_LOGIN_TIMEOUT=$(prop $dataease_conf dataease.login_timeout)
DE_MYSQL_PARAMS=$(grep -P "^\s*[^#]?spring.datasource.url=.*$" $dataease_conf | cut -d'=' --complement -f1 | awk -F'?' '{print $2}')
fi
export DE_MYSQL_PARAMS
export DE_LOGIN_TIMEOUT=$([[ -z $DE_LOGIN_TIMEOUT ]] && echo -n 480 || echo -n $DE_LOGIN_TIMEOUT)
if [[ -f $dataease_conf ]] && [[ ! ${DE_EXTERNAL_DORIS} ]]; then
export DE_DORIS_DB=$(prop $dataease_conf doris.db)
export DE_DORIS_USER=$(prop $dataease_conf doris.user)
export DE_DORIS_PASSWORD=$(prop $dataease_conf doris.password)
export DE_DORIS_HOST=$(prop $dataease_conf doris.host)
export DE_DORIS_PORT=$(prop $dataease_conf doris.port)
export DE_DORIS_HTTPPORT=$(prop $dataease_conf doris.httpPort)
if [ ${DE_DORIS_HOST} = "doris-fe" ]; then
export DE_EXTERNAL_DORIS="false"
else
export DE_EXTERNAL_DORIS="true"
fi
fi
if [ ${DE_EXTERNAL_DORIS} = "false" ] && [ ${DE_ENGINE_MODE} = "local" ]; then
compose_files="${compose_files} -f docker-compose-doris.yml"
fi
if [[ -f $dataease_conf ]] && [[ ! ${DE_EXTERNAL_KETTLE} ]]; then
export DE_CARTE_HOST=$(prop $dataease_conf carte.host)
export DE_CARTE_PORT=$(prop $dataease_conf carte.port)
export DE_CARTE_USER=$(prop $dataease_conf carte.user)
export DE_CARTE_PASSWORD=$(prop $dataease_conf carte.passwd)
if [ ${DE_CARTE_HOST} = "kettle" ]; then
export DE_EXTERNAL_KETTLE="false"
else
export DE_EXTERNAL_KETTLE="true"
fi
fi
if [ ${DE_EXTERNAL_KETTLE} = "false" ] && [ ${DE_ENGINE_MODE} = "local" ]; then
compose_files="${compose_files} -f docker-compose-kettle.yml"
fi
echo -e "======================= 开始安装 =======================" 2>&1 | tee -a ${CURRENT_DIR}/install.log
keep_doris="false"
if [[ -f ${DE_RUN_BASE}/docker-compose-doris.yml ]]; then
current_doris_version=$(grep '^ image:' ${DE_RUN_BASE}/docker-compose-doris.yml | head -1 | cut -d ':' -f3)
if [[ ! $current_doris_version =~ "v1.2.4" ]]; then
echo "不升级doris备份 docker-compose-doris.yml 文件"
keep_doris="true"
\cp ${DE_RUN_BASE}/docker-compose-doris.yml ${DE_RUN_BASE}/docker-compose-doris.yml.bak
fi
fi
mkdir -p ${DE_RUN_BASE}
read available_disk <<< $(df -m --output=avail ${DE_RUN_BASE} | tail -1)
if [[ $available_disk -lt 20480 ]];then
log "\033[31m[警告] DataEase 运行目录所在磁盘剩余空间不足 20G 可能无法正常启动!\033[0m"
fi
cp -r ./dataease/* ${DE_RUN_BASE}/
cd $DE_RUN_BASE
env | grep DE_ >.env
mkdir -p $conf_folder
mkdir -p ${DE_RUN_BASE}/data/kettle
mkdir -p ${DE_RUN_BASE}/data/fe
mkdir -p ${DE_RUN_BASE}/data/be
mkdir -p ${DE_RUN_BASE}/data/mysql
mkdir -p ${DE_RUN_BASE}/data/static-resource
mkdir -p ${DE_RUN_BASE}/custom-drivers
mkdir -p ${DE_RUN_BASE}/data/business
if [ ${keep_doris} = "true" ]; then
\mv ${DE_RUN_BASE}/docker-compose-doris.yml.bak ${DE_RUN_BASE}/docker-compose-doris.yml
fi
DE_MYSQL_HOST_ORIGIN=$DE_MYSQL_HOST
DE_MYSQL_PORT_ORIGIN=$DE_MYSQL_PORT
if [ ${DE_EXTERNAL_MYSQL} = "false" ]; then
compose_files="${compose_files} -f docker-compose-mysql.yml"
export DE_MYSQL_HOST=$mysql_container_name
export DE_MYSQL_PORT=3306
sed -i "s/^ container_name: mysql/ container_name: ${DE_MYSQL_HOST}/g" docker-compose-mysql.yml
else
sed -i -e "/^ depends_on/,+2d" docker-compose.yml
fi
log "拷贝配置文件模板文件 -> $conf_folder"
if [[ -f $dataease_conf ]]; then
grep "redis" $dataease_conf >> $templates_folder/dataease.properties
fi
cd $DE_RUN_BASE
cp -r $templates_folder/* $conf_folder
cp -r $templates_folder/.kettle $conf_folder
log "根据安装配置参数调整配置文件"
cd ${templates_folder}
templates_files=( dataease.properties mysql.env )
for i in ${templates_files[@]}; do
if [ -f $i ]; then
envsubst < $i > $conf_folder/$i
fi
done
export DE_MYSQL_HOST=$DE_MYSQL_HOST_ORIGIN
export DE_MYSQL_PORT=$DE_MYSQL_PORT_ORIGIN
cd ${CURRENT_DIR}
sed -i -e "s#DE_BASE=.*#DE_BASE=${DE_BASE}#g" dectl
\cp dectl /usr/local/bin && chmod +x /usr/local/bin/dectl
if [ ! -f /usr/bin/dectl ]; then
ln -s /usr/local/bin/dectl /usr/bin/dectl 2>/dev/null
fi
echo "time: $(date)"
if which getenforce && [ $(getenforce) == "Enforcing" ];then
log "... 关闭 SELINUX"
setenforce 0
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
fi
#Install docker & docker-compose
##Install Latest Stable Docker Release
if which docker >/dev/null 2>&1; then
log "检测到 Docker 已安装,跳过安装步骤"
log "启动 Docker "
service docker start 2>&1 | tee -a ${CURRENT_DIR}/install.log
else
if [[ -d docker ]]; then
log "... 离线安装 docker"
cp docker/bin/* /usr/bin/
cp docker/service/docker.service /etc/systemd/system/
chmod +x /usr/bin/docker*
chmod 754 /etc/systemd/system/docker.service
log "... 启动 docker"
systemctl enable docker; systemctl daemon-reload; service docker start 2>&1 | tee -a ${CURRENT_DIR}/install.log
else
log "... 在线安装 docker"
curl -fsSL https://resource.fit2cloud.com/get-docker-linux.sh -o get-docker.sh 2>&1 | tee -a ${CURRENT_DIR}/install.log
if [[ ! -f get-docker.sh ]];then
log "docker 在线安装脚本下载失败,请稍候重试"
exit 1
fi
sudo sh get-docker.sh 2>&1 | tee -a ${CURRENT_DIR}/install.log
log "... 启动 docker"
systemctl enable docker; systemctl daemon-reload; service docker start 2>&1 | tee -a ${CURRENT_DIR}/install.log
fi
if [ ! -d "$docker_config_folder" ];then
mkdir -p "$docker_config_folder"
fi
docker version >/dev/null 2>&1
if [ $? -ne 0 ]; then
log "docker 安装失败"
exit 1
else
log "docker 安装成功"
fi
fi
##Install Latest Stable Docker Compose Release
docker-compose version >/dev/null 2>&1
if [ $? -ne 0 ]; then
docker compose version >/dev/null 2>&1
if [ $? -eq 0 ]; then
echo 'docker compose "$@"' > /usr/bin/docker-compose
chmod +x /usr/bin/docker-compose
else
if [[ -d docker ]]; then
log "... 离线安装 docker-compose"
cp docker/bin/docker-compose /usr/bin/
chmod +x /usr/bin/docker-compose
else
log "... 在线安装 docker-compose"
curl -L https://resource.fit2cloud.com/docker/compose/releases/download/v2.16.0/docker-compose-$(uname -s | tr A-Z a-z)-$(uname -m) -o /usr/local/bin/docker-compose 2>&1 | tee -a ${CURRENT_DIR}/install.log
if [[ ! -f /usr/local/bin/docker-compose ]];then
log "docker-compose 下载失败,请稍候重试"
exit 1
fi
chmod +x /usr/local/bin/docker-compose
ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
fi
fi
docker-compose version >/dev/null
if [ $? -ne 0 ]; then
log "docker-compose 安装失败"
exit 1
else
log "docker-compose 安装成功"
fi
else
log "检测到 Docker Compose 已安装,跳过安装步骤"
fi
export COMPOSE_HTTP_TIMEOUT=180
cd ${CURRENT_DIR}
# 加载镜像
if [[ -d images ]]; then
log "加载镜像"
for i in $(ls images); do
if [ ${DE_ENGINE_MODE} != "local" ]; then
if [[ $i =~ "doris" ]] || [[ $i =~ "kettle" ]]; then
continue
fi
fi
docker load -i images/$i 2>&1 | tee -a ${CURRENT_DIR}/install.log
done
else
log "拉取镜像"
cd ${DE_RUN_BASE} && docker-compose $compose_files pull 2>&1
DEVERSION=$(cat ${CURRENT_DIR}/dataease/templates/version)
curl -sfL https://resource.fit2cloud.com/installation-log.sh | sh -s de ${INSTALL_TYPE} ${DEVERSION}
cd -
fi
log "配置 dataease Service"
cp ${DE_RUN_BASE}/bin/dataease/dataease.service /etc/init.d/dataease
chmod a+x /etc/init.d/dataease
if which chkconfig;then
chkconfig --add dataease
fi
if [ -f /etc/rc.d/rc.local ];then
dataeaseService=$(grep "service dataease start" /etc/rc.d/rc.local | wc -l)
if [ "$dataeaseService" -eq 0 ]; then
echo "sleep 10" >> /etc/rc.d/rc.local
echo "service dataease start" >> /etc/rc.d/rc.local
fi
chmod +x /etc/rc.d/rc.local
fi
if [[ $(grep "vm.max_map_count" /etc/sysctl.conf | wc -l) -eq 0 ]];then
sysctl -w vm.max_map_count=2000000
echo "vm.max_map_count=2000000" >> /etc/sysctl.conf
elif (( $(grep "vm.max_map_count" /etc/sysctl.conf | awk -F'=' '{print $2}') < 2000000 ));then
sysctl -w vm.max_map_count=2000000
sed -i 's/^vm\.max_map_count.*/vm\.max_map_count=2000000/' /etc/sysctl.conf
fi
if [ $(grep "net.ipv4.ip_forward" /etc/sysctl.conf | wc -l) -eq 0 ];then
sysctl -w net.ipv4.ip_forward=1
echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf
else
sed -i 's/^net\.ipv4\.ip_forward.*/net\.ipv4\.ip_forward=1/' /etc/sysctl.conf
fi
if which firewall-cmd >/dev/null; then
if systemctl is-active firewalld &>/dev/null ;then
log "防火墙端口开放"
firewall-cmd --zone=public --add-port=${DE_PORT}/tcp --permanent
firewall-cmd --reload
else
log "防火墙未开启,忽略端口开放"
fi
fi
http_code=$(curl -sILw "%{http_code}\n" http://localhost:${DE_PORT} -o /dev/null)
if [[ $http_code == 200 ]];then
log "停止服务进行升级..."
dectl uninstall
fi
log "启动服务"
dectl reload | tee -a ${CURRENT_DIR}/install.log
dectl status 2>&1 | tee -a ${CURRENT_DIR}/install.log
for b in {1..30}
do
sleep 3
http_code=$(curl -sILw "%{http_code}\n" http://localhost:${DE_PORT} -o /dev/null)
if [[ $http_code == 000 ]];then
log "服务启动中,请稍候 ..."
elif [[ $http_code == 200 ]];then
log "服务启动成功!"
break;
else
log "服务启动出错!"
exit 1
fi
done
if [[ $http_code != 200 ]];then
log "【警告】服务在等待时间内未完全启动!请稍后使用 dectl status 检查服务运行状况。"
fi
echo -e "======================= 安装完成 =======================\n" 2>&1 | tee -a ${CURRENT_DIR}/install.log
echo -e "请通过以下方式访问:\n URL: http://\$LOCAL_IP:$DE_PORT\n 用户名: admin\n 初始密码: dataease" 2>&1 | tee -a ${CURRENT_DIR}/install.log

View File

@ -0,0 +1 @@
# 日志目录

View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<repositories>
<repository> <id>KettleFileRepository</id>
<name>repo</name>
<description>File repository</description>
<is_default>false</is_default>
<base_directory>/opt/dataease/data/kettle</base_directory>
<read_only>N</read_only>
<hides_hidden_files>N</hides_hidden_files>
</repository> </repositories>

View File

@ -0,0 +1,63 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
PPROF_TMPDIR="$DORIS_HOME/log/"
# INFO, WARNING, ERROR, FATAL
sys_log_level = INFO
# ports for admin, web, heartbeat service
be_port = 9060
be_rpc_port = 9070
webserver_port = 8040
heartbeat_service_port = 9050
brpc_port = 8060
# Choose one if there are more than one ip except loopback address.
# Note that there should at most one ip match this list.
# If no ip match this rule, will choose one randomly.
# use CIDR format, e.g. 10.10.10.0/24
# Default value is empty.
# priority_networks = 10.10.10.0/24;192.168.0.0/16
# data root path, separate by ';'
# you can specify the storage medium of each root path, HDD or SSD
# you can add capacity limit at the end of each root path, seperate by ','
# eg:
# storage_root_path = /home/disk1/doris.HDD,50;/home/disk2/doris.SSD,1;/home/disk2/doris
# /home/disk1/doris.HDD, capacity limit is 50GB, HDD;
# /home/disk2/doris.SSD, capacity limit is 1GB, SSD;
# /home/disk2/doris, capacity limit is disk capacity, HDD(default)
#
# you also can specify the properties by setting '<property>:<value>', seperate by ','
# property 'medium' has a higher priority than the extension of path
#
# Default value is ${DORIS_HOME}/storage, you should create it by hand.
# storage_root_path = ${DORIS_HOME}/storage
# Advanced configurations
# sys_log_dir = ${DORIS_HOME}/log
# sys_log_roll_mode = SIZE-MB-1024
sys_log_roll_num = 1
load_error_log_reserve_hours=1
# sys_log_verbose_modules = *
# log_buffer_level = -1
# palo_cgroups
trash_file_expire_time_sec = 60
snapshot_expire_time_sec = 60
pending_data_expire_time_sec = 60
enable_spilling = true

View File

@ -0,0 +1,26 @@
# 数据库配置
spring.datasource.url=jdbc:mysql://${DE_MYSQL_HOST}:${DE_MYSQL_PORT}/${DE_MYSQL_DB}?${DE_MYSQL_PARAMS}
spring.datasource.username=${DE_MYSQL_USER}
spring.datasource.password=${DE_MYSQL_PASSWORD}
carte.host=${DE_CARTE_HOST}
carte.port=${DE_CARTE_PORT}
carte.user=${DE_CARTE_USER}
carte.passwd=${DE_CARTE_PASSWORD}
doris.db=${DE_DORIS_DB}
doris.user=${DE_DORIS_USER}
doris.password=${DE_DORIS_PASSWORD}
doris.host=${DE_DORIS_HOST}
doris.port=${DE_DORIS_PORT}
doris.httpPort=${DE_DORIS_HTTPPORT}
#新建用户初始密码
dataease.init_password=DataEase123456
#登录超时时间单位min 如果不设置 默认8小时也就是480
dataease.login_timeout=${DE_LOGIN_TIMEOUT}
logger.level=INFO
#DE运行模式可选值有 local、simple、cluster分别对应 本地模式、精简模式、集群模式
engine_mode=${DE_ENGINE_MODE}

View File

@ -0,0 +1,76 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#####################################################################
## The uppercase properties are read and exported by bin/start_fe.sh.
## To see all Frontend configurations,
## see fe/src/org/apache/doris/common/Config.java
#####################################################################
# the output dir of stderr and stdout
LOG_DIR = ${DORIS_HOME}/log
DATE = `date +%Y%m%d-%H%M%S`
JAVA_OPTS="-Xmx4096m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE"
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
JAVA_OPTS_FOR_JDK_9="-Xmx4096m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$DATE:time"
##
## the lowercase properties are read by main program.
##
# INFO, WARN, ERROR, FATAL
sys_log_level = INFO
# store metadata, must be created before start FE.
# Default value is ${DORIS_HOME}/doris-meta
# meta_dir = ${DORIS_HOME}/doris-meta
http_port = 8030
rpc_port = 9020
query_port = 9030
edit_log_port = 9010
mysql_service_nio_enabled = true
# Choose one if there are more than one ip except loopback address.
# Note that there should at most one ip match this list.
# If no ip match this rule, will choose one randomly.
# use CIDR format, e.g. 10.10.10.0/24
# Default value is empty.
# priority_networks = 10.10.10.0/24;192.168.0.0/16
# Advanced configurations
# log_roll_size_mb = 1024
# sys_log_dir = ${DORIS_HOME}/log
sys_log_roll_num = 1
# sys_log_verbose_modules =
# audit_log_dir = ${DORIS_HOME}/log
# audit_log_modules = slow_query, query
audit_log_roll_num = 1
# meta_delay_toleration_second = 10
qe_max_connection = 65535
max_conn_per_user = 1024
# qe_query_timeout_second = 300
# qe_slow_log_ms = 5000
sys_log_delete_age=1d
audit_log_delete_age=3d
exec_mem_limit=8589934592
tablet_create_timeout_second=30
catalog_trash_expire_second = 60
enable_batch_delete_by_default=true
max_layout_length_per_row=10000000

View File

@ -0,0 +1,28 @@
[mysqld]
datadir=/var/lib/mysql
default-storage-engine=INNODB
character_set_server=utf8
lower_case_table_names=1
table_open_cache=128
max_connections=2000
max_connect_errors=6000
innodb_file_per_table=1
innodb_buffer_pool_size=1G
max_allowed_packet=64M
transaction_isolation=READ-COMMITTED
innodb_flush_method=O_DIRECT
innodb_lock_wait_timeout=1800
innodb_flush_log_at_trx_commit=0
sync_binlog=0
group_concat_max_len=1024000
sql_mode=STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION
skip-name-resolve
[mysql]
default-character-set=utf8
[mysql.server]
default-character-set=utf8

View File

@ -0,0 +1 @@
MYSQL_ROOT_PASSWORD=${DE_MYSQL_PASSWORD}

View File

@ -0,0 +1 @@
13_20180929_184557

View File

@ -0,0 +1,41 @@
#!/bin/bash
DE_BASE=/opt
read -r -p "即将卸载 DataEase 服务,包括删除运行目录、数据及相关镜像,是否继续? [Y/n] " input
case $input in
[yY][eE][sS]|[yY])
echo "Yes"
;;
[nN][oO]|[nN])
echo "No"
exit 1
;;
*)
echo "无效输入..."
exit 1
;;
esac
if [ -f /usr/bin/dectl ]; then
# 获取已安装的 DataEase 的运行目录
DE_BASE=$(grep "^DE_BASE=" /usr/bin/dectl | cut -d'=' -f2)
fi
echo "停止 DataEase 服务"
dectl uninstall
# 清理 DataEase 相关镜像
if test ! -z "$(docker images -f dangling=true -q)"; then
echo "清理虚悬镜像"
docker rmi $(docker images -f dangling=true -q)
fi
if test -n "$(docker images | grep 'registry.cn-qingdao.aliyuncs.com/dataease')"; then
echo "清理 DataEase 镜像"
docker rmi $(docker images | grep "registry.cn-qingdao.aliyuncs.com/dataease" | awk -F' ' '{print $1":"$2}')
fi
# 清理 DataEase 运行目录及命令行工具 dectl
rm -rf ${DE_BASE}/dataease /usr/local/bin/dectl /usr/bin/dectl

76
installer/quick_start.sh Normal file
View File

@ -0,0 +1,76 @@
git_urls=('gitee.com' 'github.com')
if [[ -x "$(command -v python)" ]];then
py_cmd='python'
elif [[ -x "$(command -v python3)" ]]; then
py_cmd='python3'
else
git_urls=('github.com')
fi
for git_url in ${git_urls[*]}
do
success="true"
for i in {1..3}
do
echo -ne "检测 ${git_url} ... ${i} "
curl -m 5 -kIs https://${git_url} >/dev/null
if [ $? != 0 ];then
echo "failed"
success="false"
break
else
echo "ok"
fi
done
if [[ ${success} == "true" ]];then
server_url=${git_url}
break
fi
done
if [[ "x${server_url}" == "x" ]];then
echo "没有找到稳定的下载服务器,请稍候重试"
exit 1
fi
echo "使用下载服务器 ${server_url}"
if [[ "${server_url}" == "gitee.com" ]];then
owner='fit2cloud-feizhiyun'
repo='DataEase'
gitee_release_content=$(curl -s https://gitee.com/api/v5/repos/${owner}/${repo}/releases/latest)
# export LC_ALL="en_US.utf8"
DEVERSION=$($py_cmd -c "import json; obj=json.loads('$gitee_release_content', strict=False); print(obj['tag_name']);")
else
owner='dataease'
repo='dataease'
DEVERSION=$(curl -s https://api.github.com/repos/${owner}/${repo}/releases/latest | grep -e "\"tag_name\"" | sed -r 's/.*: "(.*)",/\1/')
fi
if [[ "x${DEVERSION}" == "x" ]];then
echo "获取最新版本失败,请稍候重试"
exit 1
fi
echo "开始下载 DataEase ${DEVERSION} 版本在线安装包"
dataease_online_file_name="dataease-${DEVERSION}-online.tar.gz"
download_url="https://${server_url}/${owner}/${repo}/releases/download/${DEVERSION}/${dataease_online_file_name}"
echo "下载地址: ${download_url}"
curl -LOk -m 60 -o ${dataease_online_file_name} ${download_url}
if [ ! -f ${dataease_online_file_name} ];then
echo "下载在线安装包失败,请试试重新执行一次安装命令。"
exit 1
fi
tar zxvf ${dataease_online_file_name}
if [ $? != 0 ];then
echo "下载在线安装包失败,请试试重新执行一次安装命令。"
rm -f ${dataease_online_file_name}
exit 1
fi
cd dataease-${DEVERSION}-online
/bin/bash install.sh