others linux服务器运维 django3 监控 k8s golang 数据库 大数据 前端 devops 理论基础 java oracle 运维日志

centos7系统下 dolphinscheduler 调度系统 集群安装部署

访问量:1847 创建时间:2021-03-26
IP 角色
192.168.96.72 postgresql-server 部署机器
192.168.96.105 zookeeper 待部署机器
192.168.96.106 zookeeper 待部署机器
192.168.96.107 zookeeper 待部署机器
jdk 1.8

yum安装postgresql

#默认是9.2.24版本
[root@disp pgsql]# yum install postgresql-server
#初始化
[root@disp pgsql]# postgresql-setup initdb
[root@disp pgsql]# systemctl start postgresql
[root@disp pgsql]# systemctl enable postgresql
Created symlink from /etc/systemd/system/multi-user.target.wants/postgresql.service to /usr/lib/systemd/system/postgresql.service.
[root@disp pgsql]# systemctl status postgresql
[root@disp pgsql]# netstat -tunlp | grep 5432
tcp        0      0 127.0.0.1:5432          0.0.0.0:*               LISTEN      4384/postgres       
tcp6       0      0 ::1:5432                :::*                    LISTEN      4384/postgres 
###
[root@disp pgsql]# su - postgres
-bash-4.2$ \l
-bash: l: command not found
-bash-4.2$ psql 
psql (9.2.24)
Type "help" for help.

postgres=# 

修改配置

#允许root用户通过psql -U postgres登录
[root@disp pgsql]# pwd
/var/lib/pgsql
[root@disp pgsql]# vim data/pg_hba.conf 
#local   all             all                                     peer
#增加下面两行,上面一行注释掉
local   all             all                                     trust
host    all             all             0.0.0.0/0               trust
[root@disp pgsql]# vim data/postgresql.conf 
#服务监听的地址修改为*
listen_addresses = '*':wq

[root@disp pgsql]# systemctl restart postgresql
[root@disp pgsql]# netstat -tunlp | grep 5432
tcp        0      0 0.0.0.0:5432            0.0.0.0:*               LISTEN      6991/postgres       
tcp6       0      0 :::5432                 :::*                    LISTEN      6991/postgres 
[root@disp pgsql]# psql -U postgres
psql (9.2.24)
Type "help" for help.

postgres=# \l
                                  List of databases
   Name    |  Owner   | Encoding |   Collate   |    Ctype    |   Access privileges   
-----------+----------+----------+-------------+-------------+-----------------------
 postgres  | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 | 
 template0 | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 | =c/postgres          +
           |          |          |             |             | postgres=CTc/postgres
 template1 | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 | =c/postgres          +
           |          |          |             |             | postgres=CTc/postgres
(3 rows)

postgres=# \q

设置密码

su - postgres 
psql    # 登进去(默认没有密码)

postgres=# \password postgres #执行sql语句,修改密码,postgres为用户名,其他用户修改也是如此
Enter new password:
Enter it again:
#####创建普通用户和数据库
postgres=# CREATE USER dolphinscheduler WITH PASSWORD 'dolphinscheduler@#12dg';
CREATE ROLE
postgres=# CREATE DATABASE dolphinscheduler OWNER dolphinscheduler;
CREATE DATABASE

下载dolphinscheduler

下载地址

[root@disp dolphinscheduler]# mkdir -pv /hadoop/dolphinscheduler
[root@disp dolphinscheduler]# cd  /hadoop/dolphinscheduler
[root@disp dolphinscheduler]# tar xf apache-dolphinscheduler-incubating-1.3.5-dolphinscheduler-bin.tar.gz 
[root@disp dolphinscheduler]# mv apache-dolphinscheduler-incubating-1.3.5-dolphinscheduler-bin dolphinscheduler-bin

创建部署用户和hosts映射

[root@name1 dolphinscheduler]# useradd dolphinscheduler;
[root@name1 dolphinscheduler]# echo "dolphinscheduler@axd!32#1" | passwd --stdin dolphinscheduler
[root@name1 dolphinscheduler]# echo 'dolphinscheduler  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' >> /etc/sudoers
[root@name1 dolphinscheduler]# sed -i 's/Defaults    requiretty/#Defaults    requiretty/g' /etc/sudoers

配置hosts和ssh无密码访问

hosts配置过程省略

[root@disp dolphinscheduler]# chown dolphinscheduler.dolphinscheduler -R /hadoop/dolphinscheduler
[dolphinscheduler@disp ~]$ ssh-keygen 
Generating public/private rsa key pair.
Enter file in which to save the key (/home/dolphinscheduler/.ssh/id_rsa): 
Created directory '/home/dolphinscheduler/.ssh'.
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /home/dolphinscheduler/.ssh/id_rsa.
Your public key has been saved in /home/dolphinscheduler/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:B6YBbj/lMY3aabYHzTkb8QePn1egNdZHLp4FJN6ou10 dolphinscheduler@disp.hdp.xxx.com.cn
The key's randomart image is:
+---[RSA 2048]----+
|    .       ..o .|
|   . .   o . + = |
|    o . B o + B =|
|   . . O B = O *.|
|      = S O + = .|
|       + + = o ..|
|        . +   E .|
|         . o . . |
|          . .    |
+----[SHA256]-----+
[dolphinscheduler@disp ~]$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
[dolphinscheduler@disp ~]$ chmod 600 ~/.ssh/authorized_keys
[dolphinscheduler@disp ~]$ ssh localhost
The authenticity of host 'localhost (::1)' can't be established.
ECDSA key fingerprint is SHA256:n0uDTVBk2cqSTNPc8pXsC+CGaDAIyqwBIQIorZ1yaeA.
ECDSA key fingerprint is MD5:d3:31:30:8f:37:e4:0c:f7:9c:f1:2d:52:16:1a:40:d6.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts.
Last login: Mon Mar 29 08:32:04 2021

        Welcome to Huawei Cloud Service

[dolphinscheduler@disp ~]$ exit
[dolphinscheduler@disp ~]$ ssh-copy-id name1.hdp.xxx.com.cn
[dolphinscheduler@disp ~]$ ssh-copy-id name2.hdp.xxx.com.cn
[dolphinscheduler@disp ~]$ ssh-copy-id node1.hdp.xxx.com.cn

数据库初始化

默认数据库是PostgreSQL(我数据库部署在72机器上),如选择MySQL的话,后续需要添加mysql-connector-java驱动包到DolphinScheduler的lib目录下.

[dolphinscheduler@disp dolphinscheduler-bin]$ pwd
/hadoop/dolphinscheduler/dolphinscheduler-bin
#修改配置文件
[dolphinscheduler@disp dolphinscheduler-bin]$ vim conf/datasource.properties
# postgresql
spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler
spring.datasource.username=dolphinscheduler
spring.datasource.password=dolphinscheduler@#12dg
# mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://xxx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true     需要修改ip
#spring.datasource.username=xxx
#spring.datasource.password=xxx
#修改并保存完后,执行 script 目录下的创建表及导入基础数据脚本
[dolphinscheduler@disp dolphinscheduler-bin]$ sh script/create-dolphinscheduler.sh

修改运行参数

[dolphinscheduler@disp dolphinscheduler-bin]$ vim conf/env/dolphinscheduler_env.sh
export JAVA_HOME=/usr/jdk64/jdk1.8.0_112
[dolphinscheduler@disp dolphinscheduler-bin]$ sudo ln -s /usr/jdk64/jdk1.8.0_112/bin/java  /usr/bin/java
[root@name1 dolphinscheduler]# mkdir -pv /hadoop/soft/dolphinscheduler
mkdir: created directory ‘/hadoop/soft’
mkdir: created directory ‘/hadoop/soft/dolphinscheduler’
[root@name1 dolphinscheduler]# chown -R dolphinscheduler.dolphinscheduler /hadoop/soft

配置安装文件,我这里没有配置资源上传功,开启了kerberos

[dolphinscheduler@disp dolphinscheduler-bin]$ cat !$
cat conf/config/install_config.conf
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


# NOTICE :  If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[`
# postgresql or mysql
dbtype="postgresql"

# db config
# db address and port
dbhost="192.168.96.72:5432"

# db username
username="dolphinscheduler"

# database name
dbname="dolphinscheduler"

# db passwprd
# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[`
password="dolphinscheduler@#12dg"

# zk cluster
zkQuorum="192.168.96.105:2181,192.168.96.106:2181,192.168.96.107:2181"

# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd)
installPath="/hadoop/soft/dolphinscheduler"

# deployment user
# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
deployUser="dolphinscheduler"


# alert config
# mail server host
mailServerHost="smtp.exmail.qq.com"

# mail server port
# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct.
mailServerPort="465"

# sender
mailSender="hostdata@xxx.com"

# user
mailUser="hostdata@xxx.com"

# sender password
# note: The mail.passwd is email service authorization code, not the email login password.
mailPassword="11221123"

# TLS mail protocol support
starttlsEnable="false"

# SSL mail protocol support
# only one of TLS and SSL can be in the true state.
sslEnable="true"

#note: sslTrust is the same as mailServerHost
sslTrust="smtp.exmail.qq.com"


# resource storage type:HDFS,S3,NONE
resourceStorageType="NONE"

# if resourceStorageType is HDFS,defaultFS write namenode address,HA you need to put core-site.xml and hdfs-site.xml in the conf directory.
# if S3,write S3 address,HA,for example :s3a://dolphinscheduler,
# Note,s3 be sure to create the root directory /dolphinscheduler
defaultFS="hdfs://mycluster:8020"

# if resourceStorageType is S3, the following three configuration is required, otherwise please ignore
s3Endpoint="http://192.168.xx.xx:9010"
s3AccessKey="xxxxxxxxxx"
s3SecretKey="xxxxxxxxxx"

# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
yarnHaIps=""

# if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname.
singleYarnIp="192.168.96.106"

# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。/dolphinscheduler is recommended
resourceUploadPath="/data/dolphinscheduler"

# who have permissions to create directory under HDFS/S3 root path
# Note: if kerberos is enabled, please config hdfsRootUser=
hdfsRootUser="hdfs"

# kerberos config
# whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore
kerberosStartUp="true"
# kdc krb5 config file path
krb5ConfPath="/etc/krb5.conf"
# keytab username
keytabUserName="hive@xxx.com.cn"
# username keytab path
keytabPath="/home/dolphinscheduler/hive.keytab"


# api server port
apiServerPort="12345"


# install hosts
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
ips="disp.hdp.xxx.com.cn,name1.hdp.xxx.com.cn,name2.hdp.xxx.com.cn,node1.hdp.xxx.com.cn"

# ssh port, default 22
# Note: if ssh port is not default, modify here
sshPort="22"

# run master machine
# Note: list of hosts hostname for deploying master
masters="disp.hdp.xxx.com.cn"

# run worker machine
# note: need to write the worker group name of each worker, the default value is "default"
workers="name1.hdp.xxx.com.cn:default,name2.hdp.xxx.com.cn:default,node1.hdp.xxx.com.cn:default"

# run alert machine
# note: list of machine hostnames for deploying alert server
alertServer="name1.hdp.xxx.com.cn"

# run api machine
# note: list of machine hostnames for deploying api server
apiServers="name1.hdp.xxx.com.cn"

执行一键部署脚本

[dolphinscheduler@disp dolphinscheduler-bin]$ sh install.sh

在不同的机器上查看启动进程

[dolphinscheduler@disp dolphinscheduler-bin]$ jps
12529 Jps
12338 MasterServer
[root@name1 dolphinscheduler]# su - dolphinscheduler
Last login: Mon Mar 29 08:55:12 CST 2021 from 192.168.96.72 on pts/2
[dolphinscheduler@name1 ~]$ jps
18785 LoggerServer
18641 WorkerServer
20209 Jps
18978 ApiApplicationServer
18885 AlertServer
Last login: Mon Mar 29 08:55:17 CST 2021 from 192.168.96.72 on pts/2
[dolphinscheduler@name2 ~]$ jps
26064 Jps
21219 LoggerServer
21015 WorkerServer
[dolphinscheduler@node1 ~]$ jps
29011 Jps
26920 WorkerServer
27022 LoggerServer
#以上程序被安装在/hadoop/soft/dolphinscheduler 下

登陆系统

登陆地址http://apiserverip:12345/dolphinscheduler

默认用户名密码:admin/dolphinscheduler123

服务启动关闭

#一键停止集群所有服务

sh ./bin/stop-all.sh

#一键开启集群所有服务

sh ./bin/start-all.sh

#启停Master

sh ./bin/dolphinscheduler-daemon.sh start master-server
sh ./bin/dolphinscheduler-daemon.sh stop master-server
#启停Worker
sh ./bin/dolphinscheduler-daemon.sh start worker-server
sh ./bin/dolphinscheduler-daemon.sh stop worker-server
#启停Api
sh ./bin/dolphinscheduler-daemon.sh start api-server
sh ./bin/dolphinscheduler-daemon.sh stop api-server
#启停Logger
sh ./bin/dolphinscheduler-daemon.sh start logger-server
sh ./bin/dolphinscheduler-daemon.sh stop logger-server
#启停Alert
sh ./bin/dolphinscheduler-daemon.sh start alert-server
sh ./bin/dolphinscheduler-daemon.sh stop alert-server
登陆评论: 使用GITHUB登陆