IP | 角色 |
---|---|
192.168.96.72 | postgresql-server 部署机器 |
192.168.96.105 | zookeeper 待部署机器 |
192.168.96.106 | zookeeper 待部署机器 |
192.168.96.107 | zookeeper 待部署机器 |
jdk | 1.8 |
#默认是9.2.24版本
[root@disp pgsql]# yum install postgresql-server
#初始化
[root@disp pgsql]# postgresql-setup initdb
[root@disp pgsql]# systemctl start postgresql
[root@disp pgsql]# systemctl enable postgresql
Created symlink from /etc/systemd/system/multi-user.target.wants/postgresql.service to /usr/lib/systemd/system/postgresql.service.
[root@disp pgsql]# systemctl status postgresql
[root@disp pgsql]# netstat -tunlp | grep 5432
tcp 0 0 127.0.0.1:5432 0.0.0.0:* LISTEN 4384/postgres
tcp6 0 0 ::1:5432 :::* LISTEN 4384/postgres
###
[root@disp pgsql]# su - postgres
-bash-4.2$ \l
-bash: l: command not found
-bash-4.2$ psql
psql (9.2.24)
Type "help" for help.
postgres=#
修改配置
#允许root用户通过psql -U postgres登录
[root@disp pgsql]# pwd
/var/lib/pgsql
[root@disp pgsql]# vim data/pg_hba.conf
#local all all peer
#增加下面两行,上面一行注释掉
local all all trust
host all all 0.0.0.0/0 trust
[root@disp pgsql]# vim data/postgresql.conf
#服务监听的地址修改为*
listen_addresses = '*':wq
[root@disp pgsql]# systemctl restart postgresql
[root@disp pgsql]# netstat -tunlp | grep 5432
tcp 0 0 0.0.0.0:5432 0.0.0.0:* LISTEN 6991/postgres
tcp6 0 0 :::5432 :::* LISTEN 6991/postgres
[root@disp pgsql]# psql -U postgres
psql (9.2.24)
Type "help" for help.
postgres=# \l
List of databases
Name | Owner | Encoding | Collate | Ctype | Access privileges
-----------+----------+----------+-------------+-------------+-----------------------
postgres | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 |
template0 | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | =c/postgres +
| | | | | postgres=CTc/postgres
template1 | postgres | UTF8 | en_US.UTF-8 | en_US.UTF-8 | =c/postgres +
| | | | | postgres=CTc/postgres
(3 rows)
postgres=# \q
设置密码
su - postgres
psql # 登进去(默认没有密码)
postgres=# \password postgres #执行sql语句,修改密码,postgres为用户名,其他用户修改也是如此
Enter new password:
Enter it again:
#####创建普通用户和数据库
postgres=# CREATE USER dolphinscheduler WITH PASSWORD 'dolphinscheduler@#12dg';
CREATE ROLE
postgres=# CREATE DATABASE dolphinscheduler OWNER dolphinscheduler;
CREATE DATABASE
[root@disp dolphinscheduler]# mkdir -pv /hadoop/dolphinscheduler
[root@disp dolphinscheduler]# cd /hadoop/dolphinscheduler
[root@disp dolphinscheduler]# tar xf apache-dolphinscheduler-incubating-1.3.5-dolphinscheduler-bin.tar.gz
[root@disp dolphinscheduler]# mv apache-dolphinscheduler-incubating-1.3.5-dolphinscheduler-bin dolphinscheduler-bin
[root@name1 dolphinscheduler]# useradd dolphinscheduler;
[root@name1 dolphinscheduler]# echo "dolphinscheduler@axd!32#1" | passwd --stdin dolphinscheduler
[root@name1 dolphinscheduler]# echo 'dolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' >> /etc/sudoers
[root@name1 dolphinscheduler]# sed -i 's/Defaults requiretty/#Defaults requiretty/g' /etc/sudoers
hosts配置过程省略
[root@disp dolphinscheduler]# chown dolphinscheduler.dolphinscheduler -R /hadoop/dolphinscheduler
[dolphinscheduler@disp ~]$ ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/home/dolphinscheduler/.ssh/id_rsa):
Created directory '/home/dolphinscheduler/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/dolphinscheduler/.ssh/id_rsa.
Your public key has been saved in /home/dolphinscheduler/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:B6YBbj/lMY3aabYHzTkb8QePn1egNdZHLp4FJN6ou10 dolphinscheduler@disp.hdp.xxx.com.cn
The key's randomart image is:
+---[RSA 2048]----+
| . ..o .|
| . . o . + = |
| o . B o + B =|
| . . O B = O *.|
| = S O + = .|
| + + = o ..|
| . + E .|
| . o . . |
| . . |
+----[SHA256]-----+
[dolphinscheduler@disp ~]$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
[dolphinscheduler@disp ~]$ chmod 600 ~/.ssh/authorized_keys
[dolphinscheduler@disp ~]$ ssh localhost
The authenticity of host 'localhost (::1)' can't be established.
ECDSA key fingerprint is SHA256:n0uDTVBk2cqSTNPc8pXsC+CGaDAIyqwBIQIorZ1yaeA.
ECDSA key fingerprint is MD5:d3:31:30:8f:37:e4:0c:f7:9c:f1:2d:52:16:1a:40:d6.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts.
Last login: Mon Mar 29 08:32:04 2021
Welcome to Huawei Cloud Service
[dolphinscheduler@disp ~]$ exit
[dolphinscheduler@disp ~]$ ssh-copy-id name1.hdp.xxx.com.cn
[dolphinscheduler@disp ~]$ ssh-copy-id name2.hdp.xxx.com.cn
[dolphinscheduler@disp ~]$ ssh-copy-id node1.hdp.xxx.com.cn
默认数据库是PostgreSQL(我数据库部署在72机器上),如选择MySQL的话,后续需要添加mysql-connector-java驱动包到DolphinScheduler的lib目录下.
[dolphinscheduler@disp dolphinscheduler-bin]$ pwd
/hadoop/dolphinscheduler/dolphinscheduler-bin
#修改配置文件
[dolphinscheduler@disp dolphinscheduler-bin]$ vim conf/datasource.properties
# postgresql
spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler
spring.datasource.username=dolphinscheduler
spring.datasource.password=dolphinscheduler@#12dg
# mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://xxx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true 需要修改ip
#spring.datasource.username=xxx
#spring.datasource.password=xxx
#修改并保存完后,执行 script 目录下的创建表及导入基础数据脚本
[dolphinscheduler@disp dolphinscheduler-bin]$ sh script/create-dolphinscheduler.sh
[dolphinscheduler@disp dolphinscheduler-bin]$ vim conf/env/dolphinscheduler_env.sh
export JAVA_HOME=/usr/jdk64/jdk1.8.0_112
[dolphinscheduler@disp dolphinscheduler-bin]$ sudo ln -s /usr/jdk64/jdk1.8.0_112/bin/java /usr/bin/java
[root@name1 dolphinscheduler]# mkdir -pv /hadoop/soft/dolphinscheduler
mkdir: created directory ‘/hadoop/soft’
mkdir: created directory ‘/hadoop/soft/dolphinscheduler’
[root@name1 dolphinscheduler]# chown -R dolphinscheduler.dolphinscheduler /hadoop/soft
配置安装文件,我这里没有配置资源上传功,开启了kerberos
[dolphinscheduler@disp dolphinscheduler-bin]$ cat !$
cat conf/config/install_config.conf
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# NOTICE : If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[`
# postgresql or mysql
dbtype="postgresql"
# db config
# db address and port
dbhost="192.168.96.72:5432"
# db username
username="dolphinscheduler"
# database name
dbname="dolphinscheduler"
# db passwprd
# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[`
password="dolphinscheduler@#12dg"
# zk cluster
zkQuorum="192.168.96.105:2181,192.168.96.106:2181,192.168.96.107:2181"
# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd)
installPath="/hadoop/soft/dolphinscheduler"
# deployment user
# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
deployUser="dolphinscheduler"
# alert config
# mail server host
mailServerHost="smtp.exmail.qq.com"
# mail server port
# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct.
mailServerPort="465"
# sender
mailSender="hostdata@xxx.com"
# user
mailUser="hostdata@xxx.com"
# sender password
# note: The mail.passwd is email service authorization code, not the email login password.
mailPassword="11221123"
# TLS mail protocol support
starttlsEnable="false"
# SSL mail protocol support
# only one of TLS and SSL can be in the true state.
sslEnable="true"
#note: sslTrust is the same as mailServerHost
sslTrust="smtp.exmail.qq.com"
# resource storage type:HDFS,S3,NONE
resourceStorageType="NONE"
# if resourceStorageType is HDFS,defaultFS write namenode address,HA you need to put core-site.xml and hdfs-site.xml in the conf directory.
# if S3,write S3 address,HA,for example :s3a://dolphinscheduler,
# Note,s3 be sure to create the root directory /dolphinscheduler
defaultFS="hdfs://mycluster:8020"
# if resourceStorageType is S3, the following three configuration is required, otherwise please ignore
s3Endpoint="http://192.168.xx.xx:9010"
s3AccessKey="xxxxxxxxxx"
s3SecretKey="xxxxxxxxxx"
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
yarnHaIps=""
# if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname.
singleYarnIp="192.168.96.106"
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。/dolphinscheduler is recommended
resourceUploadPath="/data/dolphinscheduler"
# who have permissions to create directory under HDFS/S3 root path
# Note: if kerberos is enabled, please config hdfsRootUser=
hdfsRootUser="hdfs"
# kerberos config
# whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore
kerberosStartUp="true"
# kdc krb5 config file path
krb5ConfPath="/etc/krb5.conf"
# keytab username
keytabUserName="hive@xxx.com.cn"
# username keytab path
keytabPath="/home/dolphinscheduler/hive.keytab"
# api server port
apiServerPort="12345"
# install hosts
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
ips="disp.hdp.xxx.com.cn,name1.hdp.xxx.com.cn,name2.hdp.xxx.com.cn,node1.hdp.xxx.com.cn"
# ssh port, default 22
# Note: if ssh port is not default, modify here
sshPort="22"
# run master machine
# Note: list of hosts hostname for deploying master
masters="disp.hdp.xxx.com.cn"
# run worker machine
# note: need to write the worker group name of each worker, the default value is "default"
workers="name1.hdp.xxx.com.cn:default,name2.hdp.xxx.com.cn:default,node1.hdp.xxx.com.cn:default"
# run alert machine
# note: list of machine hostnames for deploying alert server
alertServer="name1.hdp.xxx.com.cn"
# run api machine
# note: list of machine hostnames for deploying api server
apiServers="name1.hdp.xxx.com.cn"
[dolphinscheduler@disp dolphinscheduler-bin]$ sh install.sh
在不同的机器上查看启动进程
[dolphinscheduler@disp dolphinscheduler-bin]$ jps
12529 Jps
12338 MasterServer
[root@name1 dolphinscheduler]# su - dolphinscheduler
Last login: Mon Mar 29 08:55:12 CST 2021 from 192.168.96.72 on pts/2
[dolphinscheduler@name1 ~]$ jps
18785 LoggerServer
18641 WorkerServer
20209 Jps
18978 ApiApplicationServer
18885 AlertServer
Last login: Mon Mar 29 08:55:17 CST 2021 from 192.168.96.72 on pts/2
[dolphinscheduler@name2 ~]$ jps
26064 Jps
21219 LoggerServer
21015 WorkerServer
[dolphinscheduler@node1 ~]$ jps
29011 Jps
26920 WorkerServer
27022 LoggerServer
#以上程序被安装在/hadoop/soft/dolphinscheduler 下
登陆地址http://apiserverip:12345/dolphinscheduler
默认用户名密码:admin/dolphinscheduler123
#一键停止集群所有服务
sh ./bin/stop-all.sh
#一键开启集群所有服务
sh ./bin/start-all.sh
#启停Master
sh ./bin/dolphinscheduler-daemon.sh start master-server
sh ./bin/dolphinscheduler-daemon.sh stop master-server
#启停Worker
sh ./bin/dolphinscheduler-daemon.sh start worker-server
sh ./bin/dolphinscheduler-daemon.sh stop worker-server
#启停Api
sh ./bin/dolphinscheduler-daemon.sh start api-server
sh ./bin/dolphinscheduler-daemon.sh stop api-server
#启停Logger
sh ./bin/dolphinscheduler-daemon.sh start logger-server
sh ./bin/dolphinscheduler-daemon.sh stop logger-server
#启停Alert
sh ./bin/dolphinscheduler-daemon.sh start alert-server
sh ./bin/dolphinscheduler-daemon.sh stop alert-server