others linux服务器运维 django3 监控 k8s golang 数据库 大数据 前端 devops 理论基础 java oracle 运维日志

hdp2.6 安装apache-kylin 3.0.2

访问量:1156 创建时间:2020-06-05

下载安装

#下载到hdp的一个节点,节点必须已经安装了必要的角色client
[root@node1 ~]# wget https://downloads.apache.org/kylin/apache-kylin-3.0.2/apache-kylin-3.0.2-bin-hbase1x.tar.gz
[root@node1 ~]# tar xf apache-kylin-3.0.2-bin-hbase1x.tar.gz
[root@node1 ~]# mv apache-kylin-3.0.2-bin-hbase1x /hadoop/kylin

配置环境变量

[root@node1 ~]# vim /etc/profile
export KYLIN_HOME=/hadoop/kylin
export PATH=$KYLIN_HOME/bin:$PATH
[root@node1 ~]# source !$
source /etc/profile

检查环境

[root@node1 ~]# cd /hadoop/kylin/
[root@node1 kylin]# ./bin/check-env.sh 
Retrieving hadoop conf dir...
KYLIN_HOME is set to /hadoop/kylin
mkdir: Permission denied: user=root, access=WRITE, inode="/kylin":hdfs:hdfs:drwxr-xr-x
Failed to create /kylin. Please make sure the user has right to access /kylin
#创建kylin用户(hdp所有机器执行)
[root@node1 kylin]# groupadd kylin -g 1100 
[root@node1 kylin]# useradd -g kylin -u 1100 kylin
[root@node1 kylin]# su - hdfs
Last login: Fri Jun  5 18:41:53 CST 2020
[hdfs@node1 ~]$ hadoop fs -mkdir /kylin
[hdfs@node1 ~]$ hadoop fs -chown kylin /kylin
[hdfs@node1 ~]$ hadoop fs -chgrp kylin /kylin
[hdfs@node1 ~]$ hadoop fs -mkdir /user/kylin
[hdfs@node1 ~]$ hadoop fs -chown kylin:kylin /user/kylin
[hdfs@node1 ~]$ hadoop fs -ls /user/
[hdfs@node1 ~]$ exit
[root@node1 kylin]# chown kylin.kylin -R /hadoop/kylin/
#通过kylin用户检查环境,没有报错
[root@node1 kylin]# su - kylin
Last login: Fri Jun  5 19:22:15 CST 2020 on pts/0
[kylin@node1 ~]$ cd /hadoop/kylin/
[kylin@node1 kylin]$ ./bin/check-env.sh 
Retrieving hadoop conf dir...
KYLIN_HOME is set to /hadoop/kylin

启动

[kylin@node1 kylin]$ ./bin/kylin.sh start
[kylin@node1 kylin]$ ./bin/kylin.sh start
Retrieving hadoop conf dir...
KYLIN_HOME is set to /hadoop/kylin
Retrieving hive dependency...
Retrieving hbase dependency...
Retrieving hadoop conf dir...
Retrieving kafka dependency...
Retrieving Spark dependency...
spark not found, set SPARK_HOME, or run bin/download-spark.sh
#解决方法
[kylin@node1 kylin]$ exit
logout
[root@node1 kylin]# vim /etc/profile
export SPARK_HOME=/usr/hdp/current/spark2-client
export HIVE_CONF=/etc/hive/conf
export HCAT_HOME=/usr/hdp/current/hive-webhcat
[root@node1 kylin]#  source /etc/profile
[root@node1 kylin]# su - kylin
[kylin@node1 ~]$ cd /hadoop/kylin/
[kylin@node1 kylin]$ ./bin/kylin.sh start
A new Kylin instance is started by kylin. To stop it, run 'kylin.sh stop'
Check the log at /hadoop/kylin/logs/kylin.log
Web UI is at http://node1.work.com:7070/kylin

访问页面http://192.168.31.132:7070/kylin/login 账号ADMIN密码KYLIN

kylin 接入启用kerberos的大数据集群

如果你的大数据集群开启了kerberos,上面的安装部署完成后,启动会报错:WARN ipc.Client: Exception encountered while connecting to the server : javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]

kylin使用kerberos认证只需要运行用户环境进行了kinit即可。

在kerberos中创建

[root@ecs-hdp-dev-ambari ~]# kadmin.local 
Authenticating as principal root/admin@work.com with password.
kadmin.local:  addprinc -randkey kylin@work.com
kadmin.local:  xst -k klin.keytab kylin@work.com
#cp文件到kylin机器
[root@ecs-hdp-dev-ambari ~]# scp klin.keytab 172.16.96.109:/home/kylin/
#在kylin机器上修改文件权限
[root@node3 ~]# chown kylin.kylin /home/kylin/klin.keytab

hive授权

[root@node3 ~]# klist -e -kt /etc/security/keytabs/hive.service.keytab 
Keytab name: FILE:/etc/security/keytabs/hive.service.keytab
KVNO Timestamp           Principal
---- ------------------- ------------------------------------------------------
   1 05/27/2020 13:30:38 hive/node3.hdp.work.com@work.com (des3-cbc-sha1) 
   1 05/27/2020 13:30:38 hive/node3.hdp.work.com@work.com (aes256-cts-hmac-sha1-96) 
   1 05/27/2020 13:30:38 hive/node3.hdp.work.com@work.com (des-cbc-md5) 
   1 05/27/2020 13:30:38 hive/node3.hdp.work.com@work.com (aes128-cts-hmac-sha1-96) 
   1 05/27/2020 13:30:38 hive/node3.hdp.work.com@work.com (arcfour-hmac) 
[root@node3 ~]# kinit -kt /etc/security/keytabs/hive.service.keytab  hive/node3.hdp.work.com@work.com
You have new mail in /var/spool/mail/root
[root@node3 ~]# klist 
Ticket cache: FILE:/tmp/krb5cc_0
Default principal: hive/node3.hdp.work.com@work.com

Valid starting       Expires              Service principal
06/06/2020 08:54:21  06/07/2020 08:54:21  krbtgt/work.com@work.com

[root@node3 ~]# beeline -u "jdbc:hive2://name2.hdp.work.com:10000/default;principal=hive/_HOST@work.com"
0: jdbc:hive2://name2.hdp.work.com:1000> set hive.users.in.admin.role;
+----------------------------------------+--+
|                  set                   |
+----------------------------------------+--+
| hive.users.in.admin.role is undefined  |
+----------------------------------------+--+
#没有管理员用户,在ambari中定义hive.users.in.admin.role=hive,重启hive服务
[root@node3 ~]# beeline -u "jdbc:hive2://name2.hdp.work.com:10000/default;principal=hive/_HOST@work.com"
Connecting to jdbc:hive2://name2.hdp.work.com:10000/default;principal=hive/_HOST@work.com
Connected to: Apache Hive (version 1.2.1000.2.6.5.0-292)
Driver: Hive JDBC (version 1.2.1000.2.6.5.0-292)
Transaction isolation: TRANSACTION_REPEATABLE_READ
Beeline version 1.2.1000.2.6.5.0-292 by Apache Hive
0: jdbc:hive2://name2.hdp.work.com:1000> set hive.users.in.admin.role;  
+--------------------------------+--+
|              set               |
+--------------------------------+--+
| hive.users.in.admin.role=hive  |
+--------------------------------+--+
1 row selected (0.241 seconds)
0: jdbc:hive2://name2.hdp.work.com:1000> show current roles; 
+---------+--+
|  role   |
+---------+--+
| public  |
+---------+--+
1 row selected (1.384 seconds)
0: jdbc:hive2://name2.hdp.work.com:1000> set role admin;  
No rows affected (0.26 seconds)
0: jdbc:hive2://name2.hdp.work.com:1000> show current roles; 
+--------+--+
|  role  |
+--------+--+
| admin  |
+--------+--+
1 row selected (0.051 seconds)
0: jdbc:hive2://name2.hdp.work.com:1000> grant role admin to user kylin;
No rows affected (0.275 seconds)

hbase授权

[root@node2 ~]# kinit -kt /etc/security/keytabs/hbase.headless.keytab hbase-hdpdev@work.com
[root@node2 ~]# hbase shell
HBase Shell; enter 'help<RETURN>' for list of supported commands.
Type "exit<RETURN>" to leave the HBase Shell
Version 1.1.2.2.6.5.0-292, r897822d4dd5956ca186974c10382e9094683fa29, Fri May 11 08:00:59 UTC 2018

hbase(main):001:0> grant 'kylin','RWXCA'
0 row(s) in 0.9690 seconds

启动

[root@node3 ~]# su - kylin
[kylin@node3 ~]$ kinit -k -t /home/kylin/klin.keytab kylin@work.com
#定时刷新
[kylin@node3 kylin]$ crontab -e
3 * * * * kinit -k -t /home/kylin/klin.keytab kylin@work.com
#启动
[kylin@node3 kylin]$ ./bin/kylin.sh start
登陆评论: 使用GITHUB登陆