others linux服务器运维 django3 监控 k8s golang 数据库 大数据 前端 devops 理论基础 java oracle 运维日志

centos7 hdp 集群 安装hue 4.7

访问量:1813 创建时间:2020-06-10

版本选择

综合考虑：

下载地址：

下载后上传到你要部署的机器上，hue4.7支持python2.7与python3.5+ (建议用3.6，可以yum安装)

安装hue

#安装依赖文件
[root@name1 hue-4.7.1]# yum install zlib-devel zlibc zlib-bin sqlite-devel saslwrapper-devel.x86_64 python-devel python-dev openssl-devel openldap-devel mysql-devel mysql make lxml libxslt-devel libxml2-devel libssl-dev libmysqlclient-dev libldap2-dev libkrb5-dev libidn11-dev libidn11 libgsasl-devel.x86_64 libffi-devel krb5-devel gmp-devel gcc-c++ gcc g++ cyrus-sasl-plain cyrus-sasl-lib.x86_64 cyrus-sasl-gssapi cyrus-sasl-devel.x86_64 cyrus-sasl-devel build-essential asciidoc ant
[root@name1 hue]# pwd
/hadoop/hue
[root@name1 hue]# tar xf hue-4.7.1.tgz
[root@name1 hue]# cd hue-4.7.1
#编译的错误可能比较多，解决对应的依赖
[root@name1 hue-4.7.1]# make apps
[root@name1 hue-4.7.1]# useradd hue -G users,hadoop
#修改hue配置文件（这里只用hue连接hive，调试一下hive），如果大数据集群没有开启kerberos，[kerberos]可以不用配置
[root@name1 hue-4.7.1]# cat desktop/conf/hue.ini | egrep -v '^$|^    #|^#|^  #|^      #|^   #|        #'
[desktop]
  secret_key=
  http_host=0.0.0.0
  http_port=8888
  time_zone=Asia/Shanghai
  django_debug_mode=false
  http_500_debug_mode=false
  server_user=hue
  server_group=hadoop
  default_site_encoding=utf-8
  [[django_admins]]
  [[custom]]
  [[auth]]
    idle_session_timeout=-1
  [[ldap]]
    [[[users]]]
    [[[groups]]]
    [[[ldap_servers]]]
  [[vcs]]
  #hue使用的数据库，默认sqlite，改为mysql
  [[database]]
    engine=mysql
    host=ecs-hdp-dev-ambari
    port=3306
    user=hue
    password=yourpassword
    name=hue47
  [[session]]
  [[smtp]]
    host=localhost
    port=25
    user=
    password=
    tls=no
  [[knox]]
  #kerbero如果没开启，[kerberos]下面内容不用配置
  [[kerberos]]
    hue_keytab=/hadoop/hue/hue47.keytab
    hue_principal=hue/name1.hdp.work.com@work.com
    ccache_path=/var/run/hue/hue_krb5_ccache
    kinit_path=/usr/bin/kinit
  [[oauth]]
  [[oidc]]
  [[metrics]]
  [[tracing]]
  [[task_server]]
   [[gc_accounts]]
      [[[default]]]
[notebook]
  [[interpreters]]
[dashboard]
  [[engines]]
[hadoop]
  [[hdfs_clusters]]
    [[[default]]]
      fs_defaultfs=hdfs://name1.hdp.work.com:8020
      webhdfs_url=http://name1.hdp.work.com:50070/webhdfs/v1/
      security_enabled=true
  [[yarn_clusters]]
    [[[default]]]
      resourcemanager_host=name2.hdp.work.com
      resourcemanager_port=8050
      submit_to=True
      security_enabled=true
      resourcemanager_api_url=http://name2.hdp.work.com:8088

#hive 的配置，hive版本较低thrift_version=7，其他配置根据自己集群配置
[beeswax]
  hive_server_host=name2.hdp.work.com
  hive_server_port=10000
  hive_metastore_host=name2.hdp.work.com
  hive_metastore_port=9083
  hive_conf_dir=/etc/hive/conf
  max_number_of_sessions=-1
  close_sessions= false
  thrift_version=7
  [[ssl]]
[metastore]
#impala的配置，如果你的集群没有impala,将[impala]下面的内容删除
[impala]
  server_host=name2.hdp.work.com
  server_port=21050
  use_sasl=true
  impala_principal=impala/_HOST@work.com
  [[ssl]]
[spark]
  livy_server_url=http://localhost:8998
[oozie]
[filebrowser]
[pig]
[sqoop]
[proxy]
[hbase]
  hbase_conf_dir=/etc/hbase/conf
[search]
[libsolr]
[indexer]
[jobsub]
[jobbrowser]
[security]
[zookeeper]
  [[clusters]]
    [[[default]]]
[useradmin]
  [[password_policy]]
[liboozie]
[aws]
  [[aws_accounts]]
[azure]
  [[azure_accounts]]
    [[[default]]]
  [[adls_clusters]]
    [[[default]]]
  [[abfs_clusters]]
    [[[default]]]
[libsentry]
[libzookeeper]
[librdbms]
  [[databases]]
[libsaml]
[liboauth]
[kafka]
  [[kafka]]
[metadata]
  [[manager]]
  [[optimizer]]
  [[catalog]]
  [[navigator]]
  [[prometheus]]

创建mariadb数据库

MariaDB [(none)]> create database hue47  default character set utf8 default collate utf8_general_ci;
MariaDB [(none)]> grant all privileges on hue47.* to 'hue'@'%'  identified by 'password';

同步hue数据库

[root@name1 hue-4.7.1]# ./build/env/bin/hue syncdb
[root@name1 hue-4.7.1]# ./build/env/bin/hue migrate
#启动
[root@name1 hue-4.7.1]# nohup ./build/env/bin/supervisor &

hue安装报错(以下信息经过调整)

hue报错1：

(hue) [root@hdp01 hue-4.7.1]# pwd
/hadoop/hue/hue-4.7.1
(hue) [root@hdp01 hue-4.7.1]# tail -f logs/kt_renewer.log 
kt_renewer   ERROR    Couldn't renew kerberos ticket in order to work around Kerberos 1.8.1 issue. Please check that the ticket for 'hue/node1@EXAMPLE.COM' is still renewable:
  $ klist -f -c /var/run/hue/hue_krb5_ccache

处理方法：

[root@hdp01 ~]# kadmin.local
Authenticating as principal dmpadmin/admin@hdpprd.work.com.cn with password.
kadmin.local:   modprinc -maxrenewlife 7day +allow_renewable krbtgt/hdpprd.work.com.cn@hdpprd.work.com.cn
Principal "krbtgt/hdpprd.work.com.cn@hdpprd.work.com.cn" modified.

hue报错2：

#报错信息
Bad status for request TOpenSessionReq(username='hue', password=None, client_protocol=6, configuration={'hive.server2.proxy.user': u'hive'}): TOpenSessionResp(status=TStatus(errorCode=0, errorMessage=u'Failed to validate proxy privilege of hue for hive', sqlState=u'08S01', infoMessages=[u'*org.apache.hive.service.cli.HiveSQLException:Failed to validate proxy privilege of hue for hive:14:13', u'org.apache.hive.service.auth.HiveAuthFactory:verifyProxyAccess:HiveAuthFactory.java:394', u'org.apache.hive.service.cli.thrift.ThriftCLIService:getProxyUser:ThriftCLIService.java:777', u'org.apache.hive.service.cli.thrift.ThriftCLIService:getUserName:ThriftCLIService.java:389', u'org.apache.hive.service.cli.thrift.ThriftCLIService:getSessionHandle:ThriftCLIService.java:416', u'org.apache.hive.service.cli.thrift.ThriftCLIService:OpenSession:ThriftCLIService.java:319', u'org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession:getResult:TCLIService.java:1317', u'org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession:getResult:TCLIService.java:1302', u'org.apache.thrift.ProcessFunction:process:ProcessFunction.java:39', u'org.apache.thrift.TBaseProcessor:process:TBaseProcessor.java:39', u'org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingProcessor:process:HadoopThriftAuthBridge.java:562', u'org.apache.thrift.server.TThreadPoolServer$WorkerProcess:run:TThreadPoolServer.java:286', u'java.util.concurrent.ThreadPoolExecutor:runWorker:ThreadPoolExecutor.java:1142', u'java.util.concurrent.ThreadPoolExecutor$Worker:run:ThreadPoolExecutor.java:617', u'java.lang.Thread:run:Thread.java:745', u'*org.apache.hadoop.security.authorize.AuthorizationException:User: hue is not allowed to impersonate hive:0:-1'], statusCode=3), sessionHandle=None, configuration=None, serverProtocolVersion=7)
处理方法
配置hadoop custom core-site.xml 文件增加 hadoop.proxyuser.hue.groups=* hadoop.proxyuser.hue.hosts=*

需要重启hdp集群

hue故障3：

在hue中查询hive，卡死不反应(通常出现在几个人同时登陆一个hue账号时出现)

在hdp的ambari配置Custom hive-site hive.server2.parallel.ops.in.session=true 重启hive

hue故障4：

AnalysisException: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/HBaseConfiguration CAUSED BY: ExecutionException: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/HBaseConfiguration CAUSED BY: NoClassDefFoundError: org/apache/hadoop/hbase/HBaseConfiguration CAUSED BY: ClassNotFoundException: org.apache.hadoop.hbase.HBaseConfiguration CAUSED BY: TableLoadingException: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/HBaseConfiguration CAUSED BY: ExecutionException: java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/HBaseConfiguration CAUSED BY: NoClassDefFoundError: org/apache/hadoop/hbase/HBaseConfiguration CAUSED BY: ClassNotFoundException: org.apache.hadoop.hbase.HBaseConfiguration

解决方法：保证impala的全部机器的lib下面的hbase的jar包链接文件能够生效

(base) [root@hdp01 ~]# ll /usr/lib/impala/lib/hbase-*
lrwxrwxrwx 1 root root 36 Aug 20 16:24 /usr/lib/impala/lib/hbase-annotations.jar -> /usr/lib/hbase/hbase-annotations.jar
lrwxrwxrwx 1 root root 31 Aug 20 16:24 /usr/lib/impala/lib/hbase-client.jar -> /usr/lib/hbase/hbase-client.jar
lrwxrwxrwx 1 root root 31 Aug 20 16:24 /usr/lib/impala/lib/hbase-common.jar -> /usr/lib/hbase/hbase-common.jar
lrwxrwxrwx 1 root root 33 Aug 20 16:24 /usr/lib/impala/lib/hbase-protocol.jar -> /usr/lib/hbase/hbase-protocol.jar

故障5：

现象:hue中使用hive建表中文字符报错：'ascii' codec can't decode byte 0xe5 in position 68: ordinal not in range(128)

hue-4.7.1/desktop/core/src/desktop/lib/thrift_util.py 文件头部加一下（在35行后）
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

故障6：

现象：hue中查询sql报错:(1366, "Incorrect string value: '\xE9' for column hue47.desktop_document2.search at row 1"

处理：在hue中查询条件加了中文,或者有中文注释都会报错

解决方案如下：(原来我的mariadb是11版本，按照网上的更改字段字符集的方法不生效)
更换数据库mariadb-5.5.6，centos7   epel默认的库

故障7：

现象 ： (2006, ‘MySQL server has gone away’)

[root@hdpprde01 hue-4.7.1]# mysql -uroot -p
Welcome to the MariaDB monitor.  Commands end with ; or \g.
Your MariaDB connection id is 70139
Server version: 5.5.68-MariaDB MariaDB Server

Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

MariaDB [(none)]> show global variables like '%timeout';
+----------------------------+----------+
| Variable_name              | Value    |
+----------------------------+----------+
| connect_timeout            | 10       |
| delayed_insert_timeout     | 300      |
| innodb_lock_wait_timeout   | 50       |
| innodb_rollback_on_timeout | OFF      |
| interactive_timeout        | 28800    |
| lock_wait_timeout          | 31536000 |
| net_read_timeout           | 30       |
| net_write_timeout          | 60       |
| slave_net_timeout          | 3600     |
| thread_pool_idle_timeout   | 60       |
| wait_timeout               | 28800    |
+----------------------------+----------+
11 rows in set (0.00 sec)

MariaDB [(none)]> set global wait_timeout=60*60*96;
Query OK, 0 rows affected (0.00 sec)

MariaDB [(none)]> show global variables like '%timeout';
+----------------------------+----------+
| Variable_name              | Value    |
+----------------------------+----------+
| connect_timeout            | 10       |
| delayed_insert_timeout     | 300      |
| innodb_lock_wait_timeout   | 50       |
| innodb_rollback_on_timeout | OFF      |
| interactive_timeout        | 28800    |
| lock_wait_timeout          | 31536000 |
| net_read_timeout           | 30       |
| net_write_timeout          | 60       |
| slave_net_timeout          | 3600     |
| thread_pool_idle_timeout   | 60       |
| wait_timeout               | 345600   |
+----------------------------+----------+
11 rows in set (0.00 sec)

MariaDB [(none)]> 

#修改主配置文件
[root@hdpprde01 hue-4.7.1]# vim /etc/my.cnf

[mysqld]
wait_timeout=345600

故障8：

现象： hue web页面查看文件报错，failed to obtain user group information : java.io.IOException:Username not matched: name=hive != excepted=hue"}}(error 403) 解决方法：修改hue配置文件desktop/conf/hue.ini 配置default_user=hue 然后重启hue (我的server_user=hue ，kerberos的用户名也是hue)

<< 上一章节
登陆评论: 使用GITHUB登陆