others linux服务器运维 django3 监控 k8s golang 数据库 大数据 前端 devops 理论基础 java oracle 运维日志

centos7 hive单机版部署

访问量:20 创建时间:2025-12-27

hadoop单机版部署

修改hostname
[root@localhost ~]# hostnamectl set-hostname  node-192-168-72-119.test.com
[root@localhost ~]# vim /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.72.119  node-192-168-72-119.test.com



[root@localhost ~]# yum install java-1.8.0-openjdk*
[root@localhost ~]# mkdir /data
[root@localhost ~]# cd !$
# 下载Hadoop
[root@localhost data]# wget   https://dlcdn.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz
[root@localhost data]# tar xf hadoop-3.3.6.tar.gz

[root@localhost data]# mkdir -pv /data/dfs/{tmp,data,name}
mkdir: created directory ‘/data/dfs/tmp’
mkdir: created directory ‘/data/dfs/data’

##文件结尾增加
[root@localhost hadoop-3.3.6]# vim /etc/profile
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.412.b08-1.el7_9.x86_64/
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/tools.jar

export HADOOP_HOME=/data/hadoop-3.3.6
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

[root@localhost data]# source /etc/profile

[root@localhost data]# cd hadoop-3.3.6/
[root@localhost hadoop-3.3.6]# vim etc/hadoop/hadoop-env.sh 
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.412.b08-1.el7_9.x86_64
export HADOOP_HOME=/data/hadoop-3.3.6
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

[root@localhost data]# vim $HADOOP_HOME/etc/hadoop/core-site.xml
<configuration>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/data/dfs/tmp</value>
        <description>Abase for other temporary directories.</description>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://node-192-168-72-119.test.com:9000</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
    </property>

    <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
    </property>
</configuration>

[root@localhost data]# vim hadoop-3.3.6/etc/hadoop/hdfs-site.xml

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/data/dfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/data/dfs/data</value>
    </property>
    <property>
        <name>dfs.http.address</name>
        <value>0.0.0.0:50070</value>
    </property>

    <property>
        <name>dfs.permissions</name>
        <value>false</value>
    </property>
</configuration>

###########yarn.application.classpath  的值为echo $HADOOP_CLASSPATH  的结果
[root@node-192-168-72-119 data]# export HADOOP_CLASSPATH=$(hadoop classpath)
[root@node-192-168-72-119 data]# echo $HADOOP_CLASSPATH
/data/hadoop-3.3.6/etc/hadoop:/data/hadoop-3.3.6/share/hadoop/common/lib/*:/data/hadoop-3.3.6/share/hadoop/common/*:/data/hadoop-3.3.6/share/hadoop/hdfs:/data/hadoop-3.3.6/share/hadoop/hdfs/lib/*:/data/hadoop-3.3.6/share/hadoop/hdfs/*:/data/hadoop-3.3.6/share/hadoop/mapreduce/*:/data/hadoop-3.3.6/share/hadoop/yarn:/data/hadoop-3.3.6/share/hadoop/yarn/lib/*:/data/hadoop-3.3.6/share/hadoop/yarn/*
[root@localhost hadoop-3.3.6]# vim etc/hadoop/mapred-site.xml 
<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
    <property>
    <name>yarn.app.mapreduce.am.env</name>
    <value>HADOOP_MAPRED_HOME=/data/hadoop-3.3.6</value>
  </property>

  <property>
    <name>mapreduce.map.env</name>
    <value>HADOOP_MAPRED_HOME=/data/hadoop-3.3.6</value>
  </property>

  <property>
    <name>mapreduce.reduce.env</name>
    <value>HADOOP_MAPRED_HOME=/data/hadoop-3.3.6</value>
  </property>
</configuration>

[root@localhost hadoop-3.3.6]# vim etc/hadoop/yarn-site.xml
<configuration>

<!-- Site specific YARN configuration properties -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>

  <property>
    <name>yarn.application.classpath</name>
    <value>/data/hadoop-3.3.6/etc/hadoop:/data/hadoop-3.3.6/share/hadoop/common/lib/*:/data/hadoop-3.3.6/share/hadoop/common/*:/data/hadoop-3.3.6/share/hadoop/hdfs:/data/hadoop-3.3.6/share/hadoop/hdfs/lib/*:/data/hadoop-3.3.6/share/hadoop/hdfs/*:/data/hadoop-3.3.6/share/hadoop/mapreduce/*:/data/hadoop-3.3.6/share/hadoop/yarn:/data/hadoop-3.3.6/share/hadoop/yarn/lib/*:/data/hadoop-3.3.6/share/hadoop/yarn/*</value>
  </property>
</configuration>



##格式化dhfs文件系统
[root@localhost data]# hdfs namenode -format
###关闭防火墙
[root@localhost hadoop-3.3.6]# systemctl stop firewalld
[root@localhost hadoop-3.3.6]# systemctl disable firewalld
########启动所有服务
[root@localhost hadoop-3.3.6]# ./sbin/start-all.sh 
Starting namenodes on [node-192-168-72-119.test.com]
Last login: Thu Dec 25 15:19:15 CST 2025 from 192.168.72.1 on pts/1
node-192-168-72-119.test.com: Warning: Permanently added 'node-192-168-72-119.test.com,192.168.72.119' (ECDSA) to the list of known hosts.
Starting datanodes
Last login: Thu Dec 25 15:35:38 CST 2025 on pts/0
localhost: Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts.
Starting secondary namenodes [node-192-168-72-119.test.com]
Last login: Thu Dec 25 15:35:41 CST 2025 on pts/0
2025-12-25 15:36:00,113 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Starting resourcemanager
Last login: Thu Dec 25 15:35:55 CST 2025 on pts/0
Starting nodemanagers
Last login: Thu Dec 25 15:36:02 CST 2025 on pts/0
##########查看
[root@localhost hadoop-3.3.6]# jps
22999 SecondaryNameNode
23416 NodeManager
22650 NameNode
23258 ResourceManager
22779 DataNode
23565 Jps

hdfs页面 http://Hadoop机器IP:50070/ yarn页面 http://Hadoop机器IP:8088/

hive单机版部署

hive3版本不支持jdk8以上版本。需要安装jdk8

mariadb安装

[root@node-192-168-72-119 ~]# yum -y install mariadb-server mysql-connector-java
[root@node-192-168-72-119 ~]# systemctl start mariadb
[root@node-192-168-72-119 ~]# systemctl enable mariadb
Created symlink from /etc/systemd/system/multi-user.target.wants/mariadb.service to /usr/lib/systemd/system/mariadb.service.
[root@node-192-168-72-119 ~]# mysql_secure_installation

NOTE: RUNNING ALL PARTS OF THIS SCRIPT IS RECOMMENDED FOR ALL MariaDB
      SERVERS IN PRODUCTION USE!  PLEASE READ EACH STEP CAREFULLY!

In order to log into MariaDB to secure it, we'll need the current
password for the root user.  If you've just installed MariaDB, and
you haven't set the root password yet, the password will be blank,
so you should just press enter here.

Enter current password for root (enter for none): 
OK, successfully used password, moving on...

Setting the root password ensures that nobody can log into the MariaDB
root user without the proper authorisation.

Set root password? [Y/n] Y
New password: 
Re-enter new password: 
Password updated successfully!
Reloading privilege tables..
 ... Success!


By default, a MariaDB installation has an anonymous user, allowing anyone
to log into MariaDB without having to have a user account created for
them.  This is intended only for testing, and to make the installation
go a bit smoother.  You should remove them before moving into a
production environment.

Remove anonymous users? [Y/n] Y
 ... Success!

Normally, root should only be allowed to connect from 'localhost'.  This
ensures that someone cannot guess at the root password from the network.

Disallow root login remotely? [Y/n] n
 ... skipping.

By default, MariaDB comes with a database named 'test' that anyone can
access.  This is also intended only for testing, and should be removed
before moving into a production environment.

Remove test database and access to it? [Y/n] Y
 - Dropping test database...
 ... Success!
 - Removing privileges on test database...
 ... Success!

Reloading the privilege tables will ensure that all changes made so far
will take effect immediately.

Reload privilege tables now? [Y/n] Y
 ... Success!

Cleaning up...

All done!  If you've completed all of the above steps, your MariaDB
installation should now be secure.

Thanks for using MariaDB!

hive下载解压，下载地址https://archive.apache.org/dist/hive/hive-3.1.3/apache-hive-3.1.3-bin.tar.gz

[root@node-192-168-72-119 ~]# cd /data/
[root@node-192-168-72-119 data]# wget https://archive.apache.org/dist/hive/hive-3.1.3/apache-hive-3.1.3-bin.tar.gz  
[root@node-192-168-72-119 data]# tar xf apache-hive-3.1.3-bin.tar.gz
[root@node-192-168-72-119 data]# mv apache-hive-3.1.3-bin  hive
[root@node-192-168-72-119 data]# cp /usr/share/java/mysql-connector-java.jar /data/hive/lib/

[root@node-192-168-72-119 data]# vim /etc/profile
export HIVE_HOME=/data/hive
export PATH=$PATH:$HIVE_HOME/bin
[root@node-192-168-72-119 data]# source !$

配置hive,注意使用mysql数据库和mariadb数据库jdbc有区别。

[root@node-192-168-72-119 data]# vim hive/conf/hive-env.sh
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.412.b08-1.el7_9.x86_64
export HADOOP_HOME=/data/hadoop-3.3.6
export HIVE_CONF_DIR=/data/hive/conf

[root@node-192-168-72-119 data]# cp /data/hive/conf/{hive-default.xml.template,hive-site.xml}
cat >/data/hive/conf/hive-site.xml<<'EOF'
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
  <property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://node-192-168-72-119.test.com:3306/hive?createDatabaseIfNotExist=true</value>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>root</value>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>123456</value>
  </property>

  <property>
    <name>hive.exec.local.scratchdir</name>
    <value>/tmp/hive</value>
  </property>

  <property>
    <name>hive.downloaded.resources.dir</name>
    <value>/tmp/${hive.session.id}_resources</value>
  </property>

  <property>
    <name>hive.querylog.location</name>
    <value>/tmp/hive</value>
  </property>

  <property>
    <name>hive.server2.logging.operation.log.location</name>
    <value>/tmp/hive/operation_logs</value>
  </property>

  <property>
    <name>hive.server2.enable.doAs</name>
    <value>FALSE</value>
  </property>  
</configuration>
EOF

#########处理hadoop与hive guava 版本不一致问题
[root@node-192-168-72-119 data]# mv /data/hive/lib/guava 版本不一致问题-19.0.jar{,.bak}
[root@node-192-168-72-119 data]# cp /data/hadoop-3.3.6/share/hadoop/hdfs/lib/guava-27.0-jre.jar /data/hive/lib/

hadoop fs -mkdir /tmp
hadoop fs -mkdir -p /user/hive/warehouse
hadoop fs -chmod g+w /tmp
hadoop fs -chmod g+w /user/hive/warehouse

初始化mariadb数据库

[root@node-192-168-72-119 data]# schematool -dbType mysql -initSchema

hiveserver2部署

cat > /etc/systemd/system/hive-meta.service <<EOF
[Unit] 
Description=Hive metastore 
After=network.target 

[Service] 
User=root
Group=root
ExecStart=/data/hive/bin/hive --service metastore 

[Install] 
WantedBy=multi-user.target
EOF
[root@node-192-168-72-119 data]# systemctl enable --now hive-meta
[root@node-192-168-72-119 data]# systemctl status hive-meta
● hive-meta.service - Hive metastore
   Loaded: loaded (/etc/systemd/system/hive-meta.service; enabled; vendor preset: disabled)
   Active: active (running) since Fri 2025-12-26 10:06:22 CST; 24s ago
 Main PID: 26992 (java)
   CGroup: /system.slice/hive-meta.service
           └─26992 /usr/lib/jvm/java-11-openjdk-11.0.23.0.9-2.el7_9.x86_64/bin/java -Dproc_jar -Dproc_metastore -Dlog4j2.formatMsgNoLookups=tru...

Dec 26 10:06:22 node-192-168-72-119.test.com systemd[1]: Started Hive metastore.
Dec 26 10:06:23 node-192-168-72-119.test.com hive[26992]: 2025-12-26 10:06:23: Starting Hive Metastore Server
Dec 26 10:06:23 node-192-168-72-119.test.com hive[26992]: SLF4J: Class path contains multiple SLF4J bindings.
Dec 26 10:06:23 node-192-168-72-119.test.com hive[26992]: SLF4J: Found binding in [jar:file:/data/hive/lib/log4j-slf4j-impl-2.17.1.jar!/o...class]
Dec 26 10:06:23 node-192-168-72-119.test.com hive[26992]: SLF4J: Found binding in [jar:file:/data/hadoop-3.3.6/share/hadoop/common/lib/sl...class]
Dec 26 10:06:23 node-192-168-72-119.test.com hive[26992]: SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
Dec 26 10:06:24 node-192-168-72-119.test.com hive[26992]: SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Hint: Some lines were ellipsized, use -l to show in full.

#######查看监听端口
[root@node-192-168-72-119 data]# netstat -tunlp | grep 9083
tcp6       0      0 :::9083                 :::*                    LISTEN      26992/java
###systemd管理hive-server2服务
cat > /etc/systemd/system/hive-server2.service <<EOF
[Unit] 
Description=hive-server2
After=network.target 

[Service] 
User=root
Group=root
ExecStart=/data/hive/bin/hive --service hiveserver2

[Install] 
WantedBy=multi-user.target
EOF

###启动hive-server2服务，并设为开机启动
[root@node-192-168-72-119 data]# systemctl enable --now hive-server2
Created symlink from /etc/systemd/system/multi-user.target.wants/hive-server2.service to /etc/systemd/system/hive-server2.service.
###查看服务运行状态
[root@node-192-168-72-119 data]# systemctl status hive-server2.service 
● hive-server2.service - hive-server2
   Loaded: loaded (/etc/systemd/system/hive-server2.service; enabled; vendor preset: disabled)
   Active: active (running) since Fri 2025-12-26 10:08:20 CST; 29s ago
 Main PID: 27173 (java)
   CGroup: /system.slice/hive-server2.service
           └─27173 /usr/lib/jvm/java-11-openjdk-11.0.23.0.9-2.el7_9.x86_64/bin/java -Dproc_jar -Dproc_hiveserver2 -Dlog4j2.formatMsgNoLookups=t...

Dec 26 10:08:20 node-192-168-72-119.test.com systemd[1]: Started hive-server2.
Dec 26 10:08:22 node-192-168-72-119.test.com hive[27173]: which: no hbase in (/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin)
Dec 26 10:08:23 node-192-168-72-119.test.com hive[27173]: 2025-12-26 10:08:23: Starting HiveServer2
Dec 26 10:08:23 node-192-168-72-119.test.com hive[27173]: SLF4J: Class path contains multiple SLF4J bindings.
Dec 26 10:08:23 node-192-168-72-119.test.com hive[27173]: SLF4J: Found binding in [jar:file:/data/hive/lib/log4j-slf4j-impl-2.17.1.jar!/o...class]
Dec 26 10:08:23 node-192-168-72-119.test.com hive[27173]: SLF4J: Found binding in [jar:file:/data/hadoop-3.3.6/share/hadoop/common/lib/sl...class]
Dec 26 10:08:23 node-192-168-72-119.test.com hive[27173]: SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
Dec 26 10:08:23 node-192-168-72-119.test.com hive[27173]: SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Dec 26 10:08:29 node-192-168-72-119.test.com hive[27173]: Hive Session ID = 71731bc8-4211-48ea-a2a2-174a18bda9c2
Hint: Some lines were ellipsized, use -l to show in full.

使用hive命令

[root@node-192-168-72-119 data]# hive
which: no hbase in (/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/lib/jvm/java-11-openjdk-11.0.23.0.9-2.el7_9.x86_64/bin:/data/hadoop-3.3.6/bin:/data/hadoop-3.3.6/sbin:/root/bin:/usr/lib/jvm/java-11-openjdk-11.0.23.0.9-2.el7_9.x86_64/bin:/data/hadoop-3.3.6/bin:/data/hadoop-3.3.6/sbin:/data/hive/bin:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.412.b08-1.el7_9.x86_64//bin:/data/hadoop-3.3.6/bin:/data/hadoop-3.3.6/sbin:/data/hive/bin)
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/data/hive/lib/log4j-slf4j-impl-2.17.1.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/data/hadoop-3.3.6/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Hive Session ID = a89a4240-f8dd-4a7c-a8b5-cd44943c0804

Logging initialized using configuration in jar:file:/data/hive/lib/hive-common-3.1.3.jar!/hive-log4j2.properties Async: true
Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases.Hive Session ID = 2cdcae0d-e6da-4e75-a76f-03620ed83c9d

hive> show databases;
OK
default
Time taken: 0.827 seconds, Fetched: 1 row(s)
hive> use default;
OK
Time taken: 0.031 seconds
hive> show tables;
OK
Time taken: 0.046 seconds
hive> exit;

使用beeline连接hiveserver2

[root@node-192-168-72-119 data]# beeline -u jdbc:hive2://node-192-168-72-119.test.com:10000
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/data/hive/lib/log4j-slf4j-impl-2.17.1.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/data/hadoop-3.3.6/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Connecting to jdbc:hive2://node-192-168-72-119.test.com:10000
Connected to: Apache Hive (version 3.1.3)
Driver: Hive JDBC (version 3.1.3)
Transaction isolation: TRANSACTION_REPEATABLE_READ
Beeline version 3.1.3 by Apache Hive
0: jdbc:hive2://node-192-168-72-119.test.com:> show databases;
INFO  : Compiling command(queryId=root_20251226103845_2d338a4b-dd97-4f0e-9679-1ebe40a99c6f): show databases
INFO  : Concurrency mode is disabled, not creating a lock manager
INFO  : Semantic Analysis Completed (retrial = false)
INFO  : Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:database_name, type:string, comment:from deserializer)], properties:null)
INFO  : Completed compiling command(queryId=root_20251226103845_2d338a4b-dd97-4f0e-9679-1ebe40a99c6f); Time taken: 0.794 seconds
INFO  : Concurrency mode is disabled, not creating a lock manager
INFO  : Executing command(queryId=root_20251226103845_2d338a4b-dd97-4f0e-9679-1ebe40a99c6f): show databases
INFO  : Starting task [Stage-0:DDL] in serial mode
INFO  : Completed executing command(queryId=root_20251226103845_2d338a4b-dd97-4f0e-9679-1ebe40a99c6f); Time taken: 0.078 seconds
INFO  : OK
INFO  : Concurrency mode is disabled, not creating a lock manager
+----------------+
| database_name  |
+----------------+
| default        |
+----------------+
1 row selected (1.297 seconds)
0: jdbc:hive2://node-192-168-72-119.test.com:> !quit
Closing: 0: jdbc:hive2://node-192-168-72-119.test.com:10000

hive建表，插入数据

CREATE TABLE employees (
    employee_id INT,
    name STRING,
    department STRING
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
INSERT INTO TABLE employees VALUES
(2, 'Jane Smith', 'Marketing'),
(3, 'Alice Johnson', 'HR');
登陆评论: 使用GITHUB登陆