others linux服务器运维 django3 监控 k8s golang 数据库 大数据 前端 devops 理论基础 java oracle 运维日志

doris 对接 hive 湖仓一体

访问量:17 创建时间:2025-12-27

doris安装参考:https://www.linuxsre.cn/wiki/dbserver/350 hive安装参考: https://www.linuxsre.cn/wiki/hadoop/365

[root@localhost ~]# mysql -h192.168.72.106 -P9030 -uroot -p
Enter password: 
MySQL [(none)]> show CATALOGS;
+-----------+-------------+----------+-----------+------------+----------------+------------------------+
| CatalogId | CatalogName | Type     | IsCurrent | CreateTime | LastUpdateTime | Comment                |
+-----------+-------------+----------+-----------+------------+----------------+------------------------+
|         0 | internal    | internal | Yes       | NULL       | NULL           | Doris internal catalog |
+-----------+-------------+----------+-----------+------------+----------------+------------------------+
1 row in set (0.01 sec)
########### 创建hive CATALOG映射。通过 Apache Doris 创建 Hive 表或写入数据,需要在 Catalog 属性中显式增加 fs.defaultFS 属性。如果创建 Catalog 仅用于查询,则该参数可以省略。
CREATE CATALOG hive_hms_hdfs_test_catalog PROPERTIES (
    'type' = 'hms',
    'hive.metastore.uris' = 'thrift://192.168.72.119:9083',
    'fs.defaultFS' = 'hdfs://192.168.72.119:50070',
    'hadoop.username' = 'doris'
);



MySQL [(none)]> SWITCH hive_hms_hdfs_test_catalog;
Query OK, 0 rows affected (0.00 sec)

MySQL [(none)]> show databases;
+--------------------+
| Database           |
+--------------------+
| default            |
| information_schema |
| mysql              |
+--------------------+
3 rows in set (15.17 sec)

MySQL [(none)]> use default;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
MySQL [default]> show tables;
+-------------------+
| Tables_in_default |
+-------------------+
| employees         |
+-------------------+
1 row in set (0.00 sec)

MySQL [default]> select * from employees;
ERROR 1105 (HY000): errCode = 2, detailMessage = get file split failed for table: employees, err: java.lang.RuntimeException: errCode = 2, detailMessage = Failed to get dfs FileSystem for java.net.UnknownHostException: node-192-168-72-119.test.com
#######在doris集群配置hadoop域名解析,所有主机都需要修改
vim /etc/hosts
192.168.72.119  node-192-168-72-119.test.com
###############################################继续在doris中查询
MySQL [default]> select * from employees;
+-------------+---------------+------------+
| employee_id | name          | department |
+-------------+---------------+------------+
|           2 | Jane Smith    | Marketing  |
|           3 | Alice Johnson | HR         |
|           2 | Jane Smith    | Marketing  |
|           3 | Alice Johnson | HR         |
|           2 | Jane Smith    | Marketing  |
|           3 | Alice Johnson | HR         |
+-------------+---------------+------------+
6 rows in set (1 min 19.52 sec)
########插入数据
MySQL [default]> INSERT INTO  employees VALUES (21, 'Jane Smith', 'Marketing'), (32, 'Alice Johnson', 'HR');
Query OK, 2 rows affected (1 min 46.16 sec)
{'status':'COMMITTED', 'txnId':'10357'}
########建库
CREATE DATABASE IF NOT EXISTS hive_hms_hdfs_test_catalog.hive_db;
######建表。Doris 支持在 Hive 中创建分区或非分区表。
-- Create unpartitioned hive table
CREATE TABLE unpartitioned_table (
  `col1` BOOLEAN COMMENT 'col1',
  `col2` INT COMMENT 'col2',
  `col3` BIGINT COMMENT 'col3',
  `col4` CHAR(10) COMMENT 'col4',
  `col5` FLOAT COMMENT 'col5',
  `col6` DOUBLE COMMENT 'col6',
  `col7` DECIMAL(9,4) COMMENT 'col7',
  `col8` VARCHAR(11) COMMENT 'col8',
  `col9` STRING COMMENT 'col9'
)  ENGINE=hive
PROPERTIES (
  'file_format'='parquet'
);

-- Create partitioned hive table
-- The partition columns must be in table's column definition list
CREATE TABLE partition_table (
  `col1` BOOLEAN COMMENT 'col1',
  `col2` INT COMMENT 'col2',
  `col3` BIGINT COMMENT 'col3',
  `col4` DECIMAL(2,1) COMMENT 'col4',
  `pt1` VARCHAR COMMENT 'pt1',
  `pt2` VARCHAR COMMENT 'pt2'
)  ENGINE=hive
PARTITION BY LIST (pt1, pt2) ()
PROPERTIES (
  'file_format'='orc',
  'compression'='zlib'
);

-- Create text format table(Since 2.1.7 & 3.0.3)
CREATE TABLE text_table (
    `id` INT,
    `name` STRING
) PROPERTIES (
    'file_format'='text',
    'compression'='gzip',
    'field.delim'='\t',
    'line.delim'='\n',
    'collection.delim'=';',
    'mapkey.delim'=':',
    'serialization.null.format'='\\N',
    'escape.delim'='\\'
);
登陆评论: 使用GITHUB登陆