others linux服务器运维 django3 监控 k8s golang 数据库 大数据 前端 devops 理论基础 java oracle 运维日志

ShardingSphere proxy 分库分表

访问量:1914 创建时间:2021-01-28

ShardingSphere主要产品ShardingJDBC、ShardingProxy、ShardingSidecar、UI、Scaling。

分库分表优点 缺点
分库分表相对单库单表扩展性好,单表数据量可控,查询性能提高 分库分表后分布式事务存在不一致的问题。对sql查询语句有一定限制。集群维护管理成本提高。

分库分表示例

ShardingProxy快速使用

简介:ShardingProxy提供分库分表的功能,是一个独立部署的服务端,提供统一的数据库代理服务。目前只支持Mysql和PostgreSQL.

环境 角色
192.168.171.128 mysql server ;sharding-proxy
192.168.171.129 mysql server

本实验使用2台mysql实例,同时在128机器部署sharding-proxy

安装mariadb(128和129都安装)

[root@ ~]# yum -y install mariadb mariadb-server
[root@ ~]# systemctl start mariadb
[root@ ~]# systemctl enable mariadb
[root@ ~]# mysql_secure_installation
Enter current password for root (enter for none): 
Set root password? [Y/n] Y
New password: 
Re-enter new password: 
Remove anonymous users? [Y/n] Y
Disallow root login remotely? [Y/n] n
Remove test database and access to it? [Y/n] Y
Reload privilege tables now? [Y/n] Y
[root@ ~]# vim /etc/my.cnf
#设置字符集
character-set-server=utf8
[root@cephadm ~]# systemctl restart mariadb
[root@cephadm ~]# mysql -uroot -p
MariaDB [(none)]> show variables like "%character%";
+--------------------------+----------------------------+
| Variable_name            | Value                      |
+--------------------------+----------------------------+
| character_set_client     | utf8                       |
| character_set_connection | utf8                       |
| character_set_database   | utf8                       |
| character_set_filesystem | binary                     |
| character_set_results    | utf8                       |
| character_set_server     | utf8                       |
| character_set_system     | utf8                       |
| character_sets_dir       | /usr/share/mysql/charsets/ |
+--------------------------+----------------------------+

创建数据库128上test_db_order_0和129上test_db_order_1和test_db_order_2

#128机器
[root@cephadm ~]# mysql -uroot -p
Enter password: 
MariaDB [(none)]> create database test_db_order_0;
MariaDB [(none)]> GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456';
#129机器
[root@node1 ~]# mysql -uroot -p
Enter password: 
MariaDB [(none)]> create database test_db_order_1;
MariaDB [(none)]> create database test_db_order_2;
MariaDB [(none)]> GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456';

在128机器,下载解压配置sharding-proxy(提前安装jdk,jdk安装过程省略)

[root@cephadm ~]# tar xf apache-shardingsphere-5.0.0-alpha-shardingsphere-proxy-bin.tar.gz
[root@cephadm ~]# mv apache-shardingsphere-5.0.0-alpha-shardingsphere-proxy-bin shardingsphere-proxy
[root@cephadm ~]# wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.47/mysql-connector-java-5.1.47.jar
[root@cephadm ~]# mv mysql-connector-java-5.1.47.jar shardingsphere-proxy/lib/
[root@cephadm ~]# cd shardingsphere-proxy

配置文件:

[root@cephadm shardingsphere-proxy]# vim conf/config-sharding.yaml 
# 逻辑数据源名称
schemaName: sharding_db
# 数据源的通用配置(如果每个库的username和password不同可以在dataSources单独配置)
dataSourceCommon:
  username: root
  password: 123456
  connectionTimeoutMilliseconds: 30000
  idleTimeoutMilliseconds: 60000
  maxLifetimeMilliseconds: 1800000
  maxPoolSize: 50
  minPoolSize: 1
  maintenanceIntervalMilliseconds: 30000
dataSources:
  ds_0:
    url: jdbc:mysql://192.168.171.128:3306/test_db_order_0?serverTimezone=UTC&useSSL=false
  ds_1:
    url: jdbc:mysql://192.168.171.129:3306/test_db_order_1?serverTimezone=UTC&useSSL=false
  ds_2:
    url: jdbc:mysql://192.168.171.129:3306/test_db_order_2?serverTimezone=UTC&useSSL=false

rules:
- !SHARDING
  # 数据分片规则配置
  tables:
    #逻辑表名称
    t_order:
      ## 由数据源名 + 表名组成(参考Inline语法规则)
      actualDataNodes: ds_${0..2}.t_order_${0..1}
      # 分表策略,同分库策略
      tableStrategy:
        standard:
          shardingColumn: order_id
          shardingAlgorithmName: t_order_inline
      # 分布式序列策略
  # 默认数据库分片策略
  defaultDatabaseStrategy:
    standard:
      shardingColumn: user_id
      shardingAlgorithmName: database_inline
  # 默认表分片策略
  defaultTableStrategy:
    none:
  # 分片算法配置
  shardingAlgorithms:
    # 分片算法名称
    database_inline:
      # 分片算法类型
      type: INLINE
      # 分片算法属性配置
      props:
        algorithm-expression: ds_${user_id % 3} # 使用简单的用 user 表的 id 字段对数据库节点数 n 求余规则来分片。
    t_order_inline:
      type: INLINE
      props:
        algorithm-expression: t_order_${order_id % 2}

[root@cephadm shardingsphere-proxy]# vim conf/server.yaml 
authentication:
  users:
    ## 自定义用户名
    root:
      password: root
    ## 自定义用户名
    sharding:
      ## 自定义用户的密码
      password: sharding 
      ## 自定义用户的访问权限,多个库用逗号分隔
      authorizedSchemas: sharding_db

props:
  max-connections-size-per-query: 1
  acceptor-size: 16  # The default value is available processors count * 2.
  executor-size: 16  # Infinite by default.
  proxy-frontend-flush-threshold: 128  # The default value is 128.
    # LOCAL: Proxy will run with LOCAL transaction.
    # XA: Proxy will run with XA transaction.
    # BASE: Proxy will run with B.A.S.E transaction.
  proxy-transaction-type: LOCAL
  proxy-opentracing-enabled: false
  proxy-hint-enabled: false
  query-with-cipher-column: true
  sql-show: false
  check-table-metadata-enabled: false

启动proxy

[root@cephadm shardingsphere-proxy]# ./bin/start.sh 3307
Starting the ShardingSphere-Proxy ...
The port is 3307
./bin/start.sh: line 57: fg: no job control
The classpath is .:/root/shardingsphere-proxy/lib/*:/root/shardingsphere-proxy/ext-lib/*
Please check the STDOUT file: /root/shardingsphere-proxy/logs/stdout.log
[root@cephadm shardingsphere-proxy]# ss -an | grep 3307
tcp    LISTEN     0      128       *:3307                  *:* 

登陆,建表

[root@cephadm shardingsphere-proxy]# mysql  -usharding -p -P3307 -h192.168.171.128
Enter password: sharding
MariaDB [(none)]> show databases;
+-------------+
| Database    |
+-------------+
| sharding_db |
+-------------+
1 row in set (0.11 sec)
MariaDB [sharding_db]> show tables;
Empty set (0.01 sec)
MariaDB [sharding_db]> CREATE TABLE IF NOT EXISTS `t_order` (
    ->     `order_id` int(11) NOT NULL,
    ->     `user_id` int(11) NOT NULL,
    ->     PRIMARY KEY (`order_id`)
    -> ) ENGINE=InnoDB AUTO_INCREMENT=1;
Query OK, 0 rows affected (2.36 sec)
MariaDB [sharding_db]> show tables;
+---------------------------+
| Tables_in_test_db_order_0 |
+---------------------------+
| t_order                   |
+---------------------------+
1 row in set (0.02 sec)

查看底层数据库128与129状态。

#128数据库
[root@cephadm ~]# mysql -uroot -p123456
MariaDB [(none)]> show databases ;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test_db_order_0    |
+--------------------+
4 rows in set (0.06 sec)
MariaDB [(none)]> use test_db_order_0
Database changed
MariaDB [test_db_order_0]> show tables;
+---------------------------+
| Tables_in_test_db_order_0 |
+---------------------------+
| t_order_0                 |
| t_order_1                 |
+---------------------------+
2 rows in set (0.00 sec)
#129数据库
[root@node1 ~]# mysql -uroot -p123456 -h192.168.171.129
MariaDB [(none)]> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test_db_order_1    |
| test_db_order_2    |
+--------------------+
5 rows in set (0.06 sec)
MariaDB [(none)]> use test_db_order_2
Database changed
MariaDB [test_db_order_2]> show tables;
+---------------------------+
| Tables_in_test_db_order_2 |
+---------------------------+
| t_order_0                 |
| t_order_1                 |
+---------------------------+
2 rows in set (0.00 sec)

插入数据测试(通过mysql命令行连接的proxy 3307端口插入)

MariaDB [sharding_db]> show tables;
+---------------------------+
| Tables_in_test_db_order_0 |
+---------------------------+
| t_order                   |
+---------------------------+
1 row in set (0.00 sec)
#####下面是插入命令
INSERT INTO t_order (order_id, user_id) VALUES (0, 0);
INSERT INTO t_order (order_id, user_id) VALUES (1, 0);
INSERT INTO t_order (order_id, user_id) VALUES (2, 1);
INSERT INTO t_order (order_id, user_id) VALUES (3, 2);
INSERT INTO t_order (order_id, user_id) VALUES (4, 1);
INSERT INTO t_order (order_id, user_id) VALUES (5, 3);
INSERT INTO t_order (order_id, user_id) VALUES (6, 3);
INSERT INTO t_order (order_id, user_id) VALUES (7, 4);
INSERT INTO t_order (order_id, user_id) VALUES (8, 5);
INSERT INTO t_order (order_id, user_id) VALUES (9, 6);
MariaDB [sharding_db]> select * from t_order;
+----------+---------+
| order_id | user_id |
+----------+---------+
|        0 |       0 |
|        6 |       3 |
|        1 |       0 |
|        5 |       3 |
|        9 |       6 |
|        2 |       1 |
|        4 |       1 |
|        7 |       4 |
|        8 |       5 |
|        3 |       2 |
+----------+---------+
10 rows in set (0.09 sec)

查看底层129机器mysql状态:

MariaDB [test_db_order_2]> select * from t_order_1;
+----------+---------+
| order_id | user_id |
+----------+---------+
|        3 |       2 |
+----------+---------+
1 row in set (0.00 sec)

MariaDB [test_db_order_2]> select * from t_order_0;
+----------+---------+
| order_id | user_id |
+----------+---------+
|        8 |       5 |
+----------+---------+
1 row in set (0.00 sec)

MariaDB [test_db_order_2]> use test_db_order_1
Database changed
MariaDB [test_db_order_1]> select * from t_order_0;
+----------+---------+
| order_id | user_id |
+----------+---------+
|        2 |       1 |
|        4 |       1 |
+----------+---------+
2 rows in set (0.00 sec)

MariaDB [test_db_order_1]> select * from t_order_1;
+----------+---------+
| order_id | user_id |
+----------+---------+
|        7 |       4 |
+----------+---------+
1 row in set (0.00 sec)

至此sharding-proxy的分库分表示例演示完成。

登陆评论: 使用GITHUB登陆