diff --git a/README.md b/README.md index 56656e4cbd33f741785980d42c378e4e0a3fc7cc..44c926f77fc48a0b48a537291f198fb70a33f45b 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,22 @@ This repo stores openanolis storage sig POC feature test scripts POC test script for MySQL 16k atomic write feature. It uses sysbench to run multiple tests with different benchmark types and threads, in either fixed run time or fixed run events. + +## test-mysql-auxiliary-scripts_optional(Optional) + +Non-essential auxiliary tools + +### 01_switch-scenes/switch-scenes.sh + +Auxiliary script for test-mysql.sh. It can automatically complete the +operation of "test-mysql.sh" in six scenes and output log. + +Six scenes includes (open_double_write + close_double_write + automic_write)*(time + event) + +Crudini is the require of switch-scenes.sh. And crudini-0.9.3-1.uelc20.01.noarch.rpm could be directly installed in "Anolis8.4". + +### 02_data-processing/data_processing.py + +data_processing.py can process the test data from log which is +genetated by 01_switch-scenes/switch-scenes.sh to specified excel with +template.xlsx. diff --git a/test-mysql-auxiliary-scripts_optional/01_switch-scenes/crudini-0.9.3-1.uelc20.01.noarch.rpm b/test-mysql-auxiliary-scripts_optional/01_switch-scenes/crudini-0.9.3-1.uelc20.01.noarch.rpm new file mode 100644 index 0000000000000000000000000000000000000000..1434f52f27fc29d46eb9cb2aa11f167851059b0c Binary files /dev/null and b/test-mysql-auxiliary-scripts_optional/01_switch-scenes/crudini-0.9.3-1.uelc20.01.noarch.rpm differ diff --git a/test-mysql-auxiliary-scripts_optional/01_switch-scenes/switch-scenes.sh b/test-mysql-auxiliary-scripts_optional/01_switch-scenes/switch-scenes.sh new file mode 100755 index 0000000000000000000000000000000000000000..d75765d23d30bd34db1fbf39d2bc011bc061af2a --- /dev/null +++ b/test-mysql-auxiliary-scripts_optional/01_switch-scenes/switch-scenes.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Program: +# Automatically complete the operation of "test-mysql.sh" in six scenes and output log. +# Six scenes includes (open_double_write + close_double_write + automic_write)*(time + event) +# Requires: +# crudini needs to be installed. You can directly use rpm from uos in anolis8.4 +# such as: "yum install python3-iniparse && rpm -ivh crudini-0.9.3-1.uelc20.01.noarch.rpm" + +mysql_dev=/dev/vdb +mysql_conf_path=/etc/my.cnf.d/mariadb-server.cnf +# The section where innodb_doublewrite is +section=mysqld +test_script_time_path=/root/test-mysql-time.sh +test_script_event_path=/root/test-mysql-event.sh +test_log_dir=/root/log220316-mariadb10.3.28-512threads +service_name=mariadb.service + +function init() +{ + chmod +x ${test_script_time_path} ${test_script_event_path} + [ -d ${test_log_dir} ] || mkdir ${test_log_dir} +} + +function reset_env() +{ + automic=$1 + + mysql_process_num=$(ps -ef | grep -c mysql) + times=0 + while [ $mysql_process_num -ne 1 ]; do + systemctl stop ${service_name} + echo "execute 'stop service'" + sleep 10 + mysql_process=$(ps -ef | grep mysql) + echo -e "After sleep 10, mysql process status:\n ${mysql_process}" + mysql_process_num=$(ps -ef | grep -c mysql) + times=$((times+1)) + if [ $times -eq 10 ]; then + echo "'stop service' has been executed 10 times, stop script. Please check it!" + exit 1 + fi + done + umount ${mysql_dev} + sudo mkfs.xfs -f -m reflink=1 ${mysql_dev} + sudo mount ${mysql_dev} /var/lib/mysql + sudo chown -R mysql:mysql /var/lib/mysql + + if [ $automic -eq 1 ]; then + sudo chattr +c /var/lib/mysql + fi +} + +function run_mysql() +{ + systemctl start ${service_name} +} + +function run_test-mysql() +{ + automic=$1 + innodb_doublewrite=$2 + log_name=$3 + + crudini --set ${mysql_conf_path} ${section} innodb_doublewrite ${innodb_doublewrite} + + reset_env $automic + run_mysql + ${test_script_time_path} | tee -a ${test_log_dir}/${log_name}.log + + reset_env $automic + run_mysql + ${test_script_event_path} | tee -a ${test_log_dir}/${log_name}_event.log +} + +function run_tests() +{ + run_test-mysql 1 0 "03_atomic" + run_test-mysql 0 0 "02_close_doublewrite" + run_test-mysql 0 1 "01_open_doublewrite" +} + +init +run_tests + +exit 0 + diff --git a/test-mysql-auxiliary-scripts_optional/02_data-processing/data_processing.py b/test-mysql-auxiliary-scripts_optional/02_data-processing/data_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..eecf33e50a568e2cb7d885d256bebc9ce2140465 --- /dev/null +++ b/test-mysql-auxiliary-scripts_optional/02_data-processing/data_processing.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import re +from openpyxl import load_workbook + + +logs_path = "./log220314-mariadb10.3.28" + +JUMP_INTERVAL = 4 +JUMP_ROW = 16 +# [log_path, col, row_num, jump_row] +# start to write from line +write_arguments = [[f"{logs_path}/01_open_doublewrite.log", 4, 5, JUMP_ROW], + [f"{logs_path}/02_close_doublewrite.log", 5, 5, JUMP_ROW], + [f"{logs_path}/03_atomic.log", 6, 5, JUMP_ROW], + [f"{logs_path}/01_open_doublewrite_event.log", 4, 15, JUMP_ROW], + [f"{logs_path}/02_close_doublewrite_event.log", 5, 15, JUMP_ROW], + [f"{logs_path}/03_atomic_event.log", 6, 15, JUMP_ROW]] + +excel_path = "./template.xlsx" +work_sheet = "mariadb_data_32threads" +result_excel_path = "./test220314-mariadb10.3.28.xlsx" +# the device mounted to mysql data dir(/var/lib/mysql) +device = "vdd" + + +def read_data(log_path): + regex_tps = re.compile(r'transactions:\s+\d+\s+\((\d+\.\d{2}) per sec') + regex_kB_wrtn = re.compile(device + r'\s+\d+\.\d{2}\s+\d+\.\d{2}\s+\d+\.\d{2}\s+\d+\s+(\d+)') + + with open(log_path) as f: + log = f.read() + tps_list = regex_tps.findall(log) + kB_wrtn_list = regex_kB_wrtn.findall(log) + data_writed_list = [] + for i in range(0, len(kB_wrtn_list), 2): + data_writed_list.append(int(kB_wrtn_list[i+1]) - int(kB_wrtn_list[i])) + + return tps_list, data_writed_list + + +def write_excel(ws, tps_list, data_writed_list, col, row_num, jump_row): + count = 0 + + for i in range(0, len(tps_list)): + ws.cell(row=row_num, column=col).value = tps_list[i] + ws.cell(row=row_num+1, column=col).value = data_writed_list[i] + + count += 1 + # After counting JUMP_INTERVAL, jump to next table + if count == JUMP_INTERVAL: + count = 0 + row_num += jump_row + else: + row_num +=2 + return ws + + +def main(): + wb = load_workbook(excel_path) + ws = wb[work_sheet] + for i in write_arguments: + log_path = i[0] + col = i[1] + row_num = i[2] + jump_row = i[3] + tps_list, data_writed_list = read_data(log_path) + ws = write_excel(ws, tps_list, data_writed_list, col, row_num, jump_row) + wb.save(filename=result_excel_path) + + +if __name__ == '__main__': + main() diff --git a/test-mysql-auxiliary-scripts_optional/02_data-processing/template.xlsx b/test-mysql-auxiliary-scripts_optional/02_data-processing/template.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..aa28c8931f026d2796edbd07aee646998249c963 Binary files /dev/null and b/test-mysql-auxiliary-scripts_optional/02_data-processing/template.xlsx differ