update log and db_operator

Author: YunhuiChen

robot/Resources/config/config.py

@@ -1,9 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf8 -*-
 
-'''
-基本配置信息,若配置新的环境，需要更改此文件
-'''
 #curve info
 curve_workspace = "/var/lib/jenkins/workspace/curve_failover/"
 chunkserver_start_script = "./deploy/local/chunkserver/start_chunkservers_locally.sh"

robot/Resources/keywords/base_operate.py

@@ -5,7 +5,6 @@
 import shlex
 from config import config
 from logger.logger import *
-from lib import db_operator
 from lib import shell_operator
 from swig import swig_operate
 from multiprocessing import Pool
@@ -17,61 +16,6 @@
 import types
 import mythread
 
-#clean_db
-def clean_db():
-    try:
-        cmd_list = ["DELETE FROM curve_logicalpool;", "DELETE FROM curve_copyset;", \
-                "DELETE FROM curve_physicalpool;", "DELETE FROM curve_zone;", \
-                "DELETE FROM curve_server;", "DELETE FROM curve_chunkserver;", \
-                "DELETE FROM curve_session;", "DELETE FROM client_info;"]
-        for cmd in cmd_list:
-            conn = db_operator.conn_db(config.db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-            db_operator.exec_sql(conn, cmd)
-            logger.debug("clean db %s" %cmd)
-
-    except Exception:
-        logger.error("clean db fail.")
-        raise
-
-def create_db_table():
-    try:
-        conn = db_operator.conn_db(config.db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-        db_operator.exec_sql_file(conn, config.curve_sql)
-        conn2 = db_operator.conn_db(config.db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-        db_operator.exec_sql_file(conn2, config.snap_sql)
-
-    except Exception:
-        logger.error("创建表失败.")
-        raise
-
-
-def drop_mds_table():
-    try:
-        cmd_list = ["DROP TABLE curve_logicalpool;", "DROP TABLE curve_copyset;", \
-                    "DROP TABLE curve_physicalpool;", "DROP TABLE curve_zone;", \
-                    "DROP TABLE curve_server;", "DROP TABLE curve_chunkserver;", \
-                    "DROP TABLE curve_session;", "DROP TABLE client_info;"]
-        for cmd in cmd_list:
-            conn = db_operator.conn_db(config.db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-            db_operator.exec_sql(conn, cmd)
-            logger.debug("drop table %s" %cmd)
-    except Exception:
-        logger.error("drop db fail.")
-        raise
-
-def get_copyset_table():
-    try:
-        cmd = "select copySetID,chunkServerIDList  from curve_copyset  INTO OUTFILE " + '"'+config.mysql_log +'"'+ \
-              " fields terminated by '|' lines terminated by '|'; "
-        conn = db_operator.conn_db(config.db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-        db_operator.exec_sql(conn, cmd)
-        logger.debug("get table %s" %cmd)
-        rc = os.path.isfile(config.mysql_log)
-        assert rc == True,"exec %s"%cmd
-    except Exception:
-        logger.error("get copyset table fail.cmd is %s"%cmd)
-        raise
-
 def mv_copyset_table():
     grep_cmd = "mv %s %s"%(config.mysql_log,config.curve_workspace)
     try:
@@ -130,30 +74,6 @@ def get_copyset_scatterwith():
         logger.info("chunkserver %d copyset_num is %d \t,scatterwith is %d %s"%(cs,cs_copyset_num,len(scatterwith),scatterwith))
 #        print "chunkserver %d ,scatterwith is %d" % (cs, len(scatterwith))
 
-def drop_snap_clone_table():
-    try:
-        cmd_list = ["DROP TABLE snapshot;", "DROP TABLE clone;"]
-        for cmd in cmd_list:
-            conn = db_operator.conn_db(config.db_host, config.db_port, config.db_user, config.db_pass, config.snap_db_name)
-            db_operator.exec_sql(conn, cmd)
-            logger.debug("drop table %s" %cmd)
-    except Exception:
-        logger.error("drop db fail.")
-        raise
-
-def mock_chunkserver_registe():
-    try:
-        mysql_cmd = ["INSERT INTO  curve_chunkserver VALUES (31, 'token1', 'nvme', '127.0.0.1', 8200, 0, 1, 0, 0, '/', 0, 0);",
-                     "INSERT INTO  curve_chunkserver VALUES (32, 'token2', 'nvme', '127.0.0.1', 8201, 0, 2, 0, 0, '/', 0, 0);",
-                     "INSERT INTO  curve_chunkserver VALUES (33, 'token3', 'nvme', '127.0.0.1', 8202, 0, 3, 0, 0, '/', 0, 0);"]
-        for cmd in mysql_cmd:
-            conn = db_operator.conn_db(config.db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-            db_operator.exec_sql(conn, cmd)
-            logger.debug("insert db %s" % cmd)
-    except Exception:
-        logger.error("insert db fail. %s" % cmd)
-        raise
-
 
 def kill_process(process_name):
     grep_cmd = "ps -ef | grep %s | grep -v grep | awk '{print $2}' " %process_name
@@ -432,13 +352,6 @@ def cat_chunkserver_log(chunkserver_id):
     else:
         logger.error("chunkserver log not exists!")
 
-def check_copyset_num(copyset_num):
-    sql = "select * from curve_copyset;"
-    conn = db_operator.conn_db(config.db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-    logicalpool_dbinof = db_operator.query_db(conn, sql)
-    logger.info("logicalpool_dbinof = %s" % int(logicalpool_dbinof["rowcount"]))
-    assert int(logicalpool_dbinof["rowcount"]) == int(copyset_num)
-
 def get_buf():
     return config.buf
 

robot/Resources/keywords/deploy.py

@@ -5,7 +5,6 @@
 from config import config
 from logger.logger import *
 from lib import shell_operator
-from lib import db_operator
 import threading
 import random
 import time
@@ -279,35 +278,6 @@ def drop_all_chunkserver_dat():
         logger.debug("drop cs dat get result is %d" % t.get_result())
         assert t.get_result() == 0
 
-def drop_abnormal_test_db():
-    try:
-        cmd_list = ["DROP TABLE curve_logicalpool;", "DROP TABLE curve_copyset;", \
-                    "DROP TABLE curve_physicalpool;", "DROP TABLE curve_zone;", \
-                    "DROP TABLE curve_server;", "DROP TABLE curve_chunkserver;", \
-                    "DROP TABLE curve_session;",  "DROP TABLE client_info;"]
-        cmd_list_2 = ["DROP TABLE clone;","DROP TABLE snapshot;"]
-        for cmd in cmd_list:
-            conn = db_operator.conn_db(config.abnormal_db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-            db_operator.exec_sql(conn, cmd)
-            logger.debug("drop table %s" %cmd)
-        for cmd in cmd_list_2:
-            conn = db_operator.conn_db(config.abnormal_db_host, config.db_port, config.db_user, config.db_pass, config.snap_db_name)
-            db_operator.exec_sql(conn, cmd)
-            logger.debug("drop table %s" %cmd)
-    except Exception:
-        logger.error("drop db fail.")
-        raise
-
-def create_abnormal_db_table():
-    try:
-       conn = db_operator.conn_db(config.abnormal_db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-       db_operator.exec_sql_file(conn, config.curve_sql)
-       conn2 = db_operator.conn_db(config.abnormal_db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-       db_operator.exec_sql_file(conn2, config.snap_sql)
-    except Exception:
-        logger.error("创建表失败.")
-        raise
-
 def install_deb():
     try:
 #        mkdeb_url =  config.curve_workspace + "mk-deb.sh"
@@ -427,13 +397,6 @@ def start_abnormal_test_services():
         logger.error("up servers fail.")
         raise
 
-def get_copyset_num():
-    conn = db_operator.conn_db(config.abnormal_db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-    sql = R"select * from curve_copyset;"
-    copyset = db_operator.query_db(conn, sql)
-    logger.info("now copyset num is %d"%copyset["rowcount"])
-    return int(copyset["rowcount"])
-
 def create_pool():
     ssh = shell_operator.create_ssh_connect(config.mds_list[0], 1046, config.abnormal_user)
     mds = []

robot/Resources/keywords/fault_inject.py

@@ -7,7 +7,6 @@
 from lib import shell_operator
 import random
 import time
-from lib import db_operator
 import threading
 import time
 import mythread
@@ -188,19 +187,19 @@ def clear_RecycleBin():
     ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
     ori_cmd = "curve_ops_tool clean-recycle --isTest"
     rs = shell_operator.ssh_exec(ssh, ori_cmd)
-    assert rs[3] == 0,"clean RecyclenBin失败，msg is %s"%rs[1]
+    assert rs[3] == 0,"clean RecyclenBin fail,msg is %s"%rs[1]
     starttime = time.time()
     ori_cmd = "curve_ops_tool list -fileName=/RecycleBin |grep Total"
     while time.time() - starttime < 180:
         rs = shell_operator.ssh_exec(ssh, ori_cmd)
         if "".join(rs[1]).strip() == "Total file number: 0" and rs[3] == 0:
             break
         else:
-            logger.debug("删除中")
+            logger.debug("deleting")
             if rs[3] != 0:
-                logger.debug("list /RecycleBin 失败,error is %s"%rs[1])
+                logger.debug("list /RecycleBin fail,error is %s"%rs[1])
             time.sleep(3) 
-    assert rs[3] == 0,"删除/RecycleBin 失败，error is %s"%rs[1]
+    assert rs[3] == 0,"delete /RecycleBin fail,error is %s"%rs[1]
 
 def loop_map_unmap_file():
     thread = []
@@ -274,15 +273,6 @@ def write_full_disk(fio_size):
     assert rs[3] == 0,"write fio fail"
 
 def get_chunkserver_id(host,cs_id):
-#    conn = db_operator.conn_db(config.abnormal_db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-#    sql = R"select * from curve_chunkserver where `internalHostIP` like '%s' and `mountPoint` like 'local:///data/chunkserver%d/' and `rwstatus` like 0;;"%(host,cs_id)
-#    chunkserver = db_operator.query_db(conn, sql)
-#    if chunkserver["rowcount"] == 1:
-#        chunkserver_id = chunkserver["data"][0]["chunkServerID"]
-#        logger.info("operator chunkserver id is %d"%chunkserver_id)
-#    else:
-#            assert False,"get chunkserver id fail,retun is %s"%(chunkserver)
-#        return -1
     client_host = config.client_list[0]
     logger.info("|------begin get chunkserver %s id %d------|"%(host,cs_id))
     cmd = "curve_ops_tool chunkserver-list | grep %s |grep -w chunkserver%d"%(host,cs_id)
@@ -296,18 +286,6 @@ def get_chunkserver_id(host,cs_id):
         return -1
 
 def get_cs_copyset_num(host,cs_id):
-#    conn = db_operator.conn_db(config.abnormal_db_host, config.db_port, config.db_user, config.db_pass, config.mds_db_name)
-#    try:
-#        sql = R"select * from curve_copyset where chunkServerIDList REGEXP '\n\t%d,|,\n\t%d,|,\n\t%d\n';"\
-#                %(chunkserver_id,chunkserver_id,chunkserver_id)
-#        cs_copyset_info = db_operator.query_db(conn, sql)
-#        logger.debug("get table row is %s"%cs_copyset_info["rowcount"])
-#        logger.debug("get table %s" %(cs_copyset_info))
-#    except Exception:
-#        logger.error("get db fail.")
-#        raise
-#    logger.info("chunkserver id %d have %s copysets"%(chunkserver_id,cs_copyset_info["rowcount"]))
-#    return int(cs_copyset_info["rowcount"])
     client_host = config.client_list[0]
     cs_number = int(cs_id) + 8200
     cmd = "curve_ops_tool check-chunkserver -chunkserverAddr=%s:%d |grep 'total copysets'"%(host,cs_number)
@@ -342,27 +320,27 @@ def map_nbd():
     ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user)
     cmd = "curve create --filename /fiofile --length 10 --user test"
     rs = shell_operator.ssh_exec(ssh, cmd)
-    assert rs[3] == 0,"创建卷/fiofile 失败，失败原因：%s"%rs[2]
+    assert rs[3] == 0,"create /fiofile fail：%s"%rs[2]
     cmd = "curve create --filename /vdbenchfile --length 10 --user test"
     rs = shell_operator.ssh_exec(ssh, cmd)
-    assert rs[3] == 0,"创建卷/vdbenchfile 失败，失败原因：%s"%rs[2]
+    assert rs[3] == 0,"create /vdbenchfile fail：%s"%rs[2]
     time.sleep(3)
     cmd = "sudo curve-nbd map cbd:pool1//fiofile_test_ >/dev/null 2>&1"
     rs = shell_operator.ssh_exec(ssh, cmd)
-    assert rs[3] == 0,"map fiofile 失败，失败原因：%s"%rs[2]
+    assert rs[3] == 0,"map fiofile fail：%s"%rs[2]
     cmd = "sudo curve-nbd map cbd:pool1//vdbenchfile_test_ >/dev/null 2>&1"
     rs = shell_operator.ssh_exec(ssh, cmd)
-    assert rs[3] == 0,"map vdbenchfile 失败，失败原因：%s"%rs[2]
+    assert rs[3] == 0,"map vdbenchfile fail：%s"%rs[2]
 
 def delete_nbd():
     client_host = config.client_list[0]
     ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user)
     cmd = "curve delete --filename /fiofile --user test"
     rs = shell_operator.ssh_exec(ssh, cmd)
-    assert rs[3] == 0,"删除卷/fiofile 失败，失败原因：%s"%rs[2]
+    assert rs[3] == 0,"delete /fiofile fail：%s"%rs[2]
     cmd = "curve delete --filename /vdbenchfile --user test"
     rs = shell_operator.ssh_exec(ssh, cmd)
-    assert rs[3] == 0,"删除卷/vdbenchfile 失败，失败原因：%s"%rs[2]
+    assert rs[3] == 0,"delete /vdbenchfile fail：%s"%rs[2]
 
 def check_host_connect(ip):
     cmd = "ping %s -w3"%ip
@@ -751,7 +729,6 @@ def rapid_leader_schedule():
     ori_cmd = "curve_ops_tool rapid-leader-schedule"
     rs = shell_operator.ssh_exec(ssh, ori_cmd)
     assert rs[3] == 0,"rapid leader schedule not ok"
-    # 等待rapid leader schedule执行完成
     ori_cmd = "curve_ops_tool check-operator -opName=transfer_leader -leaderOpInterval=1| grep \"Operator num is\""
     starttime = time.time()
     while time.time() - starttime < 60:
@@ -764,7 +741,6 @@ def rapid_leader_schedule():
 
 def wait_cluster_healthy(limit_iops=8000):
     check_chunkserver_online()
-    #检测集群整体状态
     host = random.choice(config.mds_list)
     ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
     ori_cmd = "curve_ops_tool status | grep \"cluster is\""
@@ -789,7 +765,6 @@ def wait_cluster_healthy(limit_iops=8000):
         logger.debug("copysets status is %s"%copysets_status)
         assert check == 1,"cluster is not healthy in %d s,cluster status is:\n %s,copysets status is:\n %s"%(config.recover_time,cluster_status,copysets_status)
     rapid_leader_schedule() 
-#检测nbd iops    
     ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046, config.abnormal_user)
     i = 0
     while i < 300:
@@ -824,7 +799,6 @@ def check_io_error():
         ssh.close()
 
 def check_copies_consistency():
-    # 快速leader均衡
     host = random.choice(config.mds_list)
     ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
     ori_cmdpri = "curve_ops_tool check-consistency -filename=/fiofile \
@@ -1738,7 +1712,6 @@ def perf_test():
     assert rs[3] == 0,"cp fiodata fail,error is %s"%rs[1]
     analysis_data(ssh)
 
-#增加数据盘
 def add_data_disk():
     ori_cmd = "bash attach_thrash.sh"
     ssh = shell_operator.create_ssh_connect(config.nova_host, 1046, config.nova_user)
@@ -1818,7 +1791,6 @@ def get_all_curvevm_active_num(num):
         assert status == "up","get vm status fail,not up.is %s,current vm id is %s"%(status,uuid)
     return active_num
 
-# 用于初始化创建50个curve系统盘的云主机。先从镜像创建一个curve云主机，再从该云主机创建自定义镜像和克隆50个云主机。每个云主机会自动拉起1000 iops的io
 def init_create_curve_vm(num):
     image_id = config.image_id
     salt = ''.join(random.sample(string.ascii_letters + string.digits, 8))
@@ -1885,7 +1857,6 @@ def do_thrasher(action):
         logger.debug("开始启动故障XXXXXXXXXXXXXXXXXXX %s,%s XXXXXXXXXXXXXXXXXXXXXX"%(action[0],str(action[1])))
         globals()[action[0]](action[1])
 
-#加回所有retired的chunkserver
 def start_retired_and_down_chunkservers():
     for host in config.chunkserver_list:
         ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)