#!/usr/bin/env python # -*- coding: UTF-8 -*- import os import sys import json import optparse import paramiko import socket import httplib2 import time from datetime import datetime import traceback class SshExecutor(object): """class docs""" def __init__(self, serverip, username, password): self._username = username self._password = password self._host = serverip self._port = 22 self.connect() def logout(self): if not self.isClosed(): self.client.close() self.client = None def connect(self): '''Try to connect, maybe again.''' self.client = paramiko.SSHClient() self.client.load_system_host_keys() self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.client.connect(self._host, self._port, self._username, self._password) def isClosed(self): '''Try to judge whether connect is closed or not.''' if self.client is None: return True transport = self.client.get_transport() if transport is None or not transport.is_active() or not transport.is_active(): self.logout() return True return False def execute(self, cmd): print('Start to execute SSH command: %s' % cmd) ssh_stdin, ssh_stdout, ssh_stderr = self.client.exec_command(cmd) print('-' * 20 + '> STDOUT <' + '-' * 20) linestr = ssh_stdout.readline() while linestr: print linestr, linestr = ssh_stdout.readline() print('-' * 50) print('-' * 20 + '> STDERR <' + '-' * 20) linestr = ssh_stderr.read() while linestr: print linestr, linestr = ssh_stderr.read() print('-' * 50) return ssh_stdout.channel.recv_exit_status() def executeSftpGet(self, remotepath, localpath): print('SFTP Get: {remotepath} {localpath} ...'.format(remotepath=remotepath, localpath=localpath)) sftp = self.client.open_sftp() sftp.get(remotepath, localpath) sftp.close() print('SFTP Get: {remotepath} {localpath} done.'.format(remotepath=remotepath, localpath=localpath)) def executeSftpPut(self, localpath, remotepath): print('SFTP Put: {localpath} {remotepath} ...'.format(localpath=localpath, remotepath=remotepath)) sftp = self.client.open_sftp() sftp.put(localpath, remotepath) sftp.close() print('SFTP Put: {localpath} {remotepath} done'.format(localpath=localpath, remotepath=remotepath)) class InvalidConfigError(Exception): def __init__(self, conf_file, message=''): self.conf_file = conf_file self.message = message def __str__(self): return 'Invalid config file: %s%s%s' % (self.conf_file, os.linesep, self.message) class StepsExecutionFailure(Exception): def __init__(self, conf_file, message=''): self.conf_file = conf_file self.message = message def __str__(self): return 'Execute steps failure! config file: %s%s%s' % (self.conf_file, os.linesep, self.message) class StepsExecutor(object): """class docs""" CMD_DEPLOY = 'cd /root/{prepack_name}; ./deploy.sh {preconfig_arg} << EOF\n{instructions}\nEOF' CMD_UPGRADE = 'cd /root/{prepack_name}; ./upgrade.sh {preconfig_arg} << EOF\n{instructions}\nEOF' CMD_ROLLBACK = 'cd /root/{prepack_name}; ./rollback.sh << EOF\n{instructions}\nEOF' CMD_EXPORT = 'cd /root/{prepack_name}; python ./utils/Monitor.py -s install_sysInfo.json -e' CMD_EXPORT_WITH_IMPORTED_TEMPLATE = CMD_EXPORT + '; python ./utils/Monitor.py -s install_sysInfo.json -E' CMD_COMPARE = 'cd /root/{prepack_name};set -o pipefail; python ./utils/Monitor.py -s install_sysInfo.json -c -j {exclude_diff_config} | tee {report_file}' CMD_COMPARE_WITH_IMPORTED_TEMPLATE = 'cd /root/{prepack_name};set -o pipefail; python ./utils/Monitor.py -s install_sysInfo.json -c -C -j {exclude_diff_config} | tee {report_file}' CMD_REFRESH_DB = '/usr/bin/virsh destroy {db_domain}; /usr/bin/virsh snapshot-revert {db_domain} {snapshot_name}; /usr/bin/virsh start {db_domain}' CMD_REFRESH_APP = '/usr/bin/virsh destroy {app_domain}; /usr/bin/virsh snapshot-revert {app_domain} {snapshot_name}; /usr/bin/virsh start {app_domain}' CMD_REFRESH = '/usr/bin/virsh destroy {domain_name}; /usr/bin/virsh snapshot-revert {domain_name} {snapshot_name}; /usr/bin/virsh start {domain_name}' INSTALL_RPMS = 'cd /root/{prepack_name}/pysetup; sh ./pysetup.sh true;sh ./pysetup.sh false' UNINSTALL_RPMS = 'cd /root/{prepack_name}/pysetup; sh ./pyunstall.sh true;sh ./pyunstall.sh false' CMD_BACKUP_PACKAGE = 'cp -rf /root/{prepack_name} /root/Jenkins_Automation/{backup_name}' CMD_RESTORE_PACKAGE = 'mv -f /root/Jenkins_Automation/{backup_name} /root/{prepack_name}' CMD_INIT_ENV = '[ -e /root/{prepack_name}/utils/initAppServer.sh ] && {{ cd /root/{prepack_name}/utils; ./initAppServer.sh; }}' CMD_MOUNT_NFS = 'mkdir -p /root/Jenkins_Automation; echo "10.116.21.71:/ifs/Jenkins_Automation /root/Jenkins_Automation nfs nolock 0 0" >> /etc/fstab; mount -a' CMD_CLEAR_CONTENT_DIR = 'cd /data/content; ls|grep -v PKG2ING|xargs rm -rf; cd /data; tar --overwrite -xf content.tar' CMD_CLEAR_APP_DIR = 'rm -rf /data/app/*; cd /data; tar -xf app.tar' CMD_INIT_CSNODE_INFO = 'sed -i \'/cluster_service_nodes/{{n;d;}}\' /root/{prepack_name}/install_sysInfo.json;' + \ 'sed -i \'/cluster_service_nodes/a {{"ip":"{installer_ip}", "user":"cms", "password":"cms1234", "rootPasswd":"root1234"}}\' /root/{prepack_name}/install_sysInfo.json;' + \ 'sed -i \'/cluster_service_nodes/{{n;d;}}\' /root/{prepack_name}/upgrade_sysInfo.json;' + \ 'sed -i \'/cluster_service_nodes/a {{"ip":"{installer_ip}", "user":"cms", "password":"cms1234", "rootPasswd":"root1234"}}\' /root/{prepack_name}/upgrade_sysInfo.json' CMD_INIT_APP_INFO = 'sed -i \'/cluster_app_nodes/{{n;d;}}\' /root/{prepack_name}/install_sysInfo.json;' + \ 'sed -i \'/cluster_app_nodes/a {{"ip":"{app_ip}", "user":"cms", "password":"cms1234", "rootPasswd":"root1234"}}],\' /root/{prepack_name}/install_sysInfo.json;' + \ 'sed -i \'/cluster_app_nodes/{{n;d;}}\' /root/{prepack_name}/upgrade_sysInfo.json;' + \ 'sed -i \'/cluster_app_nodes/a {{"ip":"{app_ip}", "user":"cms", "password":"cms1234", "rootPasswd":"root1234"}}],\' /root/{prepack_name}/upgrade_sysInfo.json' CMD_BEFORE_REFRESH_01 = '[ -e /tmp/contentClassSrc.xml ] && cp -vf /tmp/contentClassSrc.xml /root/Jenkins_Automation/contentClassSrc.xml.{tailing_flag}' CMD_BEFORE_REFRESH_02 = '[ -e /tmp/cms_file_monitor.json ] && cp -vf /tmp/cms_file_monitor.json /root/Jenkins_Automation/cms_file_monitor.json.{tailing_flag}' CMD_BEFORE_REFRESH_03 = '[ -e /tmp/cms_data_monitor.json ] && cp -vf /tmp/cms_data_monitor.json /root/Jenkins_Automation/cms_data_monitor.json.{tailing_flag}' CMD_BEFORE_REFRESH_04 = '[ -e /tmp/cms_imported_templates_info.json ] && cp -vf /tmp/cms_imported_templates_info.json /root/Jenkins_Automation/cms_imported_templates_info.json.{tailing_flag}' CMDS_BEFORE_REFRESH = [CMD_BEFORE_REFRESH_01, CMD_BEFORE_REFRESH_02, CMD_BEFORE_REFRESH_03, CMD_BEFORE_REFRESH_04] CMD_AFTER_REFRESH_01 = '[ -e /root/Jenkins_Automation/contentClassSrc.xml.{tailing_flag} ] && cp -vf /root/Jenkins_Automation/contentClassSrc.xml.{tailing_flag} /tmp/contentClassSrc.xml' CMD_AFTER_REFRESH_02 = '[ -e /root/Jenkins_Automation/cms_file_monitor.json.{tailing_flag} ] && cp -vf /root/Jenkins_Automation/cms_file_monitor.json.{tailing_flag} /tmp/cms_file_monitor.json' CMD_AFTER_REFRESH_03 = '[ -e /root/Jenkins_Automation/cms_data_monitor.json.{tailing_flag} ] && cp -vf /root/Jenkins_Automation/cms_data_monitor.json.{tailing_flag} /tmp/cms_data_monitor.json' CMD_AFTER_REFRESH_04 = '[ -e /root/Jenkins_Automation/cms_imported_templates_info.json.{tailing_flag} ] && cp -vf /root/Jenkins_Automation/cms_imported_templates_info.json.{tailing_flag} /tmp/cms_imported_templates_info.json' CMDS_AFTER_REFRESH = [CMD_AFTER_REFRESH_01, CMD_AFTER_REFRESH_02, CMD_AFTER_REFRESH_03, CMD_AFTER_REFRESH_04] ACTION_DEPLOY = 'deploy' ACTION_UPGRADE = 'upgrade' ACTION_ROLLBACK = 'rollback' ACTION_EXPORT = 'export' ACTION_COMPARE = 'compare' ACTION_REFRESH = 'refresh' ACTION_APPLYLICENSE = 'applyLicense' CONSTANT_TRUE = 'true' CONSTANT_FALSE = 'false' CONST_LIC_LOCAL_ROOT = '/home/jenkins/ci-scripts/autotest/conf/licenses/' CONST_LIC_PATH = '/opt/tandbergtv/cms/conf/workflow/prepackLicense.xml' def __init__(self, **argsDict): self._step_json = argsDict['step_json'] if argsDict['step_json'] else '' self._app_executor = argsDict['app_executor'] self._prepack_name = argsDict['prepack_name'] if argsDict['prepack_name'] else '' self._workspace = argsDict['workspace'] if argsDict['workspace'] else '' self._virt_serv_executor = argsDict['virt_serv_executor'] self._app_ip = argsDict['app_ip'] if argsDict['app_ip'] else '' self._snapshot_name = argsDict['snapshot_name'] if argsDict['snapshot_name'] else '' self._db_domain = argsDict['db_domain'] if argsDict['db_domain'] else '' self._app_domain = argsDict['app_domain'] if argsDict['app_domain'] else '' self._below4x = argsDict['below4x'] if argsDict['below4x'] is not None else False self._installer_ip = argsDict['installer_ip'] if argsDict['installer_ip'] else '' self._installer_domain = argsDict['installer_domain'] if argsDict['installer_domain'] else '' self._db_ip = argsDict['db_ip'] if argsDict['db_ip'] else '' self._es_domain = argsDict['es_domain'] if argsDict['es_domain'] else '' self._pt_domain = argsDict['pt_domain'] if argsDict['pt_domain'] else '' def execute(self): jsonfile = open(self._step_json) try: jsonData = json.load(jsonfile) except: raise InvalidConfigError(os.path.abspath(self._step_json), 'Invalid json format.') finally: jsonfile.close() if 'steps' not in jsonData or not jsonData['steps']: print('No steps to execute.') return True app_executor = self._app_executor fs_tailing = datetime.now().strftime('%Y%m%d%H%M%S%f') + str(os.getpid()) report_file_name = 'cms_compare_report.%s.txt' % fs_tailing rpfile_path = '/tmp/' + report_file_name local_rpfile_path = self._workspace + os.sep + report_file_name for step in jsonData['steps']: if 'description' in step: StepsExecutor.printDescription(step['description']) if 'action' not in step: raise InvalidConfigError(os.path.abspath(self._step_json), 'No action value :%s' % step) instruction = step['instruction'] if 'instruction' in step else '' preconfig = step['preconfig'] if 'preconfig' in step else '' sshExecuteStatus = 0 if StepsExecutor.ACTION_DEPLOY == step['action']: if not self._below4x: app_executor.execute(StepsExecutor.CMD_INIT_CSNODE_INFO.format(prepack_name=self._prepack_name, installer_ip=self._installer_ip)) if len(self._app_ip) > 1: app_executor.execute(StepsExecutor.CMD_INIT_APP_INFO.format(prepack_name=self._prepack_name, app_ip=self._app_ip[1])) sshExecuteStatus = app_executor.execute(StepsExecutor.CMD_DEPLOY.format(prepack_name=self._prepack_name, instructions=instruction, preconfig_arg=preconfig)) elif StepsExecutor.ACTION_UPGRADE == step['action']: if not self._below4x: app_executor.execute(StepsExecutor.CMD_INIT_CSNODE_INFO.format(prepack_name=self._prepack_name, installer_ip=self._installer_ip)) if len(self._app_ip) > 1: app_executor.execute(StepsExecutor.CMD_INIT_APP_INFO.format(prepack_name=self._prepack_name, app_ip=self._app_ip[1])) sshExecuteStatus = app_executor.execute(StepsExecutor.CMD_UPGRADE.format(prepack_name=self._prepack_name, instructions=instruction, preconfig_arg=preconfig)) elif StepsExecutor.ACTION_ROLLBACK == step['action']: sshExecuteStatus = app_executor.execute(StepsExecutor.CMD_ROLLBACK.format(prepack_name=self._prepack_name, instructions=instruction)) elif StepsExecutor.ACTION_APPLYLICENSE == step['action']: app_executor.executeSftpPut(StepsExecutor.CONST_LIC_LOCAL_ROOT + str(step['license']), StepsExecutor.CONST_LIC_PATH) elif StepsExecutor.ACTION_EXPORT == step['action']: if not self._below4x: app_executor.execute(StepsExecutor.CMD_INIT_CSNODE_INFO.format(prepack_name=self._prepack_name, installer_ip=self._installer_ip)) app_executor.execute(StepsExecutor.INSTALL_RPMS.format(prepack_name=self._prepack_name)) sshExecuteStatus = app_executor.execute(StepsExecutor.CMD_EXPORT_WITH_IMPORTED_TEMPLATE.format(prepack_name=self._prepack_name)) \ if step.has_key('withImportedTemplate') and step['withImportedTemplate'] else \ app_executor.execute(StepsExecutor.CMD_EXPORT.format(prepack_name=self._prepack_name)) elif StepsExecutor.ACTION_COMPARE == step['action']: app_executor.execute(StepsExecutor.INSTALL_RPMS.format(prepack_name=self._prepack_name)) configPathOnAppServer = self.constructExcludeDiffSetFileAndUploadToAppServer(step) # compare with previous status sshExecuteStatus = app_executor.execute(StepsExecutor.CMD_COMPARE_WITH_IMPORTED_TEMPLATE.format(prepack_name=self._prepack_name, exclude_diff_config=configPathOnAppServer, report_file=rpfile_path)) \ if step.has_key('withImportedTemplate') and step['withImportedTemplate'] else \ app_executor.execute(StepsExecutor.CMD_COMPARE.format(prepack_name=self._prepack_name, exclude_diff_config=configPathOnAppServer, report_file=rpfile_path)) # save the report to local app_executor.executeSftpGet(rpfile_path, local_rpfile_path) elif StepsExecutor.ACTION_REFRESH == step['action']: if 'simple' not in step or StepsExecutor.CONSTANT_TRUE != step['simple']: # backup something before refreshing backup_pkg_name = self._prepack_name + '.' + fs_tailing for cmd in StepsExecutor.CMDS_BEFORE_REFRESH: app_executor.execute( cmd.format(tailing_flag=fs_tailing) ) app_executor.execute( StepsExecutor.CMD_BACKUP_PACKAGE.format(prepack_name=self._prepack_name, backup_name=backup_pkg_name) ) # do refreshing app_executor.logout() if self._below4x: sshExecuteStatus = self.refreshServer(self._app_ip[0]) else: sshExecuteStatus = self.refreshServerCms4x() app_executor.connect() if not self._below4x: app_executor.execute(StepsExecutor.CMD_CLEAR_CONTENT_DIR) app_executor.execute(StepsExecutor.CMD_CLEAR_APP_DIR) app_executor.execute( StepsExecutor.CMD_MOUNT_NFS ) if 'simple' not in step or StepsExecutor.CONSTANT_TRUE != step['simple']: # restore the backup just done app_executor.execute( StepsExecutor.CMD_RESTORE_PACKAGE.format(prepack_name=self._prepack_name, backup_name=backup_pkg_name) ) app_executor.execute( StepsExecutor.CMD_INIT_ENV.format(prepack_name=self._prepack_name) ) for cmd in StepsExecutor.CMDS_AFTER_REFRESH: app_executor.execute( cmd.format(tailing_flag=fs_tailing) ) else: raise InvalidConfigError(os.path.abspath(self._step_json), 'Invalid action value :%s' % step['action']) if sshExecuteStatus != 0: print 'Encountering problem while executing step: ' + step['description'] return False return True def constructExcludeDiffSetFileAndUploadToAppServer(self, step): filePath = self._step_json + ".excludeDiff.json" configObj = step["excludeDifferentSet"] if step and step.has_key("excludeDifferentSet") else {} with open(filePath, 'w') as f: json.dump(configObj, f, indent=4) fs_tailing = datetime.now().strftime('%Y%m%d%H%M%S%f') + str(os.getpid()) exclude_diff_config_file_name = 'exclude_diff_config.%s.json' % fs_tailing exclude_diff_config_file_path = '/tmp/' + exclude_diff_config_file_name self._app_executor.executeSftpPut(filePath,exclude_diff_config_file_path) os.remove(filePath) return exclude_diff_config_file_path def refreshServer(self, app_ip): if not self._virt_serv_executor: raise StepsExecutionFailure(os.path.abspath(self._step_json), "You demanded refreshing on APP Server, but didn't tell me the IP of Virtual Machine Server. Please note that some steps might have been already committed.") sshExecuteStatus = self._virt_serv_executor.execute(StepsExecutor.CMD_REFRESH_DB.format(snapshot_name=self._snapshot_name, db_domain=self._db_domain)) if sshExecuteStatus == 0: sshExecuteStatus = self._virt_serv_executor.execute(StepsExecutor.CMD_REFRESH_APP.format(snapshot_name=self._snapshot_name, app_domain=self._app_domain[0])) print('sleep a while...') time.sleep(180) print('Test if the server has started...') if not StepsExecutor.checkAppServerIfStarted(app_ip, 300): raise StepsExecutionFailure(os.path.abspath(self._step_json), 'Server started failed: services still down!') print('Server has started.') return sshExecuteStatus def refreshServerCms4x(self): self._virt_serv_executor.execute(StepsExecutor.CMD_REFRESH.format(snapshot_name=self._snapshot_name, domain_name=self._installer_domain)) print('sleep a while...') time.sleep(90) self._virt_serv_executor.execute(StepsExecutor.CMD_REFRESH.format(snapshot_name=self._snapshot_name, domain_name=self._db_domain)) print('sleep a while...') time.sleep(90) for appd in self._app_domain: self._virt_serv_executor.execute(StepsExecutor.CMD_REFRESH.format(snapshot_name=self._snapshot_name, domain_name=appd)) if self._es_domain: self._virt_serv_executor.execute(StepsExecutor.CMD_REFRESH.format(snapshot_name=self._snapshot_name, domain_name=self._es_domain)) if self._pt_domain: self._virt_serv_executor.execute(StepsExecutor.CMD_REFRESH.format(snapshot_name=self._snapshot_name, domain_name=self._pt_domain)) print('sleep a while...') time.sleep(180) print('Test if the server has started...') if self._installer_ip: installer_executor = SshExecutor(self._installer_ip, "root", "root1234") installer_executor.execute("service haproxy restart") if not StepsExecutor.checkAppServerIfStarted(self._app_ip[0], 300): raise StepsExecutionFailure(os.path.abspath(self._step_json), 'Server started failed: services still down!') print('Server has started.') return 0 @staticmethod def checkAppServerIfStarted(app_ip, max_times): serviceOk = False h = httplib2.Http(".cache", disable_ssl_certificate_validation=True, timeout=60) h.force_exception_to_status_code=True for i in range(max_times): try: print('Test request %d...' % i) # check cms service respCms = h.request("https://{app_ip}:8443/portal".format(app_ip=app_ip), "GET") # check tomcat service respTomcat = h.request("http://{app_ip}:8081/filemanager/services/FileSubsystemWebservice".format(app_ip=app_ip), "GET") if (respCms[0]["status"] == '200' or respCms[0]["status"] == '302') and \ (respTomcat[0]["status"] == '200' or respTomcat[0]["status"] == '302'): serviceOk = True break except (httplib2.SSLHandshakeError, socket.error): pass print('Still waiting...') time.sleep(5) return serviceOk @staticmethod def printDescription(msg): print print('=' * 50) print(msg) print('=' * 50) print def buildUpOptions(): optparser = optparse.OptionParser() optparser.add_option("-w", "--ws-this-time", action="store", type="string", dest="workspace", help="workspace for this time, result report is supposed to be saved here, OPTIONAL.") optparser.add_option("-s", "--steps", action="store", type="string", dest="stepsJson", help="config json for steps, relative path to current directory, REQUIRED.") optparser.add_option("-v", "--virt-server-ip", action="store", type="string", dest="virtServIp", help="IP address of VM server, OPTIONAL.") optparser.add_option("-n", "--prepack-name", action="store", type="string", dest="prepackName", help="which package to be installed, single file name, OPTIONAL.") optparser.add_option("-S", "--snapshot-name", action="store", type="string", dest="snapshotName", help="which snapshot to revert to, OPTIONAL.") optparser.add_option("-b", "--below-4x", action="store_true", dest="below4x", default=False, help="whether below 4x or not, OPTIONAL.") optparser.add_option("-i", "--installer-ip", action="store", type="string", dest="installerIp", help="IP address of the installer, OPTIONAL.") optparser.add_option("-I", "--installer-domain", action="store", type="string", dest="installerDomain", help="installer-domain to refresh server, OPTIONAL.") optparser.add_option("-d", "--db-ip", action="store", type="string", dest="dbIp", help="IP address of the database server, OPTIONAL.") optparser.add_option("-D", "--db-domain", action="store", type="string", dest="dbDomain", help="db-domain to refresh server, OPTIONAL.") optparser.add_option("-a", "--app-ip", action="append", type="string", dest="appIp", help="IP address of App server, REQUIRED.") optparser.add_option("-A", "--app-domain", action="append", type="string", dest="appDomain", help="app-domain to refresh server, OPTIONAL.") optparser.add_option("-E", "--es-domain", action="store", type="string", dest="esDomain", help="es-domain to refresh server, OPTIONAL.") optparser.add_option("-P", "--pt-domain", action="store", type="string", dest="ptDomain", help="pt-domain to refresh server, OPTIONAL.") return optparser def main(): """main function.""" optparser = buildUpOptions() (options, args) = optparser.parse_args() print("options: {0}".format(options)) if not options.stepsJson or not options.appIp: optparser.print_help() optparser.error("Sorry, require more options.") return username = "root" password = "root1234" try: app_executor = SshExecutor(options.appIp[0], username, password) virt_serv_executor = None if options.virtServIp: virt_serv_executor = SshExecutor(options.virtServIp, username, password) if options.prepackName: #app_executor = SshExecutor(options.appIp, username, password) app_executor.execute("cd /root/%s; find . -type f -name '*.sh' | xargs chmod a+x" % options.prepackName) result = app_executor.execute(StepsExecutor.CMD_INIT_ENV.format(prepack_name=options.prepackName)) if result != 0: exit(result) steps_executor = StepsExecutor(step_json=options.stepsJson, app_executor=app_executor, virt_serv_executor=virt_serv_executor, prepack_name=options.prepackName, workspace=options.workspace, db_domain=options.dbDomain, app_domain=options.appDomain, app_ip=options.appIp, snapshot_name=options.snapshotName, below4x=options.below4x, installer_ip=options.installerIp, installer_domain=options.installerDomain, db_ip=options.dbIp, es_domain=options.esDomain, pt_domain=options.ptDomain) result = steps_executor.execute() if not result: print 'Encountering problem while executing Steps.' sys.exit(2) except Exception: print "Unexpected error:" + traceback.format_exc() sys.exit(2) finally: if app_executor: app_executor.logout() if virt_serv_executor: virt_serv_executor.logout() sys.exit(0) if __name__ == "__main__": main()