more filesHEADmaster

1 files changed, 524 insertions, 0 deletions

diff --git a/tools/collect-logs.py b/tools/collect-logs.py
new file mode 100644
index 0000000..8ec3ce5
--- /dev/null
+++ b/tools/collect-logs.py
@@ -0,0 +1,524 @@
+import datetime
+import pexpect
+import os
+import logging
+import sys
+import tarfile
+import os.path
+import gzip
+import psycopg2
+
+class LogCollector:
+
+    def __init__(self):
+        pass
+        self.logger = logging.getLogger()
+
+    def get_connection(self):
+        conn = psycopg2.connect(
+            host="",
+            database="",
+            user="",
+            password="")
+        return conn
+
+    def run_command_and_collect_output(self, site_name, site_ip, password, command):
+        #print(site_name)
+        #print(site_ip)
+        #print(password)
+        #print(command)
+        #conn = self.get_connection()
+        # create a cursor
+        #try:
+        #    cur = conn.cursor()
+        
+            # execute a statement
+        #    print('PostgreSQL database version:')
+        #    cur.execute('SELECT version()')
+
+            # display the PostgreSQL database server version
+        #    db_version = cur.fetchone()
+        #    print(db_version)
+       
+            # close the communication with the PostgreSQL
+        #    cur.close()
+        #except (Exception, psycopg2.DatabaseError) as error:
+        #    print(error)
+        #finally:
+        #    if conn is not None:
+        #        conn.close()
+        #        print('Database connection closed.')
+
+        source_dir = site_name
+        if not os.path.exists(source_dir):
+            os.makedirs(source_dir)
+
+        ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t sysadmin@' + site_ip
+        debugLog = site_name + "/cmd-output.log"
+        fp = open(debugLog, "a")
+
+        parts = command.split(" ")
+
+        which_cmd = ssh_prefix + " which " + parts[0] 
+        cmds = []
+        cmds.append(which_cmd)
+        all_lines = self.run_command_get_all_lines(cmds, password, self.logger)
+        raw_cmd = ''
+        for line in all_lines.split("\n"):
+            line = line.rstrip().lstrip()
+            if line and 'Connection' not in line:
+                raw_cmd = line
+                break
+
+        #kubectl get ns --kubeconfig=/etc/kubernetes/admin.conf | grep wsbomagj | awk '{print $1}' | xargs kubectl delete ns --kubeconfig=/etc/kubernetes/admin.conf
+
+        #cmd = ssh_prefix + " /usr/sbin/" + raw_cmd
+        raw_cmd = raw_cmd + " " + ' '.join(parts[1:])
+        if 'system' in raw_cmd:
+            cmd = ssh_prefix + " source /etc/platform/openrc; " + raw_cmd
+        elif 'kubectl' in raw_cmd:
+            cmd = ssh_prefix + " " + raw_cmd + " --kubeconfig=/etc/kubernetes/admin.conf"
+        else:
+            cmd = ssh_prefix + " " + raw_cmd
+        print(cmd)
+        self._execute_cmd_and_save_op(cmd, raw_cmd, password, fp)
+        fp.close()
+
+    def collect_samsung_logs(self, site_name, site_ip, password):
+        source_dir = site_name
+        if not os.path.exists(source_dir):
+            os.makedirs(source_dir)
+
+        ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t sysadmin@' + site_ip
+        debugLog = site_name + "/DebugLog"
+        fp = open(debugLog, "w")
+
+        # 1. uptime
+        raw_cmd = " uptime "
+        uptime = ssh_prefix + raw_cmd
+        self._execute_cmd_and_save_op(uptime, raw_cmd, password, fp)
+
+        # 2. ls -l /var/lib/systemd/coredump
+        raw_cmd = " ls -l /var/lib/systemd/coredump "
+        coredump_ls = ssh_prefix + raw_cmd
+        self._execute_cmd_and_save_op(coredump_ls, raw_cmd, password, fp)
+
+        # 3. fm alarm-list
+        raw_cmd = " fm alarm-list"
+        fm_alarm_list = ssh_prefix + " source /etc/platform/openrc; " + raw_cmd
+        self._execute_cmd_and_save_op(fm_alarm_list, raw_cmd, password, fp)
+
+        # 4. date
+        raw_cmd = " date  "
+        date = ssh_prefix + raw_cmd
+        self._execute_cmd_and_save_op(date, raw_cmd, password, fp)
+
+        # 5. build.info
+        raw_cmd = " cat /etc/build.info "
+        build_info = ssh_prefix + raw_cmd
+        self._execute_cmd_and_save_op(build_info, raw_cmd, password, fp)
+
+        # 6. ip link
+        raw_cmd = " /usr/sbin/ip link "
+        ip_link = ssh_prefix + raw_cmd
+        self._execute_cmd_and_save_op(ip_link, raw_cmd, password, fp)
+        fp.close()
+
+        # 7. pmond.log
+        pmond_log = '/var/log/pmond.log'
+        cmds = []
+        cmd = 'scp -o StrictHostKeyChecking=no sysadmin@[' + site_ip + ']:' + pmond_log + " " + site_name + "/."
+        cmds.append(cmd)
+        self.run_commands_scp(cmds, password, self.logger)
+
+        # 8. /var/log/kern.log
+        kern_logs_cmd = ssh_prefix + " ls /var/log/kern.log*"
+        cmds = [kern_logs_cmd]
+        kern_logs = self.run_command_get_all_lines(cmds, password, self.logger)
+        print(kern_logs)
+        for kern_log in kern_logs.split('\n'):
+            if 'Connection to' not in kern_log:
+                kern_log = kern_log.rstrip().lstrip()
+                kern_parts = kern_log.split("\t")
+                print("Kern parts:" + str(len(kern_parts)))
+                if len(kern_parts) == 1:
+                    kern_parts = kern_log.split(" ")
+                for p in kern_parts:
+                    if p != '' and p != '\n' and p != '\t' and 'Connection' not in p:
+                        cmds = []
+                        print(p)
+                        cmd = 'scp -o StrictHostKeyChecking=no sysadmin@[' + site_ip + ']:' + p + " " + site_name + "/."
+                        print(cmd)
+                        cmds.append(cmd)
+                        self.run_commands_scp(cmds, password, self.logger)
+
+        # 6. /var/log/user.log*
+        user_logs_cmd = ssh_prefix + " ls /var/log/user.log*"
+        cmds = [user_logs_cmd]
+        user_logs = self.run_command_get_all_lines(cmds, password, self.logger)
+        print(user_logs)
+        for user_log in user_logs.split('\n'):
+            if 'Connection to' not in user_log:
+                cmds = []
+                user_log = user_log.rstrip().lstrip()
+                user_parts = user_log.split("\t")
+                print("User parts:" + str(len(user_parts)))
+                if len(user_parts) == 1:
+                    user_parts = user_log.split(" ")
+                for p in user_parts:
+                    if p != '' and p != '\n' and p != '\t' and 'Connection' not in p:
+                        print(p)
+                        cmd = 'scp -o StrictHostKeyChecking=no sysadmin@[' + site_ip + ']:' + p + " " + site_name + "/."
+                        print(cmd)
+                        cmds.append(cmd)
+                        self.run_commands_scp(cmds, password, self.logger)
+
+        # 7. /var/log/pods/*adpf*
+        pod_logs_cmd = ssh_prefix + " ls /var/log/pods/ | grep adpf"
+        cmds = [pod_logs_cmd]
+        pod_logs = self.run_command_get_all_lines(cmds, password, self.logger)
+        print(pod_logs)
+        for pod_log in pod_logs.split('\n'):
+            cmds = []
+            if ((pod_log is not None) and (pod_log != '')):
+                cmd = 'scp -o StrictHostKeyChecking=no -r sysadmin@[' + site_ip + ']:/var/log/pods/' + pod_log + " " + site_name + "/."
+                cmds.append(cmd)
+                self.run_commands_scp(cmds, password, self.logger)
+
+        # 8. /var/lib/systemd/coredump/*
+        core_dump_cmd = ssh_prefix + " ls /var/lib/systemd/coredump "
+        cmds = [core_dump_cmd]
+        core_dump_logs = self.run_command_get_all_lines(cmds, password, self.logger)
+        print(core_dump_logs)
+        #for core_dump_log in core_dump_logs.split('\n'):
+        cmds = []
+        cmd = 'scp -o StrictHostKeyChecking=no -r sysadmin@[' + site_ip + ']:/var/lib/systemd/coredump/' + " " + site_name + "/."
+        cmds.append(cmd)
+        self.run_commands_scp(cmds, password, self.logger)
+
+        # create tar ball
+        date = datetime.datetime.now().strftime("%b-%d-%Y")
+        tar_file_name = site_name + '-' + date + '.tar'
+        gzip_file_name = tar_file_name + '.gz'
+        os.system('tar -cvf ' + tar_file_name + ' ./' + site_name)
+        os.system('gzip ' + tar_file_name)
+        print(gzip_file_name + " created.")
+        print("Upload it to Samsung FTP site and send a note to Samsung team. FTP coordinates:")
+        #print("    url: https://zone.samsungtelecom.com/")
+        #print("    username:vzw_femto_ndbm_jumpbox")
+        #print("    password:S@msung123")
+        suser = "verizon_snap"
+        shost = "zone.samsungtelecom.com"
+        spasswd = "VZWSn@p786"
+        print("     ur: "+shost+"/ 65.169.250.6")
+        print("     username:"+suser)
+        print("     password:"+spasswd)
+        # sftp the file up to samsung
+        remotepath = '/VZW_4G_5G/Controller\ Logs'
+        p = pexpect.spawn('sftp %s@%s' %(suser,shost))
+        p.logfile = sys.stdout.buffer
+        try:
+            p.expect('(?i)password:')
+            x = p.sendline(spasswd)
+            x = p.expect(['Permission denied','sftp>'])
+            if x == 0:
+                print ('Permission denied for password:')
+                print (spasswd)
+                p.kill(0)
+            else:
+                x = p.sendline('cd ' + remotepath)
+                x = p.expect('sftp>')
+                x = p.sendline('put ' + gzip_file_name)
+                x = p.expect('sftp>')
+                x = p.isalive()
+                x = p.close()
+                retval = p.exitstatus
+        except EOF:
+            print (str(p))
+            print ('SFTP file transfer failed due to premature end of file.')
+        except TIMEOUT:
+            print (str(p))
+            print ('SFTP file transfer failed due to timeout.')
+
+    def _handle_collect(self, site_name, site_ip, password):
+        ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t sysadmin@' + site_ip
+        collect_cmd = ssh_prefix + " collect all"
+        cmds = []
+        cmds.append(collect_cmd)
+        all_lines = self._run_collect(cmds, password)
+        for line in all_lines.split("\n"):
+            if 'creating' in line:
+                parts = line.split(" ")
+                collect_file_location = parts[3]
+                print("Collect file location:" + collect_file_location)
+
+                cmds = []
+                cmd = 'scp -o StrictHostKeyChecking=no sysadmin@[' + site_ip + ']:' + collect_file_location + " " + site_name + "/."
+                cmds.append(cmd)
+                self.run_commands_scp(cmds, password, self.logger)
+
+    def collect_windriver_logs_steps(self, site_name, site_ip, password):
+        source_dir = site_name
+        if not os.path.exists(source_dir):
+            os.makedirs(source_dir)
+
+        # 1. collect
+        self._handle_collect(site_name, site_ip, password)
+
+        # 2. network interface details
+        ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t sysadmin@' + site_ip
+        port_list_cmd = ssh_prefix + " \"source /etc/platform/openrc; system host-port-list controller-0 | grep -v name | grep -v '+' | cut -d ' ' -f 4\""
+        cmds = []
+        cmds.append(port_list_cmd)
+        all_lines = self.run_command_get_all_lines(cmds, password, self.logger)
+
+        for interface in all_lines.split("\n"):
+            if 'Connection' not in interface:
+                print("Interface..." + interface)
+                interface_file = site_name + "/" + interface + ".txt"
+                fp = open(interface_file, "a")
+
+                raw_cmd = " /usr/sbin/ethtool -i " + interface
+                cmd1 = ssh_prefix + raw_cmd
+                print(cmd1)
+                self._execute_cmd_and_save_op(cmd1, raw_cmd, password, fp)
+
+                raw_cmd = " /usr/sbin/ethtool -l " + interface
+                cmd1 = ssh_prefix + raw_cmd
+                self._execute_cmd_and_save_op(cmd1, raw_cmd, password, fp)
+
+                raw_cmd = " /usr/sbin/ethtool -S " + interface
+                cmd1 = ssh_prefix + raw_cmd
+                self._execute_cmd_and_save_op(cmd1, raw_cmd, password, fp)
+
+                fp.close()
+
+        # 3. interrupts
+        interrupt_file = site_name + "/interrupts.txt"
+        fp = open(interrupt_file, "a")
+        raw_cmd = " cat /proc/interrupts"
+        cmd1 = ssh_prefix + raw_cmd
+        self._execute_cmd_and_save_op(cmd1, raw_cmd, password, fp)
+        fp.close()
+
+        debugLog = site_name + "/wr-debug.log"
+        fp = open(debugLog, "w")
+
+        # 4. date
+        raw_cmd = " date  "
+        date = ssh_prefix + raw_cmd
+        self._execute_cmd_and_save_op(date, raw_cmd, password, fp)
+
+        # 5. ipmitool
+        raw_cmd = " sudo ipmitool sel time get "
+        cmd1 = ssh_prefix + raw_cmd
+        print(cmd1)
+        self._execute_cmd_and_save_op(cmd1, raw_cmd, password, fp, additional_password=True)
+
+        # 6. uptime
+        raw_cmd = " uptime "
+        uptime = ssh_prefix + raw_cmd
+        self._execute_cmd_and_save_op(uptime, raw_cmd, password, fp)
+
+        # 7. host_if_list
+        raw_cmd = "system host-if-list controller-0"
+        host_if_list = ssh_prefix + " source /etc/platform/openrc; " + raw_cmd
+        self._execute_cmd_and_save_op(host_if_list, raw_cmd, password, fp)
+
+        # 8. interface_network_list
+        raw_cmd = "system interface-network-list controller-0"
+        interface_network_list = ssh_prefix + " source /etc/platform/openrc; " + raw_cmd
+        self._execute_cmd_and_save_op(interface_network_list, raw_cmd, password, fp)
+
+        # 9. fm alarm-list
+        raw_cmd = " fm alarm-list"
+        fm_alarm_list = ssh_prefix + " source /etc/platform/openrc; " + raw_cmd
+        self._execute_cmd_and_save_op(fm_alarm_list, raw_cmd, password, fp)
+
+        # 5. fm event-list
+        #raw_cmd = " fm event-list"
+        #fm_event_list = ssh_prefix + " source /etc/platform/openrc; " + raw_cmd
+        #self._execute_cmd_and_save_op(fm_event_list, raw_cmd, password, fp)
+
+        # 10. sw-patch query
+        raw_cmd = " /usr/sbin/sw-patch query"
+        sw_patch_query = ssh_prefix + " source /etc/platform/openrc; " + raw_cmd
+        self._execute_cmd_and_save_op(sw_patch_query, raw_cmd, password, fp)
+        fp.close()
+
+        # 11. drive details
+        drive_list_cmd = ssh_prefix + " \"source /etc/platform/openrc; system host-disk-list controller-0 | grep \"/dev/disk\" | awk '{print $4}' \""
+        cmds = []
+        cmds.append(drive_list_cmd)
+        all_lines = self.run_command_get_all_lines(cmds, password, self.logger)
+
+        if not os.path.exists(site_name + "/dev"):
+            os.makedirs(site_name + "/dev")
+
+        for drive in all_lines.split("\n"):
+            if 'Connection' not in drive:
+                print("Drive..." + drive)
+                drive_file = site_name + drive + ".txt"
+                fp = open(drive_file, "w")
+                raw_cmd = " sudo smartctl -a  " + drive
+                cmd1 = ssh_prefix + raw_cmd
+                print(cmd1)
+                self._execute_cmd_and_save_op(cmd1, raw_cmd, password, fp, additional_password=True)
+                fp.close()
+
+        # create tar ball
+        tar_file_name = source_dir + '.tar'
+        gzip_file_name = tar_file_name + '.gz'
+        os.system('tar -cvf ' + tar_file_name + ' ./' + source_dir)
+        os.system('gzip ' + tar_file_name)
+        #with tarfile.open(tar_file_name, "w:gz") as tar:
+        #    tar.add(source_dir, arcname=os.path.basename(source_dir))
+        print(gzip_file_name + " created.")
+
+    def collect_windriver_logs(self, site_name, site_ip, password):
+        source_dir = site_name
+        if not os.path.exists(source_dir):
+            os.makedirs(source_dir)
+
+        ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t sysadmin@' + site_ip
+
+        # Running collect
+        cmds = []
+        cmd = ssh_prefix + " collect"
+        cmds.append(cmd)
+        all_lines = self._run_collect(cmds, password)
+        for line in all_lines.split("\n"):
+            if 'creating' in line:
+                parts = line.split(" ")
+                collect_file_location = parts[3]
+                print("Collect file location:" + collect_file_location)
+
+    def _execute_cmd_and_save_op(self, cmd, raw_cmd, password, fp, additional_password=False):
+        cmds = []
+        #print(cmd)
+        cmds.append(cmd)
+        all_lines = self.run_command_get_all_lines(cmds, password, self.logger, additional_password=additional_password)
+        #print(all_lines)
+        fp.write(raw_cmd + ":\n")
+        fp.write("-------------------------\n")
+        for line in all_lines.split("\n"):
+            if 'Connection' not in line:
+                fp.write(line)
+                fp.write("\n")
+        fp.write("\n")
+
+    def _run_collect(self, commands, host_password, cmd_timeout=None):
+        all_lines = []
+        for command in commands:
+            print(command)
+            child = pexpect.spawn(command)
+            child.timeout=cmd_timeout
+            try:
+                child.expect_exact(['password: '], timeout=cmd_timeout)
+                child.sendline(host_password)
+                child.expect_exact(['[sudo] password for sysadmin:'], timeout=cmd_timeout)
+                child.sendline(host_password)
+                #child.sendline("\n")
+                line = child.readline()
+                print(line)
+                line = child.readline()
+                print(line)
+                all_lines = child.read()
+                all_lines = all_lines.rstrip().lstrip()
+                all_lines = all_lines.decode('utf-8').replace('\r\n', '\n')
+                print(all_lines)
+            except:
+                print(str(child))
+        return all_lines
+
+    def run_command_get_all_lines(self, commands, host_password, logger, block=False, timeout=None, additional_password=False):
+        #print("Inside _run_command_get_all_lines")
+        all_lines = []
+        for command in commands:
+            print("Executing.." + command)
+            child = pexpect.spawn(command)
+            child.timeout=timeout
+            try:
+                i = child.expect(['password: ','Connection refused\r\r\n'], timeout=timeout)
+                if i == 0:
+                    child.sendline(host_password)
+                    if additional_password:
+                        child.expect_exact("Password:")
+                        child.sendline(host_password)
+                    all_lines = child.read()
+                    all_lines = all_lines.rstrip().lstrip()
+                    all_lines = all_lines.decode('utf-8').replace('\r\n', '\n')
+                    #logger.info(all_lines)
+                if i == 1:
+                    print("Connection refused")
+            except:
+                print(str(child))
+        return all_lines
+
+    def run_commands_scp(self, commands, host_password, logger, block=False, timeout=None):
+        successful = False
+        #print("Inside run_command_scp")
+        #print(commands)
+        for command in commands:
+            print("Executing.." + command)
+            try:
+                child = pexpect.spawn(str(command))
+            except:
+                print(str(child))
+            child.timeout = None
+            try:
+                i = child.expect([b'', 'Are you .*(yes/no)? ', pexpect.EOF], timeout=None)
+                if i == 0:
+                    child.sendline('') 
+                if i == 1:
+                    child.sendline('yes')
+                child.expect(['password: '], timeout=timeout)
+                child.sendline(host_password)
+                all_lines = child.read()
+                all_lines = all_lines.rstrip().lstrip()
+                all_lines = all_lines.decode('utf-8').replace('\r\n', '\n')
+                successful = True # Tentative
+                for line in all_lines.split("\n"):
+                    #print(line)
+                    if re.search('error', line, re.IGNORECASE):
+                        successful = False 
+                    if re.search('unable', line, re.IGNORECASE):
+                        successful = False 
+            except:
+            	pass
+                #print(str(child))
+        #print("Status:" + str(successful))
+        return successful
+
+if __name__ == '__main__':
+
+    if len(sys.argv) < 5:
+        print("Samsung logs:")
+        print("  python collect-logs.py samsung <site-name> <site-ip> <password>")
+        print("WindRiver logs:")
+        print("  python collect-logs.py windriver <site-name> <site-ip> <password>")
+        print("One-off command o/p:")
+        print("  python collect-logs.py command <site-name> <site-ip> <password> \"<command>\"")
+        print("  python3 collect-logs.py command longmeadow-oct8 2001:4888:2a10:e0ad:122:40a:0:f400 password \"ifconfig\"")
+        print("  python3 collect-logs.py command longmeadow-oct8 2001:4888:2a10:e0ad:122:40a:0:f400 password \"ip link\"")
+        exit(0)
+
+    logs_for = sys.argv[1]
+    site_name = sys.argv[2]
+    site_ip = sys.argv[3]
+    password = sys.argv[4]
+
+    print("Option chosen:" + logs_for)
+    print("Site:" + site_name)
+    print("Site IP:" + site_ip)
+    print("Site Password:" + password)
+    log_collector = LogCollector()
+
+    if logs_for == 'samsung':
+        log_collector.collect_samsung_logs(site_name, site_ip, password)
+    elif logs_for == 'windriver':
+        log_collector.collect_windriver_logs_steps(site_name, site_ip, password)
+    else:
+        command = sys.argv[5]
+        #print(command)
+        log_collector.run_command_and_collect_output(site_name, site_ip, password, command)