diff options
Diffstat (limited to 'tests/paas-setup-check')
| -rwxr-xr-x | tests/paas-setup-check/audit-vmb-site-list.sh | 20 | ||||
| -rw-r--r-- | tests/paas-setup-check/paas-setup-check.py | 189 | ||||
| -rw-r--r-- | tests/paas-setup-check/send-vmb-messages.py | 73 | ||||
| -rw-r--r-- | tests/paas-setup-check/site_run_command.py | 130 |
4 files changed, 412 insertions, 0 deletions
diff --git a/tests/paas-setup-check/audit-vmb-site-list.sh b/tests/paas-setup-check/audit-vmb-site-list.sh new file mode 100755 index 0000000..08e24f8 --- /dev/null +++ b/tests/paas-setup-check/audit-vmb-site-list.sh @@ -0,0 +1,20 @@ +#!/bin/bash +if [ $# -lt 2 ]; then + echo "Usage: $0 [file] [filter]" + exit 1 +fi + +filter=$2 + +while read site; do + + #grep wsbomagj-d325077-001 /var/log/apache2/far-edge-ops-api_error_log |grep reportName |grep 2020-12-01 |wc -l + commandline=$(grep $site /var/log/apache2/far-edge-ops-api_error_log |grep reportName |grep $filter | wc -l) + + if [ $commandline -ge 3 ]; then + echo "All msgs sent for ($site)" + else + echo "Failed to send all msgs to ($site)" + fi + +done < $1 diff --git a/tests/paas-setup-check/paas-setup-check.py b/tests/paas-setup-check/paas-setup-check.py new file mode 100644 index 0000000..a6a5119 --- /dev/null +++ b/tests/paas-setup-check/paas-setup-check.py @@ -0,0 +1,189 @@ +import subprocess +import os +import json +import requests +import sys +import csv +import re + +class PaaSSetupCheck(): + + def run_test(self, site_name): + try: + network_ok = False + namespace_ok = False + network_unreachable = False + cc_unreachable = False + url = 'https://middleware.vcpfe.vzwops.com/orchestration/remote-regions/' + site_name + '/site-details' + network_setup_url = 'https://middleware.vcpfe.vzwops.com/orchestration/remote-regions/' + site_name + '/setup-network' + namespace_setup_url = 'https://middleware.vcpfe.vzwops.com/orchestration/setup-cluster/' + site_name + + #url = 'http://middleware.meter.vzwops.com:3080/orchestration/remote-regions/' + site_name + '/site-details' + #network_setup_url = 'http://middleware.meter.vzwops.com:3080/orchestration/remote-regions/' + site_name + '/setup-network' + #namespace_setup_url = 'http://middleware.meter.vzwops.com:3080/orchestration/setup-cluster/' + site_name + + #namespace_setup_url_1 = 'https://middleware.vcpfe.vzwops.com/orchestration/setup-cluster/' + site_name + '?network_setup=false' + #url = 'http://localhost:8000/orchestration/remote-regions/' + site_name + '/site-details' + #network_setup_url = 'http://localhost:8000/orchestration/remote-regions/' + site_name + '/setup-network' + #namespace_setup_url = 'http://localhost:8000/orchestration/setup-cluster/' + site_name + + auth = ('SVC-Far-Edge','QxbV44IU4P7t7rN44dxK4xyPB'); + + resp = requests.get(url, auth=auth, verify=False) + if resp.status_code == 500: + network_unreachable = True + print(site_name + ' ssh network unreachable') + else: + print(resp.json()) + + cc_status = resp.json()['setup_details']['online_status'] + if not cc_status: + cc_unreachable = True + else: + namespaces = resp.json()['setup_details']['namespaces'] + for namespace in namespaces: + print('namespace: ' + namespace) + if 'Unable to connect to the server' in namespace: + network_unreachable = True + print(site_name + ' k8s network unreachable') + break + else: + x = re.search("^wsbomagj-.*[0-9]$", namespace.split(' ')[0]) + if x: + # check secrets + secrets = resp.json()['setup_details']['ns_secrets'] + total_secrets = 0 + for secret in secrets: + if 'admin-registry-secret' in secret: + total_secrets = total_secrets + 1 + if 'ceph-pool-kube-rbd' in secret: + total_secrets = total_secrets + 1 + if 'default-token' in secret: + total_secrets = total_secrets + 1 + if 'vran-serviceaccount-token' in secret: + total_secrets = total_secrets + 1 + + if total_secrets == 4: + # check service accounts + total_sa = 0 + serviceaccounts = resp.json()['setup_details']['ns_service_accounts'] + for sa in serviceaccounts: + if 'default' in sa: + total_sa = total_sa + 1 + if 'vran-serviceaccount' in sa: + total_sa = total_sa + 1 + + if total_sa == 2: + print(site_name + ': ' + namespace + ' namespace is created') + namespace_ok = True + + if not network_unreachable and not cc_unreachable: + network_no = len(resp.json()['setup_details']['network_details']) + if network_no and network_no >= 15: + print(site_name + ' network setup succeeded') + network_ok = True; + + if namespace_ok: + if network_ok: + print(site_name + " both namespace and network setup correct. yay.") + else: + print(site_name + ' namespace is okay, network setup failed.') + print('calling setup-network for: ' + site_name) + # call network_setup + network_setup_resp = requests.put(network_setup_url, auth=auth, verify=False) + else: + if network_ok: + print(site_name + " namespace setup failed, network okay.") + print('calling setup-cluster with network_setup=false query parameter.') + payload = {'network_setup': 'false'} + namespace_setup_resp = requests.get(namespace_setup_url, auth=auth, params=payload, verify=False) + else: + print(site_name + 'both namespace and network NOT setup') + namespace_setup_resp = requests.get(namespace_setup_url, auth=auth, verify=False) + print('called setup-cluster') + + except Exception as e: + print(e) + + return network_ok, namespace_ok, network_unreachable, cc_unreachable + +if __name__ == '__main__': + + if len(sys.argv) < 1: + print("Usage:") + print(" python paas-set-check.py <site_list_csv_file>") + exit(0) + + site_list_file = sys.argv[1] + print("Input WR installed site list file:" + site_list_file) + paasSetupCheck = PaaSSetupCheck() + + with open('network_setup_succeed.csv', mode='w') as succeed_file: + with open('network_setup_fail.csv', mode='w') as fail_file: + with open('namespace_setup_succeed.csv', mode='w') as namespace_succeed_file: + with open('namespace_setup_fail.csv', mode='w') as namespace_fail_file: + with open('network-namespace_setup_succeed.csv', mode='w') as network_namespace_succeed_file: + with open('network-unreachable.csv', mode='w') as network_unreachable_file: + with open('cc-unreachable.csv', mode='w') as cc_unreachable_file: + writer0 = csv.writer(succeed_file, delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL) + writer1 = csv.writer(fail_file, delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL) + writer2 = csv.writer(namespace_succeed_file, delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL) + writer3 = csv.writer(namespace_fail_file, delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL) + writer4 = csv.writer(network_namespace_succeed_file, delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL) + writer5 = csv.writer(network_unreachable_file, delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL) + writer6 = csv.writer(cc_unreachable_file, delimiter=' ', quotechar='"', quoting=csv.QUOTE_MINIMAL) + with open(site_list_file) as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + line_count = 0 + ok_count = 0 + bad_count = 0 + network_ok_count = 0 + network_not_ok_count = 0 + namespace_ok_count = 0 + namespace_not_ok_count= 0 + cc_unreachable_count = 0 + for row in csv_reader: + print(f'\t{row[0]}') + #site_name = 'wsbomagj-d325030-001'; + site_name = row[0]; + network_ok, namespace_ok, network_unreachable, cc_unreachable = paasSetupCheck.run_test(site_name) + print(network_ok, namespace_ok) + + if network_ok and namespace_ok: + writer4.writerow({site_name}) + ok_count += 1 + + if network_unreachable: + writer5.writerow({site_name}) + bad_count += 1 + + if cc_unreachable: + writer6.writerow({site_name}) + cc_unreachable_count += 1 + + if network_ok: + writer0.writerow({site_name}) + network_ok_count += 1 + elif not network_unreachable: + writer1.writerow({site_name}) + network_not_ok_count += 1 + print(site_name + ' network setup failed') + + if namespace_ok: + writer2.writerow({site_name}) + namespace_ok_count += 1 + elif not network_unreachable: + writer3.writerow({site_name}) + namespace_not_ok_count += 1 + print(site_name + ' namespace setup failed') + + line_count += 1 + + print(f'Processed {line_count} lines.') + print(f'{ok_count} of {line_count} ready-to-go sites. file: network-namespace_setup_succeed.csv') + print(f'{bad_count} of {line_count} bad sites. file: network-unreachable.csv') + print(f'{cc_unreachable_count} of {line_count} central controller unreachable sites. file: cc-unreachable.csv') + print(f'{network_ok_count} of {line_count} sites with no network problem. file: network_setup_succeed.csv') + print(f'{network_not_ok_count} of {line_count} sites with network problem. file: network_setup_fail.csv') + print(f'{namespace_ok_count} of {line_count} sites with no namespace problem. file:namespace_setup_succeed.csv') + print(f'{namespace_not_ok_count} of {line_count} sites with namespace problem. file: namespace_setup_fail.csv') diff --git a/tests/paas-setup-check/send-vmb-messages.py b/tests/paas-setup-check/send-vmb-messages.py new file mode 100644 index 0000000..d94004f --- /dev/null +++ b/tests/paas-setup-check/send-vmb-messages.py @@ -0,0 +1,73 @@ +import requests +import sys +import csv +import time + +class SendVMBMessages(): + auth = ('SVC-Far-Edge','QxbV44IU4P7t7rN44dxK4xyPB'); + def _is_network_ok(self, site_name): + url = 'https://middleware.vcpfe.vzwops.com/orchestration/remote-regions/' + site_name + '/site-details' + + resp = requests.get(url, auth=self.auth, verify=False) + if resp.status_code == 500: + print(site_name + ' ssh network unreachable') + return False, resp.status_code + + print(resp.json()) + namespaces = resp.json()['setup_details']['namespaces'] + for namespace in namespaces: + print('namespace: ' + namespace) + if 'Unable to connect to the server' in namespace: + print(site_name + ' k8s network unreachable') + return False, 500 + + network_no = len(resp.json()['setup_details']['network_details']) + if network_no and network_no >= 15: + print(site_name + ' network setup succeeded') + return True, resp.status_code + else: + return False, resp.status_code #200 + + def run_setup_cluster(self, site_name, network_setup=True): + try: + namespace_setup_url = 'https://middleware.vcpfe.vzwops.com/orchestration/setup-cluster/' + site_name + #namespace_setup_url = 'http://localhost:8000/orchestration/setup-cluster/' + site_name + print('calling setup-cluster with network_setup : ' + str(network_setup) ) + payload = {'network_setup': network_setup} + resp = requests.get(namespace_setup_url, auth=self.auth, params=payload, verify=False) + if resp.status_code == 500: + print(site_name + ' setup-cluster call failed') + else: + print('called sending message to VMB for ' + site_name) + except Exception as e: + print(e) + +if __name__ == '__main__': + + if len(sys.argv) < 1: + print("Usage:") + print(" python SendVMBMessages.py <site_list_csv_file>") + exit(0) + + site_list_file = sys.argv[1] + print("Input WR installed site list file:" + site_list_file) + sendVMBMessages = SendVMBMessages() + + with open(site_list_file) as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + line_count = 0 + for row in csv_reader: + print(f'\t{row[0]}') + #site_name = 'wsbomagj-d325030-001'; + site_name = row[0]; + + network_ok, status_code = sendVMBMessages._is_network_ok(site_name) + if status_code != 500: + if network_ok: + sendVMBMessages.run_setup_cluster(site_name, network_setup=False) + else: + sendVMBMessages.run_setup_cluster(site_name, network_setup=True) + line_count += 1 + time.sleep(3) + + print(f'Processed {line_count} lines.') diff --git a/tests/paas-setup-check/site_run_command.py b/tests/paas-setup-check/site_run_command.py new file mode 100644 index 0000000..e19afac --- /dev/null +++ b/tests/paas-setup-check/site_run_command.py @@ -0,0 +1,130 @@ +import sys +import csv +import time +import pexpect +import psycopg2 + +class RunCmdFromList(): + + def run_cmd(self, site_name): + try: + + #get oam_ip + site_ip = self._get_remote_region_oam_ip(site_name) + print('site_ip ' + site_ip) + + ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t sysadmin@' + site_ip + raw_cmd = " sudo reboot " + cmd1 = ssh_prefix + raw_cmd + print(cmd1) + self._execute_cmd_and_save_op(cmd1, raw_cmd, 'xxxxxxx', additional_password=True) + print('called ' + raw_cmd + ' for ' + site_name) + + except Exception as e: + print(e) + + def _execute_cmd_and_save_op(self, cmd, raw_cmd, password, additional_password=False): + cmds = [] + #print(cmd) + cmds.append(cmd) + all_lines = self._run_command_get_all_lines(cmds, password, additional_password=additional_password) + print(all_lines) + print(raw_cmd + ":\n") + print("-------------------------\n") + for line in all_lines.split("\n"): + if 'Connection' not in line: + print(line) + print("\n") + print("\n") + + def _run_command_get_all_lines(self, commands, host_password, timeout=None, additional_password=False): + #print("Inside _run_command_get_all_lines") + all_lines = [] + for command in commands: + print("Executing.." + command) + child = pexpect.spawn(command) + child.timeout=timeout + try: + i = child.expect(['password: ','Connection refused\r\r\n'], timeout=timeout) + if i == 0: + child.sendline(host_password) + if additional_password: + child.expect_exact("Password:") + child.sendline(host_password) + all_lines = child.read() + all_lines = all_lines.rstrip().lstrip() + all_lines = all_lines.decode('utf-8').replace('\r\n', '\n') + #logger.info(all_lines) + if i == 1: + print("Connection refused") + except: + print(str(child)) + return all_lines + + def _get_remote_region_oam_ip(self, cluster_name): + + """ Connect to the PostgreSQL database server """ + conn = None + oam_ip = None + try: + print('Connecting to the PostgreSQL database...') + + conn = psycopg2.connect( + host='xxxxxxx', + port='5432', + database="faredge", + user="faredge", + password='xxxxxx') + + # create a cursor + cur = conn.cursor() + + # execute a statement + print('select oam_vip_address from caas_wrbatch where cluster_name=' + site_name ) + sql = 'select oam_vip_address from caas_wrbatch where cluster_name=' + '\'' + site_name + '\'' + cur.execute(sql) + + #print('PostgreSQL database version:') + #cur.execute('SELECT version()') + + # display the PostgreSQL database server version + oam_ip = cur.fetchone() + print( site_name + ' ' + str(oam_ip)) + + # close the communication with the PostgreSQL + cur.close() + except (Exception, psycopg2.DatabaseError) as error: + print(error) + finally: + if conn is not None: + conn.close() + print('Database connection closed.') + + print(site_name + " OAM IP:" + oam_ip[0]) + return oam_ip[0] + + +if __name__ == '__main__': + + if len(sys.argv) < 1: + print("Usage:") + print(" python site_run_command.py <site_list_csv_file>") + exit(0) + + site_list_file = sys.argv[1] + print("Input WR installed site list file:" + site_list_file) + runCmdFromList = RunCmdFromList() + + with open(site_list_file) as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + line_count = 0 + for row in csv_reader: + print(f'\t{row[0]}') + #site_name = 'wsbomagj-d325030-001'; + site_name = row[0]; + runCmdFromList.run_cmd(site_name) + line_count += 1 + time.sleep(3) + + + print(f'Processed {line_count} lines.') |