import django import os import threading from django.conf import settings from django.core.exceptions import AppRegistryNotReady from django.utils import timezone from .utils import * try: django.setup() from .models import ImageSync from caas.models import Cluster from caas.models import Location from caas.models import Namespace except django.core.exceptions.AppRegistryNotReady as exp: pass class RemoteRegionWorker: def __init__(self): pass def _get_crd_details(self): crd_cluster_role = "nad" crd_api_group = "k8s.cni.cncf.io" crd_api_resources = "network-attachment-definitions" crd_api_verbs = "*" return crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs def _read_remote_openrc(self, remote_region_oam_ip, logger): #cmd = "scp @[fd00:4888:2000:120c::290]:/etc/platform/openrc . logger.info("Inside _read_remote_openrc") host_username = settings.HOST_CREDS['username'] host_password = settings.HOST_CREDS['password'] ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + remote_region_oam_ip env_string = "" cmds_scp = [] folder = get_temp_file_location() logger.info("Tmp file location:" + folder) #tmpFilePath = os.path.join(os.path.dirname(__file__), "./" + folder) tmpFileName = 'openrc_' + str(remote_region_oam_ip) cmds_scp.append('scp -o StrictHostKeyChecking=no ' + host_username + '@[' + remote_region_oam_ip + ']:/etc/platform/openrc ' + folder + '/' + tmpFileName) logger.info(cmds_scp) run_commands_scp(cmds_scp, host_password, logger, block=True) logger.info("About to create env string") fp = open(folder + '/' + tmpFileName) lines = fp.readlines() logger.info(lines) password_line = '' for line in lines: line = line.lstrip().rstrip() logger.info(line) parts = line.split(' ') if len(parts) == 2: if parts[0] == 'export': if parts[1]: env_var = parts[1].lstrip().rstrip() logger.info(env_var) if 'OS_PASSWORD' not in env_var: env_string = env_string + " " + env_var if len(parts) >= 2 and 'OS_PASSWORD=' in parts[1]: logger.info("Parsing password") password_parts = line.split('PASSWORD=') password_command_parts = password_parts[1].split(' ') password_line = password_command_parts[1].rstrip().lstrip() logger.info("Password line:" + password_line) logger.info("Deleting openrc file ") # Delete file if os.path.exists(folder + '/' + tmpFileName): os.remove(folder + '/' + tmpFileName) logger.info("Looking for password") # Get password password_cmd = [] #cmd = 'TERM=linux /opt/platform/.keyring/20.06/.CREDENTIAL 2>/dev/null' cmd_to_run = ssh_prefix + " " + password_line logger.info(cmd_to_run) successful, value = self._run_command_get_output([cmd_to_run], host_password, logger) password_val = host_password #if successful: # if value != '': # password_val = value os_password = "OS_PASSWORD=" + password_val env_string = env_string + " " + os_password logger.info("Env string:" + env_string) return env_string def _run_command_get_all_lines(self, commands, host_password, logger, block=False, timeout=None): logger.info("Inside _run_command_get_all_lines") all_lines = [] for command in commands: logger.info(" Executing.." + str(command)) child = pexpect.spawn(command) child.timeout=timeout try: i = child.expect(['password: ','Connection refused\r\r\n'], timeout=timeout) if i == 0: child.sendline(host_password) all_lines = child.read() all_lines = all_lines.rstrip().lstrip() all_lines = all_lines.decode('utf-8').replace('\r\n', '\n') logger.info(all_lines) if i == 1: logger.info("Connection refused") except: logger.info(str(child)) return all_lines def _run_command_get_output(self, commands, host_password, logger, block=False, timeout=None): successful = False value_to_return = '' logger.info("Inside _run_command_get_output") for command in commands: logger.info(" Executing.." + str(command)) child = pexpect.spawn(command) child.timeout=timeout try: child.expect(['password: '], timeout=timeout) child.sendline(host_password) all_lines = child.read() all_lines = all_lines.rstrip().lstrip() all_lines = all_lines.decode('utf-8').replace('\r\n', '\n') successful = True # Tentative for line in all_lines.split("\n"): logger.info(line) if value_to_return == '': value_to_return = line.strip() if re.search('error', line, re.IGNORECASE): successful = False if re.search('unable', line, re.IGNORECASE): successful = False except: logger.info(str(child)) logger.info("Status:" + str(successful) + " value_to_return:" + value_to_return) return successful, value_to_return def _get_remote_region_oam_ip(self, cluster_name, logger): remoteclusterObj = Cluster.objects.filter(cluster_name=cluster_name) oam_ip = remoteclusterObj[0].oam_vip_address logger.info(" Remote region:" + cluster_name + " OAM IP:" + str(oam_ip)) return oam_ip def _get_central_region_name(self, oam_vip_address, logger): logger.info("1") remoteclusterObj = Cluster.objects.filter(oam_vip_address=oam_vip_address) logger.info("2") logger.info(remoteclusterObj) cluster_name = remoteclusterObj[0].cluster_name logger.info(" Remote region:" + str(oam_vip_address) + " Cluster Name:" + str(cluster_name)) return cluster_name def _get_namespace(self, region, logger): # Lookup database and findout namespace given region logger.info(" Inside _get_namespace") remoteclusterObj = Cluster.objects.filter(cluster_name=region) if len(remoteclusterObj) > 0: logger.info(" RemoteClusterObj:" + str(remoteclusterObj)) namespace_id = remoteclusterObj[0].namespace_id logger.info(" Namespace id:" + str(namespace_id)) namespaceObj = Namespace.objects.filter(id=namespace_id) logger.info(" NamespaceObj:" + str(namespaceObj)) namespace_name = namespaceObj[0].namespace_name logger.info(" Remote region:" + region + " Namespace:" + namespace_name) return namespace_name else: return "" def _get_central_region_oam_ip(self, remoteregion, transaction_id): remoteclusterObj = Cluster.objects.filter(cluster_name=remoteregion) parent_cluster_id = remoteclusterObj[0].parent_cluster_id self.logger.info(str(transaction_id) + " Parent cluster id.." + str(parent_cluster_id)) centralclusterObj = Cluster.objects.filter(id=parent_cluster_id) self.logger.info(str(transaction_id) + " " + str(centralclusterObj)) central_region_list = [] for central_region in centralclusterObj: central_region_list.append(central_region.oam_vip_address) self.logger.info(str(transaction_id) + " Central Region List:" + ','.join(central_region_list)) return central_region_list def get_fuze_spm_site_details(self, cluster_name, logger): fuze_spm_site_name = '' fuze_spm_site_id = '' logger.info(" Inside _get_fuze_spm_site_name " + str(cluster_name)) remoteclusterObj = Cluster.objects.filter(cluster_name=cluster_name) if len(remoteclusterObj) > 0: fuze_id = remoteclusterObj[0].location_id locationObj = Location.objects.filter(id=fuze_id) if len(locationObj) > 0: fuze_spm_site_name = locationObj[0].fuze_spm_site_name fuze_spm_site_id = locationObj[0].fuze_spm_site_id logger.info(" Fuze site name:" + fuze_spm_site_name) logger.info(" Fuze site id:" + fuze_spm_site_id) return fuze_spm_site_name, fuze_spm_site_id def check_namespaces(self, remote_region_oam_ip, logger): host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) cmds = [] cmds.append(ssh_prefix + " kubectl get namespaces --kubeconfig=/etc/kubernetes/admin.conf ") output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True) #logger.info("Returned output lines:") #logger.info(output_lines) new_op_lines = [] for line in output_lines.split("\n"): if not 'Connection to' in line: new_op_lines.append(line) return new_op_lines def check_namespace_secrets(self, region, remote_region_oam_ip, logger): namespace = self._get_namespace(region, logger) host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) cmds = [] cmds.append(ssh_prefix + " kubectl get secrets --kubeconfig=/etc/kubernetes/admin.conf -n " + namespace) output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True) #logger.info("Returned output lines:") #logger.info(output_lines) new_op_lines = [] for line in output_lines.split("\n"): if not 'Connection to' in line: new_op_lines.append(line) return new_op_lines def check_namespace_serviceaccounts(self, region, remote_region_oam_ip, logger): namespace = self._get_namespace(region, logger) host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) cmds = [] cmds.append(ssh_prefix + " kubectl get serviceaccounts --kubeconfig=/etc/kubernetes/admin.conf -n " + namespace) output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True) #logger.info("Returned output lines:") #logger.info(output_lines) new_op_lines = [] for line in output_lines.split("\n"): if not 'Connection to' in line: new_op_lines.append(line) return new_op_lines def check_online_status(self, region, remote_region_oam_ip, logger): host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) central_region_list = self._get_central_region_oam_ip(region, "-1") region_online_status = [] new_op_lines = [] for central_region_ip in central_region_list: cmds = [] cmd = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + central_region_ip + ' ' + '"source /etc/platform/openrc; dcmanager subcloud list | grep ' + region + '"' cmds.append(cmd) output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True) logger.info(output_lines) for line in output_lines.split("\n"): if not 'Connection to' in line: new_op_lines.append(line) return new_op_lines def create_host_network(self, remote_region_oam_ip, logger): logger.info("Inside _create_host_network") #host_username = settings.HOST_CREDS['username'] #host_password = settings.HOST_CREDS['password'] #ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + remote_region_oam_ip host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) env_string = self._read_remote_openrc(remote_region_oam_ip, logger) #basecmd = " OS_ENDPOINT_TYPE=internalURL CINDER_ENDPOINT_TYPE=internalURL OS_USERNAME=admin" #basecmd = basecmd + " OS_PASSWORD=`TERM=linux /opt/platform/.keyring/20.06/.CREDENTIAL 2>/dev/null`" #basecmd = basecmd + " OS_AUTH_TYPE=password OS_AUTH_URL=http://[fd00:4888:2000:120b::220]:5000/v3" #basecmd = basecmd + " OS_PROJECT_NAME=admin OS_USER_DOMAIN_NAME=Default OS_PROJECT_DOMAIN_NAME=Default" #basecmd = basecmd + " OS_IDENTITY_API_VERSION=3 OS_REGION_NAME=subcloud2 OS_INTERFACE=internal" basecmd = env_string cmds = ["system host-lock controller-0", "system host-if-modify -n f1u -c pci-sriov --num-vfs 8 controller-0 ens3f1 --vf-driver=vfio", "system host-if-add -c pci-sriov controller-0 f1c vf f1u --num-vfs 4 --vf-driver=netdevice", "system host-if-modify controller-0 f1u --imtu=1956", "system host-if-modify controller-0 f1c --imtu=1956", "system datanetwork-add f1u vlan --mtu=1956", "system datanetwork-add f1c vlan --mtu=1956", "system interface-datanetwork-assign controller-0 f1u f1u", "system interface-datanetwork-assign controller-0 f1c f1c", "system host-if-add -c pci-sriov controller-0 fh0m vf fh0 --num-vfs 4 --vf-driver=netdevice", "system host-if-modify controller-0 fh0m --imtu=9000", "system datanetwork-add fh0m flat", "system interface-datanetwork-assign controller-0 fh0m fh0m", "system host-if-modify -n fh1 -c pci-sriov --num-vfs 8 controller-0 enp181s0f0 --vf-driver=vfio", "system host-if-modify controller-0 fh1 --imtu=9000", "system datanetwork-add fh1 vlan --mtu=9000", "system interface-datanetwork-assign controller-0 fh1 fh1"] # New steps proposed by Eddy - These do not seem to create fh0m so commenting out. #cmds = ["system host-lock controller-0", # "system host-if-modify -n f1c -c pci-sriov --num-vfs 8 controller-0 ens3f1 --vf-driver=netdevice", # "system host-if-add -c pci-sriov controller-0 f1u vf f1c --num-vfs 4 --vf-driver=vfio", # "system host-if-modify controller-0 f1u --imtu=1956", # "system host-if-modify controller-0 f1c --imtu=1956", # "system datanetwork-add f1u vlan --mtu=1956", # "system datanetwork-add f1c vlan --mtu=1956", # "system interface-datanetwork-assign controller-0 f1u f1u", # "system interface-datanetwork-assign controller-0 f1c f1c", # # this one is wrong as well but for some reason I think the fh0 is setup in Carlos deployment config. So we have to delete the fh0 and recreate both. # "system host-if-modify controller0 fh0 -nc none" # "system host-if-modify -n fh0m -c pci-sriov --num-vfs 8 controller-0 ens179s0f0 --vf-driver=netdevice", # "system host-if-add -c pci-sriov controller-0 fh0 vf fh0m --num-vfs 4 --vf-driver=vfio", # "system host-if-modify controller-0 fh0m --imtu=9000", # "system datanetwork-add fh0m flat", # "system interface-datanetwork-assign controller-0 fh0m fh0m", # "system host-if-modify -n fh1 -c pci-sriov --num-vfs 8 controller-0 enp181s0f0 --vf-driver=vfio", # "system host-if-modify controller-0 fh1 --imtu=9000", # "system datanetwork-add fh1 vlan --mtu=9000", # "system interface-datanetwork-assign controller-0 fh1 fh1"] for cmd in cmds: cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd logger.info(cmd_to_run) run_commands([cmd_to_run], host_password, logger, block=True) # host_network_configured = False # while not host_network_configured: # host_network_configured = self._verify_host_network(ssh_prefix, basecmd, host_password, logger) # if not host_network_configured: # cmds = ["system host-lock controller-0"] # for cmd in cmds: # cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd # logger.info(cmd_to_run) # output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True) # logger.info("Returned output lines:") # logger.info(output_lines) # time.sleep(3) cmds = ["system host-unlock controller-0"] while True: unlock_wait = False for cmd in cmds: cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd logger.info(cmd_to_run) output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True) logger.info("Returned output lines:") logger.info(output_lines) if len(output_lines) > 0: for line in output_lines.split("\n"): logger.info("Line:" + line) if not unlock_wait: if 'retry host-unlock' in line or 'Rejected' in line: logger.info("Need to wait to call host-unlock ##### ") unlock_wait = True break if unlock_wait: time.sleep(60) else: break # for cmd in cmds: # cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd # logger.info(cmd_to_run) # output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True) # logger.info("Returned output lines:") # logger.info(output_lines) logger.info("Done setting up host network")