diff options
Diffstat (limited to 'src/orchestration/remoteregionhandler.py')
| -rw-r--r-- | src/orchestration/remoteregionhandler.py | 375 |
1 files changed, 375 insertions, 0 deletions
diff --git a/src/orchestration/remoteregionhandler.py b/src/orchestration/remoteregionhandler.py new file mode 100644 index 0000000..6754672 --- /dev/null +++ b/src/orchestration/remoteregionhandler.py @@ -0,0 +1,375 @@ +import django +import os +import threading + +from django.conf import settings +from django.core.exceptions import AppRegistryNotReady +from django.utils import timezone + +from .utils import * + +try: + django.setup() + from .models import ImageSync + from caas.models import Cluster + from caas.models import Location + from caas.models import Namespace +except django.core.exceptions.AppRegistryNotReady as exp: + pass + +class RemoteRegionWorker: + + def __init__(self): + pass + + def _get_crd_details(self): + crd_cluster_role = "nad" + crd_api_group = "k8s.cni.cncf.io" + crd_api_resources = "network-attachment-definitions" + crd_api_verbs = "*" + return crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs + + def _read_remote_openrc(self, remote_region_oam_ip, logger): + #cmd = "scp @[fd00:4888:2000:120c::290]:/etc/platform/openrc . + + logger.info("Inside _read_remote_openrc") + host_username = settings.HOST_CREDS['username'] + host_password = settings.HOST_CREDS['password'] + ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + remote_region_oam_ip + env_string = "" + cmds_scp = [] + folder = get_temp_file_location() + logger.info("Tmp file location:" + folder) + #tmpFilePath = os.path.join(os.path.dirname(__file__), "./" + folder) + tmpFileName = 'openrc_' + str(remote_region_oam_ip) + cmds_scp.append('scp -o StrictHostKeyChecking=no ' + host_username + '@[' + remote_region_oam_ip + ']:/etc/platform/openrc ' + folder + '/' + tmpFileName) + logger.info(cmds_scp) + run_commands_scp(cmds_scp, host_password, logger, block=True) + logger.info("About to create env string") + + fp = open(folder + '/' + tmpFileName) + lines = fp.readlines() + logger.info(lines) + password_line = '' + for line in lines: + line = line.lstrip().rstrip() + logger.info(line) + parts = line.split(' ') + if len(parts) == 2: + if parts[0] == 'export': + if parts[1]: + env_var = parts[1].lstrip().rstrip() + logger.info(env_var) + if 'OS_PASSWORD' not in env_var: + env_string = env_string + " " + env_var + if len(parts) >= 2 and 'OS_PASSWORD=' in parts[1]: + logger.info("Parsing password") + password_parts = line.split('PASSWORD=') + password_command_parts = password_parts[1].split(' ') + password_line = password_command_parts[1].rstrip().lstrip() + logger.info("Password line:" + password_line) + + logger.info("Deleting openrc file ") + + # Delete file + if os.path.exists(folder + '/' + tmpFileName): + os.remove(folder + '/' + tmpFileName) + + logger.info("Looking for password") + # Get password + password_cmd = [] + #cmd = 'TERM=linux /opt/platform/.keyring/20.06/.CREDENTIAL 2>/dev/null' + cmd_to_run = ssh_prefix + " " + password_line + logger.info(cmd_to_run) + successful, value = self._run_command_get_output([cmd_to_run], host_password, logger) + password_val = host_password + #if successful: + # if value != '': + # password_val = value + os_password = "OS_PASSWORD=" + password_val + env_string = env_string + " " + os_password + + logger.info("Env string:" + env_string) + return env_string + + def _run_command_get_all_lines(self, commands, host_password, logger, block=False, timeout=None): + logger.info("Inside _run_command_get_all_lines") + all_lines = [] + for command in commands: + logger.info(" Executing.." + str(command)) + child = pexpect.spawn(command) + child.timeout=timeout + try: + i = child.expect(['password: ','Connection refused\r\r\n'], timeout=timeout) + if i == 0: + child.sendline(host_password) + all_lines = child.read() + all_lines = all_lines.rstrip().lstrip() + all_lines = all_lines.decode('utf-8').replace('\r\n', '\n') + logger.info(all_lines) + if i == 1: + logger.info("Connection refused") + except: + logger.info(str(child)) + return all_lines + + def _run_command_get_output(self, commands, host_password, logger, block=False, timeout=None): + successful = False + value_to_return = '' + logger.info("Inside _run_command_get_output") + for command in commands: + logger.info(" Executing.." + str(command)) + child = pexpect.spawn(command) + child.timeout=timeout + try: + child.expect(['password: '], timeout=timeout) + child.sendline(host_password) + all_lines = child.read() + all_lines = all_lines.rstrip().lstrip() + all_lines = all_lines.decode('utf-8').replace('\r\n', '\n') + successful = True # Tentative + for line in all_lines.split("\n"): + logger.info(line) + if value_to_return == '': + value_to_return = line.strip() + if re.search('error', line, re.IGNORECASE): + successful = False + if re.search('unable', line, re.IGNORECASE): + successful = False + except: + logger.info(str(child)) + logger.info("Status:" + str(successful) + " value_to_return:" + value_to_return) + return successful, value_to_return + + def _get_remote_region_oam_ip(self, cluster_name, logger): + remoteclusterObj = Cluster.objects.filter(cluster_name=cluster_name) + oam_ip = remoteclusterObj[0].oam_vip_address + logger.info(" Remote region:" + cluster_name + " OAM IP:" + str(oam_ip)) + return oam_ip + + def _get_central_region_name(self, oam_vip_address, logger): + logger.info("1") + remoteclusterObj = Cluster.objects.filter(oam_vip_address=oam_vip_address) + logger.info("2") + logger.info(remoteclusterObj) + cluster_name = remoteclusterObj[0].cluster_name + logger.info(" Remote region:" + str(oam_vip_address) + " Cluster Name:" + str(cluster_name)) + return cluster_name + + def _get_namespace(self, region, logger): + # Lookup database and findout namespace given region + logger.info(" Inside _get_namespace") + remoteclusterObj = Cluster.objects.filter(cluster_name=region) + if len(remoteclusterObj) > 0: + logger.info(" RemoteClusterObj:" + str(remoteclusterObj)) + namespace_id = remoteclusterObj[0].namespace_id + logger.info(" Namespace id:" + str(namespace_id)) + namespaceObj = Namespace.objects.filter(id=namespace_id) + logger.info(" NamespaceObj:" + str(namespaceObj)) + namespace_name = namespaceObj[0].namespace_name + logger.info(" Remote region:" + region + " Namespace:" + namespace_name) + return namespace_name + else: + return "" + + def _get_central_region_oam_ip(self, remoteregion, transaction_id): + remoteclusterObj = Cluster.objects.filter(cluster_name=remoteregion) + parent_cluster_id = remoteclusterObj[0].parent_cluster_id + self.logger.info(str(transaction_id) + " Parent cluster id.." + str(parent_cluster_id)) + centralclusterObj = Cluster.objects.filter(id=parent_cluster_id) + self.logger.info(str(transaction_id) + " " + str(centralclusterObj)) + central_region_list = [] + for central_region in centralclusterObj: + central_region_list.append(central_region.oam_vip_address) + self.logger.info(str(transaction_id) + " Central Region List:" + ','.join(central_region_list)) + return central_region_list + + def get_fuze_spm_site_details(self, cluster_name, logger): + fuze_spm_site_name = '' + fuze_spm_site_id = '' + logger.info(" Inside _get_fuze_spm_site_name " + str(cluster_name)) + remoteclusterObj = Cluster.objects.filter(cluster_name=cluster_name) + if len(remoteclusterObj) > 0: + fuze_id = remoteclusterObj[0].location_id + locationObj = Location.objects.filter(id=fuze_id) + if len(locationObj) > 0: + fuze_spm_site_name = locationObj[0].fuze_spm_site_name + fuze_spm_site_id = locationObj[0].fuze_spm_site_id + logger.info(" Fuze site name:" + fuze_spm_site_name) + logger.info(" Fuze site id:" + fuze_spm_site_id) + return fuze_spm_site_name, fuze_spm_site_id + + def check_namespaces(self, remote_region_oam_ip, logger): + + host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) + cmds = [] + cmds.append(ssh_prefix + " kubectl get namespaces --kubeconfig=/etc/kubernetes/admin.conf ") + + output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True) + #logger.info("Returned output lines:") + #logger.info(output_lines) + + new_op_lines = [] + for line in output_lines.split("\n"): + if not 'Connection to' in line: + new_op_lines.append(line) + + return new_op_lines + + def check_namespace_secrets(self, region, remote_region_oam_ip, logger): + namespace = self._get_namespace(region, logger) + host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) + cmds = [] + cmds.append(ssh_prefix + " kubectl get secrets --kubeconfig=/etc/kubernetes/admin.conf -n " + namespace) + + output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True) + #logger.info("Returned output lines:") + #logger.info(output_lines) + + new_op_lines = [] + for line in output_lines.split("\n"): + if not 'Connection to' in line: + new_op_lines.append(line) + + return new_op_lines + + def check_namespace_serviceaccounts(self, region, remote_region_oam_ip, logger): + namespace = self._get_namespace(region, logger) + host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) + cmds = [] + cmds.append(ssh_prefix + " kubectl get serviceaccounts --kubeconfig=/etc/kubernetes/admin.conf -n " + namespace) + + output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True) + #logger.info("Returned output lines:") + #logger.info(output_lines) + + new_op_lines = [] + for line in output_lines.split("\n"): + if not 'Connection to' in line: + new_op_lines.append(line) + + return new_op_lines + + def check_online_status(self, region, remote_region_oam_ip, logger): + host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) + central_region_list = self._get_central_region_oam_ip(region, "-1") + region_online_status = [] + new_op_lines = [] + for central_region_ip in central_region_list: + cmds = [] + cmd = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + central_region_ip + ' ' + '"source /etc/platform/openrc; dcmanager subcloud list | grep ' + region + '"' + cmds.append(cmd) + output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True) + logger.info(output_lines) + + for line in output_lines.split("\n"): + if not 'Connection to' in line: + new_op_lines.append(line) + return new_op_lines + + def create_host_network(self, remote_region_oam_ip, logger): + logger.info("Inside _create_host_network") + + #host_username = settings.HOST_CREDS['username'] + #host_password = settings.HOST_CREDS['password'] + #ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + remote_region_oam_ip + + host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip) + + env_string = self._read_remote_openrc(remote_region_oam_ip, logger) + + #basecmd = " OS_ENDPOINT_TYPE=internalURL CINDER_ENDPOINT_TYPE=internalURL OS_USERNAME=admin" + #basecmd = basecmd + " OS_PASSWORD=`TERM=linux /opt/platform/.keyring/20.06/.CREDENTIAL 2>/dev/null`" + #basecmd = basecmd + " OS_AUTH_TYPE=password OS_AUTH_URL=http://[fd00:4888:2000:120b::220]:5000/v3" + #basecmd = basecmd + " OS_PROJECT_NAME=admin OS_USER_DOMAIN_NAME=Default OS_PROJECT_DOMAIN_NAME=Default" + #basecmd = basecmd + " OS_IDENTITY_API_VERSION=3 OS_REGION_NAME=subcloud2 OS_INTERFACE=internal" + + basecmd = env_string + + cmds = ["system host-lock controller-0", + "system host-if-modify -n f1u -c pci-sriov --num-vfs 8 controller-0 ens3f1 --vf-driver=vfio", + "system host-if-add -c pci-sriov controller-0 f1c vf f1u --num-vfs 4 --vf-driver=netdevice", + "system host-if-modify controller-0 f1u --imtu=1956", + "system host-if-modify controller-0 f1c --imtu=1956", + "system datanetwork-add f1u vlan --mtu=1956", + "system datanetwork-add f1c vlan --mtu=1956", + "system interface-datanetwork-assign controller-0 f1u f1u", + "system interface-datanetwork-assign controller-0 f1c f1c", + "system host-if-add -c pci-sriov controller-0 fh0m vf fh0 --num-vfs 4 --vf-driver=netdevice", + "system host-if-modify controller-0 fh0m --imtu=9000", + "system datanetwork-add fh0m flat", + "system interface-datanetwork-assign controller-0 fh0m fh0m", + "system host-if-modify -n fh1 -c pci-sriov --num-vfs 8 controller-0 enp181s0f0 --vf-driver=vfio", + "system host-if-modify controller-0 fh1 --imtu=9000", + "system datanetwork-add fh1 vlan --mtu=9000", + "system interface-datanetwork-assign controller-0 fh1 fh1"] + + # New steps proposed by Eddy - These do not seem to create fh0m so commenting out. + #cmds = ["system host-lock controller-0", + # "system host-if-modify -n f1c -c pci-sriov --num-vfs 8 controller-0 ens3f1 --vf-driver=netdevice", + # "system host-if-add -c pci-sriov controller-0 f1u vf f1c --num-vfs 4 --vf-driver=vfio", + # "system host-if-modify controller-0 f1u --imtu=1956", + # "system host-if-modify controller-0 f1c --imtu=1956", + # "system datanetwork-add f1u vlan --mtu=1956", + # "system datanetwork-add f1c vlan --mtu=1956", + # "system interface-datanetwork-assign controller-0 f1u f1u", + # "system interface-datanetwork-assign controller-0 f1c f1c", + # # this one is wrong as well but for some reason I think the fh0 is setup in Carlos deployment config. So we have to delete the fh0 and recreate both. + # "system host-if-modify controller0 fh0 -nc none" + # "system host-if-modify -n fh0m -c pci-sriov --num-vfs 8 controller-0 ens179s0f0 --vf-driver=netdevice", + # "system host-if-add -c pci-sriov controller-0 fh0 vf fh0m --num-vfs 4 --vf-driver=vfio", + # "system host-if-modify controller-0 fh0m --imtu=9000", + # "system datanetwork-add fh0m flat", + # "system interface-datanetwork-assign controller-0 fh0m fh0m", + # "system host-if-modify -n fh1 -c pci-sriov --num-vfs 8 controller-0 enp181s0f0 --vf-driver=vfio", + # "system host-if-modify controller-0 fh1 --imtu=9000", + # "system datanetwork-add fh1 vlan --mtu=9000", + # "system interface-datanetwork-assign controller-0 fh1 fh1"] + + for cmd in cmds: + cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd + logger.info(cmd_to_run) + run_commands([cmd_to_run], host_password, logger, block=True) + +# host_network_configured = False +# while not host_network_configured: +# host_network_configured = self._verify_host_network(ssh_prefix, basecmd, host_password, logger) +# if not host_network_configured: +# cmds = ["system host-lock controller-0"] +# for cmd in cmds: +# cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd +# logger.info(cmd_to_run) +# output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True) +# logger.info("Returned output lines:") +# logger.info(output_lines) +# time.sleep(3) + + cmds = ["system host-unlock controller-0"] + while True: + unlock_wait = False + for cmd in cmds: + cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd + logger.info(cmd_to_run) + output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True) + logger.info("Returned output lines:") + logger.info(output_lines) + if len(output_lines) > 0: + for line in output_lines.split("\n"): + logger.info("Line:" + line) + if not unlock_wait: + if 'retry host-unlock' in line or 'Rejected' in line: + logger.info("Need to wait to call host-unlock ##### ") + unlock_wait = True + break + if unlock_wait: + time.sleep(60) + else: + break + +# for cmd in cmds: +# cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd +# logger.info(cmd_to_run) +# output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True) +# logger.info("Returned output lines:") +# logger.info(output_lines) + logger.info("Done setting up host network") |