path: root/src/orchestration/remoteregionhandler.py

import django
import os
import threading

from django.conf import settings
from django.core.exceptions import AppRegistryNotReady
from django.utils import timezone

from .utils import *

try:
    django.setup()
    from .models import ImageSync
    from caas.models import Cluster
    from caas.models import Location 
    from caas.models import Namespace  
except django.core.exceptions.AppRegistryNotReady as exp:
    pass            
                    
class RemoteRegionWorker:

    def __init__(self):
        pass

    def _get_crd_details(self):
        crd_cluster_role = "nad"
        crd_api_group = "k8s.cni.cncf.io"
        crd_api_resources = "network-attachment-definitions"
        crd_api_verbs = "*"
        return crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs

    def _read_remote_openrc(self, remote_region_oam_ip, logger):
        #cmd = "scp @[fd00:4888:2000:120c::290]:/etc/platform/openrc .

        logger.info("Inside _read_remote_openrc")
        host_username = settings.HOST_CREDS['username']
        host_password = settings.HOST_CREDS['password']
        ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + remote_region_oam_ip
        env_string = ""
        cmds_scp = []
        folder = get_temp_file_location()
        logger.info("Tmp file location:" + folder)
        #tmpFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
        tmpFileName = 'openrc_' + str(remote_region_oam_ip)
        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + host_username + '@[' + remote_region_oam_ip + ']:/etc/platform/openrc ' + folder + '/' + tmpFileName)
        logger.info(cmds_scp)
        run_commands_scp(cmds_scp, host_password, logger, block=True)
        logger.info("About to create env string")

        fp = open(folder + '/' + tmpFileName)
        lines = fp.readlines()
        logger.info(lines)
        password_line = ''
        for line in lines:
            line = line.lstrip().rstrip()
            logger.info(line)
            parts = line.split(' ')
            if len(parts) == 2:
                if parts[0] == 'export':
                    if parts[1]:
                        env_var = parts[1].lstrip().rstrip()
                        logger.info(env_var)
                        if 'OS_PASSWORD' not in env_var:
                            env_string = env_string + " " + env_var
            if len(parts) >= 2 and 'OS_PASSWORD=' in parts[1]:
                logger.info("Parsing password")
                password_parts = line.split('PASSWORD=')
                password_command_parts = password_parts[1].split(' ')
                password_line = password_command_parts[1].rstrip().lstrip()
                logger.info("Password line:" + password_line)

        logger.info("Deleting openrc file ")

        # Delete file
        if os.path.exists(folder + '/' + tmpFileName):
            os.remove(folder + '/' + tmpFileName)

        logger.info("Looking for password")
        # Get password
        password_cmd = []
        #cmd = 'TERM=linux /opt/platform/.keyring/20.06/.CREDENTIAL 2>/dev/null'
        cmd_to_run = ssh_prefix + " " + password_line
        logger.info(cmd_to_run)
        successful, value = self._run_command_get_output([cmd_to_run], host_password, logger)
        password_val = host_password
        #if successful:
        #    if value != '':
        #        password_val = value
        os_password = "OS_PASSWORD=" + password_val
        env_string = env_string + " " + os_password

        logger.info("Env string:" + env_string)
        return env_string

    def _run_command_get_all_lines(self, commands, host_password, logger, block=False, timeout=None):
        logger.info("Inside _run_command_get_all_lines")
        all_lines = []
        for command in commands:
            logger.info(" Executing.." + str(command))
            child = pexpect.spawn(command)
            child.timeout=timeout
            try:
                i = child.expect(['password: ','Connection refused\r\r\n'], timeout=timeout)
                if i == 0:
                    child.sendline(host_password)
                    all_lines = child.read()
                    all_lines = all_lines.rstrip().lstrip()
                    all_lines = all_lines.decode('utf-8').replace('\r\n', '\n')
                    logger.info(all_lines)
                if i == 1:
                    logger.info("Connection refused")
            except:
                logger.info(str(child))
        return all_lines 
    
    def _run_command_get_output(self, commands, host_password, logger, block=False, timeout=None):
        successful = False
        value_to_return = ''
        logger.info("Inside _run_command_get_output")
        for command in commands:
            logger.info(" Executing.." + str(command))
            child = pexpect.spawn(command)
            child.timeout=timeout
            try:
                child.expect(['password: '], timeout=timeout)
                child.sendline(host_password)
                all_lines = child.read()
                all_lines = all_lines.rstrip().lstrip()
                all_lines = all_lines.decode('utf-8').replace('\r\n', '\n')
                successful = True # Tentative
                for line in all_lines.split("\n"):
                    logger.info(line)
                    if value_to_return == '':
                        value_to_return = line.strip()
                    if re.search('error', line, re.IGNORECASE):
                        successful = False 
                    if re.search('unable', line, re.IGNORECASE):
                        successful = False 
            except:
                logger.info(str(child))
        logger.info("Status:" + str(successful) + " value_to_return:" + value_to_return)
        return successful, value_to_return 

    def _get_remote_region_oam_ip(self, cluster_name, logger):
        remoteclusterObj = Cluster.objects.filter(cluster_name=cluster_name)
        oam_ip = remoteclusterObj[0].oam_vip_address
        logger.info(" Remote region:" + cluster_name + " OAM IP:" + str(oam_ip))
        return oam_ip

    def _get_central_region_name(self, oam_vip_address, logger):
        logger.info("1")
        remoteclusterObj = Cluster.objects.filter(oam_vip_address=oam_vip_address)
        logger.info("2")
        logger.info(remoteclusterObj)
        cluster_name = remoteclusterObj[0].cluster_name
        logger.info(" Remote region:" + str(oam_vip_address) + " Cluster Name:" + str(cluster_name))
        return cluster_name

    def _get_namespace(self, region, logger):
        # Lookup database and findout namespace given region
        logger.info(" Inside _get_namespace")
        remoteclusterObj = Cluster.objects.filter(cluster_name=region)
        if len(remoteclusterObj) > 0:
            logger.info(" RemoteClusterObj:" + str(remoteclusterObj))
            namespace_id = remoteclusterObj[0].namespace_id
            logger.info(" Namespace id:" + str(namespace_id))
            namespaceObj = Namespace.objects.filter(id=namespace_id)
            logger.info(" NamespaceObj:" + str(namespaceObj))
            namespace_name = namespaceObj[0].namespace_name
            logger.info(" Remote region:" + region + " Namespace:" + namespace_name)
            return namespace_name
        else:
            return ""

    def _get_central_region_oam_ip(self, remoteregion, transaction_id):
        remoteclusterObj = Cluster.objects.filter(cluster_name=remoteregion)
        parent_cluster_id = remoteclusterObj[0].parent_cluster_id
        self.logger.info(str(transaction_id) + " Parent cluster id.." + str(parent_cluster_id))
        centralclusterObj = Cluster.objects.filter(id=parent_cluster_id)
        self.logger.info(str(transaction_id) + " " + str(centralclusterObj))
        central_region_list = []
        for central_region in centralclusterObj:
            central_region_list.append(central_region.oam_vip_address)
        self.logger.info(str(transaction_id) + " Central Region List:" + ','.join(central_region_list))
        return central_region_list

    def get_fuze_spm_site_details(self, cluster_name, logger):
        fuze_spm_site_name = ''
        fuze_spm_site_id = ''
        logger.info(" Inside _get_fuze_spm_site_name " + str(cluster_name))
        remoteclusterObj = Cluster.objects.filter(cluster_name=cluster_name)
        if len(remoteclusterObj) > 0:
            fuze_id = remoteclusterObj[0].location_id
            locationObj = Location.objects.filter(id=fuze_id)
            if len(locationObj) > 0:
                fuze_spm_site_name = locationObj[0].fuze_spm_site_name
                fuze_spm_site_id = locationObj[0].fuze_spm_site_id
        logger.info(" Fuze site name:" + fuze_spm_site_name)
        logger.info(" Fuze site id:" + fuze_spm_site_id)
        return fuze_spm_site_name, fuze_spm_site_id 

    def check_namespaces(self, remote_region_oam_ip, logger):

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
        cmds = []
        cmds.append(ssh_prefix + " kubectl get namespaces --kubeconfig=/etc/kubernetes/admin.conf ")

        output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True)
        #logger.info("Returned output lines:")
        #logger.info(output_lines)

        new_op_lines = []
        for line in output_lines.split("\n"):
            if not 'Connection to' in line:
                new_op_lines.append(line)

        return new_op_lines

    def check_namespace_secrets(self, region, remote_region_oam_ip, logger):
        namespace = self._get_namespace(region, logger)
        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
        cmds = []
        cmds.append(ssh_prefix + " kubectl get secrets --kubeconfig=/etc/kubernetes/admin.conf -n " + namespace)

        output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True)
        #logger.info("Returned output lines:")
        #logger.info(output_lines)

        new_op_lines = []
        for line in output_lines.split("\n"):
            if not 'Connection to' in line:
                new_op_lines.append(line)

        return new_op_lines

    def check_namespace_serviceaccounts(self, region, remote_region_oam_ip, logger):
        namespace = self._get_namespace(region, logger)
        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
        cmds = []
        cmds.append(ssh_prefix + " kubectl get serviceaccounts --kubeconfig=/etc/kubernetes/admin.conf -n " + namespace)

        output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True)
        #logger.info("Returned output lines:")
        #logger.info(output_lines)

        new_op_lines = []
        for line in output_lines.split("\n"):
            if not 'Connection to' in line:
                new_op_lines.append(line)

        return new_op_lines

    def check_online_status(self, region, remote_region_oam_ip, logger):
        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
        central_region_list = self._get_central_region_oam_ip(region, "-1")
        region_online_status = []
        new_op_lines = []
        for central_region_ip in central_region_list:
            cmds = []
            cmd = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + central_region_ip + ' ' + '"source /etc/platform/openrc; dcmanager subcloud list | grep ' + region + '"'
            cmds.append(cmd)
            output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True)
            logger.info(output_lines)

            for line in output_lines.split("\n"):
                if not 'Connection to' in line:
                    new_op_lines.append(line)
        return new_op_lines

    def create_host_network(self, remote_region_oam_ip, logger):
        logger.info("Inside _create_host_network")

        #host_username = settings.HOST_CREDS['username']
        #host_password = settings.HOST_CREDS['password']
        #ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + remote_region_oam_ip

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        env_string = self._read_remote_openrc(remote_region_oam_ip, logger)

        #basecmd = " OS_ENDPOINT_TYPE=internalURL CINDER_ENDPOINT_TYPE=internalURL OS_USERNAME=admin"
        #basecmd = basecmd + " OS_PASSWORD=`TERM=linux /opt/platform/.keyring/20.06/.CREDENTIAL 2>/dev/null`"
        #basecmd = basecmd + " OS_AUTH_TYPE=password OS_AUTH_URL=http://[fd00:4888:2000:120b::220]:5000/v3"
        #basecmd = basecmd + " OS_PROJECT_NAME=admin OS_USER_DOMAIN_NAME=Default OS_PROJECT_DOMAIN_NAME=Default"
        #basecmd = basecmd + " OS_IDENTITY_API_VERSION=3 OS_REGION_NAME=subcloud2 OS_INTERFACE=internal"

        basecmd = env_string

        cmds = ["system host-lock controller-0",
                "system host-if-modify -n f1u -c pci-sriov --num-vfs 8 controller-0 ens3f1 --vf-driver=vfio",
                "system host-if-add -c pci-sriov controller-0 f1c vf f1u --num-vfs 4 --vf-driver=netdevice",
                "system host-if-modify controller-0 f1u --imtu=1956",
                "system host-if-modify controller-0 f1c --imtu=1956",
                "system datanetwork-add f1u vlan --mtu=1956",
                "system datanetwork-add f1c vlan --mtu=1956",
                "system interface-datanetwork-assign controller-0 f1u f1u",
                "system interface-datanetwork-assign controller-0 f1c f1c",
                "system host-if-add -c pci-sriov controller-0 fh0m vf fh0 --num-vfs 4 --vf-driver=netdevice",
                "system host-if-modify controller-0 fh0m --imtu=9000",
                "system datanetwork-add fh0m flat",
                "system interface-datanetwork-assign controller-0  fh0m fh0m",
                "system host-if-modify -n fh1 -c pci-sriov --num-vfs 8 controller-0 enp181s0f0 --vf-driver=vfio",
                "system host-if-modify controller-0 fh1 --imtu=9000",
                "system datanetwork-add fh1 vlan --mtu=9000",
                "system interface-datanetwork-assign controller-0 fh1 fh1"]

        # New steps proposed by Eddy - These do not seem to create fh0m so commenting out.
        #cmds = ["system host-lock controller-0",
        #        "system host-if-modify -n f1c -c pci-sriov --num-vfs 8 controller-0 ens3f1 --vf-driver=netdevice",
        #        "system host-if-add -c pci-sriov controller-0 f1u vf f1c --num-vfs 4 --vf-driver=vfio",
        #        "system host-if-modify controller-0 f1u --imtu=1956",
        #        "system host-if-modify controller-0 f1c --imtu=1956",
        #        "system datanetwork-add f1u vlan --mtu=1956",
        #        "system datanetwork-add f1c vlan --mtu=1956",
        #        "system interface-datanetwork-assign controller-0 f1u f1u",
        #        "system interface-datanetwork-assign controller-0 f1c f1c",
        #         # this one is wrong as well but for some reason I think the fh0 is setup in Carlos deployment config.  So we have to delete the fh0 and recreate both.
        #        "system host-if-modify controller0 fh0 -nc none"
        #        "system host-if-modify -n fh0m -c pci-sriov --num-vfs 8 controller-0 ens179s0f0 --vf-driver=netdevice",
        #        "system host-if-add -c pci-sriov controller-0 fh0 vf fh0m --num-vfs 4 --vf-driver=vfio",
        #        "system host-if-modify controller-0 fh0m --imtu=9000",
        #        "system datanetwork-add fh0m flat",
        #        "system interface-datanetwork-assign controller-0  fh0m fh0m",
        #        "system host-if-modify -n fh1 -c pci-sriov --num-vfs 8 controller-0 enp181s0f0 --vf-driver=vfio",
        #        "system host-if-modify controller-0 fh1 --imtu=9000",
        #        "system datanetwork-add fh1 vlan --mtu=9000",
        #        "system interface-datanetwork-assign controller-0 fh1 fh1"]

        for cmd in cmds:
            cmd_to_run = ssh_prefix + " " + basecmd + " "  + cmd
            logger.info(cmd_to_run)
            run_commands([cmd_to_run], host_password, logger, block=True)

#        host_network_configured = False
#        while not host_network_configured:
#            host_network_configured = self._verify_host_network(ssh_prefix, basecmd, host_password, logger)
#            if not host_network_configured:
#                cmds = ["system host-lock controller-0"]
#                for cmd in cmds:
#                    cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd
#                    logger.info(cmd_to_run)
#                    output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True)
#                    logger.info("Returned output lines:")
#                    logger.info(output_lines)
#                time.sleep(3)

        cmds = ["system host-unlock controller-0"]
        while True:
            unlock_wait = False
            for cmd in cmds:
                cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd
                logger.info(cmd_to_run)
                output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True)
                logger.info("Returned output lines:")
                logger.info(output_lines)
                if len(output_lines) > 0:
                    for line in output_lines.split("\n"):
                        logger.info("Line:" + line)
                        if not unlock_wait:
                            if 'retry host-unlock' in line or 'Rejected' in line:
                                logger.info("Need to wait to call host-unlock ##### ")
                                unlock_wait = True 
                                break
            if unlock_wait: 
                time.sleep(60)
            else:
                break

#        for cmd in cmds:
#            cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd
#            logger.info(cmd_to_run)
#            output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True)
#            logger.info("Returned output lines:")
#            logger.info(output_lines)
        logger.info("Done setting up host network")