more filesHEADmaster

1 files changed, 375 insertions, 0 deletions

diff --git a/src/orchestration/remoteregionhandler.py b/src/orchestration/remoteregionhandler.py
new file mode 100644
index 0000000..6754672
--- /dev/null
+++ b/src/orchestration/remoteregionhandler.py
@@ -0,0 +1,375 @@
+import django
+import os
+import threading
+
+from django.conf import settings
+from django.core.exceptions import AppRegistryNotReady
+from django.utils import timezone
+
+from .utils import *
+
+try:
+    django.setup()
+    from .models import ImageSync
+    from caas.models import Cluster
+    from caas.models import Location 
+    from caas.models import Namespace  
+except django.core.exceptions.AppRegistryNotReady as exp:
+    pass            
+                    
+class RemoteRegionWorker:
+
+    def __init__(self):
+        pass
+
+    def _get_crd_details(self):
+        crd_cluster_role = "nad"
+        crd_api_group = "k8s.cni.cncf.io"
+        crd_api_resources = "network-attachment-definitions"
+        crd_api_verbs = "*"
+        return crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs
+
+    def _read_remote_openrc(self, remote_region_oam_ip, logger):
+        #cmd = "scp @[fd00:4888:2000:120c::290]:/etc/platform/openrc .
+
+        logger.info("Inside _read_remote_openrc")
+        host_username = settings.HOST_CREDS['username']
+        host_password = settings.HOST_CREDS['password']
+        ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + remote_region_oam_ip
+        env_string = ""
+        cmds_scp = []
+        folder = get_temp_file_location()
+        logger.info("Tmp file location:" + folder)
+        #tmpFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
+        tmpFileName = 'openrc_' + str(remote_region_oam_ip)
+        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + host_username + '@[' + remote_region_oam_ip + ']:/etc/platform/openrc ' + folder + '/' + tmpFileName)
+        logger.info(cmds_scp)
+        run_commands_scp(cmds_scp, host_password, logger, block=True)
+        logger.info("About to create env string")
+
+        fp = open(folder + '/' + tmpFileName)
+        lines = fp.readlines()
+        logger.info(lines)
+        password_line = ''
+        for line in lines:
+            line = line.lstrip().rstrip()
+            logger.info(line)
+            parts = line.split(' ')
+            if len(parts) == 2:
+                if parts[0] == 'export':
+                    if parts[1]:
+                        env_var = parts[1].lstrip().rstrip()
+                        logger.info(env_var)
+                        if 'OS_PASSWORD' not in env_var:
+                            env_string = env_string + " " + env_var
+            if len(parts) >= 2 and 'OS_PASSWORD=' in parts[1]:
+                logger.info("Parsing password")
+                password_parts = line.split('PASSWORD=')
+                password_command_parts = password_parts[1].split(' ')
+                password_line = password_command_parts[1].rstrip().lstrip()
+                logger.info("Password line:" + password_line)
+
+        logger.info("Deleting openrc file ")
+
+        # Delete file
+        if os.path.exists(folder + '/' + tmpFileName):
+            os.remove(folder + '/' + tmpFileName)
+
+        logger.info("Looking for password")
+        # Get password
+        password_cmd = []
+        #cmd = 'TERM=linux /opt/platform/.keyring/20.06/.CREDENTIAL 2>/dev/null'
+        cmd_to_run = ssh_prefix + " " + password_line
+        logger.info(cmd_to_run)
+        successful, value = self._run_command_get_output([cmd_to_run], host_password, logger)
+        password_val = host_password
+        #if successful:
+        #    if value != '':
+        #        password_val = value
+        os_password = "OS_PASSWORD=" + password_val
+        env_string = env_string + " " + os_password
+
+        logger.info("Env string:" + env_string)
+        return env_string
+
+    def _run_command_get_all_lines(self, commands, host_password, logger, block=False, timeout=None):
+        logger.info("Inside _run_command_get_all_lines")
+        all_lines = []
+        for command in commands:
+            logger.info(" Executing.." + str(command))
+            child = pexpect.spawn(command)
+            child.timeout=timeout
+            try:
+                i = child.expect(['password: ','Connection refused\r\r\n'], timeout=timeout)
+                if i == 0:
+                    child.sendline(host_password)
+                    all_lines = child.read()
+                    all_lines = all_lines.rstrip().lstrip()
+                    all_lines = all_lines.decode('utf-8').replace('\r\n', '\n')
+                    logger.info(all_lines)
+                if i == 1:
+                    logger.info("Connection refused")
+            except:
+                logger.info(str(child))
+        return all_lines 
+    
+    def _run_command_get_output(self, commands, host_password, logger, block=False, timeout=None):
+        successful = False
+        value_to_return = ''
+        logger.info("Inside _run_command_get_output")
+        for command in commands:
+            logger.info(" Executing.." + str(command))
+            child = pexpect.spawn(command)
+            child.timeout=timeout
+            try:
+                child.expect(['password: '], timeout=timeout)
+                child.sendline(host_password)
+                all_lines = child.read()
+                all_lines = all_lines.rstrip().lstrip()
+                all_lines = all_lines.decode('utf-8').replace('\r\n', '\n')
+                successful = True # Tentative
+                for line in all_lines.split("\n"):
+                    logger.info(line)
+                    if value_to_return == '':
+                        value_to_return = line.strip()
+                    if re.search('error', line, re.IGNORECASE):
+                        successful = False 
+                    if re.search('unable', line, re.IGNORECASE):
+                        successful = False 
+            except:
+                logger.info(str(child))
+        logger.info("Status:" + str(successful) + " value_to_return:" + value_to_return)
+        return successful, value_to_return 
+
+    def _get_remote_region_oam_ip(self, cluster_name, logger):
+        remoteclusterObj = Cluster.objects.filter(cluster_name=cluster_name)
+        oam_ip = remoteclusterObj[0].oam_vip_address
+        logger.info(" Remote region:" + cluster_name + " OAM IP:" + str(oam_ip))
+        return oam_ip
+
+    def _get_central_region_name(self, oam_vip_address, logger):
+        logger.info("1")
+        remoteclusterObj = Cluster.objects.filter(oam_vip_address=oam_vip_address)
+        logger.info("2")
+        logger.info(remoteclusterObj)
+        cluster_name = remoteclusterObj[0].cluster_name
+        logger.info(" Remote region:" + str(oam_vip_address) + " Cluster Name:" + str(cluster_name))
+        return cluster_name
+
+    def _get_namespace(self, region, logger):
+        # Lookup database and findout namespace given region
+        logger.info(" Inside _get_namespace")
+        remoteclusterObj = Cluster.objects.filter(cluster_name=region)
+        if len(remoteclusterObj) > 0:
+            logger.info(" RemoteClusterObj:" + str(remoteclusterObj))
+            namespace_id = remoteclusterObj[0].namespace_id
+            logger.info(" Namespace id:" + str(namespace_id))
+            namespaceObj = Namespace.objects.filter(id=namespace_id)
+            logger.info(" NamespaceObj:" + str(namespaceObj))
+            namespace_name = namespaceObj[0].namespace_name
+            logger.info(" Remote region:" + region + " Namespace:" + namespace_name)
+            return namespace_name
+        else:
+            return ""
+
+    def _get_central_region_oam_ip(self, remoteregion, transaction_id):
+        remoteclusterObj = Cluster.objects.filter(cluster_name=remoteregion)
+        parent_cluster_id = remoteclusterObj[0].parent_cluster_id
+        self.logger.info(str(transaction_id) + " Parent cluster id.." + str(parent_cluster_id))
+        centralclusterObj = Cluster.objects.filter(id=parent_cluster_id)
+        self.logger.info(str(transaction_id) + " " + str(centralclusterObj))
+        central_region_list = []
+        for central_region in centralclusterObj:
+            central_region_list.append(central_region.oam_vip_address)
+        self.logger.info(str(transaction_id) + " Central Region List:" + ','.join(central_region_list))
+        return central_region_list
+
+    def get_fuze_spm_site_details(self, cluster_name, logger):
+        fuze_spm_site_name = ''
+        fuze_spm_site_id = ''
+        logger.info(" Inside _get_fuze_spm_site_name " + str(cluster_name))
+        remoteclusterObj = Cluster.objects.filter(cluster_name=cluster_name)
+        if len(remoteclusterObj) > 0:
+            fuze_id = remoteclusterObj[0].location_id
+            locationObj = Location.objects.filter(id=fuze_id)
+            if len(locationObj) > 0:
+                fuze_spm_site_name = locationObj[0].fuze_spm_site_name
+                fuze_spm_site_id = locationObj[0].fuze_spm_site_id
+        logger.info(" Fuze site name:" + fuze_spm_site_name)
+        logger.info(" Fuze site id:" + fuze_spm_site_id)
+        return fuze_spm_site_name, fuze_spm_site_id 
+
+    def check_namespaces(self, remote_region_oam_ip, logger):
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+        cmds = []
+        cmds.append(ssh_prefix + " kubectl get namespaces --kubeconfig=/etc/kubernetes/admin.conf ")
+
+        output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True)
+        #logger.info("Returned output lines:")
+        #logger.info(output_lines)
+
+        new_op_lines = []
+        for line in output_lines.split("\n"):
+            if not 'Connection to' in line:
+                new_op_lines.append(line)
+
+        return new_op_lines
+
+    def check_namespace_secrets(self, region, remote_region_oam_ip, logger):
+        namespace = self._get_namespace(region, logger)
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+        cmds = []
+        cmds.append(ssh_prefix + " kubectl get secrets --kubeconfig=/etc/kubernetes/admin.conf -n " + namespace)
+
+        output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True)
+        #logger.info("Returned output lines:")
+        #logger.info(output_lines)
+
+        new_op_lines = []
+        for line in output_lines.split("\n"):
+            if not 'Connection to' in line:
+                new_op_lines.append(line)
+
+        return new_op_lines
+
+    def check_namespace_serviceaccounts(self, region, remote_region_oam_ip, logger):
+        namespace = self._get_namespace(region, logger)
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+        cmds = []
+        cmds.append(ssh_prefix + " kubectl get serviceaccounts --kubeconfig=/etc/kubernetes/admin.conf -n " + namespace)
+
+        output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True)
+        #logger.info("Returned output lines:")
+        #logger.info(output_lines)
+
+        new_op_lines = []
+        for line in output_lines.split("\n"):
+            if not 'Connection to' in line:
+                new_op_lines.append(line)
+
+        return new_op_lines
+
+    def check_online_status(self, region, remote_region_oam_ip, logger):
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+        central_region_list = self._get_central_region_oam_ip(region, "-1")
+        region_online_status = []
+        new_op_lines = []
+        for central_region_ip in central_region_list:
+            cmds = []
+            cmd = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + central_region_ip + ' ' + '"source /etc/platform/openrc; dcmanager subcloud list | grep ' + region + '"'
+            cmds.append(cmd)
+            output_lines = self._run_command_get_all_lines(cmds, host_password, logger, block=True)
+            logger.info(output_lines)
+
+            for line in output_lines.split("\n"):
+                if not 'Connection to' in line:
+                    new_op_lines.append(line)
+        return new_op_lines
+
+    def create_host_network(self, remote_region_oam_ip, logger):
+        logger.info("Inside _create_host_network")
+
+        #host_username = settings.HOST_CREDS['username']
+        #host_password = settings.HOST_CREDS['password']
+        #ssh_prefix = 'ssh -o StrictHostKeyChecking=no -t ' + host_username + '@' + remote_region_oam_ip
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        env_string = self._read_remote_openrc(remote_region_oam_ip, logger)
+
+        #basecmd = " OS_ENDPOINT_TYPE=internalURL CINDER_ENDPOINT_TYPE=internalURL OS_USERNAME=admin"
+        #basecmd = basecmd + " OS_PASSWORD=`TERM=linux /opt/platform/.keyring/20.06/.CREDENTIAL 2>/dev/null`"
+        #basecmd = basecmd + " OS_AUTH_TYPE=password OS_AUTH_URL=http://[fd00:4888:2000:120b::220]:5000/v3"
+        #basecmd = basecmd + " OS_PROJECT_NAME=admin OS_USER_DOMAIN_NAME=Default OS_PROJECT_DOMAIN_NAME=Default"
+        #basecmd = basecmd + " OS_IDENTITY_API_VERSION=3 OS_REGION_NAME=subcloud2 OS_INTERFACE=internal"
+
+        basecmd = env_string
+
+        cmds = ["system host-lock controller-0",
+                "system host-if-modify -n f1u -c pci-sriov --num-vfs 8 controller-0 ens3f1 --vf-driver=vfio",
+                "system host-if-add -c pci-sriov controller-0 f1c vf f1u --num-vfs 4 --vf-driver=netdevice",
+                "system host-if-modify controller-0 f1u --imtu=1956",
+                "system host-if-modify controller-0 f1c --imtu=1956",
+                "system datanetwork-add f1u vlan --mtu=1956",
+                "system datanetwork-add f1c vlan --mtu=1956",
+                "system interface-datanetwork-assign controller-0 f1u f1u",
+                "system interface-datanetwork-assign controller-0 f1c f1c",
+                "system host-if-add -c pci-sriov controller-0 fh0m vf fh0 --num-vfs 4 --vf-driver=netdevice",
+                "system host-if-modify controller-0 fh0m --imtu=9000",
+                "system datanetwork-add fh0m flat",
+                "system interface-datanetwork-assign controller-0  fh0m fh0m",
+                "system host-if-modify -n fh1 -c pci-sriov --num-vfs 8 controller-0 enp181s0f0 --vf-driver=vfio",
+                "system host-if-modify controller-0 fh1 --imtu=9000",
+                "system datanetwork-add fh1 vlan --mtu=9000",
+                "system interface-datanetwork-assign controller-0 fh1 fh1"]
+
+        # New steps proposed by Eddy - These do not seem to create fh0m so commenting out.
+        #cmds = ["system host-lock controller-0",
+        #        "system host-if-modify -n f1c -c pci-sriov --num-vfs 8 controller-0 ens3f1 --vf-driver=netdevice",
+        #        "system host-if-add -c pci-sriov controller-0 f1u vf f1c --num-vfs 4 --vf-driver=vfio",
+        #        "system host-if-modify controller-0 f1u --imtu=1956",
+        #        "system host-if-modify controller-0 f1c --imtu=1956",
+        #        "system datanetwork-add f1u vlan --mtu=1956",
+        #        "system datanetwork-add f1c vlan --mtu=1956",
+        #        "system interface-datanetwork-assign controller-0 f1u f1u",
+        #        "system interface-datanetwork-assign controller-0 f1c f1c",
+        #         # this one is wrong as well but for some reason I think the fh0 is setup in Carlos deployment config.  So we have to delete the fh0 and recreate both.
+        #        "system host-if-modify controller0 fh0 -nc none"
+        #        "system host-if-modify -n fh0m -c pci-sriov --num-vfs 8 controller-0 ens179s0f0 --vf-driver=netdevice",
+        #        "system host-if-add -c pci-sriov controller-0 fh0 vf fh0m --num-vfs 4 --vf-driver=vfio",
+        #        "system host-if-modify controller-0 fh0m --imtu=9000",
+        #        "system datanetwork-add fh0m flat",
+        #        "system interface-datanetwork-assign controller-0  fh0m fh0m",
+        #        "system host-if-modify -n fh1 -c pci-sriov --num-vfs 8 controller-0 enp181s0f0 --vf-driver=vfio",
+        #        "system host-if-modify controller-0 fh1 --imtu=9000",
+        #        "system datanetwork-add fh1 vlan --mtu=9000",
+        #        "system interface-datanetwork-assign controller-0 fh1 fh1"]
+
+        for cmd in cmds:
+            cmd_to_run = ssh_prefix + " " + basecmd + " "  + cmd
+            logger.info(cmd_to_run)
+            run_commands([cmd_to_run], host_password, logger, block=True)
+
+#        host_network_configured = False
+#        while not host_network_configured:
+#            host_network_configured = self._verify_host_network(ssh_prefix, basecmd, host_password, logger)
+#            if not host_network_configured:
+#                cmds = ["system host-lock controller-0"]
+#                for cmd in cmds:
+#                    cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd
+#                    logger.info(cmd_to_run)
+#                    output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True)
+#                    logger.info("Returned output lines:")
+#                    logger.info(output_lines)
+#                time.sleep(3)
+
+        cmds = ["system host-unlock controller-0"]
+        while True:
+            unlock_wait = False
+            for cmd in cmds:
+                cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd
+                logger.info(cmd_to_run)
+                output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True)
+                logger.info("Returned output lines:")
+                logger.info(output_lines)
+                if len(output_lines) > 0:
+                    for line in output_lines.split("\n"):
+                        logger.info("Line:" + line)
+                        if not unlock_wait:
+                            if 'retry host-unlock' in line or 'Rejected' in line:
+                                logger.info("Need to wait to call host-unlock ##### ")
+                                unlock_wait = True 
+                                break
+            if unlock_wait: 
+                time.sleep(60)
+            else:
+                break
+
+#        for cmd in cmds:
+#            cmd_to_run = ssh_prefix + " " + basecmd + " " + cmd
+#            logger.info(cmd_to_run)
+#            output_lines = self._run_command_get_all_lines([cmd_to_run], host_password, logger, block=True)
+#            logger.info("Returned output lines:")
+#            logger.info(output_lines)
+        logger.info("Done setting up host network")