more filesHEADmaster

1 files changed, 926 insertions, 0 deletions

diff --git a/src/orchestration/kubeconfighandler.py b/src/orchestration/kubeconfighandler.py
new file mode 100644
index 0000000..d7318b2
--- /dev/null
+++ b/src/orchestration/kubeconfighandler.py
@@ -0,0 +1,926 @@
+import django
+
+from django.conf import settings
+from django.core.exceptions import AppRegistryNotReady
+from django.utils import timezone
+
+from multiprocessing import Process, Pool, Queue
+import pexpect
+import logging
+import multiprocessing
+from logging.handlers import QueueHandler
+import sys
+import time
+import datetime
+from .utils import *
+
+import os
+import signal, psutil
+import json
+
+from jinja2 import Environment, FileSystemLoader
+
+from .vmb_messages import Kubeconfig, KubeconfigMessage 
+from .remoteregionhandler import RemoteRegionWorker
+from .vendorhandler import Vendor
+
+try:
+    django.setup()
+    from .models import RemoteRegionSetup
+    from caas.models import Cluster
+    from caas.models import Namespace  
+except django.core.exceptions.AppRegistryNotReady as exp:
+    pass
+
+class KubeconfigGenerator(RemoteRegionWorker):
+
+    ORCHESTRATION_TEAM = 'orchestration-team'
+    APPLICATION_TEAM = 'application-team'
+    VENDOR_TEAM_SAMSUNG = 'samsung-team'
+    OPS_TEAM = 'ops-team'
+
+    def __init__(self, loggerQueue, requestQueue, dbQueue, vmbQueue, dbCoordinationQueue, vmbCoordinationQueue, doneQueue):
+        self.loggerQueue = loggerQueue
+        self.requestQueue = requestQueue
+        self.dbQueue = dbQueue
+        self.vmbQueue = vmbQueue
+        self.dbCoordinationQueue = dbCoordinationQueue
+        self.vmbCoordinationQueue = vmbCoordinationQueue
+        self.doneQueue = doneQueue
+
+    def run(self):
+        qh = QueueHandler(self.loggerQueue)
+        self.logger = logging.getLogger()
+        self.logger.addHandler(qh)
+        self.logger.setLevel(logging.DEBUG)
+
+        self.logger.info("KubeconfigGenerator started...")
+
+        while True:
+            try:
+                item = self.requestQueue.get(block=True)
+                if item:
+                    self.logger.info(item)
+                    self.get_kubeconfig(item)
+                    self.logger.info("Done generating kubeconfig")
+                    self.logger.info("Performing vendor setup...")
+                    vendor_provisioner = Vendor(self.logger, self.dbCoordinationQueue, self.vmbCoordinationQueue, self.doneQueue)
+                    vendor_provisioner.perform_vendor_setup(item)
+            except:
+                pass
+            time.sleep(1)
+
+    def get_kubeconfig(self, request):
+        transaction_id = request['transaction_id']
+        region = request['remote_region']
+        kubeconfig_for = request['kubeconfig_for']
+        kubeconfig_approach = request['kubeconfig_approach']
+        namespace = self._get_namespace(region, self.logger)
+        if namespace != "": 
+            self.logger.info(" Transaction:" + transaction_id)
+            self.logger.info(" Region:" + region)
+            self.logger.info(" Namespace:" + namespace)
+            kubeconfig = self._create_kubeconfig(kubeconfig_for, region, namespace, kubeconfig_approach, transaction_id)
+            return kubeconfig
+        else:
+            message = " Could not find Namespace for region " + region 
+            self.logger.info(message)
+            status = "FAILED"
+            self._update_status_async(region, namespace, '', message, status, transaction_id, self.logger, kubeconfig='', done=True)
+            return json.dumps({}) 
+
+    def _get_serviceaccount_name(self, team, transaction_id):
+        if team == KubeconfigGenerator.ORCHESTRATION_TEAM:
+            return "orchestration-sa"
+        if team == KubeconfigGenerator.APPLICATION_TEAM:
+            return "application-sa-" + str(transaction_id)
+        if team == KubeconfigGenerator.OPS_TEAM:
+            return "ops-sa"
+        if team == KubeconfigGenerator.VENDOR_TEAM_SAMSUNG:
+            return "samsung-sa"
+
+    def _get_user_name(self, team):
+        if team == KubeconfigGenerator.ORCHESTRATION_TEAM:
+            return "SVC-FE-Atlas"
+        if team == KubeconfigGenerator.APPLICATION_TEAM:
+            return "SVC-Edge-Eng"
+
+    def _apply_application_rbac_policies_sa(self, region, remote_region_oam_ip, app_sa_namespace, namespace, saName, saFilePath, logger):
+        logger.info("Inside _apply_application_rbac_policies")
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        sa_files_dir = []
+        folder = 'cluster-setup-files'
+        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
+        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
+
+        logger.info("About to render Application RBAC files")
+        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
+        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
+        logger.info(env)
+
+        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
+        app_config = {}
+        app_config["application_sa"] = saName
+        app_config["namespace"] = namespace
+        app_config["crd_cluster_role"] = crd_cluster_role
+        app_config["crd_api_group"] = crd_api_group
+        app_config["crd_api_resources"] = crd_api_resources 
+        app_rbac_template = env.get_template('application-sa.yaml')
+        temp_file_location = get_temp_file_location()
+        policy_file_location = temp_file_location + "/" + region
+        logger.info("Policy file location:" + policy_file_location)
+        if not os.path.exists(policy_file_location):
+            os.makedirs(policy_file_location)
+        fp = open(policy_file_location + "/rendered-application-sa.yaml", "w")
+        fp.write(app_rbac_template.render(app_config))
+        fp.close()
+
+        logger.info("About to copy rendered-application-rbac.yaml")
+        cmds_scp = []
+        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-application-sa.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
+        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)
+
+        logger.info("About to apply rendered-application-sa.yaml")
+        cmds = []
+        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-application-sa.yaml')
+        self._run_commands(cmds, host_password, self.logger, block=True)
+
+    def _apply_application_rbac_policies_ldap(self, region, remote_region_oam_ip, namespace, logger):
+        logger.info("Inside _apply_application_rbac_policies ldap")
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        sa_files_dir = []
+        folder = 'cluster-setup-files'
+        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
+        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
+
+        logger.info("About to render Application RBAC files")
+        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
+        self.logger.info(" RBAC File Path:" + str(saFilePath))
+        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
+        logger.info(env)
+
+        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
+        orch_config = {}
+        orch_config["namespace"] = namespace
+        orch_config["crd_cluster_role"] = crd_cluster_role
+        orch_config["crd_api_group"] = crd_api_group
+        orch_config["crd_api_resources"] = crd_api_resources
+        orch_rbac_template = env.get_template('application-ldap.yaml')
+        temp_file_location = get_temp_file_location()
+        policy_file_location = temp_file_location + "/" + region
+        logger.info("Policy file location:" + policy_file_location)
+        if not os.path.exists(policy_file_location):
+            os.makedirs(policy_file_location)
+        fp = open(policy_file_location + "/rendered-application-ldap.yaml", "w")
+        fp.write(orch_rbac_template.render(orch_config))
+        fp.close()
+
+        logger.info("About to copy rendered-application-ldap.yaml")
+        cmds_scp = []
+        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-application-ldap.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
+        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)
+
+        logger.info("About to apply rendered-application-ldap.yaml")
+        cmds = []
+        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-application-ldap.yaml')
+        self._run_commands(cmds, host_password, self.logger, block=True)
+
+    def _apply_vendor_samsung_rbac_policies(self, region, remote_region_oam_ip, app_sa_namespace, namespace, saName, saFilePath, logger):
+        logger.info("Inside _apply_vendor_samsung_rbac_policies")
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        sa_files_dir = []
+        folder = 'cluster-setup-files'
+        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
+        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
+
+        logger.info("About to render Vendor Samsung RBAC files")
+        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
+        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
+        logger.info(env)
+
+        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
+        app_config = {}
+        app_config["application_sa"] = saName
+        app_config["namespace"] = namespace
+        app_config["crd_cluster_role"] = crd_cluster_role
+        app_config["crd_api_group"] = crd_api_group
+        app_config["crd_api_resources"] = crd_api_resources 
+        app_rbac_template = env.get_template('samsung-sa.yaml')
+        temp_file_location = get_temp_file_location()
+        policy_file_location = temp_file_location + "/" + region
+        logger.info("Policy file location:" + policy_file_location)
+        if not os.path.exists(policy_file_location):
+            os.makedirs(policy_file_location)
+        fp = open(policy_file_location + "/rendered-samsung-sa.yaml", "w")
+        fp.write(app_rbac_template.render(app_config))
+        fp.close()
+
+        logging.info("About to copy rendered-samsung-rbac.yaml")
+        cmds_scp = []
+        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-samsung-sa.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
+        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)
+
+        logger.info("About to apply rendered-samsung-sa.yaml")
+        cmds = []
+        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-samsung-sa.yaml')
+        self._run_commands(cmds, host_password, self.logger, block=True)
+
+    def _apply_orchestration_rbac_policies_ldap(self, region, remote_region_oam_ip, namespace, logger):
+        logger.info("Inside _apply_orchestration_rbac_policies ldap")
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        sa_files_dir = []
+        folder = 'cluster-setup-files'
+        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
+        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
+
+        logger.info("About to render Orchestration RBAC files")
+        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
+        self.logger.info(" RBAC File Path:" + str(saFilePath))
+        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
+        logger.info(env)
+
+        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
+        orch_config = {}
+        orch_config["namespace"] = namespace
+        orch_config["crd_cluster_role"] = crd_cluster_role 
+        orch_rbac_template = env.get_template('orchestration-ldap.yaml')
+        temp_file_location = get_temp_file_location()
+        policy_file_location = temp_file_location + "/" + region
+        logger.info("Policy file location:" + policy_file_location)
+        if not os.path.exists(policy_file_location):
+            os.makedirs(policy_file_location)
+        fp = open(policy_file_location + "/rendered-orchestration-ldap.yaml", "w")
+        fp.write(orch_rbac_template.render(orch_config))
+        fp.close()
+
+        logger.info("About to copy rendered-orchestration-ldap.yaml")
+        cmds_scp = []
+        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-orchestration-ldap.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
+        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)
+
+        logger.info("About to apply rendered-orchestration-ldap.yaml")
+        cmds = []
+        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-orchestration-ldap.yaml')
+        self._run_commands(cmds, host_password, self.logger, block=True)
+
+    def _apply_orchestration_rbac_policies_sa(self, region, remote_region_oam_ip, orch_sa_namespace, namespace, saName, saFilePath, logger):
+        logger.info("Inside _apply_orchestration_rbac_policies sa")
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        sa_files_dir = []
+        folder = 'cluster-setup-files'
+        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
+        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
+
+        logger.info("About to render Orchestration RBAC files")
+        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
+        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
+        logger.info(env)
+
+        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
+        orch_config = {}
+        orch_config["orchestration_sa"] = saName
+        orch_config["namespace"] = namespace
+        orch_config["crd_cluster_role"] = crd_cluster_role 
+        orch_rbac_template = env.get_template('orchestration-sa.yaml')
+        temp_file_location = get_temp_file_location()
+        policy_file_location = temp_file_location + "/" + region
+        logger.info("Policy file location:" + policy_file_location)
+        if not os.path.exists(policy_file_location):
+            os.makedirs(policy_file_location)
+        fp = open(policy_file_location + "/rendered-orchestration-sa.yaml", "w")
+        fp.write(orch_rbac_template.render(orch_config))
+        fp.close()
+
+        logger.info("About to copy rendered-orchestration-rbac.yaml")
+        cmds_scp = []
+        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-orchestration-sa.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
+        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)
+
+        logger.info("About to apply rendered-orchestration-sa.yaml")
+        cmds = []
+        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-orchestration-sa.yaml')
+        self._run_commands(cmds, host_password, self.logger, block=True)
+
+    def _create_rbac_files_clusteradmin(self, region, namespace, saName, saFilePath, logger):
+        logger.info("Inside _create_rbac_files")
+        if not os.path.exists(saFilePath):
+            os.makedirs(saFilePath)
+
+        sa_metadata = {}
+        sa_metadata["namespace"] = namespace
+        sa_metadata["name"] = saName
+
+        subjects_list = []
+        subjects = {}
+        subjects["kind"] = "ServiceAccount"
+        subjects["name"] = saName
+        subjects["namespace"] = namespace
+        subjects_list.append(subjects)
+
+        temp_file_location = get_temp_file_location()
+        policy_file_location = temp_file_location + "/" + region
+        logger.info("Policy file location:" + policy_file_location)
+        if not os.path.exists(policy_file_location):
+            os.makedirs(policy_file_location)
+        fp = open(policy_file_location + "/sa-role.json", "w")
+        sa_role = {}
+        sa_role["apiVersion"] = "rbac.authorization.k8s.io/v1"
+        sa_role["kind"] = "Role"
+        sa_role["metadata"] = sa_metadata
+        sa_rules_list = []
+        sa_rules = {}
+        sa_rules["apiGroups"] = ["*"]
+        sa_rules["resources"] = ["*"]
+        sa_rules["verbs"] = ["*"]
+        sa_rules_list.append(sa_rules)
+        sa_role["rules"] = sa_rules_list
+        sa_role_json = json.dumps(sa_role)
+        logger.info("sa_role_json:" + str(sa_role_json))
+        fp.write(sa_role_json)
+
+        fp = open(policy_file_location + "/sa-rolebinding.json", "w")
+        sa_role_binding = {}
+        sa_role_binding["apiVersion"] = "rbac.authorization.k8s.io/v1"
+        sa_role_binding["kind"] = "RoleBinding"
+        sa_role_binding["metadata"] = sa_metadata
+        sa_role_binding["subjects"] = subjects_list
+        role_ref = {}
+        role_ref["kind"] = "Role"
+        role_ref["name"] = saName
+        role_ref["apiGroup"] = "rbac.authorization.k8s.io"
+        sa_role_binding["roleRef"] = role_ref
+        sa_role_binding_json = json.dumps(sa_role_binding)
+        fp.write(sa_role_binding_json)
+
+        fp = open(temp_file_location + "/sa-clusterrole.json", "w")
+        sa_clusterrole = {}
+        sa_clusterrole["apiVersion"] = "rbac.authorization.k8s.io/v1"
+        sa_clusterrole["kind"] = "ClusterRole"
+        sa_clusterrole["metadata"] = sa_metadata
+        sa_rules_list = []
+        sa_rules = {}
+        sa_rules["apiGroups"] = [""]
+        sa_rules["resources"] = ["*"]
+        sa_rules["verbs"] = ["*"]
+        sa_rules_list.append(sa_rules)
+        sa_clusterrole["rules"] = sa_rules_list
+        sa_clusterrole_json = json.dumps(sa_clusterrole)
+        fp.write(sa_clusterrole_json)
+
+        fp = open(temp_file_location + "/sa-clusterrolebinding.json", "w")
+        sa_clusterrole_binding = {}
+        sa_clusterrole_binding["apiVersion"] = "rbac.authorization.k8s.io/v1"
+        sa_clusterrole_binding["kind"] = "ClusterRoleBinding"
+        sa_clusterrole_binding["metadata"] = sa_metadata
+        sa_clusterrole_binding["subjects"] = subjects_list
+        clusterrole_ref = {}
+        clusterrole_ref["kind"] = "ClusterRole"
+        clusterrole_ref["name"] = saName
+        clusterrole_ref["apiGroup"] = "rbac.authorization.k8s.io"
+        sa_clusterrole_binding["roleRef"] = clusterrole_ref
+        sa_clusterrole_binding_json = json.dumps(sa_clusterrole_binding)
+        fp.write(sa_clusterrole_binding_json)
+        fp.close()
+
+    def _wait_for_oidc_app(self, region, namespace):
+        self.logger.info("Inside checking _wait_for_oidc_app...")
+
+        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        env_string = self._read_remote_openrc(remote_region_oam_ip, self.logger)
+        basecmd = env_string
+
+        status_applied = False
+        while not status_applied:
+            cmds = []
+            cmd = ssh_prefix + " " + basecmd + " system application-show oidc-auth-apps"
+            self.logger.info("cmd:" + cmd)
+            cmds.append(cmd)
+            output_lines = self._run_command_get_all_lines(cmds, host_password, self.logger)
+            self.logger.info("Returned output lines:")
+            self.logger.info(output_lines)
+            if len(output_lines) > 0:
+                for line in output_lines.split("\n"):
+                    self.logger.info("Line:" + line)
+                    if 'status' in line and ('applied' in line or 'apply-failed' in line):
+                        self.logger.info("oidc-auth-apps applied: " + line)
+                        if 'applied' in line:
+                            status_applied = True
+                        if 'apply-failed' in line:
+                            cmds1 = []
+                            cmd1 = ssh_prefix + " " + basecmd + " system application-remove oidc-auth-apps"
+                            self.logger.info("cmd:" + cmd1)
+                            cmd2 = ssh_prefix + " " + basecmd + " system application-apply oidc-auth-apps"
+                            self.logger.info("cmd:" + cmd2)
+                            cmds1.append(cmd1)
+                            cmds1.append(cmd2)
+                            output_lines1 = self._run_command_get_all_lines(cmds1 , host_password, self.logger)
+                            self.logger.info("Returned output lines:")
+                            self.logger.info(output_lines1)
+            time.sleep(3)
+
+    def _create_kubeconfig(self, kubeconfig_for, region, namespace, kubeconfig_approach, transaction_id):
+        self.logger.info("Kubeconfig for:" + kubeconfig_for)
+        kubeconfig_src = kubeconfig_approach
+        if kubeconfig_for == KubeconfigGenerator.ORCHESTRATION_TEAM:
+            if kubeconfig_src == 'SA':
+                self._create_kubeconfig_orchestration_sa(region, namespace, transaction_id)
+            if kubeconfig_src == 'LDAP':
+                #self._wait_for_oidc_app(region, namespace)
+                self._create_kubeconfig_orchestration_ldap(region, namespace, transaction_id)
+        elif kubeconfig_for == KubeconfigGenerator.APPLICATION_TEAM:
+            if kubeconfig_src == 'SA':
+                self._create_kubeconfig_application_sa(region, namespace, transaction_id)
+            if kubeconfig_src == 'LDAP':
+                self._create_kubeconfig_application_ldap(region, namespace, transaction_id)
+            self.vmbCoordinationQueue.put("Done")
+        elif kubeconfig_for == KubeconfigGenerator.OPS_TEAM:
+            self._create_kubeconfig_ops(region, namespace, transaction_id)
+        elif kubeconfig_for == KubeconfigGenerator.VENDOR_TEAM_SAMSUNG:
+            self._create_kubeconfig_samsung(region, namespace, transaction_id)
+            self.vmbCoordinationQueue.put("Done")
+
+    def _create_kubeconfig_application_sa(self, region, namespace, transaction_id):
+        self.logger.info("Inside ..kubeconfig application")
+        saName = self._get_serviceaccount_name(KubeconfigGenerator.APPLICATION_TEAM, transaction_id)
+        self.logger.info("Service Account name:" + saName)
+
+        message = 'Starting kubeconfig creation. '
+        status = 'STARTING'
+        self.logger.info(message + ' ' + status)
+        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
+
+        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        app_sa_namespace = 'default' # default ns is better as it already exists 
+
+        sa_files_dir = []
+        folder = 'cluster-setup-files'
+        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
+        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
+  
+        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
+
+        message = 'Applying RBAC to Service Account'
+        status = 'CREATING'
+        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
+        self.logger.info(" Service Account File Path:" + str(saFilePath))
+
+        self._apply_application_rbac_policies_sa(region, remote_region_oam_ip, app_sa_namespace, namespace, saName, saFilePath, self.logger)
+        
+        cmds_token_name = []
+        cmds_token_name.append(ssh_prefix + " kubectl describe serviceaccount --kubeconfig=/etc/kubernetes/admin.conf -n " + app_sa_namespace + " " + saName  + "| grep Tokens ")
+        all_lines = self._run_commands(cmds_token_name, host_password, self.logger, block=True)
+        secretname = self._parse_token_name(all_lines)
+
+        message = 'Parsing token'
+        status = 'CREATING'
+        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
+        self.logger.info(" Secret name:" + secretname)
+        if secretname != None:
+            cmds1 = []
+            cmds1.append(ssh_prefix + " kubectl describe secret --kubeconfig=/etc/kubernetes/admin.conf -n " + app_sa_namespace + " " + secretname + " | grep token:")
+            all_lines = self._run_commands(cmds1, host_password, self.logger, block=True)
+            token = self._parse_token(all_lines)
+            #self.logger.info("TOKEN TO USE:" + token)
+            token = token.strip()
+            self.logger.info(" Token:[" + str(token) + "]")
+
+            # Generate kubeconfig
+            kubeconfig_value = self._generate_kubeconfig(region, namespace, saName, token, transaction_id, self.logger)
+
+            # Update database
+            message = 'kubeconfig creation done. '
+            status = 'COMPLETE'
+            self.logger.info(message + ' ' + status)
+            self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)
+
+    def _create_kubeconfig_application_ldap(self, region, namespace, transaction_id):
+        self.logger.info("Inside kubeconfig application ldap...")
+        message = 'Starting kubeconfig creation. '
+        status = 'STARTING'
+        self.logger.info(message + ' ' + status)
+        app_user = self._get_user_name(KubeconfigGenerator.APPLICATION_TEAM)
+        self.logger.info("Application User:" + app_user)
+        self._update_status_async(region, namespace, app_user, message, status, transaction_id, self.logger, kubeconfig='')
+
+        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        message = 'Applying RBAC to User Account'
+        status = 'CREATING'
+        self._update_status_async(region, namespace, app_user, message, status, transaction_id, self.logger, kubeconfig='')
+
+        self.logger.info("Applying RBAC policies to the Application User...")
+        self._apply_application_rbac_policies_ldap(region, remote_region_oam_ip, namespace, self.logger)
+ 
+        ldap_account = 'edge_eng'
+        token = self._get_ldap_token(host_password, ssh_prefix, app_user, ldap_account, remote_region_oam_ip)
+
+        # Generate kubeconfig
+        kubeconfig_value = self._generate_kubeconfig(region, namespace, app_user, token, transaction_id, self.logger)
+
+        # Update database
+        message = 'kubeconfig creation done. '
+        status = 'COMPLETE'
+        self.logger.info(message + ' ' + status)
+        self._update_status_async(region, namespace, app_user, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)
+
+        #mv /home/sysadmin/.kube/config /home/sysadmin/.kube/config-for-orchestration
+        self.logger.info("Moving /home/sysadmin/.kube/config to /home/sysadmin/.kube/config-for-edge-eng...")
+        set_kubeconfig_cmd = []
+        set_kubeconfig_cmd.append(ssh_prefix + ' mv /home/sysadmin/.kube/config /home/sysadmin/.kube/config-for-edge-eng')
+        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)
+
+        #export KUBECONFIG=/etc/kubernetes/admin.conf
+        self.logger.info("Resetting KUBECONFIG...")
+        set_kubeconfig_cmd = []
+        set_kubeconfig_cmd.append(ssh_prefix + ' export KUBECONFIG=/etc/kubernetes/admin.conf ')
+        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)
+        return kubeconfig_value
+
+    def _create_kubeconfig_samsung(self, region, namespace, transaction_id):
+        self.logger.info("Inside ..kubeconfig samsung")
+        saName = self._get_serviceaccount_name(KubeconfigGenerator.VENDOR_TEAM_SAMSUNG, transaction_id)
+        self.logger.info("Service Account name:" + saName)
+
+        message = 'Starting kubeconfig creation. '
+        status = 'STARTING'
+        self.logger.info(message + ' ' + status)
+        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
+
+        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        app_sa_namespace = 'default' # default ns is better as it already exists 
+
+        sa_files_dir = []
+        folder = 'cluster-setup-files'
+        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
+        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
+  
+        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
+
+        message = 'Applying RBAC to Service Account'
+        status = 'CREATING'
+        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
+        self.logger.info(" Service Account File Path:" + str(saFilePath))
+
+        self._apply_vendor_samsung_rbac_policies(region, remote_region_oam_ip, app_sa_namespace, namespace, saName, saFilePath, self.logger)
+        
+        cmds_token_name = []
+        cmds_token_name.append(ssh_prefix + " kubectl describe serviceaccount --kubeconfig=/etc/kubernetes/admin.conf -n " + app_sa_namespace + " " + saName  + "| grep Tokens ")
+        all_lines = self._run_commands(cmds_token_name, host_password, self.logger, block=True)
+        secretname = self._parse_token_name(all_lines)
+
+        message = 'Parsing token'
+        status = 'CREATING'
+        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
+        self.logger.info(" Secret name:" + secretname)
+        if secretname != None:
+            cmds1 = []
+            cmds1.append(ssh_prefix + " kubectl describe secret --kubeconfig=/etc/kubernetes/admin.conf -n " + app_sa_namespace + " " + secretname + " | grep token:")
+            all_lines = self._run_commands(cmds1, host_password, self.logger, block=True)
+            token = self._parse_token(all_lines)
+            #self.logger.info("TOKEN TO USE:" + token)
+            token = token.strip()
+            self.logger.info(" Token:[" + str(token) + "]")
+
+            # Generate kubeconfig
+            kubeconfig_value = self._generate_kubeconfig(region, namespace, saName, token, transaction_id, self.logger)
+
+            # Update database
+            message = 'kubeconfig creation done. '
+            status = 'COMPLETE'
+            self.logger.info(message + ' ' + status)
+            self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)
+
+
+    def _create_kubeconfig_ops(self, region, namespace, transaction_id):
+        pass
+
+    def _create_kubeconfig_orchestration_ldap(self, region, namespace, transaction_id):
+        self.logger.info("Inside kubeconfig orchestration ldap...")
+        message = 'Starting kubeconfig creation. '
+        status = 'STARTING'
+        self.logger.info(message + ' ' + status)
+        orch_user = self._get_user_name(KubeconfigGenerator.ORCHESTRATION_TEAM)
+        self.logger.info("Orchestration User:" + orch_user)
+        self._update_status_async(region, namespace, orch_user, message, status, transaction_id, self.logger, kubeconfig='')
+
+        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+
+        message = 'Applying RBAC to User Account'
+        status = 'CREATING'
+        self._update_status_async(region, namespace, orch_user, message, status, transaction_id, self.logger, kubeconfig='')
+
+        self.logger.info("Applying RBAC policies to the Orchestration User...")
+        self._apply_orchestration_rbac_policies_ldap(region, remote_region_oam_ip, namespace, self.logger)
+ 
+        ldap_account = 'orchestration'
+        token = self._get_ldap_token(host_password, ssh_prefix, orch_user, ldap_account, remote_region_oam_ip)
+
+        # Generate kubeconfig
+        kubeconfig_value = self._generate_kubeconfig(region, namespace, orch_user, token, transaction_id, self.logger)
+
+        # Update database
+        message = 'kubeconfig creation done. '
+        status = 'COMPLETE'
+        self.logger.info(message + ' ' + status)
+        self._update_status_async(region, namespace, orch_user, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)
+
+        #mv /home/sysadmin/.kube/config /home/sysadmin/.kube/config-for-orchestration
+        self.logger.info("Moving /home/sysadmin/.kube/config to /home/sysadmin/.kube/config-for-orchestration...")
+        set_kubeconfig_cmd = []
+        set_kubeconfig_cmd.append(ssh_prefix + ' mv /home/sysadmin/.kube/config /home/sysadmin/.kube/config-for-orchestration')
+        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)
+
+        #export KUBECONFIG=/etc/kubernetes/admin.conf
+        self.logger.info("Resetting KUBECONFIG...")
+        set_kubeconfig_cmd = []
+        set_kubeconfig_cmd.append(ssh_prefix + ' export KUBECONFIG=/etc/kubernetes/admin.conf ')
+        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)
+
+        # Send notification on VMB
+        self._send_vmb_notification(region, namespace, transaction_id, kubeconfig_value)
+        return kubeconfig_value
+
+    def _get_ldap_token(self, host_password, ssh_prefix, user, ldap_account, remote_region_oam_ip):
+        #cp /etc/kubernetes/admin.conf /home/sysadmin/.kube/config
+        self.logger.info("Starting token generation...")
+        cp_cmd = []
+        cp_cmd.append(ssh_prefix + ' cp /etc/kubernetes/admin.conf /home/sysadmin/.kube/config')
+        self._run_commands(cp_cmd, host_password, self.logger, block=True)
+
+        #export KUBECONFIG=/home/sysadmin/.kube/config
+        self.logger.info("Setting KUBECONFIG...")
+        set_kubeconfig_cmd = []
+        set_kubeconfig_cmd.append(ssh_prefix + ' export KUBECONFIG=/home/sysadmin/.kube/config ')
+        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)
+
+        #kubectl config set-context --kubeconfig=/home/sysadmin/.kube/config SVC-Edge-Eng@kubernetes --cluster=kubernetes --user=SVC-Edge-Eng
+        self.logger.info("Performing set-context...")
+        set_context = ' kubectl config set-context --kubeconfig=/home/sysadmin/.kube/config ' + user + '@kubernetes --cluster=kubernetes --user=' + user
+        self.logger.info("Set context cmd:" + set_context)
+        set_context_cmd = []
+        set_context_cmd.append(ssh_prefix + set_context)
+        self._run_commands(set_context_cmd, host_password, self.logger, block=True)
+
+        #oidc-auth -c <OAM-IP> -u SVC-Edge-Eng -p 322C6v22acuhAGdyce22S3w282
+        self.logger.info("Executing oidc-auth...")
+        orch_password = settings.LDAP_USER_CREDS[ldap_account]
+        oidc_auth = ' oidc-auth -c ' + remote_region_oam_ip + ' -u ' + user + ' -p ' + orch_password
+        self.logger.info("OIDC Auth..:" + oidc_auth)
+        oidc_auth_cmd = []
+        oidc_auth_cmd.append(ssh_prefix + oidc_auth)
+        self._run_commands(oidc_auth_cmd, host_password, self.logger, block=True)
+
+        #grep token /home/sysadmin/.kube/config
+        self.logger.info("Retrieving token...")
+        cmds1 = []
+        cmds1.append(ssh_prefix + " grep token /home/sysadmin/.kube/config ")
+        all_lines = self._run_commands(cmds1, host_password, self.logger, block=True)
+        token = self._parse_token(all_lines)
+        #self.logger.info("TOKEN TO USE:" + token)
+        token = token.strip()
+        self.logger.info(" Token:[" + str(token) + "]")
+        return token
+
+    def _create_kubeconfig_orchestration_sa(self, region, namespace, transaction_id):
+        # For orchestration SA, we use either the 'orchestration' namespace or the 'default' namespace
+        self.logger.info("Inside ..kubeconfig orchestration sa")
+        saName = self._get_serviceaccount_name(KubeconfigGenerator.ORCHESTRATION_TEAM, transaction_id)
+        self.logger.info("Service Account name:" + saName)
+
+        message = 'Starting kubeconfig creation. '
+        status = 'STARTING'
+        self.logger.info(message + ' ' + status)
+        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
+
+        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
+        orch_sa_namespace = 'default' # default ns is better as it already exists 
+
+        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
+        
+        #cmds_sa = []
+        #cmds_sa.append(ssh_prefix + ' kubectl create serviceaccount --kubeconfig=/etc/kubernetes/admin.conf ' + saName + ' -n ' + orch_sa_namespace)
+        #self._run_commands(cmds_sa, host_password, self.logger, block=True)
+
+        sa_files_dir = []
+        folder = 'cluster-setup-files'
+        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
+        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
+  
+        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
+
+        message = 'Applying RBAC to Service Account'
+        status = 'CREATING'
+        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
+        self.logger.info(" Service Account File Path:" + str(saFilePath))
+
+        self._apply_orchestration_rbac_policies_sa(region, remote_region_oam_ip, orch_sa_namespace, namespace, saName, saFilePath, self.logger)
+        
+        cmds_token_name = []
+        secret_name_found = False
+        cmds_token_name.append(ssh_prefix + " kubectl describe serviceaccount --kubeconfig=/etc/kubernetes/admin.conf -n " + orch_sa_namespace + " " + saName  + "| grep Tokens ")
+        while not secret_name_found:
+            all_lines = self._run_commands(cmds_token_name, host_password, self.logger, block=True)
+            # Check if secretname != <none>; if so, repeat the command
+            ## kubeconfighandler.py _create_kubeconfig_orchestration 351  Secret name:<none>
+            secretname = self._parse_token_name(all_lines)
+            if secretname != "<none>":
+                secret_name_found = True
+            else:
+                time.sleep(60)
+
+        message = 'Parsing token'
+        status = 'CREATING'
+        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
+        self.logger.info(" Secret name:" + secretname)
+        if secretname != None:
+            #self.logger.info("ABC")
+            cmds1 = []
+            cmds1.append(ssh_prefix + " kubectl describe secret --kubeconfig=/etc/kubernetes/admin.conf -n " + orch_sa_namespace + " " + secretname + " | grep token:")
+            all_lines = self._run_commands(cmds1, host_password, self.logger, block=True)
+            token = self._parse_token(all_lines)
+            #self.logger.info("TOKEN TO USE:" + token)
+            token = token.strip()
+            self.logger.info(" Token:[" + str(token) + "]")
+
+            # Generate kubeconfig
+            kubeconfig_value = self._generate_kubeconfig(region, namespace, saName, token, transaction_id, self.logger)
+
+            # Update database
+            message = 'kubeconfig creation done. '
+            status = 'COMPLETE'
+            self.logger.info(message + ' ' + status)
+            self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)
+            
+            # Send notification on VMB
+            self._send_vmb_notification(region, namespace, transaction_id, kubeconfig_value)
+            return kubeconfig_value
+
+    def _send_vmb_notification(self, region, namespace, transaction_id, kubeconfig_value):
+            created_at_time = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+
+            site_name, site_location = self.get_fuze_spm_site_details(region, self.logger)
+            self.logger.info("Site name:" + site_name)
+            self.logger.info("Site location:" + site_location)
+
+            # Send VMB Notification
+            kubeconfig = Kubeconfig(
+                cluster=region,
+                namespace=namespace,
+                location=site_location,
+                kubeconfig='Example kubeconfig',
+                created_at=str(created_at_time),
+                updated_at=str(created_at_time),
+                transactionId=str(transaction_id)
+            )
+
+            kubeconfig.kubeconfig = kubeconfig_value
+
+            reportDescription = 'kubeconfig file: ' + site_name
+            kubeconfig_message = KubeconfigMessage(
+                reportName='vcp_fe_kubeconfig',
+                reportDescription=reportDescription,
+                reportGeneratedOn=str(created_at_time),
+                rowCount = 1,
+                reportDataRows=[kubeconfig],
+            )
+
+            item = {}
+            item['payload'] = kubeconfig_message
+            item['message'] = 'Kubeconfig'
+            self.logger.info("Sending to VMB")
+            self.logger.info(kubeconfig_message)
+            self.logger.info(kubeconfig)
+            self.vmbQueue.put(item)
+
+    def _generate_kubeconfig(self, region, namespace, saName, token, transaction_id, logger):
+        logger.info(" Inside _generate_kubeconfig")
+
+        user_list = []
+        tokendata = {}
+        tokendata['token'] = token
+        userdata = {}
+        userdata['name'] = saName
+        userdata['user'] = tokendata
+        user_list.append(userdata)
+        
+        context_list = []
+        contextdatawrapper = {}
+        contextdata = {}
+        contextdata['cluster'] = region
+        contextdata['user'] = saName
+        contextdata['namespace'] = namespace
+        contextdatawrapper['context'] = contextdata
+        contextdatawrapper['name'] = region
+        context_list.append(contextdatawrapper)
+
+        cluster_list = []
+        clusterdata = {}
+        clusterdata['name'] = region
+        cluster_detail = {}
+        cluster_detail['insecure-skip-tls-verify'] = True 
+        remote_region_oam_ip = self._get_remote_region_oam_ip(region, logger)
+        cluster_detail['server'] = 'https://[' + remote_region_oam_ip + ']:6443'
+        clusterdata["cluster"] = cluster_detail
+        cluster_list.append(clusterdata)
+        
+        outer_dict = {}
+        outer_dict['apiVersion'] = 'v1'
+        outer_dict['kind'] = 'Config'
+        outer_dict['current-context'] = region
+        outer_dict['users'] = user_list
+        outer_dict['contexts'] = context_list
+        outer_dict['clusters'] = cluster_list
+
+        kubeconfig_json = json.dumps(outer_dict)
+        logger.info(str(transaction_id) + " Kubeconfig:" + kubeconfig_json)
+        return kubeconfig_json
+
+    def _parse_token_name(self, all_lines):
+        for line in all_lines.split("\n"):
+            if 'Tokens' in line:
+                parts = line.split(":")
+                tokenName = parts[1].rstrip().lstrip()
+                return tokenName
+
+    def _parse_token(self, all_lines):
+        for line in all_lines.split("\n"):
+            if 'token:' in line:
+                parts = line.split(":")
+                token = parts[1].rstrip().lstrip()
+                return token
+
+    def _run_commands_scp(self, commands, host_password, logger, block=False):
+        logger.info("Inside _run_commands_scp..:")
+        status = run_commands_scp(commands, host_password, logger, block=False)
+        return status
+
+    def _run_commands(self, commands, host_password, logger, block=False):
+        #logger.info(commands)
+        all_lines1 = []
+        for command in commands:
+            logger.info(" Executing.." + str(command))
+            child = pexpect.spawn(command)
+            try:
+                if block:
+                    child.timeout=None
+                    child.expect(['password: '], timeout=None)
+                    child.sendline(host_password)
+                    all_lines1 = child.read()
+                    all_lines1 = all_lines1.rstrip().lstrip()
+                    all_lines1 = all_lines1.decode('utf-8').replace('\r\n', '\n')
+                    logger.info(all_lines1)
+                    return all_lines1
+                else:
+                    child.expect(['(yes/no)? '])
+                    child.sendline('yes')
+                    child.expect(['password: '])
+                    child.sendline(host_password)
+                    #child.interact()
+                    #child.close()
+                    child.expect(pexpect.EOF, timeout=5)
+            except:
+                pass
+        return all_lines1
+
+    def _update_status_async(self, region, namespace, saName, message, status, transaction_id, logger, kubeconfig='', done=False):
+        item = {}
+        item['type'] = 'kubeconfig'
+        item['region'] = region
+        item['namespace'] = namespace
+        item['saName'] = saName
+        item['message'] = message
+        item['status'] = status
+        item['transaction_id'] = transaction_id
+        item['kubeconfig'] = kubeconfig
+        item['done'] = done
+        self.dbQueue.put(item)
+
+    def _update_status(self, region, namespace, saName, message, status, transaction_id, logger, kubeconfig=''):
+        RemoteRegionSetup.objects.filter(transaction_id=transaction_id).update(message=message,
+                                                                               status=status,
+                                                                               serviceaccount=saName,
+                                                                               kubernetes_namespace=namespace,
+                                                                               remote_region_name=region,
+                                                                               kubeconfig=kubeconfig)
+        return
+