path: root/src/orchestration/kubeconfighandler.py

import django

from django.conf import settings
from django.core.exceptions import AppRegistryNotReady
from django.utils import timezone

from multiprocessing import Process, Pool, Queue
import pexpect
import logging
import multiprocessing
from logging.handlers import QueueHandler
import sys
import time
import datetime
from .utils import *

import os
import signal, psutil
import json

from jinja2 import Environment, FileSystemLoader

from .vmb_messages import Kubeconfig, KubeconfigMessage 
from .remoteregionhandler import RemoteRegionWorker
from .vendorhandler import Vendor

try:
    django.setup()
    from .models import RemoteRegionSetup
    from caas.models import Cluster
    from caas.models import Namespace  
except django.core.exceptions.AppRegistryNotReady as exp:
    pass

class KubeconfigGenerator(RemoteRegionWorker):

    ORCHESTRATION_TEAM = 'orchestration-team'
    APPLICATION_TEAM = 'application-team'
    VENDOR_TEAM_SAMSUNG = 'samsung-team'
    OPS_TEAM = 'ops-team'

    def __init__(self, loggerQueue, requestQueue, dbQueue, vmbQueue, dbCoordinationQueue, vmbCoordinationQueue, doneQueue):
        self.loggerQueue = loggerQueue
        self.requestQueue = requestQueue
        self.dbQueue = dbQueue
        self.vmbQueue = vmbQueue
        self.dbCoordinationQueue = dbCoordinationQueue
        self.vmbCoordinationQueue = vmbCoordinationQueue
        self.doneQueue = doneQueue

    def run(self):
        qh = QueueHandler(self.loggerQueue)
        self.logger = logging.getLogger()
        self.logger.addHandler(qh)
        self.logger.setLevel(logging.DEBUG)

        self.logger.info("KubeconfigGenerator started...")

        while True:
            try:
                item = self.requestQueue.get(block=True)
                if item:
                    self.logger.info(item)
                    self.get_kubeconfig(item)
                    self.logger.info("Done generating kubeconfig")
                    self.logger.info("Performing vendor setup...")
                    vendor_provisioner = Vendor(self.logger, self.dbCoordinationQueue, self.vmbCoordinationQueue, self.doneQueue)
                    vendor_provisioner.perform_vendor_setup(item)
            except:
                pass
            time.sleep(1)

    def get_kubeconfig(self, request):
        transaction_id = request['transaction_id']
        region = request['remote_region']
        kubeconfig_for = request['kubeconfig_for']
        kubeconfig_approach = request['kubeconfig_approach']
        namespace = self._get_namespace(region, self.logger)
        if namespace != "": 
            self.logger.info(" Transaction:" + transaction_id)
            self.logger.info(" Region:" + region)
            self.logger.info(" Namespace:" + namespace)
            kubeconfig = self._create_kubeconfig(kubeconfig_for, region, namespace, kubeconfig_approach, transaction_id)
            return kubeconfig
        else:
            message = " Could not find Namespace for region " + region 
            self.logger.info(message)
            status = "FAILED"
            self._update_status_async(region, namespace, '', message, status, transaction_id, self.logger, kubeconfig='', done=True)
            return json.dumps({}) 

    def _get_serviceaccount_name(self, team, transaction_id):
        if team == KubeconfigGenerator.ORCHESTRATION_TEAM:
            return "orchestration-sa"
        if team == KubeconfigGenerator.APPLICATION_TEAM:
            return "application-sa-" + str(transaction_id)
        if team == KubeconfigGenerator.OPS_TEAM:
            return "ops-sa"
        if team == KubeconfigGenerator.VENDOR_TEAM_SAMSUNG:
            return "samsung-sa"

    def _get_user_name(self, team):
        if team == KubeconfigGenerator.ORCHESTRATION_TEAM:
            return "SVC-FE-Atlas"
        if team == KubeconfigGenerator.APPLICATION_TEAM:
            return "SVC-Edge-Eng"

    def _apply_application_rbac_policies_sa(self, region, remote_region_oam_ip, app_sa_namespace, namespace, saName, saFilePath, logger):
        logger.info("Inside _apply_application_rbac_policies")

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        sa_files_dir = []
        folder = 'cluster-setup-files'
        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
        self._run_commands(sa_files_dir, host_password, self.logger, block=True)

        logger.info("About to render Application RBAC files")
        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
        logger.info(env)

        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
        app_config = {}
        app_config["application_sa"] = saName
        app_config["namespace"] = namespace
        app_config["crd_cluster_role"] = crd_cluster_role
        app_config["crd_api_group"] = crd_api_group
        app_config["crd_api_resources"] = crd_api_resources 
        app_rbac_template = env.get_template('application-sa.yaml')
        temp_file_location = get_temp_file_location()
        policy_file_location = temp_file_location + "/" + region
        logger.info("Policy file location:" + policy_file_location)
        if not os.path.exists(policy_file_location):
            os.makedirs(policy_file_location)
        fp = open(policy_file_location + "/rendered-application-sa.yaml", "w")
        fp.write(app_rbac_template.render(app_config))
        fp.close()

        logger.info("About to copy rendered-application-rbac.yaml")
        cmds_scp = []
        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-application-sa.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)

        logger.info("About to apply rendered-application-sa.yaml")
        cmds = []
        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-application-sa.yaml')
        self._run_commands(cmds, host_password, self.logger, block=True)

    def _apply_application_rbac_policies_ldap(self, region, remote_region_oam_ip, namespace, logger):
        logger.info("Inside _apply_application_rbac_policies ldap")

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        sa_files_dir = []
        folder = 'cluster-setup-files'
        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
        self._run_commands(sa_files_dir, host_password, self.logger, block=True)

        logger.info("About to render Application RBAC files")
        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
        self.logger.info(" RBAC File Path:" + str(saFilePath))
        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
        logger.info(env)

        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
        orch_config = {}
        orch_config["namespace"] = namespace
        orch_config["crd_cluster_role"] = crd_cluster_role
        orch_config["crd_api_group"] = crd_api_group
        orch_config["crd_api_resources"] = crd_api_resources
        orch_rbac_template = env.get_template('application-ldap.yaml')
        temp_file_location = get_temp_file_location()
        policy_file_location = temp_file_location + "/" + region
        logger.info("Policy file location:" + policy_file_location)
        if not os.path.exists(policy_file_location):
            os.makedirs(policy_file_location)
        fp = open(policy_file_location + "/rendered-application-ldap.yaml", "w")
        fp.write(orch_rbac_template.render(orch_config))
        fp.close()

        logger.info("About to copy rendered-application-ldap.yaml")
        cmds_scp = []
        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-application-ldap.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)

        logger.info("About to apply rendered-application-ldap.yaml")
        cmds = []
        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-application-ldap.yaml')
        self._run_commands(cmds, host_password, self.logger, block=True)

    def _apply_vendor_samsung_rbac_policies(self, region, remote_region_oam_ip, app_sa_namespace, namespace, saName, saFilePath, logger):
        logger.info("Inside _apply_vendor_samsung_rbac_policies")

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        sa_files_dir = []
        folder = 'cluster-setup-files'
        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
        self._run_commands(sa_files_dir, host_password, self.logger, block=True)

        logger.info("About to render Vendor Samsung RBAC files")
        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
        logger.info(env)

        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
        app_config = {}
        app_config["application_sa"] = saName
        app_config["namespace"] = namespace
        app_config["crd_cluster_role"] = crd_cluster_role
        app_config["crd_api_group"] = crd_api_group
        app_config["crd_api_resources"] = crd_api_resources 
        app_rbac_template = env.get_template('samsung-sa.yaml')
        temp_file_location = get_temp_file_location()
        policy_file_location = temp_file_location + "/" + region
        logger.info("Policy file location:" + policy_file_location)
        if not os.path.exists(policy_file_location):
            os.makedirs(policy_file_location)
        fp = open(policy_file_location + "/rendered-samsung-sa.yaml", "w")
        fp.write(app_rbac_template.render(app_config))
        fp.close()

        logging.info("About to copy rendered-samsung-rbac.yaml")
        cmds_scp = []
        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-samsung-sa.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)

        logger.info("About to apply rendered-samsung-sa.yaml")
        cmds = []
        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-samsung-sa.yaml')
        self._run_commands(cmds, host_password, self.logger, block=True)

    def _apply_orchestration_rbac_policies_ldap(self, region, remote_region_oam_ip, namespace, logger):
        logger.info("Inside _apply_orchestration_rbac_policies ldap")

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        sa_files_dir = []
        folder = 'cluster-setup-files'
        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
        self._run_commands(sa_files_dir, host_password, self.logger, block=True)

        logger.info("About to render Orchestration RBAC files")
        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
        self.logger.info(" RBAC File Path:" + str(saFilePath))
        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
        logger.info(env)

        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
        orch_config = {}
        orch_config["namespace"] = namespace
        orch_config["crd_cluster_role"] = crd_cluster_role 
        orch_rbac_template = env.get_template('orchestration-ldap.yaml')
        temp_file_location = get_temp_file_location()
        policy_file_location = temp_file_location + "/" + region
        logger.info("Policy file location:" + policy_file_location)
        if not os.path.exists(policy_file_location):
            os.makedirs(policy_file_location)
        fp = open(policy_file_location + "/rendered-orchestration-ldap.yaml", "w")
        fp.write(orch_rbac_template.render(orch_config))
        fp.close()

        logger.info("About to copy rendered-orchestration-ldap.yaml")
        cmds_scp = []
        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-orchestration-ldap.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)

        logger.info("About to apply rendered-orchestration-ldap.yaml")
        cmds = []
        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-orchestration-ldap.yaml')
        self._run_commands(cmds, host_password, self.logger, block=True)

    def _apply_orchestration_rbac_policies_sa(self, region, remote_region_oam_ip, orch_sa_namespace, namespace, saName, saFilePath, logger):
        logger.info("Inside _apply_orchestration_rbac_policies sa")

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        sa_files_dir = []
        folder = 'cluster-setup-files'
        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
        self._run_commands(sa_files_dir, host_password, self.logger, block=True)

        logger.info("About to render Orchestration RBAC files")
        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)
        env = Environment(loader = FileSystemLoader(saFilePath), trim_blocks=True, lstrip_blocks=True)
        logger.info(env)

        crd_cluster_role, crd_api_group, crd_api_resources, crd_api_verbs = self._get_crd_details()
        orch_config = {}
        orch_config["orchestration_sa"] = saName
        orch_config["namespace"] = namespace
        orch_config["crd_cluster_role"] = crd_cluster_role 
        orch_rbac_template = env.get_template('orchestration-sa.yaml')
        temp_file_location = get_temp_file_location()
        policy_file_location = temp_file_location + "/" + region
        logger.info("Policy file location:" + policy_file_location)
        if not os.path.exists(policy_file_location):
            os.makedirs(policy_file_location)
        fp = open(policy_file_location + "/rendered-orchestration-sa.yaml", "w")
        fp.write(orch_rbac_template.render(orch_config))
        fp.close()

        logger.info("About to copy rendered-orchestration-rbac.yaml")
        cmds_scp = []
        cmds_scp.append('scp -o StrictHostKeyChecking=no ' + policy_file_location + '/rendered-orchestration-sa.yaml ' + host_username + '@[' + remote_region_oam_ip + ']:~/' + folder + '/.')
        self._run_commands_scp(cmds_scp, host_password, self.logger, block=True)

        logger.info("About to apply rendered-orchestration-sa.yaml")
        cmds = []
        cmds.append(ssh_prefix + ' kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f ./' + folder + '/rendered-orchestration-sa.yaml')
        self._run_commands(cmds, host_password, self.logger, block=True)

    def _create_rbac_files_clusteradmin(self, region, namespace, saName, saFilePath, logger):
        logger.info("Inside _create_rbac_files")
        if not os.path.exists(saFilePath):
            os.makedirs(saFilePath)

        sa_metadata = {}
        sa_metadata["namespace"] = namespace
        sa_metadata["name"] = saName

        subjects_list = []
        subjects = {}
        subjects["kind"] = "ServiceAccount"
        subjects["name"] = saName
        subjects["namespace"] = namespace
        subjects_list.append(subjects)

        temp_file_location = get_temp_file_location()
        policy_file_location = temp_file_location + "/" + region
        logger.info("Policy file location:" + policy_file_location)
        if not os.path.exists(policy_file_location):
            os.makedirs(policy_file_location)
        fp = open(policy_file_location + "/sa-role.json", "w")
        sa_role = {}
        sa_role["apiVersion"] = "rbac.authorization.k8s.io/v1"
        sa_role["kind"] = "Role"
        sa_role["metadata"] = sa_metadata
        sa_rules_list = []
        sa_rules = {}
        sa_rules["apiGroups"] = ["*"]
        sa_rules["resources"] = ["*"]
        sa_rules["verbs"] = ["*"]
        sa_rules_list.append(sa_rules)
        sa_role["rules"] = sa_rules_list
        sa_role_json = json.dumps(sa_role)
        logger.info("sa_role_json:" + str(sa_role_json))
        fp.write(sa_role_json)

        fp = open(policy_file_location + "/sa-rolebinding.json", "w")
        sa_role_binding = {}
        sa_role_binding["apiVersion"] = "rbac.authorization.k8s.io/v1"
        sa_role_binding["kind"] = "RoleBinding"
        sa_role_binding["metadata"] = sa_metadata
        sa_role_binding["subjects"] = subjects_list
        role_ref = {}
        role_ref["kind"] = "Role"
        role_ref["name"] = saName
        role_ref["apiGroup"] = "rbac.authorization.k8s.io"
        sa_role_binding["roleRef"] = role_ref
        sa_role_binding_json = json.dumps(sa_role_binding)
        fp.write(sa_role_binding_json)

        fp = open(temp_file_location + "/sa-clusterrole.json", "w")
        sa_clusterrole = {}
        sa_clusterrole["apiVersion"] = "rbac.authorization.k8s.io/v1"
        sa_clusterrole["kind"] = "ClusterRole"
        sa_clusterrole["metadata"] = sa_metadata
        sa_rules_list = []
        sa_rules = {}
        sa_rules["apiGroups"] = [""]
        sa_rules["resources"] = ["*"]
        sa_rules["verbs"] = ["*"]
        sa_rules_list.append(sa_rules)
        sa_clusterrole["rules"] = sa_rules_list
        sa_clusterrole_json = json.dumps(sa_clusterrole)
        fp.write(sa_clusterrole_json)

        fp = open(temp_file_location + "/sa-clusterrolebinding.json", "w")
        sa_clusterrole_binding = {}
        sa_clusterrole_binding["apiVersion"] = "rbac.authorization.k8s.io/v1"
        sa_clusterrole_binding["kind"] = "ClusterRoleBinding"
        sa_clusterrole_binding["metadata"] = sa_metadata
        sa_clusterrole_binding["subjects"] = subjects_list
        clusterrole_ref = {}
        clusterrole_ref["kind"] = "ClusterRole"
        clusterrole_ref["name"] = saName
        clusterrole_ref["apiGroup"] = "rbac.authorization.k8s.io"
        sa_clusterrole_binding["roleRef"] = clusterrole_ref
        sa_clusterrole_binding_json = json.dumps(sa_clusterrole_binding)
        fp.write(sa_clusterrole_binding_json)
        fp.close()

    def _wait_for_oidc_app(self, region, namespace):
        self.logger.info("Inside checking _wait_for_oidc_app...")

        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        env_string = self._read_remote_openrc(remote_region_oam_ip, self.logger)
        basecmd = env_string

        status_applied = False
        while not status_applied:
            cmds = []
            cmd = ssh_prefix + " " + basecmd + " system application-show oidc-auth-apps"
            self.logger.info("cmd:" + cmd)
            cmds.append(cmd)
            output_lines = self._run_command_get_all_lines(cmds, host_password, self.logger)
            self.logger.info("Returned output lines:")
            self.logger.info(output_lines)
            if len(output_lines) > 0:
                for line in output_lines.split("\n"):
                    self.logger.info("Line:" + line)
                    if 'status' in line and ('applied' in line or 'apply-failed' in line):
                        self.logger.info("oidc-auth-apps applied: " + line)
                        if 'applied' in line:
                            status_applied = True
                        if 'apply-failed' in line:
                            cmds1 = []
                            cmd1 = ssh_prefix + " " + basecmd + " system application-remove oidc-auth-apps"
                            self.logger.info("cmd:" + cmd1)
                            cmd2 = ssh_prefix + " " + basecmd + " system application-apply oidc-auth-apps"
                            self.logger.info("cmd:" + cmd2)
                            cmds1.append(cmd1)
                            cmds1.append(cmd2)
                            output_lines1 = self._run_command_get_all_lines(cmds1 , host_password, self.logger)
                            self.logger.info("Returned output lines:")
                            self.logger.info(output_lines1)
            time.sleep(3)

    def _create_kubeconfig(self, kubeconfig_for, region, namespace, kubeconfig_approach, transaction_id):
        self.logger.info("Kubeconfig for:" + kubeconfig_for)
        kubeconfig_src = kubeconfig_approach
        if kubeconfig_for == KubeconfigGenerator.ORCHESTRATION_TEAM:
            if kubeconfig_src == 'SA':
                self._create_kubeconfig_orchestration_sa(region, namespace, transaction_id)
            if kubeconfig_src == 'LDAP':
                #self._wait_for_oidc_app(region, namespace)
                self._create_kubeconfig_orchestration_ldap(region, namespace, transaction_id)
        elif kubeconfig_for == KubeconfigGenerator.APPLICATION_TEAM:
            if kubeconfig_src == 'SA':
                self._create_kubeconfig_application_sa(region, namespace, transaction_id)
            if kubeconfig_src == 'LDAP':
                self._create_kubeconfig_application_ldap(region, namespace, transaction_id)
            self.vmbCoordinationQueue.put("Done")
        elif kubeconfig_for == KubeconfigGenerator.OPS_TEAM:
            self._create_kubeconfig_ops(region, namespace, transaction_id)
        elif kubeconfig_for == KubeconfigGenerator.VENDOR_TEAM_SAMSUNG:
            self._create_kubeconfig_samsung(region, namespace, transaction_id)
            self.vmbCoordinationQueue.put("Done")

    def _create_kubeconfig_application_sa(self, region, namespace, transaction_id):
        self.logger.info("Inside ..kubeconfig application")
        saName = self._get_serviceaccount_name(KubeconfigGenerator.APPLICATION_TEAM, transaction_id)
        self.logger.info("Service Account name:" + saName)

        message = 'Starting kubeconfig creation. '
        status = 'STARTING'
        self.logger.info(message + ' ' + status)
        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')

        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        app_sa_namespace = 'default' # default ns is better as it already exists 

        sa_files_dir = []
        folder = 'cluster-setup-files'
        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
  
        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)

        message = 'Applying RBAC to Service Account'
        status = 'CREATING'
        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
        self.logger.info(" Service Account File Path:" + str(saFilePath))

        self._apply_application_rbac_policies_sa(region, remote_region_oam_ip, app_sa_namespace, namespace, saName, saFilePath, self.logger)
        
        cmds_token_name = []
        cmds_token_name.append(ssh_prefix + " kubectl describe serviceaccount --kubeconfig=/etc/kubernetes/admin.conf -n " + app_sa_namespace + " " + saName  + "| grep Tokens ")
        all_lines = self._run_commands(cmds_token_name, host_password, self.logger, block=True)
        secretname = self._parse_token_name(all_lines)

        message = 'Parsing token'
        status = 'CREATING'
        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
        self.logger.info(" Secret name:" + secretname)
        if secretname != None:
            cmds1 = []
            cmds1.append(ssh_prefix + " kubectl describe secret --kubeconfig=/etc/kubernetes/admin.conf -n " + app_sa_namespace + " " + secretname + " | grep token:")
            all_lines = self._run_commands(cmds1, host_password, self.logger, block=True)
            token = self._parse_token(all_lines)
            #self.logger.info("TOKEN TO USE:" + token)
            token = token.strip()
            self.logger.info(" Token:[" + str(token) + "]")

            # Generate kubeconfig
            kubeconfig_value = self._generate_kubeconfig(region, namespace, saName, token, transaction_id, self.logger)

            # Update database
            message = 'kubeconfig creation done. '
            status = 'COMPLETE'
            self.logger.info(message + ' ' + status)
            self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)

    def _create_kubeconfig_application_ldap(self, region, namespace, transaction_id):
        self.logger.info("Inside kubeconfig application ldap...")
        message = 'Starting kubeconfig creation. '
        status = 'STARTING'
        self.logger.info(message + ' ' + status)
        app_user = self._get_user_name(KubeconfigGenerator.APPLICATION_TEAM)
        self.logger.info("Application User:" + app_user)
        self._update_status_async(region, namespace, app_user, message, status, transaction_id, self.logger, kubeconfig='')

        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        message = 'Applying RBAC to User Account'
        status = 'CREATING'
        self._update_status_async(region, namespace, app_user, message, status, transaction_id, self.logger, kubeconfig='')

        self.logger.info("Applying RBAC policies to the Application User...")
        self._apply_application_rbac_policies_ldap(region, remote_region_oam_ip, namespace, self.logger)
 
        ldap_account = 'edge_eng'
        token = self._get_ldap_token(host_password, ssh_prefix, app_user, ldap_account, remote_region_oam_ip)

        # Generate kubeconfig
        kubeconfig_value = self._generate_kubeconfig(region, namespace, app_user, token, transaction_id, self.logger)

        # Update database
        message = 'kubeconfig creation done. '
        status = 'COMPLETE'
        self.logger.info(message + ' ' + status)
        self._update_status_async(region, namespace, app_user, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)

        #mv /home/sysadmin/.kube/config /home/sysadmin/.kube/config-for-orchestration
        self.logger.info("Moving /home/sysadmin/.kube/config to /home/sysadmin/.kube/config-for-edge-eng...")
        set_kubeconfig_cmd = []
        set_kubeconfig_cmd.append(ssh_prefix + ' mv /home/sysadmin/.kube/config /home/sysadmin/.kube/config-for-edge-eng')
        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)

        #export KUBECONFIG=/etc/kubernetes/admin.conf
        self.logger.info("Resetting KUBECONFIG...")
        set_kubeconfig_cmd = []
        set_kubeconfig_cmd.append(ssh_prefix + ' export KUBECONFIG=/etc/kubernetes/admin.conf ')
        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)
        return kubeconfig_value

    def _create_kubeconfig_samsung(self, region, namespace, transaction_id):
        self.logger.info("Inside ..kubeconfig samsung")
        saName = self._get_serviceaccount_name(KubeconfigGenerator.VENDOR_TEAM_SAMSUNG, transaction_id)
        self.logger.info("Service Account name:" + saName)

        message = 'Starting kubeconfig creation. '
        status = 'STARTING'
        self.logger.info(message + ' ' + status)
        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')

        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        app_sa_namespace = 'default' # default ns is better as it already exists 

        sa_files_dir = []
        folder = 'cluster-setup-files'
        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
  
        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)

        message = 'Applying RBAC to Service Account'
        status = 'CREATING'
        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
        self.logger.info(" Service Account File Path:" + str(saFilePath))

        self._apply_vendor_samsung_rbac_policies(region, remote_region_oam_ip, app_sa_namespace, namespace, saName, saFilePath, self.logger)
        
        cmds_token_name = []
        cmds_token_name.append(ssh_prefix + " kubectl describe serviceaccount --kubeconfig=/etc/kubernetes/admin.conf -n " + app_sa_namespace + " " + saName  + "| grep Tokens ")
        all_lines = self._run_commands(cmds_token_name, host_password, self.logger, block=True)
        secretname = self._parse_token_name(all_lines)

        message = 'Parsing token'
        status = 'CREATING'
        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
        self.logger.info(" Secret name:" + secretname)
        if secretname != None:
            cmds1 = []
            cmds1.append(ssh_prefix + " kubectl describe secret --kubeconfig=/etc/kubernetes/admin.conf -n " + app_sa_namespace + " " + secretname + " | grep token:")
            all_lines = self._run_commands(cmds1, host_password, self.logger, block=True)
            token = self._parse_token(all_lines)
            #self.logger.info("TOKEN TO USE:" + token)
            token = token.strip()
            self.logger.info(" Token:[" + str(token) + "]")

            # Generate kubeconfig
            kubeconfig_value = self._generate_kubeconfig(region, namespace, saName, token, transaction_id, self.logger)

            # Update database
            message = 'kubeconfig creation done. '
            status = 'COMPLETE'
            self.logger.info(message + ' ' + status)
            self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)


    def _create_kubeconfig_ops(self, region, namespace, transaction_id):
        pass

    def _create_kubeconfig_orchestration_ldap(self, region, namespace, transaction_id):
        self.logger.info("Inside kubeconfig orchestration ldap...")
        message = 'Starting kubeconfig creation. '
        status = 'STARTING'
        self.logger.info(message + ' ' + status)
        orch_user = self._get_user_name(KubeconfigGenerator.ORCHESTRATION_TEAM)
        self.logger.info("Orchestration User:" + orch_user)
        self._update_status_async(region, namespace, orch_user, message, status, transaction_id, self.logger, kubeconfig='')

        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)

        message = 'Applying RBAC to User Account'
        status = 'CREATING'
        self._update_status_async(region, namespace, orch_user, message, status, transaction_id, self.logger, kubeconfig='')

        self.logger.info("Applying RBAC policies to the Orchestration User...")
        self._apply_orchestration_rbac_policies_ldap(region, remote_region_oam_ip, namespace, self.logger)
 
        ldap_account = 'orchestration'
        token = self._get_ldap_token(host_password, ssh_prefix, orch_user, ldap_account, remote_region_oam_ip)

        # Generate kubeconfig
        kubeconfig_value = self._generate_kubeconfig(region, namespace, orch_user, token, transaction_id, self.logger)

        # Update database
        message = 'kubeconfig creation done. '
        status = 'COMPLETE'
        self.logger.info(message + ' ' + status)
        self._update_status_async(region, namespace, orch_user, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)

        #mv /home/sysadmin/.kube/config /home/sysadmin/.kube/config-for-orchestration
        self.logger.info("Moving /home/sysadmin/.kube/config to /home/sysadmin/.kube/config-for-orchestration...")
        set_kubeconfig_cmd = []
        set_kubeconfig_cmd.append(ssh_prefix + ' mv /home/sysadmin/.kube/config /home/sysadmin/.kube/config-for-orchestration')
        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)

        #export KUBECONFIG=/etc/kubernetes/admin.conf
        self.logger.info("Resetting KUBECONFIG...")
        set_kubeconfig_cmd = []
        set_kubeconfig_cmd.append(ssh_prefix + ' export KUBECONFIG=/etc/kubernetes/admin.conf ')
        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)

        # Send notification on VMB
        self._send_vmb_notification(region, namespace, transaction_id, kubeconfig_value)
        return kubeconfig_value

    def _get_ldap_token(self, host_password, ssh_prefix, user, ldap_account, remote_region_oam_ip):
        #cp /etc/kubernetes/admin.conf /home/sysadmin/.kube/config
        self.logger.info("Starting token generation...")
        cp_cmd = []
        cp_cmd.append(ssh_prefix + ' cp /etc/kubernetes/admin.conf /home/sysadmin/.kube/config')
        self._run_commands(cp_cmd, host_password, self.logger, block=True)

        #export KUBECONFIG=/home/sysadmin/.kube/config
        self.logger.info("Setting KUBECONFIG...")
        set_kubeconfig_cmd = []
        set_kubeconfig_cmd.append(ssh_prefix + ' export KUBECONFIG=/home/sysadmin/.kube/config ')
        self._run_commands(set_kubeconfig_cmd, host_password, self.logger, block=True)

        #kubectl config set-context --kubeconfig=/home/sysadmin/.kube/config SVC-Edge-Eng@kubernetes --cluster=kubernetes --user=SVC-Edge-Eng
        self.logger.info("Performing set-context...")
        set_context = ' kubectl config set-context --kubeconfig=/home/sysadmin/.kube/config ' + user + '@kubernetes --cluster=kubernetes --user=' + user
        self.logger.info("Set context cmd:" + set_context)
        set_context_cmd = []
        set_context_cmd.append(ssh_prefix + set_context)
        self._run_commands(set_context_cmd, host_password, self.logger, block=True)

        #oidc-auth -c <OAM-IP> -u SVC-Edge-Eng -p 322C6v22acuhAGdyce22S3w282
        self.logger.info("Executing oidc-auth...")
        orch_password = settings.LDAP_USER_CREDS[ldap_account]
        oidc_auth = ' oidc-auth -c ' + remote_region_oam_ip + ' -u ' + user + ' -p ' + orch_password
        self.logger.info("OIDC Auth..:" + oidc_auth)
        oidc_auth_cmd = []
        oidc_auth_cmd.append(ssh_prefix + oidc_auth)
        self._run_commands(oidc_auth_cmd, host_password, self.logger, block=True)

        #grep token /home/sysadmin/.kube/config
        self.logger.info("Retrieving token...")
        cmds1 = []
        cmds1.append(ssh_prefix + " grep token /home/sysadmin/.kube/config ")
        all_lines = self._run_commands(cmds1, host_password, self.logger, block=True)
        token = self._parse_token(all_lines)
        #self.logger.info("TOKEN TO USE:" + token)
        token = token.strip()
        self.logger.info(" Token:[" + str(token) + "]")
        return token

    def _create_kubeconfig_orchestration_sa(self, region, namespace, transaction_id):
        # For orchestration SA, we use either the 'orchestration' namespace or the 'default' namespace
        self.logger.info("Inside ..kubeconfig orchestration sa")
        saName = self._get_serviceaccount_name(KubeconfigGenerator.ORCHESTRATION_TEAM, transaction_id)
        self.logger.info("Service Account name:" + saName)

        message = 'Starting kubeconfig creation. '
        status = 'STARTING'
        self.logger.info(message + ' ' + status)
        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')

        remote_region_oam_ip = self._get_remote_region_oam_ip(region, self.logger)
        orch_sa_namespace = 'default' # default ns is better as it already exists 

        host_username, host_password, ssh_prefix = get_host_connection_details(remote_region_oam_ip)
        
        #cmds_sa = []
        #cmds_sa.append(ssh_prefix + ' kubectl create serviceaccount --kubeconfig=/etc/kubernetes/admin.conf ' + saName + ' -n ' + orch_sa_namespace)
        #self._run_commands(cmds_sa, host_password, self.logger, block=True)

        sa_files_dir = []
        folder = 'cluster-setup-files'
        sa_files_dir.append(ssh_prefix + ' mkdir -p /home/sysadmin/' + folder)
        self._run_commands(sa_files_dir, host_password, self.logger, block=True)
  
        saFilePath = os.path.join(os.path.dirname(__file__), "./" + folder)

        message = 'Applying RBAC to Service Account'
        status = 'CREATING'
        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
        self.logger.info(" Service Account File Path:" + str(saFilePath))

        self._apply_orchestration_rbac_policies_sa(region, remote_region_oam_ip, orch_sa_namespace, namespace, saName, saFilePath, self.logger)
        
        cmds_token_name = []
        secret_name_found = False
        cmds_token_name.append(ssh_prefix + " kubectl describe serviceaccount --kubeconfig=/etc/kubernetes/admin.conf -n " + orch_sa_namespace + " " + saName  + "| grep Tokens ")
        while not secret_name_found:
            all_lines = self._run_commands(cmds_token_name, host_password, self.logger, block=True)
            # Check if secretname != <none>; if so, repeat the command
            ## kubeconfighandler.py _create_kubeconfig_orchestration 351  Secret name:<none>
            secretname = self._parse_token_name(all_lines)
            if secretname != "<none>":
                secret_name_found = True
            else:
                time.sleep(60)

        message = 'Parsing token'
        status = 'CREATING'
        self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig='')
        self.logger.info(" Secret name:" + secretname)
        if secretname != None:
            #self.logger.info("ABC")
            cmds1 = []
            cmds1.append(ssh_prefix + " kubectl describe secret --kubeconfig=/etc/kubernetes/admin.conf -n " + orch_sa_namespace + " " + secretname + " | grep token:")
            all_lines = self._run_commands(cmds1, host_password, self.logger, block=True)
            token = self._parse_token(all_lines)
            #self.logger.info("TOKEN TO USE:" + token)
            token = token.strip()
            self.logger.info(" Token:[" + str(token) + "]")

            # Generate kubeconfig
            kubeconfig_value = self._generate_kubeconfig(region, namespace, saName, token, transaction_id, self.logger)

            # Update database
            message = 'kubeconfig creation done. '
            status = 'COMPLETE'
            self.logger.info(message + ' ' + status)
            self._update_status_async(region, namespace, saName, message, status, transaction_id, self.logger, kubeconfig=kubeconfig_value, done=True)
            
            # Send notification on VMB
            self._send_vmb_notification(region, namespace, transaction_id, kubeconfig_value)
            return kubeconfig_value

    def _send_vmb_notification(self, region, namespace, transaction_id, kubeconfig_value):
            created_at_time = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")

            site_name, site_location = self.get_fuze_spm_site_details(region, self.logger)
            self.logger.info("Site name:" + site_name)
            self.logger.info("Site location:" + site_location)

            # Send VMB Notification
            kubeconfig = Kubeconfig(
                cluster=region,
                namespace=namespace,
                location=site_location,
                kubeconfig='Example kubeconfig',
                created_at=str(created_at_time),
                updated_at=str(created_at_time),
                transactionId=str(transaction_id)
            )

            kubeconfig.kubeconfig = kubeconfig_value

            reportDescription = 'kubeconfig file: ' + site_name
            kubeconfig_message = KubeconfigMessage(
                reportName='vcp_fe_kubeconfig',
                reportDescription=reportDescription,
                reportGeneratedOn=str(created_at_time),
                rowCount = 1,
                reportDataRows=[kubeconfig],
            )

            item = {}
            item['payload'] = kubeconfig_message
            item['message'] = 'Kubeconfig'
            self.logger.info("Sending to VMB")
            self.logger.info(kubeconfig_message)
            self.logger.info(kubeconfig)
            self.vmbQueue.put(item)

    def _generate_kubeconfig(self, region, namespace, saName, token, transaction_id, logger):
        logger.info(" Inside _generate_kubeconfig")

        user_list = []
        tokendata = {}
        tokendata['token'] = token
        userdata = {}
        userdata['name'] = saName
        userdata['user'] = tokendata
        user_list.append(userdata)
        
        context_list = []
        contextdatawrapper = {}
        contextdata = {}
        contextdata['cluster'] = region
        contextdata['user'] = saName
        contextdata['namespace'] = namespace
        contextdatawrapper['context'] = contextdata
        contextdatawrapper['name'] = region
        context_list.append(contextdatawrapper)

        cluster_list = []
        clusterdata = {}
        clusterdata['name'] = region
        cluster_detail = {}
        cluster_detail['insecure-skip-tls-verify'] = True 
        remote_region_oam_ip = self._get_remote_region_oam_ip(region, logger)
        cluster_detail['server'] = 'https://[' + remote_region_oam_ip + ']:6443'
        clusterdata["cluster"] = cluster_detail
        cluster_list.append(clusterdata)
        
        outer_dict = {}
        outer_dict['apiVersion'] = 'v1'
        outer_dict['kind'] = 'Config'
        outer_dict['current-context'] = region
        outer_dict['users'] = user_list
        outer_dict['contexts'] = context_list
        outer_dict['clusters'] = cluster_list

        kubeconfig_json = json.dumps(outer_dict)
        logger.info(str(transaction_id) + " Kubeconfig:" + kubeconfig_json)
        return kubeconfig_json

    def _parse_token_name(self, all_lines):
        for line in all_lines.split("\n"):
            if 'Tokens' in line:
                parts = line.split(":")
                tokenName = parts[1].rstrip().lstrip()
                return tokenName

    def _parse_token(self, all_lines):
        for line in all_lines.split("\n"):
            if 'token:' in line:
                parts = line.split(":")
                token = parts[1].rstrip().lstrip()
                return token

    def _run_commands_scp(self, commands, host_password, logger, block=False):
        logger.info("Inside _run_commands_scp..:")
        status = run_commands_scp(commands, host_password, logger, block=False)
        return status

    def _run_commands(self, commands, host_password, logger, block=False):
        #logger.info(commands)
        all_lines1 = []
        for command in commands:
            logger.info(" Executing.." + str(command))
            child = pexpect.spawn(command)
            try:
                if block:
                    child.timeout=None
                    child.expect(['password: '], timeout=None)
                    child.sendline(host_password)
                    all_lines1 = child.read()
                    all_lines1 = all_lines1.rstrip().lstrip()
                    all_lines1 = all_lines1.decode('utf-8').replace('\r\n', '\n')
                    logger.info(all_lines1)
                    return all_lines1
                else:
                    child.expect(['(yes/no)? '])
                    child.sendline('yes')
                    child.expect(['password: '])
                    child.sendline(host_password)
                    #child.interact()
                    #child.close()
                    child.expect(pexpect.EOF, timeout=5)
            except:
                pass
        return all_lines1

    def _update_status_async(self, region, namespace, saName, message, status, transaction_id, logger, kubeconfig='', done=False):
        item = {}
        item['type'] = 'kubeconfig'
        item['region'] = region
        item['namespace'] = namespace
        item['saName'] = saName
        item['message'] = message
        item['status'] = status
        item['transaction_id'] = transaction_id
        item['kubeconfig'] = kubeconfig
        item['done'] = done
        self.dbQueue.put(item)

    def _update_status(self, region, namespace, saName, message, status, transaction_id, logger, kubeconfig=''):
        RemoteRegionSetup.objects.filter(transaction_id=transaction_id).update(message=message,
                                                                               status=status,
                                                                               serviceaccount=saName,
                                                                               kubernetes_namespace=namespace,
                                                                               remote_region_name=region,
                                                                               kubeconfig=kubeconfig)
        return