import csv import ipaddress from io import TextIOWrapper from django.db import connections from django.conf import settings from .helpers import Address, RawQuery from ..models import Vlan, Namespace, CellSiteRouter, SapClli, Location, Cluster, Server """ Derive IPs from rules defined in the LLD: https://oneconfluence.verizon.com/display/NTD/VRAN+2.0+Far+Edge?preview=/554294908/592448982/vRAN%202.0%20on%20VCP%20Far%20Edge_WebScale%20Transport%20LLD%20v2.3.pdf """ class CiqService(): @staticmethod def import_ciq_data(filehandle, assign_parent = False): with connections["default"].cursor() as cursor: f = TextIOWrapper(filehandle, encoding = "ascii", errors = "replace") csv_reader = csv.reader(f, delimiter = ",") for row in csv_reader: # Fetch data from CSV row fuze_spm_site_id = row[1] fuze_spm_site_name = row[3] fuze_spm_market = row[4] fuze_spm_submarket = row[5] fuze_spm_site_type = row[6] central_controller_sap_clli = row[7] server_number = int(row[9]) ilo_hostname = row[10].lower() cluster_name = row[11].lower() namespace_name = row[12].lower() host_vlan = int(row[13]) oam_vlan = int(row[14]) mgmt_vlan = int(row[15]) csr_hostname = row[17] ilo_host_address = ipaddress.IPv6Address(row[23]) ilo_default_gateway = ipaddress.IPv6Address(row[24]) mgmt_default_gateway = ipaddress.IPv6Address(row[25]) oam_default_gateway = ipaddress.IPv6Address(row[26]) central_controller_p = bool(row[28]) # Use the minimum uniquely identifying data to load existing matching records if available # location location_kwargs = {"fuze_spm_site_id": fuze_spm_site_id} location, location_created = Location.objects.get_or_create(**location_kwargs) location.fuze_spm_site_name = fuze_spm_site_name location.fuze_spm_market = fuze_spm_market location.fuze_spm_submarket = fuze_spm_submarket location.fuze_spm_site_type = fuze_spm_site_type # vlan vlan_kwargs = {"host_vlan": host_vlan, "oam_vlan": oam_vlan, "mgmt_vlan": mgmt_vlan} vlan, vlan_created = Vlan.objects.get_or_create(**vlan_kwargs) # csr csr_kwargs = {"csr_hostname": csr_hostname} csr, csr_created = CellSiteRouter.objects.get_or_create(**csr_kwargs) # sap_clli sap_clli_kwargs = {"clli": central_controller_sap_clli} sap_clli, sap_clli_created = SapClli.objects.get_or_create(**sap_clli_kwargs) # cluster cluster_kwargs = {"cluster_name": cluster_name} cluster, cluster_created = Cluster.objects.get_or_create(**cluster_kwargs) # namespaces namespace_kwargs = {"namespace_name": namespace_name} namespace, namespace_created = Namespace.objects.get_or_create(**namespace_kwargs) # server server_kwargs = {"ilo_hostname": ilo_hostname} server, server_created = Server.objects.get_or_create(**server_kwargs) parent_cluster = None if not central_controller_p: # find least-used parent central controller cluster parent_clusters_kwargs = {"sap_clli_id__exact": sap_clli.pk, "parent_cluster_id__isnull": True, "is_central_controller": 1} parent_clusters = Cluster.objects.filter(**parent_clusters_kwargs) subcloud_counts = {} for parent in parent_clusters: num_children = RawQuery.query_single_value(cursor, """ select count(*) as num_children from caas_cluster where parent_cluster_id = %s and is_central_controller = 0""" % parent.pk, "num_children") if num_children < settings.WR_MAX_CHILD_CLUSTERS: subcloud_counts[parent.pk] = [num_children, parent] if len(subcloud_counts) != 0: parent_cluster = min(subcloud_counts.items(), key=lambda x: x[1][0])[1][1] # Derive subnets from gateways oam_subnet = Address.add_40a(Address.gateway_to_subnet(oam_default_gateway)) mgmt_subnet = Address.add_40a(Address.gateway_to_subnet(mgmt_default_gateway)) ilo_subnet = Address.add_40a(Address.gateway_to_subnet(ilo_default_gateway)) # oam vip need_ip = False if csr.oam_subnet == str(oam_subnet) and csr.oam_default_gateway == str(oam_default_gateway): if cluster.oam_vip_address is None: need_ip = True else: csr.oam_subnet = str(oam_subnet) csr.oam_default_gateway = str(oam_default_gateway) need_ip = True if need_ip: oam_vip_address_max = RawQuery.query_single_value(cursor, """ with oam_ips as ( select csr.id as csr_id, c.oam_vip_address from caas_server s inner join caas_cluster c on s.cluster_id = c.id inner join caas_cellsiterouter csr on c.csr_id = csr.id ) select max(oam_vip_address) as max from oam_ips where csr_id = %s""" % csr.id, "max") if oam_vip_address_max: oam_vip_address_max = str(Address.next_address(oam_vip_address_max, oam_subnet, 0x10)) else: oam_vip_address_max = str(Address.next_address(oam_vip_address_max, oam_subnet, 0xf400)) cluster.oam_vip_address = oam_vip_address_max # oam host need_ip = False if csr.oam_subnet == str(oam_subnet) and csr.oam_default_gateway == str(oam_default_gateway): if server.oam_host_address is None: need_ip = True else: csr.oam_subnet = str(oam_subnet) csr.oam_default_gateway = str(oam_default_gateway) need_ip = True if need_ip: oam_host_address_max = RawQuery.query_single_value(cursor, """ with oam_ips as ( select csr.id as csr_id, s.oam_host_address from caas_server s inner join caas_cluster c on s.cluster_id = c.id inner join caas_cellsiterouter csr on c.csr_id = csr.id ) select max(oam_host_address) as max from oam_ips where csr_id = %s""" % csr.id, "max") if oam_host_address_max: oam_host_address_max = str(Address.next_address(oam_host_address_max, oam_subnet, 0x1)) else: oam_host_address_max = str(Address.next_address(oam_host_address_max, oam_subnet, 0x400)) server.oam_host_address = oam_host_address_max # mgmt cluster.mgmt_subnet = str(mgmt_subnet) cluster.mgmt_default_gateway = str(mgmt_default_gateway) cluster.mgmt_address_range_start = str(mgmt_subnet) cluster.mgmt_address_range_end = str(Address.address_add(cluster.mgmt_address_range_start, 0xf)) # don't step on central controllers TODO: separate central and remote CIQ ingestion csr.ilo_subnet = str(ilo_subnet) csr.ilo_default_gateway = str(ilo_default_gateway) server.server_number = server_number server.ilo_host_address = str(ilo_host_address) else: cluster.is_central_controller = 1 # foreign-key relationships cluster.sap_clli = sap_clli if assign_parent is True and cluster.parent_cluster is None: cluster.parent_cluster = parent_cluster cluster.location = location cluster.vlan = vlan cluster.csr = csr cluster.namespace = namespace server.cluster = cluster # upsert vlan.save() namespace.save() csr.save() sap_clli.save() location.save() cluster.save() server.save()