more filesHEADmaster

1 files changed, 193 insertions, 0 deletions

diff --git a/src/caas/services/ciqservice.py b/src/caas/services/ciqservice.py
new file mode 100644
index 0000000..a97e9e0
--- /dev/null
+++ b/src/caas/services/ciqservice.py
@@ -0,0 +1,193 @@
+import csv
+import ipaddress
+from io import TextIOWrapper
+from django.db import connections
+from django.conf import settings
+from .helpers import Address, RawQuery
+from ..models import Vlan, Namespace, CellSiteRouter, SapClli, Location, Cluster, Server
+
+
+"""
+Derive IPs from rules defined in the LLD:
+https://oneconfluence.verizon.com/display/NTD/VRAN+2.0+Far+Edge?preview=/554294908/592448982/vRAN%202.0%20on%20VCP%20Far%20Edge_WebScale%20Transport%20LLD%20v2.3.pdf
+"""
+class CiqService():
+    @staticmethod
+    def import_ciq_data(filehandle, assign_parent = False):
+        with connections["default"].cursor() as cursor:
+            f = TextIOWrapper(filehandle, encoding = "ascii", errors = "replace")
+            csv_reader = csv.reader(f, delimiter = ",")
+            for row in csv_reader:
+                # Fetch data from CSV row
+                fuze_spm_site_id = row[1]
+                fuze_spm_site_name = row[3]
+                fuze_spm_market = row[4]
+                fuze_spm_submarket = row[5]
+                fuze_spm_site_type = row[6]
+                central_controller_sap_clli = row[7]
+                server_number = int(row[9])
+                ilo_hostname = row[10].lower()
+                cluster_name = row[11].lower()
+                namespace_name = row[12].lower()
+                host_vlan = int(row[13])
+                oam_vlan = int(row[14])
+                mgmt_vlan = int(row[15])
+                csr_hostname = row[17]
+                ilo_host_address = ipaddress.IPv6Address(row[23])
+                ilo_default_gateway = ipaddress.IPv6Address(row[24])
+                mgmt_default_gateway = ipaddress.IPv6Address(row[25])
+                oam_default_gateway = ipaddress.IPv6Address(row[26])
+                central_controller_p = bool(row[28])
+
+                # Use the minimum uniquely identifying data to load existing matching records if available
+
+                # location
+                location_kwargs = {"fuze_spm_site_id": fuze_spm_site_id}
+                location, location_created = Location.objects.get_or_create(**location_kwargs)
+                location.fuze_spm_site_name = fuze_spm_site_name
+                location.fuze_spm_market = fuze_spm_market
+                location.fuze_spm_submarket = fuze_spm_submarket
+                location.fuze_spm_site_type = fuze_spm_site_type
+
+                # vlan
+                vlan_kwargs = {"host_vlan": host_vlan,
+                               "oam_vlan": oam_vlan,
+                               "mgmt_vlan": mgmt_vlan}
+                vlan, vlan_created = Vlan.objects.get_or_create(**vlan_kwargs)
+
+                # csr
+                csr_kwargs = {"csr_hostname": csr_hostname}
+                csr, csr_created = CellSiteRouter.objects.get_or_create(**csr_kwargs)
+
+                # sap_clli
+                sap_clli_kwargs = {"clli": central_controller_sap_clli}
+                sap_clli, sap_clli_created = SapClli.objects.get_or_create(**sap_clli_kwargs)
+
+                # cluster
+                cluster_kwargs = {"cluster_name": cluster_name}
+                cluster, cluster_created = Cluster.objects.get_or_create(**cluster_kwargs)
+
+                # namespaces
+                namespace_kwargs = {"namespace_name": namespace_name}
+                namespace, namespace_created = Namespace.objects.get_or_create(**namespace_kwargs)
+
+                # server
+                server_kwargs = {"ilo_hostname": ilo_hostname}
+                server, server_created = Server.objects.get_or_create(**server_kwargs)
+
+                parent_cluster = None
+
+                if not central_controller_p:
+                    # find least-used parent central controller cluster
+                    parent_clusters_kwargs = {"sap_clli_id__exact": sap_clli.pk,
+                                              "parent_cluster_id__isnull": True,
+                                              "is_central_controller": 1}
+                    parent_clusters = Cluster.objects.filter(**parent_clusters_kwargs)
+                    subcloud_counts = {}
+
+                    for parent in parent_clusters:
+                        num_children = RawQuery.query_single_value(cursor, """
+select count(*) as num_children
+from caas_cluster
+where parent_cluster_id = %s and is_central_controller = 0""" % parent.pk, "num_children")
+
+                        if num_children < settings.WR_MAX_CHILD_CLUSTERS:
+                            subcloud_counts[parent.pk] = [num_children, parent]
+
+                    if len(subcloud_counts) != 0:
+                        parent_cluster = min(subcloud_counts.items(), key=lambda x: x[1][0])[1][1]
+
+                    # Derive subnets from gateways
+                    oam_subnet = Address.add_40a(Address.gateway_to_subnet(oam_default_gateway))
+                    mgmt_subnet = Address.add_40a(Address.gateway_to_subnet(mgmt_default_gateway))
+                    ilo_subnet = Address.add_40a(Address.gateway_to_subnet(ilo_default_gateway))
+
+                    # oam vip
+                    need_ip = False
+
+                    if csr.oam_subnet == str(oam_subnet) and csr.oam_default_gateway == str(oam_default_gateway):
+                        if cluster.oam_vip_address is None:
+                            need_ip = True
+                    else:
+                        csr.oam_subnet = str(oam_subnet)
+                        csr.oam_default_gateway = str(oam_default_gateway)
+                        need_ip = True
+                    if need_ip:
+                        oam_vip_address_max = RawQuery.query_single_value(cursor, """
+with oam_ips as (
+    select csr.id as csr_id,
+        c.oam_vip_address
+    from caas_server s
+    inner join caas_cluster c
+        on s.cluster_id = c.id
+    inner join caas_cellsiterouter csr
+        on c.csr_id = csr.id
+)
+select max(oam_vip_address) as max
+from oam_ips
+where csr_id = %s""" % csr.id, "max")
+                        if oam_vip_address_max:
+                            oam_vip_address_max = str(Address.next_address(oam_vip_address_max, oam_subnet, 0x10))
+                        else:
+                            oam_vip_address_max = str(Address.next_address(oam_vip_address_max, oam_subnet, 0xf400))
+                        cluster.oam_vip_address = oam_vip_address_max
+                    # oam host
+                    need_ip = False
+                    if csr.oam_subnet == str(oam_subnet) and csr.oam_default_gateway == str(oam_default_gateway):
+                        if server.oam_host_address is None:
+                            need_ip = True
+                    else:
+                        csr.oam_subnet = str(oam_subnet)
+                        csr.oam_default_gateway = str(oam_default_gateway)
+                        need_ip = True
+                    if need_ip:
+                        oam_host_address_max = RawQuery.query_single_value(cursor, """
+with oam_ips as (
+    select csr.id as csr_id,
+        s.oam_host_address
+    from caas_server s
+    inner join caas_cluster c
+        on s.cluster_id = c.id
+    inner join caas_cellsiterouter csr
+        on c.csr_id = csr.id
+)
+select max(oam_host_address) as max
+from oam_ips
+where csr_id = %s""" % csr.id, "max")
+                        if oam_host_address_max:
+                            oam_host_address_max = str(Address.next_address(oam_host_address_max, oam_subnet, 0x1))
+                        else:
+                            oam_host_address_max = str(Address.next_address(oam_host_address_max, oam_subnet, 0x400))
+                        server.oam_host_address = oam_host_address_max
+
+                    # mgmt
+                    cluster.mgmt_subnet = str(mgmt_subnet)
+                    cluster.mgmt_default_gateway = str(mgmt_default_gateway)
+                    cluster.mgmt_address_range_start = str(mgmt_subnet)
+                    cluster.mgmt_address_range_end = str(Address.address_add(cluster.mgmt_address_range_start, 0xf))
+
+                    # don't step on central controllers TODO: separate central and remote CIQ ingestion
+                    csr.ilo_subnet = str(ilo_subnet)
+                    csr.ilo_default_gateway = str(ilo_default_gateway)
+                    server.server_number = server_number
+                    server.ilo_host_address = str(ilo_host_address)
+                else:
+                    cluster.is_central_controller = 1
+
+                # foreign-key relationships
+                cluster.sap_clli = sap_clli
+                if assign_parent is True and cluster.parent_cluster is None:
+                    cluster.parent_cluster = parent_cluster
+                cluster.location = location
+                cluster.vlan = vlan
+                cluster.csr = csr
+                cluster.namespace = namespace
+                server.cluster = cluster
+                # upsert
+                vlan.save()
+                namespace.save()
+                csr.save()
+                sap_clli.save()
+                location.save()
+                cluster.save()
+                server.save()