From 640ff61422bc0ee3966941b93d9889ddbbd38d77 Mon Sep 17 00:00:00 2001 From: ckonstanski Date: Thu, 30 Jul 2026 18:17:14 -0600 Subject: more files --- src/caas/services/ansibleservice.py | 52 ++++++ src/caas/services/bmcservice.py | 50 ++++++ src/caas/services/ciqservice.py | 193 ++++++++++++++++++++++ src/caas/services/dnsservice.py | 60 +++++++ src/caas/services/firmwareservice.py | 87 ++++++++++ src/caas/services/helpers.py | 136 +++++++++++++++ src/caas/services/httpclient.py | 18 ++ src/caas/services/icingapollservice.py | 47 ++++++ src/caas/services/icingaservice.py | 46 ++++++ src/caas/services/jinja/bmc_host_vars.j2 | 13 ++ src/caas/services/jinja/bmc_inventory.j2 | 6 + src/caas/services/jinja/dns_host_vars.j2 | 12 ++ src/caas/services/jinja/dns_inventory.j2 | 6 + src/caas/services/jinja/firmware_host_vars.j2 | 13 ++ src/caas/services/jinja/firmware_inventory.j2 | 7 + src/caas/services/jinja/icinga_host_vars.j2 | 38 +++++ src/caas/services/jinja/icinga_inventory.j2 | 6 + src/caas/services/jinja/icinga_poll_host_vars.j2 | 11 ++ src/caas/services/jinja/icinga_poll_inventory.j2 | 6 + src/caas/services/jinja/wr_host_vars.j2 | 74 +++++++++ src/caas/services/jinja/wr_inventory.j2 | 11 ++ src/caas/services/macaddressservice.py | 38 +++++ src/caas/services/nicfirmwareupgradenowservice.py | 68 ++++++++ src/caas/services/nicfirmwareversionservice.py | 37 +++++ src/caas/services/passwordservice.py | 180 ++++++++++++++++++++ src/caas/services/patchservice.py | 106 ++++++++++++ src/caas/services/playbookreportservice.py | 147 ++++++++++++++++ src/caas/services/subcloudservice.py | 12 ++ src/caas/services/wrinstallnowservice.py | 107 ++++++++++++ src/caas/services/wrservice.py | 75 +++++++++ 30 files changed, 1662 insertions(+) create mode 100644 src/caas/services/ansibleservice.py create mode 100644 src/caas/services/bmcservice.py create mode 100644 src/caas/services/ciqservice.py create mode 100644 src/caas/services/dnsservice.py create mode 100644 src/caas/services/firmwareservice.py create mode 100644 src/caas/services/helpers.py create mode 100644 src/caas/services/httpclient.py create mode 100644 src/caas/services/icingapollservice.py create mode 100644 src/caas/services/icingaservice.py create mode 100644 src/caas/services/jinja/bmc_host_vars.j2 create mode 100644 src/caas/services/jinja/bmc_inventory.j2 create mode 100644 src/caas/services/jinja/dns_host_vars.j2 create mode 100644 src/caas/services/jinja/dns_inventory.j2 create mode 100644 src/caas/services/jinja/firmware_host_vars.j2 create mode 100644 src/caas/services/jinja/firmware_inventory.j2 create mode 100644 src/caas/services/jinja/icinga_host_vars.j2 create mode 100644 src/caas/services/jinja/icinga_inventory.j2 create mode 100644 src/caas/services/jinja/icinga_poll_host_vars.j2 create mode 100644 src/caas/services/jinja/icinga_poll_inventory.j2 create mode 100644 src/caas/services/jinja/wr_host_vars.j2 create mode 100644 src/caas/services/jinja/wr_inventory.j2 create mode 100644 src/caas/services/macaddressservice.py create mode 100644 src/caas/services/nicfirmwareupgradenowservice.py create mode 100644 src/caas/services/nicfirmwareversionservice.py create mode 100644 src/caas/services/passwordservice.py create mode 100644 src/caas/services/patchservice.py create mode 100644 src/caas/services/playbookreportservice.py create mode 100644 src/caas/services/subcloudservice.py create mode 100644 src/caas/services/wrinstallnowservice.py create mode 100644 src/caas/services/wrservice.py (limited to 'src/caas/services') diff --git a/src/caas/services/ansibleservice.py b/src/caas/services/ansibleservice.py new file mode 100644 index 0000000..148bcf4 --- /dev/null +++ b/src/caas/services/ansibleservice.py @@ -0,0 +1,52 @@ +from django.conf import settings +import json +import zmq +import logging + + +logger = logging.getLogger("caas") + + +""" +Superclass for classes that manage ansible playbooks, for example +icinga and wr-installer. +""" +class AnsibleService(): + def __init__(self, ansible_queue, target, git, branch, playbook, zmq, maint_window_p = False): + self.ansible_queue = ansible_queue + self.target = target + self.git = git + self.branch = branch + self.playbook = playbook + self.zmq = zmq + self.maint_window_p = maint_window_p + + def run_playbook(self, inventory, inventory_content, hostvars, hostvars_content, extra_args = {}): + payload = [{"git": self.git, + "playbook": self.playbook, + "maintWindowP": self.maint_window_p, + "inventory": inventory, + "inventoryContent": inventory_content, + "hostvars": hostvars, + "hostvarsContent": hostvars_content}] + payload[0].update(extra_args) + payload_json = json.dumps(payload) + context = zmq.Context() + retries = 10 + while True: + socket = context.socket(zmq.REQ) + if settings.ZEROMQ_IPV6: + socket.setsockopt(zmq.IPV6, 1) + # logger.debug("sending playbook: %s" % payload_json) + socket.connect(self.zmq) + socket.send_string(payload_json) + logger.debug("sent playbook") + if (socket.poll(5000) & zmq.POLLIN) != 0: + return socket.recv() + retries -= 1 + socket.setsockopt(zmq.LINGER, 0) + socket.close() + if retries == 0: + errmsg = "Failed to send zmq message to ansible-queue: %s" % payload_json + logger.error(errmsg) + raise Exception(errmsg) diff --git a/src/caas/services/bmcservice.py b/src/caas/services/bmcservice.py new file mode 100644 index 0000000..43fdb51 --- /dev/null +++ b/src/caas/services/bmcservice.py @@ -0,0 +1,50 @@ +from .helpers import Jinja +from .ansibleservice import AnsibleService +from ..models import IcingaBmc +from django.conf import settings +import logging + + +logger = logging.getLogger("caas") + + +class BmcService(AnsibleService): + def __init__(self, target, git, branch, playbook, zmq): + super().__init__("bmc", target, git, branch, playbook, zmq) + + def deploy(self): + logger.debug("%s Enter" % self.target) + try: + bmc_kwargs = {"ilo_host_address": self.target} + bmc = IcingaBmc.objects.get(**bmc_kwargs) + except IcingaBmc.DoesNotExist: + errmsg = "%s FAILED: while running BMC playbook: failed to look up BMC" % self.target + logger.error(errmsg) + return -1 + # The ansible target is the hostname, but until now we only had the ilo IP. + self.target = bmc.ilo_hostname + # host_vars file + hostvars_config = {"bmc_ip": bmc.ilo_host_address, + "bmc_password_admin_default": bmc.bmc_password_admin_default, + "bmc_password_admin": bmc.bmc_password_admin, + "bmc_username": bmc.bmc_username, + "bmc_password": bmc.bmc_password, + "dns_nameservers": (" ").join(settings.DNS_SERVERS), + "syslog_server": settings.SYSLOG_SERVER, + "syslog_server_port": settings.SYSLOG_SERVER_PORT, + "ntp_servers": (" ").join(settings.NTP_SERVERS), + "middleware_endpoint": settings.CAAS_MIDDLEWARE_ENDPOINT, + "middleware_username": settings.CAAS_MIDDLEWARE_USERNAME, + "middleware_password": settings.CAAS_MIDDLEWARE_PASSWORD} + hostvars_content = Jinja.render("bmc_host_vars.j2", **hostvars_config) + hostvars = self.target + # inventory file + inventory_config = {"vendor_name": bmc.vendor_name, + "target": self.target} + inventory_content = Jinja.render("bmc_inventory.j2", **inventory_config) + inventory = "%s.yaml" % self.target + logger.debug("%s Sending playbook to queue" % self.target) + self.run_playbook(inventory, inventory_content, hostvars, hostvars_content) + logger.debug("%s Sent playbook to queue" % self.target) + return 0 + diff --git a/src/caas/services/ciqservice.py b/src/caas/services/ciqservice.py new file mode 100644 index 0000000..a97e9e0 --- /dev/null +++ b/src/caas/services/ciqservice.py @@ -0,0 +1,193 @@ +import csv +import ipaddress +from io import TextIOWrapper +from django.db import connections +from django.conf import settings +from .helpers import Address, RawQuery +from ..models import Vlan, Namespace, CellSiteRouter, SapClli, Location, Cluster, Server + + +""" +Derive IPs from rules defined in the LLD: +https://oneconfluence.verizon.com/display/NTD/VRAN+2.0+Far+Edge?preview=/554294908/592448982/vRAN%202.0%20on%20VCP%20Far%20Edge_WebScale%20Transport%20LLD%20v2.3.pdf +""" +class CiqService(): + @staticmethod + def import_ciq_data(filehandle, assign_parent = False): + with connections["default"].cursor() as cursor: + f = TextIOWrapper(filehandle, encoding = "ascii", errors = "replace") + csv_reader = csv.reader(f, delimiter = ",") + for row in csv_reader: + # Fetch data from CSV row + fuze_spm_site_id = row[1] + fuze_spm_site_name = row[3] + fuze_spm_market = row[4] + fuze_spm_submarket = row[5] + fuze_spm_site_type = row[6] + central_controller_sap_clli = row[7] + server_number = int(row[9]) + ilo_hostname = row[10].lower() + cluster_name = row[11].lower() + namespace_name = row[12].lower() + host_vlan = int(row[13]) + oam_vlan = int(row[14]) + mgmt_vlan = int(row[15]) + csr_hostname = row[17] + ilo_host_address = ipaddress.IPv6Address(row[23]) + ilo_default_gateway = ipaddress.IPv6Address(row[24]) + mgmt_default_gateway = ipaddress.IPv6Address(row[25]) + oam_default_gateway = ipaddress.IPv6Address(row[26]) + central_controller_p = bool(row[28]) + + # Use the minimum uniquely identifying data to load existing matching records if available + + # location + location_kwargs = {"fuze_spm_site_id": fuze_spm_site_id} + location, location_created = Location.objects.get_or_create(**location_kwargs) + location.fuze_spm_site_name = fuze_spm_site_name + location.fuze_spm_market = fuze_spm_market + location.fuze_spm_submarket = fuze_spm_submarket + location.fuze_spm_site_type = fuze_spm_site_type + + # vlan + vlan_kwargs = {"host_vlan": host_vlan, + "oam_vlan": oam_vlan, + "mgmt_vlan": mgmt_vlan} + vlan, vlan_created = Vlan.objects.get_or_create(**vlan_kwargs) + + # csr + csr_kwargs = {"csr_hostname": csr_hostname} + csr, csr_created = CellSiteRouter.objects.get_or_create(**csr_kwargs) + + # sap_clli + sap_clli_kwargs = {"clli": central_controller_sap_clli} + sap_clli, sap_clli_created = SapClli.objects.get_or_create(**sap_clli_kwargs) + + # cluster + cluster_kwargs = {"cluster_name": cluster_name} + cluster, cluster_created = Cluster.objects.get_or_create(**cluster_kwargs) + + # namespaces + namespace_kwargs = {"namespace_name": namespace_name} + namespace, namespace_created = Namespace.objects.get_or_create(**namespace_kwargs) + + # server + server_kwargs = {"ilo_hostname": ilo_hostname} + server, server_created = Server.objects.get_or_create(**server_kwargs) + + parent_cluster = None + + if not central_controller_p: + # find least-used parent central controller cluster + parent_clusters_kwargs = {"sap_clli_id__exact": sap_clli.pk, + "parent_cluster_id__isnull": True, + "is_central_controller": 1} + parent_clusters = Cluster.objects.filter(**parent_clusters_kwargs) + subcloud_counts = {} + + for parent in parent_clusters: + num_children = RawQuery.query_single_value(cursor, """ +select count(*) as num_children +from caas_cluster +where parent_cluster_id = %s and is_central_controller = 0""" % parent.pk, "num_children") + + if num_children < settings.WR_MAX_CHILD_CLUSTERS: + subcloud_counts[parent.pk] = [num_children, parent] + + if len(subcloud_counts) != 0: + parent_cluster = min(subcloud_counts.items(), key=lambda x: x[1][0])[1][1] + + # Derive subnets from gateways + oam_subnet = Address.add_40a(Address.gateway_to_subnet(oam_default_gateway)) + mgmt_subnet = Address.add_40a(Address.gateway_to_subnet(mgmt_default_gateway)) + ilo_subnet = Address.add_40a(Address.gateway_to_subnet(ilo_default_gateway)) + + # oam vip + need_ip = False + + if csr.oam_subnet == str(oam_subnet) and csr.oam_default_gateway == str(oam_default_gateway): + if cluster.oam_vip_address is None: + need_ip = True + else: + csr.oam_subnet = str(oam_subnet) + csr.oam_default_gateway = str(oam_default_gateway) + need_ip = True + if need_ip: + oam_vip_address_max = RawQuery.query_single_value(cursor, """ +with oam_ips as ( + select csr.id as csr_id, + c.oam_vip_address + from caas_server s + inner join caas_cluster c + on s.cluster_id = c.id + inner join caas_cellsiterouter csr + on c.csr_id = csr.id +) +select max(oam_vip_address) as max +from oam_ips +where csr_id = %s""" % csr.id, "max") + if oam_vip_address_max: + oam_vip_address_max = str(Address.next_address(oam_vip_address_max, oam_subnet, 0x10)) + else: + oam_vip_address_max = str(Address.next_address(oam_vip_address_max, oam_subnet, 0xf400)) + cluster.oam_vip_address = oam_vip_address_max + # oam host + need_ip = False + if csr.oam_subnet == str(oam_subnet) and csr.oam_default_gateway == str(oam_default_gateway): + if server.oam_host_address is None: + need_ip = True + else: + csr.oam_subnet = str(oam_subnet) + csr.oam_default_gateway = str(oam_default_gateway) + need_ip = True + if need_ip: + oam_host_address_max = RawQuery.query_single_value(cursor, """ +with oam_ips as ( + select csr.id as csr_id, + s.oam_host_address + from caas_server s + inner join caas_cluster c + on s.cluster_id = c.id + inner join caas_cellsiterouter csr + on c.csr_id = csr.id +) +select max(oam_host_address) as max +from oam_ips +where csr_id = %s""" % csr.id, "max") + if oam_host_address_max: + oam_host_address_max = str(Address.next_address(oam_host_address_max, oam_subnet, 0x1)) + else: + oam_host_address_max = str(Address.next_address(oam_host_address_max, oam_subnet, 0x400)) + server.oam_host_address = oam_host_address_max + + # mgmt + cluster.mgmt_subnet = str(mgmt_subnet) + cluster.mgmt_default_gateway = str(mgmt_default_gateway) + cluster.mgmt_address_range_start = str(mgmt_subnet) + cluster.mgmt_address_range_end = str(Address.address_add(cluster.mgmt_address_range_start, 0xf)) + + # don't step on central controllers TODO: separate central and remote CIQ ingestion + csr.ilo_subnet = str(ilo_subnet) + csr.ilo_default_gateway = str(ilo_default_gateway) + server.server_number = server_number + server.ilo_host_address = str(ilo_host_address) + else: + cluster.is_central_controller = 1 + + # foreign-key relationships + cluster.sap_clli = sap_clli + if assign_parent is True and cluster.parent_cluster is None: + cluster.parent_cluster = parent_cluster + cluster.location = location + cluster.vlan = vlan + cluster.csr = csr + cluster.namespace = namespace + server.cluster = cluster + # upsert + vlan.save() + namespace.save() + csr.save() + sap_clli.save() + location.save() + cluster.save() + server.save() diff --git a/src/caas/services/dnsservice.py b/src/caas/services/dnsservice.py new file mode 100644 index 0000000..bd8c388 --- /dev/null +++ b/src/caas/services/dnsservice.py @@ -0,0 +1,60 @@ +from django.conf import settings +from .helpers import Jinja, Hostname +from .ansibleservice import AnsibleService +from ..models import Server, Cluster +import logging + + +logger = logging.getLogger("caas") + + +class DnsService(AnsibleService): + def __init__(self, target, git, branch, playbook, zmq): + super().__init__("dns", target, git, branch, playbook, zmq) + + def deploy(self): + logger.debug("%s Enter" % self.target) + try: + server_kwargs = {"ilo_host_address": self.target} + server = Server.objects.get(**server_kwargs) + except Server.DoesNotExist: + errmsg = "%s FAILED: while running DNS playbook: failed to look up server" % self.target + logger.error(errmsg) + return -1 + try: + cluster_kwargs = {"id": server.cluster.pk} + cluster = Cluster.objects.get(**cluster_kwargs) + except Cluster.DoesNotExist: + errmsg = "%s FAILED: while running DNS playbook: failed to look up cluster" % self.target + logger.error(errmsg) + return -1 + server.oam_hostname = Hostname.make_oam_hostname(server.ilo_hostname) + if cluster.oam_vip_hostname is None: + cluster.oam_vip_hostname = Hostname.make_vip_hostname(server.ilo_hostname, cluster.cluster_name) + cluster.save() + server.save() + # The ansible target is the ilo hostname, but until now we only had the ilo IP. + self.target = server.ilo_hostname + # host_vars file + hostvars_config = {"ilo_hostname": server.ilo_hostname, + "ilo_host_address": server.ilo_host_address, + "oam_hostname": server.oam_hostname, + "oam_host_address": server.oam_host_address, + "oam_vip_hostname": cluster.oam_vip_hostname, + "oam_vip_address": cluster.oam_vip_address, + "cluster_name": cluster.cluster_name, + "dns_admin_endpoint": settings.DNS_ADMIN_ENDPOINT, + "dns_domain": settings.DNS_DOMAIN, + "infoblox_username": settings.INFOBLOX_USERNAME, + "infoblox_password": settings.INFOBLOX_PASSWORD} + hostvars_content = Jinja.render("dns_host_vars.j2", **hostvars_config) + hostvars = server.ilo_hostname + # inventory file + inventory_config = {"target": self.target} + inventory_content = Jinja.render("dns_inventory.j2", **inventory_config) + inventory = "%s.yaml" % self.target + logger.debug("%s Sending playbook to queue" % self.target) + self.run_playbook(inventory, inventory_content, hostvars, hostvars_content) + logger.debug("%s Sent playbook to queue" % self.target) + return 0 + diff --git a/src/caas/services/firmwareservice.py b/src/caas/services/firmwareservice.py new file mode 100644 index 0000000..97bfecc --- /dev/null +++ b/src/caas/services/firmwareservice.py @@ -0,0 +1,87 @@ +import datetime +from .helpers import Jinja +from .ansibleservice import AnsibleService +from ..models import FirmwareUpgradeSchedule, FirmwareBatch +from django.conf import settings +import logging + + +logger = logging.getLogger("caas") + + +""" +Expected versions. The bottom one is OK; the top one is too old. +firmware-version: 6.00 0x800036cb 1.1747.0 +firmware-version: 7.20 0x80007fdf 1.2585.0 +""" +class FirmwareService(AnsibleService): + """ + target == ilo_host_address + """ + def __init__(self, target, git, branch, playbook, zmq): + super().__init__("nic", target, git, branch, playbook, zmq) + + def schedule_upgrade(self): + logger.debug("%s Enter schedule_upgrade" % self.target) + try: + firmwarebatch_kwargs = {"ilo_host_address": self.target} + firmwarebatch = FirmwareBatch.objects.get(**firmwarebatch_kwargs) + except FirmwareBatch.DoesNotExist: + errmsg = "%s FAILED: while checking Intel NIC firmware version: cannot find server" % self.target + logger.error(errmsg) + return -1 + if firmwarebatch.intel_nic_firmware_version != settings.INTEL_NIC_FIRMWARE_VERSION: + firmware_kwargs = {"server_id": firmwarebatch.pk} + firmware, firmware_created = FirmwareUpgradeSchedule.objects.get_or_create(**firmware_kwargs) + if firmware.date_scheduled == None: + firmware.date_scheduled = datetime.datetime.now() + if firmware.kirke_ticket_number == None: + # TODO bealer: create KIRKE ticket + pass + firmware.save() + return 0 + + def deploy(self): + logger.debug("%s Enter deploy" % self.target) + kirke_inprocess_kwargs = {"kirke_ticket_number__isnull": False, + "kirke_ticket_completed__isnull": True} + kirke_inprocess = FirmwareUpgradeSchedule.objects.filter(**kirke_inprocess_kwargs) + for rec in kirke_inprocess: + # TODO bealer: query KIRKE and update status + rec.save() + firmware_schedule_kwargs = {"kirke_ticket_completed__isnull": False, + "date_completed__isnull": True} + firmware_schedule_ready = FirmwareUpgradeSchedule.objects.filter(**firmware_schedule_kwargs) + for sched in firmware_schedule_ready: + try: + firmwarebatch_kwargs = {"pk": sched.server.pk} + firmwarebatch = FirmwareBatch.objects.get(**firmwarebatch_kwargs) + except FirmwareBatch.DoesNotExist: + errmsg = "%s FAILED: while running firmware batch playbook: failed to look up batch" % self.target + logger.error(errmsg) + return -1 + self.target = firmwarebatch.ilo_hostname + # host_vars file + hostvars_config = {"ilo_host_address": firmwarebatch.ilo_host_address, + "wr_bmc_username": firmwarebatch.bmc_username, + "wr_bmc_password": firmwarebatch.bmc_password, + "wr_webdav_firmwarebatch": settings.WR_WEBDAV_SERVER, + "wr_webdav_server": settings.WR_WEBDAV_SERVER, + "wr_webdav_username": settings.WR_WEBDAV_USERNAME, + "wr_webdav_password": settings.WR_WEBDAV_PASSWORD, + "middleware_endpoint": settings.CAAS_MIDDLEWARE_ENDPOINT, + "middleware_username": settings.CAAS_MIDDLEWARE_USERNAME, + "middleware_password": settings.CAAS_MIDDLEWARE_PASSWORD, + "wr_registry_server": settings.WR_REGISTRY_SERVER, + "firmware_iso": settings.FIRMWARE_ISO} + hostvars_content = Jinja.render("firmware_host_vars.j2", **hostvars_config) + hostvars = self.target + # inventory file + inventory_config = {"target": firmwarebatch.ilo_hostname, + "ilo_host_address": firmwarebatch.ilo_host_address} + inventory_content = Jinja.render("firmware_inventory.j2", **inventory_config) + inventory = "%s.yaml" % self.target + logger.debug("%s Sending playbook to queue" % self.target) + self.run_playbook(inventory, inventory_content, hostvars, hostvars_content, extra_args = {"autoremediate": firmwarebatch.ilo_host_address}) + logger.debug("%s Sent playbook to queue" % self.target) + return 0 diff --git a/src/caas/services/helpers.py b/src/caas/services/helpers.py new file mode 100644 index 0000000..9145d1f --- /dev/null +++ b/src/caas/services/helpers.py @@ -0,0 +1,136 @@ +import ipaddress +import jinja2 +import os +import subprocess + + +class Shell(): + @staticmethod + def run_shell_command(cmd): + p = subprocess.Popen([cmd], + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={}) + out, err = p.communicate() + if p.returncode != 0: + raise Exception("Error running shell command:\n%s\n%s" % (out, err)) + return out, err + + +class Jinja(): + @staticmethod + def render(template_name, **kwargs): + cwd = os.path.dirname(os.path.abspath(__file__)) + loader = jinja2.FileSystemLoader([".", "/"]) + extensions = ["jinja2.ext.with_", "jinja2.ext.loopcontrols"] + environment = jinja2.Environment(trim_blocks = True, + lstrip_blocks = True, + extensions = extensions, + loader = loader) + template_names = ["%s/jinja/%s" % (cwd, template_name)] + template = environment.select_template(template_names) + return template.render(kwargs) + + +class RawQuery(): + @staticmethod + def dict_fetchall(cursor): + rows = cursor.fetchall() + if len(rows) == 0: + return None + else: + columns = [col[0] for col in cursor.description] + return [dict(zip(columns, row)) for row in rows] + + @staticmethod + def query_single_value(cursor, query, column_name): + cursor.execute(query) + rows = RawQuery.dict_fetchall(cursor) + if rows == {}: + return None + else: + return rows[0][column_name] + + +class Address(): + @staticmethod + def coalesce(ip, default): + if ip is None: + return default + else: + return ip + + @staticmethod + def get_quads(addr): + return list(filter(lambda x: x, str(addr).split("/")[0].split(":"))) + + @staticmethod + def address_add(ip, index): + return ipaddress.IPv6Address(ip) + index + + """ + Assumes a 5-quad 'subnet' as import input. Adds the 6th quad which is always 40a. + """ + @staticmethod + def add_40a(subnet): + return Address.address_add(subnet, 0x040a00000000) + + @staticmethod + def next_address(ip, subnet, block_begin_hex): + beginning_address = Address.coalesce(ip, Address.address_add(subnet, block_begin_hex)) + if ip is None: + return beginning_address + else: + return Address.address_add(beginning_address, 0x1) + + @staticmethod + def add_cidr(subnet, cidr = 64): + subnet_quads = Address.get_quads(subnet) + return "%s::/%s" % (":".join(subnet_quads), cidr) + + @staticmethod + def remove_cidr(subnet): + return ipaddress.IPv6Address(str(subnet).split("/")[0]) + + @staticmethod + def gateway_to_subnet(gateway, numquads = 5): + # The subnet is always the first 5 quads of the gateway. + # TODO: make sure this holds true in MTCE and labs as well as production. + quads = Address.get_quads(gateway) + return ipaddress.IPv6Address("%s::" % ":".join(quads[:numquads])) + + @staticmethod + def gateway_to_subnet_new(gateway): + quads = Address.get_quads(gateway) + return ipaddress.IPv6Address("%s:40a::" % ":".join(quads[:5])) + + @staticmethod + def subnet_to_4_quads(subnet): + quads = Address.get_quads(gateway) + return ipaddress.IPv6Address("%s::" % ":".join(quads[:4])) + + @staticmethod + def strip_brackets(ip): + return ip.replace("[", "").replace("]", "") + + +class Hostname(): + @staticmethod + def make_oam_hostname(ilo_hostname): + return ilo_hostname.replace("-pe0", "-pe2") + + """ + Note: this method does not deal with the index at the end of the + hostname. Doing so would require database access. This is the + responsibility of the service classes. See dnsservice.py + """ + @staticmethod + def make_vip_hostname(ilo_hostname, cluster_name): + index = int(cluster_name[-3:]) + return "%s-%s" % (ilo_hostname.replace("-pe0", "-pe1")[:-4], f"{index + 0:03}") + + @staticmethod + def increment_hostname(hostname): + index = int(hostname[-3:]) + return "%s-%s" % (hostname[:-4], f"{index + 1:03}") diff --git a/src/caas/services/httpclient.py b/src/caas/services/httpclient.py new file mode 100644 index 0000000..ad8270b --- /dev/null +++ b/src/caas/services/httpclient.py @@ -0,0 +1,18 @@ +import requests +from requests.packages.urllib3.exceptions import InsecureRequestWarning +from requests.auth import HTTPBasicAuth + + +class HttpClient(object): + def __init__(self): + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + self.session = requests.Session() + + def post(self, url, username, password, data = None, headers = {}, timeout = 60): + resp = self.session.post(url = url, + auth = HTTPBasicAuth(username, password), + json = data, + headers = headers, + timeout = timeout, + verify = False) + return resp diff --git a/src/caas/services/icingapollservice.py b/src/caas/services/icingapollservice.py new file mode 100644 index 0000000..d1796d0 --- /dev/null +++ b/src/caas/services/icingapollservice.py @@ -0,0 +1,47 @@ +from .helpers import Jinja +from .ansibleservice import AnsibleService +from ..models import IcingaBmc +from django.conf import settings +import logging + + +logger = logging.getLogger("caas") + + +class IcingaPollService(AnsibleService): + def __init__(self, target, git, branch, playbook, zmq, url): + super().__init__("icinga_poll", target, git, branch, playbook, zmq) + self.url = url + + def deploy(self): + logger.debug("%s Enter" % self.target) + try: + icinga_poll_kwargs = {"ilo_host_address": self.target} + icinga_poll = IcingaBmc.objects.get(**icinga_poll_kwargs) + except IcingaBmc.DoesNotExist: + logger.error("%s FAILED: while running icinga_poll playbook: failed to look up BMC" % self.target) + return -1 + + self.target = icinga_poll.ilo_hostname + # host_vars file + hostvars_config = {"bmc_ip": icinga_poll.ilo_host_address, + "bmc_username": icinga_poll.bmc_username, + "bmc_password": icinga_poll.bmc_password, + "icinga_api_url": settings.ICINGA_API_URL, + "icinga_api_username": settings.ICINGA_API_USERNAME, + "icinga_api_password": settings.ICINGA_API_PASSWORD, + "middleware_endpoint": settings.CAAS_MIDDLEWARE_ENDPOINT, + "middleware_username": settings.CAAS_MIDDLEWARE_USERNAME, + "middleware_password": settings.CAAS_MIDDLEWARE_PASSWORD, + "middleware_url": self.url} + hostvars_content = Jinja.render("icinga_poll_host_vars.j2", **hostvars_config) + hostvars = self.target + # inventory file + inventory_config = {"vendor_name": icinga_poll.vendor_name, "target": self.target} + inventory_content = Jinja.render("icinga_poll_inventory.j2", **inventory_config) + inventory = "%s.yaml" % self.target + logger.debug("%s Sending playbook to queue" % self.target) + self.run_playbook(inventory, inventory_content, hostvars, hostvars_content) + logger.debug("%s Sent playbook to queue" % self.target) + return 0 + diff --git a/src/caas/services/icingaservice.py b/src/caas/services/icingaservice.py new file mode 100644 index 0000000..4d1ff67 --- /dev/null +++ b/src/caas/services/icingaservice.py @@ -0,0 +1,46 @@ +from .helpers import Jinja +from .ansibleservice import AnsibleService +from .macaddressservice import MacAddressService +from ..models import IcingaBmc, IcingaRouter +from django.conf import settings +import logging + + +logger = logging.getLogger("caas") + + +class IcingaService(AnsibleService): + def __init__(self, target, git, branch, playbook, zmq): + super().__init__("icinga", target, git, branch, playbook, zmq) + + def deploy(self, mockciq = False): + logger.debug("%s Enter" % self.target) + # host_vars file + hostvars_config = {"icinga_routers": [], "icinga_bmcs": []} + for router in IcingaRouter.objects.all(): + hostvars_config["icinga_routers"].append({"name": router.csr_hostname, + "address6": router.oam_default_gateway}) + server_kwargs = {"date_completed": None} + servers = IcingaBmc.objects.filter(**server_kwargs) + for server in servers: + hostvars_config["icinga_bmcs"].append({"name": server.ilo_hostname, + "address6": server.ilo_host_address, + "preauth": True if mockciq or server.bmc_password_admin_default is None or MacAddressService.get_server_mac_address(server.ilo_host_address) is None else False, + "bmc_password_admin_default": server.bmc_password_admin_default, + "bmc_password_admin": server.bmc_password_admin, + "bmc_username": server.bmc_username, + "bmc_password": server.bmc_password, + "middleware_endpoint": settings.CAAS_MIDDLEWARE_ENDPOINT, + "middleware_username": settings.CAAS_MIDDLEWARE_USERNAME, + "middleware_password": settings.CAAS_MIDDLEWARE_PASSWORD}) + + hostvars_content = Jinja.render("icinga_host_vars.j2", **hostvars_config) + hostvars = self.target + # inventory file + inventory_config = {"target": self.target} + inventory_content = Jinja.render("icinga_inventory.j2", **inventory_config) + inventory = "%s.yaml" % self.target + logger.debug("%s Sending playbook to queue" % self.target) + self.run_playbook(inventory, inventory_content, hostvars, hostvars_content) + logger.debug("%s Sent playbook to queue" % self.target) + return 0 diff --git a/src/caas/services/jinja/bmc_host_vars.j2 b/src/caas/services/jinja/bmc_host_vars.j2 new file mode 100644 index 0000000..eb56bdd --- /dev/null +++ b/src/caas/services/jinja/bmc_host_vars.j2 @@ -0,0 +1,13 @@ +--- +bmc_ip: "{{ bmc_ip }}" +bmc_password_admin_default: "{{ bmc_password_admin_default }}" +bmc_password_admin: "{{ bmc_password_admin }}" +bmc_username: "{{ bmc_username }}" +bmc_password: "{{ bmc_password }}" +dns_nameservers: "{{ dns_nameservers }}" +syslog_server: "{{ syslog_server }}" +syslog_server_port: "{{ syslog_server_port }}" +ntp_servers: "{{ ntp_servers }}" +middleware_endpoint: "{{ middleware_endpoint }}" +middleware_username: "{{ middleware_username }}" +middleware_password: "{{ middleware_password }}" diff --git a/src/caas/services/jinja/bmc_inventory.j2 b/src/caas/services/jinja/bmc_inventory.j2 new file mode 100644 index 0000000..6ef8ca2 --- /dev/null +++ b/src/caas/services/jinja/bmc_inventory.j2 @@ -0,0 +1,6 @@ +--- +all: + children: + {{ vendor_name }}: + hosts: + {{ target }} diff --git a/src/caas/services/jinja/dns_host_vars.j2 b/src/caas/services/jinja/dns_host_vars.j2 new file mode 100644 index 0000000..2e66a8f --- /dev/null +++ b/src/caas/services/jinja/dns_host_vars.j2 @@ -0,0 +1,12 @@ +--- +ilo_hostname: "{{ ilo_hostname }}" +ilo_host_address: "{{ ilo_host_address }}" +oam_hostname: "{{ oam_hostname }}" +oam_host_address: "{{ oam_host_address }}" +oam_vip_hostname: "{{ oam_vip_hostname }}" +oam_vip_address: "{{ oam_vip_address }}" +cluster_name: "{{ cluster_name }}" +dns_admin_endpoint: "{{ dns_admin_endpoint }}" +dns_domain: "{{ dns_domain }}" +infoblox_username: "{{ infoblox_username }}" +infoblox_password: "{{ infoblox_password }}" diff --git a/src/caas/services/jinja/dns_inventory.j2 b/src/caas/services/jinja/dns_inventory.j2 new file mode 100644 index 0000000..c003afe --- /dev/null +++ b/src/caas/services/jinja/dns_inventory.j2 @@ -0,0 +1,6 @@ +--- +all: + children: + target: + hosts: + {{ target }} diff --git a/src/caas/services/jinja/firmware_host_vars.j2 b/src/caas/services/jinja/firmware_host_vars.j2 new file mode 100644 index 0000000..c029def --- /dev/null +++ b/src/caas/services/jinja/firmware_host_vars.j2 @@ -0,0 +1,13 @@ +--- +wr_distribution_zip: "WRCP-2006.20.03.zip" +ilo_host_address: "{{ ilo_host_address }}" +wr_bmc_username: "{{ wr_bmc_username }}" +wr_bmc_password: "{{ wr_bmc_password }}" +wr_webdav_server: "{{ wr_webdav_server }}" +wr_webdav_username: "{{ wr_webdav_username }}" +wr_webdav_password: "{{ wr_webdav_password }}" +middleware_endpoint: "{{ middleware_endpoint }}" +middleware_username: "{{ middleware_username }}" +middleware_password: "{{ middleware_password }}" +wr_registry_server: "{{ wr_registry_server }}" +firmware_iso: "{{ firmware_iso }}" diff --git a/src/caas/services/jinja/firmware_inventory.j2 b/src/caas/services/jinja/firmware_inventory.j2 new file mode 100644 index 0000000..b7d566c --- /dev/null +++ b/src/caas/services/jinja/firmware_inventory.j2 @@ -0,0 +1,7 @@ +--- +all: + children: + target: + hosts: + {{ target }}: + ansible_host: {{ ilo_host_address }} diff --git a/src/caas/services/jinja/icinga_host_vars.j2 b/src/caas/services/jinja/icinga_host_vars.j2 new file mode 100644 index 0000000..c25b0ec --- /dev/null +++ b/src/caas/services/jinja/icinga_host_vars.j2 @@ -0,0 +1,38 @@ +--- +icinga_routers: +{% for router in icinga_routers %} + - name: "{{ router.name }}" + address6: "{{ router.address6 }}" + group: "icingaadmins" +{% endfor %} + +icinga_bmcs: +{% for bmc in icinga_bmcs %} + - name: "{{ bmc.name }}" + address6: "{{ bmc.address6 }}" + group: "icingaadmins" +{% if bmc.preauth %} + preauth: True +{% endif %} +{% if bmc.bmc_password_admin_default is not none %} + bmc_username_admin_default: "Administrator" + bmc_password_admin_default: "{{ bmc.bmc_password_admin_default }}" +{% endif %} +{% if bmc.bmc_password_admin is not none %} + bmc_username_admin: "Administrator" + bmc_password_admin: "{{ bmc.bmc_password_admin }}" +{% endif %} +{% if bmc.bmc_username is not none and bmc.bmc_password is not none %} + bmc_username: "{{ bmc.bmc_username }}" + bmc_password: "{{ bmc.bmc_password }}" +{% endif %} +{% if bmc.middleware_endpoint is not none %} + middleware_endpoint: "{{ bmc.middleware_endpoint }}" +{% endif %} +{% if bmc.middleware_username is not none %} + middleware_username: "{{ bmc.middleware_username }}" +{% endif %} +{% if bmc.middleware_password is not none %} + middleware_password: "{{ bmc.middleware_password }}" +{% endif %} +{% endfor %} diff --git a/src/caas/services/jinja/icinga_inventory.j2 b/src/caas/services/jinja/icinga_inventory.j2 new file mode 100644 index 0000000..c003afe --- /dev/null +++ b/src/caas/services/jinja/icinga_inventory.j2 @@ -0,0 +1,6 @@ +--- +all: + children: + target: + hosts: + {{ target }} diff --git a/src/caas/services/jinja/icinga_poll_host_vars.j2 b/src/caas/services/jinja/icinga_poll_host_vars.j2 new file mode 100644 index 0000000..e2aaac7 --- /dev/null +++ b/src/caas/services/jinja/icinga_poll_host_vars.j2 @@ -0,0 +1,11 @@ +--- +bmc_ip: "{{ bmc_ip }}" +bmc_username: "{{ bmc_username }}" +bmc_password: "{{ bmc_password }}" +icinga_api_url: "{{ icinga_api_url }}" +icinga_api_username: "{{ icinga_api_username }}" +icinga_api_password: "{{ icinga_api_password }}" +middleware_endpoint: "{{ middleware_endpoint }}" +middleware_username: "{{ middleware_username }}" +middleware_password: "{{ middleware_password }}" +middleware_url: "{{ middleware_url }}" diff --git a/src/caas/services/jinja/icinga_poll_inventory.j2 b/src/caas/services/jinja/icinga_poll_inventory.j2 new file mode 100644 index 0000000..6ef8ca2 --- /dev/null +++ b/src/caas/services/jinja/icinga_poll_inventory.j2 @@ -0,0 +1,6 @@ +--- +all: + children: + {{ vendor_name }}: + hosts: + {{ target }} diff --git a/src/caas/services/jinja/wr_host_vars.j2 b/src/caas/services/jinja/wr_host_vars.j2 new file mode 100644 index 0000000..15a6f4d --- /dev/null +++ b/src/caas/services/jinja/wr_host_vars.j2 @@ -0,0 +1,74 @@ +--- +wr_vendor_name: "{{ wr_vendor_name }}" +wr_region_name: "{{ wr_region_name }}" +wr_location: "{{ wr_region_name }}" +wr_bmc_username: "{{ wr_bmc_username }}" +wr_bmc_password: "{{ wr_bmc_password }}" +wr_bmc_controller_0_address: "{{ wr_bmc_controller_0_address }}" +{% if wr_bmc_controller_1_address %} +wr_bmc_controller_1_address: "{{ wr_bmc_controller_1_address }}" +{% endif %} +wr_mgmt_controller_0_mac: "{{ wr_mgmt_controller_0_mac }}" +{% if wr_bmc_controller_1_mac %} +wr_mgmt_controller_1_mac: "{{ wr_mgmt_controller_1_mac }}" +{% endif %} +wr_oam_floating_address: "{{ wr_oam_floating_address }}/64" +wr_oam_node_0_address: "{{ wr_oam_node_0_address }}/64" +{% if wr_oam_node_1_address %} +wr_oam_node_1_address: "{{ wr_oam_node_1_address }}/64" +{% endif %} +wr_oam_gateway_address: "{{ wr_oam_gateway_address }}" +wr_mgmt_start_address: "{{ wr_mgmt_start_address }}" +wr_mgmt_end_address: "{{ wr_mgmt_end_address }}" +wr_mgmt_gateway_address: "{{ wr_mgmt_gateway_address }}" +wr_mgmt_subnet: "{{ wr_mgmt_subnet }}/64" +wr_central_mgmt_start_address: "{{ wr_central_mgmt_start_address }}" +wr_central_mgmt_default_gateway: "{{ wr_central_mgmt_default_gateway }}" +wr_central_mgmt_subnet: "{{ wr_central_mgmt_subnet }}" +wr_host_vlan: {{ wr_host_vlan }} +wr_oam_vlan: {{ wr_oam_vlan }} +wr_mgmt_vlan: {{ wr_mgmt_vlan }} +wr_cluster_pod_subnet: "fd00:4888::/64" +wr_cluster_host_subnet: "fd00:4888:0:1::/64" +wr_cluster_service_subnet: "fd00:4888:0:2::/112" +wr_mgmt_multicast_subnet: "ff05::18:1:0/124" +{% if wr_dns_servers %} +wr_dns_servers: +{% for server in wr_dns_servers %} + - "{{ server }}" +{% endfor %} +{% endif %} +{% if wr_ntp_servers %} +wr_ntp_servers: +{% for server in wr_ntp_servers %} + - "{{ server }}" +{% endfor %} +{% endif %} +wr_no_check_cert: True +wr_registry_server: "{{ wr_registry_server }}" +wr_registry_username: "{{ wr_registry_username }}" +wr_registry_password: "{{ wr_registry_password }}" +wr_webdav_server: "{{ wr_webdav_server }}" +wr_webdav_username: "{{ wr_webdav_username }}" +wr_webdav_password: "{{ wr_webdav_password }}" +wr_middleware_endpoint: "{{ wr_middleware_endpoint }}" +wr_middleware_username: "{{ wr_middleware_username }}" +wr_middleware_password: "{{ wr_middleware_password }}" +{% if wr_docker_http_proxy %} +wr_docker_http_proxy: "{{ wr_docker_http_proxy }}" +{% endif %} +{% if wr_docker_https_proxy %} +wr_docker_https_proxy: "{{ wr_docker_https_proxy }}" +{% endif %} +{% if wr_docker_no_proxy %} +wr_docker_no_proxy: {{ wr_docker_no_proxy }} +{% endif %} +wr_oam_nic_0: ens3f0 +wr_oam_nic_1: ens3f1 +wr_nic_main: ens3f0 +wr_no_check_cert: True + +wr_ssl_ca_cert: | +{% filter indent(width=2) %} + {{ wr_ssl_ca_cert }} +{% endfilter %} diff --git a/src/caas/services/jinja/wr_inventory.j2 b/src/caas/services/jinja/wr_inventory.j2 new file mode 100644 index 0000000..814d692 --- /dev/null +++ b/src/caas/services/jinja/wr_inventory.j2 @@ -0,0 +1,11 @@ +--- +all: + children: + central: + hosts: + {{ central_target }}: + ansible_host: {{ central_target_ip }} + remote: + hosts: + {{ remote_target }}: + ansible_host: {{ remote_target_ip }} diff --git a/src/caas/services/macaddressservice.py b/src/caas/services/macaddressservice.py new file mode 100644 index 0000000..745f5fa --- /dev/null +++ b/src/caas/services/macaddressservice.py @@ -0,0 +1,38 @@ +from ..models import Server +from .helpers import Address +import logging + + +logger = logging.getLogger("caas") + + +class MacAddressService(): + @staticmethod + def set_server_mac_address(ilo_host_address, pxe_mac_address, intel_nic_firmware_version): + try: + server_kwargs = {"ilo_host_address": Address.strip_brackets(ilo_host_address)} + server = Server.objects.get(**server_kwargs) + except Server.DoesNotExist: + logger.error("%s FAILED: while setting server MAC address: failed to retrieve server" % ilo_host_address) + return -1 + changed_p = False + if server.pxe_mac_address != pxe_mac_address: + server.pxe_mac_address = pxe_mac_address + changed_p = True + if server.intel_nic_firmware_version != intel_nic_firmware_version: + server.intel_nic_firmware_version = intel_nic_firmware_version + changed_p = True + if changed_p: + server.save() + return 0 + + @staticmethod + def get_server_mac_address(ilo_host_address): + try: + server_kwargs = {"ilo_host_address": Address.strip_brackets(ilo_host_address)} + server = Server.objects.get(**server_kwargs) + except Server.DoesNotExist: + logger.error("%s FAILED: while getting server MAC address: failed to retrieve server" % ilo_host_address) + return None + + return server.pxe_mac_address diff --git a/src/caas/services/nicfirmwareupgradenowservice.py b/src/caas/services/nicfirmwareupgradenowservice.py new file mode 100644 index 0000000..d31fd74 --- /dev/null +++ b/src/caas/services/nicfirmwareupgradenowservice.py @@ -0,0 +1,68 @@ +from .helpers import Jinja +from .ansibleservice import AnsibleService +from .firmwareservice import FirmwareService +from .wrinstallnowservice import WrInstallNowService +from ..models import FirmwareBatch, WrBatch, WrInstallSchedule, FirmwareUpgradeSchedule +from django.conf import settings +import logging +import datetime + + +logger = logging.getLogger("caas") + + +class NicFirmwareUpgradeNowService(AnsibleService): + def __init__(self, target, git, branch, playbook, zmq): + super().__init__("nic", target, git, branch, playbook, zmq) + + def deploy(self): + try: + firmwarebatch_kwargs = {"ilo_host_address": self.target} + firmwarebatch = FirmwareBatch.objects.get(**firmwarebatch_kwargs) + except FirmwareBatch.DoesNotExist: + errmsg = "FAILED: while checking Intel NIC firmware version: cannot find server with ilo_host_address: %s" % self.target + logger.error(errmsg) + return -1 + completed = datetime.datetime.utcnow() + if firmwarebatch.intel_nic_firmware_version == settings.INTEL_NIC_FIRMWARE_VERSION: + try: + wrbatch_kwargs = {"ilo_host_address": firmwarebatch.ilo_host_address} + wrbatch = WrBatch.objects.get(**wrbatch_kwargs) + except WrBatch.DoesNotExist: + logger.error("FAILED: while scheduling Wind River installation: cannot find wrbatch with identifier: %s" % icingabmc.ilo_host_address) + raise + try: + wr_kwargs = {"cluster_id": wrbatch.cluster_id} + wr = WrInstallSchedule.objects.get(**wr_kwargs) + except WrInstallSchedule.DoesNotExist: + return "FAILED: while performing Wind River playbook callback: cannot find WrInstallSchedule associated with the cluster with cluster_id: %s" % cluster.pk + # never run an installation on top of a finished subcloud + if wr.kirke_ticket_number is None: + if wr.kirke_ticket_completed is None: + wr.kirke_ticket_completed = completed + wr.date_completed = None + wr.kirke_ticket_number = None + wr.save() + wrinstallnow = WrInstallNowService("wr", + firmwarebatch.ilo_host_address, + settings.ANSIBLE["wr"]["git"], + settings.ANSIBLE["wr"]["branch"], + settings.ANSIBLE["wr"]["playbook"], + settings.ANSIBLE["wr"]["zmq"]) + wrinstallnow.deploy() + else: + try: + nic_kwargs = {"server_id": firmwarebatch.pk} + nic = FirmwareUpgradeSchedule.objects.get(**nic_kwargs) + except FirmwareUpgradeSchedule.DoesNotExist: + logger.error("FAILED: while deploying Wind River installation: cannot find FirmwareUpgradeSchedule with ID: %s" % icingabmc.pk) + raise + if nic.kirke_ticket_completed is None: + nic.kirke_ticket_completed = completed + nic.date_completed = None + nic.kirke_ticket_number = None + nic.save() + firmwareservice = FirmwareService(firmwarebatch.ilo_host_address, self.git, self.branch, self.playbook, self.zmq) + firmwareservice.deploy() + # The playbook callback will trigger the wr install + return 0 diff --git a/src/caas/services/nicfirmwareversionservice.py b/src/caas/services/nicfirmwareversionservice.py new file mode 100644 index 0000000..b8a9d43 --- /dev/null +++ b/src/caas/services/nicfirmwareversionservice.py @@ -0,0 +1,37 @@ +import datetime +from .helpers import Address +from ..models import FirmwareUpgradeSchedule, Server +from django.conf import settings +import logging + + +logger = logging.getLogger("caas") + +class NicFirmwareVersionService(): + @staticmethod + def set_server_nic_firmware_version(ilo_host_address, intel_nic_firmware_version): + try: + server_kwargs = {"ilo_host_address": ilo_host_address} + server = Server.objects.get(**server_kwargs) + except Server.DoesNotExist: + errmsg = "FAILED: while upgrading Intel NIC firmware: cannot find server with ilo_host_address: %s" % ilo_host_address + logger.error(errmsg) + return -1 + + try: + firmware_kwargs = {"server_id": server.pk} + firmware = FirmwareUpgradeSchedule.objects.get(**firmware_kwargs) + except FirmwareUpgradeSchedule.DoesNotExist: + errmsg = "FAILED: while upgrading Intel NIC firmware: cannot find FirmwareUpgradeSchedule associated with the server with ilo_hostname: %s" % server.ilo_hostname + logger.error(errmsg) + return -1 + + if intel_nic_firmware_version == settings.INTEL_NIC_FIRMWARE_VERSION: + firmware.date_completed = datetime.datetime.now() + else: + firmware.date_last_failed = datetime.datetime.now() + + server.intel_nic_firmware_version = intel_nic_firmware_version + server.save() + firmware.save() + return 0 diff --git a/src/caas/services/passwordservice.py b/src/caas/services/passwordservice.py new file mode 100644 index 0000000..ab666c8 --- /dev/null +++ b/src/caas/services/passwordservice.py @@ -0,0 +1,180 @@ +import csv +import hashlib +import os +from io import TextIOWrapper +from django.conf import settings +from ..models import Cluster, Server, Credential, Chassis, Blade, Vendor +import logging + +logger = logging.getLogger("caas") + +class PasswordService(): + @staticmethod + def import_password_data(filehandle): + f = TextIOWrapper(filehandle, encoding = "ascii", errors = "replace") + csv_reader = csv.reader(f, delimiter=",") + for row in csv_reader: + serial_number_chassis = "ILO%s" % row[1] + serial_number_blade = "ILO%s" % row[2] + password = row[3] + credential_kwargs = {"bmc_password_admin_default": password, + "bmc_password_admin__isnull": True, + "bmc_username__isnull": True, + "bmc_password__isnull": True} + credential, credential_created = Credential.objects.get_or_create(**credential_kwargs) + chassis_kwargs = {"serial_number": serial_number_chassis} + chassis, chassis_created = Chassis.objects.get_or_create(**chassis_kwargs) + blade_kwargs = {"serial_number": serial_number_blade} + blade, blade_created = Blade.objects.get_or_create(**blade_kwargs) + blade.credential = credential + blade.chassis = chassis + credential.save() + chassis.save() + blade.save() + + @staticmethod + def get_server_link_data(ilo_host_address): + try: + server_kwargs = {"ilo_host_address": ilo_host_address} + server = Server.objects.get(**server_kwargs) + except Server.DoesNotExist: + logger.error("FAILED: while linking server to blade: failed to look up server ilo_host_address=%s" % ilo_host_address) + return None + try: + blade_kwargs = {"id": server.blade.pk} + blade = Blade.objects.get(**blade_kwargs) + except Blade.DoesNotExist: + logger.error("FAILED: while linking server to blade: failed to look up blade ilo_host_address=%s" % ilo_host_address) + return None + try: + vendor_kwargs = {"id": blade.vendor.pk} + vendor = Vendor.objects.get(**vendor_kwargs) + except Vendor.DoesNotExist: + logger.error("FAILED: while linking server to blade: failed to look up vendor ilo_host_address=%s" % ilo_host_address) + return None + return {"vendor": vendor.vendor_name, + "serial_number": blade.serial_number, + "ilo_host_address": server.ilo_host_address} + + @staticmethod + def link_server_to_blade(vendor_name, serial_number, ilo_host_address): + try: + server_kwargs = {"ilo_host_address": ilo_host_address} + server = Server.objects.get(**server_kwargs) + except Server.DoesNotExist: + logger.error("FAILED: while linking server to blade: failed to look up server! ilo_host_address=%s" % ilo_host_address) + return None + try: + blade_kwargs = {"serial_number": serial_number} + blade = Blade.objects.get(**blade_kwargs) + except Blade.DoesNotExist: + logger.error("FAILED: while linking server to blade: failed to look up blade! ilo_host_address=%s" % ilo_host_address) + return None + try: + vendor_kwargs = {"vendor_name": vendor_name} + vendor = Vendor.objects.get(**vendor_kwargs) + except Vendor.DoesNotExist: + logger.error("FAILED: while linking server to blade: failed to look up vendor! ilo_host_address=%s" % ilo_host_address) + return None + server.blade = blade + blade.vendor = vendor + server.save() + blade.save() + return server + + @staticmethod + def is_server_linked_to_blade(ilo_host_address): + try: + server_kwargs = {"ilo_host_address": ilo_host_address} + server = Server.objects.get(**server_kwargs) + except Server.DoesNotExist: + return True # server doesn't exist, no action should be triggered + + if server.blade is None or server.pxe_mac_address is None: + return False # server is not yet linked with blade + else: + return True # server is already linked with blade + + @staticmethod + def generate_unique_password(): + m = hashlib.sha1() + m.update(os.urandom(16)) + return m.hexdigest()[:20] + + @staticmethod + def create_bmc_admin_password(serial_number): + try: + blade_kwargs = {"serial_number": serial_number} + blade = Blade.objects.get(**blade_kwargs) + except Blade.DoesNotExist: + logger.error("FAILED: while creating BMC account: failed to look up blade for serial %s" % serial_number) + return -1 + if blade.credential is None: + logger.error("FAILED: while creating BMC account: failed to look up credential for serial %s" % serial_number) + return -1 + if blade.credential.bmc_password is None: + # We don't want to update this Credential because it will affect + # every Blade that is tied to it. We need to create a new + # Credential and link the Blade to the new Credential. + admin_password = PasswordService.generate_unique_password() + ospctl_password = PasswordService.generate_unique_password() + credential_kwargs = {"bmc_password_admin_default": blade.credential.bmc_password_admin_default} + credential, credential_created = Credential.objects.get_or_create(**credential_kwargs) + blade.credential = credential + credential.bmc_password_admin = admin_password + credential.bmc_username = settings.BMC_USERNAME + credential.bmc_password = ospctl_password + blade.save() + credential.save() + logger.debug("OK: passwords created for serial %s" % serial_number) + return 0 + logger.debug("OK: no-op for serial %s" % serial_number) + return 0 + + @staticmethod + def central_server_p(ilo_host_address): + try: + server_kwargs = {"ilo_host_address": ilo_host_address} + server = Server.objects.get(**server_kwargs) + except Server.DoesNotExist: + logger.error("FAILED: while determining if the server with ilo_host_address %s is a central server: server not found." % ilo_host_address) + return True + try: + cluster_kwargs = {"id": server.cluster.pk} + cluster = Cluster.objects.get(**cluster_kwargs) + except Cluster.DoesNotExist: + logger.error("FAILED: while determining if the server with ilo_host_address %s is a central server: cluster not found." % ilo_host_address) + return True + return False if cluster.is_central_controller==0 else True + + @staticmethod + def fix_creds(): + bad_creds = Credential.objects.raw(""" +with dupes (bmc_password_admin_default, zahl) as ( + select bmc_password_admin_default, + count(bmc_password_admin_default) as zahl + from caas_credential + group by bmc_password_admin_default +) +select c.id, + c.bmc_password_admin_default, + c.bmc_password_admin, + c.bmc_username, + c.bmc_password +from caas_credential c +left outer join dupes d + on c.bmc_password_admin_default = d.bmc_password_admin_default +where d.zahl > 1 + and bmc_password is null""") + for bad_cred in bad_creds: + good_cred_kwargs = {"bmc_password_admin_default": bad_cred.bmc_password_admin_default, + "bmc_password__isnull": False} + good_cred = Credential.objects.get(**good_cred_kwargs) + try: + blade_kwargs = {"credential_id": bad_cred.id} + blade = Blade.objects.get(**blade_kwargs) + blade.credential_id = good_cred.pk + blade.save() + bad_cred.delete() + except Blade.DoesNotExist: + pass diff --git a/src/caas/services/patchservice.py b/src/caas/services/patchservice.py new file mode 100644 index 0000000..c986051 --- /dev/null +++ b/src/caas/services/patchservice.py @@ -0,0 +1,106 @@ +import datetime +from ..models import Cluster, WrInstallSchedule, PatchSchedule, WrBatch +from django.conf import settings +from .helpers import Jinja +from .ansibleservice import AnsibleService + + +class PatchService(AnsibleService): + """ + target == ilo_host_address, later becomes cluster_name + """ + def __init__(self, target, git, branch, playbook, zmq): + super().__init__("patch", target, git, branch, playbook, zmq) + + def schedule_install(self): + pass + # try: + # patchbatch_kwargs = {"ilo_host_address": self.target} + # # repurpose WrBatch + # patchbatch = WrBatch.objects.get(**patchbatch_kwargs) + # except WrBatch.DoesNotExist: + # return "FAILED: while scheduling Wind River installation: cannot find wrbatch with ilo_host_address: %s" % self.target + # try: + # cluster_kwargs = {"pk": patchbatch.cluster_id} + # cluster = Cluster.objects.get(**cluster_kwargs) + # except Cluster.DoesNotExist: + # return "FAILED: while scheduling Wind River installation: cannot find cluster associated with wrbatch with ilo_host_address: %s" % patchbatch.ilo_host_address + # patch_kwargs = {"cluster_id": cluster.pk} + # patch, patch_created = PatchSchedule.objects.get_or_create(**patch_kwargs) + # if patch.date_scheduled is None: + # patch.date_scheduled = datetime.datetime.now() + # if patch.kirke_ticket_number is None: + # # TODO bealer: create KIRKE ticket + # pass + # patch.save() + # # one playbook per cluster + # self.target = cluster.cluster_name + # with tempfile.TemporaryDirectory() as tempdir: + # builddir = "%s/%s" % (tempdir, self.repo_name) + # self.clone_repo(builddir) + # # host_vars file + # patch_config = {"wr_vendor_name": patchbatch.vendor_name, + # "wr_region_name": cluster.cluster_name, + # "wr_central_p": False if patchbatch.parent_oam_vip_hostname else True, + # "wr_bmc_controller_0_address": patchbatch.ilo_host_address, + # "wr_mgmt_controller_0_mac": patchbatch.pxe_mac_address, + # "wr_oam_floating_address": patchbatch.oam_vip_address, + # "wr_oam_node_0_address": patchbatch.oam_host_address, + # "wr_oam_gateway_address": patchbatch.oam_default_gateway, + # "wr_mgmt_start_address": patchbatch.mgmt_address_range_start, + # "wr_mgmt_end_address": patchbatch.mgmt_address_range_end, + # "wr_mgmt_gateway_address": patchbatch.mgmt_default_gateway, + # "wr_mgmt_subnet": patchbatch.mgmt_subnet, + # "wr_central_mgmt_start_address": patchbatch.parent_mgmt_address_range_start, + # "wr_central_mgmt_default_gateway": patchbatch.parent_mgmt_default_gateway, + # "wr_host_vlan": patchbatch.host_vlan, + # "wr_oam_vlan": patchbatch.oam_vlan, + # "wr_mgmt_vlan": patchbatch.mgmt_vlan, + # "wr_bmc_username": patchbatch.bmc_username, + # "wr_bmc_password": patchbatch.bmc_password, + # "wr_dns_servers": settings.DNS_SERVERS, + # "wr_ntp_servers": settings.NTP_SERVERS, + # "wr_registry_server": settings.WR_REGISTRY_SERVER, + # "wr_registry_username": settings.WR_REGISTRY_USERNAME, + # "wr_registry_password": settings.WR_REGISTRY_PASSWORD, + # "wr_webdav_server": settings.WR_WEBDAV_SERVER, + # "wr_webdav_username": settings.WR_WEBDAV_USERNAME, + # "wr_webdav_password": settings.WR_WEBDAV_PASSWORD, + # "wr_docker_http_proxy": settings.WR_DOCKER_HTTP_PROXY, + # "wr_docker_https_proxy": settings.WR_DOCKER_HTTPS_PROXY, + # "wr_docker_no_proxy": settings.WR_DOCKER_NO_PROXY, + # "wr_ssl_ca_cert": settings.WR_SSL_CA_CERT} + # content = Jinja.render("patch_host_vars.j2", **patch_config) + # filename = "%s/host_vars/%s" % (builddir, patchbatch.oam_hostname) + # with AnsibleVaultFile(filename) as f: + # f.write(content) + # # inventory file + # patch_config = {"central_target": patchbatch.parent_cluster_name, + # "central_target_ip": patchbatch.parent_oam_vip_address, + # "remote_target": patchbatch.oam_hostname, + # "remote_target_ip": patchbatch.oam_vip_address} + # content = Jinja.render("patch_inventory.j2", **patch_config) + # filename = "%s/inventory/%s.yaml" % (builddir, cluster.cluster_name) + # with open(filename, "w") as f: + # f.write(content) + # self.push() + + def deploy(self): + pass + # kirke_inprocess = PatchSchedule.objects.filter(kirke_ticket_number__isnull = False, + # kirke_ticket_completed__isnull = True) + # for rec in kirke_inprocess: + # # TODO bealer: query KIRKE and update status + # rec.save() + # patch_schedule_kwargs = {"kirke_ticket_completed__isnull": False, + # "date_completed__isnull": True} + # patch_schedule_ready = PatchSchedule.objects.filter(**patch_schedule_kwargs) + # for sched in patch_schedule_ready: + # try: + # patchbatch_kwargs = {"cluster_id": sched.cluster.pk, + # "intel_nic_firmware_version": settings.INTEL_NIC_FIRMWARE_VERSION} + # patchbatch = WrBatch.objects.filter(**patchbatch_kwargs) + # except WrBatch.DoesNotExist: + # return "FAILED: while running PATCH batch playbook: failed to look up wrbatch" + # self.target = patchbatch.cluster_name + # self.run_playbook() diff --git a/src/caas/services/playbookreportservice.py b/src/caas/services/playbookreportservice.py new file mode 100644 index 0000000..73abdfa --- /dev/null +++ b/src/caas/services/playbookreportservice.py @@ -0,0 +1,147 @@ +import datetime +from django.conf import settings +from ..models import FirmwareUpgradeSchedule, WrInstallSchedule, Location, Cluster, Server, IcingaBmc, WrBatch +from .wrinstallnowservice import WrInstallNowService +from .icingapollservice import IcingaPollService +from .bmcservice import BmcService +from .httpclient import HttpClient +import logging + + +logger = logging.getLogger("caas") + + +class PlaybookReportService(): + def __init__(self, git, playbook, target, failed): + self.git = git + self.playbook = playbook + self.target = target + self.failed = failed + + def process(self): + if ".yaml" in self.playbook: + ansible_config = {k: v for (k, v) in settings.ANSIBLE.items() if self.playbook in v.values() and v["git"] == self.git} + else: + ansible_config = {k: v for (k, v) in settings.ANSIBLE.items() if k == self.playbook and v["git"] == self.git} + key = [k for k in ansible_config.keys()][0] + if key == "wr": + self.target = self.target.split(".yaml")[0] + try: + cluster_kwargs = {"cluster_name": self.target} + cluster = Cluster.objects.get(**cluster_kwargs) + except Server.DoesNotExist: + return "FAILED: while performing Wind River playbook callback: cannot find cluster with cluster_name: %s" % self.target + try: + wr_kwargs = {"cluster_id": cluster.pk} + wr = WrInstallSchedule.objects.get(**wr_kwargs) + except WrInstallSchedule.DoesNotExist: + return "FAILED: while performing Wind River playbook callback: cannot find WrInstallSchedule associated with the cluster with cluster_name: %s" % cluster.cluster_name + completed = datetime.datetime.utcnow() + if wr.kirke_ticket_completed is None: + wr.kirke_ticket_completed = completed + if self.failed == "0": + wr.date_completed = completed + wr.kirke_ticket_number = None # Still needs manual check/remediation + # # Change default BMC password + # server_kwargs = {"cluster_id": cluster.pk} + # servers = Server.objects.filter(**server_kwargs) + # for server in servers: + # bmc = BmcService(server.ilo_host_address, + # settings.ANSIBLE["bmc"]["git"], + # settings.ANSIBLE["bmc"]["branch"], + # settings.ANSIBLE["bmc"]["playbook_adminpassword"], + # settings.ANSIBLE["bmc"]["zmq"]) + # bmc.deploy() + # Orchstration callback + try: + location_kwargs = {"pk": cluster.location_id} + location = Location.objects.get(**location_kwargs) + except Location.DoesNotExist: + return "FAILED: while performing Wind River playbook callback: cannot find Location associated with the cluster with cluster_name: %s" % cluster.cluster_name + payload = {"cluster_status": [{ + "name": cluster.cluster_name, + "description": location.fuze_spm_site_name, + "location": location.fuze_spm_site_id, + "software_version": settings.WR_VERSION, + "availability": "ONLINE", + "deploy_status": "COMPLETE", + "created_at": completed.strftime("%Y-%m-%d %H:%M:%S"), + "updated_at": completed.strftime("%Y-%m-%d %H:%M:%S")}]} + httpclient = HttpClient() + resp = httpclient.post(url = "%s/orchestration/caas-status" % settings.CAAS_MIDDLEWARE_ENDPOINT, + username = settings.CAAS_MIDDLEWARE_USERNAME, + password = settings.CAAS_MIDDLEWARE_PASSWORD, + data = payload) + if resp.status_code >= 300: + return "Error calling orchestration endpoint. Status code: %s , message: %s" % (resp.status_code, resp.text) + else: + wr.date_completed = None + wr.kirke_ticket_number = None + wr.date_last_failed = completed + wr.save() + elif key == "nic": + self.target = self.target.split(".yaml")[0] + try: + icingabmc_kwargs = {"ilo_hostname": self.target} + icingabmc = IcingaBmc.objects.get(**icingabmc_kwargs) + except IcingaBmc.DoesNotExist: + logger.error("FAILED: while deploying Wind River installation: cannot find IcingaBmc with identifier: %s" % self.target) + raise + try: + nic_kwargs = {"server_id": icingabmc.pk} + nic = FirmwareUpgradeSchedule.objects.get(**nic_kwargs) + except FirmwareUpgradeSchedule.DoesNotExist: + logger.error("FAILED: while deploying Wind River installation: cannot find FirmwareUpgradeSchedule with ID: %s" % icingabmc.pk) + raise + completed = datetime.datetime.utcnow() + if nic.kirke_ticket_completed is None: + nic.kirke_ticket_completed = completed + if self.failed == "0": + try: + wrbatch_kwargs = {"ilo_host_address": icingabmc.ilo_host_address} + wrbatch = WrBatch.objects.get(**wrbatch_kwargs) + except WrBatch.DoesNotExist: + logger.error("FAILED: while scheduling Wind River installation: cannot find wrbatch with identifier: %s" % icingabmc.ilo_host_address) + raise + # never run an installation on top of a finished subcloud + try: + wr_kwargs = {"cluster_id": wrbatch.cluster_id} + wr = WrInstallSchedule.objects.get(**wr_kwargs) + except WrInstallSchedule.DoesNotExist: + return "FAILED: while performing Wind River playbook callback: cannot find WrInstallSchedule associated with the cluster with cluster_name: %s" % cluster.cluster_name + if wr.kirke_ticket_number is None: + nic.date_completed = completed + nic.kirke_ticket_number = nic.pk + wrinstallnow = WrInstallNowService("wr", + wrbatch.ilo_host_address, + settings.ANSIBLE["wr"]["git"], + settings.ANSIBLE["wr"]["branch"], + settings.ANSIBLE["wr"]["playbook"], + settings.ANSIBLE["wr"]["zmq"]) + wrinstallnow.deploy() + else: + nic.date_completed = None + nic.kirke_ticket_number = None + nic.date_last_failed = completed + nic.save() + elif key == "bmc": + self.target = self.target.split(".yaml")[0] + if self.failed == "0": + try: + icingabmc_kwargs = {"ilo_hostname": self.target} + icingabmc = IcingaBmc.objects.get(**icingabmc_kwargs) + except IcingaBmc.DoesNotExist: + logger.error("FAILED: while deploying Wind River installation: cannot find IcingaBmc with identifier: %s" % self.target) + raise + icinga_poll = IcingaPollService(icingabmc.ilo_host_address, + settings.ANSIBLE["icinga_poll"]["git"], + settings.ANSIBLE["icinga_poll"]["branch"], + settings.ANSIBLE["icinga_poll"]["playbook"], + settings.ANSIBLE["icinga_poll"]["zmq"], + "adminpassword") + rc = icinga_poll.deploy() + return {"git": self.git, + "playbook": self.playbook, + "target": self.target, + "failed": self.failed, + "key": key} diff --git a/src/caas/services/subcloudservice.py b/src/caas/services/subcloudservice.py new file mode 100644 index 0000000..5095bbb --- /dev/null +++ b/src/caas/services/subcloudservice.py @@ -0,0 +1,12 @@ +from ..models import WrBatch +import logging + + +logger = logging.getLogger("caas") + + +class SubcloudService(): + @staticmethod + def find_subclouds(fuze_id): + wrbatch_kwargs = {"fuze_spm_site_id": fuze_id} + return WrBatch.objects.filter(**wrbatch_kwargs) diff --git a/src/caas/services/wrinstallnowservice.py b/src/caas/services/wrinstallnowservice.py new file mode 100644 index 0000000..4db0ec4 --- /dev/null +++ b/src/caas/services/wrinstallnowservice.py @@ -0,0 +1,107 @@ +import tempfile +import datetime +from ..models import Cluster, WrInstallSchedule, WrBatch, Namespace +from django.conf import settings +from .helpers import Address, Jinja +from .ansibleservice import AnsibleService +import logging + + +logger = logging.getLogger("caas") + + +class WrInstallNowService(AnsibleService): + """ + target == ilo_host_address, later becomes cluster_name + """ + def __init__(self, ansible_queue, target, git, branch, playbook, zmq): + super().__init__(ansible_queue, target, git, branch, playbook, zmq) + + def get_wrbatch(self): + if self.ansible_queue == "wr": + wrbatch_kwargs = {"ilo_host_address": self.target} + elif self.ansible_queue in ("wr_remediate", "wr_wipedisk", "wr_ptp", "wr_ptp_config", "wr_unlock", "wr_wrap"): + wrbatch_kwargs = {"cluster_name": self.target} + elif self.ansible_queue == "wr_reboot": + wrbatch_kwargs = {"namespace_name": self.target} + try: + wrbatch = WrBatch.objects.get(**wrbatch_kwargs) + except WrBatch.DoesNotExist: + logger.error("FAILED: while scheduling Wind River installation: cannot find wrbatch with identifier: %s" % self.target) + raise + return wrbatch + + def deploy(self): + logger.debug("%s Enter" % self.target) + wrbatch = self.get_wrbatch() + self.do_deploy(wrbatch) + + def remediate(self): + logger.debug("%s Enter" % self.target) + wrbatch = self.get_wrbatch() + self.do_deploy(wrbatch) + + def do_deploy(self, wrbatch): + try: + cluster_kwargs = {"pk": wrbatch.cluster_id} + cluster = Cluster.objects.get(**cluster_kwargs) + except Cluster.DoesNotExist: + logger.error("FAILED: while scheduling Wind River installation: cannot find cluster associated with wrbatch with identifier: %s" % wrbatch.ilo_host_address) + raise + + # one playbook per cluster + self.target = cluster.cluster_name + self.maint_window_p = wrbatch.maint_window_p + # host_vars file + hostvars_config = {"wr_vendor_name": wrbatch.vendor_name, + "wr_region_name": cluster.cluster_name, + "wr_bmc_controller_0_address": wrbatch.ilo_host_address, + "wr_mgmt_controller_0_mac": wrbatch.pxe_mac_address, + "wr_oam_floating_address": wrbatch.oam_vip_address, + "wr_oam_node_0_address": wrbatch.oam_host_address, + "wr_oam_gateway_address": wrbatch.oam_default_gateway, + "wr_mgmt_start_address": wrbatch.mgmt_address_range_start, + "wr_mgmt_end_address": wrbatch.mgmt_address_range_end, + "wr_mgmt_gateway_address": wrbatch.mgmt_default_gateway, + "wr_mgmt_subnet": Address.gateway_to_subnet(wrbatch.mgmt_subnet, numquads = 4), + "wr_central_mgmt_start_address": wrbatch.parent_mgmt_address_range_start, + "wr_central_mgmt_default_gateway": wrbatch.parent_mgmt_default_gateway, + "wr_central_mgmt_subnet": Address.gateway_to_subnet(wrbatch.parent_mgmt_subnet, numquads = 4), + "wr_host_vlan": wrbatch.host_vlan, + "wr_oam_vlan": wrbatch.oam_vlan, + "wr_mgmt_vlan": wrbatch.mgmt_vlan, + "wr_bmc_username": wrbatch.bmc_username, + "wr_bmc_password": wrbatch.bmc_password, + "wr_dns_servers": settings.DNS_SERVERS, + "wr_ntp_servers": settings.NTP_SERVERS, + "wr_registry_server": settings.WR_REGISTRY_SERVER, + "wr_registry_username": settings.WR_REGISTRY_USERNAME, + "wr_registry_password": settings.WR_REGISTRY_PASSWORD, + "wr_webdav_server": settings.WR_WEBDAV_SERVER, + "wr_webdav_username": settings.WR_WEBDAV_USERNAME, + "wr_webdav_password": settings.WR_WEBDAV_PASSWORD, + "wr_middleware_endpoint": settings.CAAS_MIDDLEWARE_ENDPOINT, + "wr_middleware_username": settings.CAAS_MIDDLEWARE_USERNAME, + "wr_middleware_password": settings.CAAS_MIDDLEWARE_PASSWORD, + "wr_docker_http_proxy": settings.WR_DOCKER_HTTP_PROXY, + "wr_docker_https_proxy": settings.WR_DOCKER_HTTPS_PROXY, + "wr_docker_no_proxy": settings.WR_DOCKER_NO_PROXY, + "wr_ssl_ca_cert": settings.WR_SSL_CA_CERT} + # revisit for NVPA-279 + # if len(servers) > 1: + # wr_config["wr_bmc_controller_1_address"] = servers[1].ilo_host_address + # wr_config["wr_mgmt_controller_1_mac"] = servers[1].pxe_mac_address + # wr_config["wr_oam_node_1_address"] = servers[1].oam_host_address + hostvars_content = Jinja.render("wr_host_vars.j2", **hostvars_config) + hostvars = wrbatch.oam_hostname + # inventory file + inventory_config = {"central_target": wrbatch.parent_cluster_name, + "central_target_ip": wrbatch.parent_oam_vip_address, + "remote_target": wrbatch.oam_hostname, + "remote_target_ip": wrbatch.oam_vip_address} + inventory_content = Jinja.render("wr_inventory.j2", **inventory_config) + inventory = "%s.yaml" % self.target + logger.debug("%s Sending playbook to queue" % self.target) + self.run_playbook(inventory, inventory_content, hostvars, hostvars_content, extra_args = {"clusterName": wrbatch.parent_cluster_name}) + logger.debug("%s Sent playbook to queue" % self.target) + return 0 diff --git a/src/caas/services/wrservice.py b/src/caas/services/wrservice.py new file mode 100644 index 0000000..51b3c02 --- /dev/null +++ b/src/caas/services/wrservice.py @@ -0,0 +1,75 @@ +import datetime +from ..models import Cluster, WrInstallSchedule, WrBatch +from django.conf import settings +from .helpers import Address, Jinja +from .ansibleservice import AnsibleService +from .icingapollservice import IcingaPollService +import logging + + +logger = logging.getLogger("caas") + + +class WrService(AnsibleService): + """ + target == ilo_host_address, later becomes cluster_name + """ + def __init__(self, target, git, branch, playbook, zmq): + super().__init__("wr", target, git, branch, playbook, zmq) + + def schedule_install(self): + logger.debug("%s Enter schedule_install" % self.target) + try: + wrbatch_kwargs = {"ilo_host_address": self.target} + wrbatch = WrBatch.objects.get(**wrbatch_kwargs) + except WrBatch.DoesNotExist: + logger.error("FAILED: while scheduling Wind River installation: cannot find wrbatch with ilo_host_address: %s" % self.target) + raise + try: + cluster_kwargs = {"pk": wrbatch.cluster_id} + cluster = Cluster.objects.get(**cluster_kwargs) + except Cluster.DoesNotExist: + logger.error("FAILED: while scheduling Wind River installation: cannot find cluster associated with wrbatch with ilo_host_address: %s" % wrbatch.ilo_host_address) + raise + wr_kwargs = {"cluster_id": cluster.pk} + wr, wr_created = WrInstallSchedule.objects.get_or_create(**wr_kwargs) + if wr.date_scheduled is None: + wr.date_scheduled = datetime.datetime.now() + if wr.kirke_ticket_number is None: + # TODO bealer: create KIRKE ticket + pass + wr.save() + + def deploy(self): + kirke_inprocess_kwargs = {"kirke_ticket_number__isnull": False, + "kirke_ticket_completed__isnull": True} + kirke_inprocess = WrInstallSchedule.objects.filter(**kirke_inprocess_kwargs) + + for rec in kirke_inprocess: + # TODO bealer: query KIRKE and update status + rec.save() + + wr_schedule_kwargs = {"kirke_ticket_completed__isnull": False, + "date_completed__isnull": True} + wr_schedule_ready = WrInstallSchedule.objects.filter(**wr_schedule_kwargs) + + for sched in wr_schedule_ready: + try: + wrbatch_kwargs = {"cluster_id": sched.cluster.pk, + "intel_nic_firmware_version": settings.INTEL_NIC_FIRMWARE_VERSION} + wrbatch_results = WrBatch.objects.filter(**wrbatch_kwargs) + + except WrBatch.DoesNotExist: + logger.error("FAILED: while running WR batch playbook: failed to look up wrbatch") + raise + + # check hardware before wr isntall + wrbatch = wrbatch_results[0] + icingapoll = IcingaPollService(wrbatch.ilo_host_address, + settings.ANSIBLE["icinga_poll"]["git"], + settings.ANSIBLE["icinga_poll"]["branch"], + settings.ANSIBLE["icinga_poll"]["playbook"], + settings.ANSIBLE["icinga_poll"]["zmq"], + "wrinstallnow") + icingapoll.deploy() + -- cgit v1.3