more filesHEADmaster

18 files changed, 2051 insertions, 0 deletions

diff --git a/doc/markdown/.gitignore b/doc/markdown/.gitignore
new file mode 100644
index 0000000..2d19fc7
--- /dev/null
+++ b/doc/markdown/.gitignore
@@ -0,0 +1 @@
+*.html
diff --git a/doc/markdown/automation-stack-architecture.md b/doc/markdown/automation-stack-architecture.md
new file mode 100644
index 0000000..2ae775c
--- /dev/null
+++ b/doc/markdown/automation-stack-architecture.md
@@ -0,0 +1,767 @@
+# Far-Edge Automation Stack Deployment Guide
+
+The Far-Edge automation stack consists of nine Ubuntu 18.04 hosts:
+
+- Two load balancers
+- Two middleware application hosts
+- Two ansible hosts
+- Three database hosts
+
+The provisioning of these hosts in openstack is outside the scope of
+this document. There will be a separate document to address this.
+
+There is an additional host in the production stack to handle LDAP
+dual-stack proxying to reach USWIN. This host does not live in the
+automation stack tenant space. It is a standalone VM managed by VCPe.
+
+## OpenStack tenant space
+
+In both production and MTCE the far-edge automation stack runs in an
+OpenStack tenant space. (Production runs in VCPe.) Creating the
+OpenStack objects (neutron networks, cinder volumes, instances) is
+beyond the scope of this document and will be documented separately.
+
+Secgroup rules must be created to allow network traffic. Use a script
+like the following to create the secgroup rules:
+
+    #!/bin/bash
+
+    private_network_cidr="2001:4888:a21:3102:245:29::/112"
+    public_network_cidrs="2001:4888::/32"
+
+    for name in default-grp; do
+        openstack security group show ${name} || openstack security group create --description "${name}" ${name}
+        for protocol in tcp udp; do
+            if [ "openstack security group show ${name} -f shell -c rules | grep -F 'ingress' | grep -F \"port_range_min='1'\" | grep -F \"port_range_max='65535'\" | grep -F \"${protocol}\" | grep -F 'IPv6'" == "" ]; then
+                openstack security group rule create --egress --protocol ${protocol} --src-ip ::/0 --dst-port 1:65535 --ethertype IPv6 ${name}
+            fi
+        done
+        for cidr in ${private_network_cidr} ${public_network_cidrs}; do
+            if [ "$(openstack security group show ${name} -f shell -c rules | grep -F \"icmp\" | grep -F \"remote_ip_prefix='${cidr}'\")" == "" ]; then
+                openstack security group rule create --ingress --protocol icmp --src-ip ${cidr} --ethertype IPv6 ${name}
+            fi
+        done
+        for port in 22; do
+            for cidr in ${private_network_cidr} ${public_network_cidrs}; do
+                if [ "$(openstack security group show ${name} -f shell -c rules | grep -F \"port_range_min='${port}'\" | grep -F \"remote_ip_prefix='${cidr}'\")" == "" ]; then
+                    openstack security group rule create --ingress --protocol tcp --dst-port ${port} --src-ip ${cidr} --ethertype IPv6 ${name}
+                fi
+            done
+        done
+    done
+
+    for name in icinga; do
+        openstack security group show ${name} || openstack security group create --description "${name}" ${name}
+        for port in 5665; do
+            for cidr in ${private_network_cidr}; do
+                if [ "$(openstack security group show ${name} -f shell -c rules | grep -F \"port_range_min='${port}'\" | grep -F \"remote_ip_prefix='${cidr}'\")" == "" ]; then
+                    openstack security group rule create --ingress --protocol tcp --dst-port ${port} --src-ip ${cidr} --ethertype IPv6 ${name}
+                fi
+            done
+        done
+    done
+
+    for name in ldap; do
+        openstack security group show ${name} || openstack security group create --description "${name}" ${name}
+        for port in 636; do
+            for cidr in ${private_network_cidr}; do
+                if [ "$(openstack security group show ${name} -f shell -c rules | grep -F \"port_range_min='${port}'\" | grep -F \"remote_ip_prefix='${cidr}'\")" == "" ]; then
+                    openstack security group rule create --ingress --protocol tcp --dst-port ${port} --src-ip ${cidr} --ethertype IPv6 ${name}
+                fi
+            done
+        done
+    done
+
+    for name in web; do
+        openstack security group show ${name} || openstack security group create --description "${name}" ${name}
+        for port in 443 3000; do
+            for cidr in ${private_network_cidr} ${public_network_cidrs}; do
+                if [ "$(openstack security group show ${name} -f shell -c rules | grep -F \"port_range_min='${port}'\" | grep -F \"remote_ip_prefix='${cidr}'\")" == "" ]; then
+                    openstack security group rule create --ingress --protocol tcp --dst-port ${port} --src-ip ${cidr} --ethertype IPv6 ${name}
+                fi
+            done
+        done
+        for port in 80 3000 3080 3128:3130 8000; do
+            for cidr in ${private_network_cidr}; do
+                if [ "$(openstack security group show ${name} -f shell -c rules | grep -F \"port_range_min='${port}'\" | grep -F \"remote_ip_prefix='${cidr}'\")" == "" ]; then
+                    openstack security group rule create --ingress --protocol tcp --dst-port ${port} --src-ip ${cidr} --ethertype IPv6 ${name}
+                fi
+            done
+        done
+    done
+
+    for name in galera; do
+        openstack security group show ${name} || openstack security group create --description "${name}" ${name}
+        for port in 3306 4444 4567 9200 13306; do
+            for cidr in ${private_network_cidr}; do
+                if [ "$(openstack security group show ${name} -f shell -c rules | grep -F \"port_range_min='${port}'\" | grep -F \"remote_ip_prefix='${cidr}'\")" == "" ]; then
+                    openstack security group rule create --ingress --protocol tcp --dst-port ${port} --src-ip ${cidr} --ethertype IPv6 ${name}
+                fi
+            done
+        done
+    done
+
+    for name in zmq; do
+        openstack security group show ${name} || openstack security group create --description "${name}" ${name}
+        for port in 5555:5559; do
+            for cidr in ${private_network_cidr}; do
+                if [ "$(openstack security group show ${name} -f shell -c rules | grep -F \"port_range_min='${port}'\" | grep -F \"remote_ip_prefix='${cidr}'\")" == "" ]; then
+                    openstack security group rule create --ingress --protocol tcp --dst-port ${port} --src-ip ${cidr} --ethertype IPv6 ${name}
+                fi
+            done
+        done
+    done
+
+    exit 0
+
+## Database
+
+```far_edge_ops_api``` and icinga2 require a database backend. We
+currently use postgresql. The goal is to migrate to
+galera/mariadb. For now we are using only one database host, but when
+we switch to galera we will use all three. The galera solution is
+dockerized. It is a mature and well-tested product of the VCP Metering
+project.
+
+### Installation
+
+Install postgresql on the first database host per the typical
+procedure that can be found on the internet. TODO: provide a link to
+an internet HOWTO that best fits our needs.
+
+#### postgresql.conf:
+
+    data_directory = '/var/lib/postgresql/12/data'
+    listen_addresses = '*'
+    port = 5432
+    max_connections = 2000
+    password_encryption = md5
+    ssl = off
+    shared_buffers = 2000MB
+    work_mem = 4MB
+    maintenance_work_mem = 256MB
+    dynamic_shared_memory_type = posix
+    wal_buffers = 2MB
+    checkpoint_timeout = 15min
+    max_wal_size = 1GB
+    min_wal_size = 80MB
+    checkpoint_completion_target = 0.9
+    logging_collector = on
+    log_directory = '/var/lib/postgresql/12_log/'
+    log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'
+    log_rotation_size = 50MB
+    log_min_duration_statement = 5
+    log_checkpoints = on
+    log_connections = on
+    log_disconnections = on
+    log_duration = on
+    log_error_verbosity = default
+    log_line_prefix = '%m, %d, %a. %r, %s, %x, %p '
+    log_timezone = 'GMT'
+    autovacuum = on
+    idle_in_transaction_session_timeout = 43200000
+    datestyle = 'iso, dmy'
+    timezone = 'GMT'
+    lc_messages = 'en_US.utf8'
+    lc_monetary = 'en_US.utf8'
+    lc_numeric = 'en_US.utf8'
+    lc_time = 'en_US.utf8'
+    default_text_search_config = 'pg_catalog.simple'
+    plperl.on_init = 'use utf8; use re; package utf8; require "utf8_heavy.pl";'
+
+#### pg_hba.conf:
+
+    local   icinga          icinga                                  md5
+    local   all             all                                     password
+    local   replication     postgres                                trust
+    host    icinga          icinga          127.0.0.1/32            md5
+    host    icinga          icinga          ::1/128                 md5
+    host    all             all             127.0.0.1/32            password
+    host    all             all             ::1/128                 password
+    host    icinga          icinga          2001:4888:a21:3102::/32 md5
+    host    all             all             2001:4888:a21:3102::/32 password
+
+#### sysctl.conf:
+
+Configure ```kernel.shmmax``` and ```kernel.shmall``` to be 25\% of
+available RAM. For example on a 16GB host:
+
+    kernel.shmmax=4199411712
+    kernel.shmall=4199411712
+
+### User and database creation
+
+As the postgres user:
+
+    # createuser -l -P -e faredge
+    # createdb -e -O faredge faredge
+
+We do not need to populate the database with a schema. Django will do
+this automatically.
+
+## Load Balancers
+
+The services which run on the loadbalancer hosts are as follows:
+
+- apache
+- haproxy
+- icinga2
+
+### Apache
+
+#### SSL Certificates:
+
+A prerequisite for configuring apache is an SSL x509 certificate that
+answers to the following names:
+
+    middleware.<fqdn>
+    icinga.<fqdn>
+    dns-admin.<fqdn>
+
+Place the certificate in ```/etc/ssl/certs/``` and the key in
+```/etc/ssl/private```.
+
+#### Modules:
+
+Apache needs the following modules enabled:
+
+    - access_compat.load
+    - alias.conf
+    - alias.load
+    - auth_basic.load
+    - authn_core.load
+    - authn_file.load
+    - authz_host.load
+    - authz_user.load
+    - authz_core.load
+    - autoindex.conf
+    - autoindex.load
+    - deflate.conf
+    - deflate.load
+    - dir.conf
+    - dir.load
+    - env.load
+    - filter.load
+    - http2.load
+    - mime.conf
+    - mime.load
+    - mpm_prefork.conf
+    - mpm_prefork.load
+    - negotiation.conf
+    - negotiation.load
+    - php7.2.conf
+    - php7.2.load
+    - proxy.conf
+    - proxy.load
+    - proxy_http.load
+    - proxy_http2.load
+    - proxy_wstunnel.load
+    - reqtimeout.conf
+    - reqtimeout.load
+    - rewrite.load
+    - setenvif.conf
+    - setenvif.load
+    - socache_shmcb.load
+    - ssl.conf
+    - ssl.load
+    - status.conf
+    - status.load
+
+#### VirtualHosts:
+
+Apache needs the following sites enabled:
+
+    - dns-admin.conf
+    - far_edge_ops_api.conf
+    - icinga.conf
+    - webdav.conf
+
+```webdav.conf``` is only needed when an external HTTP share is not
+available.
+
+##### dns-admin.conf:
+
+    <VirtualHost [2607:f160:b:10f1::e]:443>
+        ServerName dns-admin.faredge.vzwops.com
+        ServerAdmin carlos.konstanski@verizonwireless.com
+        Protocols http/1.1
+        SSLEngine on
+        SSLProxyEngine on
+        SSLCertificateFile /etc/ssl/certs/vcpfe-lb-vip.faredge.vzwops.com.pem
+        SSLCertificateKeyFile /etc/ssl/private/vcpfe-lb-vip.faredge.vzwops.com.key
+        SSLProtocol all -SSLv2 -SSLv3
+        SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+        SSLHonorCipherOrder on
+        SSLCompression off
+        SSLOptions +StrictRequire
+        ProxyPass "/" "http://[::1]:13000/"
+        ProxyPassReverse "/" "http://[::1]:13000/"
+        SetEnvIf User-Agent ".*MSIE.*" nokeepalive ssl-unclean-shutdown
+        ErrorLog "/var/log/apache2/dnsadmin_error_log"
+        CustomLog "/var/log/apache2/dnsadmin_access_log" common
+    </VirtualHost>
+
+##### far\_edge\_ops\_api.conf:
+
+    <VirtualHost [2607:f160:b:10f1::e]:443>
+        ServerName middleware.faredge.vzwops.com
+        ServerAdmin carlos.konstanski@verizonwireless.com
+        Protocols http/1.1
+        SSLEngine on
+        SSLProxyEngine on
+        SSLCertificateFile /etc/ssl/certs/vcpfe-lb-vip.faredge.vzwops.com.pem
+        SSLCertificateKeyFile /etc/ssl/private/vcpfe-lb-vip.faredge.vzwops.com.key
+        SSLProtocol all -SSLv2 -SSLv3
+        SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+        SSLHonorCipherOrder on
+        SSLCompression off
+        SSLOptions +StrictRequire
+        ProxyPass "/" "http://[::1]:13080/"
+        ProxyPassReverse "/" "http://[::1]:13080/"
+        ErrorLog "/var/log/apache2/middleware_error_log"
+        CustomLog "/var/log/apache2/middleware_access_log" common
+    </VirtualHost>
+
+##### icinga.conf:
+
+    <VirtualHost [2607:f160:b:10f1::e]:443>
+        ServerName icinga.faredge.vzwops.com
+        ServerAdmin carlos.konstanski@verizonwireless.com
+        DocumentRoot "/usr/share/icingaweb2/public"
+        SSLEngine on
+        SSLProxyEngine on
+        SSLCertificateFile /etc/ssl/certs/vcpfe-lb-vip.faredge.vzwops.com.pem
+        SSLCertificateKeyFile /etc/ssl/private/vcpfe-lb-vip.faredge.vzwops.com.key
+        SSLProtocol all -SSLv2 -SSLv3
+        SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+        SSLHonorCipherOrder on
+        SSLCompression off
+        SSLOptions +StrictRequire
+
+        <Directory "/usr/share/icingaweb2/public">
+            Options SymLinksIfOwnerMatch
+            AllowOverride None
+
+            <IfModule mod_authz_core.c>
+                <RequireAll>
+                    Require all granted
+                </RequireAll>
+            </IfModule>
+
+            SetEnv ICINGAWEB_CONFIGDIR "/etc/icingaweb2"
+            EnableSendfile Off
+
+            <IfModule mod_rewrite.c>
+                RewriteEngine on
+                RewriteBase /
+                RewriteCond %{REQUEST_FILENAME} -s [OR]
+                RewriteCond %{REQUEST_FILENAME} -l [OR]
+                RewriteCond %{REQUEST_FILENAME} -d
+                RewriteRule ^.*$ - [NC,L]
+                RewriteRule ^.*$ index.php [NC,L]
+            </IfModule>
+        </Directory>
+    </VirtualHost>
+
+##### webdav.conf:
+
+    <VirtualHost *:80>
+        ServerName webdav.faredge.vzwops.com
+        ServerAdmin carlos.konstanski@verizonwireless.com
+        DocumentRoot "/var/www/html/webdav"
+        SetEnvIf User-Agent ".*MSIE.*" nokeepalive ssl-unclean-shutdown
+        ErrorLog "/var/log/apache2/webdav_error_log"
+        CustomLog "/var/log/apache2/webdav_access_log" common
+        <Directory "/var/www/html/webdav">
+            AllowOverride limit
+            Options Indexes FollowSymLinks
+        </Directory>
+    </VirtualHost>
+
+##### Webdav directory:
+
+Create this directory only if you don't have access to an external HTTP share and you are using the webdav VirtualHost above:
+
+    - /var/www/html/webdav
+
+### HAProxy
+
+Use the following haproxy config, adjusting IP addresses, hostnames and credentials as needed:
+
+    global
+        tune.ssl.default-dh-param 2048
+        log /dev/log local0
+        log /dev/log local1 notice
+        chroot /var/lib/haproxy
+        stats socket /run/haproxy/admin.sock mode 660 level admin
+        stats timeout 30s
+        maxconn 5000
+        user haproxy
+        group haproxy
+        daemon
+        ca-base /etc/ssl/certs
+        crt-base /etc/ssl/private
+        ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
+        ssl-default-bind-options no-sslv3
+
+    defaults
+        log global
+        option dontlognull
+        timeout connect 10s
+        timeout client 60s
+        timeout server 180s
+        errorfile 400 /etc/haproxy/errors/400.http
+        errorfile 403 /etc/haproxy/errors/403.http
+        errorfile 408 /etc/haproxy/errors/408.http
+        errorfile 500 /etc/haproxy/errors/500.http
+        errorfile 502 /etc/haproxy/errors/502.http
+        errorfile 503 /etc/haproxy/errors/503.http
+        errorfile 504 /etc/haproxy/errors/504.http
+
+    listen stats
+        bind 127.0.0.1:1993
+        mode http
+        option httplog
+        option forwardfor
+        stats enable
+        stats uri /stats
+        stats auth someuser:somepass
+        stats refresh 30s
+        stats show-node
+        stats hide-version
+        log global
+
+    frontend dns-admin
+        bind 0.0.0.0:13000
+        bind :::13000
+        mode http
+        option httplog
+        option forwardfor
+        default_backend dns-admin-backend
+
+    backend dns-admin-backend
+        balance roundrobin
+        mode http
+        option log-health-checks
+        option httpchk GET /dns HTTP/1.1\r\nHost:\ dns-admin.faredge.vzwops.com\r\nUser-Agent:\ curl/7.35.0\r\nAccept:\ */*\r\nAuthorization:\ Basic\ b3BlbmJvb2s6WnlsejNmZE1TZDhTVmlod0w5TnlaTHpU
+        default-server inter 10s fall 1 rise 1
+        server server-1 [2607:f160:b:10f1::7]:80 check
+        #server server-2 [2607:f160:b:10f1::8]:80 check
+
+    frontend far_edge_ops_api
+        bind 0.0.0.0:13080
+        bind :::13080
+        mode http
+        option httplog
+        option forwardfor
+        reqdel ^Host:
+        reqadd Host:\ middleware.faredge.vzwops.com
+        default_backend far_edge_ops_api-backend
+
+    backend far_edge_ops_api-backend
+        balance roundrobin
+        mode http
+        option log-health-checks
+        option httpchk GET /caas/ HTTP/1.1\r\nHost:\ middleware.faredge.vzwops.com\r\nUser-Agent:\ curl/7.35.0\r\nAccept:\ */*\r\nAuthorization:\ Basic\ bWlkZGxld2FyZXVzZXI6bWlkZGxld2FyZXBhc3N3b3Jk
+        default-server inter 10s fall 1 rise 1
+        server server-1 [2607:f160:b:10f1::6]:3080 check
+        #server server-2 [2607:f160:b:10f1::5]:3080 check
+
+We currently only proxy to one backend for both ```dns-admin``` and
+```far_edge_ops_api``` because there has been no opportunity to test
+the behavior of the apps in a multiple-backend configuration. This is
+why ```server-2``` is commented out. TODO: update this document when
+this testing is complete.
+
+### Icinga2
+
+The icinga2 configuration is mostly standard. TODO: provide a link to
+an internet HOWTO from which the rest of this documentation can
+build. There will be only minor additions to the standard base
+install.
+
+## Jenkins
+
+The applications are built and deployed from jenkins. The build jobs
+already exist. But a new deploy job needs to be added for the new
+environment which you are building.
+
+### env.yaml
+
+All of the far-edge builds and deploymenets are run on the jenkins
+slave ```openbook-docker-slave-4.meter.vzwops.com```. Log onto this
+host and edit three files in the jenkins home directory
+```/var/lib/jenkins/```:
+
+    - env.yaml.ansible-queue
+    - env.yaml.dns-admin
+    - env.yaml.far-edge-ops-api
+
+A new deployment target section needs to be added to each of these
+files. Give the environment a new unique name and use the same name in
+all three files.
+
+### jenkins-job-builder (jjb)
+
+Clone the following git repo onto your laptop:
+
+    git@gitlab.verizon.com:ONV6661_VCPSPUBLIC/jjb.git
+
+Create an ini file called ```verizoncloudplatform.com.ini``` in the
+top-level directory of your jjb git checkout with the following
+contents:
+
+#### verizoncloudplatform.com.ini:
+    [job_builder]
+    ignore_cache=True
+    keep_descriptions=False
+    include_path=.:scripts:~/git/
+    recursive=False
+    exclude=.*:manual:./development
+    allow_duplicates=False
+
+    [jenkins]
+    user=<your_jenkins_username>
+    password=<your_jenkins_api_token>
+    url=https://openbook-jenkins-master.meter.vzwops.com:8080/
+    timeout=120
+    query_plugins_info=False
+
+Follow these steps to create the new deploy jobs:
+
+- Open the file ```yaml/verizoncloudplatform.com/deploy-vcpfe.yaml```
+  in an editor.
+
+- Add three new jobs to the list, one for each app
+  (```ansible-queue```, ```dns-admin``` and
+  ```far-edge-ops-api```). Set the target attribute to the new ansible
+  target which you created in env.yaml in the previous section.
+
+- ```# ./run.sh verizoncloudplatform.com```
+
+Once this is done, log into the jenkins UI to verify that the new jobs
+were created.
+
+Push your jjb changes to git. (The ini file is gitignored; it is your
+private file.)
+
+## Application hosts
+
+The services which run on the application hosts are as follows:
+
+    - far_edge_ops_api
+
+### far\_edge\_ops\_api
+
+The middleware (as ```far_edge_ops_api``` is ubiquitously known) is a
+docker containerized django web application.
+
+#### Build/deploy
+
+The application can be built and deployed from jenkins:
+
+https://openbook-jenkins-master.meter.vzwops.com:8080
+
+Run the job ```docker-container-far-edge-ops-api``` to build the
+docker container, and the appropriate
+```far-edge-ops-api-deploy-vcpfe-<env>``` job to deploy it to an
+environment.
+
+#### Configure
+
+It might seem backwards to perform the build and deployment before
+configuring. It's a chicken-and-egg problem: the deployment needs to
+create the required files and directories before configuration can
+commence. For first-time deployments the process will look like: build
+-> deploy -> configure -> deploy. This is true of all three
+applications.
+
+The main configuration directory is ```/etc/far-edge-ops-api/```. It
+looks like the following on a working system:
+
+    $ tree /etc/far-edge-ops-api/
+    /etc/far-edge-ops-api/
+    ├── modules.d
+    │   ├── wsgi.conf
+    │   └── wsgi.load
+    ├── settings.py
+    └── vhosts.d
+        └── far-edge-ops-api.conf
+
+```settings.py``` is a rather long file, too long to include here. Get
+a copy from a known good source (a production or lab server) and edit
+to suit the new environment.
+
+```far-edge-ops-api.conf``` is a rather typical-looking apache
+VirtualHosts file, but with LDAP auth added. First the file:
+
+##### far-edge-ops-api.conf:
+
+    LDAPTrustedGlobalCert CA_BASE64 "/etc/apache2/auth/uswin_ca.pem"
+    LDAPTrustedGlobalCert CA_BASE64 "/etc/ssl/certs/selfsigned_root_cert.crt"
+    LDAPTrustedMode SSL
+    LDAPLibraryDebug 7
+
+    <VirtualHost *:3080>
+        ServerName middleware.<fqdn>
+        DocumentRoot "/opt/far-edge-ops-api/src/src"
+        WSGIScriptAlias / /opt/far-edge-ops-api/src/src/far_edge_ops_api/wsgi.py
+        WSGIDaemonProcess middleware.<fqdn> processes=20 threads=15 display-name=%{GROUP} python-home=/opt/far-edge-ops-api/src/venv
+        WSGIProcessGroup middleware.<fqdn>
+        Alias /static/ /opt/far-edge-ops-api/src/src/static/
+        TimeOut 600
+        SetEnvIf User-Agent ".*MSIE.*" nokeepalive ssl-unclean-shutdown
+        ErrorLog "/var/log/apache2/far-edge-ops-api_error_log"
+        CustomLog "/var/log/apache2/far-edge-ops-api_access_log" common
+    </VirtualHost>
+
+    <Directory "/opt/far-edge-ops-api/src/src">
+        AllowOverride all
+        Options FollowSymlinks
+        AuthName "USWIN"
+        AuthType Basic
+        AuthBasicProvider ldap
+        AuthLDAPURL "ldaps://vcpfe-ldap-prod.vcpfe.vzwops.com:636/DC=uswin,DC=ad,DC=vzwcorp,DC=com?sAMAccountName?sub?(objectclass=*)"
+        AuthLDAPBindDN "CN=SVC-VCP-LDAP,OU=SVC,OU=FNA,DC=uswin,DC=ad,DC=vzwcorp,DC=com"
+        AuthLDAPBindPassword "<password_for_SVC-VCP-LDAP_account>"
+        Require valid-user
+    </Directory>
+
+    <Directory "/opt/far-edge-ops-api/src/src/static">
+        AllowOverride limit
+        Require all granted
+        Options Indexes FollowSymLinks
+    </Directory>
+
+There are two SSL certs because the far-edge stack is IPv6-only while
+the production LDAP endpoint (USWIN) is IPv4-only. So we must use a
+proxy. The first cert is the actual uswin cert, while the second cert
+is the one for the proxy. Both of these certs must be placed on the
+app hosts.
+
+Replace ```<fqdn>``` with the DNS domain. Replace
+```<password_for_SVC-VCP-LDAP_account>``` with the correct password.
+
+OpenLDAP needs to be configured with these same certs. Since they are
+self-signed, openldap will reject them unless they are listed as
+exceptions.
+
+#### /etc/ldap/ldap.conf:
+
+    TLS_CACERT /etc/apache2/auth/uswin_ca.pem
+    LDAPTLS_CACERT /etc/ssl/certs/self_ca_signed_cert_and_key_2.pem
+
+Now that the configuration is complete, rerun the deployment from
+jenkins. This time the application should start normally.
+
+## Queue hosts
+
+The services which run on the application hosts are as follows:
+
+    - ansible-queue
+    - dns-admin
+
+### ansible-queue
+
+```ansible-queue``` is a docker containerized common lisp
+application.
+
+#### Build/deploy
+
+The application can be built and deployed from jenkins:
+
+https://openbook-jenkins-master.meter.vzwops.com:8080
+
+Run the job ```docker-container-ansible-queue``` to build the docker
+container, and the appropriate ```ansible-queue-deploy-vcpfe-<env>```
+job to deploy it to an environment.
+
+#### Configure
+
+Two files are required in the directory ```/etc/ansible-queue/```:
+
+    - .ansible_pass.far_edge
+    - options.lisp
+
+```.ansible_pass.far_edge``` is the ansible vault password
+file. Obtain a copy from an existing queue host.
+
+```options.lisp``` is the config file for the ```ansible-queue```
+application.
+
+##### options.lisp:
+
+    ((:queue (:num-icinga-process-threads 1
+              :num-bmc-process-threads 30
+              :num-nic-process-threads 20
+              :num-wr-process-threads 25
+              :num-dns-process-threads 1
+              :num-patch-process-threads 1
+              :num-hw-process-threads 20
+              :num-fix-process-threads 20
+              :icinga-wait-interval 0
+              :bmc-wait-interval 1
+              :nic-wait-interval 30
+              :wr-wait-interval 0
+              :dns-wait-interval 0
+              :patch-wait-interval 30
+              :hw-wait-interval 1
+              :fix-wait-interval 1
+              :sleep-interval 60
+              :ipv6 t
+              :vault-password-file "/etc/ansible-queue/.ansible_pass.far_edge"
+              :icinga-queue-url "tcp://*:5555"
+              :bmc-queue-url "tcp://*:5556"
+              :nic-queue-url "tcp://*:5557"
+              :wr-queue-url "tcp://*:5558"
+              :dns-queue-url "tcp://*:5559"
+              :patch-queue-url "tcp://*:5560"
+              :hw-queue-url "tcp://*:5561"
+              :fix-queue-url "tcp://*:5562"
+              :middleware-url "https://middleware.vcpfe.vzwops.com"
+              :middleware-username "SVC-Far-Edge"
+              :middleware-password "<password_for_SVC-Far-Edge_account>")))
+
+Replace ```<password_for_SVC-Far-Edge_account>``` with the correct
+password.
+
+Now that the configuration is complete, rerun the deployment from
+jenkins. This time the application should start normally.
+
+### dns-admin
+
+```dns-admin``` is a docker containerized web application written in
+common lisp and clojurescript.
+
+#### Build/deploy
+
+The application can be built and deployed from jenkins:
+
+https://openbook-jenkins-master.meter.vzwops.com:8080
+
+Run the job ```docker-container-dns-admin``` to build the docker
+container, and the appropriate ```dns-admin-deploy-vcpfe-<env>``` job
+to deploy it to an environment.
+
+#### Configure
+
+Only one new config file is required. Use the following content
+verbatim:
+
+##### /etc/dns-admin/conf.lisp:
+
+    (:name "dns-admin"
+     :document-root "dns-admin"
+     :title "DNS Administration"
+     :meta-description "A website for administering DNS. Supports nsupdate and infoblox."
+     :dns (:label "CDS Infoblox"
+           :backend-type "infoblox"
+           :url "https://infoblox.alias/wapi/v2.6"))
+
+Notice the odd-looking hostname ```infoblox.alias```. The following
+needs to be added to ```/etc/hosts```:
+
+    2607:f160:8a02:8016:a0:103:0:10 infoblox.alias
+
+Now that the configuration is complete, rerun the deployment from
+jenkins. This time the application should start normally.
diff --git a/doc/markdown/subcloud-install-remediation.md b/doc/markdown/subcloud-install-remediation.md
new file mode 100644
index 0000000..753c5fb
--- /dev/null
+++ b/doc/markdown/subcloud-install-remediation.md
@@ -0,0 +1,253 @@
+# Far-Edge Subcloud Installation and Remediation Guide
+
+## Installation
+
+A Wind River subcloud installation normally runs with no manual
+intervention. In the event that an installation needs to be started
+manually:
+
+- Visit https://middleware.vcpfe.vzwops.com/caas/wrfix/ in a browser.
+
+- Enter the iLO IP address into the first text field (even though it's
+  labeled "cluster name").
+
+- Type ```wr``` in the playbook key field.
+
+- Click the submit button.
+
+If an OK message appears, the job has been successfully sent to
+ansible-queue. The remaining steps require an SSH shell on
+```root@vcpfe-queue-2-prod-birmingham```:
+
+- View ```/var/log/lisp/ansible-queue.log``` to ensure that the
+  playbook started.
+
+- View the specific ansible log in ```/var/log/ansible/queue/``` to
+  watch the playbook's progress.
+
+If you want to watch the OS installation (the first stage of the
+overall deployment), attach to the iLO console:
+
+- Find the subcloud data (see below for the query) and find the
+  ```ilo_host_address``` and ```bmc_password```.
+
+- ```# ssh bladecenter-lb-vip.meter.vzwops.com```
+
+- ```# do-bmc <ilo_host_address>```
+
+- Enter the password when prompted.
+
+- Type ```vsp``` to start the serial console.
+
+## Remediation
+
+All too often it is necessary to remediate failed Wind River
+installations. Even successful ones need a visit by an engineer to
+ensure that they are in fact complete. It is not possible to discover
+this information from the middleware database, the subcloud, nor the
+central controller alone. All three of these data sources must be
+accessed. TODO: write a dashboard which aggregates the required
+information from the central controller, subcloud and database.
+
+Common causes of failure (this is not an exhaustive list):
+
+- OAM network unreachable.
+- MGMT network unreachable.
+- MGMT subnet overlaps with another already-installed subcloud.
+- The subcloud does not sync in time after coming online.
+- The platform-integ-apps application fails to apply.
+
+(This does not include servers that have issues which prevent them
+from advancing through the automation pipeline far enough to begin a
+Wind River installation. These include, but are not limited to,
+hardware problems and incorrect firmware versions.)
+
+### Inspecting a subcloud
+
+Three data sources were listed above. Let's visit each one in detail.
+
+#### Middleware database
+
+The middlware database is our record of what should be installed and
+whether or not we have marked it as complete. The ```caas_wrbatch```
+view and ```caas_wrinstallschedule``` table are important in this
+context.
+
+Here is a handy query to pull up all relevant information by cluster
+name:
+
+    select b.cluster_name || '.vcpfe.vzwops.com' as cluster_name,
+      b.*,
+      s.*
+    from caas_wrbatch b
+    left outer join caas_wrinstallschedule s
+      on b.cluster_id = s.cluster_id
+    inner join caas_location l
+      on b.fuze_spm_site_id = l.fuze_spm_site_id
+    where b.cluster_name in ('<cluster_name>')
+    order by b.parent_cluster_name asc,
+      b.cluster_name asc;
+
+(The expression in the first column is a convenience: it builds a FQDN
+for the subcloud that can be directly used in an SSH command from your
+laptop.)
+
+Here is a sample record showing a completed subcloud:
+
+    cluster_name                    | wsbomagj-d663366-001.vcpfe.vzwops.com
+    id                              | 423
+    cluster_id                      | 423
+    fuze_spm_site_id                | 663366
+    parent_oam_vip_hostname         | 
+    parent_oam_vip_address          | 2001:4888:a12:3221:106:290:0:10
+    parent_mgmt_address_range_start | 2001:4888:a12:3220:106:290:0:10
+    parent_mgmt_address_range_end   | 2001:4888:a12:3220:106:290:0:ffff
+    parent_mgmt_default_gateway     | 2001:4888:a12:3220:106:28::
+    parent_mgmt_subnet              | 2001:4888:a12:3220::
+    pxe_mac_address                 | 48:df:37:e2:67:34
+    ilo_hostname                    | wsbomagj-663366-rh-pe0e910-001
+    ilo_host_address                | 2001:4888:2a10:90fd:103:40a:0:e001
+    oam_hostname                    | wsbomagj-663366-rh-pe2e910-001
+    oam_host_address                | 2001:4888:2a10:90fd:103:40a:0:400
+    oam_vip_address                 | 2001:4888:2a10:90fd:103:40a:0:f400
+    oam_default_gateway             | 2001:4888:2a10:90fd:103:2a0::
+    mgmt_address_range_start        | 2001:4888:2a10:982b:103:40a::
+    mgmt_address_range_end          | 2001:4888:2a10:982b:103:40a:0:f
+    mgmt_default_gateway            | 2001:4888:2a10:982b:103:2a0::
+    mgmt_subnet                     | 2001:4888:2a10:982b:103:40a::
+    host_vlan                       | 2845
+    oam_vlan                        | 410
+    mgmt_vlan                       | 420
+    cluster_name                    | wsbomagj-d663366-001
+    parent_cluster_name             | wsbomagj-c319918-003
+    vendor_name                     | HPE
+    intel_nic_firmware_version      | 1.2585.0
+    maint_window_p                  | f
+    namespace_id                    | 425
+    namespace_name                  | wsbomagj-663366vzwcvdu-y-ss-x-05690012222
+    bmc_username                    | k8sctl
+    bmc_password                    | <redacted>
+    id                              | 709
+    date_scheduled                  | 2020-11-19 19:15:16.885864+00
+    date_completed                  | 2020-12-04 19:28:17.644994+00
+    date_last_failed                | 2020-12-03 20:47:18.199344+00
+    kirke_ticket_number             | 709
+    kirke_ticket_status             | 
+    kirke_ticket_completed          | 2020-11-20 02:28:30.908054+00
+    cluster_id                      | 423
+
+The key points:
+
+- ```pxe_mac_address``` is not null
+- ```intel_nic_firmware_version``` == 1.2585.0
+- ```date_completed``` is not null
+- ```kirke_ticket_number``` is not null
+
+The last point is the crucial one. Setting ```kirke_ticket_number```
+to something other than null is the way we manually mark the Wind
+River installation as complete. It must be done with a query similar
+to the following:
+
+    update caas_wrinstallschedule
+    set kirke_ticket_number = id
+    where id = 709;
+
+#### Central controller
+
+Use the following command to see the status of every subcloud attached
+to a central controller:
+
+    dcmanager subcloud list
+
+If the subcloud in question is complete and healthy from the central
+controller's point of view, it will look like the following. This does
+not necessarily mean that all is well. The subcloud itself must be
+visited and checked.
+
+    [sysadmin@controller-0 ~(keystone_admin)]$ dcmanager subcloud list | grep wsbomagj-d663366-001
+    | 546 | wsbomagj-d663366-001    | managed    | online       | complete      | in-sync |
+
+#### Subcloud
+
+If the central controller says that the subcloud is offline, the first
+thing to check is the mgmt network. If you cannot ping the central
+controller's mgmt IP, there is no hope for remediating the subcloud.
+
+    ping6 <parent_mgmt_address_range_start>
+
+If the subcloud is managed, online and in-sync, then the remaining
+checks can be performed. Check the alarms:
+
+    fm alarm-list
+
+Check the applied status of the applications:
+
+    [sysadmin@controller-0 ~(keystone_admin)]$ system application-list 
+    +--------------------------+----------+-----------------------------------+------------------------------------+---------+-----------+
+    | application              | version  | manifest name                     | manifest file                      | status  | progress  |
+    +--------------------------+----------+-----------------------------------+------------------------------------+---------+-----------+
+    | cert-manager             | 20.06-4  | cert-manager-manifest             | certmanager-manifest.yaml          | applied | completed |
+    | nginx-ingress-controller | 20.06-0  | nginx-ingress-controller-manifest | nginx_ingress_controller_manifest. | applied | completed |
+    |                          |          |                                   | yaml                               |         |           |
+    |                          |          |                                   |                                    |         |           |
+    | oidc-auth-apps           | 20.06-26 | oidc-auth-manifest                | manifest.yaml                      | applied | completed |
+    | platform-integ-apps      | 20.06-9  | platform-integration-manifest     | manifest.yaml                      | applied | completed |
+    +--------------------------+----------+-----------------------------------+------------------------------------+---------+-----------+
+
+Check the drbd resize. drbd8 should be set to 32GB:
+
+    [sysadmin@controller-0 ~(keystone_admin)]$ lsblk | grep drbd8
+      │ └─drbd8                         147:8    0   32G  0 disk /var/lib/docker-distribution
+
+Check the PTP configuration. ```tx_timestamp_timeout``` should be set
+to 50:
+
+    [sysadmin@controller-0 ~(keystone_admin)]$ grep tx_timestamp_timeout /etc/ptp4l.conf 
+    tx_timestamp_timeout 50
+
+Check the ISOL CPU configuration which should be enabled on cores 3,
+4, 27 and 28:
+
+    [sysadmin@controller-0 ~(keystone_admin)]$ system host-cpu-list controller-0 | grep -F 'Application-isolated'
+    | 72499a12-e2f2-40fc-a069-a2b002908843 | 3     | 0         | 3     | 0      | Intel(R) Xeon(R) Gold 6212U CPU @ 2.40GHz | Application-isolated |
+    | 8498669d-9048-45f6-b9dd-c81262934b70 | 4     | 0         | 4     | 0      | Intel(R) Xeon(R) Gold 6212U CPU @ 2.40GHz | Application-isolated |
+    | 4a70893f-b2cf-4405-9b69-11ed823a81b3 | 27    | 0         | 3     | 1      | Intel(R) Xeon(R) Gold 6212U CPU @ 2.40GHz | Application-isolated |
+    | 1d6e330d-7d2f-416c-b375-9b909d6a1f3c | 28    | 0         | 4     | 1      | Intel(R) Xeon(R) Gold 6212U CPU @ 2.40GHz | Application-isolated |
+
+### Marking an installation as complete
+
+If the subcloud passes all of the checks, it should be marked complete
+with the update query:
+
+    update caas_wrinstallschedule
+    set kirke_ticket_number = id
+    where id = <id>;
+
+### Remediating
+
+If the subcloud fails to pass all of the checks, it must be
+remediated. Deciding which steps to take requires operational
+experience. Here is a guide to some common problems and solutions:
+
+- The subcloud is unmanaged: ping the central controller mgmt IP. If
+  it fails, the network is misconfigured and must be fixed by the
+  region.
+
+- ```platform-integ-apps``` is not applied: perform a wipedisk and
+  reinstall.
+
+- ```oidc-auth-apps``` is uploaded but not applied: run the
+  ```wr_remediate``` playbook.
+
+- The drbd resize hasn't happened: run the ```wr_remediate```
+  playbook.
+
+- PTP isn't configured: run the ```wr_remediate``` playbook.
+
+- ISOL CPUS are not configured: run the ```wr_remediate``` playbook.
+
+The very last thing that the ```wr-installer``` playbook does is a
+```collect all```. You can tell if the playbook ran to completion by
+checking the contents of the ```/scratch``` directory on the
+subcloud. If it contains a collect tarball, the playbook completed.
+
diff --git a/doc/swagger/restapi-swagger.yaml b/doc/swagger/restapi-swagger.yaml
new file mode 100644
index 0000000..b0a2752
--- /dev/null
+++ b/doc/swagger/restapi-swagger.yaml
@@ -0,0 +1,93 @@
+swagger: '2.0'
+info:
+  description: FarEdge REST API
+  version: 0.0.1
+  title: FarEdge REST API
+  contact:
+    email: subin.ren@verizonwireless.com
+host: faredge.restapi.io
+basePath: /v1
+tags:
+- name: "CAAS"
+  description: "cluster status API"
+schemes:
+  - https
+paths:
+  /caas-status:
+    post:
+      tags:
+        - CAAS
+      summary: Send cluster status to VMB
+      description: Send cluster status to VMB
+      operationId: clusterStatus
+      consumes:
+        - application/json
+      produces:
+        - application/json
+      parameters:
+        - name: api_key
+          in: header
+          required: true
+          type: string
+        - in: body
+          name: body
+          description: caas readyness status
+          required: true
+          schema:
+            $ref: '#/definitions/CaaSStatusMap'
+      responses:
+        '200':
+          description: "successful operation"
+        '403':
+          description: Forbidden
+        '405':
+          description: Invalid input
+securityDefinitions:
+  api_key:
+    type: apiKey
+    name: api_key
+    in: header
+definitions:
+  CaaSStatusMap:
+    type: object
+    properties:
+      cluster_status:
+        type: array
+        items:
+          $ref: "#/definitions/Status"
+  Status:
+    type: object
+    required:
+      - name
+      - location
+      - software_version
+      - availability
+      - deploy_status
+      - created_at
+    properties:
+      name:
+        type: string
+        example: wsbomagj-d654321-001
+      description:
+        type: string
+        example: NE CONCORD 8_N
+      location:
+        type: string
+        example: 654321
+      software_version:
+        type: string
+        example: 20.06       
+      availability:
+        type: string
+        enum:
+          - ONLINE
+      deploy_status:
+        type: string
+        enum:
+          - COMPLETE
+      created_at:
+        type: string
+        example: "2020-01-07 04:16:15.743617"
+      updated_at:
+        type: string
+        example: "2020-01-07 06:03:10.854598"
diff --git a/doc/tex/break-fix/.gitignore b/doc/tex/break-fix/.gitignore
new file mode 100644
index 0000000..99eff73
--- /dev/null
+++ b/doc/tex/break-fix/.gitignore
@@ -0,0 +1,9 @@
+*.aux
+*.bbl
+*.blg
+*.log
+*.nav
+*.out
+*.pdf
+*.snm
+*.toc
diff --git a/doc/tex/break-fix/compile.sh b/doc/tex/break-fix/compile.sh
new file mode 100755
index 0000000..6e2d731
--- /dev/null
+++ b/doc/tex/break-fix/compile.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+rm *-blx.bib *.aux *.bbl *.bcf *.blg *.log *.nav *.out *.run.xml *.snm *.toc
+file="$(ls *.tex | awk -F. '{print $1}')"
+pdflatex ${file}
+exit 0
diff --git a/doc/tex/break-fix/slides.tex b/doc/tex/break-fix/slides.tex
new file mode 100644
index 0000000..a3a0215
--- /dev/null
+++ b/doc/tex/break-fix/slides.tex
@@ -0,0 +1,82 @@
+% To make a PDF from this source, use the pdflatex command.
+\documentclass[helvetica,english,utf8,notitle,nologo,8pt]{beamer}
+\usetheme{boxes}
+\usecolortheme{seahorse}
+\usepackage[american]{babel}
+\usepackage[babel]{csquotes}
+
+\begin{document}
+
+\title{Wind River Subcloud Break-Fix Procedure}
+\author{Carlos Konstanski\\
+Far-Edge Engineering/Operations}
+
+\frame{\titlepage}
+
+\begin{frame}
+  \frametitle{Introduction}
+
+  This document will describe the steps required to replace a Wind
+  River subcloud server, specifically the portion of the process that
+  is handled by Far-Edge Engineering/Operations. This group is
+  concerned with the installation of Wind River CaaS and the vDU
+  workloads. It is assumed that the prerequisite steps have already
+  been completed. These include but are not necessarily limited to:
+
+  \begin{itemize}
+  \item Physical server racking
+  \item Network cabling
+  \item Capturing the serial number and iLO password from the sticker
+    on the bottom of the blade
+  \item Capturing the iLO IPv6 address
+  \end{itemize}
+\end{frame}
+
+\begin{frame}
+  \frametitle{What Eng/Ops Needs in the Handoff}
+
+  Once a new server has been racked and cabled, Far-Edge Eng/Ops needs
+  to be notified so that we can provision the new host. We require
+  three pieces of information in order to do this:
+
+  \begin{enumerate}
+  \item The iLO IPv6 address. This is the key for looking up the
+    remainder of the server information in our system.
+  \item The iLO password. It is printed on a sticker on the bottom of
+    the blade.
+  \item The blade serial number. It is printed on the same sticker.
+  \end{enumerate}
+
+  Workbench 360 has been proposed as the notification channel.
+\end{frame}
+
+\begin{frame}
+  \frametitle{How to Procure the Subcloud Information}
+
+  An interim web UI page has been set up to query server information
+  based on the FUZE ID:
+
+  https://middleware.vcpfe.vzwops.com/caas/subcloud/
+\end{frame}
+
+\begin{frame}
+  \frametitle{What Eng/Ops Does Next}
+
+  We will watch the automation pipeline to ensure that the server
+  completes the zero-touch provisioning process successfully.
+
+  \begin{itemize}
+  \item On the central controller: dcmanager subcloud unmanage cluster\_name
+  \item On the central controller: dcmanager subcloud delete cluster\_name
+  \item Disassociate the server record from the old blade record in
+    the middleware database.
+  \item Ingest the new blade info (serial number and password).
+  \item Update icinga to pick up the new model\_check service which
+    will relink the server to the new blade.
+  \end{itemize}
+
+  At this point the server will enter the automation pipeline and
+  proceed just like a normal installation.
+\end{frame}
+
+\end{document}
diff --git a/doc/uml/.gitignore b/doc/uml/.gitignore
new file mode 100644
index 0000000..fb59d0f
--- /dev/null
+++ b/doc/uml/.gitignore
@@ -0,0 +1,6 @@
+hardware_install_activity_diagram.png
+wr_install_schedule_activity_diagram.png
+use_case_diagram.png
+security_use_case_diagram.png
+middleware-stack-prod-birmingham.png
+middleware-stack-mtce.png
diff --git a/doc/uml/actor.png b/doc/uml/actor.png
new file mode 100644
index 0000000..5fa3808
--- /dev/null
+++ b/doc/uml/actor.png
diff --git a/doc/uml/computer.png b/doc/uml/computer.png
new file mode 100644
index 0000000..36f5cdb
--- /dev/null
+++ b/doc/uml/computer.png
diff --git a/doc/uml/drive-removable-media.png b/doc/uml/drive-removable-media.png
new file mode 100644
index 0000000..673284c
--- /dev/null
+++ b/doc/uml/drive-removable-media.png
diff --git a/doc/uml/hardware_install_activity_diagram.dot b/doc/uml/hardware_install_activity_diagram.dot
new file mode 100644
index 0000000..446773f
--- /dev/null
+++ b/doc/uml/hardware_install_activity_diagram.dot
@@ -0,0 +1,209 @@
+// This is a graphviz file. To generate a diagram from this source,
+// you must first install graphviz (available from homebrew). Then run
+// the following command:
+//
+// dot -Tpng -o hardware_install_activity_diagram.png hardware_install_activity_diagram.dot
+
+digraph G {
+    graph [fontsize = 14, penwidth = 0, rankdir = LR];
+    node [shape = "rect"];
+    edge [fontzize = 9, arrowhead = "empty"];
+
+    subgraph cluster_activity {
+        label = "CaaS/PaaS Hardware Installation";
+        color = black;
+        start_automation [shape = "circle", style = "filled", color = "grey", label = "Start\nautomation"];
+        start_server_install [shape = "circle", style = "filled", color = "grey", label = "Start\nphysical server\ninstallation"];
+        start_monitoring [shape = "circle", style = "filled", color = "grey", label = "Start\nmonitoring"];
+        end_monitoring_fail [shape = "doublecircle", style = "filled", color = "brown1", label = "Monitoring\nCRIT"];
+        start_firmware_upgrade [shape = "circle", style = "filled", color = "grey", label = "Schedule\nIntel NIC\nfirmware\nupgrade"];
+        start_wr_install [shape = "circle", style = "filled", color = "grey", label = "Schedule\nWind River\ninstallation"];
+        end_schedulewr_fail [shape = "doublecircle", style = "filled", color = "brown1", label = "Wind River\ninstallation\nfailed"];
+        end_schedulefirmware_fail [shape = "doublecircle", style = "filled", color = "brown1", label = "NIC\nfirmware\nupgrade\nfailed"];
+        start_orchestration [shape = "doublecircle", style = "filled", color = "grey", label = "Start\norchestration"];
+
+        subgraph cluster_hardwareinstall {
+            label = "Install hardware";
+            penwidth = 1;
+            color = "chocolate1";
+
+            subgraph cluster_dataimport {
+                label = "Data import";
+                dataimport_password_import [label = "Import HP passwords"];
+                dataimport_ciq_import [label = "Import CIQ"];
+                dataimport_icinga_batch [label = "Icinga job is sent to ansible-queue"];
+            }
+
+            subgraph cluster_serialnumber {
+                label = "Link serial numbers/passwords to CIQ";
+                serialnumber_server_hostalive_p [shape = "diamond", label = "Server pingble\nfrom model_check?"];
+                serialnumber_collect_vendor [label = "Icinga passes vendor,\nserial number and\nBMC IP to middleware"];
+                serialnumber_link_blade [label = "Middleware links\nblade to server"];
+            }
+
+            subgraph cluster_biosconfig {
+                label = "Configure passwords, DNS, icinga and BIOS";
+                biosconfig_generate_password [label = "Middleware generates\nunique BMC passwords"];
+                biosconfig_bmc_batch [label = "BMC job is sent\nto ansible-queue"];
+                biosconfig_dns_batch [label = "DNS job is sent\nto ansible-queue"];
+            }
+        }
+
+        subgraph cluster_icingamonitoring {
+            label = "Monitor hardware";
+            penwidth = 1;
+            color = "deepskyblue";
+
+            subgraph cluster_macnic {
+                label = "Collect BMC MAC and NIC firmware version";
+                macnic_bmc_push_pxe_mac [label = "BMC playbook pushes\nMAC address and\nNIC firmware version\nto middleware"];
+                macnic_icinga_monitoring [label = "icinga job is sent\nto ansible-queue"];
+                macnic_poll_monitoring [label = "icinga_poll job is sent\nto ansible-queue"];
+            }
+
+            subgraph cluster_nicfirmware {
+                label = "Intel NIC firmware version";
+                firmware_upgrade_p [shape = "diamond", label = "Is\nan Intel NIC\nfirmware upgrade\nneeded?"];
+            }
+        }
+
+        subgraph cluster_firmwarescheduler {
+            label = "Intel NIC firmware upgrade";
+            penwidth = 1;
+            color = "darkorchid1";
+
+            subgraph cluster_schedulefirmwareschedule {
+                label = "Schedule";
+                schedulefirmwareschedule_schedule [label = "Schedule firmware\nupgrade"];
+            }
+
+            subgraph cluster_schedulefirmwarerun {
+                label = "Run";
+                schedulefirmwarerun_run [label = "firmware upgrade\njob is sent\nto ansible-queue"];
+            }
+
+            subgraph cluster_schedulefirmwareresults {
+                label = "Results";
+                schedulefirmwareresults_p [shape = "diamond", label = "Did firmware\ninstall successfully?"];
+            }
+        }
+
+        subgraph cluster_wrscheduler {
+            label = "Schedule Wind River installation";
+            penwidth = 1;
+            color = "darkolivegreen4";
+
+            subgraph cluster_schedulewrschedule {
+                label = "Schedule";
+                schedulewrschedule_schedule [label = "Schedule Wind\nRiver install"];
+            }
+
+            subgraph cluster_schedulewrrun {
+                label = "Run";
+                schedulewrrun_run [label = "Wind River\ninstall job is sent\nto ansible-queue"];
+            }
+
+            subgraph cluster_schedulewrresults {
+                label = "Results";
+                schedulewrresults_p [shape = "diamond", label = "Did Wind River\ninstall successfully?"];
+            }
+
+            subgraph cluster_schedulewrorchestration {
+                label = "Orchestration";
+                schedulewrorchestration_orch [label = "Call\norchestration\nendpoint"];
+            }
+        }
+
+        subgraph cluster_ansiblequeue {
+            label = "ansible-queue";
+            penwidth = 1;
+            color = "yellow3";
+
+            subgraph cluster_icingaplaybook {
+                label = "Icinga queue";
+                icingaplaybook_run [label = "Manage ansible host_vars\nand run playbook"];
+            }
+
+            subgraph cluster_icingapollplaybook {
+                label = "Icinga Poll queue";
+                icingapollplaybook_run [label = "Manage ansible host_vars\nand run playbook"];
+                icingapollplaybook_report [label = "Playbook informs\nmiddleware of result"];
+            }
+
+            subgraph cluster_bmcplaybook {
+                label = "BMC queue";
+                bmcplaybook_run_1 [label = "Manage ansible host_vars\nand run playbook"];
+                bmcplaybook_run_2 [label = "Manage ansible host_vars\nand run playbook"];
+            }
+
+            subgraph cluster_dnsplaybook {
+                label = "DNS queue";
+                dnsplaybook_run [label = "Manage ansible host_vars\nand run playbook"];
+            }
+
+            subgraph cluster_firmwareplaybook {
+                label = "NIC queue";
+                firmwareplaybook_run [label = "Manage ansible host_vars\nand run playbook"];
+                firmwareplaybook_report [label = "Playbook informs\nmiddleware of result"];
+            }
+ 
+            subgraph cluster_wrplaybook {
+                label = "Wind River queue";
+                wrplaybook_run [label = "Manage ansible host_vars\nand run playbook"];
+                wrplaybook_report [label = "Playbook informs\nmiddleware of result"];
+            }
+        }
+
+        subgraph cluster_legend {
+            label = "Legend";
+            penwidth = 1;
+            node [shape = "rect", style = "filled"];
+            legend_hardware [label = "Hardware", fillcolor = "chocolate1"];
+            legend_monitoring [label = "Monitoring", fillcolor = "deepskyblue"];
+            legend_firmware [label = "NIC firmware", fillcolor = "darkorchid1"];
+            legend_wr [label = "Wind River", fillcolor = "darkolivegreen4"];
+            legend_ansible [label = "ansible-queue", fillcolor = "yellow3"];
+        }
+        
+        start_automation -> dataimport_password_import [color = "chocolate1"];
+        dataimport_password_import -> dataimport_ciq_import [color = "chocolate1"];
+        dataimport_ciq_import -> dataimport_icinga_batch [color = "chocolate1"];
+        dataimport_icinga_batch -> icingaplaybook_run [color = "chocolate1"];
+        dataimport_icinga_batch -> serialnumber_server_hostalive_p [color = "chocolate1"];
+        start_server_install -> serialnumber_server_hostalive_p [color = "chocolate1"];
+        serialnumber_server_hostalive_p -> serialnumber_server_hostalive_p [color = "chocolate1", label = "No"];
+        serialnumber_server_hostalive_p -> serialnumber_collect_vendor [color = "chocolate1", label = "Yes"];
+        serialnumber_collect_vendor -> serialnumber_link_blade [color = "chocolate1"];
+        serialnumber_link_blade -> biosconfig_generate_password [color = "chocolate1"];
+        biosconfig_generate_password -> biosconfig_dns_batch [color = "chocolate1"];
+        biosconfig_generate_password -> biosconfig_bmc_batch [color = "chocolate1"];
+        schedulefirmwareschedule_schedule -> schedulefirmwarerun_run [color = "darkorchid1"];
+        schedulefirmwarerun_run -> firmwareplaybook_run [color = "darkorchid1"];
+        firmwareplaybook_run -> firmwareplaybook_report [color = "yellow3"];
+        firmwareplaybook_report -> schedulefirmwareresults_p [color = "darkorchid1"];
+        schedulefirmwareresults_p -> end_schedulefirmware_fail [color = "darkorchid1", label = "No"];
+        schedulefirmwareresults_p -> start_wr_install [color = "darkorchid1", label = "Yes"];
+        biosconfig_bmc_batch -> bmcplaybook_run_1 [color = "chocolate1"];
+        biosconfig_dns_batch -> dnsplaybook_run [color = "chocolate1"];
+        bmcplaybook_run_1 -> start_monitoring [color = "deepskyblue"];
+        start_monitoring -> macnic_bmc_push_pxe_mac [color = "deepskyblue"];
+        macnic_bmc_push_pxe_mac -> macnic_icinga_monitoring [color = "deepskyblue"];
+        macnic_bmc_push_pxe_mac -> macnic_poll_monitoring [color = "deepskyblue"];
+        macnic_icinga_monitoring -> icingaplaybook_run [color = "deepskyblue"];
+        macnic_poll_monitoring -> icingapollplaybook_run [color = "deepskyblue"];
+        icingapollplaybook_run -> icingapollplaybook_report [color = "yellow3"];
+        icingapollplaybook_report -> firmware_upgrade_p [color = "deepskyblue", label = "Cleared"];
+        icingapollplaybook_report -> end_monitoring_fail [color = "deepskyblue", label = "Not cleared"];
+        firmware_upgrade_p -> start_firmware_upgrade [color = "deepskyblue", label = "Yes"];
+        firmware_upgrade_p -> start_wr_install [color = "deepskyblue", label = "No"];
+        start_firmware_upgrade -> schedulefirmwareschedule_schedule [color = "darkorchid1"];
+        start_wr_install -> schedulewrschedule_schedule [color = "darkolivegreen4"];
+        schedulewrschedule_schedule -> schedulewrrun_run [color = "darkolivegreen4"];
+        schedulewrrun_run -> wrplaybook_run [color = "darkolivegreen4"];
+        wrplaybook_run -> wrplaybook_report [color = "yellow3"];
+        wrplaybook_report -> schedulewrresults_p [color = "darkolivegreen4"];
+        schedulewrresults_p -> end_schedulewr_fail [color = "darkolivegreen4", label = "No"];
+        schedulewrresults_p -> schedulewrorchestration_orch [color = "darkolivegreen4", label = "Yes"];
+        schedulewrorchestration_orch -> start_orchestration [color = "darkolivegreen4"];
+    }
+}
diff --git a/doc/uml/middleware-stack-mtce.dot b/doc/uml/middleware-stack-mtce.dot
new file mode 100644
index 0000000..3bde052
--- /dev/null
+++ b/doc/uml/middleware-stack-mtce.dot
@@ -0,0 +1,200 @@
+// This is a graphviz file. To generate a diagram from this source,
+// you must first install graphviz (available from homebrew). Then run
+// the following command:
+//
+// dot -Tpng -o security_use_case_diagram.png security_use_case_diagram.dot
+
+digraph G {
+    graph [fontsize = 14, penwidth = 0, rankdir = LR];
+    node [shape = "plaintext"];
+    edge [color = "deepskyblue", fontsize = 9, dir = none];
+
+    subgraph cluster_vm {
+        label = "Nova instances";
+        penwidth = 1;
+
+        subgraph cluster_vmmiddleware {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_vmmiddlewareapp1 {
+                label = "Django/icinga host 1\nSOLKTX01VZBvVFE-Y-WR-X-00-MW-001";
+                penwidth = 0;
+                vmmiddlewareapp1_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmmiddlewareapp2 {
+                label = "Django/icinga host 2\nSOLKTX01VZBvVFE-Y-WR-X-00-MW-002";
+                penwidth = 0;
+                vmmiddlewareapp2_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmmiddlewaredatabase1 {
+                label = "Database host 1\nSOLKTX01VZBvVFE-Y-WR-X-00-DB-001";
+                penwidth = 0;
+                vmmiddlewaredatabase1_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmmiddlewaredatabase2 {
+                label = "Database host 2\nSOLKTX01VZBvVFE-Y-WR-X-00-DB-002";
+                penwidth = 0;
+                vmmiddlewaredatabase2_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmmiddlewaredatabase3 {
+                label = "Database host 3\nSOLKTX01VZBvVFE-Y-WR-X-00-DB-003";
+                penwidth = 0;
+                vmmiddlewaredatabase3_icon [label = "", image = "computer.png"];
+            }
+        }
+        
+        subgraph cluster_vmservices {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_vmservices1 {
+                label = "ansible-queue/dns-admin host 1\nSOLKTX01VZBvVFE-Y-WR-X-00-CM-001";
+                penwidth = 0;
+                vmservices1_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmservices2 {
+                label = "ansible-queue/dns-admin host 2\nSOLKTX01VZBvVFE-Y-WR-X-00-CM-002";
+                penwidth = 0;
+                vmservices2_icon [label = "", image = "computer.png"];
+            }
+        }
+            
+        subgraph cluster_vmlb {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_vmlb1 {
+                label = "Load balancer host 1\nSOLKTX01VZBvVFE-Y-WR-X-00-LB-001";
+                penwidth = 0;
+                vmlb1_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmlb2 {
+                label = "Load balancer host 2\nSOLKTX01VZBvVFE-Y-WR-X-00-LB-002";
+                penwidth = 0;
+                vmlb2_icon [label = "", image = "computer.png"];
+            }
+        }
+    }
+
+    subgraph cluster_port {
+        label = "Neutron ports";
+        penwidth = 1;
+
+        subgraph cluster_portmiddleware {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_portmiddlewareapp1 {
+                label = "Django/icinga host 1\nSOLKTX01VZBvVFE-Y-WR-X-00-MW-001\n2607:f160:b:10f1::6";
+                penwidth = 0;
+                portmiddlewareapp1_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portmiddlewareapp2 {
+                label = "Django/icinga host 2\nSOLKTX01VZBvVFE-Y-WR-X-00-MW-002\n2607:f160:b:10f1::5";
+                penwidth = 0;
+                portmiddlewareapp2_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portmiddlewaredatabase1 {
+                label = "Database host 1\nSOLKTX01VZBvVFE-Y-WR-X-00-DB-001\n2607:f160:b:10f1::b";
+                penwidth = 0;
+                portmiddlewaredatabase1_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portmiddlewaredatabase2 {
+                label = "Database host 2\nSOLKTX01VZBvVFE-Y-WR-X-00-DB-002\n2607:f160:b:10f1::3";
+                penwidth = 0;
+                portmiddlewaredatabase2_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portmiddlewaredatabase3 {
+                label = "Database host 3\nSOLKTX01VZBvVFE-Y-WR-X-00-DB-003\n2607:f160:b:10f1::c";
+                penwidth = 0;
+                portmiddlewaredatabase3_icon [label = "", image = "network-wired.png"];
+            }
+        }
+        
+        subgraph cluster_portservices {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_portservices1 {
+                label = "ansible-queue/\ndns-admin host 1\nSOLKTX01VZBvVFE-Y-WR-X-00-CM-001\n2607:f160:b:10f1::7";
+                penwidth = 0;
+                portservices1_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portservices2 {
+                label = "ansible-queue/\ndns-admin host 2\nSOLKTX01VZBvVFE-Y-WR-X-00-CM-002\n2607:f160:b:10f1::8";
+                penwidth = 0;
+                portservices2_icon [label = "", image = "network-wired.png"];
+            }
+        }
+            
+        subgraph cluster_portlb {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_portlb1 {
+                label = "Load balancer host 1\nSOLKTX01VZBvVFE-Y-WR-X-00-LB-001\n2607:f160:b:10f1::e";
+                penwidth = 0;
+                portlb1_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portlb2 {
+                label = "Load balancer host 2\nSOLKTX01VZBvVFE-Y-WR-X-00-LB-002\n2607:f160:b:10f1::d";
+                penwidth = 0;
+                portlb2_icon [label = "", image = "network-wired.png"];
+            }
+        }
+    }
+
+    subgraph cluster_volume {
+        label = "Cinder volumes";
+        penwidth = 1;
+
+        subgraph cluster_volumemiddleware {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_volumemiddlewaredatabase1 {
+                label = "Database host 1\nSOLKTX01VZBvVFE-Y-WR-X-00-DB-01";
+                penwidth = 0;
+                volumemiddlewaredatabase1_icon [label = "", image = "drive-removable-media.png"];
+            }
+
+            subgraph cluster_volumemiddlewaredatabase2 {
+                label = "Database host 2\nSOLKTX01VZBvVFE-Y-WR-X-00-DB-02";
+                penwidth = 0;
+                volumemiddlewaredatabase2_icon [label = "", image = "drive-removable-media.png"];
+            }
+
+            subgraph cluster_volumemiddlewaredatabase3 {
+                label = "Database host 3\nSOLKTX01VZBvVFE-Y-WR-X-00-DB-03";
+                penwidth = 0;
+                volumemiddlewaredatabase3_icon [label = "", image = "drive-removable-media.png"];
+            }
+        }
+    }
+
+    vmmiddlewareapp1_icon -> portmiddlewareapp1_icon;
+    vmmiddlewareapp2_icon -> portmiddlewareapp2_icon;
+    vmmiddlewaredatabase1_icon -> portmiddlewaredatabase1_icon;
+    vmmiddlewaredatabase2_icon -> portmiddlewaredatabase2_icon;
+    vmmiddlewaredatabase3_icon -> portmiddlewaredatabase3_icon;
+    vmservices1_icon -> portservices1_icon;
+    vmservices2_icon -> portservices2_icon;
+    vmlb1_icon -> portlb1_icon;
+    vmlb2_icon -> portlb2_icon;
+    vmmiddlewaredatabase1_icon -> volumemiddlewaredatabase1_icon;
+    vmmiddlewaredatabase2_icon -> volumemiddlewaredatabase2_icon;
+    vmmiddlewaredatabase3_icon -> volumemiddlewaredatabase3_icon;
+}
diff --git a/doc/uml/middleware-stack-prod-birmingham.dot b/doc/uml/middleware-stack-prod-birmingham.dot
new file mode 100644
index 0000000..5250381
--- /dev/null
+++ b/doc/uml/middleware-stack-prod-birmingham.dot
@@ -0,0 +1,200 @@
+// This is a graphviz file. To generate a diagram from this source,
+// you must first install graphviz (available from homebrew). Then run
+// the following command:
+//
+// dot -Tpng -o security_use_case_diagram.png security_use_case_diagram.dot
+
+digraph G {
+    graph [fontsize = 14, penwidth = 0, rankdir = LR];
+    node [shape = "plaintext"];
+    edge [color = "deepskyblue", fontsize = 9, dir = none];
+
+    subgraph cluster_vm {
+        label = "Nova instances";
+        penwidth = 1;
+
+        subgraph cluster_vmmiddleware {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_vmmiddlewareapp1 {
+                label = "Django/icinga host 1\nBRHOALTBVZBvVFE-Y-WR-X-00-MW-001";
+                penwidth = 0;
+                vmmiddlewareapp1_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmmiddlewareapp2 {
+                label = "Django/icinga host 2\nBRHOALTBVZBvVFE-Y-WR-X-00-MW-002";
+                penwidth = 0;
+                vmmiddlewareapp2_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmmiddlewaredatabase1 {
+                label = "Database host 1\nBRHOALTBVZBvVFE-Y-WR-X-00-DB-001";
+                penwidth = 0;
+                vmmiddlewaredatabase1_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmmiddlewaredatabase2 {
+                label = "Database host 2\nBRHOALTBVZBvVFE-Y-WR-X-00-DB-002";
+                penwidth = 0;
+                vmmiddlewaredatabase2_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmmiddlewaredatabase3 {
+                label = "Database host 3\nBRHOALTBVZBvVFE-Y-WR-X-00-DB-003";
+                penwidth = 0;
+                vmmiddlewaredatabase3_icon [label = "", image = "computer.png"];
+            }
+        }
+        
+        subgraph cluster_vmservices {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_vmservices1 {
+                label = "ansible-queue/dns-admin host 1\nBRHOALTBVZBvVFE-Y-WR-X-00-CM-001";
+                penwidth = 0;
+                vmservices1_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmservices2 {
+                label = "ansible-queue/dns-admin host 2\nBRHOALTBVZBvVFE-Y-WR-X-00-CM-002";
+                penwidth = 0;
+                vmservices2_icon [label = "", image = "computer.png"];
+            }
+        }
+            
+        subgraph cluster_vmlb {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_vmlb1 {
+                label = "Load balancer host 1\nBRHOALTBVZBvVFE-Y-WR-X-00-LB-001";
+                penwidth = 0;
+                vmlb1_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_vmlb2 {
+                label = "Load balancer host 2\nBRHOALTBVZBvVFE-Y-WR-X-00-LB-002";
+                penwidth = 0;
+                vmlb2_icon [label = "", image = "computer.png"];
+            }
+        }
+    }
+
+    subgraph cluster_port {
+        label = "Neutron ports";
+        penwidth = 1;
+
+        subgraph cluster_portmiddleware {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_portmiddlewareapp1 {
+                label = "Django/icinga host 1\nBRHOALTBVZBvVFE-Y-WR-X-00-MW-001\n2001:4888:a21:3102:245:29::";
+                penwidth = 0;
+                portmiddlewareapp1_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portmiddlewareapp2 {
+                label = "Django/icinga host 2\nBRHOALTBVZBvVFE-Y-WR-X-00-MW-002\n2001:4888:a21:3102:245:29:0:a";
+                penwidth = 0;
+                portmiddlewareapp2_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portmiddlewaredatabase1 {
+                label = "Database host 1\nBRHOALTBVZBvVFE-Y-WR-X-00-DB-001\n2001:4888:a21:3102:245:29:0:1";
+                penwidth = 0;
+                portmiddlewaredatabase1_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portmiddlewaredatabase2 {
+                label = "Database host 2\nBRHOALTBVZBvVFE-Y-WR-X-00-DB-002\n2001:4888:a21:3102:245:29:0:7";
+                penwidth = 0;
+                portmiddlewaredatabase2_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portmiddlewaredatabase3 {
+                label = "Database host 3\nBRHOALTBVZBvVFE-Y-WR-X-00-DB-003\n2001:4888:a21:3102:245:29:0:2";
+                penwidth = 0;
+                portmiddlewaredatabase3_icon [label = "", image = "network-wired.png"];
+            }
+        }
+        
+        subgraph cluster_portservices {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_portservices1 {
+                label = "ansible-queue/\ndns-admin host 1\nBRHOALTBVZBvVFE-Y-WR-X-00-CM-001\n2001:4888:a21:3102:245:29:0:b";
+                penwidth = 0;
+                portservices1_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portservices2 {
+                label = "ansible-queue/\ndns-admin host 2\nBRHOALTBVZBvVFE-Y-WR-X-00-CM-002\n2001:4888:a21:3102:245:29:0:10";
+                penwidth = 0;
+                portservices2_icon [label = "", image = "network-wired.png"];
+            }
+        }
+            
+        subgraph cluster_portlb {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_portlb1 {
+                label = "Load balancer host 1\nBRHOALTBVZBvVFE-Y-WR-X-00-LB-001\n2001:4888:a21:3102:245:29:0:11";
+                penwidth = 0;
+                portlb1_icon [label = "", image = "network-wired.png"];
+            }
+
+            subgraph cluster_portlb2 {
+                label = "Load balancer host 2\nBRHOALTBVZBvVFE-Y-WR-X-00-LB-002\n2001:4888:a21:3102:245:29:0:8";
+                penwidth = 0;
+                portlb2_icon [label = "", image = "network-wired.png"];
+            }
+        }
+    }
+
+    subgraph cluster_volume {
+        label = "Cinder volumes";
+        penwidth = 1;
+
+        subgraph cluster_volumemiddleware {
+            label = "";
+            penwidth = 0;
+
+            subgraph cluster_volumemiddlewaredatabase1 {
+                label = "Database host 1\nBRHOALTBVZBvVFE-Y-WR-X-00-DB-01";
+                penwidth = 0;
+                volumemiddlewaredatabase1_icon [label = "", image = "drive-removable-media.png"];
+            }
+
+            subgraph cluster_volumemiddlewaredatabase2 {
+                label = "Database host 2\nBRHOALTBVZBvVFE-Y-WR-X-00-DB-02";
+                penwidth = 0;
+                volumemiddlewaredatabase2_icon [label = "", image = "drive-removable-media.png"];
+            }
+
+            subgraph cluster_volumemiddlewaredatabase3 {
+                label = "Database host 3\nBRHOALTBVZBvVFE-Y-WR-X-00-DB-03";
+                penwidth = 0;
+                volumemiddlewaredatabase3_icon [label = "", image = "drive-removable-media.png"];
+            }
+        }
+    }
+
+    vmmiddlewareapp1_icon -> portmiddlewareapp1_icon;
+    vmmiddlewareapp2_icon -> portmiddlewareapp2_icon;
+    vmmiddlewaredatabase1_icon -> portmiddlewaredatabase1_icon;
+    vmmiddlewaredatabase2_icon -> portmiddlewaredatabase2_icon;
+    vmmiddlewaredatabase3_icon -> portmiddlewaredatabase3_icon;
+    vmservices1_icon -> portservices1_icon;
+    vmservices2_icon -> portservices2_icon;
+    vmlb1_icon -> portlb1_icon;
+    vmlb2_icon -> portlb2_icon;
+    vmmiddlewaredatabase1_icon -> volumemiddlewaredatabase1_icon;
+    vmmiddlewaredatabase2_icon -> volumemiddlewaredatabase2_icon;
+    vmmiddlewaredatabase3_icon -> volumemiddlewaredatabase3_icon;
+}
diff --git a/doc/uml/network-wired.png b/doc/uml/network-wired.png
new file mode 100644
index 0000000..6417065
--- /dev/null
+++ b/doc/uml/network-wired.png
diff --git a/doc/uml/security_use_case_diagram.dot b/doc/uml/security_use_case_diagram.dot
new file mode 100644
index 0000000..c939cc6
--- /dev/null
+++ b/doc/uml/security_use_case_diagram.dot
@@ -0,0 +1,153 @@
+// This is a graphviz file. To generate a diagram from this source,
+// you must first install graphviz (available from homebrew). Then run
+// the following command:
+//
+// dot -Tpng -o security_use_case_diagram.png security_use_case_diagram.dot
+
+digraph G {
+    graph [fontsize = 14, penwidth = 0, rankdir = UD];
+    node [shape = "plaintext"];
+    edge [fontsize = 9, style = dotted, arrowhead = empty];
+
+    subgraph cluster_external {
+        label = "External systems";
+        penwidth = 1;
+
+        subgraph cluster_externalbrowser {
+            label = "EDN-attached web browser";
+            penwidth = 0;
+            externalbrowser_icon [label = "", image = "computer.png"];
+        }
+
+        subgraph cluster_externalorchestration {
+            label = "ATLAS/Orchestration";
+            penwidth = 0;
+            externalorchestration_icon [label = "", image = "computer.png"];
+        }
+
+        subgraph cluster_externalvmb{
+            label = "VMB";
+            penwidth = 0;
+            externalvmb_icon [label = "", image = "computer.png"];
+        }
+
+        subgraph cluster_externalgitlab {
+            label = "GitLab";
+            penwidth = 0;
+            externalgitlab_icon [label = "", image = "computer.png"];
+        }
+
+        subgraph cluster_externalartifactory {
+            label = "Artifactory";
+            penwidth = 0;
+            externalartifactory_icon [label = "", image = "computer.png"];
+        }
+
+        subgraph cluster_externalinfoblox {
+            label = "CDS Infoblox";
+            penwidth = 0;
+            externalinfoblox_icon [label = "", image = "computer.png"];
+        }
+
+        subgraph cluster_externalldap {
+            label = "USWIN LDAP";
+            penwidth = 0;
+            externalldap_icon [label = "", image = "computer.png"];
+        }
+    }
+
+    subgraph cluster_middlewarestack {
+        label = "Automation stack";
+        penwidth = 1;
+
+        subgraph cluster_middlewarestackansible {
+            label = "Ansible/DNS hosts";
+            penwidth = 0;
+            middlewarestackansible_icon [label = "", image = "computer.png"];
+        }
+
+        subgraph cluster_middlewarestackapp {
+            label = "Middleware/Icinga hosts";
+            penwidth = 0;
+            middlewarestackapp_icon [label = "", image = "computer.png"];
+        }
+
+        subgraph cluster_middlewarestackdatabase {
+            label = "Database hosts";
+            penwidth = 0;
+            middlewarestackdatabase_icon [label = "", image = "drive-removable-media.png"];
+        }
+
+        subgraph cluster_middlewarestacklb {
+            label = "Load balancer hosts";
+            penwidth = 0;
+            middlewarestacklb_icon [label = "", image = "computer.png"];
+        }
+    }
+
+    subgraph cluster_wr {
+        label = "Wind River platform";
+        penwidth = 1;
+
+        subgraph cluster_wrcentral {
+            label = "Central controller cluster";
+
+            subgraph cluster_wrcentralhosts {
+                label = "Central controller hosts";
+                penwidth = 0;
+                wrcentralhosts_icon [label = "", image = "computer.png"];
+            }
+
+            subgraph cluster_wrcentralbmc {
+                label = "Central controller BMC";
+                penwidth = 0;
+                wrcentralbmc_icon [label = "", image = "network-wired.png"];
+            }
+        }
+
+        subgraph cluster_wrremote {
+            label = "Remote subcloud cluster";
+
+            subgraph cluster_wrremotehosts {
+                label = "Remote subcloud hosts";
+                penwidth = 0;
+                wrremotehosts_icon [label = "", image = "computer.png"];
+            }
+            
+            subgraph cluster_wrremotebmc {
+                label = "Remote subcloud BMC";
+                penwidth = 0;
+                wrremotebmc_icon [label = "", image = "network-wired.png"];
+            }
+        }
+    }
+
+    subgraph cluster_legend {
+        label = "Legend";
+        penwidth = 1;
+        node [shape = "rect", style = filled];
+        legend_external [label = "Connections\nto external\nsystems", fillcolor = "chocolate1"];
+        legend_wr [label = "Connections\nto Wind River\nplatform", fillcolor = "darkolivegreen3"];
+        legend_middleware_external [label = "Connections\nto middleware", fillcolor = "deepskyblue"];
+    }
+
+    externalbrowser_icon -> middlewarestacklb_icon [color = "deepskyblue", headlabel = "TCP *:443"];
+    externalorchestration_icon -> middlewarestacklb_icon [color = "deepskyblue"];
+    middlewarestackapp_icon -> externalvmb_icon [color = "chocolate1", headlabel = "TCP *:6651"];
+    externalgitlab_icon -> middlewarestacklb_icon  [color = "deepskyblue"];
+    middlewarestackansible_icon -> externalartifactory_icon [color = "chocolate1", headlabel = "TCP *:443"];
+    middlewarestackapp_icon -> externalgitlab_icon [color = "chocolate1", headlabel = "TCP *:22"];
+    middlewarestackansible_icon -> externalgitlab_icon [color = "chocolate1"];
+    middlewarestackansible_icon -> wrcentralhosts_icon [color = "darkolivegreen3", headlabel = "TCP *:22"];
+    middlewarestackansible_icon -> wrremotehosts_icon [color = "darkolivegreen3", headlabel = "TCP *:22"];
+    middlewarestackapp_icon -> middlewarestackansible_icon [color = "deepskyblue", headlabel = "TCP *:80, *:5555-5561"];
+    middlewarestacklb_icon -> externalldap_icon [color = "chocolate1", headlabel = "TCP *:636"];
+    middlewarestackansible_icon -> externalinfoblox_icon [color = "chocolate1", headlabel = "TCP *:443"];
+    middlewarestackapp_icon -> wrcentralbmc_icon [color = "darkolivegreen3", headlabel = "TCP *:443"];
+    middlewarestackapp_icon -> wrremotebmc_icon [color = "darkolivegreen3", headlabel = "TCP *:443"];
+    middlewarestacklb_icon -> middlewarestackapp_icon [color = "deepskyblue", headlabel = "TCP *:80"];
+    middlewarestackapp_icon -> middlewarestackdatabase_icon [color = "deepskyblue", headlabel = "TCP *:5432"];
+    middlewarestacklb_icon -> middlewarestackansible_icon [color = "deepskyblue"];
+    wrcentralhosts_icon -> externalartifactory_icon [color = "chocolate1"];
+    wrcentralhosts_icon -> wrremotehosts_icon [color = "darkolivegreen3"];
+}
diff --git a/doc/uml/security_use_case_diagram.pdf b/doc/uml/security_use_case_diagram.pdf
new file mode 100644
index 0000000..2a5bfc0
--- /dev/null
+++ b/doc/uml/security_use_case_diagram.pdf
diff --git a/doc/uml/use_case_diagram.dot b/doc/uml/use_case_diagram.dot
new file mode 100644
index 0000000..dfa7692
--- /dev/null
+++ b/doc/uml/use_case_diagram.dot
@@ -0,0 +1,72 @@
+// This is a graphviz file. To generate a diagram from this source,
+// you must first install graphviz (available from homebrew). Then run
+// the following command:
+//
+// dot -Tpng -o use_case_diagram.png use_case_diagram.dot
+
+digraph G {
+    fontname = "Bitstream Vera Sans";
+    fontsize = 12;
+
+    node [
+        fontname = "Bitstream Vera Sans";
+        fontsize = 12;
+        shape = "ellipse";
+    ]
+
+    edge [
+        fontname = "Bitstream Vera Sans";
+        fontsize = 12;
+    ]
+
+    edge [
+        arrowhead = "empty";
+    ]
+
+    subgraph cluster_actors {
+        color = white;
+        node [
+            shape = custom;
+            shapefile = "actor.png";
+	    width = 0.352;
+            height = 0.77;
+            fixedsize = true;
+	    color = "#ffffff";
+        ]
+	ciq [label = "CIQ"];
+        cron [label = "cron"]
+	technician [label = "Technician"];
+    }
+
+    subgraph cluster_processes { 
+        style = filled;
+        color = azure2;
+        populate_database [label = "Populate database"];
+        monitor_hardware [label = "Monitor hardware"];
+        install_caas [label = "Install CaaS"];
+    }
+
+    {
+        edge [color = red;]
+        ciq -> populate_database [label = "Populates"];
+    }
+    
+    {
+        edge [color = chocolate4;]
+        cron -> install_caas [label = "Starts"];
+    }
+    
+    {
+        edge [color = blue;]
+        technician -> monitor_hardware [label = "Waits for"];
+    }
+    
+    {
+        edge [
+            label = "Uses";
+            color = green4;
+        ]
+        install_caas -> monitor_hardware;
+        monitor_hardware -> populate_database;
+    }
+}