diff options
95 files changed, 5555 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80fb99f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*~ +*swp diff --git a/README.md b/README.md new file mode 100644 index 0000000..309c4a2 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +Openbook Installer +================== + +Invocation +---------- + +Two pillar values must be passed in on the command line: + +- openbook_database_password: the mysql root password +- openbook_gcomm_addresses: the IP addresses of the galera nodes + (comma-separated list, no spaces) + +Customize bootstrap/config.yaml with these values. If you leave them +as null (represented by a ~ in yaml), default values will be supplied. + + + +# Ansible + +cd into the conf/ directory and set a symlink called env.yaml that +points to either env.yaml.openbook or env.yaml.docker_registry. + + + +## docker-registry ansible playbook + +The certs must be placed in /etc/ssl/certs and /etc/ssl/private prior +to running the playbook. Also update-cs-certificates needs to be run +before attempting to start docker-registry. + + diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore new file mode 100644 index 0000000..5b6b072 --- /dev/null +++ b/bootstrap/.gitignore @@ -0,0 +1 @@ +config.yaml diff --git a/bootstrap/config.yaml.example b/bootstrap/config.yaml.example new file mode 100644 index 0000000..ffadfd0 --- /dev/null +++ b/bootstrap/config.yaml.example @@ -0,0 +1,8 @@ +registry: + url: ~ + username: ~ + password: ~ +jars: + url: ~ + username: ~ + password: ~ diff --git a/bootstrap/http_client.rb b/bootstrap/http_client.rb new file mode 100644 index 0000000..58eea23 --- /dev/null +++ b/bootstrap/http_client.rb @@ -0,0 +1,90 @@ +require 'json' +require 'net/http' +require 'net/https' +require 'uri' +require 'openssl' + +class HttpClient + attr_accessor :uri + attr_accessor :header + attr_accessor :cookies + attr_accessor :body + attr_accessor :response + attr_accessor :response_code + + @@req_xml = <<EOF +<?xml version="1.0"?> +<a:propfind xmlns:a="DAV:"> +<a:prop><a:resourcetype/></a:prop> +</a:propfind> +EOF + + def initialize(url: nil, username: nil, password: nil, headers: Hash.new, cookies: Hash.new, body: nil) + @uri = URI.parse(url) + @username = username + @password = password + @headers = headers + @cookies = cookies + @body = body + end + + def do_http(method: :get, do_cookies_p: false, read_timeout: 600, extra_headers: Hash.new) + resp = nil + http = Net::HTTP.new(@uri.host, @uri.port) + if uri.scheme == 'https' + http.use_ssl = true + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + end + http.read_timeout = read_timeout + req = case method + when :get + Net::HTTP::Get.new(uri.request_uri, @headers.merge(extra_headers)) + when :delete + Net::HTTP::Delete.new(uri.request_uri, @headers.merge(extra_headers)) + when :post + Net::HTTP::Post.new(uri.request_uri, @headers.merge(extra_headers)) + when :put + Net::HTTP::Put.new(uri.request_uri, @headers.merge(extra_headers)) + when :propfind + Net::HTTP::Propfind.new(uri.request_uri, {"Depth" => "1"}) + end + req.basic_auth(@username, @password) if @username and @password + if method == :propfind + req.body = @@req_xml + else + req.body = @body if @body + end + resp = http.request(req) + @response = resp.body + @response_code = resp.class + get_cookies(resp) if do_cookies_p + end + + # Below is a pair of example Set-Cookie headers after + # Net::HttpClient turns them into a concatenated string. Note the + # bad use of a comma to separate the cookies. It's bad because a + # comma already appears in the date. We deal with that by eating + # the date with gsub() before splitting. + # + # csrftoken=urvsX10TfzWIK1fgctZ2HjHuNQZDr3E7; expires=Wed, 16-Nov-2016 16:57:20 GMT; Max-Age=31449600; Path=/, sessionid=e0t7304jn44ssc7gw412abup4aasehgb; expires=Wed, 18-Nov-2015 17:57:20 GMT; httponly; Max-Age=3600; Path=/ + # + # rememberMe=deleteMe; Path=/Openbook; Max-Age=0; Expires=Wed, 16-Dec-2015 22:53:58 GMT + def get_cookies(resp) + if resp.response['Set-Cookie'] + rawstring = resp.response['Set-Cookie'].gsub(/expires.*?Path=/, '') + rawstring.chomp.split(",").each do |entry| + entryparts = entry.split(";") + if entryparts.length >= 1 + cookieparts = entryparts[0].split("=") + name = cookieparts[0].strip + value = cookieparts[1].strip + @cookies[name] = value + end + end + end + end + + def create_cookie(name, value) + "#{name}=#{value}; Path=/" + end +end diff --git a/bootstrap/pillar_config.rb b/bootstrap/pillar_config.rb new file mode 100755 index 0000000..60460b1 --- /dev/null +++ b/bootstrap/pillar_config.rb @@ -0,0 +1,84 @@ +#!/usr/bin/ruby + +require_relative 'http_client' +require 'digest' +require "json" +require 'socket' +require "yaml" + +def sha() + seed = "" + File.open("/dev/urandom", "r") do |f| + seed = "#{f.sysread(1024)}" + end + Digest::SHA256.hexdigest(seed) +end + +def get_ip_address() + ip_arr = Socket.ip_address_list.map { |addr| + addr.ip_address + }.reject { |addr| + addr == "172.17.0.1" or addr.start_with?("127") or addr.include?("::") + } + if ip_arr.length == nil + "127.0.0.1" + else + ip_arr[0] + end +end + +def get_jar_versions(yml) + ret = Hash.new + begin + http_client = HttpClient.new(url: yml["jars"]["url"], + username: yml["jars"]["username"], + password: yml["jars"]["password"]) + http_client.do_http(method: :propfind) + http_client.response.each_line do |line| + if line =~ /D:href.*\.jar/ + jar_name = line.chomp.split(/[<>]/)[2] + jarparts = jar_name.split(/\//) + name, *version_parts, junk1 = jarparts[jarparts.length - 1].split(".") + ret["#{name.gsub("-", "_")}"] = { + "jar_name" => jarparts[jarparts.length - 1], + "jar_version" => version_parts.join("."), + "image_name" => "openbook_#{name.gsub("-", "_")}", + "container_name" => "#{name.gsub("-", "_")}", + "build_dir" => "/root/openbook_build/#{name.gsub("-", "_")}" + } + end + end + rescue Exception => e + STDERR.puts "Error while contacting webdav endpoint: #{e.message}" + exit(1) + end + ret +end + +def process_yaml() + env_yaml_path = "#{File.dirname($0)}/config.yaml" + if ! File.file?(env_yaml_path) + STDERR.puts "Unable to find config.yaml. Exiting." + exit(1) + end + yml = YAML.load_file(env_yaml_path) + yml["baseimage"] = {} + yml["database"] = {} + yml["baseimage"]["image_name"] = "openbook_baseimage" + yml["baseimage"]["container_name"] = "baseimage" + yml["baseimage"]["build_dir"] = "/root/openbook_build/baseimage" + yml["database"]["image_name"] = "openbook_database" + yml["database"]["container_name"] = "database" + yml["database"]["build_dir"] = "/root/openbook_build/database" + yml["ansible"] = Hash.new + yml["ansible"]["build_dir"] = "/root/openbook_build/ansible" + yml +end + +def main() + yml = process_yaml() + puts yml.merge(get_jar_versions(yml)).to_json +end + +main() +exit(0) diff --git a/bootstrap/run_salt.sh b/bootstrap/run_salt.sh new file mode 100755 index 0000000..869b52b --- /dev/null +++ b/bootstrap/run_salt.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +salt '*' state.highstate pillar="$($(dirname ${0})/pillar_config.rb)" +exit 0 diff --git a/ci-script.sh b/ci-script.sh new file mode 100644 index 0000000..7865a83 --- /dev/null +++ b/ci-script.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +buildroot="$(pwd)" + +# clean docker containers +if [ -x ${buildroot}/docker_build/ci/openbookctl ]; then + ${buildroot}/docker_build/ci/openbookctl stop +fi +for i in $(docker ps | grep -v 'COMMAND' | grep -v 'database' | awk '{print $1}'); do + docker stop $i +done +for i in $(docker ps | grep -v 'COMMAND' | grep -F 'database' | awk '{print $1}'); do + docker stop $i +done +docker container prune -f +for i in $(docker images | grep -v 'REPOSITORY' | awk '{print $3}'); do + docker rmi --force $i +done + +# clean the build files +rm -rf ${buildroot}/openbook_build /etc/openbook /var/lib/mysql /var/log/mysql /var/log/openbook + +# clone repo +git clone git@txslnno-github.cds.eng.vzwcorp.com:konstca/openbook-installer.git +cp ${HOME}/config.yaml /srv/bootstrap/ + +# run salt to create build scripts +/srv/bootstrap/run_salt.sh + +# build new docker containers +openbook_build/ci/build.sh + +exit 0 diff --git a/pillar/openbook/admin_web_ui.sls b/pillar/openbook/admin_web_ui.sls new file mode 100644 index 0000000..40e63a7 --- /dev/null +++ b/pillar/openbook/admin_web_ui.sls @@ -0,0 +1,72 @@ +admin_web_ui: + dirs: + data: + name: data + mode: "0755" + relative_to_buildroot: True + etc_openbook_admin_web_ui: + name: /etc/openbook/admin-web-ui + mode: "0777" + relative_to_buildroot: False + data_etc_openbook_admin_web_ui: + name: data/etc/openbook/admin-web-ui + mode: "0777" + relative_to_buildroot: True + var_log_openbook_admin_web_ui: + name: /var/log/openbook/admin-web-ui + mode: "0777" + relative_to_buildroot: False + data_var_log_openbook_admin_web_ui: + name: data/var/log/openbook/admin-web-ui + mode: "0777" + relative_to_buildroot: True + files: + build_sh: + name: build.sh + path: . + source: build.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + run_sh: + name: run.sh + path: . + source: run.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: True + dockerfile: + name: Dockerfile + path: . + source: Dockerfile + mode: "0644" + is_template: True + relative_to_buildroot: True + ansible: False + installdeps: + name: installdeps.sh + path: data + source: installdeps.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + start_app: + name: start_app.sh + path: data + source: start_app.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + admin_web_ui_json: + name: admin-web-ui.json + path: /etc/openbook/admin-web-ui + source: admin_web_ui.json + mode: "0644" + is_template: True + relative_to_buildroot: False + ansible: True + log: /var/log/openbook/admin-web-ui/admin-web-ui.log diff --git a/pillar/openbook/ansible.sls b/pillar/openbook/ansible.sls new file mode 100644 index 0000000..f60f4f1 --- /dev/null +++ b/pillar/openbook/ansible.sls @@ -0,0 +1,173 @@ +ansible: + dirs: + conf: + name: conf + mode: "0755" + relative_to_buildroot: True + playbooks: + name: playbooks + mode: "0755" + relative_to_buildroot: True + files: + name: files + mode: "0755" + relative_to_buildroot: True + templates: + name: templates + mode: "0755" + relative_to_buildroot: True + output: + name: /opt/openbook + mode: "0755" + relative_to_buildroot: False + files: + playbooks_install_openbook_yaml: + name: playbooks/install_openbook.yaml + source: install_openbook.yaml + mode: "0644" + is_template: False + playbooks_install_docker_registry_yaml: + name: playbooks/install_docker_registry.yaml + source: install_docker_registry.yaml + mode: "0644" + is_template: False + playbooks_install_openbook_docker_slave_yaml: + name: playbooks/install_openbook_docker_slave.yaml + source: install_openbook_docker_slave.yaml + mode: "0644" + is_template: False + playbooks_salt_yaml: + name: playbooks/salt.yaml + source: salt.yaml + mode: "0644" + is_template: False + playbooks_docker_yaml: + name: playbooks/docker.yaml + source: docker.yaml + mode: "0644" + is_template: False + playbooks_docker_compose_yaml: + name: playbooks/docker_compose.yaml + source: docker_compose.yaml + mode: "0644" + is_template: False + playbooks_docker_registry_yaml: + name: playbooks/docker_registry.yaml + source: docker_registry.yaml + mode: "0644" + is_template: False + playbooks_mysql_client_yaml: + name: playbooks/mysql_client.yaml + source: mysql_client.yaml + mode: "0644" + is_template: False + playbooks_openjdk_yaml: + name: playbooks/openjdk.yaml + source: openjdk.yaml + mode: "0644" + is_template: False + playbooks_ssh_keys_yaml: + name: playbooks/ssh_keys.yaml + source: ssh_keys.yaml + mode: "0644" + is_template: False + playbooks_jenkins_user_yaml: + name: playbooks/jenkins_user.yaml + source: jenkins_user.yaml + mode: "0644" + is_template: False + playbooks_build_yaml: + name: playbooks/build.yaml + source: build.yaml + mode: "0644" + is_template: False + playbooks_openbook_yaml: + name: playbooks/openbook.yaml + source: openbook.yaml + mode: "0644" + is_template: True + saltstack_list: + name: files/saltstack.list + source: saltstack.list + mode: "0644" + is_template: False + docker_list: + name: files/docker.list + source: docker.list + mode: "0644" + is_template: False + mariadb_list: + name: files/mariadb.list + source: mariadb.list + mode: "0644" + is_template: False + salt_master: + name: files/master + source: salt_master + mode: "0644" + is_template: False + docker_registry_ubuntu: + name: files/docker-registry.ubuntu + source: docker-registry.ubuntu + mode: "0644" + is_template: False + docker_registry_gentoo: + name: files/docker-registry.gentoo + source: docker-registry.gentoo + mode: "0644" + is_template: False + docker_compose_yaml: + name: files/docker-compose.yaml + source: docker-compose.yaml + mode: "0644" + is_template: False + ci_script_sh: + name: files/ci-script.sh + source: ci-script.sh + mode: "0644" + is_template: False + slave_jar: + name: files/slave.jar + source: slave.jar + mode: "0644" + is_template: False + registry_conf: + name: templates/registry.conf + source: registry.conf + mode: "0644" + is_template: True + conf_env_yaml_openbook_example: + name: conf/env.yaml.openbook.example + source: env.yaml.openbook.example + mode: "0644" + is_template: False + conf_env_yaml_docker_registry_example: + name: conf/env.yaml.docker_registry.example + source: env.yaml.docker_registry.example + mode: "0644" + is_template: False + conf_env_yaml_openbook_docker_slave_example: + name: conf/env.yaml.openbook_docker_slave.example + source: env.yaml.openbook_docker_slave.example + mode: "0644" + is_template: False + inventory_json_rb: + name: inventory_json.rb + source: inventory_json.rb + mode: "0755" + is_template: False + openbook_installer_yaml: + name: openbook_installer.yaml + source: openbook_installer.yaml + mode: "0644" + is_template: False + docker_registry_installer_yaml: + name: docker_registry_installer.yaml + source: docker_registry_installer.yaml + mode: "0644" + is_template: False + openbook_docker_slave_installer_yaml: + name: openbook_docker_slave_installer.yaml + source: openbook_docker_slave_installer.yaml + mode: "0644" + is_template: False diff --git a/pillar/openbook/api_server.sls b/pillar/openbook/api_server.sls new file mode 100644 index 0000000..6ee24c7 --- /dev/null +++ b/pillar/openbook/api_server.sls @@ -0,0 +1,72 @@ +api_server: + dirs: + data: + name: data + mode: "0755" + relative_to_buildroot: True + etc_openbook_api_server: + name: /etc/openbook/api-server + mode: "0777" + relative_to_buildroot: False + data_etc_openbook_api_server: + name: data/etc/openbook/api-server + mode: "0777" + relative_to_buildroot: True + var_log_openbook_api_server: + name: /var/log/openbook/api-server + mode: "0777" + relative_to_buildroot: False + data_var_log_openbook_api_server: + name: data/var/log/openbook/api-server + mode: "0777" + relative_to_buildroot: True + files: + build_sh: + name: build.sh + path: . + source: build.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + run_sh: + name: run.sh + path: . + source: run.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: True + dockerfile: + name: Dockerfile + path: . + source: Dockerfile + mode: "0644" + is_template: True + relative_to_buildroot: True + ansible: False + installdeps: + name: installdeps.sh + path: data + source: installdeps.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + start_app: + name: start_app.sh + path: data + source: start_app.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + api_server_json: + name: api-server.json + path: /etc/openbook/api-server + source: api_server.json + mode: "0644" + is_template: True + relative_to_buildroot: False + ansible: True + log: /var/log/openbook/api-server/api-server.log diff --git a/pillar/openbook/baseimage.sls b/pillar/openbook/baseimage.sls new file mode 100644 index 0000000..89c3df1 --- /dev/null +++ b/pillar/openbook/baseimage.sls @@ -0,0 +1,36 @@ +baseimage: + build_dir: /root/openbook_build/baseimage + dirs: + data: + name: data + mode: "0755" + relative_to_data: True + data_etc_apt: + name: data/etc/apt + mode: "0755" + relative_to_data: True + files: + build_sh: + name: build.sh + source: build.sh + mode: "0755" + is_template: True + relative_to_data: True + dockerfile: + name: Dockerfile + source: Dockerfile + mode: "0644" + is_template: False + relative_to_data: True + sources_list: + name: data/etc/apt/sources.list + source: sources.list + mode: "0644" + is_template: False + relative_to_data: True + installdeps: + name: data/installdeps.sh + source: installdeps.sh + mode: "0755" + is_template: True + relative_to_data: True diff --git a/pillar/openbook/ci.sls b/pillar/openbook/ci.sls new file mode 100644 index 0000000..37bf397 --- /dev/null +++ b/pillar/openbook/ci.sls @@ -0,0 +1,56 @@ +ci: + build_dir: /root/openbook_build/ci + dirs: + data: + name: data + mode: "0755" + relative_to_buildroot: True + files: + build_sh: + name: build.sh + path: . + source: build.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + openbookctl: + name: openbookctl + path: . + source: openbookctl + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: True + cloudcfg_sh: + name: cloudcfg.sh + path: . + source: cloudcfg.sh + mode: "0755" + is_template: False + relative_to_buildroot: True + ansible: True + openbook_ubuntu: + name: openbook.ubuntu + path: . + source: openbook.ubuntu + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: True + openbook_redhat: + name: openbook.redhat + path: . + source: openbook.redhat + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: True + openbook_gentoo: + name: openbook.gentoo + path: . + source: openbook.gentoo + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: True diff --git a/pillar/openbook/customer_web_ui.sls b/pillar/openbook/customer_web_ui.sls new file mode 100644 index 0000000..e38d4ef --- /dev/null +++ b/pillar/openbook/customer_web_ui.sls @@ -0,0 +1,72 @@ +customer_web_ui: + dirs: + data: + name: data + mode: "0755" + relative_to_buildroot: True + etc_openbook_customer_web_ui: + name: /etc/openbook/customer-web-ui + mode: "0777" + relative_to_buildroot: False + data_etc_openbook_customer_web_ui: + name: data/etc/openbook/customer-web-ui + mode: "0777" + relative_to_buildroot: True + var_log_openbook_customer_web_ui: + name: /var/log/openbook/customer-web-ui + mode: "0777" + relative_to_buildroot: False + data_log_openbook_customer_web_ui: + name: data/var/log/openbook/customer-web-ui + mode: "0777" + relative_to_buildroot: True + files: + build_sh: + name: build.sh + path: . + source: build.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + run_sh: + name: run.sh + path: . + source: run.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: True + dockerfile: + name: Dockerfile + path: . + source: Dockerfile + mode: "0644" + is_template: True + relative_to_buildroot: True + ansible: False + installdeps: + name: installdeps.sh + path: data + source: installdeps.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + start_app: + name: start_app.sh + path: data + source: start_app.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + customer_web_ui_json: + name: customer-web-ui.json + path: /etc/openbook/customer-web-ui + source: customer_web_ui.json + mode: "0644" + is_template: True + relative_to_buildroot: False + ansible: True + log: /var/log/openbook/customer-web-ui/customer-web-ui.log diff --git a/pillar/openbook/database.sls b/pillar/openbook/database.sls new file mode 100644 index 0000000..e495cf2 --- /dev/null +++ b/pillar/openbook/database.sls @@ -0,0 +1,135 @@ +database: + dirs: + var_lib_mysql: + name: /var/lib/mysql + mode: "0777" + relative_to_buildroot: False + var_log_mysql: + name: /var/log/mysql + mode: "0777" + relative_to_buildroot: False + data_usr_local_bin: + name: data/usr/local/bin + mode: "0755" + relative_to_buildroot: True + etc_openbook_database: + name: /etc/openbook/database + mode: "0755" + relative_to_buildroot: False + etc_openbook_database_conf.d: + name: /etc/openbook/database/conf.d + mode: "0755" + relative_to_buildroot: False + etc_openbook_database_bin: + name: /etc/openbook/database/bin + mode: "0755" + relative_to_buildroot: False + data_root: + name: data/root + mode: "0700" + relative_to_buildroot: True + files: + build_sh: + name: build.sh + path: . + source: build.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + run_sh: + name: run.sh + path: . + source: run.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: True + dockerfile: + name: Dockerfile + path: . + source: Dockerfile + mode: "0644" + is_template: True + relative_to_buildroot: True + ansible: False + root_my_cnf: + name: .my.cnf + path: /root + source: _my.cnf + mode: "0644" + is_template: True + relative_to_buildroot: False + ansible: True + _my_cnf: + name: .my.cnf + path: data/root + source: _my.cnf + mode: "0644" + is_template: True + relative_to_buildroot: True + ansible: False + my_cnf: + name: my.cnf + path: /etc/openbook/database + source: my.cnf + mode: "0644" + is_template: False + relative_to_buildroot: False + ansible: True + create_openbook_schemas: + name: create_openbook_schemas.sql + path: /etc/openbook/database + source: create_openbook_schemas.sql + mode: "0644" + is_template: True + relative_to_buildroot: False + ansible: True + debian_cnf: + name: debian.cnf + path: /etc/openbook/database + source: debian.cnf + mode: "0644" + is_template: True + relative_to_buildroot: False + ansible: True + cluster_cnf: + name: cluster.cnf + path: /etc/openbook/database/conf.d + source: cluster.cnf + mode: "0644" + is_template: True + relative_to_buildroot: False + ansible: True + debconfrc: + name: debconfrc + path: /etc/openbook/database/conf.d + source: debconfrc + mode: "0644" + is_template: True + relative_to_buildroot: False + ansible: True + installdeps: + name: installdeps.sh + path: data + source: installdeps.sh + mode: "0755" + is_template: True + relative_to_buildroot: True + ansible: False + start_galera_node_sh: + name: start_galera_node.sh + path: /etc/openbook/database/bin + source: start_galera_node.sh + mode: "0755" + is_template: True + relative_to_buildroot: False + ansible: True + fix_permissions_sh: + name: fix_permissions.sh + path: /etc/openbook/database/bin + source: fix_permissions.sh + mode: "0755" + is_template: True + relative_to_buildroot: False + ansible: True diff --git a/pillar/top.sls b/pillar/top.sls new file mode 100644 index 0000000..f471540 --- /dev/null +++ b/pillar/top.sls @@ -0,0 +1,9 @@ +base: + "*": + - openbook.ci + - openbook.ansible + - openbook.baseimage + - openbook.database + - openbook.api_server + - openbook.admin_web_ui + - openbook.customer_web_ui diff --git a/salt/_modules/logger_mod.py b/salt/_modules/logger_mod.py new file mode 100644 index 0000000..708483c --- /dev/null +++ b/salt/_modules/logger_mod.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +''' +Execution module to provide logging +========================== +:maintainer: Joe Julian <me@joejulian.name> +:maturity: new +:platform: Linux +.. versionadded:: 2014.7.0 +''' + +__virtualname__ = 'logger' + +import logging as log +from pprint import pformat +import json +import yaml + +def pretty(var): + return pformat(yaml.load(json.dumps(var))) + +def __virtual__(): + return __virtualname__ + +def debug(var, string = ""): + ''' + Print a var to a debug output. + ''' + ret = string + pretty(var) + log.debug(ret) + return ret + +def info(var, string = ""): + ''' + Print a var to a info output. + ''' + ret = string + pretty(var) + log.info(ret) + return ret + +def warning(var, string = ""): + ''' + Print a var to a warning output. + ''' + ret = string + pretty(var) + log.warning(ret) + return ret + +def error(var, string = ""): + ''' + Print a var to a error output. + ''' + ret = string + pretty(var) + log.error(ret) + return ret + +def critical(var, string = ""): + ''' + Print a var to a critical output. + ''' + ret = string + pretty(var) + log.critical(ret) + return ret diff --git a/salt/_states/logging.py b/salt/_states/logging.py new file mode 100644 index 0000000..35856cc --- /dev/null +++ b/salt/_states/logging.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +''' +Logging output +========================== +:maintainer: Joe Julian <me@joejulian.name> +:maturity: new +:platform: Linux +.. versionadded:: 2014.7.0 +:configuration: See :py:mod:`salt.modules.logging` for setup instructions. +.. code-block:: yaml + debugmyvar: + logger.debug: + - name: myvar + - string: "myvar is :" +''' + +__virtual_name__ = 'logging' + +def __virtual__(): + return __virtual_name__ + +def debug(name, obj, string = ''): + ret = {'name': name, + 'changes': [], + 'result': True, + 'comment': __salt__['logger.debug'](obj, string)} + return ret + +def info(name, obj, string = ''): + ret = {'name': name, + 'changes': [], + 'result': True, + 'comment': __salt__['logger.info'](obj, string)} + return ret + +def warning(name, obj, string = ''): + ret = {'name': name, + 'changes': [], + 'result': True, + 'comment': __salt__['logger.warning'](obj, string)} + return ret + +def error(name, obj, string = ''): + ret = {'name': name, + 'changes': [], + 'result': True, + 'comment': __salt__['logger.error'](obj, string)} + return ret + +def critical(name, obj, string = ''): + ret = {'name': name, + 'changes': [], + 'result': True, + 'comment': __salt__['logger.critical'](obj, string)} + return ret diff --git a/salt/openbook/admin_web_ui/files/Dockerfile b/salt/openbook/admin_web_ui/files/Dockerfile new file mode 100644 index 0000000..4b57ce5 --- /dev/null +++ b/salt/openbook/admin_web_ui/files/Dockerfile @@ -0,0 +1,8 @@ +FROM {{ pillar["baseimage"]["image_name"] }} +MAINTAINER Carlos Konstanski <carlos@talligent.com> +ADD docker-start.tar.xz / +EXPOSE 8082 5701 5702 +VOLUME {{ pillar["admin_web_ui"]["dirs"]["etc_openbook_admin_web_ui"]["name"] }} +VOLUME {{ pillar["admin_web_ui"]["dirs"]["var_log_openbook_admin_web_ui"]["name"] }} +RUN /installdeps.sh +ENTRYPOINT [ "/start_app.sh" ] diff --git a/salt/openbook/admin_web_ui/files/admin_web_ui.json b/salt/openbook/admin_web_ui/files/admin_web_ui.json new file mode 100644 index 0000000..6008b92 --- /dev/null +++ b/salt/openbook/admin_web_ui/files/admin_web_ui.json @@ -0,0 +1,3 @@ +{ + "log.path": "{{ pillar["admin_web_ui"]["log"] }}" +} diff --git a/salt/openbook/admin_web_ui/files/build.sh b/salt/openbook/admin_web_ui/files/build.sh new file mode 100644 index 0000000..0964047 --- /dev/null +++ b/salt/openbook/admin_web_ui/files/build.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Do not call directly. Meant to be called from CI build script. + +pushd {{ pillar["admin_web_ui"]["build_dir"] }} 2>/dev/null + +echo "Creating tmp directory" +rm -rf tmp +mkdir tmp +rm -f docker-start.tar.xz + +echo "Populating with custom files" +rsync -aq data/ tmp/ + +echo "Repackaging into .xz file" +tar Jcvf docker-start.tar.xz -C tmp/ . + +echo "Deleting existing docker container" +docker stop {{ pillar["admin_web_ui"]["container_name"] }} +docker container prune -f +docker rmi -f {{ pillar["admin_web_ui"]["image_name"] }} + +echo "Building docker Ubuntu image" +docker build -t {{ pillar["admin_web_ui"]["image_name"] }} . + +echo "Push docker image to the registry" +docker tag {{ pillar["admin_web_ui"]["image_name"] }} {{ pillar["registry"]["url"] }}/{{ pillar["admin_web_ui"]["image_name"] }} +docker push {{ pillar["registry"]["url"] }}/{{ pillar["admin_web_ui"]["image_name"] }} + +echo "Cleaning up" +rm -rf tmp +rm -f docker-start.tar.xz + +popd 2>/dev/null +exit 0 diff --git a/salt/openbook/admin_web_ui/files/installdeps.sh b/salt/openbook/admin_web_ui/files/installdeps.sh new file mode 100644 index 0000000..7de3c7c --- /dev/null +++ b/salt/openbook/admin_web_ui/files/installdeps.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +apt-get -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" openjdk-8-jre-headless + +curl -k \ + -u '{{ pillar["jars"]["username"] }}:{{ pillar["jars"]["password"] }}' \ + -o "{{ pillar["admin_web_ui"]["jar_name"] }}" \ + "{{ pillar["jars"]["url"] }}{{ pillar["admin_web_ui"]["jar_name"] }}" + +exit 0 diff --git a/salt/openbook/admin_web_ui/files/run.sh b/salt/openbook/admin_web_ui/files/run.sh new file mode 100644 index 0000000..9669172 --- /dev/null +++ b/salt/openbook/admin_web_ui/files/run.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +docker run -d --net=host -v {{ pillar["admin_web_ui"]["dirs"]["etc_openbook_admin_web_ui"]["name"] }}:{{ pillar["admin_web_ui"]["dirs"]["etc_openbook_admin_web_ui"]["name"] }} -v {{ pillar["admin_web_ui"]["dirs"]["var_log_openbook_admin_web_ui"]["name"] }}:{{ pillar["admin_web_ui"]["dirs"]["var_log_openbook_admin_web_ui"]["name"] }} --name {{ pillar["admin_web_ui"]["container_name"] }} {{ pillar["registry"]["url"] }}/{{ pillar["admin_web_ui"]["image_name"] }} +exit 0 diff --git a/salt/openbook/admin_web_ui/files/start_app.sh b/salt/openbook/admin_web_ui/files/start_app.sh new file mode 100644 index 0000000..86daad4 --- /dev/null +++ b/salt/openbook/admin_web_ui/files/start_app.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exec java -jar {{ pillar["admin_web_ui"]["jar_name"] }} run com.talligent.openbook.verticle.Startup --cluster diff --git a/salt/openbook/admin_web_ui/init.sls b/salt/openbook/admin_web_ui/init.sls new file mode 100644 index 0000000..814cedb --- /dev/null +++ b/salt/openbook/admin_web_ui/init.sls @@ -0,0 +1,55 @@ +{% for dir in pillar["admin_web_ui"]["dirs"].items() %} +adminui_dirs_{{ dir[0] }}: + file.directory: +{% if dir[1]["relative_to_buildroot"] %} + - name: {{ pillar["admin_web_ui"]["build_dir"] }}/{{ dir[1]["name"] }} +{% else %} + - name: {{ dir[1]["name"] }} +{% endif %} + - user: root + - group: root + - mode: {{ dir[1]["mode"] }} + - makedirs: True +{% endfor %} + +{% for file in pillar["admin_web_ui"]["files"].items() %} +adminui_files_{{ file[0] }}: + file.managed: +{% if file[1]["relative_to_buildroot"] %} + - name: {{ pillar["admin_web_ui"]["build_dir"] }}/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% else %} + - name: {{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/admin_web_ui/files/{{ file[1]["source"] }} +{% if file[1]["is_template"] %} + - template: jinja +{% endif %} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% if file[1]["ansible"] %} +ansible_adminui_dirs_{{ file[0] }}: + file.directory: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/admin_web_ui/{{ file[1]["path"] }} +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/admin_web_ui/{{ file[1]["path"] }} +{% endif %} + - user: root + - group: root + - mode: "0755" + - makedirs: True +ansible_adminui_files_{{ file[0] }}: + file.managed: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/admin_web_ui/{{ file[1]["path"] }}/{{ file[1]["name"] }} + - template: jinja +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/admin_web_ui/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/admin_web_ui/files/{{ file[1]["source"] }} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% endif %} +{% endfor %} diff --git a/salt/openbook/ansible/files/docker-compose.yaml b/salt/openbook/ansible/files/docker-compose.yaml new file mode 100644 index 0000000..befd77f --- /dev/null +++ b/salt/openbook/ansible/files/docker-compose.yaml @@ -0,0 +1,16 @@ +nginx: + image: "nginx:1.9" + ports: + - 443:443 + links: + - registry:registry + volumes: + - ./nginx/:/etc/nginx/conf.d:ro +registry: + image: registry:2 + ports: + - 127.0.0.1:5000:5000 + environment: + REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /data + volumes: + - ./data:/data diff --git a/salt/openbook/ansible/files/docker-registry.gentoo b/salt/openbook/ansible/files/docker-registry.gentoo new file mode 100644 index 0000000..02883f6 --- /dev/null +++ b/salt/openbook/ansible/files/docker-registry.gentoo @@ -0,0 +1,30 @@ +#!/sbin/openrc-run +# Copyright 1999-2008 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: $ + +depend() { + docker +} + +start() { + ebegin "openbook: start" + docker-compose -f /root/docker-registry/docker-compose.yaml up -d + eend $? +} + +stop() { + ebegin "openstack: stop" + docker-compose -f /root/docker-registry/docker-compose.yaml stop + docker-compose -f /root/docker-registry/docker-compose.yaml rm -f + eend $? +} + +restart() { + stop + start +} + +status() { + docker-compose -f /root/docker-registry/docker-compose.yaml ps +} diff --git a/salt/openbook/ansible/files/docker-registry.ubuntu b/salt/openbook/ansible/files/docker-registry.ubuntu new file mode 100644 index 0000000..cd2e676 --- /dev/null +++ b/salt/openbook/ansible/files/docker-registry.ubuntu @@ -0,0 +1,44 @@ +#!/bin/sh +set -e + +### BEGIN INIT INFO +# Provides: docker-registry +# Required-Start: $docker +# Required-Stop: $docker +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Docker Registry +# Description: +# Docker Registry +### END INIT INFO + +export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin + +do_start() { + docker-compose -f /root/docker-registry/docker-compose.yaml up -d +} + +do_stop() { + docker-compose -f /root/docker-registry/docker-compose.yaml stop + docker-compose -f /root/docker-registry/docker-compose.yaml rm -f +} + +do_status() { + docker-compose -f /root/docker-registry/docker-compose.yaml ps +} + +case $1 in + start*) + do_start + ;; + stop*) + do_stop + ;; + restart*) + do_stop + do_start + ;; + status*) + do_status + ;; +esac diff --git a/salt/openbook/ansible/files/docker.list b/salt/openbook/ansible/files/docker.list new file mode 100644 index 0000000..318b353 --- /dev/null +++ b/salt/openbook/ansible/files/docker.list @@ -0,0 +1 @@ +deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable diff --git a/salt/openbook/ansible/files/docker.yaml b/salt/openbook/ansible/files/docker.yaml new file mode 100644 index 0000000..9ac5499 --- /dev/null +++ b/salt/openbook/ansible/files/docker.yaml @@ -0,0 +1,88 @@ +--- +- name: "Install docker" + hosts: target + tasks: + - name: "Install kernel extras" + apt: + pkg: "{{ item }}" + state: present + with_items: + - "linux-image-extra-{{ ansible_kernel }}" + - linux-image-extra-virtual + - apt-transport-https + - ca-certificates + - software-properties-common + when: ansible_distribution == "Ubuntu" + + - name: "Install docker apt key" + shell: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + when: ansible_distribution == "Ubuntu" + + - name: "Configure docker.list" + copy: + src: "../files/{{ item }}" + dest: "/etc/apt/sources.list.d/{{ item }}" + mode: 0644 + owner: root + group: root + with_items: + - docker.list + when: ansible_distribution == "Ubuntu" + + - name: "Update apt" + apt: + update_cache: yes + when: ansible_distribution == "Ubuntu" + + - name: "Install docker packages" + apt: + pkg: "{{ item }}" + state: present + with_items: + - docker-ce + when: ansible_distribution == "Ubuntu" + + - name: "Install required packages" + yum: + name: "{{ item }}" + state: present + with_items: + - yum-utils + - device-mapper-persistent-data + - lvm2 + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Add docker repository" + shell: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Update yum cache" + yum: + update_cache: yes + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Install required packages" + yum: + name: "{{ item }}" + state: present + with_items: + - docker-ce + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Install docker" + portage: + package: "{{ item }}" + update: yes + changed_use: yes + with_items: + - app-emulation/docker + when: ansible_distribution == "Gentoo" + + - name: "Enable docker" + service: + name: "{{ item.name }}" + enabled: yes + state: started + runlevel: "{{ item.runlevel }}" + with_items: + - { name: docker, runlevel: default } diff --git a/salt/openbook/ansible/files/docker_compose.yaml b/salt/openbook/ansible/files/docker_compose.yaml new file mode 100644 index 0000000..8333a4b --- /dev/null +++ b/salt/openbook/ansible/files/docker_compose.yaml @@ -0,0 +1,22 @@ +--- +- name: "Install docker-compose" + hosts: target + tasks: + - name: "Install packages" + apt: + pkg: "{{ item }}" + state: present + with_items: + - docker-compose + - apache2-utils + when: ansible_distribution == "Ubuntu" + + - name: "Install packsges" + portage: + package: "{{ item }}" + update: yes + changed_use: yes + with_items: + - app-emulation/docker-compose + - app-admin/apache-tools + when: ansible_distribution == "Gentoo" diff --git a/salt/openbook/ansible/files/docker_registry.yaml b/salt/openbook/ansible/files/docker_registry.yaml new file mode 100644 index 0000000..20dfb73 --- /dev/null +++ b/salt/openbook/ansible/files/docker_registry.yaml @@ -0,0 +1,65 @@ +--- +- name: "Install docker-registry" + hosts: target + tasks: + - name: "Create docker-registry directories" + file: + path: "{{ item.path }}" + state: directory + mode: "{{ item.mode }}" + with_items: + - { path: /root/docker-registry/nginx, mode: "0755" } + - { path: /root/docker-registry/data, mode: "0755" } + + - name: "Create docker-registry files" + template: + src: "../templates/{{ item.src }}" + dest: "{{ item.dest }}" + mode: "{{ item.mode }}" + owner: root + group: root + with_items: + - { src: registry.conf, dest: /root/docker-registry/nginx/registry.conf, mode: "0644" } + + - name: "Create docker-registry files" + copy: + src: "../files/{{ item.src }}" + dest: "{{ item.dest }}" + mode: "{{ item.mode }}" + owner: root + group: root + with_items: + - { src: docker-compose.yaml, dest: /root/docker-registry/docker-compose.yaml, mode: "0644" } + + - name: "Copy SSL certs into nginx directory" + shell: | + cp "/etc/ssl/certs/{{ cn }}.crt" /root/docker-registry/nginx/ + cp "/etc/ssl/certs/{{ cn }}.ca" /root/docker-registry/nginx/ + cp "/etc/ssl/private/{{ cn }}.key" /root/docker-registry/nginx/ + when: ansible_distribution == "Ubuntu" or ansible_distribution == "Gentoo" + + - name: "Create password file" + shell: htpasswd -c -b -s /root/docker-registry/nginx/registry.password '{{ username }}' '{{ password }}' + + - name: "Install init script" + copy: + src: ../files/docker-registry.ubuntu + dest: /etc/init.d/docker-registry + mode: "0755" + when: ansible_distribution == "Ubuntu" + + - name: "Install init script" + copy: + src: ../files/docker-registry.gentoo + dest: /etc/init.d/docker-registry + mode: "0755" + when: ansible_distribution == "Gentoo" + + - name: "Enable docker-registry" + service: + name: "{{ item.name }}" + enabled: yes + state: started + runlevel: "{{ item.runlevel }}" + with_items: + - { name: docker-registry, runlevel: default } diff --git a/salt/openbook/ansible/files/docker_registry_installer.yaml b/salt/openbook/ansible/files/docker_registry_installer.yaml new file mode 100644 index 0000000..a38e8eb --- /dev/null +++ b/salt/openbook/ansible/files/docker_registry_installer.yaml @@ -0,0 +1,3 @@ +--- +- name: "Install Openbook: Docker Edition" + include: playbooks/install_docker_registry.yaml diff --git a/salt/openbook/ansible/files/env.yaml.docker_registry.example b/salt/openbook/ansible/files/env.yaml.docker_registry.example new file mode 100644 index 0000000..ccdef39 --- /dev/null +++ b/salt/openbook/ansible/files/env.yaml.docker_registry.example @@ -0,0 +1,10 @@ +--- +globals: + ansible_ssh_user: root + ansible_python_interpreter: /usr/bin/python2 + +hosts: + - host1: + cn: ~ + username: ~ + password: ~ diff --git a/salt/openbook/ansible/files/env.yaml.openbook.example b/salt/openbook/ansible/files/env.yaml.openbook.example new file mode 100644 index 0000000..b8971e3 --- /dev/null +++ b/salt/openbook/ansible/files/env.yaml.openbook.example @@ -0,0 +1,11 @@ +--- +globals: + ansible_ssh_user: root + ansible_python_interpreter: /usr/bin/python2 + openbook_gcomm_addresses: ~ + +hosts: + - host1: + openbook_database_password: ~ + - host2: + openbook_database_password: ~ diff --git a/salt/openbook/ansible/files/env.yaml.openbook_docker_slave.example b/salt/openbook/ansible/files/env.yaml.openbook_docker_slave.example new file mode 100644 index 0000000..72a8549 --- /dev/null +++ b/salt/openbook/ansible/files/env.yaml.openbook_docker_slave.example @@ -0,0 +1,14 @@ +--- +globals: + ansible_ssh_user: root + locale: en_US.utf8 + ansible_python_interpreter: /usr/bin/python2 + salt_master: localhost + +hosts: + - host1: + database_root_password: ~ + docker_username: ~ + docker_password: ~ + webdav_username: ~ + webdav_password: ~ diff --git a/salt/openbook/ansible/files/install_docker_registry.yaml b/salt/openbook/ansible/files/install_docker_registry.yaml new file mode 100644 index 0000000..f22e643 --- /dev/null +++ b/salt/openbook/ansible/files/install_docker_registry.yaml @@ -0,0 +1,4 @@ +--- +- include: docker.yaml +- include: docker_compose.yaml +- include: docker_registry.yaml diff --git a/salt/openbook/ansible/files/install_openbook.yaml b/salt/openbook/ansible/files/install_openbook.yaml new file mode 100644 index 0000000..30b12c9 --- /dev/null +++ b/salt/openbook/ansible/files/install_openbook.yaml @@ -0,0 +1,4 @@ +--- +- include: docker.yaml +- include: mysql_client.yaml +- include: openbook.yaml diff --git a/salt/openbook/ansible/files/install_openbook_docker_slave.yaml b/salt/openbook/ansible/files/install_openbook_docker_slave.yaml new file mode 100644 index 0000000..05b6c50 --- /dev/null +++ b/salt/openbook/ansible/files/install_openbook_docker_slave.yaml @@ -0,0 +1,7 @@ +--- +- include: salt.yaml +- include: docker.yaml +- include: mysql_client.yaml +- include: openjdk.yaml +- include: jenkins_user.yaml +- include: ssh_keys.yaml diff --git a/salt/openbook/ansible/files/inventory_json.rb b/salt/openbook/ansible/files/inventory_json.rb new file mode 100644 index 0000000..8a01004 --- /dev/null +++ b/salt/openbook/ansible/files/inventory_json.rb @@ -0,0 +1,34 @@ +#!/usr/bin/ruby + +require "json" +require "yaml" + +def doit() + env_yaml_path = "#{File.dirname($0)}/conf/env.yaml" + if ! File.file?(env_yaml_path) + STDERR.puts "Unable to find env.yaml. Exiting." + exit(1) + end + globs = {} + hostvars = {} + outobj = {} + hosts = [] + yml = YAML.load_file(env_yaml_path) + yml["globals"].each do |key, value| + globs[key] = value + end + yml["hosts"].each do |h| + h.each do |host, attributes| + hosts.push(host) + attributes.each do |attribute, value| + hostvars[host] = globs.merge(attributes) + end + end + end + outobj["target"] = {"hosts" => hosts} + outobj["_meta"] = {"hostvars" => hostvars} + puts outobj.to_json +end + +doit() +exit(0) diff --git a/salt/openbook/ansible/files/jenkins_user.yaml b/salt/openbook/ansible/files/jenkins_user.yaml new file mode 100644 index 0000000..14085da --- /dev/null +++ b/salt/openbook/ansible/files/jenkins_user.yaml @@ -0,0 +1,15 @@ +- name: "Create jenkins user" + hosts: all + tasks: + - name: "Create jenkins group" + group: + name: jenkins + state: present + + - name: "Create jenkins user" + user: + name: jenkins + shell: /bin/bash + comment: "jenkins user" + home: /var/lib/jenkins + group: jenkins diff --git a/salt/openbook/ansible/files/mariadb.list b/salt/openbook/ansible/files/mariadb.list new file mode 100644 index 0000000..b357315 --- /dev/null +++ b/salt/openbook/ansible/files/mariadb.list @@ -0,0 +1 @@ +deb [arch=amd64,i386] http://sgp1.mirrors.digitalocean.com/mariadb/repo/10.1/ubuntu xenial main diff --git a/salt/openbook/ansible/files/mysql_client.yaml b/salt/openbook/ansible/files/mysql_client.yaml new file mode 100644 index 0000000..6ee58be --- /dev/null +++ b/salt/openbook/ansible/files/mysql_client.yaml @@ -0,0 +1,60 @@ +--- +- name: "Install mysql client" + hosts: target + tasks: + - name: "Install mariadb apt key" + shell: apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 + when: ansible_distribution == "Ubuntu" + + - name: "Configure mariadb.list" + copy: + src: "../files/{{ item }}" + dest: "/etc/apt/sources.list.d/{{ item }}" + mode: 0644 + owner: root + group: root + with_items: + - mariadb.list + when: ansible_distribution == "Ubuntu" + + - name: "Update apt" + apt: + update_cache: yes + when: ansible_distribution == "Ubuntu" + + - name: "Install mariadb-client package" + apt: + pkg: "{{ item }}" + state: present + with_items: + - mariadb-client-10.1 + when: ansible_distribution == "Ubuntu" + + - name: "Add mariadb repository" + yum_repository: + name: MariaDB + description: MariaDB + baseurl: http://yum.mariadb.org/10.1/centos7-amd64 + gpgkey: https://yum.mariadb.org/RPM-GPG-KEY-MariaDB + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Update yum cache" + yum: + update_cache: yes + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Install mariadb-client package" + yum: + name: "{{ item }}" + state: present + with_items: + - MariaDB-client + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Install mariadb-client package" + portage: + name: "{{ item }}" + state: present + with_items: + - dev-db/mariadb + when: ansible_distribution == "Gentoo" diff --git a/salt/openbook/ansible/files/openbook.yaml b/salt/openbook/ansible/files/openbook.yaml new file mode 100644 index 0000000..b6992ba --- /dev/null +++ b/salt/openbook/ansible/files/openbook.yaml @@ -0,0 +1,157 @@ +--- +- name: "Install Openbook" + hosts: target + tasks: + - name: "Create Openbook directories" + file: + path: "{% raw %}{{ item.path }}{% endraw %}" + state: directory + mode: "{% raw %}{{ item.mode }}{% endraw %}" + with_items: + - { path: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/database, mode: "0755" } + - { path: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/admin_web_ui, mode: "0755" } + - { path: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/api_server, mode: "0755" } + - { path: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/customer_web_ui, mode: "0755" } + - { path: {{ pillar["database"]["dirs"]["etc_openbook_database_conf.d"]["name"] }}, mode: "0755" } + - { path: {{ pillar["database"]["dirs"]["etc_openbook_database_bin"]["name"] }}, mode: "0755" } + - { path: {{ pillar["database"]["dirs"]["var_lib_mysql"]["name"] }}, mode: "0777" } + - { path: {{ pillar["database"]["dirs"]["var_log_mysql"]["name"] }}, mode: "0777" } + - { path: {{ pillar["api_server"]["dirs"]["etc_openbook_api_server"]["name"] }}, mode: "0777" } + - { path: {{ pillar["customer_web_ui"]["dirs"]["etc_openbook_customer_web_ui"]["name"] }}, mode: "0777" } + - { path: {{ pillar["admin_web_ui"]["dirs"]["etc_openbook_admin_web_ui"]["name"] }}, mode: "0777" } + - { path: {{ pillar["admin_web_ui"]["log"] }}, mode: "0777" } + - { path: {{ pillar["api_server"]["log"] }}, mode: "0777" } + - { path: {{ pillar["customer_web_ui"]["log"] }}, mode: "0777" } + + - name: "Create Openbook files" + template: + src: "../templates/{% raw %}{{ item.src }}{% endraw %}" + dest: "{% raw %}{{ item.dest }}{% endraw %}" + mode: "{% raw %}{{ item.mode }}{% endraw %}" + owner: root + group: root + with_items: + - { src: ci/{{ pillar["ci"]["files"]["openbookctl"]["path"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }}, dest: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }}, mode: "0755" } + - { src: database/{{ pillar["database"]["files"]["run_sh"]["path"] }}/{{ pillar["database"]["files"]["run_sh"]["name"] }}, dest: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/database/{{ pillar["database"]["files"]["run_sh"]["name"] }}, mode: "0755" } + - { src: admin_web_ui/{{ pillar["admin_web_ui"]["files"]["run_sh"]["path"] }}/{{ pillar["admin_web_ui"]["files"]["run_sh"]["name"] }}, dest: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/admin_web_ui/{{ pillar["admin_web_ui"]["files"]["run_sh"]["name"] }}, mode: "0755" } + - { src: api_server/{{ pillar["api_server"]["files"]["run_sh"]["path"] }}/{{ pillar["api_server"]["files"]["run_sh"]["name"] }}, dest: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/api_server/{{ pillar["api_server"]["files"]["run_sh"]["name"] }}, mode: "0755" } + - { src: customer_web_ui/{{ pillar["customer_web_ui"]["files"]["run_sh"]["path"] }}/{{ pillar["customer_web_ui"]["files"]["run_sh"]["name"] }}, dest: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/customer_web_ui/{{ pillar["customer_web_ui"]["files"]["run_sh"]["name"] }}, mode: "0755" } + - { src: admin_web_ui/{{ pillar["admin_web_ui"]["files"]["admin_web_ui_json"]["path"] }}/{{ pillar["admin_web_ui"]["files"]["admin_web_ui_json"]["name"] }}, dest: {{ pillar["admin_web_ui"]["files"]["admin_web_ui_json"]["path"] }}/{{ pillar["admin_web_ui"]["files"]["admin_web_ui_json"]["name"] }}, mode: "0644" } + - { src: api_server/{{ pillar["api_server"]["files"]["api_server_json"]["path"] }}/{{ pillar["api_server"]["files"]["api_server_json"]["name"] }}, dest: {{ pillar["api_server"]["files"]["api_server_json"]["path"] }}/{{ pillar["api_server"]["files"]["api_server_json"]["name"] }}, mode: "0644" } + - { src: customer_web_ui/{{ pillar["customer_web_ui"]["files"]["customer_web_ui_json"]["path"] }}/{{ pillar["customer_web_ui"]["files"]["customer_web_ui_json"]["name"] }}, dest: {{ pillar["customer_web_ui"]["files"]["customer_web_ui_json"]["path"] }}/{{ pillar["customer_web_ui"]["files"]["customer_web_ui_json"]["name"] }}, mode: "0644" } + - { src: database/{{ pillar["database"]["files"]["root_my_cnf"]["path"] }}/{{ pillar["database"]["files"]["root_my_cnf"]["name"] }}, dest: {{ pillar["database"]["files"]["root_my_cnf"]["path"] }}/{{ pillar["database"]["files"]["root_my_cnf"]["name"] }}, mode: "0644" } + - { src: database/{{ pillar["database"]["files"]["create_openbook_schemas"]["path"] }}/{{ pillar["database"]["files"]["create_openbook_schemas"]["name"] }}, dest: {{ pillar["database"]["files"]["create_openbook_schemas"]["path"] }}/{{ pillar["database"]["files"]["create_openbook_schemas"]["name"] }}, mode: "0755" } + - { src: database/{{ pillar["database"]["files"]["debian_cnf"]["path"] }}/{{ pillar["database"]["files"]["debian_cnf"]["name"] }}, dest: {{ pillar["database"]["files"]["debian_cnf"]["path"] }}/{{ pillar["database"]["files"]["debian_cnf"]["name"] }}, mode: "0644" } + - { src: database/{{ pillar["database"]["files"]["cluster_cnf"]["path"] }}/{{ pillar["database"]["files"]["cluster_cnf"]["name"] }}, dest: {{ pillar["database"]["files"]["cluster_cnf"]["path"] }}/{{ pillar["database"]["files"]["cluster_cnf"]["name"] }}, mode: "0644" } + - { src: database/{{ pillar["database"]["files"]["debconfrc"]["path"] }}/{{ pillar["database"]["files"]["debconfrc"]["name"] }}, dest: {{ pillar["database"]["files"]["debconfrc"]["path"] }}/{{ pillar["database"]["files"]["debconfrc"]["name"] }}, mode: "0644" } + - { src: database/{{ pillar["database"]["files"]["fix_permissions_sh"]["path"] }}/{{ pillar["database"]["files"]["fix_permissions_sh"]["name"] }}, dest: {{ pillar["database"]["files"]["fix_permissions_sh"]["path"] }}/{{ pillar["database"]["files"]["fix_permissions_sh"]["name"] }}, mode: "0755" } + - { src: database/{{ pillar["database"]["files"]["start_galera_node_sh"]["path"] }}/{{ pillar["database"]["files"]["start_galera_node_sh"]["name"] }}, dest: {{ pillar["database"]["files"]["start_galera_node_sh"]["path"] }}/{{ pillar["database"]["files"]["start_galera_node_sh"]["name"] }}, mode: "0755" } + + - name: "Create Openbook files" + copy: + src: "../files/{% raw %}{{ item.src }}{% endraw %}" + dest: "{% raw %}{{ item.dest }}{% endraw %}" + mode: "{% raw %}{{ item.mode }}{% endraw %}" + owner: root + group: root + with_items: + - { src: ci/{{ pillar["ci"]["files"]["cloudcfg_sh"]["path"] }}/{{ pillar["ci"]["files"]["cloudcfg_sh"]["name"] }}, dest: {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["cloudcfg_sh"]["name"] }}, mode: "0755" } + - { src: database/{{ pillar["database"]["files"]["my_cnf"]["path"] }}/{{ pillar["database"]["files"]["my_cnf"]["name"] }}, dest: {{ pillar["database"]["files"]["my_cnf"]["path"] }}/{{ pillar["database"]["files"]["my_cnf"]["name"] }}, mode: "0644" } + + - name: "Create Openbook init script: Ubuntu" + template: + src: "../templates/{% raw %}{{ item.src }}{% endraw %}" + dest: "{% raw %}{{ item.dest }}{% endraw %}" + mode: "{% raw %}{{ item.mode }}{% endraw %}" + owner: root + group: root + with_items: + - { src: ci/{{ pillar["ci"]["files"]["openbook_ubuntu"]["path"] }}/{{ pillar["ci"]["files"]["openbook_ubuntu"]["name"] }}, dest: /etc/init.d/openbook, mode: "0755" } + when: ansible_distribution == "Ubuntu" + + - name: "Create Openbook init script: Redhat" + template: + src: "../templates/{% raw %}{{ item.src }}{% endraw %}" + dest: "{% raw %}{{ item.dest }}{% endraw %}" + mode: "{% raw %}{{ item.mode }}{% endraw %}" + owner: root + group: root + with_items: + - { src: ci/{{ pillar["ci"]["files"]["openbook_redhat"]["path"] }}/{{ pillar["ci"]["files"]["openbook_redhat"]["name"] }}, dest: /etc/init.d/openbook, mode: "0755" } + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Create Openbook init script: Gentoo" + template: + src: "../templates/{% raw %}{{ item.src }}{% endraw %}" + dest: "{% raw %}{{ item.dest }}{% endraw %}" + mode: "{% raw %}{{ item.mode }}{% endraw %}" + owner: root + group: root + with_items: + - { src: ci/{{ pillar["ci"]["files"]["openbook_gentoo"]["path"] }}/{{ pillar["ci"]["files"]["openbook_gentoo"]["name"] }}, dest: /etc/init.d/openbook, mode: "0755" } + when: ansible_distribution == "Gentoo" + + - name: "Add service script to systemd" + shell: systemctl enable openbook + when: ansible_distribution == "Ubuntu" or ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Enable openbook service" + service: + name: "{% raw %}{{ item.name }}{% endraw %}" + enabled: yes + runlevel: "{% raw %}{{ item.runlevel }}{% endraw %}" + with_items: + - { name: openbook, runlevel: default } + + - name: "Delete existing docker images" + shell: | + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} stop || true + for i in $(docker images | grep -v CREATED | awk '{print $3}'); do + docker rmi --force $i + done + docker container prune -f + ignore_errors: True + + - name: "Pull docker images" + shell: | + docker login --username {{ pillar["registry"]["username"] }} --password {{ pillar["registry"]["password"] }} {{ pillar["registry"]["url"] }} + docker pull {{ pillar["registry"]["url"] }}/{{ pillar["database"]["image_name"] }} + docker pull {{ pillar["registry"]["url"] }}/{{ pillar["api_server"]["image_name"] }} + docker pull {{ pillar["registry"]["url"] }}/{{ pillar["admin_web_ui"]["image_name"] }} + docker pull {{ pillar["registry"]["url"] }}/{{ pillar["customer_web_ui"]["image_name"] }} + + - name: "Write hosts entry" + lineinfile: + dest: /etc/hosts + regexp: "{% raw %}{{ item.regexp }}{% endraw %}" + line: "{% raw %}{{ item.line }}{% endraw %}" + with_items: + - { regexp: '^127\.0\.0\.1.*$', line: "127.0.0.1 localhost {% raw %}{{ ansible_nodename }}{% endraw %}" } + + - name: "Install network tools" + apt: + pkg: "{% raw %}{{ item }}{% endraw %}" + state: present + with_items: + - netcat-openbsd + - lsof + when: ansible_distribution == "Ubuntu" + + - name: "Install network tools" + yum: + name: "{% raw %}{{ item }}{% endraw %}" + state: present + with_items: + - nmap-ncat + - lsof + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Install network tools" + portage: + package: "{% raw %}{{ item }}{% endraw %}" + update: yes + changed_use: yes + with_items: + - net-analyzer/netcat + - sys-process/lsof + when: ansible_distribution == "Gentoo" diff --git a/salt/openbook/ansible/files/openbook_docker_slave_installer.yaml b/salt/openbook/ansible/files/openbook_docker_slave_installer.yaml new file mode 100644 index 0000000..4f3dd07 --- /dev/null +++ b/salt/openbook/ansible/files/openbook_docker_slave_installer.yaml @@ -0,0 +1,3 @@ +--- +- name: "Install Openbook jenkins slave for building docker images" + include: playbooks/install_openbook_docker_slave.yaml diff --git a/salt/openbook/ansible/files/openbook_installer.yaml b/salt/openbook/ansible/files/openbook_installer.yaml new file mode 100644 index 0000000..3e1f01a --- /dev/null +++ b/salt/openbook/ansible/files/openbook_installer.yaml @@ -0,0 +1,3 @@ +--- +- name: "Install Openbook: Docker Edition" + include: playbooks/install_openbook.yaml diff --git a/salt/openbook/ansible/files/openjdk.yaml b/salt/openbook/ansible/files/openjdk.yaml new file mode 100644 index 0000000..a016866 --- /dev/null +++ b/salt/openbook/ansible/files/openjdk.yaml @@ -0,0 +1,44 @@ +--- +- name: "Install openjdk" + hosts: target + tasks: + - name: "Update apt" + apt: + update_cache: yes + when: ansible_distribution == "Ubuntu" + + - name: "Install openjdk package" + apt: + pkg: "{{ item }}" + state: present + with_items: + - openjdk-8-jre-headless + when: ansible_distribution == "Ubuntu" + + - name: "Update yum cache" + yum: + update_cache: yes + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Install openjdk package" + yum: + name: "{{ item }}" + state: present + with_items: + - openjdk + when: ansible_distribution == "RedHat" or ansible_distribution == "CentOS" + + - name: "Install icedtea package" + portage: + name: "{{ item }}" + state: present + with_items: + - dev-java/icedtea:8 + when: ansible_distribution == "Gentoo" +<<<<<<< HEAD +======= + + - name: "Set icedtea-8 as default java" + shell: java-config -S icedtea-8 + when: ansible_distribution == "Gentoo" +>>>>>>> 5d6764d17db282814ad18a15f08a74115dfa36f6 diff --git a/salt/openbook/ansible/files/registry.conf b/salt/openbook/ansible/files/registry.conf new file mode 100644 index 0000000..e97729a --- /dev/null +++ b/salt/openbook/ansible/files/registry.conf @@ -0,0 +1,39 @@ +upstream docker-registry { + server registry:5000; +} + +server { + listen 443; + server_name {% raw %}{{ cn }}{% endraw %}; + + # SSL + ssl on; + ssl_certificate /etc/nginx/conf.d/{% raw %}{{ cn }}{% endraw %}.crt; + ssl_certificate_key /etc/nginx/conf.d/{% raw %}{{ cn }}{% endraw %}.key; + + # disable any limits to avoid HTTP 413 for large image uploads + client_max_body_size 0; + + # required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486) + chunked_transfer_encoding on; + + location /v2/ { + # Do not allow connections from docker 1.5 and earlier + # docker pre-1.6.0 did not properly set the user agent on ping, catch "Go *" user agents + if ($http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" ) { + return 404; + } + + # To add basic authentication to v2 use auth_basic setting plus add_header + auth_basic "registry.localhost"; + auth_basic_user_file /etc/nginx/conf.d/registry.password; + add_header 'Docker-Distribution-Api-Version' 'registry/2.0' always; + + proxy_pass http://docker-registry; + proxy_set_header Host $http_host; # required for docker client's sake + proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + } +} diff --git a/salt/openbook/ansible/files/salt.yaml b/salt/openbook/ansible/files/salt.yaml new file mode 100644 index 0000000..d0581ae --- /dev/null +++ b/salt/openbook/ansible/files/salt.yaml @@ -0,0 +1,96 @@ +--- +- name: "Install salt" + hosts: all + tasks: + - name: "Create salt-master directories" + file: + path: "{{ item }}" + state: directory + mode: 0755 + owner: root + group: root + with_items: + - /srv + - /srv/keys + - /srv/reactor + + - name: "Create salt-master symlinks" + file: + src: "/root/openbook-installer/{{ item }}" + dest: "/srv/{{ item }}" + state: link + force: yes + with_items: + - bootstrap + - pillar + - salt + + - name: "Install salt apt key" + shell: curl -fsSL https://repo.saltstack.com/apt/ubuntu/16.04/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key add - + when: ansible_distribution == "Ubuntu" + + - name: "Configure saltstack.list" + copy: + src: "../files/{{ item }}" + dest: "/etc/apt/sources.list.d/{{ item }}" + mode: 0644 + owner: root + group: root + with_items: + - saltstack.list + when: ansible_distribution == "Ubuntu" + + - name: "Update apt" + apt: + update_cache: yes + when: ansible_distribution == "Ubuntu" + + - name: "Install salt packages" + apt: + pkg: "{{ item }}" + state: present + with_items: + - jq + - ruby + - salt-master + - salt-minion + when: ansible_distribution == "Ubuntu" + + - name: "Install salt packages" + portage: + package: "{{ item }}" + update: yes + changed_use: yes + with_items: + - app-misc/jq + - dev-lang/ruby + - app-admin/salt + when: ansible_distribution == "Gentoo" + + - name: "Configure salt-master" + copy: + src: "../files/{{ item }}" + dest: "/etc/salt/{{ item }}" + mode: 0644 + owner: root + group: root + with_items: + - master + + - name: "Configure salt-minion" + lineinfile: + dest: "/etc/salt/minion" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - { regexp: '^#master:.*$', line: "master: {{ salt_master }}" } + + - name: "Enable salt services" + service: + name: "{{ item.name }}" + enabled: yes + state: started + runlevel: "{{ item.runlevel }}" + with_items: + - { name: salt-master, runlevel: default } + - { name: salt-minion, runlevel: default } diff --git a/salt/openbook/ansible/files/salt_master b/salt/openbook/ansible/files/salt_master new file mode 100644 index 0000000..aa49105 --- /dev/null +++ b/salt/openbook/ansible/files/salt_master @@ -0,0 +1,1088 @@ +##### Primary configuration settings ##### +########################################## +# This configuration file is used to manage the behavior of the Salt Master. +# Values that are commented out but have an empty line after the comment are +# defaults that do not need to be set in the config. If there is no blank line +# after the comment then the value is presented as an example and is not the +# default. + +# Per default, the master will automatically include all config files +# from master.d/*.conf (master.d is a directory in the same directory +# as the main master config file). +#default_include: master.d/*.conf + +# The address of the interface to bind to: +interface: 127.0.0.1 + + +# Whether the master should listen for IPv6 connections. If this is set to True, +# the interface option must be adjusted, too. (For example: "interface: '::'") +#ipv6: False + +# The tcp port used by the publisher: +#publish_port: 4505 + +# The user under which the salt master will run. Salt will update all +# permissions to allow the specified user to run the master. The exception is +# the job cache, which must be deleted if this user is changed. If the +# modified files cause conflicts, set verify_env to False. +#user: root + +# The port used by the communication interface. The ret (return) port is the +# interface used for the file server, authentication, job returns, etc. +#ret_port: 4506 + +# Specify the location of the daemon process ID file: +#pidfile: /var/run/salt-master.pid + +# The root directory prepended to these options: pki_dir, cachedir, +# sock_dir, log_file, autosign_file, autoreject_file, extension_modules, +# key_logfile, pidfile: +#root_dir: / + +# The path to the master's configuration file. +#conf_file: /etc/salt/master + +# Directory used to store public key data: +#pki_dir: /etc/salt/pki/master + +# Key cache. Increases master speed for large numbers of accepted +# keys. Available options: 'sched'. (Updates on a fixed schedule.) +# Note that enabling this feature means that minions will not be +# available to target for up to the length of the maintanence loop +# which by default is 60s. +#key_cache: '' + +# Directory to store job and cache data: +# This directory may contain sensitive data and should be protected accordingly. +# +#cachedir: /var/cache/salt/master + +# Directory for custom modules. This directory can contain subdirectories for +# each of Salt's module types such as "runners", "output", "wheel", "modules", +# "states", "returners", etc. +#extension_modules: <no default> + +# Directory for custom modules. This directory can contain subdirectories for +# each of Salt's module types such as "runners", "output", "wheel", "modules", +# "states", "returners", "engines", etc. +# Like 'extension_modules' but can take an array of paths +#module_dirs: <no default> +# - /var/cache/salt/minion/extmods + +# Verify and set permissions on configuration directories at startup: +#verify_env: True + +# Set the number of hours to keep old job information in the job cache: +#keep_jobs: 24 + +# The number of seconds to wait when the client is requesting information +# about running jobs. +#gather_job_timeout: 10 + +# Set the default timeout for the salt command and api. The default is 5 +# seconds. +#timeout: 5 + +# The loop_interval option controls the seconds for the master's maintenance +# process check cycle. This process updates file server backends, cleans the +# job cache and executes the scheduler. +#loop_interval: 60 + +# Set the default outputter used by the salt command. The default is "nested". +#output: nested + +# Set the default output file used by the salt command. Default is to output +# to the CLI and not to a file. Functions the same way as the "--out-file" +# CLI option, only sets this to a single file for all salt commands. +#output_file: None + +# Return minions that timeout when running commands like test.ping +#show_timeout: True + +# By default, output is colored. To disable colored output, set the color value +# to False. +#color: True + +# Do not strip off the colored output from nested results and state outputs +# (true by default). +# strip_colors: False + +# To display a summary of the number of minions targeted, the number of +# minions returned, and the number of minions that did not return, set the +# cli_summary value to True. (False by default.) +# +#cli_summary: False + +# Set the directory used to hold unix sockets: +#sock_dir: /var/run/salt/master + +# The master can take a while to start up when lspci and/or dmidecode is used +# to populate the grains for the master. Enable if you want to see GPU hardware +# data for your master. +# enable_gpu_grains: False + +# The master maintains a job cache. While this is a great addition, it can be +# a burden on the master for larger deployments (over 5000 minions). +# Disabling the job cache will make previously executed jobs unavailable to +# the jobs system and is not generally recommended. +#job_cache: True + +# Cache minion grains, pillar and mine data via the cache subsystem in the +# cachedir or a database. +#minion_data_cache: True + +# Cache subsystem module to use for minion data cache. +#cache: localfs +# Enables a fast in-memory cache booster and sets the expiration time. +#memcache_expire_seconds: 0 +# Set a memcache limit in items (bank + key) per cache storage (driver + driver_opts). +#memcache_max_items: 1024 +# Each time a cache storage got full cleanup all the expired items not just the oldest one. +#memcache_full_cleanup: False +# Enable collecting the memcache stats and log it on `debug` log level. +#memcache_debug: False + +# Store all returns in the given returner. +# Setting this option requires that any returner-specific configuration also +# be set. See various returners in salt/returners for details on required +# configuration values. (See also, event_return_queue below.) +# +#event_return: mysql + +# On busy systems, enabling event_returns can cause a considerable load on +# the storage system for returners. Events can be queued on the master and +# stored in a batched fashion using a single transaction for multiple events. +# By default, events are not queued. +#event_return_queue: 0 + +# Only return events matching tags in a whitelist, supports glob matches. +#event_return_whitelist: +# - salt/master/a_tag +# - salt/run/*/ret + +# Store all event returns **except** the tags in a blacklist, supports globs. +#event_return_blacklist: +# - salt/master/not_this_tag +# - salt/wheel/*/ret + +# Passing very large events can cause the minion to consume large amounts of +# memory. This value tunes the maximum size of a message allowed onto the +# master event bus. The value is expressed in bytes. +#max_event_size: 1048576 + +# By default, the master AES key rotates every 24 hours. The next command +# following a key rotation will trigger a key refresh from the minion which may +# result in minions which do not respond to the first command after a key refresh. +# +# To tell the master to ping all minions immediately after an AES key refresh, set +# ping_on_rotate to True. This should mitigate the issue where a minion does not +# appear to initially respond after a key is rotated. +# +# Note that ping_on_rotate may cause high load on the master immediately after +# the key rotation event as minions reconnect. Consider this carefully if this +# salt master is managing a large number of minions. +# +# If disabled, it is recommended to handle this event by listening for the +# 'aes_key_rotate' event with the 'key' tag and acting appropriately. +# ping_on_rotate: False + +# By default, the master deletes its cache of minion data when the key for that +# minion is removed. To preserve the cache after key deletion, set +# 'preserve_minion_cache' to True. +# +# WARNING: This may have security implications if compromised minions auth with +# a previous deleted minion ID. +#preserve_minion_cache: False + +# Allow or deny minions from requesting their own key revocation +#allow_minion_key_revoke: True + +# If max_minions is used in large installations, the master might experience +# high-load situations because of having to check the number of connected +# minions for every authentication. This cache provides the minion-ids of +# all connected minions to all MWorker-processes and greatly improves the +# performance of max_minions. +# con_cache: False + +# The master can include configuration from other files. To enable this, +# pass a list of paths to this option. The paths can be either relative or +# absolute; if relative, they are considered to be relative to the directory +# the main master configuration file lives in (this file). Paths can make use +# of shell-style globbing. If no files are matched by a path passed to this +# option, then the master will log a warning message. +# +# Include a config file from some other path: +# include: /etc/salt/extra_config +# +# Include config from several files and directories: +# include: +# - /etc/salt/extra_config + + +##### Large-scale tuning settings ##### +########################################## +# Max open files +# +# Each minion connecting to the master uses AT LEAST one file descriptor, the +# master subscription connection. If enough minions connect you might start +# seeing on the console (and then salt-master crashes): +# Too many open files (tcp_listener.cpp:335) +# Aborted (core dumped) +# +# By default this value will be the one of `ulimit -Hn`, ie, the hard limit for +# max open files. +# +# If you wish to set a different value than the default one, uncomment and +# configure this setting. Remember that this value CANNOT be higher than the +# hard limit. Raising the hard limit depends on your OS and/or distribution, +# a good way to find the limit is to search the internet. For example: +# raise max open files hard limit debian +# +#max_open_files: 100000 + +# The number of worker threads to start. These threads are used to manage +# return calls made from minions to the master. If the master seems to be +# running slowly, increase the number of threads. This setting can not be +# set lower than 3. +#worker_threads: 5 + +# Set the ZeroMQ high water marks +# http://api.zeromq.org/3-2:zmq-setsockopt + +# The listen queue size / backlog +#zmq_backlog: 1000 + +# The publisher interface ZeroMQPubServerChannel +#pub_hwm: 1000 + +# These two ZMQ HWM settings, salt_event_pub_hwm and event_publisher_pub_hwm +# are significant for masters with thousands of minions. When these are +# insufficiently high it will manifest in random responses missing in the CLI +# and even missing from the job cache. Masters that have fast CPUs and many +# cores with appropriate worker_threads will not need these set as high. + +# On deployment with 8,000 minions, 2.4GHz CPUs, 24 cores, 32GiB memory has +# these settings: +# +# salt_event_pub_hwm: 128000 +# event_publisher_pub_hwm: 64000 + +# ZMQ high-water-mark for SaltEvent pub socket +#salt_event_pub_hwm: 20000 + +# ZMQ high-water-mark for EventPublisher pub socket +#event_publisher_pub_hwm: 10000 + +# The master may allocate memory per-event and not +# reclaim it. +# To set a high-water mark for memory allocation, use +# ipc_write_buffer to set a high-water mark for message +# buffering. +# Value: In bytes. Set to 'dynamic' to have Salt select +# a value for you. Default is disabled. +# ipc_write_buffer: 'dynamic' + + +##### Security settings ##### +########################################## +# Enable "open mode", this mode still maintains encryption, but turns off +# authentication, this is only intended for highly secure environments or for +# the situation where your keys end up in a bad state. If you run in open mode +# you do so at your own risk! +#open_mode: False + +# Enable auto_accept, this setting will automatically accept all incoming +# public keys from the minions. Note that this is insecure. +#auto_accept: False + +# Time in minutes that an incoming public key with a matching name found in +# pki_dir/minion_autosign/keyid is automatically accepted. Expired autosign keys +# are removed when the master checks the minion_autosign directory. +# 0 equals no timeout +# autosign_timeout: 120 + +# If the autosign_file is specified, incoming keys specified in the +# autosign_file will be automatically accepted. This is insecure. Regular +# expressions as well as globing lines are supported. +#autosign_file: /etc/salt/autosign.conf + +# Works like autosign_file, but instead allows you to specify minion IDs for +# which keys will automatically be rejected. Will override both membership in +# the autosign_file and the auto_accept setting. +#autoreject_file: /etc/salt/autoreject.conf + +# Enable permissive access to the salt keys. This allows you to run the +# master or minion as root, but have a non-root group be given access to +# your pki_dir. To make the access explicit, root must belong to the group +# you've given access to. This is potentially quite insecure. If an autosign_file +# is specified, enabling permissive_pki_access will allow group access to that +# specific file. +#permissive_pki_access: False + +# Allow users on the master access to execute specific commands on minions. +# This setting should be treated with care since it opens up execution +# capabilities to non root users. By default this capability is completely +# disabled. +#publisher_acl: +# larry: +# - test.ping +# - network.* +# +# Blacklist any of the following users or modules +# +# This example would blacklist all non sudo users, including root from +# running any commands. It would also blacklist any use of the "cmd" +# module. This is completely disabled by default. +# +# +# Check the list of configured users in client ACL against users on the +# system and throw errors if they do not exist. +#client_acl_verify: True +# +#publisher_acl_blacklist: +# users: +# - root +# - '^(?!sudo_).*$' # all non sudo users +# modules: +# - cmd +# +# WARNING: client_acl and client_acl_blacklist options are deprecated and will +# be removed in the future releases. Use publisher_acl and +# publisher_acl_blacklist instead. + +# Enforce publisher_acl & publisher_acl_blacklist when users have sudo +# access to the salt command. +# +#sudo_acl: False + +# The external auth system uses the Salt auth modules to authenticate and +# validate users to access areas of the Salt system. +#external_auth: +# pam: +# fred: +# - test.* +# +# Time (in seconds) for a newly generated token to live. Default: 12 hours +#token_expire: 43200 +# +# Allow eauth users to specify the expiry time of the tokens they generate. +# A boolean applies to all users or a dictionary of whitelisted eauth backends +# and usernames may be given. +# token_expire_user_override: +# pam: +# - fred +# - tom +# ldap: +# - gary +# +#token_expire_user_override: False + +# Allow minions to push files to the master. This is disabled by default, for +# security purposes. +#file_recv: False + +# Set a hard-limit on the size of the files that can be pushed to the master. +# It will be interpreted as megabytes. Default: 100 +#file_recv_max_size: 100 + +# Signature verification on messages published from the master. +# This causes the master to cryptographically sign all messages published to its event +# bus, and minions then verify that signature before acting on the message. +# +# This is False by default. +# +# Note that to facilitate interoperability with masters and minions that are different +# versions, if sign_pub_messages is True but a message is received by a minion with +# no signature, it will still be accepted, and a warning message will be logged. +# Conversely, if sign_pub_messages is False, but a minion receives a signed +# message it will be accepted, the signature will not be checked, and a warning message +# will be logged. This behavior went away in Salt 2014.1.0 and these two situations +# will cause minion to throw an exception and drop the message. +# sign_pub_messages: False + +# Use TLS/SSL encrypted connection between master and minion. +# Can be set to a dictionary containing keyword arguments corresponding to Python's +# 'ssl.wrap_socket' method. +# Default is None. +#ssl: +# keyfile: <path_to_keyfile> +# certfile: <path_to_certfile> +# ssl_version: PROTOCOL_TLSv1_2 + +##### Salt-SSH Configuration ##### +########################################## + +# Pass in an alternative location for the salt-ssh roster file +#roster_file: /etc/salt/roster + +# Define locations for roster files so they can be chosen when using Salt API. +# An administrator can place roster files into these locations. Then when +# calling Salt API, parameter 'roster_file' should contain a relative path to +# these locations. That is, "roster_file=/foo/roster" will be resolved as +# "/etc/salt/roster.d/foo/roster" etc. This feature prevents passing insecure +# custom rosters through the Salt API. +# +#rosters: +# - /etc/salt/roster.d +# - /opt/salt/some/more/rosters + +# The log file of the salt-ssh command: +#ssh_log_file: /var/log/salt/ssh + +# Pass in minion option overrides that will be inserted into the SHIM for +# salt-ssh calls. The local minion config is not used for salt-ssh. Can be +# overridden on a per-minion basis in the roster (`minion_opts`) +#ssh_minion_opts: +# gpg_keydir: /root/gpg + +# Set this to True to default to using ~/.ssh/id_rsa for salt-ssh +# authentication with minions +#ssh_use_home_key: False + +##### Master Module Management ##### +########################################## +# Manage how master side modules are loaded. + +# Add any additional locations to look for master runners: +#runner_dirs: [] + +# Enable Cython for master side modules: +#cython_enable: False + + +##### State System settings ##### +########################################## +# The state system uses a "top" file to tell the minions what environment to +# use and what modules to use. The state_top file is defined relative to the +# root of the base environment as defined in "File Server settings" below. +#state_top: top.sls + +# The master_tops option replaces the external_nodes option by creating +# a plugable system for the generation of external top data. The external_nodes +# option is deprecated by the master_tops option. +# +# To gain the capabilities of the classic external_nodes system, use the +# following configuration: +# master_tops: +# ext_nodes: <Shell command which returns yaml> +# +#master_tops: {} + +# The external_nodes option allows Salt to gather data that would normally be +# placed in a top file. The external_nodes option is the executable that will +# return the ENC data. Remember that Salt will look for external nodes AND top +# files and combine the results if both are enabled! +#external_nodes: None + +# The renderer to use on the minions to render the state data +#renderer: yaml_jinja + +# The Jinja renderer can strip extra carriage returns and whitespace +# See http://jinja.pocoo.org/docs/api/#high-level-api +# +# If this is set to True the first newline after a Jinja block is removed +# (block, not variable tag!). Defaults to False, corresponds to the Jinja +# environment init variable "trim_blocks". +#jinja_trim_blocks: False +# +# If this is set to True leading spaces and tabs are stripped from the start +# of a line to a block. Defaults to False, corresponds to the Jinja +# environment init variable "lstrip_blocks". +#jinja_lstrip_blocks: False + +# The failhard option tells the minions to stop immediately after the first +# failure detected in the state execution, defaults to False +#failhard: False + +# The state_verbose and state_output settings can be used to change the way +# state system data is printed to the display. By default all data is printed. +# The state_verbose setting can be set to True or False, when set to False +# all data that has a result of True and no changes will be suppressed. +#state_verbose: True + +# The state_output setting changes if the output is the full multi line +# output for each changed state if set to 'full', but if set to 'terse' +# the output will be shortened to a single line. If set to 'mixed', the output +# will be terse unless a state failed, in which case that output will be full. +# If set to 'changes', the output will be full unless the state didn't change. +#state_output: full + +# Automatically aggregate all states that have support for mod_aggregate by +# setting to 'True'. Or pass a list of state module names to automatically +# aggregate just those types. +# +# state_aggregate: +# - pkg +# +#state_aggregate: False + +# Send progress events as each function in a state run completes execution +# by setting to 'True'. Progress events are in the format +# 'salt/job/<JID>/prog/<MID>/<RUN NUM>'. +#state_events: False + +##### File Server settings ##### +########################################## +# Salt runs a lightweight file server written in zeromq to deliver files to +# minions. This file server is built into the master daemon and does not +# require a dedicated port. + +# The file server works on environments passed to the master, each environment +# can have multiple root directories, the subdirectories in the multiple file +# roots cannot match, otherwise the downloaded files will not be able to be +# reliably ensured. A base environment is required to house the top file. +# Example: +# file_roots: +# base: +# - /srv/salt/ +# dev: +# - /srv/salt/dev/services +# - /srv/salt/dev/states +# prod: +# - /srv/salt/prod/services +# - /srv/salt/prod/states +# +#file_roots: +# base: +# - /srv/salt +# +file_roots: + base: + - /srv/salt + +# When using multiple environments, each with their own top file, the +# default behaviour is an unordered merge. To prevent top files from +# being merged together and instead to only use the top file from the +# requested environment, set this value to 'same'. +#top_file_merging_strategy: merge + +# To specify the order in which environments are merged, set the ordering +# in the env_order option. Given a conflict, the last matching value will +# win. +#env_order: ['base', 'dev', 'prod'] + +# If top_file_merging_strategy is set to 'same' and an environment does not +# contain a top file, the top file in the environment specified by default_top +# will be used instead. +#default_top: base + +# The hash_type is the hash to use when discovering the hash of a file on +# the master server. The default is md5 but sha1, sha224, sha256, sha384 +# and sha512 are also supported. +# +# WARNING: While md5 is also supported, do not use it due to the high chance +# of possible collisions and thus security breach. +# +# Prior to changing this value, the master should be stopped and all Salt +# caches should be cleared. +#hash_type: sha256 + +# The buffer size in the file server can be adjusted here: +#file_buffer_size: 1048576 + +# A regular expression (or a list of expressions) that will be matched +# against the file path before syncing the modules and states to the minions. +# This includes files affected by the file.recurse state. +# For example, if you manage your custom modules and states in subversion +# and don't want all the '.svn' folders and content synced to your minions, +# you could set this to '/\.svn($|/)'. By default nothing is ignored. +#file_ignore_regex: +# - '/\.svn($|/)' +# - '/\.git($|/)' + +# A file glob (or list of file globs) that will be matched against the file +# path before syncing the modules and states to the minions. This is similar +# to file_ignore_regex above, but works on globs instead of regex. By default +# nothing is ignored. +# file_ignore_glob: +# - '*.pyc' +# - '*/somefolder/*.bak' +# - '*.swp' + +# File Server Backend +# +# Salt supports a modular fileserver backend system, this system allows +# the salt master to link directly to third party systems to gather and +# manage the files available to minions. Multiple backends can be +# configured and will be searched for the requested file in the order in which +# they are defined here. The default setting only enables the standard backend +# "roots" which uses the "file_roots" option. +#fileserver_backend: +# - roots +# +# To use multiple backends list them in the order they are searched: +#fileserver_backend: +# - git +# - roots +# +# Uncomment the line below if you do not want the file_server to follow +# symlinks when walking the filesystem tree. This is set to True +# by default. Currently this only applies to the default roots +# fileserver_backend. +#fileserver_followsymlinks: False +# +# Uncomment the line below if you do not want symlinks to be +# treated as the files they are pointing to. By default this is set to +# False. By uncommenting the line below, any detected symlink while listing +# files on the Master will not be returned to the Minion. +#fileserver_ignoresymlinks: True +# +# By default, the Salt fileserver recurses fully into all defined environments +# to attempt to find files. To limit this behavior so that the fileserver only +# traverses directories with SLS files and special Salt directories like _modules, +# enable the option below. This might be useful for installations where a file root +# has a very large number of files and performance is impacted. Default is False. +# fileserver_limit_traversal: False +# +# The fileserver can fire events off every time the fileserver is updated, +# these are disabled by default, but can be easily turned on by setting this +# flag to True +#fileserver_events: False + +# Git File Server Backend Configuration +# +# Optional parameter used to specify the provider to be used for gitfs. Must +# be one of the following: pygit2, gitpython, or dulwich. If unset, then each +# will be tried in that same order, and the first one with a compatible +# version installed will be the provider that is used. +#gitfs_provider: pygit2 + +# Along with gitfs_password, is used to authenticate to HTTPS remotes. +# gitfs_user: '' + +# Along with gitfs_user, is used to authenticate to HTTPS remotes. +# This parameter is not required if the repository does not use authentication. +#gitfs_password: '' + +# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote. +# This parameter enables authentication over HTTP. Enable this at your own risk. +#gitfs_insecure_auth: False + +# Along with gitfs_privkey (and optionally gitfs_passphrase), is used to +# authenticate to SSH remotes. This parameter (or its per-remote counterpart) +# is required for SSH remotes. +#gitfs_pubkey: '' + +# Along with gitfs_pubkey (and optionally gitfs_passphrase), is used to +# authenticate to SSH remotes. This parameter (or its per-remote counterpart) +# is required for SSH remotes. +#gitfs_privkey: '' + +# This parameter is optional, required only when the SSH key being used to +# authenticate is protected by a passphrase. +#gitfs_passphrase: '' + +# When using the git fileserver backend at least one git remote needs to be +# defined. The user running the salt master will need read access to the repo. +# +# The repos will be searched in order to find the file requested by a client +# and the first repo to have the file will return it. +# When using the git backend branches and tags are translated into salt +# environments. +# Note: file:// repos will be treated as a remote, so refs you want used must +# exist in that repo as *local* refs. +#gitfs_remotes: +# - git://github.com/saltstack/salt-states.git +# - file:///var/git/saltmaster +# +# The gitfs_ssl_verify option specifies whether to ignore ssl certificate +# errors when contacting the gitfs backend. You might want to set this to +# false if you're using a git backend that uses a self-signed certificate but +# keep in mind that setting this flag to anything other than the default of True +# is a security concern, you may want to try using the ssh transport. +#gitfs_ssl_verify: True +# +# The gitfs_root option gives the ability to serve files from a subdirectory +# within the repository. The path is defined relative to the root of the +# repository and defaults to the repository root. +#gitfs_root: somefolder/otherfolder +# +# +##### Pillar settings ##### +########################################## +# Salt Pillars allow for the building of global data that can be made selectively +# available to different minions based on minion grain filtering. The Salt +# Pillar is laid out in the same fashion as the file server, with environments, +# a top file and sls files. However, pillar data does not need to be in the +# highstate format, and is generally just key/value pairs. +pillar_roots: + base: + - /srv/pillar + +#ext_pillar: +# - hiera: /etc/hiera.yaml +# - cmd_yaml: cat /etc/salt/yaml + +# The ext_pillar_first option allows for external pillar sources to populate +# before file system pillar. This allows for targeting file system pillar from +# ext_pillar. +#ext_pillar_first: False + +# The external pillars permitted to be used on-demand using pillar.ext +#on_demand_ext_pillar: +# - libvirt +# - virtkey + +# The pillar_gitfs_ssl_verify option specifies whether to ignore ssl certificate +# errors when contacting the pillar gitfs backend. You might want to set this to +# false if you're using a git backend that uses a self-signed certificate but +# keep in mind that setting this flag to anything other than the default of True +# is a security concern, you may want to try using the ssh transport. +#pillar_gitfs_ssl_verify: True + +# The pillar_opts option adds the master configuration file data to a dict in +# the pillar called "master". This is used to set simple configurations in the +# master config file that can then be used on minions. +#pillar_opts: False + +# The pillar_safe_render_error option prevents the master from passing pillar +# render errors to the minion. This is set on by default because the error could +# contain templating data which would give that minion information it shouldn't +# have, like a password! When set true the error message will only show: +# Rendering SLS 'my.sls' failed. Please see master log for details. +#pillar_safe_render_error: True + +# The pillar_source_merging_strategy option allows you to configure merging strategy +# between different sources. It accepts five values: none, recurse, aggregate, overwrite, +# or smart. None will not do any merging at all. Recurse will merge recursively mapping of data. +# Aggregate instructs aggregation of elements between sources that use the #!yamlex renderer. Overwrite +# will overwrite elements according the order in which they are processed. This is +# behavior of the 2014.1 branch and earlier. Smart guesses the best strategy based +# on the "renderer" setting and is the default value. +#pillar_source_merging_strategy: smart + +# Recursively merge lists by aggregating them instead of replacing them. +#pillar_merge_lists: False + +# Set this option to 'True' to force a 'KeyError' to be raised whenever an +# attempt to retrieve a named value from pillar fails. When this option is set +# to 'False', the failed attempt returns an empty string. Default is 'False'. +#pillar_raise_on_missing: False + +# Git External Pillar (git_pillar) Configuration Options +# +# Specify the provider to be used for git_pillar. Must be either pygit2 or +# gitpython. If unset, then both will be tried in that same order, and the +# first one with a compatible version installed will be the provider that +# is used. +#git_pillar_provider: pygit2 + +# If the desired branch matches this value, and the environment is omitted +# from the git_pillar configuration, then the environment for that git_pillar +# remote will be base. +#git_pillar_base: master + +# If the branch is omitted from a git_pillar remote, then this branch will +# be used instead +#git_pillar_branch: master + +# Environment to use for git_pillar remotes. This is normally derived from +# the branch/tag (or from a per-remote env parameter), but if set this will +# override the process of deriving the env from the branch/tag name. +#git_pillar_env: '' + +# Path relative to the root of the repository where the git_pillar top file +# and SLS files are located. +#git_pillar_root: '' + +# Specifies whether or not to ignore SSL certificate errors when contacting +# the remote repository. +#git_pillar_ssl_verify: False + +# When set to False, if there is an update/checkout lock for a git_pillar +# remote and the pid written to it is not running on the master, the lock +# file will be automatically cleared and a new lock will be obtained. +#git_pillar_global_lock: True + +# Git External Pillar Authentication Options +# +# Along with git_pillar_password, is used to authenticate to HTTPS remotes. +#git_pillar_user: '' + +# Along with git_pillar_user, is used to authenticate to HTTPS remotes. +# This parameter is not required if the repository does not use authentication. +#git_pillar_password: '' + +# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote. +# This parameter enables authentication over HTTP. +#git_pillar_insecure_auth: False + +# Along with git_pillar_privkey (and optionally git_pillar_passphrase), +# is used to authenticate to SSH remotes. +#git_pillar_pubkey: '' + +# Along with git_pillar_pubkey (and optionally git_pillar_passphrase), +# is used to authenticate to SSH remotes. +#git_pillar_privkey: '' + +# This parameter is optional, required only when the SSH key being used +# to authenticate is protected by a passphrase. +#git_pillar_passphrase: '' + +# A master can cache pillars locally to bypass the expense of having to render them +# for each minion on every request. This feature should only be enabled in cases +# where pillar rendering time is known to be unsatisfactory and any attendant security +# concerns about storing pillars in a master cache have been addressed. +# +# When enabling this feature, be certain to read through the additional ``pillar_cache_*`` +# configuration options to fully understand the tunable parameters and their implications. +# +# Note: setting ``pillar_cache: True`` has no effect on targeting Minions with Pillars. +# See https://docs.saltstack.com/en/latest/topics/targeting/pillar.html +#pillar_cache: False + +# If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount +# of time, in seconds, before the cache is considered invalid by a master and a fresh +# pillar is recompiled and stored. +#pillar_cache_ttl: 3600 + +# If and only if a master has set `pillar_cache: True`, one of several storage providers +# can be utililzed. +# +# `disk`: The default storage backend. This caches rendered pillars to the master cache. +# Rendered pillars are serialized and deserialized as msgpack structures for speed. +# Note that pillars are stored UNENCRYPTED. Ensure that the master cache +# has permissions set appropriately. (Same defaults are provided.) +# +# memory: [EXPERIMENTAL] An optional backend for pillar caches which uses a pure-Python +# in-memory data structure for maximal performance. There are several caveats, +# however. First, because each master worker contains its own in-memory cache, +# there is no guarantee of cache consistency between minion requests. This +# works best in situations where the pillar rarely if ever changes. Secondly, +# and perhaps more importantly, this means that unencrypted pillars will +# be accessible to any process which can examine the memory of the ``salt-master``! +# This may represent a substantial security risk. +# +#pillar_cache_backend: disk + + +##### Syndic settings ##### +########################################## +# The Salt syndic is used to pass commands through a master from a higher +# master. Using the syndic is simple. If this is a master that will have +# syndic servers(s) below it, then set the "order_masters" setting to True. +# +# If this is a master that will be running a syndic daemon for passthrough, then +# the "syndic_master" setting needs to be set to the location of the master server +# to receive commands from. + +# Set the order_masters setting to True if this master will command lower +# masters' syndic interfaces. +#order_masters: False + +# If this master will be running a salt syndic daemon, syndic_master tells +# this master where to receive commands from. +#syndic_master: masterofmasters + +# This is the 'ret_port' of the MasterOfMaster: +#syndic_master_port: 4506 + +# PID file of the syndic daemon: +#syndic_pidfile: /var/run/salt-syndic.pid + +# The log file of the salt-syndic daemon: +#syndic_log_file: /var/log/salt/syndic + +# The behaviour of the multi-syndic when connection to a master of masters failed. +# Can specify ``random`` (default) or ``ordered``. If set to ``random``, masters +# will be iterated in random order. If ``ordered`` is specified, the configured +# order will be used. +#syndic_failover: random + +# The number of seconds for the salt client to wait for additional syndics to +# check in with their lists of expected minions before giving up. +#syndic_wait: 5 + + +##### Peer Publish settings ##### +########################################## +# Salt minions can send commands to other minions, but only if the minion is +# allowed to. By default "Peer Publication" is disabled, and when enabled it +# is enabled for specific minions and specific commands. This allows secure +# compartmentalization of commands based on individual minions. + +# The configuration uses regular expressions to match minions and then a list +# of regular expressions to match functions. The following will allow the +# minion authenticated as foo.example.com to execute functions from the test +# and pkg modules. +#peer: +# foo.example.com: +# - test.* +# - pkg.* +# +# This will allow all minions to execute all commands: +#peer: +# .*: +# - .* +# +# This is not recommended, since it would allow anyone who gets root on any +# single minion to instantly have root on all of the minions! + +# Minions can also be allowed to execute runners from the salt master. +# Since executing a runner from the minion could be considered a security risk, +# it needs to be enabled. This setting functions just like the peer setting +# except that it opens up runners instead of module functions. +# +# All peer runner support is turned off by default and must be enabled before +# using. This will enable all peer runners for all minions: +#peer_run: +# .*: +# - .* +# +# To enable just the manage.up runner for the minion foo.example.com: +#peer_run: +# foo.example.com: +# - manage.up +# +# +##### Mine settings ##### +##################################### +# Restrict mine.get access from minions. By default any minion has a full access +# to get all mine data from master cache. In acl definion below, only pcre matches +# are allowed. +# mine_get: +# .*: +# - .* +# +# The example below enables minion foo.example.com to get 'network.interfaces' mine +# data only, minions web* to get all network.* and disk.* mine data and all other +# minions won't get any mine data. +# mine_get: +# foo.example.com: +# - network.interfaces +# web.*: +# - network.* +# - disk.* + + +##### Logging settings ##### +########################################## +# The location of the master log file +# The master log can be sent to a regular file, local path name, or network +# location. Remote logging works best when configured to use rsyslogd(8) (e.g.: +# ``file:///dev/log``), with rsyslogd(8) configured for network logging. The URI +# format is: <file|udp|tcp>://<host|socketpath>:<port-if-required>/<log-facility> +#log_file: /var/log/salt/master +#log_file: file:///dev/log +#log_file: udp://loghost:10514 + +#log_file: /var/log/salt/master +#key_logfile: /var/log/salt/key + +# The level of messages to send to the console. +# One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'. +# +# The following log levels are considered INSECURE and may log sensitive data: +# ['garbage', 'trace', 'debug'] +# +#log_level: warning + +# The level of messages to send to the log file. +# One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'. +# If using 'log_granular_levels' this must be set to the highest desired level. +#log_level_logfile: warning + +# The date and time format used in log messages. Allowed date/time formatting +# can be seen here: http://docs.python.org/library/time.html#time.strftime +#log_datefmt: '%H:%M:%S' +#log_datefmt_logfile: '%Y-%m-%d %H:%M:%S' + +# The format of the console logging messages. Allowed formatting options can +# be seen here: http://docs.python.org/library/logging.html#logrecord-attributes +# +# Console log colors are specified by these additional formatters: +# +# %(colorlevel)s +# %(colorname)s +# %(colorprocess)s +# %(colormsg)s +# +# Since it is desirable to include the surrounding brackets, '[' and ']', in +# the coloring of the messages, these color formatters also include padding as +# well. Color LogRecord attributes are only available for console logging. +# +#log_fmt_console: '%(colorlevel)s %(colormsg)s' +#log_fmt_console: '[%(levelname)-8s] %(message)s' +# +#log_fmt_logfile: '%(asctime)s,%(msecs)03d [%(name)-17s][%(levelname)-8s] %(message)s' + +# This can be used to control logging levels more specificically. This +# example sets the main salt library at the 'warning' level, but sets +# 'salt.modules' to log at the 'debug' level: +# log_granular_levels: +# 'salt': 'warning' +# 'salt.modules': 'debug' +# +#log_granular_levels: {} + + +##### Node Groups ###### +########################################## +# Node groups allow for logical groupings of minion nodes. A group consists of +# a group name and a compound target. Nodgroups can reference other nodegroups +# with 'N@' classifier. Ensure that you do not have circular references. +# +#nodegroups: +# group1: 'L@foo.domain.com,bar.domain.com,baz.domain.com or bl*.domain.com' +# group2: 'G@os:Debian and foo.domain.com' +# group3: 'G@os:Debian and N@group1' +# group4: +# - 'G@foo:bar' +# - 'or' +# - 'G@foo:baz' + + +##### Range Cluster settings ##### +########################################## +# The range server (and optional port) that serves your cluster information +# https://github.com/ytoolshed/range/wiki/%22yamlfile%22-module-file-spec +# +#range_server: range:80 + + +##### Windows Software Repo settings ##### +########################################### +# Location of the repo on the master: +#winrepo_dir_ng: '/srv/salt/win/repo-ng' +# +# List of git repositories to include with the local repo: +#winrepo_remotes_ng: +# - 'https://github.com/saltstack/salt-winrepo-ng.git' + + +##### Windows Software Repo settings - Pre 2015.8 ##### +######################################################## +# Legacy repo settings for pre-2015.8 Windows minions. +# +# Location of the repo on the master: +#winrepo_dir: '/srv/salt/win/repo' +# +# Location of the master's repo cache file: +#winrepo_mastercachefile: '/srv/salt/win/repo/winrepo.p' +# +# List of git repositories to include with the local repo: +#winrepo_remotes: +# - 'https://github.com/saltstack/salt-winrepo.git' + + +##### Returner settings ###### +############################################ +# Which returner(s) will be used for minion's result: +#return: mysql + + +###### Miscellaneous settings ###### +############################################ +# Default match type for filtering events tags: startswith, endswith, find, regex, fnmatch +#event_match_type: startswith + +# Save runner returns to the job cache +#runner_returns: True + +# Permanently include any available Python 3rd party modules into Salt Thin +# when they are generated for Salt-SSH or other purposes. +# The modules should be named by the names they are actually imported inside the Python. +# The value of the parameters can be either one module or a comma separated list of them. +#thin_extra_mods: foo,bar + diff --git a/salt/openbook/ansible/files/saltstack.list b/salt/openbook/ansible/files/saltstack.list new file mode 100644 index 0000000..cda4b32 --- /dev/null +++ b/salt/openbook/ansible/files/saltstack.list @@ -0,0 +1 @@ +deb http://repo.saltstack.com/apt/ubuntu/16.04/amd64/latest xenial main diff --git a/salt/openbook/ansible/files/slave.jar b/salt/openbook/ansible/files/slave.jar Binary files differnew file mode 100644 index 0000000..37e0ac5 --- /dev/null +++ b/salt/openbook/ansible/files/slave.jar diff --git a/salt/openbook/ansible/files/ssh_keys.yaml b/salt/openbook/ansible/files/ssh_keys.yaml new file mode 100644 index 0000000..e0046b2 --- /dev/null +++ b/salt/openbook/ansible/files/ssh_keys.yaml @@ -0,0 +1,39 @@ +--- +- name: "Generate SSH keypair" + hosts: all + vars: + ssh_privkey_path: /root/.ssh/id_rsa + tasks: + - name: "Detect existing SSH privkey" + stat: + path: "{{ ssh_privkey_path }}" + register: ssh_privkey + + - name: "Detect existing SSH pubkey" + stat: + path: "{{ ssh_privkey_path }}.pub" + register: ssh_pubkey + + - name: "Generate SSH keypair" + shell: ssh-keygen -q -t rsa -b 4096 -f "{{ ssh_privkey_path }}" -N '' + when: ssh_privkey.stat.exists == False or ssh_pubkey.stat.exists == False + +- name: "Generate SSH keypair" + hosts: all + become_user: jenkins + vars: + ssh_privkey_path: /var/lib/jenkins/.ssh/id_rsa + tasks: + - name: "Detect existing SSH privkey" + stat: + path: "{{ ssh_privkey_path }}" + register: ssh_privkey + + - name: "Detect existing SSH pubkey" + stat: + path: "{{ ssh_privkey_path }}.pub" + register: ssh_pubkey + + - name: "Generate SSH keypair" + shell: ssh-keygen -q -t rsa -b 4096 -f "{{ ssh_privkey_path }}" -N '' + when: ssh_privkey.stat.exists == False or ssh_pubkey.stat.exists == False diff --git a/salt/openbook/ansible/init.sls b/salt/openbook/ansible/init.sls new file mode 100644 index 0000000..e337964 --- /dev/null +++ b/salt/openbook/ansible/init.sls @@ -0,0 +1,27 @@ +{% for dir in pillar["ansible"]["dirs"].items() %} +anaible_dirs_{{ dir[0] }}: + file.directory: +{% if dir[1]["relative_to_buildroot"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/{{ dir[1]["name"] }} +{% else %} + - name: {{ dir[1]["name"] }} +{% endif %} + - user: root + - group: root + - mode: {{ dir[1]["mode"] }} + - makedirs: True +{% endfor %} + +{% for file in pillar["ansible"]["files"].items() %} +ansible_files_{{ file[0] }}: + file.managed: + - name: {{ pillar["ansible"]["build_dir"] }}/{{ file[1]["name"] }} + - source: salt://openbook/ansible/files/{{ file[1]["source"] }} +{% if file[1]["is_template"] %} + - template: jinja +{% endif %} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% endfor %} + diff --git a/salt/openbook/api_server/files/Dockerfile b/salt/openbook/api_server/files/Dockerfile new file mode 100644 index 0000000..57c48ea --- /dev/null +++ b/salt/openbook/api_server/files/Dockerfile @@ -0,0 +1,8 @@ +FROM {{ pillar["baseimage"]["image_name"] }} +MAINTAINER Carlos Konstanski <carlos@talligent.com> +ADD docker-start.tar.xz / +EXPOSE 8081 5701 5702 +VOLUME {{ pillar["api_server"]["dirs"]["etc_openbook_api_server"]["name"] }} +VOLUME {{ pillar["api_server"]["dirs"]["var_log_openbook_api_server"]["name"] }} +RUN /installdeps.sh +ENTRYPOINT [ "/start_app.sh" ] diff --git a/salt/openbook/api_server/files/api_server.json b/salt/openbook/api_server/files/api_server.json new file mode 100644 index 0000000..8e75119 --- /dev/null +++ b/salt/openbook/api_server/files/api_server.json @@ -0,0 +1,15 @@ +{% raw %} +{% set ips = openbook_gcomm_addresses.split(",") %} +{% if ips|length > 1 %} +{% set failover = "failover:" %} +{% else %} +{% set failover = "" %} +{% endif %} +{% endraw %} + +{ + "log.path": "{{ pillar["api_server"]["log"] }}", + "entityDS.url": "jdbc:mariadb:{% raw %}{{ failover }}{% endraw %}//{% raw %}{{ openbook_gcomm_addresses }}{% endraw %}/openbook", + "workflowDS.url": "jdbc:mariadb:{% raw %}{{ failover }}{% endraw %}//{% raw %}{{ openbook_gcomm_addresses }}{% endraw %}/openbook_workflow", + "jobsDS.url": "jdbc:mariadb:{% raw %}{{ failover }}{% endraw %}//{% raw %}{{ openbook_gcomm_addresses }}{% endraw %}/openbook_jobs" +} diff --git a/salt/openbook/api_server/files/build.sh b/salt/openbook/api_server/files/build.sh new file mode 100644 index 0000000..95f7077 --- /dev/null +++ b/salt/openbook/api_server/files/build.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Do not call directly. Meant to be called from CI build script. + +pushd {{ pillar["api_server"]["build_dir"] }} 2>/dev/null + +echo "Creating tmp directory" +rm -rf tmp +mkdir tmp +rm -f docker-start.tar.xz + +echo "Populating with custom files" +rsync -aq data/ tmp/ + +echo "Repackaging into .xz file" +tar Jcvf docker-start.tar.xz -C tmp/ . + +echo "Deleting existing docker container" +docker stop {{ pillar["api_server"]["container_name"] }} +docker container prune -f +docker rmi -f {{ pillar["api_server"]["image_name"] }} + +echo "Building docker Ubuntu image" +docker build -t {{ pillar["api_server"]["image_name"] }} . + +echo "Push docker image to the registry" +docker tag {{ pillar["api_server"]["image_name"] }} {{ pillar["registry"]["url"] }}/{{ pillar["api_server"]["image_name"] }} +docker push {{ pillar["registry"]["url"] }}/{{ pillar["api_server"]["image_name"] }} + +echo "Cleaning up" +rm -rf tmp +rm -f docker-start.tar.xz + +popd 2>/dev/null +exit 0 diff --git a/salt/openbook/api_server/files/installdeps.sh b/salt/openbook/api_server/files/installdeps.sh new file mode 100644 index 0000000..94a6353 --- /dev/null +++ b/salt/openbook/api_server/files/installdeps.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +apt-get -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" openjdk-8-jre-headless + +curl -k \ + -u '{{ pillar["jars"]["username"] }}:{{ pillar["jars"]["password"] }}' \ + -o "{{ pillar["api_server"]["jar_name"] }}" \ + "{{ pillar["jars"]["url"] }}{{ pillar["api_server"]["jar_name"] }}" + +exit 0 diff --git a/salt/openbook/api_server/files/run.sh b/salt/openbook/api_server/files/run.sh new file mode 100644 index 0000000..86f5ede --- /dev/null +++ b/salt/openbook/api_server/files/run.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +docker run -d --net=host -v {{ pillar["api_server"]["dirs"]["etc_openbook_api_server"]["name"] }}:{{ pillar["api_server"]["dirs"]["etc_openbook_api_server"]["name"] }} -v {{ pillar["api_server"]["dirs"]["var_log_openbook_api_server"]["name"] }}:{{ pillar["api_server"]["dirs"]["var_log_openbook_api_server"]["name"] }} --name {{ pillar["api_server"]["container_name"] }} {{ pillar["registry"]["url"] }}/{{ pillar["api_server"]["image_name"] }} +exit 0 diff --git a/salt/openbook/api_server/files/start_app.sh b/salt/openbook/api_server/files/start_app.sh new file mode 100644 index 0000000..87cea8b --- /dev/null +++ b/salt/openbook/api_server/files/start_app.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exec java -jar {{ pillar["api_server"]["jar_name"] }} run com.talligent.openbook.verticle.Startup --cluster diff --git a/salt/openbook/api_server/init.sls b/salt/openbook/api_server/init.sls new file mode 100644 index 0000000..f0fdb7c --- /dev/null +++ b/salt/openbook/api_server/init.sls @@ -0,0 +1,55 @@ +{% for dir in pillar["api_server"]["dirs"].items() %} +apiserver_dirs_{{ dir[0] }}: + file.directory: +{% if dir[1]["relative_to_buildroot"] %} + - name: {{ pillar["api_server"]["build_dir"] }}/{{ dir[1]["name"] }} +{% else %} + - name: {{ dir[1]["name"] }} +{% endif %} + - user: root + - group: root + - mode: {{ dir[1]["mode"] }} + - makedirs: True +{% endfor %} + +{% for file in pillar["api_server"]["files"].items() %} +apiserver_files_{{ file[0] }}: + file.managed: +{% if file[1]["relative_to_buildroot"] %} + - name: {{ pillar["api_server"]["build_dir"] }}/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% else %} + - name: {{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/api_server/files/{{ file[1]["source"] }} +{% if file[1]["is_template"] %} + - template: jinja +{% endif %} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% if file[1]["ansible"] %} +ansible_apiserver_dirs_{{ file[0] }}: + file.directory: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/api_server/{{ file[1]["path"] }} +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/api_server/{{ file[1]["path"] }} +{% endif %} + - user: root + - group: root + - mode: "0755" + - makedirs: True +ansible_apiserver_files_{{ file[0] }}: + file.managed: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/api_server/{{ file[1]["path"] }}/{{ file[1]["name"] }} + - template: jinja +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/api_server/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/api_server/files/{{ file[1]["source"] }} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% endif %} +{% endfor %} diff --git a/salt/openbook/baseimage/files/Dockerfile b/salt/openbook/baseimage/files/Dockerfile new file mode 100644 index 0000000..1c02714 --- /dev/null +++ b/salt/openbook/baseimage/files/Dockerfile @@ -0,0 +1,4 @@ +FROM ubuntu:xenial +MAINTAINER Carlos Konstanski <carlos@talligent.com> +ADD docker-start.tar.xz / +RUN /installdeps.sh diff --git a/salt/openbook/baseimage/files/build.sh b/salt/openbook/baseimage/files/build.sh new file mode 100644 index 0000000..d5cad7a --- /dev/null +++ b/salt/openbook/baseimage/files/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Do not call directly. Meant to be called from CI build script. + +pushd {{ pillar["baseimage"]["build_dir"] }} 2>/dev/null + +echo "Creating tmp directory" +rm -rf tmp +mkdir tmp +rm -f docker-start.tar.xz + +echo "Populating with custom files" +rsync -aq data/ tmp/ + +echo "Repackaging into .xz file" +tar Jcvf docker-start.tar.xz -C tmp/ . + +echo "Deleting existing docker container" +docker stop {{ pillar["baseimage"]["container_name"] }} +docker container prune -f +docker rmi -f {{ pillar["baseimage"]["image_name"] }} + +echo "Building docker Ubuntu image" +docker build -t {{ pillar["baseimage"]["image_name"] }} . + +echo "Cleaning up" +rm -rf tmp +rm -f docker-start.tar.xz + +popd 2>/dev/null +exit 0 diff --git a/salt/openbook/baseimage/files/installdeps.sh b/salt/openbook/baseimage/files/installdeps.sh new file mode 100644 index 0000000..1bcccdb --- /dev/null +++ b/salt/openbook/baseimage/files/installdeps.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade +apt-get -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" curl python +exit 0 diff --git a/salt/openbook/baseimage/files/sources.list b/salt/openbook/baseimage/files/sources.list new file mode 100644 index 0000000..4e624e5 --- /dev/null +++ b/salt/openbook/baseimage/files/sources.list @@ -0,0 +1,54 @@ +# deb http://de.archive.ubuntu.com/ubuntu/ xenial main restricted + +# deb http://de.archive.ubuntu.com/ubuntu/ xenial-updates main restricted +# deb http://security.ubuntu.com/ubuntu xenial-security main restricted + +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://de.archive.ubuntu.com/ubuntu/ xenial main restricted +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial main restricted + +## Major bug fix updates produced after the final release of the +## distribution. +deb http://de.archive.ubuntu.com/ubuntu/ xenial-updates main restricted +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial-updates main restricted + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team. Also, please note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +deb http://de.archive.ubuntu.com/ubuntu/ xenial universe +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial universe +deb http://de.archive.ubuntu.com/ubuntu/ xenial-updates universe +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial-updates universe + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in +## multiverse WILL NOT receive any review or updates from the Ubuntu +## security team. +deb http://de.archive.ubuntu.com/ubuntu/ xenial multiverse +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial multiverse +deb http://de.archive.ubuntu.com/ubuntu/ xenial-updates multiverse +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial-updates multiverse + +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +deb http://de.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse + +## Uncomment the following two lines to add software from Canonical's +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu xenial partner +# deb-src http://archive.canonical.com/ubuntu xenial partner + +deb http://security.ubuntu.com/ubuntu xenial-security main restricted +# deb-src http://security.ubuntu.com/ubuntu xenial-security main restricted +deb http://security.ubuntu.com/ubuntu xenial-security universe +# deb-src http://security.ubuntu.com/ubuntu xenial-security universe +deb http://security.ubuntu.com/ubuntu xenial-security multiverse +# deb-src http://security.ubuntu.com/ubuntu xenial-security multiverse diff --git a/salt/openbook/baseimage/init.sls b/salt/openbook/baseimage/init.sls new file mode 100644 index 0000000..f23989d --- /dev/null +++ b/salt/openbook/baseimage/init.sls @@ -0,0 +1,30 @@ +{% for dir in pillar["baseimage"]["dirs"].items() %} +baseimage_dirs_{{ dir[0] }}: + file.directory: +{% if dir[1]["relative_to_data"] %} + - name: {{ pillar["baseimage"]["build_dir"] }}/{{ dir[1]["name"] }} +{% else %} + - name: {{ dir[1]["name"] }} +{% endif %} + - user: root + - group: root + - mode: {{ dir[1]["mode"] }} + - makedirs: True +{% endfor %} + +{% for file in pillar["baseimage"]["files"].items() %} +baseimage_files_{{ file[0] }}: + file.managed: +{% if file[1]["relative_to_data"] %} + - name: {{ pillar["baseimage"]["build_dir"] }}/{{ file[1]["name"] }} +{% else %} + - name: {{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/baseimage/files/{{ file[1]["source"] }} +{% if file[1]["is_template"] %} + - template: jinja +{% endif %} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% endfor %} diff --git a/salt/openbook/ci/files/build.sh b/salt/openbook/ci/files/build.sh new file mode 100644 index 0000000..b225c85 --- /dev/null +++ b/salt/openbook/ci/files/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +for i in $(docker images | grep -v CREATED | awk '{print $3}'); do + docker rmi --force $i +done +docker container prune -f + +pushd {{ pillar["ci"]["build_dir"] }} 2>/dev/null + +echo "Login to docker registry" +docker login --username {{ pillar["registry"]["username"] }} --password {{ pillar["registry"]["password"] }} {{ pillar["registry"]["url"] }} + +# for i in $(curl -u '{{ pillar["registry"]["username"] }}:{{ pillar["registry"]["password"] }}' https://{{ pillar["registry"]["url"] }}/v2/_catalog | jq '.repositories' | fgrep 'openbook' | awk -F\" '{print $2}'); do +# # get reference digest +# digest=$(curl -u '{{ pillar["registry"]["username"] }}:{{ pillar["registry"]["password"] }}' -H "Accept: application/vnd.docker.distribution.manifest.v2+json" https://{{ pillar["registry"]["url"] }}/v2/${i}/manifests/latest | jq '.config | .digest' | awk -F\" '{print $2}') +# curl -u '{{ pillar["registry"]["username"] }}:{{ pillar["registry"]["password"] }}' -X DELETE https://{{ pillar["registry"]["url"] }}/v2/${i}/manifests/${digest} +# done + +{{ pillar["baseimage"]["build_dir"] }}/build.sh +{{ pillar["database"]["build_dir"] }}/build.sh +{{ pillar["api_server"]["build_dir"] }}/build.sh +{{ pillar["admin_web_ui"]["build_dir"] }}/build.sh +{{ pillar["customer_web_ui"]["build_dir"] }}/build.sh + +popd 2>/dev/null +exit 0 diff --git a/salt/openbook/ci/files/cloudcfg.sh b/salt/openbook/ci/files/cloudcfg.sh new file mode 100644 index 0000000..f7dffd6 --- /dev/null +++ b/salt/openbook/ci/files/cloudcfg.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Prepare the instance for ansible. This amounts to installing python2. + +python=$(which python || which python3 || which python2 || false) + +case "$(${python} -m platform)" in + *Ubuntu*) + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade + apt-get -y install python python2.7 + ;; + *centos*|*redhat*) + # RHEL systems still use python2 by default; nothing to do + ;; + *gentoo*) + if [ ! -d /usr/portage/sys-devel/gcc ]; then + emerge-webrsync + fi + emerge --oneshot --update dev-lang/python:2.7 app-portage/gentoolkit + ;; +esac + +exit 0 diff --git a/salt/openbook/ci/files/openbook.gentoo b/salt/openbook/ci/files/openbook.gentoo new file mode 100644 index 0000000..4a639d4 --- /dev/null +++ b/salt/openbook/ci/files/openbook.gentoo @@ -0,0 +1,25 @@ +#!/sbin/openrc-run + +extra_commands=test_database + +depend() { + need docker +} + +start() { + ebegin "Starting Openbook" + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} start + eend $? +} + +stop() { + ebegin "Stopping Openbook" + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} stop + eend $? +} + +test_database() { + ebegin "Testing database" + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} test_database + eend $? +} diff --git a/salt/openbook/ci/files/openbook.redhat b/salt/openbook/ci/files/openbook.redhat new file mode 100644 index 0000000..571894a --- /dev/null +++ b/salt/openbook/ci/files/openbook.redhat @@ -0,0 +1,29 @@ +#!/bin/bash + +# openbook +# +# chkconfig: 2345 10 90 +# description: Manage the Openbook application +# +### BEGIN INIT INFO +# Provides: openbook +# Required-Start: docker +# Required-Stop: docker +# Short-Description: Bring up/down networking +# Description: Manage the Openbook application +### END INIT INFO + +case $1 in + start*) + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} start + ;; + stop*) + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} stop + ;; + test_database*) + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} test_database + ;; + *) + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} help + ;; +esac diff --git a/salt/openbook/ci/files/openbook.ubuntu b/salt/openbook/ci/files/openbook.ubuntu new file mode 100644 index 0000000..cda042d --- /dev/null +++ b/salt/openbook/ci/files/openbook.ubuntu @@ -0,0 +1,25 @@ +#!/bin/sh + +### BEGIN INIT INFO +# Provides: openbook +# Required-Start: docker +# Required-Stop: docker +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Manage the Openbook application +### END INIT INFO + +case $1 in + start*) + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} start + ;; + stop*) + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} stop + ;; + test_database*) + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} test_database + ;; + *) + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/{{ pillar["ci"]["files"]["openbookctl"]["name"] }} help + ;; +esac diff --git a/salt/openbook/ci/files/openbookctl b/salt/openbook/ci/files/openbookctl new file mode 100644 index 0000000..894368d --- /dev/null +++ b/salt/openbook/ci/files/openbookctl @@ -0,0 +1,85 @@ +#!/bin/bash + +echo 'mysql root password: {% raw %}{{ openbook_database_password }}{% endraw %}' +echo 'wsrep_node_address: {% raw %}{{ ansible_default_ipv4.address }}{% endraw %}' +echo 'wsrep_cluster_address: gcomm://{% raw %}{{ openbook_gcomm_addresses }}{% endraw %}' + +wait_for_database() { + echo "Waiting for database to start..." + started=1 + for i in {120..0}; do + if echo 'SELECT 1' | mysql -h 127.0.0.1 -u root --password='{% raw %}{{ openbook_database_password }}{% endraw %}' &>/dev/null; then + started=0 + break + fi + sleep 1 + done + if [ $started -eq 1 ]; then + echo "Error: database did not start." + exit 1 + fi +} + +ensure_openbook_databases_created() { + if ! echo "SHOW DATABASES;" | mysql -h 127.0.0.1 -u root --password='{% raw %}{{ openbook_database_password }}{% endraw %}' 2>/dev/null | fgrep 'openbook'; then + echo "Creating Openbook schema..." + mysql -h 127.0.0.1 -u root --password='{% raw %}{{ openbook_database_password }}{% endraw %}' &>/dev/null < {{ pillar["database"]["files"]["create_openbook_schemas"]["path"] }}/{{ pillar["database"]["files"]["create_openbook_schemas"]["name"] }} + fi +} + +do_start() { + chmod 777 {{ pillar["database"]["dirs"]["var_lib_mysql"]["name"] }} + chmod 777 {{ pillar["database"]["dirs"]["var_log_mysql"]["name"] }} + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/database/run.sh + wait_for_database + ensure_openbook_databases_created + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/api_server/run.sh + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/admin_web_ui/run.sh + {{ pillar["ansible"]["dirs"]["output"]["name"] }}/customer_web_ui/run.sh + echo + echo "Openbook started. All components may not be fully initialized and available." + echo "It is highly recommended to wait until initialization is complete before" + echo "starting the next node in the cluster. To monitor the state, run:" + echo + echo "watch -n 5 \"lsof -i -n -P 2>/dev/null | fgrep '*:808'\"" + echo + echo "When the output looks like the following, the node is fully initialized:" + echo "(Your PIDs will vary)" + echo + echo "java 27424 root 73u IPv6 65463 0t0 TCP *:8080 (LISTEN)" + echo "java 27264 root 73u IPv6 65430 0t0 TCP *:8081 (LISTEN)" + echo "java 27349 root 73u IPv6 65430 0t0 TCP *:8082 (LISTEN)" +} + +do_stop() { + docker stop {{ pillar["customer_web_ui"]["container_name"] }} + docker stop {{ pillar["admin_web_ui"]["container_name"] }} + docker stop {{ pillar["api_server"]["container_name"] }} + docker stop {{ pillar["database"]["container_name"] }} + docker container prune -f +} + +do_help() { + echo "Usage:" + echo " $0 start" + echo " $0 stop" + echo " $0 test_database" +} + +case $1 in + start*) + do_start + ;; + stop*) + do_stop + ;; + test_database*) + wait_for_database + echo "Database is up." + ;; + *) + do_help + ;; +esac + +exit 0 diff --git a/salt/openbook/ci/init.sls b/salt/openbook/ci/init.sls new file mode 100644 index 0000000..d121326 --- /dev/null +++ b/salt/openbook/ci/init.sls @@ -0,0 +1,55 @@ +{% for dir in pillar["ci"]["dirs"].items() %} +database_dirs_{{ dir[0] }}: + file.directory: +{% if dir[1]["relative_to_buildroot"] %} + - name: {{ pillar["ci"]["build_dir"] }}/{{ dir[1]["name"] }} +{% else %} + - name: {{ dir[1]["name"] }} +{% endif %} + - user: root + - group: root + - mode: {{ dir[1]["mode"] }} + - makedirs: True +{% endfor %} + +{% for file in pillar["ci"]["files"].items() %} +ci_files_{{ file[0] }}: + file.managed: +{% if file[1]["relative_to_buildroot"] %} + - name: {{ pillar["ci"]["build_dir"] }}/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% else %} + - name: {{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/ci/files/{{ file[1]["source"] }} +{% if file[1]["is_template"] %} + - template: jinja +{% endif %} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% if file[1]["ansible"] %} +ansible_ci_dirs_{{ file[0] }}: + file.directory: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/ci/{{ file[1]["path"] }} +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/ci/{{ file[1]["path"] }} +{% endif %} + - user: root + - group: root + - mode: "0755" + - makedirs: True +ansible_ci_files_{{ file[0] }}: + file.managed: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/ci/{{ file[1]["path"] }}/{{ file[1]["name"] }} + - template: jinja +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/ci/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/ci/files/{{ file[1]["source"] }} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% endif %} +{% endfor %} diff --git a/salt/openbook/customer_web_ui/files/Dockerfile b/salt/openbook/customer_web_ui/files/Dockerfile new file mode 100644 index 0000000..b5885c1 --- /dev/null +++ b/salt/openbook/customer_web_ui/files/Dockerfile @@ -0,0 +1,8 @@ +FROM {{ pillar["baseimage"]["image_name"] }} +MAINTAINER Carlos Konstanski <carlos@talligent.com> +ADD docker-start.tar.xz / +EXPOSE 8080 5701 5702 +VOLUME {{ pillar["customer_web_ui"]["dirs"]["etc_openbook_customer_web_ui"]["name"] }} +VOLUME {{ pillar["customer_web_ui"]["dirs"]["var_log_openbook_customer_web_ui"]["name"] }} +RUN /installdeps.sh +ENTRYPOINT [ "/start_app.sh" ] diff --git a/salt/openbook/customer_web_ui/files/build.sh b/salt/openbook/customer_web_ui/files/build.sh new file mode 100644 index 0000000..6119a0c --- /dev/null +++ b/salt/openbook/customer_web_ui/files/build.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Do not call directly. Meant to be called from CI build script. + +pushd {{ pillar["customer_web_ui"]["build_dir"] }} 2>/dev/null + +echo "Creating tmp directory" +rm -rf tmp +mkdir tmp +rm -f docker-start.tar.xz + +echo "Populating with custom files" +rsync -aq data/ tmp/ + +echo "Repackaging into .xz file" +tar Jcvf docker-start.tar.xz -C tmp/ . + +echo "Deleting existing docker container" +docker stop {{ pillar["customer_web_ui"]["container_name"] }} +docker container prune -f +docker rmi -f {{ pillar["customer_web_ui"]["image_name"] }} + +echo "Building docker Ubuntu image" +docker build -t {{ pillar["customer_web_ui"]["image_name"] }} . + +echo "Push docker image to the registry" +docker tag {{ pillar["customer_web_ui"]["image_name"] }} {{ pillar["registry"]["url"] }}/{{ pillar["customer_web_ui"]["image_name"] }} +docker push {{ pillar["registry"]["url"] }}/{{ pillar["customer_web_ui"]["image_name"] }} + +echo "Cleaning up" +rm -rf tmp +rm -f docker-start.tar.xz + +popd 2>/dev/null +exit 0 diff --git a/salt/openbook/customer_web_ui/files/customer_web_ui.json b/salt/openbook/customer_web_ui/files/customer_web_ui.json new file mode 100644 index 0000000..db06bc4 --- /dev/null +++ b/salt/openbook/customer_web_ui/files/customer_web_ui.json @@ -0,0 +1,3 @@ +{ + "log.path": "{{ pillar["customer_web_ui"]["log"] }}" +} diff --git a/salt/openbook/customer_web_ui/files/installdeps.sh b/salt/openbook/customer_web_ui/files/installdeps.sh new file mode 100644 index 0000000..f76ccfa --- /dev/null +++ b/salt/openbook/customer_web_ui/files/installdeps.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +apt-get -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" openjdk-8-jre-headless + +curl -k \ + -u '{{ pillar["jars"]["username"] }}:{{ pillar["jars"]["password"] }}' \ + -o "{{ pillar["customer_web_ui"]["jar_name"] }}" \ + "{{ pillar["jars"]["url"] }}{{ pillar["customer_web_ui"]["jar_name"] }}" + +exit 0 diff --git a/salt/openbook/customer_web_ui/files/run.sh b/salt/openbook/customer_web_ui/files/run.sh new file mode 100644 index 0000000..ce6b485 --- /dev/null +++ b/salt/openbook/customer_web_ui/files/run.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +docker run -d --net=host -v {{ pillar["customer_web_ui"]["dirs"]["etc_openbook_customer_web_ui"]["name"] }}:{{ pillar["customer_web_ui"]["dirs"]["etc_openbook_customer_web_ui"]["name"] }} -v {{ pillar["customer_web_ui"]["dirs"]["var_log_openbook_customer_web_ui"]["name"] }}:{{ pillar["customer_web_ui"]["dirs"]["var_log_openbook_customer_web_ui"]["name"] }} --name {{ pillar["customer_web_ui"]["container_name"] }} {{ pillar["registry"]["url"] }}/{{ pillar["customer_web_ui"]["image_name"] }} +exit 0 diff --git a/salt/openbook/customer_web_ui/files/start_app.sh b/salt/openbook/customer_web_ui/files/start_app.sh new file mode 100644 index 0000000..7f24089 --- /dev/null +++ b/salt/openbook/customer_web_ui/files/start_app.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exec java -jar {{ pillar["customer_web_ui"]["jar_name"] }} run com.talligent.openbook.verticle.Startup --cluster diff --git a/salt/openbook/customer_web_ui/init.sls b/salt/openbook/customer_web_ui/init.sls new file mode 100644 index 0000000..957e220 --- /dev/null +++ b/salt/openbook/customer_web_ui/init.sls @@ -0,0 +1,55 @@ +{% for dir in pillar["customer_web_ui"]["dirs"].items() %} +customerui_dirs_{{ dir[0] }}: + file.directory: +{% if dir[1]["relative_to_buildroot"] %} + - name: {{ pillar["customer_web_ui"]["build_dir"] }}/{{ dir[1]["name"] }} +{% else %} + - name: {{ dir[1]["name"] }} +{% endif %} + - user: root + - group: root + - mode: {{ dir[1]["mode"] }} + - makedirs: True +{% endfor %} + +{% for file in pillar["customer_web_ui"]["files"].items() %} +customerui_files_{{ file[0] }}: + file.managed: +{% if file[1]["relative_to_buildroot"] %} + - name: {{ pillar["customer_web_ui"]["build_dir"] }}/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% else %} + - name: {{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/customer_web_ui/files/{{ file[1]["source"] }} +{% if file[1]["is_template"] %} + - template: jinja +{% endif %} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% if file[1]["ansible"] %} +ansible_customerui_dirs_{{ file[0] }}: + file.directory: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/customer_web_ui/{{ file[1]["path"] }} +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/customer_web_ui/{{ file[1]["path"] }} +{% endif %} + - user: root + - group: root + - mode: "0755" + - makedirs: True +ansible_customerui_files_{{ file[0] }}: + file.managed: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/customer_web_ui/{{ file[1]["path"] }}/{{ file[1]["name"] }} + - template: jinja +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/customer_web_ui/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/customer_web_ui/files/{{ file[1]["source"] }} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% endif %} +{% endfor %} diff --git a/salt/openbook/database/files/Dockerfile b/salt/openbook/database/files/Dockerfile new file mode 100644 index 0000000..3e7848e --- /dev/null +++ b/salt/openbook/database/files/Dockerfile @@ -0,0 +1,10 @@ +FROM {{ pillar["baseimage"]["image_name"] }} +MAINTAINER Carlos Konstanski <carlos@talligent.com> +ADD docker-start.tar.xz / +EXPOSE 3306 4444 4567 4567/udp 4568 +VOLUME /etc/mysql +VOLUME /var/lib/mysql +VOLUME /var/log/mysql +RUN /installdeps.sh +ENTRYPOINT [ "/etc/mysql/bin/start_galera_node.sh" ] +CMD [] diff --git a/salt/openbook/database/files/_my.cnf b/salt/openbook/database/files/_my.cnf new file mode 100644 index 0000000..e158680 --- /dev/null +++ b/salt/openbook/database/files/_my.cnf @@ -0,0 +1,6 @@ +[mysql] +user=root +password={% raw %}{{ openbook_database_password }}{% endraw %} +[mysqldump] +user=root +password={% raw %}{{ openbook_database_password }}{% endraw %} diff --git a/salt/openbook/database/files/build.sh b/salt/openbook/database/files/build.sh new file mode 100644 index 0000000..49c9aba --- /dev/null +++ b/salt/openbook/database/files/build.sh @@ -0,0 +1,36 @@ +- +#!/bin/bash + +# Do not call directly. Meant to be called from CI build script. + +pushd {{ pillar["database"]["build_dir"] }} 2>/dev/null + +echo "Creating tmp directory" +rm -rf tmp +mkdir tmp +rm -f docker-start.tar.xz + +echo "Populating with custom file" +rsync -aq data/ tmp/ + +echo "Repackaging into .xz file" +tar Jcvf docker-start.tar.xz -C tmp/ . + +echo "Deleting existing docker container" +docker stop {{ pillar["database"]["container_name"] }} +docker container prune -f +docker rmi -f {{ pillar["database"]["image_name"] }} + +echo "Building docker Ubuntu image" +docker build -t {{ pillar["database"]["image_name"] }} . + +echo "Push docker image to the registry" +docker tag {{ pillar["database"]["image_name"] }} {{ pillar["registry"]["url"] }}/{{ pillar["database"]["image_name"] }} +docker push {{ pillar["registry"]["url"] }}/{{ pillar["database"]["image_name"] }} + +echo "Cleaning up" +rm -rf tmp +rm -f docker-start.tar.xz + +popd 2>/dev/null +exit 0 diff --git a/salt/openbook/database/files/cluster.cnf b/salt/openbook/database/files/cluster.cnf new file mode 100644 index 0000000..5dbd3cf --- /dev/null +++ b/salt/openbook/database/files/cluster.cnf @@ -0,0 +1,17 @@ +[mysqld] +query_cache_size=0 +default-storage-engine=innodb +query_cache_type=0 +group_concat_max_len=1000000 +innodb_autoinc_lock_mode=2 +innodb_locks_unsafe_for_binlog=1 +binlog_format=row + +[galera] +wsrep_on=ON +wsrep_provider=/usr/lib/galera/libgalera_smm.so +wsrep_cluster_name="openbook_cluster" +wsrep_cluster_address="gcomm://{% raw %}{{ openbook_gcomm_addresses }}{% endraw %}" +wsrep_sst_method=rsync +wsrep_node_address="{% raw %}{{ ansible_default_ipv4.address }}{% endraw %}" +wsrep_node_name="{% raw %}{{ ansible_hostname }}{% endraw %}" diff --git a/salt/openbook/database/files/create_openbook_schemas.sql b/salt/openbook/database/files/create_openbook_schemas.sql new file mode 100644 index 0000000..f6544fc --- /dev/null +++ b/salt/openbook/database/files/create_openbook_schemas.sql @@ -0,0 +1,1416 @@ +SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0; +SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0; +SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL,ALLOW_INVALID_DATES'; + +DROP SCHEMA IF EXISTS openbook; +CREATE SCHEMA openbook DEFAULT CHARACTER SET utf8; + +GRANT ALL PRIVILEGES ON openbook.* TO openbook@'%' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook.* TO openbook@localhost IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +FLUSH PRIVILEGES; + +USE openbook; + +CREATE TABLE t_audit_event ( + event_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + event_time DATETIME(6) NOT NULL, + event_type VARCHAR(32) NOT NULL, + entity_id VARCHAR(64) NOT NULL, + entity_name VARCHAR(255) NULL, + entity_type VARCHAR(45) NOT NULL, + user_id VARCHAR(128) NULL, + event_data LONGTEXT NULL, + PRIMARY KEY (event_id), + INDEX ix_audit_event_1 (event_time, event_type, entity_id), + INDEX ix_audit_event_2 (user_id, event_time)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_billable_feature ( + billable_feature_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + billable_feature_category VARCHAR(128) NOT NULL, + billable_feature_vendor_type VARCHAR(128) NOT NULL, + billable_feature_type VARCHAR(128) NOT NULL, + billable_feature_subtypes LONGTEXT NULL DEFAULT NULL, + billable_feature_subtype_classifier LONGTEXT NULL DEFAULT NULL, + billable_feature_description LONGTEXT NULL DEFAULT NULL, + billable_feature_details LONGTEXT NULL DEFAULT NULL, + PRIMARY KEY (billable_feature_id), + UNIQUE INDEX ix_billable_feature_1 (billable_feature_vendor_type ASC, billable_feature_type ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_credit ( + credit_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + payer_id INT UNSIGNED NOT NULL, + payer_type VARCHAR(64) NOT NULL, + credit_amount DECIMAL(19,2) NULL, + remaining_amount DECIMAL(19,2) NULL, + credit_type VARCHAR(16) NOT NULL, + granted_by VARCHAR(256) NOT NULL, + posted_date DATETIME(6) NOT NUll, + expire_date DATETIME(6) NOT NULL, + currency VARCHAR(3) NOT NULL, + PRIMARY KEY (credit_id), + INDEX ix_credit_1 (payer_id, payer_type, expire_date)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_customer ( + customer_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + resource_manager_id VARCHAR(36) NULL DEFAULT NULL, + resource_manager_foreign_id VARCHAR(512) NULL DEFAULT NULL, + customer_name VARCHAR(512) NOT NULL, + other_foreign_ids LONGTEXT NULL DEFAULT NULL, + tax_rate DECIMAL(18,4) NULL DEFAULT 0.0000, + currency CHAR(6) NULL DEFAULT NULL, + discount_percent DECIMAL(18,4) NULL DEFAULT 0.0000, + billing_frequency VARCHAR(40) NULL DEFAULT NULL, + last_bill_date DATETIME(6) NULL DEFAULT NULL, + next_bill_date DATETIME(6) NULL DEFAULT NULL, + is_active TINYINT(1) NULL DEFAULT 1, + create_date DATETIME(6) NULL DEFAULT NULL, + activate_date DATETIME(6) NULL DEFAULT NULL, + billing_start_date DATETIME(6) NULL DEFAULT NULL, + last_payment_date DATETIME(6) NULL DEFAULT NULL, + deactivate_date DATETIME(6) NULL DEFAULT NULL, + description VARCHAR(5120) NULL DEFAULT NULL, + minimum_invoice_commitment DECIMAL(19,4) NULL, + invoicing_method VARCHAR(32) NOT NULL, + management_region_id VARCHAR(64) NULL, + infrastructure_region_id VARCHAR(64) NULL, + billing_address_line_1 VARCHAR(512) NULL, + billing_address_line_2 VARCHAR(512) NULL, + billing_address_city VARCHAR(512) NULL, + billing_address_region VARCHAR(128) NULL, + billing_address_country VARCHAR(2) NULL, + billing_address_postal_code VARCHAR(64) NULL, + payer_payment_details VARCHAR(5120) NULL, + payment_method_confirmed TINYINT(1) DEFAULT 0, + balance_due DECIMAL(19,4) NULL DEFAULT 0, + PRIMARY KEY (customer_id), + INDEX ix_customer_1 (resource_manager_id ASC), + INDEX ix_customer_2 (resource_manager_foreign_id(255) ASC), + UNIQUE INDEX ix_customer_3 (management_region_id, customer_name(255))) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_discount ( + discount_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + payer_id INT UNSIGNED NOT NULL, + payer_type VARCHAR(64) NOT NULL, + discount_percent DECIMAL(5,2) NOT NULL, + granted_by VARCHAR(256) NOT NULL, + apply_to_charge_category VARCHAR(32) NOT NULL, + trigger_charge_category VARCHAR(32) NULL, + trigger_amount DECIMAL(19,2) NULL, + start_date DATETIME(6) NOT NULL, + end_date DATETIME(6) NOT NULL, + PRIMARY KEY (discount_id), + INDEX ix_discount_1 (payer_id, payer_type, end_date)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_email_notification ( + message_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + recipient VARCHAR(128) NOT NULL, + sender VARCHAR(128) NOT NULL, + subject VARCHAR(256) NOT NULL, + message_text LONGTEXT NOT NULL, + smtp_server_details LONGTEXT NOT NULL, + date_generated DATETIME(6) NOT NULL, + date_sent DATETIME(6) NULL, + date_last_send_attempt DATETIME(6) NULL, + send_status VARCHAR(16) NULL, + send_status_detail VARCHAR(5120) NULL, + send_attempt_count SMALLINT NULL, + PRIMARY KEY (message_id), + INDEX ix_email_notification_1 (recipient, date_generated), + INDEX ix_email_notification_2 (sender, date_generated), + INDEX ix_email_notification_3 (date_sent)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_email_template ( + provider_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + template_subject VARCHAR(256) NOT NULL, + template_name VARCHAR(128) NOT NULL, + template_text LONGTEXT NOT NULL, + PRIMARY KEY (provider_id, template_name)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_entity_attribute ( + entity_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + attribute_class VARCHAR(32) NOT NULL, + attribute_category VARCHAR(128) NOT NULL, + attribute_name VARCHAR(128) NOT NULL, + attribute_value VARCHAR(512) NOT NULL, + attribute_unit VARCHAR(64) NULL DEFAULT NULL, + add_date DATETIME(6) NULL DEFAULT NULL, + expire_date DATETIME(6) NOT NULL, + PRIMARY KEY (entity_id ASC, expire_date ASC, attribute_class ASC, attribute_category ASC, attribute_name ASC), + INDEX ix_entity_attribute_1 (attribute_class ASC), + INDEX ix_entity_attribute_2 (attribute_category ASC), + INDEX ix_entity_attribute_3 (attribute_name ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_invoice ( + invoice_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + payer_id INT UNSIGNED NOT NULL, + payer_type VARCHAR(64) NOT NULL, + payer_name VARCHAR(512) NOT NULL, + invoice_number VARCHAR(45) NULL, + invoice_date DATETIME(6) NOT NULL, + period_begin DATETIME(6) NOT NULL, + period_end DATETIME(6) NOT NULL, + invoice_amount DECIMAL(19,2) NOT NULL, + actual_charges DECIMAL(19,2) NOT NULL, + invoice_currency VARCHAR(16) NULL, + management_region_id INT UNSIGNED NOT NULL, + balance_due DECIMAL(19,2) NOT NULL, + PRIMARY KEY (invoice_id), + INDEX ix_invoice_1 (management_region_id, payer_type, payer_id ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_invoice_detail ( + invoice_id INT UNSIGNED NOT NULL, + customer_id INT UNSIGNED NOT NULL, + resource_group_id VARCHAR(36) NULL, + invoice_group VARCHAR(128) NULL, + item_name VARCHAR(512) NULL, + item_vendor_type VARCHAR(128) NULL, + item_type VARCHAR(128) NOT NULL, + item_subtype VARCHAR(128) NOT NULL, + item_description VARCHAR(512) NOT NULL, + rate_plan_id VARCHAR(36) NULL, + rate_plan_billable_feature_id VARCHAR(36) NULL DEFAULT NULL, + billing_interval VARCHAR(64) NULL DEFAULT NULL, + item_invoice_amount DECIMAL(19,2) NOT NULL, + num_units VARCHAR(32) NOT NULL, + num_units_unit VARCHAR(16) NULL, + INDEX ix_invoice_detail_1 (invoice_id ASC), + INDEX ix_invoice_detail_2 (customer_id ASC), + INDEX ix_invoice_detail_3 (item_vendor_type, item_type, item_subtype ASC), + INDEX ix_invoice_detail_4 (rate_plan_id ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_job_status ( + job_id VARCHAR(256) NOT NULL, + start_time DATETIME(6) NOT NULL, + end_time DATETIME(6) NULL DEFAULT NULL, + status VARCHAR(32) NULL DEFAULT NULL, + result VARCHAR(32) NULL DEFAULT NULL, + result_detail LONGTEXT NULL DEFAULT NULL, + PRIMARY KEY (job_id(255), start_time)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_lookup_values ( + type VARCHAR(128) NOT NULL, + name VARCHAR(256) NOT NULL, + value LONGTEXT NOT NULL, + PRIMARY KEY (type, name(255))) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_parent_child_relationships ( + parent_id INT UNSIGNED NOT NULL, + parent_type VARCHAR(64) NOT NULL, + child_id INT UNSIGNED NOT NULL, + child_type VARCHAR(64) NOT NULL, + assigned_quantity INT NOT NULL DEFAULT 1, + add_date DATETIME(6) NOT NULL, + remove_date DATETIME(6) NOT NULL, + PRIMARY KEY (parent_id, parent_type, child_id, child_type, add_date, remove_date), + INDEX ix_parent_child_relationships_1 (child_id, child_type ASC), + INDEX ix_parent_child_relationships_2 (parent_id, parent_type ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_payment ( + payment_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + payer_id INT UNSIGNED NOT NULL, + payer_type VARCHAR(64) NOT NULL, + payment_date DATETIME(6) NOT NULL, + amount DECIMAL(19,2) NOT NULL, + payment_method VARCHAR(64) NOT NULL, + payment_reference VARCHAR(256) NULL, + payment_method_detail LONGTEXT NULL DEFAULT NULL, + payment_status VARCHAR(64) NULL DEFAULT NULL, + payment_status_detail LONGTEXT NULL DEFAULT NULL, + PRIMARY KEY (payment_id), + INDEX ix_payment_1 (payer_type, payer_id, payment_date)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_promo_code ( + management_region_id INT UNSIGNED NOT NULL, + promo_code_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + promo_code VARCHAR(128) NOT NULL, + salesperson VARCHAR(128) NULL, + discount_or_credit VARCHAR(8) NOT NULL, + credit_amount DECIMAL(19,2) NULL, + credit_type VARCHAR(16) NULL, + apply_to_charge_category VARCHAR(32) NULL, + trigger_charge_category VARCHAR(32) NULL, + trigger_amount DECIMAL(19,2) NULL, + discount_percent DECIMAL(5,2) NULL, + available_invoice_periods INT NULL, + expire_date DATETIME(6) NULL, + currency VARCHAR(3) NULL DEFAULT NULL, + PRIMARY KEY (promo_code_id), + INDEX ix_promo_code_1 (promo_code), + INDEX ix_promo_code_2 (salesperson), + UNIQUE INDEX ix_promo_code_3 (management_region_id, promo_code)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_promo_code_redemption ( + promo_code_id INT UNSIGNED NOT NULL, + payer_id INT UNSIGNED NOT NULL, + payer_type VARCHAR(64) NOT NULL, + redemption_date DATETIME(6) NULL, + PRIMARY KEY (payer_id, promo_code_id)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8; + + +CREATE TABLE t_provider ( + provider_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + provider_name VARCHAR(512) NOT NULL, + provider_type VARCHAR(64) NOT NULL, + other_foreign_ids LONGTEXT NULL DEFAULT NULL, + tax_rate DECIMAL(18,4) NULL DEFAULT 0.0000, + currency CHAR(6) NULL DEFAULT NULL, + discount_percent DECIMAL(18,4) NULL DEFAULT 0.0000, + billing_frequency VARCHAR(40) NULL DEFAULT NULL, + last_bill_date DATETIME(6) NULL DEFAULT NULL, + next_bill_date DATETIME(6) NULL DEFAULT NULL, + last_payment_date DATETIME(6) NULL DEFAULT NULL, + is_active TINYINT(1) NULL DEFAULT 1, + create_date DATETIME(6) NULL DEFAULT NULL, + activate_date DATETIME(6) NULL DEFAULT NULL, + billing_start_date DATETIME(6) NULL DEFAULT NULL, + deactivate_date DATETIME(6) NULL DEFAULT NULL, + description LONGTEXT NULL DEFAULT NULL, + minimum_invoice_commitment DECIMAL(19,2) NULL, + invoicing_method VARCHAR(32) NULL DEFAULT NULL, + invoice_processing_method VARCHAR(32) NULL DEFAULT NULL, + billing_address_line_1 VARCHAR(512) NULL, + billing_address_line_2 VARCHAR(512) NULL, + billing_address_city VARCHAR(512) NULL, + billing_address_region VARCHAR(128) NULL, + billing_address_country VARCHAR(2) NULL, + billing_address_postal_code VARCHAR(64) NULL, + legal_address_line_1 VARCHAR(512) NULL, + legal_address_line_2 VARCHAR(512) NULL, + legal_address_city VARCHAR(512) NULL, + legal_address_region VARCHAR(128) NULL, + legal_address_country VARCHAR(2) NULL, + legal_address_postal_code VARCHAR(64) NULL, + payer_payment_details VARCHAR (5120) NULL, + payee_payment_details VARCHAR(5120) NULL, + public_url VARCHAR(512) NOT NULL, + private_url VARCHAR(512) NOT NULL, + management_region_id VARCHAR(64) NOT NULL, + notification_settings LONGTEXT NOT NULL, + personalization_settings LONGTEXT NOT NULL, + balance_due DECIMAL(19,2) NULL, + openbook_processes_payments TINYINT(1) NULL DEFAULT 1, + openbook_processes_past_due_invoices TINYINT(1) NULL DEFAULT 1, + openstack_tenant_id VARCHAR(36) NULL DEFAULT NULL, + PRIMARY KEY (provider_id), + UNIQUE INDEX ix_provider_1 (public_url(255)), + UNIQUE INDEX ix_provider_2 (management_region_id)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_provider_region ( + provider_id INT UNSIGNED NOT NULL, + region_id INT UNSIGNED NOT NULL, + PRIMARY KEY (provider_id, region_id)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_provisioned_entity ( + entity_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + billable_feature_id INT UNSIGNED NOT NULL, + entity_name VARCHAR(256) NOT NULL, + entity_subtype VARCHAR(128) NULL DEFAULT NULL, + customer_id INT UNSIGNED NOT NULL, + resource_manager_id INT UNSIGNED NOT NULL, + infrastructure_region_id INT UNSIGNED NOT NULL, + entity_foreign_id VARCHAR(256) NULL DEFAULT NULL, + provisioned_date DATETIME(6) NULL DEFAULT NULL, + deprovisioned_date DATETIME(6) NULL DEFAULT NULL, + PRIMARY KEY (entity_id), + INDEX ix_provisioned_entity_1 (entity_foreign_id(255) ASC), + INDEX ix_provisioned_entity_2 (customer_id), + INDEX ix_provisioned_entity_3 (resource_manager_id ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_rate_plan ( + rate_plan_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + rate_plan_name VARCHAR(255) NULL DEFAULT NULL, + applies_to VARCHAR(32) NOT NULL, + rate_plan_description VARCHAR(1024) NULL DEFAULT NULL, + proration_details LONGTEXT NULL DEFAULT NULL, + currency CHAR(6) NULL DEFAULT NULL, + discount_percent VARCHAR(20) NULL DEFAULT NULL, + management_region_id VARCHAR(64) NOT NULL, + PRIMARY KEY (rate_plan_id), + INDEX ix_rate_plan_1 (management_region_id ASC), + UNIQUE INDEX ix_rate_plan_2 (management_region_id, rate_plan_name)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_rate_plan_assignment ( + rate_plan_id INT UNSIGNED NOT NULL, + assignee_id INT UNSIGNED NOT NULL, + assignee_type VARCHAR(64) NOT NULL, + assignment_start_date DATETIME(6) NOT NULL, + assignment_end_date DATETIME(6) NOT NULL, + assignment_type VARCHAR(64) NOT NULL, + PRIMARY KEY (rate_plan_id, assignee_id, assignment_start_date, assignment_end_date), + INDEX ix_rate_plan_assignment_1 (assignee_type, assignee_id, assignment_start_date, assignment_end_date ASC), + INDEX ix_rate_plan_assignment_2 (assignment_type, assignee_id, assignment_start_date, assignment_end_date ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_rate_plan_billable_attribute ( + rate_plan_id INT UNSIGNED NOT NULL, + feature_type VARCHAR(128) NOT NULL, + attribute_class VARCHAR(32) NOT NULL, + attribute_category VARCHAR(128) NOT NULL, + attribute_name VARCHAR(128) NOT NULL, + attribute_value VARCHAR(512) NOT NULL, + attribute_unit VARCHAR(64) NULL DEFAULT NULL, + attribute_datatype VARCHAR(32) NOT NULL, + charge VARCHAR(32) NOT NULL, + operator VARCHAR(32) NULL DEFAULT NULL, + billing_scheme VARCHAR(64) NOT NULL, + cost_calculation_frequency VARCHAR(64) NOT NULL, + filter TEXT NULL, + infrastructure_region_id INT UNSIGNED NOT NULL DEFAULT 0, + INDEX ix_rate_plan_billable_attribute_1 (rate_plan_id, infrastructure_region_id)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_rate_plan_billable_feature ( + rate_plan_id INT UNSIGNED NOT NULL, + billable_feature_id INT UNSIGNED NOT NULL, + billable_feature_subtype VARCHAR(128) NOT NULL, + billable_interval_type VARCHAR(64) NOT NULL, + cost_calculation_frequency VARCHAR(64) NOT NULL, + charge VARCHAR(100) NULL DEFAULT NULL, + infrastructure_region_id INT UNSIGNED NOT NULL DEFAULT 0, + PRIMARY KEY (rate_plan_id, infrastructure_region_id, billable_feature_id, billable_feature_subtype, billable_interval_type)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_rate_plan_billable_metric ( + rate_plan_id INT UNSIGNED NOT NULL, + vendor_type VARCHAR(128) NOT NULL, + metric_name VARCHAR(128) NOT NULL, + metric_unit VARCHAR(32) NULL DEFAULT NULL, + metric_type VARCHAR(32) NULL DEFAULT NULL, + billing_scheme VARCHAR(64) NOT NULL, + billing_scheme_details LONGTEXT NULL DEFAULT NULL, + infrastructure_region_id INT UNSIGNED NOT NULL DEFAULT 0, + PRIMARY KEY (rate_plan_id, infrastructure_region_id, vendor_type, metric_name, billing_scheme)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_rate_plan_billable_event ( + rate_plan_id INT UNSIGNED NOT NULL, + billable_feature_id INT UNSIGNED NOT NULL, + billable_feature_subtype VARCHAR(128) NOT NULL, + event_type VARCHAR(128) NOT NULL, + charge VARCHAR(100) NOT NULL, + billing_scheme_details LONGTEXT NULL DEFAULT NULL, + infrastructure_region_id INT UNSIGNED NOT NULL DEFAULT 0, + PRIMARY KEY (rate_plan_id, infrastructure_region_id, billable_feature_id, billable_feature_subtype, event_type)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_region ( + region_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + region_name VARCHAR(128) NOT NULL, + region_type VARCHAR(64) NOT NULL, + parent_region_id INT UNSIGNED NULL DEFAULT NULL, + PRIMARY KEY (region_id), + UNIQUE INDEX ix_region_1 (parent_region_id, region_type, region_name ASC), + INDEX ix_region_2 (parent_region_id)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_resource_configuration_history ( + resource_id INT UNSIGNED NOT NULL, + resource_name VARCHAR(512) NULL DEFAULT NULL, + resource_foreign_id VARCHAR(146) NULL DEFAULT NULL, + resource_configuration LONGTEXT NOT NULL, + start_date DATETIME(6) NOT NULL, + end_date DATETIME(6) NOT NULL, + customer_id LONGTEXT NULL DEFAULT NULL, + customer_foreign_id VARCHAR(146) NULL DEFAULT NULL, + customer_name VARCHAR(512) NULL DEFAULT NULL, + resource_manager_id VARCHAR(72) NULL DEFAULT NULL, + resource_manager_name VARCHAR(512) NULL DEFAULT NULL, + resource_manager_foreign_id VARCHAR(72) NULL DEFAULT NULL) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_resource_event ( + event_id VARCHAR(100) NOT NULL, + event_type VARCHAR(128) NOT NULL, + event_date DATETIME(6) NOT NULL, + resource_id VARCHAR(36) NULL DEFAULT NULL, + resource_name VARCHAR(256) NULL DEFAULT NULL, + resource_foreign_id VARCHAR(512) NULL DEFAULT NULL, + customer_id VARCHAR(36) NULL DEFAULT NULL, + customer_name VARCHAR(512) NULL DEFAULT NULL, + customer_foreign_id VARCHAR(512) NULL DEFAULT NULL, + resource_manager_id VARCHAR(36) NOT NULL, + resource_manager_foreign_id VARCHAR(512) NULL DEFAULT NULL, + infrastructure_region_id INT UNSIGNED NULL DEFAULT NULL, + management_region_id INT UNSIGNED NULL DEFAULT NULL, + PRIMARY KEY (resource_manager_id, event_id), + INDEX ix_vm_event_1 (resource_foreign_id(255) ASC, event_date ASC), + INDEX ix_vm_event_2 (customer_id ASC), + INDEX ix_vm_event_3 (resource_name(255), event_date DESC), + INDEX ix_vm_event_4 (resource_manager_id ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_resource_group ( + resource_group_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + resource_group_name VARCHAR(512) NOT NULL, + resource_group_type VARCHAR(64) NULL DEFAULT NULL, + customer_id INT UNSIGNED NOT NULL, + parent_id INT UNSIGNED NOT NULL, + parent_type VARCHAR(64) NOT NULL, + PRIMARY KEY (resource_group_id), + INDEX ix_resource_group_1 (parent_id ASC), + INDEX ix_resource_group_2 (customer_id ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_resource_in_use_interval ( + resource_id VARCHAR(36) NULL DEFAULT NULL, + resource_name VARCHAR(512) NULL DEFAULT NULL, + resource_foreign_id LONGTEXT NULL DEFAULT NULL, + interval_type VARCHAR(64) NOT NULL, + start_date DATETIME(6) NOT NULL, + end_date DATETIME(6) NOT NULL, + interval_length_milliseconds BIGINT(20) NULL DEFAULT NULL, + resource_manager_id VARCHAR(36) NULL DEFAULT NULL, + resource_manager_name VARCHAR(512) NULL DEFAULT NULL, + resource_manager_foreign_id VARCHAR(512) NULL DEFAULT NULL, + customer_id VARCHAR(36) NULL DEFAULT NULL, + customer_name VARCHAR(512) NULL DEFAULT NULL, + customer_foreign_id VARCHAR(512) NULL DEFAULT NULL, + management_region_id INT UNSIGNED NOT NULL, + PRIMARY KEY (resource_id, interval_type, start_date, end_date), + INDEX ix_resource_in_use_interval_1 (resource_name(255) ASC), + INDEX ix_resource_in_use_interval_2 (customer_name(255) ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_resource_manager ( + resource_manager_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + resource_manager_name VARCHAR(512) NOT NULL, + resource_manager_description VARCHAR(1024) NULL DEFAULT NULL, + resource_manager_vendor_type VARCHAR(64) NULL DEFAULT NULL, + resource_manager_foreign_id VARCHAR(512) NULL DEFAULT NULL, + metadata LONGTEXT NULL DEFAULT NULL, + last_metric_sync_times VARCHAR(5120) NULL DEFAULT NULL, + last_entity_sync_times VARCHAR(5120) NULL DEFAULT NULL, + infrastructure_region_id INT UNSIGNED NOT NULL, + PRIMARY KEY (resource_manager_id)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_resource_manager_sync_times ( + resource_manager_id INT UNSIGNED NOT NULL, + sync_time_key VARCHAR(255) NOT NULL, + sync_time VARCHAR(255) NOT NULL, + PRIMARY KEY (resource_manager_id, sync_time_key)) +ENGINE=InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_usage_metric ( + resource_id INT UNSIGNED NULL DEFAULT 0, + resource_foreign_id VARCHAR(255) NOT NULL, + resource_name VARCHAR(256) NULL DEFAULT NULL, + resource_type VARCHAR(256) NULL DEFAULT NULL, + metric_name VARCHAR(128) NOT NULL, + metric_interval VARCHAR(64) NULL DEFAULT 600, + collection_time DATETIME(6) NOT NULL, + collection_hour varchar(10) NOT NULL, + metric_value DOUBLE NOT NULL, + metric_unit VARCHAR(64) NOT NULL, + customer_id INT UNSIGNED NOT NULL, + customer_name VARCHAR(256) NULL DEFAULT NULL, + customer_foreign_id VARCHAR(255) NULL DEFAULT NULL, + resource_manager_id INT UNSIGNED NOT NULL, + infrastructure_region_id INT UNSIGNED NOT NULL, + management_region_id INT UNSIGNED NOT NULL, + PRIMARY KEY (resource_foreign_id, metric_name, collection_time), + INDEX ix_usage_metrics_2 (customer_id, collection_time, resource_foreign_id, metric_name)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_user_permissions ( + user_id INT UNSIGNED NOT NULL, + permission LONGTEXT NOT NULL, + PRIMARY KEY (user_id, permission(255))) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_user_roles ( + user_id INT UNSIGNED NOT NULL, + role_name VARCHAR(128) NOT NULL, + role_context INT UNSIGNED NOT NULL, + context_type VARCHAR(64) NOT NULL, + UNIQUE INDEX (user_id, role_name, role_context, context_type)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_users ( + user_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + username VARCHAR(128) NOT NULL, + first_name VARCHAR(256) NULL DEFAULT NULL, + last_name VARCHAR(256) NULL DEFAULT NULL, + email_address VARCHAR(512) NULL DEFAULT NULL, + phone_number VARCHAR(64) NULL DEFAULT NULL, + password VARCHAR(512) NOT NULL, + is_active TINYINT(1) NULL DEFAULT NULL, + password_change_token VARCHAR(64) NULL, + last_password_change_date DATETIME(6) NULL, + password_change_token_expire_date DATETIME(6) NULL, + PRIMARY KEY (user_id), + UNIQUE INDEX (username)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_unbilled_charges ( + invoice_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + payer_id INT UNSIGNED NOT NULL, + payer_type VARCHAR(64) NOT NULL, + payer_name VARCHAR(512) NOT NULL, + invoice_number VARCHAR(45) NULL, + invoice_date DATETIME(6) NOT NULL, + period_begin DATETIME(6) NOT NULL, + period_end DATETIME(6) NOT NULL, + invoice_amount DECIMAL(19,2) NOT NULL, + actual_charges DECIMAL(19,2) NOT NULL, + invoice_currency VARCHAR(16) NULL, + management_region_id INT UNSIGNED NOT NULL, + PRIMARY KEY (invoice_id), + INDEX t_unbilled_charges_1 (management_region_id, payer_type, payer_id ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +CREATE TABLE t_unbilled_charges_detail ( + invoice_id INT UNSIGNED NOT NULL, + customer_id INT UNSIGNED NOT NULL, + resource_group_id VARCHAR(36) NULL, + invoice_group VARCHAR(128) NULL, + item_name VARCHAR(512) NULL, + item_vendor_type VARCHAR(128) NULL, + item_type VARCHAR(128) NOT NULL, + item_subtype VARCHAR(128) NOT NULL, + item_description VARCHAR(512) NOT NULL, + rate_plan_id VARCHAR(36) NULL, + rate_plan_billable_feature_id VARCHAR(36) NULL DEFAULT NULL, + billing_interval VARCHAR(64) NULL DEFAULT NULL, + item_invoice_amount DECIMAL(19,2) NOT NULL, + num_units VARCHAR(32) NOT NULL, + num_units_unit VARCHAR(16) NULL, + INDEX ix_unbilled_charges_detail_1 (invoice_id ASC), + INDEX ix_unbilled_charges_detail_2 (customer_id ASC), + INDEX ix_unbilled_charges_detail_3 (item_vendor_type, item_type, item_subtype ASC), + INDEX ix_unbilled_charges_detail_4 (rate_plan_id ASC)) +ENGINE = InnoDB +DEFAULT CHARACTER SET = utf8; + +-- ----------------------------------------------------- +-- Views +-- ----------------------------------------------------- + +CREATE VIEW v_invoice_detail AS SELECT + i.*, + c.customer_name + FROM + (t_invoice_detail i + LEFT JOIN t_customer c ON ((i.customer_id = c.customer_id))); + +CREATE VIEW v_customer AS SELECT + c.*, + r1.region_name as management_region_name, + r2.region_name as infrastructure_region_name + FROM + (t_customer c + LEFT JOIN t_region r1 ON (c.management_region_id = r1.region_id) + LEFT JOIN t_region r2 ON (c.infrastructure_region_id = r2.region_id)); + +CREATE VIEW v_rate_plan_assignment AS SELECT + a.*, + r.management_region_id + FROM + (t_rate_plan_assignment a + LEFT JOIN t_rate_plan r ON (a.rate_plan_id = r.rate_plan_id)); + +CREATE VIEW v_users AS SELECT + u.*, + r.role_name, + r.role_context + FROM + (t_users u + LEFT JOIN t_user_roles r ON (u.user_id = r.user_id)); + +CREATE VIEW v_provider AS SELECT + t_provider.*, + t_region.parent_region_id as parent_management_region_id + FROM + (t_provider + LEFT JOIN t_region ON (t_region.region_id = t_provider.management_region_id)); + +CREATE VIEW v_provider_region AS SELECT + t_region.*, + t_provider_region.provider_id + FROM + (t_region + LEFT JOIN t_provider_region ON (t_region.region_id = t_provider_region.region_id)); + +CREATE VIEW v_unbilled_charges_detail AS SELECT + i.*, + c.customer_name + FROM + (t_unbilled_charges_detail i + LEFT JOIN t_customer c ON ((i.customer_id = c.customer_id))); + +commit; + +SET SQL_MODE=@OLD_SQL_MODE; +SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; +SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS; + +-- ----------------------------------------------------- +-- Recreate workflow schema and tables. +-- ----------------------------------------------------- + +DROP SCHEMA IF EXISTS openbook_workflow; +CREATE SCHEMA openbook_workflow DEFAULT CHARACTER SET utf8; + +GRANT ALL PRIVILEGES ON openbook_workflow.* TO openbook@'%' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_workflow.* TO openbook@localhost IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +FLUSH PRIVILEGES; + +USE openbook_workflow; + +create table ACT_GE_PROPERTY ( + NAME_ varchar(64), + VALUE_ varchar(300), + REV_ integer, + primary key (NAME_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +insert into ACT_GE_PROPERTY +values ('schema.version', '5.20.0.1', 1); + +insert into ACT_GE_PROPERTY +values ('schema.history', 'create(5.20.0.1)', 1); + +insert into ACT_GE_PROPERTY +values ('next.dbid', '1', 1); + +create table ACT_GE_BYTEARRAY ( + ID_ varchar(64), + REV_ integer, + NAME_ varchar(255), + DEPLOYMENT_ID_ varchar(64), + BYTES_ LONGBLOB, + GENERATED_ TINYINT, + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_RE_DEPLOYMENT ( + ID_ varchar(64), + NAME_ varchar(255), + CATEGORY_ varchar(255), + TENANT_ID_ varchar(255) default '', + DEPLOY_TIME_ timestamp(3) NULL, + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_RE_MODEL ( + ID_ varchar(64) not null, + REV_ integer, + NAME_ varchar(255), + KEY_ varchar(255), + CATEGORY_ varchar(255), + CREATE_TIME_ timestamp(3) null, + LAST_UPDATE_TIME_ timestamp(3) null, + VERSION_ integer, + META_INFO_ varchar(4000), + DEPLOYMENT_ID_ varchar(64), + EDITOR_SOURCE_VALUE_ID_ varchar(64), + EDITOR_SOURCE_EXTRA_VALUE_ID_ varchar(64), + TENANT_ID_ varchar(255) default '', + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_RU_EXECUTION ( + ID_ varchar(64), + REV_ integer, + PROC_INST_ID_ varchar(64), + BUSINESS_KEY_ varchar(255), + PARENT_ID_ varchar(64), + PROC_DEF_ID_ varchar(64), + SUPER_EXEC_ varchar(64), + ACT_ID_ varchar(255), + IS_ACTIVE_ TINYINT, + IS_CONCURRENT_ TINYINT, + IS_SCOPE_ TINYINT, + IS_EVENT_SCOPE_ TINYINT, + SUSPENSION_STATE_ integer, + CACHED_ENT_STATE_ integer, + TENANT_ID_ varchar(255) default '', + NAME_ varchar(255), + LOCK_TIME_ timestamp(3) NULL, + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_RU_JOB ( + ID_ varchar(64) NOT NULL, + REV_ integer, + TYPE_ varchar(255) NOT NULL, + LOCK_EXP_TIME_ timestamp(3) NULL, + LOCK_OWNER_ varchar(255), + EXCLUSIVE_ boolean, + EXECUTION_ID_ varchar(64), + PROCESS_INSTANCE_ID_ varchar(64), + PROC_DEF_ID_ varchar(64), + RETRIES_ integer, + EXCEPTION_STACK_ID_ varchar(64), + EXCEPTION_MSG_ varchar(4000), + DUEDATE_ timestamp(3) NULL, + REPEAT_ varchar(255), + HANDLER_TYPE_ varchar(255), + HANDLER_CFG_ varchar(4000), + TENANT_ID_ varchar(255) default '', + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_RE_PROCDEF ( + ID_ varchar(64) not null, + REV_ integer, + CATEGORY_ varchar(255), + NAME_ varchar(255), + KEY_ varchar(255) not null, + VERSION_ integer not null, + DEPLOYMENT_ID_ varchar(64), + RESOURCE_NAME_ varchar(4000), + DGRM_RESOURCE_NAME_ varchar(4000), + DESCRIPTION_ varchar(4000), + HAS_START_FORM_KEY_ TINYINT, + HAS_GRAPHICAL_NOTATION_ TINYINT, + SUSPENSION_STATE_ integer, + TENANT_ID_ varchar(255) default '', + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_RU_TASK ( + ID_ varchar(64), + REV_ integer, + EXECUTION_ID_ varchar(64), + PROC_INST_ID_ varchar(64), + PROC_DEF_ID_ varchar(64), + NAME_ varchar(255), + PARENT_TASK_ID_ varchar(64), + DESCRIPTION_ varchar(4000), + TASK_DEF_KEY_ varchar(255), + OWNER_ varchar(255), + ASSIGNEE_ varchar(255), + DELEGATION_ varchar(64), + PRIORITY_ integer, + CREATE_TIME_ timestamp(3) NULL, + DUE_DATE_ datetime(3), + CATEGORY_ varchar(255), + SUSPENSION_STATE_ integer, + TENANT_ID_ varchar(255) default '', + FORM_KEY_ varchar(255), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_RU_IDENTITYLINK ( + ID_ varchar(64), + REV_ integer, + GROUP_ID_ varchar(255), + TYPE_ varchar(255), + USER_ID_ varchar(255), + TASK_ID_ varchar(64), + PROC_INST_ID_ varchar(64), + PROC_DEF_ID_ varchar(64), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_RU_VARIABLE ( + ID_ varchar(64) not null, + REV_ integer, + TYPE_ varchar(255) not null, + NAME_ varchar(255) not null, + EXECUTION_ID_ varchar(64), + PROC_INST_ID_ varchar(64), + TASK_ID_ varchar(64), + BYTEARRAY_ID_ varchar(64), + DOUBLE_ double, + LONG_ bigint, + TEXT_ varchar(4000), + TEXT2_ varchar(4000), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_RU_EVENT_SUBSCR ( + ID_ varchar(64) not null, + REV_ integer, + EVENT_TYPE_ varchar(255) not null, + EVENT_NAME_ varchar(255), + EXECUTION_ID_ varchar(64), + PROC_INST_ID_ varchar(64), + ACTIVITY_ID_ varchar(64), + CONFIGURATION_ varchar(255), + CREATED_ timestamp(3) not null DEFAULT CURRENT_TIMESTAMP(3), + PROC_DEF_ID_ varchar(64), + TENANT_ID_ varchar(255) default '', + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_EVT_LOG ( + LOG_NR_ bigint auto_increment, + TYPE_ varchar(64), + PROC_DEF_ID_ varchar(64), + PROC_INST_ID_ varchar(64), + EXECUTION_ID_ varchar(64), + TASK_ID_ varchar(64), + TIME_STAMP_ timestamp(3) not null, + USER_ID_ varchar(255), + DATA_ LONGBLOB, + LOCK_OWNER_ varchar(255), + LOCK_TIME_ timestamp(3) null, + IS_PROCESSED_ tinyint default 0, + primary key (LOG_NR_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_PROCDEF_INFO ( + ID_ varchar(64) not null, + PROC_DEF_ID_ varchar(64) not null, + REV_ integer, + INFO_JSON_ID_ varchar(64), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create index ACT_IDX_EXEC_BUSKEY on ACT_RU_EXECUTION(BUSINESS_KEY_); +create index ACT_IDX_TASK_CREATE on ACT_RU_TASK(CREATE_TIME_); +create index ACT_IDX_IDENT_LNK_USER on ACT_RU_IDENTITYLINK(USER_ID_); +create index ACT_IDX_IDENT_LNK_GROUP on ACT_RU_IDENTITYLINK(GROUP_ID_); +create index ACT_IDX_EVENT_SUBSCR_CONFIG_ on ACT_RU_EVENT_SUBSCR(CONFIGURATION_); +create index ACT_IDX_VARIABLE_TASK_ID on ACT_RU_VARIABLE(TASK_ID_); +create index ACT_IDX_ATHRZ_PROCEDEF on ACT_RU_IDENTITYLINK(PROC_DEF_ID_); +create index ACT_IDX_INFO_PROCDEF on ACT_PROCDEF_INFO(PROC_DEF_ID_); + +alter table ACT_GE_BYTEARRAY + add constraint ACT_FK_BYTEARR_DEPL +foreign key (DEPLOYMENT_ID_) +references ACT_RE_DEPLOYMENT (ID_); + +alter table ACT_RE_PROCDEF + add constraint ACT_UNIQ_PROCDEF +unique (KEY_,VERSION_, TENANT_ID_); + +alter table ACT_RU_EXECUTION + add constraint ACT_FK_EXE_PROCINST +foreign key (PROC_INST_ID_) +references ACT_RU_EXECUTION (ID_) on delete cascade on update cascade; + +alter table ACT_RU_EXECUTION + add constraint ACT_FK_EXE_PARENT +foreign key (PARENT_ID_) +references ACT_RU_EXECUTION (ID_); + +alter table ACT_RU_EXECUTION + add constraint ACT_FK_EXE_SUPER +foreign key (SUPER_EXEC_) +references ACT_RU_EXECUTION (ID_); + +alter table ACT_RU_EXECUTION + add constraint ACT_FK_EXE_PROCDEF +foreign key (PROC_DEF_ID_) +references ACT_RE_PROCDEF (ID_); + +alter table ACT_RU_IDENTITYLINK + add constraint ACT_FK_TSKASS_TASK +foreign key (TASK_ID_) +references ACT_RU_TASK (ID_); + +alter table ACT_RU_IDENTITYLINK + add constraint ACT_FK_ATHRZ_PROCEDEF +foreign key (PROC_DEF_ID_) +references ACT_RE_PROCDEF(ID_); + +alter table ACT_RU_IDENTITYLINK + add constraint ACT_FK_IDL_PROCINST +foreign key (PROC_INST_ID_) +references ACT_RU_EXECUTION (ID_); + +alter table ACT_RU_TASK + add constraint ACT_FK_TASK_EXE +foreign key (EXECUTION_ID_) +references ACT_RU_EXECUTION (ID_); + +alter table ACT_RU_TASK + add constraint ACT_FK_TASK_PROCINST +foreign key (PROC_INST_ID_) +references ACT_RU_EXECUTION (ID_); + +alter table ACT_RU_TASK + add constraint ACT_FK_TASK_PROCDEF +foreign key (PROC_DEF_ID_) +references ACT_RE_PROCDEF (ID_); + +alter table ACT_RU_VARIABLE + add constraint ACT_FK_VAR_EXE +foreign key (EXECUTION_ID_) +references ACT_RU_EXECUTION (ID_); + +alter table ACT_RU_VARIABLE + add constraint ACT_FK_VAR_PROCINST +foreign key (PROC_INST_ID_) +references ACT_RU_EXECUTION(ID_); + +alter table ACT_RU_VARIABLE + add constraint ACT_FK_VAR_BYTEARRAY +foreign key (BYTEARRAY_ID_) +references ACT_GE_BYTEARRAY (ID_); + +alter table ACT_RU_JOB + add constraint ACT_FK_JOB_EXCEPTION +foreign key (EXCEPTION_STACK_ID_) +references ACT_GE_BYTEARRAY (ID_); + +alter table ACT_RU_EVENT_SUBSCR + add constraint ACT_FK_EVENT_EXEC +foreign key (EXECUTION_ID_) +references ACT_RU_EXECUTION(ID_); + +alter table ACT_RE_MODEL + add constraint ACT_FK_MODEL_SOURCE +foreign key (EDITOR_SOURCE_VALUE_ID_) +references ACT_GE_BYTEARRAY (ID_); + +alter table ACT_RE_MODEL + add constraint ACT_FK_MODEL_SOURCE_EXTRA +foreign key (EDITOR_SOURCE_EXTRA_VALUE_ID_) +references ACT_GE_BYTEARRAY (ID_); + +alter table ACT_RE_MODEL + add constraint ACT_FK_MODEL_DEPLOYMENT +foreign key (DEPLOYMENT_ID_) +references ACT_RE_DEPLOYMENT (ID_); + +alter table ACT_PROCDEF_INFO + add constraint ACT_FK_INFO_JSON_BA +foreign key (INFO_JSON_ID_) +references ACT_GE_BYTEARRAY (ID_); + +alter table ACT_PROCDEF_INFO + add constraint ACT_FK_INFO_PROCDEF +foreign key (PROC_DEF_ID_) +references ACT_RE_PROCDEF (ID_); + +alter table ACT_PROCDEF_INFO + add constraint ACT_UNIQ_INFO_PROCDEF +unique (PROC_DEF_ID_); + +create table ACT_HI_PROCINST ( + ID_ varchar(64) not null, + PROC_INST_ID_ varchar(64) not null, + BUSINESS_KEY_ varchar(255), + PROC_DEF_ID_ varchar(64) not null, + START_TIME_ datetime(3) not null, + END_TIME_ datetime(3), + DURATION_ bigint, + START_USER_ID_ varchar(255), + START_ACT_ID_ varchar(255), + END_ACT_ID_ varchar(255), + SUPER_PROCESS_INSTANCE_ID_ varchar(64), + DELETE_REASON_ varchar(4000), + TENANT_ID_ varchar(255) default '', + NAME_ varchar(255), + primary key (ID_), + unique (PROC_INST_ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_HI_ACTINST ( + ID_ varchar(64) not null, + PROC_DEF_ID_ varchar(64) not null, + PROC_INST_ID_ varchar(64) not null, + EXECUTION_ID_ varchar(64) not null, + ACT_ID_ varchar(255) not null, + TASK_ID_ varchar(64), + CALL_PROC_INST_ID_ varchar(64), + ACT_NAME_ varchar(255), + ACT_TYPE_ varchar(255) not null, + ASSIGNEE_ varchar(255), + START_TIME_ datetime(3) not null, + END_TIME_ datetime(3), + DURATION_ bigint, + TENANT_ID_ varchar(255) default '', + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_HI_TASKINST ( + ID_ varchar(64) not null, + PROC_DEF_ID_ varchar(64), + TASK_DEF_KEY_ varchar(255), + PROC_INST_ID_ varchar(64), + EXECUTION_ID_ varchar(64), + NAME_ varchar(255), + PARENT_TASK_ID_ varchar(64), + DESCRIPTION_ varchar(4000), + OWNER_ varchar(255), + ASSIGNEE_ varchar(255), + START_TIME_ datetime(3) not null, + CLAIM_TIME_ datetime(3), + END_TIME_ datetime(3), + DURATION_ bigint, + DELETE_REASON_ varchar(4000), + PRIORITY_ integer, + DUE_DATE_ datetime(3), + FORM_KEY_ varchar(255), + CATEGORY_ varchar(255), + TENANT_ID_ varchar(255) default '', + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_HI_VARINST ( + ID_ varchar(64) not null, + PROC_INST_ID_ varchar(64), + EXECUTION_ID_ varchar(64), + TASK_ID_ varchar(64), + NAME_ varchar(255) not null, + VAR_TYPE_ varchar(100), + REV_ integer, + BYTEARRAY_ID_ varchar(64), + DOUBLE_ double, + LONG_ bigint, + TEXT_ varchar(4000), + TEXT2_ varchar(4000), + CREATE_TIME_ datetime(3), + LAST_UPDATED_TIME_ datetime(3), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_HI_DETAIL ( + ID_ varchar(64) not null, + TYPE_ varchar(255) not null, + PROC_INST_ID_ varchar(64), + EXECUTION_ID_ varchar(64), + TASK_ID_ varchar(64), + ACT_INST_ID_ varchar(64), + NAME_ varchar(255) not null, + VAR_TYPE_ varchar(255), + REV_ integer, + TIME_ datetime(3) not null, + BYTEARRAY_ID_ varchar(64), + DOUBLE_ double, + LONG_ bigint, + TEXT_ varchar(4000), + TEXT2_ varchar(4000), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_HI_COMMENT ( + ID_ varchar(64) not null, + TYPE_ varchar(255), + TIME_ datetime(3) not null, + USER_ID_ varchar(255), + TASK_ID_ varchar(64), + PROC_INST_ID_ varchar(64), + ACTION_ varchar(255), + MESSAGE_ varchar(4000), + FULL_MSG_ LONGBLOB, + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_HI_ATTACHMENT ( + ID_ varchar(64) not null, + REV_ integer, + USER_ID_ varchar(255), + NAME_ varchar(255), + DESCRIPTION_ varchar(4000), + TYPE_ varchar(255), + TASK_ID_ varchar(64), + PROC_INST_ID_ varchar(64), + URL_ varchar(4000), + CONTENT_ID_ varchar(64), + TIME_ datetime(3), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_HI_IDENTITYLINK ( + ID_ varchar(64), + GROUP_ID_ varchar(255), + TYPE_ varchar(255), + USER_ID_ varchar(255), + TASK_ID_ varchar(64), + PROC_INST_ID_ varchar(64), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + + +create index ACT_IDX_HI_PRO_INST_END on ACT_HI_PROCINST(END_TIME_); +create index ACT_IDX_HI_PRO_I_BUSKEY on ACT_HI_PROCINST(BUSINESS_KEY_); +create index ACT_IDX_HI_ACT_INST_START on ACT_HI_ACTINST(START_TIME_); +create index ACT_IDX_HI_ACT_INST_END on ACT_HI_ACTINST(END_TIME_); +create index ACT_IDX_HI_DETAIL_PROC_INST on ACT_HI_DETAIL(PROC_INST_ID_); +create index ACT_IDX_HI_DETAIL_ACT_INST on ACT_HI_DETAIL(ACT_INST_ID_); +create index ACT_IDX_HI_DETAIL_TIME on ACT_HI_DETAIL(TIME_); +create index ACT_IDX_HI_DETAIL_NAME on ACT_HI_DETAIL(NAME_); +create index ACT_IDX_HI_DETAIL_TASK_ID on ACT_HI_DETAIL(TASK_ID_); +create index ACT_IDX_HI_PROCVAR_PROC_INST on ACT_HI_VARINST(PROC_INST_ID_); +create index ACT_IDX_HI_PROCVAR_NAME_TYPE on ACT_HI_VARINST(NAME_, VAR_TYPE_); +create index ACT_IDX_HI_PROCVAR_TASK_ID on ACT_HI_VARINST(TASK_ID_); +create index ACT_IDX_HI_ACT_INST_PROCINST on ACT_HI_ACTINST(PROC_INST_ID_, ACT_ID_); +create index ACT_IDX_HI_ACT_INST_EXEC on ACT_HI_ACTINST(EXECUTION_ID_, ACT_ID_); +create index ACT_IDX_HI_IDENT_LNK_USER on ACT_HI_IDENTITYLINK(USER_ID_); +create index ACT_IDX_HI_IDENT_LNK_TASK on ACT_HI_IDENTITYLINK(TASK_ID_); +create index ACT_IDX_HI_IDENT_LNK_PROCINST on ACT_HI_IDENTITYLINK(PROC_INST_ID_); +create index ACT_IDX_HI_TASK_INST_PROCINST on ACT_HI_TASKINST(PROC_INST_ID_); + +create table ACT_ID_GROUP ( + ID_ varchar(64), + REV_ integer, + NAME_ varchar(255), + TYPE_ varchar(255), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_ID_MEMBERSHIP ( + USER_ID_ varchar(64), + GROUP_ID_ varchar(64), + primary key (USER_ID_, GROUP_ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_ID_USER ( + ID_ varchar(64), + REV_ integer, + FIRST_ varchar(255), + LAST_ varchar(255), + EMAIL_ varchar(255), + PWD_ varchar(255), + PICTURE_ID_ varchar(64), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +create table ACT_ID_INFO ( + ID_ varchar(64), + REV_ integer, + USER_ID_ varchar(64), + TYPE_ varchar(64), + KEY_ varchar(255), + VALUE_ varchar(255), + PASSWORD_ LONGBLOB, + PARENT_ID_ varchar(255), + primary key (ID_) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_bin; + +alter table ACT_ID_MEMBERSHIP + add constraint ACT_FK_MEMB_GROUP +foreign key (GROUP_ID_) +references ACT_ID_GROUP (ID_); + +alter table ACT_ID_MEMBERSHIP + add constraint ACT_FK_MEMB_USER +foreign key (USER_ID_) +references ACT_ID_USER (ID_); + +COMMIT; + +-- ----------------------------------------------------- +-- Create Quartz tables +-- ----------------------------------------------------- + +DROP SCHEMA IF EXISTS openbook_jobs; +CREATE SCHEMA openbook_jobs DEFAULT CHARACTER SET utf8; + +GRANT ALL PRIVILEGES ON openbook_jobs.* TO openbook@'%' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_jobs.* TO openbook@localhost IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +FLUSH PRIVILEGES; + +USE openbook_jobs; + +CREATE TABLE QRTZ_JOB_DETAILS( + SCHED_NAME VARCHAR(120) NOT NULL, + JOB_NAME VARCHAR(200) NOT NULL, + JOB_GROUP VARCHAR(200) NOT NULL, + DESCRIPTION VARCHAR(250) NULL, + JOB_CLASS_NAME VARCHAR(250) NOT NULL, + IS_DURABLE VARCHAR(1) NOT NULL, + IS_NONCONCURRENT VARCHAR(1) NOT NULL, + IS_UPDATE_DATA VARCHAR(1) NOT NULL, + REQUESTS_RECOVERY VARCHAR(1) NOT NULL, + JOB_DATA BLOB NULL, +PRIMARY KEY (SCHED_NAME,JOB_NAME,JOB_GROUP)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_TRIGGERS ( + SCHED_NAME VARCHAR(120) NOT NULL, + TRIGGER_NAME VARCHAR(200) NOT NULL, + TRIGGER_GROUP VARCHAR(200) NOT NULL, + JOB_NAME VARCHAR(200) NOT NULL, + JOB_GROUP VARCHAR(200) NOT NULL, + DESCRIPTION VARCHAR(250) NULL, + NEXT_FIRE_TIME BIGINT(13) NULL, + PREV_FIRE_TIME BIGINT(13) NULL, + PRIORITY INTEGER NULL, + TRIGGER_STATE VARCHAR(16) NOT NULL, + TRIGGER_TYPE VARCHAR(8) NOT NULL, + START_TIME BIGINT(13) NOT NULL, + END_TIME BIGINT(13) NULL, + CALENDAR_NAME VARCHAR(200) NULL, + MISFIRE_INSTR SMALLINT(2) NULL, + JOB_DATA BLOB NULL, +PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), +FOREIGN KEY (SCHED_NAME,JOB_NAME,JOB_GROUP) +REFERENCES QRTZ_JOB_DETAILS(SCHED_NAME,JOB_NAME,JOB_GROUP)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_SIMPLE_TRIGGERS ( + SCHED_NAME VARCHAR(120) NOT NULL, + TRIGGER_NAME VARCHAR(200) NOT NULL, + TRIGGER_GROUP VARCHAR(200) NOT NULL, + REPEAT_COUNT BIGINT(7) NOT NULL, + REPEAT_INTERVAL BIGINT(12) NOT NULL, + TIMES_TRIGGERED BIGINT(10) NOT NULL, +PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), +FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) +REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_CRON_TRIGGERS ( + SCHED_NAME VARCHAR(120) NOT NULL, + TRIGGER_NAME VARCHAR(200) NOT NULL, + TRIGGER_GROUP VARCHAR(200) NOT NULL, + CRON_EXPRESSION VARCHAR(120) NOT NULL, + TIME_ZONE_ID VARCHAR(80), +PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), +FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) +REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_SIMPROP_TRIGGERS ( + SCHED_NAME VARCHAR(120) NOT NULL, + TRIGGER_NAME VARCHAR(200) NOT NULL, + TRIGGER_GROUP VARCHAR(200) NOT NULL, + STR_PROP_1 VARCHAR(512) NULL, + STR_PROP_2 VARCHAR(512) NULL, + STR_PROP_3 VARCHAR(512) NULL, + INT_PROP_1 INT NULL, + INT_PROP_2 INT NULL, + LONG_PROP_1 BIGINT NULL, + LONG_PROP_2 BIGINT NULL, + DEC_PROP_1 NUMERIC(13,4) NULL, + DEC_PROP_2 NUMERIC(13,4) NULL, + BOOL_PROP_1 VARCHAR(1) NULL, + BOOL_PROP_2 VARCHAR(1) NULL, +PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), +FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) +REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_BLOB_TRIGGERS ( + SCHED_NAME VARCHAR(120) NOT NULL, + TRIGGER_NAME VARCHAR(200) NOT NULL, + TRIGGER_GROUP VARCHAR(200) NOT NULL, + BLOB_DATA BLOB NULL, +PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), +INDEX (SCHED_NAME,TRIGGER_NAME, TRIGGER_GROUP), +FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) +REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_CALENDARS ( + SCHED_NAME VARCHAR(120) NOT NULL, + CALENDAR_NAME VARCHAR(200) NOT NULL, + CALENDAR BLOB NOT NULL, +PRIMARY KEY (SCHED_NAME,CALENDAR_NAME)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_PAUSED_TRIGGER_GRPS ( + SCHED_NAME VARCHAR(120) NOT NULL, + TRIGGER_GROUP VARCHAR(200) NOT NULL, +PRIMARY KEY (SCHED_NAME,TRIGGER_GROUP)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_FIRED_TRIGGERS ( + SCHED_NAME VARCHAR(120) NOT NULL, + ENTRY_ID VARCHAR(95) NOT NULL, + TRIGGER_NAME VARCHAR(200) NOT NULL, + TRIGGER_GROUP VARCHAR(200) NOT NULL, + INSTANCE_NAME VARCHAR(200) NOT NULL, + FIRED_TIME BIGINT(13) NOT NULL, + SCHED_TIME BIGINT(13) NOT NULL, + PRIORITY INTEGER NOT NULL, + STATE VARCHAR(16) NOT NULL, + JOB_NAME VARCHAR(200) NULL, + JOB_GROUP VARCHAR(200) NULL, + IS_NONCONCURRENT VARCHAR(1) NULL, + REQUESTS_RECOVERY VARCHAR(1) NULL, +PRIMARY KEY (SCHED_NAME,ENTRY_ID)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_SCHEDULER_STATE ( + SCHED_NAME VARCHAR(120) NOT NULL, + INSTANCE_NAME VARCHAR(200) NOT NULL, + LAST_CHECKIN_TIME BIGINT(13) NOT NULL, + CHECKIN_INTERVAL BIGINT(13) NOT NULL, +PRIMARY KEY (SCHED_NAME,INSTANCE_NAME)) +ENGINE=InnoDB; + +CREATE TABLE QRTZ_LOCKS ( + SCHED_NAME VARCHAR(120) NOT NULL, + LOCK_NAME VARCHAR(40) NOT NULL, +PRIMARY KEY (SCHED_NAME,LOCK_NAME)) +ENGINE=InnoDB; + +CREATE INDEX IDX_QRTZ_J_REQ_RECOVERY ON QRTZ_JOB_DETAILS(SCHED_NAME,REQUESTS_RECOVERY); +CREATE INDEX IDX_QRTZ_J_GRP ON QRTZ_JOB_DETAILS(SCHED_NAME,JOB_GROUP); + +CREATE INDEX IDX_QRTZ_T_J ON QRTZ_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP); +CREATE INDEX IDX_QRTZ_T_JG ON QRTZ_TRIGGERS(SCHED_NAME,JOB_GROUP); +CREATE INDEX IDX_QRTZ_T_C ON QRTZ_TRIGGERS(SCHED_NAME,CALENDAR_NAME); +CREATE INDEX IDX_QRTZ_T_G ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP); +CREATE INDEX IDX_QRTZ_T_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE); +CREATE INDEX IDX_QRTZ_T_N_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP,TRIGGER_STATE); +CREATE INDEX IDX_QRTZ_T_N_G_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP,TRIGGER_STATE); +CREATE INDEX IDX_QRTZ_T_NEXT_FIRE_TIME ON QRTZ_TRIGGERS(SCHED_NAME,NEXT_FIRE_TIME); +CREATE INDEX IDX_QRTZ_T_NFT_ST ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE,NEXT_FIRE_TIME); +CREATE INDEX IDX_QRTZ_T_NFT_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME); +CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_STATE); +CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE_GRP ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_GROUP,TRIGGER_STATE); + +CREATE INDEX IDX_QRTZ_FT_TRIG_INST_NAME ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME); +CREATE INDEX IDX_QRTZ_FT_INST_JOB_REQ_RCVRY ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME,REQUESTS_RECOVERY); +CREATE INDEX IDX_QRTZ_FT_J_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP); +CREATE INDEX IDX_QRTZ_FT_JG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_GROUP); +CREATE INDEX IDX_QRTZ_FT_T_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP); +CREATE INDEX IDX_QRTZ_FT_TG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_GROUP); + +COMMIT; diff --git a/salt/openbook/database/files/debconfrc b/salt/openbook/database/files/debconfrc new file mode 100644 index 0000000..689cd64 --- /dev/null +++ b/salt/openbook/database/files/debconfrc @@ -0,0 +1,2 @@ +mariadb-server-10.1 mysql-server/root_password password {% raw %}{{ openbook_database_password }}{% endraw %} +mariadb-server-10.1 mysql-server/root_password_again password {% raw %}{{ openbook_database_password }}{% endraw %} diff --git a/salt/openbook/database/files/debian.cnf b/salt/openbook/database/files/debian.cnf new file mode 100644 index 0000000..1da5473 --- /dev/null +++ b/salt/openbook/database/files/debian.cnf @@ -0,0 +1,11 @@ +[client] +host = localhost +user = debian-sys-maint +password = {% raw %}{{ openbook_database_password }}{% endraw %} +socket = /var/run/mysqld/mysqld.sock +[mysql_upgrade] +host = localhost +user = debian-sys-maint +password = {% raw %}{{ openbook_database_password }}{% endraw %} +socket = /var/run/mysqld/mysqld.sock +basedir = /usr diff --git a/salt/openbook/database/files/fix_permissions.sh b/salt/openbook/database/files/fix_permissions.sh new file mode 100644 index 0000000..0bf9c64 --- /dev/null +++ b/salt/openbook/database/files/fix_permissions.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# wait for mysql to start +for i in {120..0}; do + if echo 'SELECT 1' | mysql &>/dev/null; then + break + fi + sleep 1 +done + +# privilege fixes - must be idempotent! +mysql <<EOF +DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('%', 'localhost', '127.0.0.1', '127.0.2.1', '::', '::1'); +DELETE FROM mysql.user WHERE User=''; +DELETE FROM mysql.db WHERE Db='test' OR Db='test\_%'; +FLUSH PRIVILEGES; +GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}' WITH GRANT OPTION; +GRANT ALL PRIVILEGES ON *.* TO 'root'@'127.0.0.1' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}' WITH GRANT OPTION; +GRANT ALL PRIVILEGES ON *.* TO 'root'@'::1' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}' WITH GRANT OPTION; +GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}' WITH GRANT OPTION; +GRANT ALL PRIVILEGES ON *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook.* TO 'openbook'@'localhost' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook.* TO 'openbook'@'127.0.0.1' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook.* TO 'openbook'@'::1' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook.* TO 'openbook'@'%' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_jobs.* TO 'openbook'@'localhost' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_jobs.* TO 'openbook'@'127.0.0.1' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_jobs.* TO 'openbook'@'::1' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_jobs.* TO 'openbook'@'%' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_workflow.* TO 'openbook'@'localhost' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_workflow.* TO 'openbook'@'127.0.0.1' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_workflow.* TO 'openbook'@'::1' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +GRANT ALL PRIVILEGES ON openbook_workflow.* TO 'openbook'@'%' IDENTIFIED BY '{% raw %}{{ openbook_database_password }}{% endraw %}'; +FLUSH PRIVILEGES; +EOF + +exit 0 diff --git a/salt/openbook/database/files/installdeps.sh b/salt/openbook/database/files/installdeps.sh new file mode 100644 index 0000000..cd068fa --- /dev/null +++ b/salt/openbook/database/files/installdeps.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +apt-get -y install software-properties-common +apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 +add-apt-repository 'deb [arch=amd64,i386] http://sgp1.mirrors.digitalocean.com/mariadb/repo/10.1/ubuntu xenial main' +apt-get update +apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade +apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install netcat mariadb-server-10.1 +service mysql stop + +exit 0 diff --git a/salt/openbook/database/files/my.cnf b/salt/openbook/database/files/my.cnf new file mode 100644 index 0000000..4752ae1 --- /dev/null +++ b/salt/openbook/database/files/my.cnf @@ -0,0 +1,169 @@ +# MariaDB database server configuration file. +# +# You can copy this file to one of: +# - "/etc/mysql/my.cnf" to set global options, +# - "~/.my.cnf" to set user-specific options. +# +# One can use all long options that the program supports. +# Run program with --help to get a list of available options and with +# --print-defaults to see which it would actually understand and use. +# +# For explanations see +# http://dev.mysql.com/doc/mysql/en/server-system-variables.html + +# This will be passed to all mysql clients +# It has been reported that passwords should be enclosed with ticks/quotes +# escpecially if they contain "#" chars... +# Remember to edit /etc/mysql/debian.cnf when changing the socket location. +[client] +port = 3306 +socket = /var/run/mysqld/mysqld.sock + +# Here is entries for some specific programs +# The following values assume you have at least 32M ram + +# This was formally known as [safe_mysqld]. Both versions are currently parsed. +[mysqld_safe] +socket = /var/run/mysqld/mysqld.sock +nice = 0 + +[mysqld] +# +# * Basic Settings +# +user = mysql +pid-file = /var/run/mysqld/mysqld.pid +socket = /var/run/mysqld/mysqld.sock +port = 3306 +basedir = /usr +datadir = /var/lib/mysql +tmpdir = /tmp +lc_messages_dir = /usr/share/mysql +lc_messages = en_US +skip-external-locking +# +# Instead of skip-networking the default is now to listen only on +# localhost which is more compatible and is not less secure. +bind-address = 0.0.0.0 +# +# * Fine Tuning +# +max_connections = 1000 +connect_timeout = 5 +wait_timeout = 600 +max_allowed_packet = 16M +thread_cache_size = 128 +sort_buffer_size = 4M +bulk_insert_buffer_size = 16M +tmp_table_size = 32M +max_heap_table_size = 32M +# +# * MyISAM +# +# This replaces the startup script and checks MyISAM tables if needed +# the first time they are touched. On error, make copy and try a repair. +myisam_recover_options = BACKUP +key_buffer_size = 128M +#open-files-limit = 2000 +table_open_cache = 400 +myisam_sort_buffer_size = 512M +concurrent_insert = 2 +read_buffer_size = 2M +read_rnd_buffer_size = 1M +# +# * Query Cache Configuration +# +# Cache only tiny result sets, so we can fit more in the query cache. +query_cache_limit = 128K +query_cache_size = 64M +# for more write intensive setups, set to DEMAND or OFF +#query_cache_type = DEMAND +# +# * Logging and Replication +# +# Both location gets rotated by the cronjob. +# Be aware that this log type is a performance killer. +# As of 5.1 you can enable the log at runtime! +#general_log_file = /var/log/mysql/mysql.log +#general_log = 1 +# +# Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. +# +# we do want to know about network errors and such +log_warnings = 2 +# +# Enable the slow query log to see queries with especially long duration +#slow_query_log[={0|1}] +slow_query_log_file = /var/log/mysql/mariadb-slow.log +long_query_time = 10 +#log_slow_rate_limit = 1000 +log_slow_verbosity = query_plan + +#log-queries-not-using-indexes +#log_slow_admin_statements +# +# The following can be used as easy to replay backup logs or for replication. +# note: if you are setting up a replication slave, see README.Debian about +# other settings you may need to change. +#server-id = 1 +#report_host = master1 +#auto_increment_increment = 2 +#auto_increment_offset = 1 +log_bin = /var/log/mysql/mariadb-bin +log_bin_index = /var/log/mysql/mariadb-bin.index +# not fab for performance, but safer +#sync_binlog = 1 +expire_logs_days = 10 +max_binlog_size = 100M +# slaves +#relay_log = /var/log/mysql/relay-bin +#relay_log_index = /var/log/mysql/relay-bin.index +#relay_log_info_file = /var/log/mysql/relay-bin.info +#log_slave_updates +#read_only +# +# If applications support it, this stricter sql_mode prevents some +# mistakes like inserting invalid dates etc. +#sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL +# +# * InnoDB +# +# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. +# Read the manual for more InnoDB related options. There are many! +default_storage_engine = InnoDB +# you can't just change log file size, requires special procedure +#innodb_log_file_size = 50M +innodb_buffer_pool_size = 256M +innodb_log_buffer_size = 8M +innodb_file_per_table = 1 +innodb_open_files = 400 +innodb_io_capacity = 400 +innodb_flush_method = O_DIRECT +# +# * Security Features +# +# Read the manual, too, if you want chroot! +# chroot = /var/lib/mysql/ +# +# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". +# +# ssl-ca=/etc/mysql/cacert.pem +# ssl-cert=/etc/mysql/server-cert.pem +# ssl-key=/etc/mysql/server-key.pem + +[mysqldump] +quick +quote-names +max_allowed_packet = 16M + +[mysql] +#no-auto-rehash # faster start of mysql but no tab completion + +[isamchk] +key_buffer = 16M + +# +# * IMPORTANT: Additional settings that can override those from this file! +# The files must end with '.cnf', otherwise they'll be ignored. +# +!includedir /etc/mysql/conf.d/ diff --git a/salt/openbook/database/files/run.sh b/salt/openbook/database/files/run.sh new file mode 100644 index 0000000..a956c28 --- /dev/null +++ b/salt/openbook/database/files/run.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# find other running nodes +other_nodes_found=1 +for i in $(echo "{% raw %}{{ openbook_gcomm_addresses }}{% endraw %}" | sed -e 's|,| |g'); do + if [ "${i}" != "{% raw %}{{ ansible_default_ipv4.address }}{% endraw %}" ]; then + nc ${i} 4567 </dev/null >/dev/null + if [ "${?}" == "0" ]; then + other_nodes_found=0 + fi + fi +done + +# bootstrap or cluster join depending on whether other nodes were found +if [ "${other_nodes_found}" == "0" ]; then + mysql_args="" +else + mysql_args="--wsrep-new-cluster" +fi + +docker run -d --net=host -v {{ pillar["database"]["dirs"]["etc_openbook_database"]["name"] }}:/etc/mysql -v {{ pillar["database"]["dirs"]["var_lib_mysql"]["name"] }}:/var/lib/mysql -v {{ pillar["database"]["dirs"]["var_log_mysql"]["name"] }}:/var/log/mysql --name {{ pillar["database"]["container_name"] }} {{ pillar["registry"]["url"] }}/{{ pillar["database"]["image_name"] }} ${mysql_args} + +exit 0 diff --git a/salt/openbook/database/files/sources.list b/salt/openbook/database/files/sources.list new file mode 100644 index 0000000..4e624e5 --- /dev/null +++ b/salt/openbook/database/files/sources.list @@ -0,0 +1,54 @@ +# deb http://de.archive.ubuntu.com/ubuntu/ xenial main restricted + +# deb http://de.archive.ubuntu.com/ubuntu/ xenial-updates main restricted +# deb http://security.ubuntu.com/ubuntu xenial-security main restricted + +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://de.archive.ubuntu.com/ubuntu/ xenial main restricted +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial main restricted + +## Major bug fix updates produced after the final release of the +## distribution. +deb http://de.archive.ubuntu.com/ubuntu/ xenial-updates main restricted +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial-updates main restricted + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team. Also, please note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +deb http://de.archive.ubuntu.com/ubuntu/ xenial universe +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial universe +deb http://de.archive.ubuntu.com/ubuntu/ xenial-updates universe +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial-updates universe + +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in +## multiverse WILL NOT receive any review or updates from the Ubuntu +## security team. +deb http://de.archive.ubuntu.com/ubuntu/ xenial multiverse +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial multiverse +deb http://de.archive.ubuntu.com/ubuntu/ xenial-updates multiverse +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial-updates multiverse + +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +deb http://de.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse +# deb-src http://de.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse + +## Uncomment the following two lines to add software from Canonical's +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu xenial partner +# deb-src http://archive.canonical.com/ubuntu xenial partner + +deb http://security.ubuntu.com/ubuntu xenial-security main restricted +# deb-src http://security.ubuntu.com/ubuntu xenial-security main restricted +deb http://security.ubuntu.com/ubuntu xenial-security universe +# deb-src http://security.ubuntu.com/ubuntu xenial-security universe +deb http://security.ubuntu.com/ubuntu xenial-security multiverse +# deb-src http://security.ubuntu.com/ubuntu xenial-security multiverse diff --git a/salt/openbook/database/files/start_galera_node.sh b/salt/openbook/database/files/start_galera_node.sh new file mode 100644 index 0000000..29b7e55 --- /dev/null +++ b/salt/openbook/database/files/start_galera_node.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# the mysql database files will be missing if this is the first startup after a fresh install +if [ ! -d "{{ pillar["database"]["dirs"]["var_lib_mysql"]["name"] }}/mysql" ]; then + export DEBIAN_FRONTEND=noninteractive + . /etc/profile + debconf-set-selections /etc/mysql/conf.d/debconfrc + dpkg-reconfigure mariadb-server-10.1 +fi + +/etc/mysql/bin/fix_permissions.sh & +exec /usr/sbin/mysqld ${@} diff --git a/salt/openbook/database/init.sls b/salt/openbook/database/init.sls new file mode 100644 index 0000000..be87bf6 --- /dev/null +++ b/salt/openbook/database/init.sls @@ -0,0 +1,55 @@ +{% for dir in pillar["database"]["dirs"].items() %} +database_dirs_{{ dir[0] }}: + file.directory: +{% if dir[1]["relative_to_buildroot"] %} + - name: {{ pillar["database"]["build_dir"] }}/{{ dir[1]["name"] }} +{% else %} + - name: {{ dir[1]["name"] }} +{% endif %} + - user: root + - group: root + - mode: {{ dir[1]["mode"] }} + - makedirs: True +{% endfor %} + +{% for file in pillar["database"]["files"].items() %} +database_files_{{ file[0] }}: + file.managed: +{% if file[1]["relative_to_buildroot"] %} + - name: {{ pillar["database"]["build_dir"] }}/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% else %} + - name: {{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/database/files/{{ file[1]["source"] }} +{% if file[1]["is_template"] %} + - template: jinja +{% endif %} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% if file[1]["ansible"] %} +ansible_database_dirs_{{ file[0] }}: + file.directory: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/database/{{ file[1]["path"] }} +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/database/{{ file[1]["path"] }} +{% endif %} + - user: root + - group: root + - mode: "0755" + - makedirs: True +ansible_database_files_{{ file[0] }}: + file.managed: +{% if file[1]["is_template"] %} + - name: {{ pillar["ansible"]["build_dir"] }}/templates/database/{{ file[1]["path"] }}/{{ file[1]["name"] }} + - template: jinja +{% else %} + - name: {{ pillar["ansible"]["build_dir"] }}/files/database/{{ file[1]["path"] }}/{{ file[1]["name"] }} +{% endif %} + - source: salt://openbook/database/files/{{ file[1]["source"] }} + - user: root + - group: root + - mode: {{ file[1]["mode"] }} +{% endif %} +{% endfor %} diff --git a/salt/top.sls b/salt/top.sls new file mode 100644 index 0000000..f471540 --- /dev/null +++ b/salt/top.sls @@ -0,0 +1,9 @@ +base: + "*": + - openbook.ci + - openbook.ansible + - openbook.baseimage + - openbook.database + - openbook.api_server + - openbook.admin_web_ui + - openbook.customer_web_ui |