diff --git a/.kitchen.yml b/.kitchen.yml
index 41f524c..18f797d 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -55,6 +55,30 @@
       repo: git
       source: https://gerrit.mcp.mirantis.com/salt-formulas/reclass
       branch: <%=ENV['GERRIT_BRANCH'] || 'master' %>
+    - name: backupninja
+      repo: git
+      source: https://gerrit.mcp.mirantis.com/salt-formulas/backupninja
+      branch: <%=ENV['GERRIT_BRANCH'] || 'master' %>
+    - name: logrotate
+      repo: git
+      source: https://gerrit.mcp.mirantis.com/salt-formulas/logrotate
+      branch: <%=ENV['GERRIT_BRANCH'] || 'master' %>
+    - name: sphinx
+      repo: git
+      source: https://gerrit.mcp.mirantis.com/salt-formulas/sphinx
+      branch: <%=ENV['GERRIT_BRANCH'] || 'master' %>
+    - name: sensu
+      repo: git
+      source: https://gerrit.mcp.mirantis.com/salt-formulas/sensu
+      branch: <%=ENV['GERRIT_BRANCH'] || 'master' %>
+    - name: prometheus
+      repo: git
+      source: https://gerrit.mcp.mirantis.com/salt-formulas/prometheus
+      branch: <%=ENV['GERRIT_BRANCH'] || 'master' %>
+    - name: telegraph
+      repo: git
+      source: https://gerrit.mcp.mirantis.com/salt-formulas/telegraph
+      branch: <%=ENV['GERRIT_BRANCH'] || 'master' %>
 
 verifier:
   name: inspec
diff --git a/README.rst b/README.rst
index 06e79d7..a14d847 100644
--- a/README.rst
+++ b/README.rst
@@ -978,6 +978,54 @@
 
   salt-call health_checks.keystone_keys_check keys_type='credential'
 
+Retrieve cinder db release codename
+
+.. code-block:: bash
+
+  salt-call health_checks.get_cinder_db_versions db_host='192.168.0.50' db_user='cinder' \
+    db_pass='password'
+
+Retrieve glance db release codename
+
+.. code-block:: bash
+
+  salt-call health_checks.get_glance_db_versions db_host='192.168.0.50' db_user='glance' \
+    db_pass='password'
+
+Retrieve heat db release codename
+
+.. code-block:: bash
+
+  salt-call health_checks.get_heat_db_versions db_host='192.168.0.50' db_user='heat' \
+    db_pass='password'
+
+Retrieve neutron db release codename
+
+.. code-block:: bash
+
+  salt-call health_checks.get_neutron_db_versions db_host='192.168.0.50' db_user='neutron' \
+    db_pass='password'
+
+Retrieve keystone db release codename
+
+.. code-block:: bash
+
+  salt-call health_checks.get_keystone_db_versions db_host='192.168.0.50' db_user='keystone' \
+    db_pass='password'
+
+Retrieve nova db release codename
+
+.. code-block:: bash
+
+  salt-call health_checks.get_nova_db_versions db_host='192.168.0.50' db_user='nova' \
+    db_pass='password' db_api_user='nova' db_api_pass='api_password'
+
+Retrieve openstack db release codenames
+
+.. code-block:: bash
+
+  salt-call health_checks.list_db_versions
+
 
 Encrypted pillars
 ~~~~~~~~~~~~~~~~~
diff --git a/_modules/health_checks.py b/_modules/health_checks.py
index 97d1703..fab223c 100644
--- a/_modules/health_checks.py
+++ b/_modules/health_checks.py
@@ -23,6 +23,215 @@
 except ImportError:
     from yaml import Loader, Dumper
 
+db_ver_map=yaml.load("""
+kilo:
+  cinder: 41
+  glance: 41
+  heat: 62
+  keystone: 67
+  neutron: [ kilo ]
+  nova:
+    api_db: 2
+    db: 280
+liberty:
+  cinder: 60
+  glance: 42
+  heat: 65
+  keystone: 75
+  neutron:
+  - 1b4c6e320f79
+  - 26c371498592
+  - 599c6a226151
+  - 45f955889773
+  - 1c844d1677f7
+  - 52c5312f6baf
+  - 9859ac9c136
+  - 8675309a5c4f
+  - 48153cb5f051
+  - 31337ec0ffee
+  - 34af2b5c5a59
+  - 354db87e3225
+  - 11926bcfe72d
+  - 5498d17be016
+  - 4af11ca47297
+  - 2e5352a0ad4d
+  - 2a16083502f3
+  - 4ffceebfada
+  - 30018084ec99
+  nova:
+    api_db: 3
+    db: 302
+mitaka:
+  cinder: 72
+  glance: 44
+  heat: 71
+  keystone: 97
+  neutron:
+  - 15be73214821
+  - dce3ec7a25c9
+  - 659bf3d90664
+  - 19f26505c74f
+  - 0e66c5227a8a
+  - ec7fcfbf72ee
+  - 32e5974ada25
+  - 3894bccad37f
+  - c3a73f615e4
+  - 13cfb89f881a
+  - 1df244e556f5
+  - 2f9e956e7532
+  - 15e43b934f81
+  - 59cb5b6cf4d
+  - b4caf27aae4
+  - 31ed664953e6
+  - 8a6d8bdae39
+  - c6c112992c9
+  - 2b4c2465d44b
+  - 5ffceebfada
+  - 1b294093239c
+  - 4ffceebfcdc
+  - e3278ee65050
+  nova:
+    api_db: 7
+    db: 319
+newton:
+  cinder: 79
+  glance: 44
+  heat: 73
+  keystone:
+    contract: 1
+    data: 4
+    db: 109
+    expand: 1
+  neutron:
+  - 030a959ceafa
+  - 67daae611b6e
+  - a5648cfeeadf
+  - a963b38d82f4
+  - 6b461a21bcfc
+  - 0f5bef0f87d4
+  - d3435b514502
+  - 5cd92597d11d
+  - 3d0e74aa7d37
+  - 5abc0278ca73
+  - 30107ab6a3ee
+  - 45f8dd33480b
+  - c415aab1c048
+  - 2e0d7a8a1586
+  - 5c85685d616d
+  - a8b517cff8ab
+  - a84ccf28f06a
+  - 7d9d8eeec6ad
+  - 7bbb25278f53
+  - 89ab9a816d70
+  - 8fd3918ef6f4
+  - c879c5e1ee90
+  - b67e765a3524
+  - 3b935b28e7a0
+  - b12a3ef66e62
+  - 4bcd4df1f426
+  - 97c25b0d2353
+  nova:
+    api_db: 22
+    db: 334
+ocata:
+  cinder: 96
+  glance: 45
+  heat: 79
+  keystone:
+    contract: 1
+    data: 16
+    db: 109
+    expand: 1
+  neutron:
+  - a9c43481023c
+  - 929c968efe70
+  nova:
+    api_db: 31
+    db: 347
+pike:
+  cinder: 105
+  glance: 45
+  heat: 80
+  keystone:
+    contract: 1
+    data: 24
+    db: 109
+    expand: 1
+  neutron:
+  - 62c781cb6192
+  - 2b42d90729da
+  - 7d32f979895f
+  - 349b6fd605a6
+  - 804a3c76314c
+  - c8c222d42aa9
+  nova:
+    api_db: 45
+    db: 362
+queens:
+  cinder: 117
+  glance: 45
+  heat: 85
+  keystone:
+    contract: 1
+    data: 44
+    db: 109
+    expand: 1
+  neutron:
+  - 594422d373ee
+  nova:
+    api_db: 52
+    db: 378
+rocky:
+  cinder: 123
+  glance: 45
+  heat: 86
+  keystone:
+    contract: 1
+    data: 52
+    db: 109
+    expand: 1
+  neutron:
+  - 61663558142c
+  - 867d39095bf4
+  nova:
+    api_db: 61
+    db: 390
+stein:
+  cinder: 128
+  glance: 45
+  heat: 86
+  keystone:
+    contract: 1
+    data: 61
+    db: 109
+    expand: 1
+  neutron:
+  - 0ff9e3881597
+  - 195176fb410d
+  - d72db3e25539
+  - fb0167bd9639
+  - 9bfad3f1e780
+  - cada2437bf41
+  nova:
+    api_db: 62
+    db: 391
+train:
+  cinder: 132
+  glance: 45
+  heat: 86
+  keystone:
+    contract: 1
+    data: 71
+    db: 109
+    expand: 1
+  neutron:
+  - 63fd95af7dcd
+  - c613d0b82681
+  nova:
+    api_db: 67
+    db: 402
+""")
+
 default_vrouter_info_map = yaml.load("""
 ContrailConfig:
 - deleted
@@ -1956,3 +2165,291 @@
             namespaces.append(netns)
 
     return namespaces
+
+
+def _load_mysql_module():
+
+    # Check if module is loaded
+    # It can be loaded by parent function
+    # In case of direct funtction call, we load it
+    mod_not_loaded = False
+    try:
+        dir(MySQLdb)
+    except:
+        # Not loaded. Trying to load
+        mod_not_loaded = True
+
+    if mod_not_loaded:
+        try:
+            import MySQLdb
+        except:
+            logger.error("Python library MySQLdb could not be loaded. Install it first")
+            __context__['retcode'] = 2
+            return False
+
+    return MySQLdb
+
+
+def get_keystone_db_versions(db_host, db_user, db_pass, db_name="keystone"):
+
+    ''' Return dict of keystone DB versions '''
+
+    MySQLdb = _load_mysql_module()
+    if not MySQLdb:
+        return MySQLdb
+
+    keystone_versions = {}
+    db = MySQLdb.connect(db_host, db_user, db_pass, db_name)
+    cursor = db.cursor()
+
+    cursor.execute("select `version` from `migrate_version` where `repository_id` like 'keystone'")
+    if cursor.rowcount == 1:
+        keystone_db_ver = cursor.fetchone()[0]
+    else:
+        keystone_db_ver = 0
+
+    if keystone_db_ver > 0:
+        cursor.execute("select `version` from `migrate_version` where `repository_id` like 'keystone_data_migrate'")
+        if cursor.rowcount == 1:
+            keystone_data_ver = cursor.fetchone()[0]
+        else:
+            keystone_data_ver = 0
+    db.close()
+
+    keystone_versions['db'] = keystone_db_ver
+    keystone_versions['data_db'] = keystone_data_ver
+    for release in db_ver_map:
+        keystone_obj = db_ver_map[release]["keystone"]
+        if isinstance(keystone_obj, dict):
+            keystone_db_map = int(keystone_obj["db"])
+            keystone_data_map = int(keystone_obj["data"])
+            if keystone_db_map == keystone_db_ver and keystone_data_map == keystone_data_ver:
+                keystone_versions['os_release'] = release
+        else:
+            keystone_db_map = int(keystone_obj)
+            if keystone_db_map == keystone_db_ver:
+                keystone_versions['os_release'] = release
+
+    return keystone_versions
+
+
+def get_glance_db_versions(db_host, db_user, db_pass, db_name="glance"):
+
+    ''' Return dict of glance DB versions '''
+
+    MySQLdb = _load_mysql_module()
+    if not MySQLdb:
+        return MySQLdb
+
+    glance_versions = {}
+    db = MySQLdb.connect(db_host, db_user, db_pass, db_name)
+    cursor = db.cursor()
+
+    glance_release = "unknown"
+    cursor.execute("select `version` from `migrate_version` where `repository_id` like 'Glance Migrations'")
+    if cursor.rowcount == 1:
+        glance_db_ver = cursor.fetchone()[0]
+        for release in db_ver_map:
+            glance_obj = db_ver_map[release]["glance"]
+            glance_db_map = int(glance_obj)
+            if glance_db_map == glance_db_ver:
+                glance_release = release
+    else:
+        glance_db_ver = 0
+    db.close()
+
+    db = MySQLdb.connect(db_host, db_user, db_pass, db_name)
+    cursor = db.cursor()
+    try:
+        cursor.execute("select `version_num` from `alembic_version`")
+    except:
+        pass
+    if cursor.rowcount == 1:
+        glance_release = cursor.fetchone()[0]
+        for release in db_ver_map:
+            if release in glance_release:
+                glance_db_ver = db_ver_map[release]["glance"]
+                glance_release = "%s (alembic)" % release
+    db.close()
+
+    glance_versions['db'] = glance_db_ver
+    glance_versions['os_release'] = glance_release
+
+    return glance_versions
+
+
+def get_cinder_db_versions(db_host, db_user, db_pass, db_name="cinder"):
+
+    ''' Return dict of cinder DB versions '''
+
+    MySQLdb = _load_mysql_module()
+    if not MySQLdb:
+        return MySQLdb
+
+    cinder_versions = {}
+    db = MySQLdb.connect(db_host, db_user, db_pass, db_name)
+    cursor = db.cursor()
+
+    cursor.execute("select `version` from `migrate_version` where `repository_id` like 'cinder'")
+    if cursor.rowcount == 1:
+        cinder_db_ver = cursor.fetchone()[0]
+    else:
+        cinder_db_ver = 0
+    db.close()
+
+    cinder_release = ""
+    for release in db_ver_map:
+        cinder_obj = db_ver_map[release]["cinder"]
+        cinder_db_map = int(cinder_obj)
+        if cinder_db_map == cinder_db_ver:
+            cinder_release = release
+
+    cinder_versions['db'] = cinder_db_ver
+    cinder_versions['os_release'] = cinder_release
+
+    return cinder_versions
+
+
+def get_heat_db_versions(db_host, db_user, db_pass, db_name="heat"):
+
+    ''' Return dict of heat DB versions '''
+
+    MySQLdb = _load_mysql_module()
+    if not MySQLdb:
+        return MySQLdb
+
+    heat_versions = {}
+    db = MySQLdb.connect(db_host, db_user, db_pass, db_name)
+    cursor = db.cursor()
+
+    cursor.execute("select `version` from `migrate_version` where `repository_id` like 'heat'")
+    if cursor.rowcount == 1:
+        heat_db_ver = cursor.fetchone()[0]
+    else:
+        heat_db_ver = 0
+    db.close()
+
+    heat_release = ""
+    for release in db_ver_map:
+        heat_obj = db_ver_map[release]["heat"]
+        heat_db_map = int(heat_obj)
+        if heat_db_map == heat_db_ver:
+            heat_release = release
+
+    heat_versions['db'] = heat_db_ver
+    heat_versions['os_release'] = heat_release
+
+    return heat_versions
+
+
+def get_neutron_db_versions(db_host, db_user, db_pass, db_name="neutron"):
+
+    ''' Return dict of neutron DB versions '''
+
+    MySQLdb = _load_mysql_module()
+    if not MySQLdb:
+        return MySQLdb
+
+    neutron_versions = {}
+    db = MySQLdb.connect(db_host, db_user, db_pass, db_name)
+    cursor = db.cursor()
+
+    try:
+        cursor.execute("select `version_num` from `alembic_version`")
+        neutron_db_versions_raw = cursor.fetchall()
+        neutron_db_versions = []
+        for el in neutron_db_versions_raw:
+            neutron_db_versions.append(el[0])
+    except:
+        neutron_db_versions = []
+
+    db.close()
+
+    neutron_release = "unknown (no marker found)"
+    for release in db_ver_map:
+        for commit_id in neutron_db_versions:
+            if commit_id in db_ver_map[release]["neutron"]:
+                neutron_release = release
+
+    neutron_versions['db_versions'] = neutron_db_versions
+    neutron_versions['os_release'] = neutron_release
+
+    return neutron_versions
+
+
+def get_nova_db_versions(db_host, db_user, db_pass, db_name="nova", db_api_name="nova_api", db_api_pass=""):
+
+    ''' Return dict of nova DB versions '''
+
+    MySQLdb = _load_mysql_module()
+    if not MySQLdb:
+        return MySQLdb
+
+    if not db_api_pass:
+        db_api_pass = db_pass
+
+    nova_versions = {}
+    db = MySQLdb.connect(db_host, db_user, db_pass, db_name)
+    cursor = db.cursor()
+
+    cursor.execute("select `version` from `migrate_version` where `repository_id` like 'nova'")
+    if cursor.rowcount == 1:
+        nova_db_ver = cursor.fetchone()[0]
+    else:
+        nova_db_ver = 0
+    db.close()
+
+    db = MySQLdb.connect(db_host, db_user, db_pass, db_api_name)
+    cursor = db.cursor()
+    if nova_db_ver > 0:
+        cursor.execute("select `version` from `migrate_version` where `repository_id` like 'nova_api'")
+        if cursor.rowcount == 1:
+            nova_apidb_ver = cursor.fetchone()[0]
+        else:
+            nova_apidb_ver = 0
+    db.close()
+
+    nova_versions['db'] = nova_db_ver
+    nova_versions['api_db'] = nova_apidb_ver
+    for release in db_ver_map:
+        nova_obj = db_ver_map[release]["nova"]
+        if isinstance(nova_obj, dict):
+            nova_db_map = int(nova_obj["db"])
+            nova_apidb_map = int(nova_obj["api_db"])
+            if nova_db_map == nova_db_ver and nova_apidb_map == nova_apidb_ver:
+                nova_versions['os_release'] = release
+        else:
+            nova_db_map = int(nova_obj)
+            if nova_db_map == nova_db_ver:
+                nova_versions['os_release'] = release
+
+    return nova_versions
+
+
+def list_db_versions():
+
+    ''' Retrieve openstack DB release codenames '''
+
+    db_host = str(__salt__['pillar.get']('_param:openstack_database_address'))
+    cinder_db_pass = str(__salt__['pillar.get']('_param:mysql_cinder_password'))
+    glance_db_pass = str(__salt__['pillar.get']('_param:mysql_glance_password'))
+    heat_db_pass = str(__salt__['pillar.get']('_param:mysql_heat_password'))
+    keystone_db_pass = str(__salt__['pillar.get']('_param:mysql_keystone_password'))
+    neutron_db_pass = str(__salt__['pillar.get']('_param:mysql_neutron_password'))
+    nova_db_pass = str(__salt__['pillar.get']('_param:mysql_nova_password'))
+    cinder_db_user = str(__salt__['pillar.get']('_param:mysql_cinder_username'))
+    glance_db_user = str(__salt__['pillar.get']('_param:mysql_glance_username'))
+    heat_db_user = str(__salt__['pillar.get']('_param:mysql_heat_username'))
+    keystone_db_user = str(__salt__['pillar.get']('_param:mysql_keystone_username'))
+    neutron_db_user = str(__salt__['pillar.get']('_param:mysql_neutron_username'))
+    nova_db_user = str(__salt__['pillar.get']('_param:mysql_nova_username'))
+
+    os_db_releases = {}
+    os_db_releases['cinder'] = get_cinder_db_versions(db_host, cinder_db_user, cinder_db_pass)
+    os_db_releases['glance'] = get_glance_db_versions(db_host, glance_db_user, glance_db_pass)
+    os_db_releases['heat'] = get_heat_db_versions(db_host, heat_db_user, heat_db_pass)
+    os_db_releases['neutron'] = get_neutron_db_versions(db_host, neutron_db_user, neutron_db_pass)
+    os_db_releases['keystone'] = get_keystone_db_versions(db_host, keystone_db_user, keystone_db_pass)
+    os_db_releases['nova'] = get_nova_db_versions(db_host, nova_db_user, nova_db_pass)
+
+    return os_db_releases
diff --git a/salt/meta/backupninja.yml b/salt/meta/backupninja.yml
index 1948dd8..887fce3 100644
--- a/salt/meta/backupninja.yml
+++ b/salt/meta/backupninja.yml
@@ -1,23 +1,37 @@
 {%- if pillar.salt is defined %}
-{%- if pillar.salt.get('master', {}).get('enabled', False) or (pillar.salt.get('minion', {}).get('enabled', False) and pillar.salt.get('minion', {}).ca is defined) %}
-  {%- from "salt/map.jinja" import master, minion with context %}
+
+  {%- if pillar.salt.get('master', {}).get('enabled', False) %}
+    {%- from "salt/map.jinja" import master with context %}
+    {%- set master_jinja = True %}
+  {%- else %}
+    {%- set master_jinja = False %}
+  {%- endif %}
+  {%- if pillar.salt.get('minion', {}).get('enabled', False) and pillar.salt.get('minion', {}).ca is defined %}
+    {%- from "salt/map.jinja" import minion with context %}
+    {%- set minion_jinja = True %}
+  {%- else %}
+    {%- set minion_jinja = False %}
+  {%- endif %}
+
+  {%- if master_jinja or minion_jinja %}
 backup:
   salt:
-    {%- if master.get('backup', False) %}
+    {%- if master_jinja and master.get('backup', False) %}
     fs_includes:
-    {%- if master.pillar.engine == 'reclass' or (master.pillar.engine == 'composite' and master.pillar.reclass is defined) %}
+    {%- if master_jinja and master.pillar.engine == 'reclass' or (master_jinja and master.pillar.engine == 'composite' and master.pillar.reclass is defined) %}
     - /srv/salt/reclass
     {%- endif %}
     - /etc/salt/pki
-    {%- if minion.get('backup', False) %}
+    {%- if minion_jinja and minion.get('backup', False) %}
     - /etc/pki/ca
     {%- endif %}
-    {%- elif minion.get('backup', False) %}
+    {%- elif minion_jinja and minion.get('backup', False) %}
     fs_includes:
     - /etc/pki/ca
     {%- else %}
     fs_includes: []
     {%- endif %}
     fs_excludes: []
-{%- endif %}
+  {%- endif %}
+
 {%- endif %}
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index 316692b..36c0c97 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -1,24 +1,40 @@
 #!/usr/bin/env bash
 
+###
+# Script source: https://gerrit.mcp.mirantis.com/#/admin/projects/salt-formulas/cookiecutter-salt-formula
+# Script requirments:
+#apt-get install -y python-yaml virtualenv git
+
+__ScriptVersion="2018.11.21"
+__ScriptName="run_tests.sh"
+__ScriptFullName="$0"
+__ScriptArgs="$*"
+
 set -e
 [ -n "$DEBUG" ] && set -x
 
 CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 METADATA=${CURDIR}/../metadata.yml
 FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']")
+FORMULA_META_DIR=${CURDIR}/../${FORMULA_NAME}/meta
 
 ## Overrideable parameters
 PILLARDIR=${PILLARDIR:-${CURDIR}/pillar}
 BUILDDIR=${BUILDDIR:-${CURDIR}/build}
 VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
+MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
 DEPSDIR=${BUILDDIR}/deps
+SCHEMARDIR=${SCHEMARDIR:-"${CURDIR}/../${FORMULA_NAME}/schemas/"}
 
 SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
 SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root}
 SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
 SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
+SALT_CACHE_EXTMODS_DIR=${SALT_CACHE_EXTMODS_DIR:-${SALT_CONFIG_DIR}/cache_master_extmods}
 
-SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR}"
+SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
+
+IGNORE_MODELVALIDATE_MASK=${IGNORE_MODELVALIDATE_MASK:-"novalidate"}
 
 if [ "x${SALT_VERSION}" != "x" ]; then
     PIP_SALT_VERSION="==${SALT_VERSION}"
@@ -26,24 +42,38 @@
 
 ## Functions
 log_info() {
-    echo "[INFO] $*"
+    echo -e "[INFO] $*"
 }
 
 log_err() {
-    echo "[ERROR] $*" >&2
+    echo -e "[ERROR] $*" >&2
 }
 
 setup_virtualenv() {
     log_info "Setting up Python virtualenv"
+    dependency_check virtualenv
     virtualenv $VENV_DIR
     source ${VENV_DIR}/bin/activate
     python -m pip install salt${PIP_SALT_VERSION}
+    if [[ -f ${CURDIR}/test-requirements.txt ]]; then
+       python -m pip install -r ${CURDIR}/test-requirements.txt
+    fi
+}
+
+setup_mock_bin() {
+    # If some state requires a binary, a lightweight replacement for
+    # such binary can be put into MOCK_BIN_DIR for test purposes
+    if [ -d "${MOCK_BIN_DIR}" ]; then
+        PATH="${MOCK_BIN_DIR}:$PATH"
+        export PATH
+    fi
 }
 
 setup_pillar() {
     [ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR}
     echo "base:" > ${SALT_PILLAR_DIR}/top.sls
     for pillar in ${PILLARDIR}/*; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
         state_name=$(basename ${pillar%.sls})
         echo -e "  ${state_name}:\n    - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
     done
@@ -53,9 +83,11 @@
     [ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR}
     [ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR}
     [ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR}
+    [ ! -d ${SALT_CACHE_EXTMODS_DIR} ] && mkdir -p ${SALT_CACHE_EXTMODS_DIR}
 
     echo "base:" > ${SALT_FILE_DIR}/top.sls
     for pillar in ${PILLARDIR}/*.sls; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
         state_name=$(basename ${pillar%.sls})
         echo -e "  ${state_name}:\n    - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
     done
@@ -63,13 +95,14 @@
     cat << EOF > ${SALT_CONFIG_DIR}/minion
 file_client: local
 cachedir: ${SALT_CACHE_DIR}
+extension_modules:  ${SALT_CACHE_EXTMODS_DIR}
 verify_env: False
+minion_id_caching: False
 
 file_roots:
   base:
   - ${SALT_FILE_DIR}
   - ${CURDIR}/..
-  - /usr/share/salt-formulas/env
 
 pillar_roots:
   base:
@@ -79,13 +112,14 @@
 }
 
 fetch_dependency() {
+    # example: fetch_dependency "linux:https://github.com/salt-formulas/salt-formula-linux"
     dep_name="$(echo $1|cut -d : -f 1)"
     dep_source="$(echo $1|cut -d : -f 2-)"
     dep_root="${DEPSDIR}/$(basename $dep_source .git)"
     dep_metadata="${dep_root}/metadata.yml"
 
-    [ -d /usr/share/salt-formulas/env/${dep_name} ] && log_info "Dependency $dep_name already present in system-wide salt env" && return 0
-    [ -d $dep_root ] && log_info "Dependency $dep_name already fetched" && return 0
+    dependency_check git
+    [ -d $dep_root ] && { log_info "Dependency $dep_name already fetched"; return 0; }
 
     log_info "Fetching dependency $dep_name"
     [ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR}
@@ -95,6 +129,19 @@
     METADATA="${dep_metadata}" install_dependencies
 }
 
+link_modules(){
+    # Link modules *.py files to temporary salt-root
+    local SALT_ROOT=${1:-$SALT_FILE_DIR}
+    local SALT_ENV=${2:-$DEPSDIR}
+
+    mkdir -p "${SALT_ROOT}/_modules/"
+    # from git, development versions
+    find ${SALT_ENV} -maxdepth 3 -mindepth 3 -path '*_modules*' -iname "*.py" -type f -print0 | while read -d $'\0' file; do
+      ln -fs $(readlink -e ${file}) "$SALT_ROOT"/_modules/$(basename ${file}) ;
+    done
+    salt_run saltutil.sync_all
+}
+
 install_dependencies() {
     grep -E "^dependencies:" ${METADATA} >/dev/null || return 0
     (python - | while read dep; do fetch_dependency "$dep"; done) << EOF
@@ -115,21 +162,96 @@
 }
 
 prepare() {
-    [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR}
+    if [[ -f ${BUILDDIR}/.prepare_done ]]; then
+      log_info "${BUILDDIR}/.prepare_done exist, not rebuilding BUILDDIR"
+      return
+    fi
+    [[ -d ${BUILDDIR} ]] && mkdir -p ${BUILDDIR}
 
-    which salt-call || setup_virtualenv
+    [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
+    setup_mock_bin
     setup_pillar
     setup_salt
     install_dependencies
+    link_modules
+    # Patch for load grains issue - https://github.com/saltstack/salt/commit/211c88bfbc8158b7ca16e429b366e403edfd39f3
+    sed -i 's/def find_module(self, module_name, package_path):/def find_module(self, module_name, package_path=None):/' ./build/virtualenv/lib/python2.7/site-packages/salt/__init__.py
+    touch ${BUILDDIR}/.prepare_done
+}
+
+lint_releasenotes() {
+    [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
+    source ${VENV_DIR}/bin/activate
+    reno lint ${CURDIR}/../
+}
+
+lint() {
+#    lint_releasenotes
+    log_err "TODO: lint_releasenotes"
 }
 
 run() {
     for pillar in ${PILLARDIR}/*.sls; do
+        grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
         state_name=$(basename ${pillar%.sls})
+        salt_run grains.set 'noservices' False force=True
+
+        echo "Checking state ${FORMULA_NAME}.${state_name} ..."
         salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
+
+        # Check that all files in 'meta' folder can be rendered using any valid pillar
+        for meta in `find ${FORMULA_META_DIR} -type f`; do
+            meta_name=$(basename ${meta})
+            echo "Checking meta ${meta_name} ..."
+            salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
+              || { log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1; }
+            cat ${SALT_CACHE_DIR}/${meta_name}
+        done
     done
 }
 
+real_run() {
+    for pillar in ${PILLARDIR}/*.sls; do
+        state_name=$(basename ${pillar%.sls})
+        salt_run --id=${state_name} state.sls ${FORMULA_NAME} || { log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1; }
+    done
+}
+
+run_model_validate(){
+  # Run modelschema.model_validate validation.
+  # TEST iterateble, run for `each formula ROLE against each ROLE_PILLARNAME`
+  # Pillars should be named in conviend ROLE_XXX.sls or ROLE.sls
+  # Example:
+  # client.sls  client_auth.sls  server.sls  server_auth.sls
+  if [ -d ${SCHEMARDIR} ]; then
+    # model validator require py modules
+    fetch_dependency "salt:https://github.com/salt-formulas/salt-formula-salt"
+    link_modules
+    salt_run saltutil.clear_cache; salt_run saltutil.refresh_pillar; salt_run saltutil.sync_all;
+    for role in ${SCHEMARDIR}/*.yaml; do
+      role_name=$(basename "${role%*.yaml}")
+      for pillar in $(ls pillar/${role_name}*.sls | grep -v ${IGNORE_MODELVALIDATE_MASK} ); do
+        pillar_name=$(basename "${pillar%*.sls}")
+        local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${pillar_name}"
+        log_info "model_validate ${_message}"
+        # Rendered Example:
+        # python $(which salt-call) --local -c /test1/maas/tests/build/salt --id=maas_cluster modelschema.model_validate maas cluster
+        salt_run -m ${DEPSDIR}/salt-formula-salt --id=${pillar_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; }
+      done
+    done
+  else
+    log_info "${SCHEMARDIR} not found!";
+  fi
+}
+
+dependency_check() {
+  local DEPENDENCY_COMMANDS=$*
+
+  for DEPENDENCY_COMMAND in $DEPENDENCY_COMMANDS; do
+    which $DEPENDENCY_COMMAND > /dev/null || ( log_err "Command \"$DEPENDENCY_COMMAND\" can not be found in default path."; exit 1; )
+  done
+}
+
 _atexit() {
     RETVAL=$?
     trap true INT TERM EXIT
@@ -143,6 +265,10 @@
 }
 
 ## Main
+
+log_info "Running version: ${__ScriptVersion}"
+log_info "Command line: '${__ScriptFullName} ${__ScriptArgs}'"
+
 trap _atexit INT TERM EXIT
 
 case $1 in
@@ -152,11 +278,23 @@
     prepare)
         prepare
         ;;
+    lint)
+        lint
+        ;;
     run)
         run
         ;;
+    real-run)
+        real_run
+        ;;
+    model-validate)
+       prepare
+       run_model_validate
+        ;;
     *)
         prepare
+#        lint
         run
+#         run_model_validate
         ;;
 esac
diff --git a/tests/test-requirements.txt b/tests/test-requirements.txt
new file mode 100644
index 0000000..bc7dc7f
--- /dev/null
+++ b/tests/test-requirements.txt
@@ -0,0 +1,4 @@
+jsonschema
+reno
+setuptools<45.0.0
+msgpack<1.0.0