Testcases for services failover

- keepalived restart # 4756965
- keepalived stop # 3385682

Changes:
- RallyManager refactored to use updated rally container with tempest
- Added 'rally.create_rally_task' and 'rally.run_task' methods to
  generate load on the OpenStack cluster with the specified task config
- new mark for test cases that configure 'rally' fixture:
  @pytest.mark.with_rally(rally_node=<str>,
                          prepare_openstack=<bool>,
                          prepare_tempest=<bool>)
- a new method common_services_deployed.check_keepalived_pillar()
  to check the keepalived pillar settings consistency
- a new fixture 'func_name' returns the current test function name
- a new method 'underlay.get_target_node_names(target='ctl')' to get
  a list of all nodes which name starts with the specified target string
- a new method underlay.delayed_call() which can postpone the specified
  shell command to run in several minutes later in the background
  on the specified node
- fixture 'grab_versions' now works also for failed tests

Change-Id: Icede63163ae0b3569e8463563cb548e2d314899d
diff --git a/tcp_tests/fixtures/common_fixtures.py b/tcp_tests/fixtures/common_fixtures.py
index 9d6ccba..31f0f1e 100644
--- a/tcp_tests/fixtures/common_fixtures.py
+++ b/tcp_tests/fixtures/common_fixtures.py
@@ -52,6 +52,7 @@
     finish_step = "FINISH {} TEST. TOOK {} min {} sec".format(
         step_name, minutes, seconds
     )
+    print("\n\n")
     foot = "\n" + "<" * 5 + "#" * 30 + "[ {} ]" + "#" * 30 + ">" * 5
     foot = foot.format(finish_step)
     LOG.info(foot)
@@ -69,3 +70,10 @@
     steps_mark = request.keywords.get('steps', None)
     steps = steps_mark.args[0]
     return steps
+
+
+@pytest.fixture(scope='function', autouse=True)
+def func_name(request):
+    """Name of the current test function"""
+    return getattr(request.node.function, '_name',
+                   request.node.function.__name__)
diff --git a/tcp_tests/fixtures/openstack_fixtures.py b/tcp_tests/fixtures/openstack_fixtures.py
index 98e367c..1926299 100644
--- a/tcp_tests/fixtures/openstack_fixtures.py
+++ b/tcp_tests/fixtures/openstack_fixtures.py
@@ -41,7 +41,7 @@
 @pytest.fixture(scope='function')
 def openstack_deployed(revert_snapshot, request, config,
                        hardware, underlay, common_services_deployed,
-                       openstack_actions):
+                       openstack_actions, rally):
     """Fixture to get or install OpenStack services on environment
 
     :param revert_snapshot: fixture that reverts snapshot that is specified
@@ -52,11 +52,13 @@
     :param underlay: fixture provides underlay manager
     :param common_services_deployed: fixture provides CommonServicesManager
     :param openstack_actions: fixture provides OpenstackManager instance
+    :param rally: fixture provides RallyManager instance
     :rtype: OpenstackManager
 
     If config.openstack.openstack_installed is not set, this fixture assumes
     that the openstack services were not installed, and do the following:
     - install openstack services
+    - [optional] prepare docker with rally container
     - make snapshot with name 'openstack_deployed'
     - return OpenstackManager instance
 
@@ -64,14 +66,38 @@
     the openstack services were already installed, and do the following:
     - return OpenstackManager instance
 
+    If you want to prepare 'rally', please use mark:
+    @pytest.mark.with_rally(rally_node=<str>,
+                            prepare_openstack=<bool>,
+                            prepare_tempest=<bool>)
+    :param rally_node: first chars of the node name where rally should
+                       be started
+    :param prepare_openstack: if True, prepare OpenStack objects for
+                              rally tasks: cirros image, private net04
+
     If you want to revert 'openstack_deployed' snapshot, please use mark:
     @pytest.mark.revert_snapshot("openstack_deployed")
     """
+
     # Deploy Openstack cluster
     if not config.openstack.openstack_installed:
         steps_path = config.openstack_deploy.openstack_steps_path
         commands = underlay.read_template(steps_path)
         openstack_actions.install(commands)
+
+        # If @pytest.mark.with_rally() is set, then prepare Rally
+        # container for 'openstack_deployed' snapshot.
+        with_rally = request.keywords.get('with_rally', None)
+        if with_rally:
+            prepare_openstack = with_rally.kwargs.get("prepare_openstack",
+                                                      False)
+            prepare_tempest = with_rally.kwargs.get("prepare_tempest", False)
+            if prepare_openstack:
+                rally.prepare_rally_task(target_node='ctl01')
+            if prepare_tempest:
+                rally.prepare_tempest_task()
+            rally.run_container()
+
         hardware.create_snapshot(ext.SNAPSHOT.openstack_deployed)
 
     else:
diff --git a/tcp_tests/fixtures/rally_fixtures.py b/tcp_tests/fixtures/rally_fixtures.py
index 18fa179..335ab88 100644
--- a/tcp_tests/fixtures/rally_fixtures.py
+++ b/tcp_tests/fixtures/rally_fixtures.py
@@ -18,13 +18,19 @@
 
 
 @pytest.fixture(scope='function')
-def rally(config, underlay):
+def rally(request, config, underlay):
     """Fixture that provides various actions for TCP
 
+    :param request: fixture provides pytest data
     :param config: fixture provides oslo.config
     :param underlay: fixture provides underlay manager
     :rtype: RallyManager
 
     For use in tests or fixtures to deploy a custom TCP
     """
-    return rallymanager.RallyManager(underlay, config.salt.salt_master_host)
+    with_rally = request.keywords.get('with_rally', None)
+    rally_node = "gtw01."
+    if with_rally:
+        rally_node = with_rally.kwargs.get("rally_node", "gtw01.")
+
+    return rallymanager.RallyManager(underlay, rally_node)
diff --git a/tcp_tests/fixtures/underlay_fixtures.py b/tcp_tests/fixtures/underlay_fixtures.py
index a1476e3..eacbec9 100644
--- a/tcp_tests/fixtures/underlay_fixtures.py
+++ b/tcp_tests/fixtures/underlay_fixtures.py
@@ -199,7 +199,7 @@
 
 
 @pytest.fixture(scope='function', autouse=True)
-def grab_versions(request, underlay):
+def grab_versions(request, func_name, underlay):
     """Fixture for grab package versions at the end of test
 
     Marks:
@@ -211,11 +211,10 @@
     grab_version = request.keywords.get('grab_versions', None)
 
     def test_fin():
-        default_name = getattr(request.node.function, '_name',
-                               request.node.function.__name__)
-        if hasattr(request.node, 'rep_call') and request.node.rep_call.passed \
+        if hasattr(request.node, 'rep_call') and \
+                (request.node.rep_call.passed or request.node.rep_call.failed)\
                 and grab_version:
             artifact_name = utils.extract_name_from_mark(grab_version) or \
-                "{}".format(default_name)
+                "{}".format(func_name)
             underlay.get_logs(artifact_name)
     request.addfinalizer(test_fin)