Improve service validation during upgrade Cinder

  * Added modules which allow to check and manage services statuses.

Change-Id: Ib51fd5b65c332aa1d9cae2ebcf9ebb7d510a4df3
Related-PROD: PROD-25157
diff --git a/_modules/cinderv3/services.py b/_modules/cinderv3/services.py
new file mode 100644
index 0000000..b0ed38e
--- /dev/null
+++ b/_modules/cinderv3/services.py
@@ -0,0 +1,62 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import time
+from salt.exceptions import CommandExecutionError
+from cinderv3.common import send
+from cinderv3.lists import service_list
+
+@send("put")
+def service_update(host, binary, action, **kwargs):
+    """
+    Enable/Disable Cinder service
+    """
+    req = {"host": host, "binary": binary}
+    if kwargs.get('disabled_reason') and action == 'disable':
+      url = '/os-services/disable-log-reason'
+      req['disabled_reason'] = kwargs['disabled_reason']
+    else:
+      url = '/os-services/{}'.format(action)
+
+    return url, req
+
+
+def wait_for_service(cloud_name, host=None, binary=None, admin_up_only=True, retries=18, timeout=10, **kwargs):
+    """
+    Ensure the service is up and running on specified host.
+
+    :param host:           name of a host where service is running
+    :param admin_up_only:  do not check status for admin disabled service
+    :param binary:         name of the service
+    :param timeout:        number of seconds to wait before retries
+    :param retries:        number of retries
+    """
+
+    if host:
+        kwargs['host'] = host
+    if binary:
+        kwargs['binary'] = binary
+
+    for _i in range(retries):
+        services = service_list(cloud_name=cloud_name, **kwargs)['services']
+
+        down_services = [s for s in services if s['state'] == 'down']
+
+        if admin_up_only:
+            down_services = [s for s in down_services if s['status'] == 'enabled']
+
+        if not down_services:
+            return 'Cinder services with admin_up_only=%s are up or disabled administratively' % (admin_up_only)
+
+        time.sleep(timeout)
+
+    raise CommandExecutionError("Cinder services {} are still down or disabled".format(down_services))