Improve service validation during upgrade Cinder
* Added modules which allow to check and manage services statuses.
Change-Id: Ib51fd5b65c332aa1d9cae2ebcf9ebb7d510a4df3
Related-PROD: PROD-25157
diff --git a/_modules/cinderv3/__init__.py b/_modules/cinderv3/__init__.py
index 9bb71f2..218a23f 100644
--- a/_modules/cinderv3/__init__.py
+++ b/_modules/cinderv3/__init__.py
@@ -8,6 +8,7 @@
from cinderv3 import volumes
from cinderv3 import volume_types
from cinderv3 import volume_actions
+from cinderv3 import services
volume_list = lists.volume_list
volume_list_get_details = lists.volume_list_get_details
@@ -28,6 +29,9 @@
keys_volume_type_get = volume_types.keys_volume_type_get
keys_volume_type_set = volume_types.keys_volume_type_set
image_upload_volume = volume_actions.image_upload_volume
+service_list = lists.service_list
+service_update = services.service_update
+service_wait = services.wait_for_service
__all__ = ('volume_list', 'volume_create', 'volume_delete',
'volume_get_details', 'volume_list_get_details', 'volume_update',
@@ -37,8 +41,7 @@
'volume_type_list', 'volume_type_get_details',
'volume_type_create', 'keys_volume_type_get',
'keys_volume_type_set', 'volume_type_delete',
- 'image_upload_volume',)
-
+ 'image_upload_volume', 'service_list', 'service_update', 'service_wait')
def __virtual__():
if REQUIREMENTS_MET:
diff --git a/_modules/cinderv3/lists.py b/_modules/cinderv3/lists.py
index d0bf4e9..5982f06 100644
--- a/_modules/cinderv3/lists.py
+++ b/_modules/cinderv3/lists.py
@@ -5,6 +5,7 @@
except ImportError:
from urllib import urlencode
+
@send("get")
def volume_list(**kwargs):
"""
@@ -27,3 +28,12 @@
"""
url = '/types?{}'.format(urlencode(kwargs))
return url, None
+
+
+@send("get")
+def service_list(**kwargs):
+ """
+ Return list of Cinder services.
+ """
+ url = '/os-services?{}'.format(urlencode(kwargs))
+ return url, None
diff --git a/_modules/cinderv3/services.py b/_modules/cinderv3/services.py
new file mode 100644
index 0000000..b0ed38e
--- /dev/null
+++ b/_modules/cinderv3/services.py
@@ -0,0 +1,62 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import time
+from salt.exceptions import CommandExecutionError
+from cinderv3.common import send
+from cinderv3.lists import service_list
+
+@send("put")
+def service_update(host, binary, action, **kwargs):
+ """
+ Enable/Disable Cinder service
+ """
+ req = {"host": host, "binary": binary}
+ if kwargs.get('disabled_reason') and action == 'disable':
+ url = '/os-services/disable-log-reason'
+ req['disabled_reason'] = kwargs['disabled_reason']
+ else:
+ url = '/os-services/{}'.format(action)
+
+ return url, req
+
+
+def wait_for_service(cloud_name, host=None, binary=None, admin_up_only=True, retries=18, timeout=10, **kwargs):
+ """
+ Ensure the service is up and running on specified host.
+
+ :param host: name of a host where service is running
+ :param admin_up_only: do not check status for admin disabled service
+ :param binary: name of the service
+ :param timeout: number of seconds to wait before retries
+ :param retries: number of retries
+ """
+
+ if host:
+ kwargs['host'] = host
+ if binary:
+ kwargs['binary'] = binary
+
+ for _i in range(retries):
+ services = service_list(cloud_name=cloud_name, **kwargs)['services']
+
+ down_services = [s for s in services if s['state'] == 'down']
+
+ if admin_up_only:
+ down_services = [s for s in down_services if s['status'] == 'enabled']
+
+ if not down_services:
+ return 'Cinder services with admin_up_only=%s are up or disabled administratively' % (admin_up_only)
+
+ time.sleep(timeout)
+
+ raise CommandExecutionError("Cinder services {} are still down or disabled".format(down_services))
diff --git a/_states/cinderv3.py b/_states/cinderv3.py
index 15cff40..3e2e21c 100644
--- a/_states/cinderv3.py
+++ b/_states/cinderv3.py
@@ -78,6 +78,52 @@
return _succeeded('delete', name, resource)
+def service_enabled(name, binary, cloud_name):
+ """Ensures that the service is enabled on the host
+
+ :param name: name of a host where service is running
+ :param binary: name of the service have to be run
+ """
+ changes = {}
+
+ services = _cinder_call('service_list', host=name, binary=binary, cloud_name=cloud_name)['services']
+
+ enabled_service = [s for s in services if s['status'] == 'enabled']
+
+ if len(enabled_service) > 0:
+ ret = _succeeded('no_changes', name, binary)
+ else:
+ changes = _cinder_call('service_update', name, binary, 'enable', cloud_name=cloud_name)
+ ret = _succeeded('update', name, binary, changes)
+
+ return ret
+
+
+def service_disabled(name, binary, cloud_name, disabled_reason=None):
+ """Ensures that the service is disabled on the host
+
+ :param name: name of a host where service is running
+ :param binary: name of the service have to be disabled
+ """
+ changes = {}
+ kwargs = {}
+
+ if disabled_reason is not None:
+ kwargs['disabled_reason'] = disabled_reason
+
+ services = _cinder_call('service_list', host=name, binary=binary, cloud_name=cloud_name)['services']
+
+ disabled_service = [s for s in services if s['status'] == 'disabled']
+
+ if len(disabled_service) > 0:
+ ret = _succeeded('no_changes', name, binary)
+ else:
+ changes = _cinder_call('service_update', name, binary, 'disable', cloud_name=cloud_name, **kwargs)
+ ret = _succeeded('update', name, binary, changes)
+
+ return ret
+
+
def volume_type_present(name, cloud_name, **kwargs):
return _resource_present('volume_type', name, cloud_name, **kwargs)
@@ -153,4 +199,4 @@
'comment': msg_map[op].format(resource, name),
'changes': {},
}
- return changes_dict
\ No newline at end of file
+ return changes_dict
diff --git a/cinder/upgrade/verify/_service.sls b/cinder/upgrade/verify/_service.sls
new file mode 100644
index 0000000..23b4105
--- /dev/null
+++ b/cinder/upgrade/verify/_service.sls
@@ -0,0 +1,13 @@
+cinder_task_uprade_verify_service:
+ test.show_notification:
+ - text: "Running cinder.upgrade.verify.service"
+
+{% set host_id = salt['network.get_hostname']() %}
+
+wait_for_service:
+ module.run:
+ - name: cinderv3.service_wait
+ - cloud_name: admin_identity
+ - host: {{ host_id }}
+ - retries: 30
+ - timeout: 10
diff --git a/cinder/upgrade/verify/init.sls b/cinder/upgrade/verify/init.sls
index 5b7aa99..623fcc3 100644
--- a/cinder/upgrade/verify/init.sls
+++ b/cinder/upgrade/verify/init.sls
@@ -1,2 +1,3 @@
include:
- - cinder.upgrade.verify._api
+ - cinder.upgrade.verify._service
+ - cinder.upgrade.verify._api