blob: f913544b766775857df532ce46a9213b8f3b8979 [file] [log] [blame]
Dennis Dmitriev6f59add2016-10-18 13:45:27 +03001# Copyright 2016 Mirantis, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import copy
16import os
17import shutil
18import tempfile
19import time
20import traceback
21
22import paramiko
23import yaml
24from devops.helpers import helpers
25from devops.helpers import ssh_client
Dennis Dmitriev6f59add2016-10-18 13:45:27 +030026
27from tcp_tests import logger
28from tcp_tests import settings
29from tcp_tests.helpers import ext
30
31LOG = logger.logger
32
33
34def get_test_method_name():
35 raise NotImplementedError
36
37
38def update_yaml(yaml_tree=None, yaml_value='', is_uniq=True,
39 yaml_file=settings.TIMESTAT_PATH_YAML, remote=None):
40 """Store/update a variable in YAML file.
41
42 yaml_tree - path to the variable in YAML file, will be created if absent,
43 yaml_value - value of the variable, will be overwritten if exists,
44 is_uniq - If false, add the unique two-digit suffix to the variable name.
45 """
46 def get_file(path, remote=None, mode="r"):
47 if remote:
48 return remote.open(path, mode)
49 else:
50 return open(path, mode)
51
52 if yaml_tree is None:
53 yaml_tree = []
54 with get_file(yaml_file, remote) as file_obj:
55 yaml_data = yaml.safe_load(file_obj)
56
57 # Walk through the 'yaml_data' dict, find or create a tree using
58 # sub-keys in order provided in 'yaml_tree' list
59 item = yaml_data
60 for n in yaml_tree[:-1]:
61 if n not in item:
62 item[n] = {}
63 item = item[n]
64
65 if is_uniq:
66 last = yaml_tree[-1]
67 else:
68 # Create an uniq suffix in range '_00' to '_99'
69 for n in range(100):
70 last = str(yaml_tree[-1]) + '_' + str(n).zfill(2)
71 if last not in item:
72 break
73
74 item[last] = yaml_value
75 with get_file(yaml_file, remote, mode='w') as file_obj:
76 yaml.dump(yaml_data, file_obj, default_flow_style=False)
77
78
79class TimeStat(object):
80 """Context manager for measuring the execution time of the code.
81
82 Usage:
83 with TimeStat([name],[is_uniq=True]):
84 """
85
86 def __init__(self, name=None, is_uniq=False):
87 if name:
88 self.name = name
89 else:
90 self.name = 'timestat'
91 self.is_uniq = is_uniq
92 self.begin_time = 0
93 self.end_time = 0
94 self.total_time = 0
95
96 def __enter__(self):
97 self.begin_time = time.time()
98 return self
99
100 def __exit__(self, exc_type, exc_value, exc_tb):
101 self.end_time = time.time()
102 self.total_time = self.end_time - self.begin_time
103
104 # Create a path where the 'self.total_time' will be stored.
105 yaml_path = []
106
107 # There will be a list of one or two yaml subkeys:
108 # - first key name is the method name of the test
109 method_name = get_test_method_name()
110 if method_name:
111 yaml_path.append(method_name)
112
113 # - second (subkey) name is provided from the decorator (the name of
114 # the just executed function), or manually.
115 yaml_path.append(self.name)
116
117 try:
118 update_yaml(yaml_path, '{:.2f}'.format(self.total_time),
119 self.is_uniq)
120 except Exception:
121 LOG.error("Error storing time statistic for {0}"
122 " {1}".format(yaml_path, traceback.format_exc()))
123 raise
124
125 @property
126 def spent_time(self):
127 return time.time() - self.begin_time
128
129
130def reduce_occurrences(items, text):
131 """ Return string without items(substrings)
132 Args:
133 items: iterable of strings
134 test: string
135 Returns:
136 string
137 Raise:
138 AssertionError if any substing not present in source text
139 """
140 for item in items:
141 LOG.debug(
142 "Verifying string {} is shown in "
143 "\"\"\"\n{}\n\"\"\"".format(item, text))
144 assert text.count(item) != 0
145 text = text.replace(item, "", 1)
146 return text
147
148
149def generate_keys():
150 key = paramiko.RSAKey.generate(1024)
151 public = key.get_base64()
152 dirpath = tempfile.mkdtemp()
153 key.write_private_key_file(os.path.join(dirpath, 'id_rsa'))
154 with open(os.path.join(dirpath, 'id_rsa.pub'), 'w') as pub_file:
155 pub_file.write(public)
156 return dirpath
157
158
159def clean_dir(dirpath):
160 shutil.rmtree(dirpath)
161
162
163def retry(tries_number=3, exception=Exception):
164 def _retry(func):
165 assert tries_number >= 1, 'ERROR! @retry is called with no tries!'
166
167 def wrapper(*args, **kwargs):
168 iter_number = 1
169 while True:
170 try:
171 LOG.debug('Calling function "{0}" with args "{1}" and '
172 'kwargs "{2}". Try # {3}.'.format(func.__name__,
173 args,
174 kwargs,
175 iter_number))
176 return func(*args, **kwargs)
177 except exception as e:
178 if iter_number > tries_number:
179 LOG.debug('Failed to execute function "{0}" with {1} '
180 'tries!'.format(func.__name__, tries_number))
181 raise e
182 iter_number += 1
183 return wrapper
184 return _retry
185
186
Dennis Dmitriev6f59add2016-10-18 13:45:27 +0300187class YamlEditor(object):
188 """Manipulations with local or remote .yaml files.
189
190 Usage:
191
192 with YamlEditor("tasks.yaml") as editor:
193 editor.content[key] = "value"
194
195 with YamlEditor("astute.yaml", ip=self.admin_ip) as editor:
196 editor.content[key] = "value"
197 """
198
199 def __init__(self, file_path, host=None, port=None,
200 username=None, password=None, private_keys=None,
201 document_id=0,
202 default_flow_style=False, default_style=None):
203 self.__file_path = file_path
204 self.host = host
205 self.port = port or 22
206 self.username = username
207 self.__password = password
208 self.__private_keys = private_keys or []
209 self.__content = None
210 self.__documents = [{}, ]
211 self.__document_id = document_id
212 self.__original_content = None
213 self.default_flow_style = default_flow_style
214 self.default_style = default_style
215
216 @property
217 def file_path(self):
218 """Open file path
219
220 :rtype: str
221 """
222 return self.__file_path
223
224 @property
225 def content(self):
226 if self.__content is None:
227 self.__content = self.get_content()
228 return self.__content
229
230 @content.setter
231 def content(self, new_content):
232 self.__content = new_content
233
234 def __get_file(self, mode="r"):
235 if self.host:
236 remote = ssh_client.SSHClient(
237 host=self.host,
238 port=self.port,
239 username=self.username,
240 password=self.__password,
241 private_keys=self.__private_keys)
242
243 return remote.open(self.__file_path, mode=mode)
244 else:
245 return open(self.__file_path, mode=mode)
246
247 def get_content(self):
248 """Return a single document from YAML"""
249 def multi_constructor(loader, tag_suffix, node):
250 """Stores all unknown tags content into a dict
251
252 Original yaml:
253 !unknown_tag
254 - some content
255
256 Python object:
257 {"!unknown_tag": ["some content", ]}
258 """
259 if type(node.value) is list:
260 if type(node.value[0]) is tuple:
261 return {node.tag: loader.construct_mapping(node)}
262 else:
263 return {node.tag: loader.construct_sequence(node)}
264 else:
265 return {node.tag: loader.construct_scalar(node)}
266
267 yaml.add_multi_constructor("!", multi_constructor)
268 with self.__get_file() as file_obj:
269 self.__documents = [x for x in yaml.load_all(file_obj)]
270 return self.__documents[self.__document_id]
271
272 def write_content(self, content=None):
273 if content:
274 self.content = content
275 self.__documents[self.__document_id] = self.content
276
277 def representer(dumper, data):
278 """Represents a dict key started with '!' as a YAML tag
279
280 Assumes that there is only one !tag in the dict at the
281 current indent.
282
283 Python object:
284 {"!unknown_tag": ["some content", ]}
285
286 Resulting yaml:
287 !unknown_tag
288 - some content
289 """
290 key = data.keys()[0]
291 if key.startswith("!"):
292 value = data[key]
293 if type(value) is dict:
294 node = dumper.represent_mapping(key, value)
295 elif type(value) is list:
296 node = dumper.represent_sequence(key, value)
297 else:
298 node = dumper.represent_scalar(key, value)
299 else:
300 node = dumper.represent_mapping(u'tag:yaml.org,2002:map', data)
301 return node
302
303 yaml.add_representer(dict, representer)
304 with self.__get_file("w") as file_obj:
305 yaml.dump_all(self.__documents, file_obj,
306 default_flow_style=self.default_flow_style,
307 default_style=self.default_style)
308
309 def __enter__(self):
310 self.__content = self.get_content()
311 self.__original_content = copy.deepcopy(self.content)
312 return self
313
314 def __exit__(self, x, y, z):
315 if self.content == self.__original_content:
316 return
317 self.write_content()
Dennis Dmitriev535869c2016-11-16 22:38:06 +0200318
319
320def extract_name_from_mark(mark):
321 """Simple function to extract name from pytest mark
322
323 :param mark: pytest.mark.MarkInfo
324 :rtype: string or None
325 """
326 if mark:
327 if len(mark.args) > 0:
328 return mark.args[0]
329 elif 'name' in mark.kwargs:
330 return mark.kwargs['name']
331 return None
332
333
334def get_top_fixtures_marks(request, mark_name):
335 """Order marks according to fixtures order
336
337 When a test use fixtures that depend on each other in some order,
338 that fixtures can have the same pytest mark.
339
340 This method extracts such marks from fixtures that are used in the
341 current test and return the content of the marks ordered by the
342 fixture dependences.
343 If the test case have the same mark, than the content of this mark
344 will be the first element in the resulting list.
345
346 :param request: pytest 'request' fixture
347 :param mark_name: name of the mark to search on the fixtures and the test
348
349 :rtype list: marks content, from last to first executed.
350 """
351
352 fixtureinfo = request.session._fixturemanager.getfixtureinfo(
353 request.node, request.function, request.cls)
354
355 top_fixtures_names = []
356 for _ in enumerate(fixtureinfo.name2fixturedefs):
357 parent_fixtures = set()
358 child_fixtures = set()
359 for name in sorted(fixtureinfo.name2fixturedefs):
360 if name in top_fixtures_names:
361 continue
362 parent_fixtures.add(name)
363 child_fixtures.update(
364 fixtureinfo.name2fixturedefs[name][0].argnames)
365 top_fixtures_names.extend(list(parent_fixtures - child_fixtures))
366
367 top_fixtures_marks = []
368
369 if mark_name in request.function.func_dict:
370 # The top priority is the 'revert_snapshot' mark on the test
371 top_fixtures_marks.append(
372 extract_name_from_mark(
373 request.function.func_dict[mark_name]))
374
375 for top_fixtures_name in top_fixtures_names:
376 fd = fixtureinfo.name2fixturedefs[top_fixtures_name][0]
377 if mark_name in fd.func.func_dict:
378 fixture_mark = extract_name_from_mark(
379 fd.func.func_dict[mark_name])
380 # Append the snapshot names in the order that fixtures are called
381 # starting from the last called fixture to the first one
382 top_fixtures_marks.append(fixture_mark)
383
384 LOG.debug("Fixtures ordered from last to first called: {0}"
385 .format(top_fixtures_names))
386 LOG.debug("Marks ordered from most to least preffered: {0}"
387 .format(top_fixtures_marks))
388
389 return top_fixtures_marks