| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| import copy |
| import json |
| |
| from testtools import matchers |
| |
| from heat_integrationtests.common import test |
| |
| |
| class InstanceGroupTest(test.HeatIntegrationTest): |
| |
| template = ''' |
| { |
| "AWSTemplateFormatVersion" : "2010-09-09", |
| "Description" : "Template to create multiple instances.", |
| "Parameters" : {"size": {"Type": "String", "Default": "1"}, |
| "AZ": {"Type": "String", "Default": "nova"}, |
| "image": {"Type": "String"}, |
| "flavor": {"Type": "String"}}, |
| "Resources": { |
| "JobServerGroup": { |
| "Type": "OS::Heat::InstanceGroup", |
| "Properties": { |
| "LaunchConfigurationName" : {"Ref": "JobServerConfig"}, |
| "Size" : {"Ref": "size"}, |
| "AvailabilityZones" : [{"Ref": "AZ"}] |
| } |
| }, |
| |
| "JobServerConfig" : { |
| "Type" : "AWS::AutoScaling::LaunchConfiguration", |
| "Metadata": {"foo": "bar"}, |
| "Properties": { |
| "ImageId" : {"Ref": "image"}, |
| "InstanceType" : {"Ref": "flavor"}, |
| "SecurityGroups" : [ "sg-1" ], |
| "UserData" : "jsconfig data" |
| } |
| } |
| }, |
| "Outputs": { |
| "InstanceList": {"Value": { |
| "Fn::GetAtt": ["JobServerGroup", "InstanceList"]}}, |
| "JobServerConfigRef": {"Value": { |
| "Ref": "JobServerConfig"}} |
| } |
| } |
| ''' |
| |
| instance_template = ''' |
| heat_template_version: 2013-05-23 |
| parameters: |
| ImageId: {type: string} |
| InstanceType: {type: string} |
| SecurityGroups: {type: comma_delimited_list} |
| UserData: {type: string} |
| Tags: {type: comma_delimited_list} |
| |
| resources: |
| random1: |
| type: OS::Heat::RandomString |
| properties: |
| salt: {get_param: ImageId} |
| outputs: |
| PublicIp: |
| value: {get_attr: [random1, value]} |
| ''' |
| |
| # This is designed to fail. |
| bad_instance_template = ''' |
| heat_template_version: 2013-05-23 |
| parameters: |
| ImageId: {type: string} |
| InstanceType: {type: string} |
| SecurityGroups: {type: comma_delimited_list} |
| UserData: {type: string} |
| Tags: {type: comma_delimited_list} |
| |
| resources: |
| random1: |
| type: OS::Heat::RandomString |
| depends_on: waiter |
| ready_poster: |
| type: AWS::CloudFormation::WaitConditionHandle |
| waiter: |
| type: AWS::CloudFormation::WaitCondition |
| properties: |
| Handle: {Ref: ready_poster} |
| Timeout: 1 |
| outputs: |
| PublicIp: |
| value: {get_attr: [random1, value]} |
| ''' |
| |
| def setUp(self): |
| super(InstanceGroupTest, self).setUp() |
| self.client = self.orchestration_client |
| if not self.conf.image_ref: |
| raise self.skipException("No image configured to test") |
| if not self.conf.minimal_image_ref: |
| raise self.skipException("No minimal image configured to test") |
| if not self.conf.instance_type: |
| raise self.skipException("No flavor configured to test") |
| |
| def assert_instance_count(self, stack, expected_count): |
| inst_list = self._stack_output(stack, 'InstanceList') |
| self.assertEqual(expected_count, len(inst_list.split(','))) |
| |
| def _assert_instance_state(self, nested_identifier, |
| num_complete, num_failed): |
| for res in self.client.resources.list(nested_identifier): |
| if 'COMPLETE' in res.resource_status: |
| num_complete = num_complete - 1 |
| elif 'FAILED' in res.resource_status: |
| num_failed = num_failed - 1 |
| self.assertEqual(0, num_failed) |
| self.assertEqual(0, num_complete) |
| |
| |
| class InstanceGroupBasicTest(InstanceGroupTest): |
| |
| def test_basic_create_works(self): |
| """Make sure the working case is good. |
| Note this combines test_override_aws_ec2_instance into this test as |
| well, which is: |
| If AWS::EC2::Instance is overridden, InstanceGroup will automatically |
| use that overridden resource type. |
| """ |
| |
| files = {'provider.yaml': self.instance_template} |
| env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'}, |
| 'parameters': {'size': 4, |
| 'image': self.conf.image_ref, |
| 'flavor': self.conf.instance_type}} |
| stack_identifier = self.stack_create(template=self.template, |
| files=files, environment=env) |
| initial_resources = { |
| 'JobServerConfig': 'AWS::AutoScaling::LaunchConfiguration', |
| 'JobServerGroup': 'OS::Heat::InstanceGroup'} |
| self.assertEqual(initial_resources, |
| self.list_resources(stack_identifier)) |
| |
| stack = self.client.stacks.get(stack_identifier) |
| self.assert_instance_count(stack, 4) |
| |
| def test_size_updates_work(self): |
| files = {'provider.yaml': self.instance_template} |
| env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'}, |
| 'parameters': {'size': 2, |
| 'image': self.conf.image_ref, |
| 'flavor': self.conf.instance_type}} |
| |
| stack_identifier = self.stack_create(template=self.template, |
| files=files, |
| environment=env) |
| stack = self.client.stacks.get(stack_identifier) |
| self.assert_instance_count(stack, 2) |
| |
| # Increase min size to 5 |
| env2 = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'}, |
| 'parameters': {'size': 5, |
| 'image': self.conf.image_ref, |
| 'flavor': self.conf.instance_type}} |
| self.update_stack(stack_identifier, self.template, |
| environment=env2, files=files) |
| stack = self.client.stacks.get(stack_identifier) |
| self.assert_instance_count(stack, 5) |
| |
| def test_update_group_replace(self): |
| """Make sure that during a group update the non updatable |
| properties cause a replacement. |
| """ |
| files = {'provider.yaml': self.instance_template} |
| env = {'resource_registry': |
| {'AWS::EC2::Instance': 'provider.yaml'}, |
| 'parameters': {'size': 1, |
| 'image': self.conf.image_ref, |
| 'flavor': self.conf.instance_type}} |
| |
| stack_identifier = self.stack_create(template=self.template, |
| files=files, |
| environment=env) |
| rsrc = self.client.resources.get(stack_identifier, 'JobServerGroup') |
| orig_asg_id = rsrc.physical_resource_id |
| |
| env2 = {'resource_registry': |
| {'AWS::EC2::Instance': 'provider.yaml'}, |
| 'parameters': {'size': '2', |
| 'AZ': 'wibble', |
| 'image': self.conf.image_ref, |
| 'flavor': self.conf.instance_type}} |
| self.update_stack(stack_identifier, self.template, |
| environment=env2, files=files) |
| |
| # replacement will cause the resource physical_resource_id to change. |
| rsrc = self.client.resources.get(stack_identifier, 'JobServerGroup') |
| self.assertNotEqual(orig_asg_id, rsrc.physical_resource_id) |
| |
| def test_create_instance_error_causes_group_error(self): |
| """If a resource in an instance group fails to be created, the instance |
| group itself will fail and the broken inner resource will remain. |
| """ |
| stack_name = self._stack_rand_name() |
| files = {'provider.yaml': self.bad_instance_template} |
| env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'}, |
| 'parameters': {'size': 2, |
| 'image': self.conf.image_ref, |
| 'flavor': self.conf.instance_type}} |
| |
| self.client.stacks.create( |
| stack_name=stack_name, |
| template=self.template, |
| files=files, |
| disable_rollback=True, |
| parameters={}, |
| environment=env |
| ) |
| self.addCleanup(self.client.stacks.delete, stack_name) |
| stack = self.client.stacks.get(stack_name) |
| stack_identifier = '%s/%s' % (stack_name, stack.id) |
| self._wait_for_stack_status(stack_identifier, 'CREATE_FAILED') |
| initial_resources = { |
| 'JobServerConfig': 'AWS::AutoScaling::LaunchConfiguration', |
| 'JobServerGroup': 'OS::Heat::InstanceGroup'} |
| self.assertEqual(initial_resources, |
| self.list_resources(stack_identifier)) |
| |
| nested_ident = self.assert_resource_is_a_stack(stack_identifier, |
| 'JobServerGroup') |
| self._assert_instance_state(nested_ident, 0, 2) |
| |
| def test_update_instance_error_causes_group_error(self): |
| """If a resource in an instance group fails to be created during an |
| update, the instance group itself will fail and the broken inner |
| resource will remain. |
| """ |
| files = {'provider.yaml': self.instance_template} |
| env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'}, |
| 'parameters': {'size': 2, |
| 'image': self.conf.image_ref, |
| 'flavor': self.conf.instance_type}} |
| |
| stack_identifier = self.stack_create(template=self.template, |
| files=files, |
| environment=env) |
| initial_resources = { |
| 'JobServerConfig': 'AWS::AutoScaling::LaunchConfiguration', |
| 'JobServerGroup': 'OS::Heat::InstanceGroup'} |
| self.assertEqual(initial_resources, |
| self.list_resources(stack_identifier)) |
| |
| stack = self.client.stacks.get(stack_identifier) |
| self.assert_instance_count(stack, 2) |
| nested_ident = self.assert_resource_is_a_stack(stack_identifier, |
| 'JobServerGroup') |
| self._assert_instance_state(nested_ident, 2, 0) |
| initial_list = [res.resource_name |
| for res in self.client.resources.list(nested_ident)] |
| |
| env['parameters']['size'] = 3 |
| files2 = {'provider.yaml': self.bad_instance_template} |
| self.client.stacks.update( |
| stack_id=stack_identifier, |
| template=self.template, |
| files=files2, |
| disable_rollback=True, |
| parameters={}, |
| environment=env |
| ) |
| self._wait_for_stack_status(stack_identifier, 'UPDATE_FAILED') |
| |
| nested_ident = self.assert_resource_is_a_stack(stack_identifier, |
| 'JobServerGroup') |
| # assert that there are 3 bad instances |
| # 2 resources should be in update failed, and one create failed. |
| for res in self.client.resources.list(nested_ident): |
| if res.resource_name in initial_list: |
| self._wait_for_resource_status(nested_ident, |
| res.resource_name, |
| 'UPDATE_FAILED') |
| else: |
| self._wait_for_resource_status(nested_ident, |
| res.resource_name, |
| 'CREATE_FAILED') |
| |
| |
| class InstanceGroupUpdatePolicyTest(InstanceGroupTest): |
| |
| def ig_tmpl_with_updt_policy(self): |
| templ = json.loads(copy.deepcopy(self.template)) |
| up = {"RollingUpdate": { |
| "MinInstancesInService": "1", |
| "MaxBatchSize": "2", |
| "PauseTime": "PT1S"}} |
| templ['Resources']['JobServerGroup']['UpdatePolicy'] = up |
| return templ |
| |
| def update_instance_group(self, updt_template, |
| num_updates_expected_on_updt, |
| num_creates_expected_on_updt, |
| num_deletes_expected_on_updt, |
| update_replace): |
| |
| # setup stack from the initial template |
| files = {'provider.yaml': self.instance_template} |
| size = 5 |
| env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'}, |
| 'parameters': {'size': size, |
| 'image': self.conf.image_ref, |
| 'flavor': self.conf.instance_type}} |
| stack_name = self._stack_rand_name() |
| stack_identifier = self.stack_create( |
| stack_name=stack_name, |
| template=self.ig_tmpl_with_updt_policy(), |
| files=files, |
| environment=env) |
| stack = self.client.stacks.get(stack_identifier) |
| nested_ident = self.assert_resource_is_a_stack(stack_identifier, |
| 'JobServerGroup') |
| |
| # test that physical resource name of launch configuration is used |
| conf_name = self._stack_output(stack, 'JobServerConfigRef') |
| conf_name_pattern = '%s-JobServerConfig-[a-zA-Z0-9]+$' % stack_name |
| self.assertThat(conf_name, |
| matchers.MatchesRegex(conf_name_pattern)) |
| |
| # test the number of instances created |
| self.assert_instance_count(stack, size) |
| # saves info from initial list of instances for comparison later |
| init_instances = self.client.resources.list(nested_ident) |
| init_names = [inst.resource_name for inst in init_instances] |
| |
| # test stack update |
| self.update_stack(stack_identifier, updt_template, |
| environment=env, files=files) |
| updt_stack = self.client.stacks.get(stack_identifier) |
| |
| # test that the launch configuration is replaced |
| updt_conf_name = self._stack_output(updt_stack, 'JobServerConfigRef') |
| self.assertThat(updt_conf_name, |
| matchers.MatchesRegex(conf_name_pattern)) |
| self.assertNotEqual(conf_name, updt_conf_name) |
| |
| # test that the group size are the same |
| updt_instances = self.client.resources.list(nested_ident) |
| updt_names = [inst.resource_name for inst in updt_instances] |
| self.assertEqual(len(init_names), len(updt_names)) |
| for res in updt_instances: |
| self.assertEqual('UPDATE_COMPLETE', res.resource_status) |
| |
| # test that the appropriate number of instance names are the same |
| matched_names = set(updt_names) & set(init_names) |
| self.assertEqual(num_updates_expected_on_updt, len(matched_names)) |
| |
| # test that the appropriate number of new instances are created |
| self.assertEqual(num_creates_expected_on_updt, |
| len(set(updt_names) - set(init_names))) |
| |
| # test that the appropriate number of instances are deleted |
| self.assertEqual(num_deletes_expected_on_updt, |
| len(set(init_names) - set(updt_names))) |
| |
| # test that the older instances are the ones being deleted |
| if num_deletes_expected_on_updt > 0: |
| deletes_expected = init_names[:num_deletes_expected_on_updt] |
| self.assertNotIn(deletes_expected, updt_names) |
| |
| def test_instance_group_update_replace(self): |
| """ |
| Test simple update replace with no conflict in batch size and |
| minimum instances in service. |
| """ |
| updt_template = self.ig_tmpl_with_updt_policy() |
| grp = updt_template['Resources']['JobServerGroup'] |
| policy = grp['UpdatePolicy']['RollingUpdate'] |
| policy['MinInstancesInService'] = '1' |
| policy['MaxBatchSize'] = '3' |
| config = updt_template['Resources']['JobServerConfig'] |
| config['Properties']['ImageId'] = self.conf.minimal_image_ref |
| |
| self.update_instance_group(updt_template, |
| num_updates_expected_on_updt=5, |
| num_creates_expected_on_updt=0, |
| num_deletes_expected_on_updt=0, |
| update_replace=True) |
| |
| def test_instance_group_update_replace_with_adjusted_capacity(self): |
| """ |
| Test update replace with capacity adjustment due to conflict in |
| batch size and minimum instances in service. |
| """ |
| updt_template = self.ig_tmpl_with_updt_policy() |
| grp = updt_template['Resources']['JobServerGroup'] |
| policy = grp['UpdatePolicy']['RollingUpdate'] |
| policy['MinInstancesInService'] = '4' |
| policy['MaxBatchSize'] = '4' |
| config = updt_template['Resources']['JobServerConfig'] |
| config['Properties']['ImageId'] = self.conf.minimal_image_ref |
| |
| self.update_instance_group(updt_template, |
| num_updates_expected_on_updt=2, |
| num_creates_expected_on_updt=3, |
| num_deletes_expected_on_updt=3, |
| update_replace=True) |
| |
| def test_instance_group_update_replace_huge_batch_size(self): |
| """ |
| Test update replace with a huge batch size. |
| """ |
| updt_template = self.ig_tmpl_with_updt_policy() |
| group = updt_template['Resources']['JobServerGroup'] |
| policy = group['UpdatePolicy']['RollingUpdate'] |
| policy['MinInstancesInService'] = '0' |
| policy['MaxBatchSize'] = '20' |
| config = updt_template['Resources']['JobServerConfig'] |
| config['Properties']['ImageId'] = self.conf.minimal_image_ref |
| |
| self.update_instance_group(updt_template, |
| num_updates_expected_on_updt=5, |
| num_creates_expected_on_updt=0, |
| num_deletes_expected_on_updt=0, |
| update_replace=True) |
| |
| def test_instance_group_update_replace_huge_min_in_service(self): |
| """ |
| Test update replace with a huge number of minimum instances in service. |
| """ |
| updt_template = self.ig_tmpl_with_updt_policy() |
| group = updt_template['Resources']['JobServerGroup'] |
| policy = group['UpdatePolicy']['RollingUpdate'] |
| policy['MinInstancesInService'] = '20' |
| policy['MaxBatchSize'] = '2' |
| policy['PauseTime'] = 'PT0S' |
| config = updt_template['Resources']['JobServerConfig'] |
| config['Properties']['ImageId'] = self.conf.minimal_image_ref |
| |
| self.update_instance_group(updt_template, |
| num_updates_expected_on_updt=3, |
| num_creates_expected_on_updt=2, |
| num_deletes_expected_on_updt=2, |
| update_replace=True) |
| |
| def test_instance_group_update_no_replace(self): |
| """ |
| Test simple update only and no replace (i.e. updated instance flavor |
| in Launch Configuration) with no conflict in batch size and |
| minimum instances in service. |
| """ |
| updt_template = self.ig_tmpl_with_updt_policy() |
| group = updt_template['Resources']['JobServerGroup'] |
| policy = group['UpdatePolicy']['RollingUpdate'] |
| policy['MinInstancesInService'] = '1' |
| policy['MaxBatchSize'] = '3' |
| policy['PauseTime'] = 'PT0S' |
| config = updt_template['Resources']['JobServerConfig'] |
| config['Properties']['InstanceType'] = 'm1.tiny' |
| |
| self.update_instance_group(updt_template, |
| num_updates_expected_on_updt=5, |
| num_creates_expected_on_updt=0, |
| num_deletes_expected_on_updt=0, |
| update_replace=False) |
| |
| def test_instance_group_update_no_replace_with_adjusted_capacity(self): |
| """ |
| Test update only and no replace (i.e. updated instance flavor in |
| Launch Configuration) with capacity adjustment due to conflict in |
| batch size and minimum instances in service. |
| """ |
| updt_template = self.ig_tmpl_with_updt_policy() |
| group = updt_template['Resources']['JobServerGroup'] |
| policy = group['UpdatePolicy']['RollingUpdate'] |
| policy['MinInstancesInService'] = '4' |
| policy['MaxBatchSize'] = '4' |
| policy['PauseTime'] = 'PT0S' |
| config = updt_template['Resources']['JobServerConfig'] |
| config['Properties']['InstanceType'] = 'm1.tiny' |
| |
| self.update_instance_group(updt_template, |
| num_updates_expected_on_updt=2, |
| num_creates_expected_on_updt=3, |
| num_deletes_expected_on_updt=3, |
| update_replace=False) |