Merge "Fix symlink to bootstrap.saltstack.com.sh"
diff --git a/common/ubuntu_base.sh b/common/ubuntu_base.sh
index fbb475e..383e33d 100644
--- a/common/ubuntu_base.sh
+++ b/common/ubuntu_base.sh
@@ -41,7 +41,9 @@
# Pretty tools
EXTRA_PKGS="${EXTRA_PKGS} byobu htop tmux tree vim-nox mc"
# Common prerequisites
-EXTRA_PKGS="${EXTRA_PKGS} apt-transport-https libmnl0 python-apt python-m2crypto python-psutil acpid virt-what"
+# growlvm.py dependencies
+GROWLVM_PKGS="python-jsonschema python-yaml"
+EXTRA_PKGS="${EXTRA_PKGS} apt-transport-https libmnl0 python-apt python-m2crypto python-psutil acpid virt-what dbus $GROWLVM_PKGS"
apt-get -y install ${EXTRA_PKGS}
# Cleanup old kernels, ensure latest is installed via metapackage package
diff --git a/common/ubuntu_security.sh b/common/ubuntu_security.sh
index f2641c3..03bb54b 100644
--- a/common/ubuntu_security.sh
+++ b/common/ubuntu_security.sh
@@ -42,7 +42,7 @@
usermod -p '!' root
# Drop default 'ubuntu' user
-userdel -rf ubuntu
+userdel -rf ubuntu || true
# Disable SSH password authentication and permit root login
sed -i 's|[#]*PasswordAuthentication yes|PasswordAuthentication no|g' /etc/ssh/sshd_config
diff --git a/ubuntu-16.04/files/scripts/growlvm.py b/ubuntu-16.04/files/scripts/growlvm.py
new file mode 100755
index 0000000..ae2ebd3
--- /dev/null
+++ b/ubuntu-16.04/files/scripts/growlvm.py
@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+#
+# Copyright 2018 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+"""Growpart python module
+
+The module is aimed to extend logical volumes according to sizes provided
+in image layout.
+
+Example:
+ python growlvm --image-layout-file '/root/mylayout.yml'
+
+Attributes:
+ image-layout - json string with image layout. Supported params and
+ description might be found in IMAGE_LAYOUT_SCHEMA
+
+"""
+
+__version__ = '1.0'
+
+import argparse
+import collections
+import yaml
+from jsonschema import validate
+import logging
+import os
+import re
+import subprocess
+import sys
+
+
+LOG = logging.getLogger(__name__)
+
+DECIMAL_REG = re.compile(r"(\d+)")
+
+IMAGE_LAYOUT_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "title": "Image partition layout",
+ "type": "object",
+ "patternProperties": {
+ ".*": {"$ref": "#/definitions/logical_volume_layout"}
+ },
+ "definitions": {
+ "logical_volume_layout": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "description": "Logical Volume Name",
+ "type": "string"
+ },
+ "size": {
+ "description": (
+ "Size of Logical volume in units of logical extents. "
+ "The number might be volume size in units of "
+ "megabytes. A size suffix of M for megabytes, G for "
+ "gigabytes, T for terabytes, P for petabytes or E for "
+ "exabytes is optional. The number can also be "
+ "expressed as a percentage of the total space in the "
+ "Volume Group with the suffix %VG. Percentage of the "
+ "changeble values like free space is not supported."
+ ),
+ },
+ "resizefs": {
+ "description": (
+ "Resize underlying filesystem together with the "
+ "logical volume using fsadm(8)."
+ ),
+ "type": "boolean"
+ },
+ "vg": {
+ "description": ("Volume group name to resize logical "
+ "volume on."),
+ "type": "string"
+ }
+ },
+ "additionalProperties": False,
+ "required": ["size"]
+ }
+ },
+}
+
+
+def get_volume_groups_info(unit, vg):
+ cmd = ("vgs --noheadings -o vg_name,size,free,vg_extent_size --units %s "
+ "--separator ';' %s") % (unit, vg)
+ try:
+ output = subprocess.check_output(cmd, shell=True,
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as exc:
+ raise Exception("Failed to get volume group info", exc.output)
+
+ vgs = []
+ for line in output.splitlines():
+ parts = line.strip().split(';')
+ vgs.append({
+ 'name': parts[0],
+ 'size': int(DECIMAL_REG.match(parts[1]).group(1)),
+ 'free': int(DECIMAL_REG.match(parts[2]).group(1)),
+ 'ext_size': int(DECIMAL_REG.match(parts[3]).group(1))
+ })
+ return vgs
+
+
+def get_logical_volume_info(unit, vg):
+ cmd = ("lvs -a --noheadings --nosuffix -o lv_name,size,lv_attr --units %s "
+ "--separator ';' %s") % (unit, vg)
+ try:
+ output = subprocess.check_output(cmd, shell=True,
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as exc:
+ raise Exception("Failed to get volume info", exc.output)
+
+ lvs = []
+
+ for line in output.splitlines():
+ parts = line.strip().split(';')
+ lvs.append({
+ 'name': parts[0].replace('[', '').replace(']', ''),
+ 'size': int(DECIMAL_REG.match(parts[1]).group(1)),
+ })
+ return lvs
+
+
+def normalize_to_pe(size, pe):
+ """ Make sure requested size is multiply of PE
+
+ PE is gathered from system with honor of provided units,
+ when volume size is set in Gigabytes, PE (4mb default) will
+ be shown as 0.
+ """
+
+ if pe > 0:
+ return (size // pe + 1) * pe
+
+ return size
+
+
+def main():
+ logging.basicConfig(
+ format='%(asctime)s - %(levelname)s - %(message)s'
+ )
+ LOG.setLevel(logging.INFO)
+
+ parser = argparse.ArgumentParser(
+ description=('Grow lvm partitions and theirs filesystems to '
+ 'specified sizes.')
+ )
+
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument(
+ '--image-layout',
+ help='json based image layout',
+ )
+ group.add_argument(
+ '--image-layout-file',
+ help='Path to file with image layout',
+ )
+ args = parser.parse_args()
+
+ if args.image_layout_file:
+ with open(args.image_layout_file) as f:
+ layout_info = yaml.load(f)
+ else:
+ layout_info = yaml.load(args.image_layout)
+
+ LOG.info("Validating provided layout.")
+ try:
+ validate(layout_info, IMAGE_LAYOUT_SCHEMA)
+ except Exception as e:
+ LOG.error("Validation of provided layout failed.")
+ raise e
+
+ for part_name, part_layout in layout_info.iteritems():
+
+ size_opt = '--size'
+ size_unit = 'm'
+
+ size = part_layout['size']
+ vg = part_layout.get('vg', 'vg0')
+ resizefs = part_layout.get('resizefs', True)
+
+ if size:
+ if '+' in size:
+ raise Exception("Setting relative size is not supported.")
+ # LVCREATE(8) -l --extents option with percentage
+ elif '%' in size:
+ size_parts = size.split('%', 1)
+ size_percent = int(size_parts[0])
+ if size_percent > 100:
+ raise Exception(
+ "Size percentage cannot be larger than 100%")
+ size_whole = size_parts[1]
+ if size_whole == 'ORIGIN':
+ raise Exception("Snapshot Volumes are not supported")
+ elif size_whole not in ['VG']:
+ raise Exception("Relative sizes are not supported.")
+ size_opt = '--extents'
+ size_unit = ''
+ else:
+ # LVCREATE(8) -L --size option unit
+ if size[-1].lower() in 'bskmgtpe':
+ size_unit = size[-1].lower()
+ size = size[0:-1]
+
+ # when no unit, megabytes by default
+ if size_opt == '--extents':
+ unit = 'm'
+ else:
+ unit = size_unit
+
+ vgs = get_volume_groups_info(unit, vg)
+ this_volume_group = vgs[0]
+ pe = this_volume_group['ext_size']
+
+ lvs = get_logical_volume_info(unit, vg)
+
+ LOG.info("Volume group info: %s", vgs)
+ LOG.info("Logical Volume info: %s", lvs)
+
+ this_volume = [v for v in lvs if v['name'] == part_name][0]
+
+ LOG.info("Using %s for volume: %s", size_opt, this_volume)
+ if size_opt == '--extents':
+ size_free = this_volume_group['free']
+ if size_whole == 'VG':
+ size_requested = normalize_to_pe(
+ size_percent * this_volume_group['size'] / 100, pe)
+
+ LOG.info("Request %s size for volume %s",
+ size_requested, this_volume)
+ if this_volume['size'] > size_requested:
+ raise Exception("Redusing volume size in not supported.")
+ elif this_volume['size'] < size_requested:
+ if (size_free > 0) and (('+' not in size) or
+ (size_free >= (size_requested - this_volume['size']))):
+ cmd = "lvextend "
+ else:
+ raise Exception(
+ ("Logical Volume %s could not be extended. Not "
+ "enough free space left (%s%s required / %s%s "
+ "available)"),
+ this_volume['name'],
+ size_requested - this_volume['size'],
+ unit, size_free, unit
+ )
+ if resizefs:
+ cmd += "--resizefs "
+
+ cmd = "%s -%s %s%s %s/%s" % (
+ cmd, size_opt, size, size_unit, vg, this_volume['name'])
+ try:
+ LOG.debug("Executing command: %s", cmd)
+ output = subprocess.check_output(
+ cmd,
+ shell=True,
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as exc:
+ raise Exception(
+ "Failed to resize volume %s. Exception: %s" % (
+ part_name, exc.output))
+ else:
+ LOG.info("No need to resize volume %s.", part_name)
+ else:
+ cmd = "lvresize "
+ if normalize_to_pe(int(size), pe) > this_volume['size']:
+ if resizefs:
+ cmd += "--resizefs "
+ cmd = "%s -%s %s%s %s/%s" % (
+ cmd, size_opt, size, size_unit, vg, this_volume['name'])
+ try:
+ LOG.debug("Executing command: %s", cmd)
+ output = subprocess.check_output(
+ cmd,
+ shell=True,
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as exc:
+ raise Exception(
+ "Failed to resize volume %s. Exception: %s" % (
+ part_name, exc.output))
+
+ elif normalize_to_pe(int(size), pe) == this_volume['size']:
+ LOG.info("No need to resize volume %s.", part_name)
+ else:
+ raise Exception(
+ "Redusing size in not supported for provided layout.")
+
+
+if __name__ == '__main__':
+ try:
+ main()
+ except Exception as e:
+ LOG.exception("Failed to apply image layout: %s", e)
+ sys.exit(1)
diff --git a/ubuntu-16.04/http/preseed.cfg b/ubuntu-16.04/http/preseed.cfg
index 269a2bc..8eaf33e 100644
--- a/ubuntu-16.04/http/preseed.cfg
+++ b/ubuntu-16.04/http/preseed.cfg
@@ -17,23 +17,126 @@
d-i netcfg/get_hostname string unassigned-hostname
d-i partman-auto/disk string /dev/sda /dev/vda
-d-i partman-auto/method string regular
-d-i partman/choose_partition select finish
-d-i partman/confirm boolean true
-d-i partman/confirm_nooverwrite boolean true
-d-i partman/confirm_write_new_label boolean true
-d-i partman-basicfilesystems/no_swap boolean false
-d-i partman-auto/choose_recipe select boot-root
+# Use LVM for partitioning
+d-i partman-auto/method string lvm
-d-i partman-auto/expert_recipe string \
- boot-root :: \
- 512 512 -1 ext4 \
- $primary{ } $bootable{ } \
- method{ format } format{ } \
- use_filesystem{ } filesystem{ ext4 } \
- mountpoint{ / } \
- options/noatime{ noatime } \
- . \
+# GPT
+
+d-i partman-basicfilesystems/choose_label string gpt
+d-i partman-basicfilesystems/default_label string gpt
+d-i partman-partitioning/choose_label string gpt
+d-i partman-partitioning/default_label string gpt
+d-i partman/choose_label string gpt
+d-i partman/default_label string gpt
+
+# GPT
+
+d-i partman-auto-lvm/guided_size string max
+
+# If one of the disks that are going to be automatically partitioned
+# contains an old LVM configuration, the user will normally receive a
+# warning. Preseed this away
+d-i partman-lvm/device_remove_lvm boolean true
+
+# And the same goes for the confirmation to write the lvm partitions.
+d-i partman-lvm/confirm boolean true
+
+# Really, please don't prompt me!
+d-i partman-lvm/confirm_nooverwrite boolean true
+
+d-i partman-auto/expert_recipe string \
+ vcp_lvm :: \
+ 1 1 1 free \
+ $gptonly{ } \
+ $primary{ } \
+ $bios_boot{ } \
+ method{ biosgrub } \
+ . \
+ 1051 2 1051 ext3 \
+ $gptonly{ } \
+ $primary{ } \
+ method{ format } format{ } \
+ use_filesystem{ } filesystem{ ext3 } \
+ lable{ lable-boot } \
+ mountpoint{ /boot } \
+ . \
+ 10 10 10 ext4 \
+ $defaultignore{ } \
+ $gptonly{ } \
+ $primary{ } \
+ method{ lvm } \
+ vg_name{ vg0 } \
+ . \
+ 105 2 105 ext4 \
+ $lvmok{ } \
+ in_vg{ vg0 } \
+ lv_name{ home } \
+ method{ lvm } format{ } \
+ use_filesystem{ } filesystem{ ext4 } \
+ mountpoint{ /home } \
+ options/nodev{ nodev } \
+ . \
+ 105 2 105 ext4 ext4 \
+ $lvmok{ } \
+ in_vg{ vg0 } \
+ lv_name{ tmp } \
+ method{ lvm } format{ } \
+ use_filesystem{ } filesystem{ ext4 } \
+ mountpoint{ /tmp } \
+ options/nodev{ nodev } \
+ options/nosuid{ nosuid } \
+ options/noexec{ noexec } \
+ . \
+ 105 2 105 ext4 \
+ $lvmok{ } \
+ in_vg{ vg0 } \
+ lv_name{ var_log } \
+ method{ format } format{ } \
+ use_filesystem{ } filesystem{ ext4 } \
+ mountpoint{ /var/log } \
+ options/noexec{ noexec } \
+ . \
+ 315 2 315 ext4 ext4 \
+ $lvmok{ } \
+ in_vg{ vg0 } \
+ lv_name{ var_tmp } \
+ method{ format } format{ } \
+ use_filesystem{ } filesystem{ ext4 } \
+ mountpoint{ /var/tmp } \
+ options/nodev{ nodev } \
+ options/nosuid{ nosuid } \
+ options/noexec{ noexec } \
+ . \
+ 105 2 105 ext4 ext4 \
+ $lvmok{ } \
+ in_vg{ vg0 } \
+ lv_name{ var_log_audit } \
+ method{ format } format{ } \
+ use_filesystem{ } filesystem{ ext4 } \
+ mountpoint{ /var/log/audit } \
+ options/noexec{ noexec } \
+ . \
+ 2154 2 65536 ext4 \
+ $lvmok{ } \
+ method{ format } format{ } \
+ use_filesystem{ } filesystem{ ext4 } \
+ mountpoint{ / } \
+ in_vg{ vg0 } \
+ lv_name{ root } \
+ .
+
+d-i partman-auto/choose_recipe select vcp_lvm
+
+partman-basicfilesystems partman-basicfilesystems/no_swap boolean false
+
+# This makes partman automatically partition without confirmation, provided
+# that you told it what to do using one of the methods above.
+d-i partman-partitioning/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+
+
d-i pkgsel/include string openssh-server
d-i pkgsel/install-language-support boolean false
diff --git a/ubuntu-16.04/template.json b/ubuntu-16.04/template.json
index 1cfc211..3ac2970 100644
--- a/ubuntu-16.04/template.json
+++ b/ubuntu-16.04/template.json
@@ -2,7 +2,7 @@
"variables": {
"user": "root",
"password": "ho5uo7Uome5d",
- "disk_size": "8000",
+ "disk_size": "5000",
"images_cache": "{{ env `PACKER_IMAGES_CACHE` }}",
"ubuntu_baseurl": "{{ env `UBUNTU_BASEURL` }}",
"saltstack_gpg": "{{ env `SALTSTACK_GPG` }}",
@@ -101,6 +101,11 @@
"type": "file",
"source": "files/etc/",
"destination": "/etc"
+ },
+ {
+ "type": "file",
+ "source": "files/scripts/",
+ "destination": "/usr/bin"
}
],
"post-processors": [
@@ -270,35 +275,37 @@
{
"type": "qemu",
"qemuargs": [
- [ "-m", "1024M" ],
- [ "-cdrom", "config-drive/cloudata.iso" ],
+ ["-m", "1024M"],
["-device", "virtio-net,netdev=user.0"],
["-object","rng-random,id=objrng0,filename=/dev/urandom"],
- ["-device", "virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0x10" ]
+ ["-device", "virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0x10"]
],
- "vm_name": "{{ user `image_name` }}-{{ isotime \"200601021504\" }}",
- "output_directory": "images/{{ user `image_name` }}-qemu-{{ isotime \"200601021504\" }}",
- "format": "qcow2",
- "iso_checksum": "c71d27a86a168f28097bc30004b54c1f",
- "iso_checksum_type": "md5",
- "iso_url": "http://releases.ubuntu.com/16.04.5/ubuntu-16.04.5-server-amd64.iso",
- "iso_target_path": "{{ user `images_cache`}}/ubuntu-16.04.5-server-amd64.iso",
- "disk_image": true,
- "disk_compression": true,
+ "vm_name": "{{ user `image_name` }}",
+ "output_directory": "images/{{ user `image_name`}}",
"accelerator": "kvm",
"disk_size": "{{ user `disk_size`}}",
+ "iso_checksum": "24636fd103a2a43c95659f1c3c63718e",
+ "iso_checksum_type": "md5",
+ "iso_url": "http://releases.ubuntu.com/16.04.5/ubuntu-16.04.5-server-amd64.iso",
+ "iso_target_path": "{{ user `images_cache` }}/ubuntu-16.04.5-server-amd64.iso",
+ "http_directory": "http",
"headless": true,
- "ssh_username": "{{ user `user` }}",
- "ssh_password": "{{ user `password` }}",
- "ssh_host_port_min": 7000,
- "ssh_host_port_max": 7050,
- "shutdown_command": "shutdown -P now",
+ "ssh_username": "{{user `user`}}",
+ "ssh_password": "{{user `password`}}",
+ "shutdown_command": "sync; sudo /sbin/shutdown -hP now",
"boot_wait": "2s",
- "ssh_wait_timeout": "360s",
+ "ssh_wait_timeout": "700s",
"boot_command": [
- "<wait50ms>",
- "<esc><wait>",
- "<enter><wait>",
+ "<enter><wait><f6><esc>",
+ "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
+ "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
+ "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
+ "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
+ "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
+ "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
+ "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
+ "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
+ "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
"/install/vmlinuz<wait>",
" auto<wait>",
" console-setup/ask_detect=false<wait>",