Add ubuntu18
* Info about Ubiquity https://wiki.ubuntu.com/UbiquityAutomation
- we do exactly partitioning=> no possibility for now, to use
new installer type.
* Move growlvm.py to commons
Change-Id: Iabda3770b368a48919fef89d3a78cae388f60168
Prod-related: PROD-28155 (PROD:28155)
diff --git a/common/files/scripts/growlvm.py b/common/files/scripts/growlvm.py
new file mode 100755
index 0000000..ae2ebd3
--- /dev/null
+++ b/common/files/scripts/growlvm.py
@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+#
+# Copyright 2018 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+"""Growpart python module
+
+The module is aimed to extend logical volumes according to sizes provided
+in image layout.
+
+Example:
+ python growlvm --image-layout-file '/root/mylayout.yml'
+
+Attributes:
+ image-layout - json string with image layout. Supported params and
+ description might be found in IMAGE_LAYOUT_SCHEMA
+
+"""
+
+__version__ = '1.0'
+
+import argparse
+import collections
+import yaml
+from jsonschema import validate
+import logging
+import os
+import re
+import subprocess
+import sys
+
+
+LOG = logging.getLogger(__name__)
+
+DECIMAL_REG = re.compile(r"(\d+)")
+
+IMAGE_LAYOUT_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "title": "Image partition layout",
+ "type": "object",
+ "patternProperties": {
+ ".*": {"$ref": "#/definitions/logical_volume_layout"}
+ },
+ "definitions": {
+ "logical_volume_layout": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "description": "Logical Volume Name",
+ "type": "string"
+ },
+ "size": {
+ "description": (
+ "Size of Logical volume in units of logical extents. "
+ "The number might be volume size in units of "
+ "megabytes. A size suffix of M for megabytes, G for "
+ "gigabytes, T for terabytes, P for petabytes or E for "
+ "exabytes is optional. The number can also be "
+ "expressed as a percentage of the total space in the "
+ "Volume Group with the suffix %VG. Percentage of the "
+ "changeble values like free space is not supported."
+ ),
+ },
+ "resizefs": {
+ "description": (
+ "Resize underlying filesystem together with the "
+ "logical volume using fsadm(8)."
+ ),
+ "type": "boolean"
+ },
+ "vg": {
+ "description": ("Volume group name to resize logical "
+ "volume on."),
+ "type": "string"
+ }
+ },
+ "additionalProperties": False,
+ "required": ["size"]
+ }
+ },
+}
+
+
+def get_volume_groups_info(unit, vg):
+ cmd = ("vgs --noheadings -o vg_name,size,free,vg_extent_size --units %s "
+ "--separator ';' %s") % (unit, vg)
+ try:
+ output = subprocess.check_output(cmd, shell=True,
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as exc:
+ raise Exception("Failed to get volume group info", exc.output)
+
+ vgs = []
+ for line in output.splitlines():
+ parts = line.strip().split(';')
+ vgs.append({
+ 'name': parts[0],
+ 'size': int(DECIMAL_REG.match(parts[1]).group(1)),
+ 'free': int(DECIMAL_REG.match(parts[2]).group(1)),
+ 'ext_size': int(DECIMAL_REG.match(parts[3]).group(1))
+ })
+ return vgs
+
+
+def get_logical_volume_info(unit, vg):
+ cmd = ("lvs -a --noheadings --nosuffix -o lv_name,size,lv_attr --units %s "
+ "--separator ';' %s") % (unit, vg)
+ try:
+ output = subprocess.check_output(cmd, shell=True,
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as exc:
+ raise Exception("Failed to get volume info", exc.output)
+
+ lvs = []
+
+ for line in output.splitlines():
+ parts = line.strip().split(';')
+ lvs.append({
+ 'name': parts[0].replace('[', '').replace(']', ''),
+ 'size': int(DECIMAL_REG.match(parts[1]).group(1)),
+ })
+ return lvs
+
+
+def normalize_to_pe(size, pe):
+ """ Make sure requested size is multiply of PE
+
+ PE is gathered from system with honor of provided units,
+ when volume size is set in Gigabytes, PE (4mb default) will
+ be shown as 0.
+ """
+
+ if pe > 0:
+ return (size // pe + 1) * pe
+
+ return size
+
+
+def main():
+ logging.basicConfig(
+ format='%(asctime)s - %(levelname)s - %(message)s'
+ )
+ LOG.setLevel(logging.INFO)
+
+ parser = argparse.ArgumentParser(
+ description=('Grow lvm partitions and theirs filesystems to '
+ 'specified sizes.')
+ )
+
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument(
+ '--image-layout',
+ help='json based image layout',
+ )
+ group.add_argument(
+ '--image-layout-file',
+ help='Path to file with image layout',
+ )
+ args = parser.parse_args()
+
+ if args.image_layout_file:
+ with open(args.image_layout_file) as f:
+ layout_info = yaml.load(f)
+ else:
+ layout_info = yaml.load(args.image_layout)
+
+ LOG.info("Validating provided layout.")
+ try:
+ validate(layout_info, IMAGE_LAYOUT_SCHEMA)
+ except Exception as e:
+ LOG.error("Validation of provided layout failed.")
+ raise e
+
+ for part_name, part_layout in layout_info.iteritems():
+
+ size_opt = '--size'
+ size_unit = 'm'
+
+ size = part_layout['size']
+ vg = part_layout.get('vg', 'vg0')
+ resizefs = part_layout.get('resizefs', True)
+
+ if size:
+ if '+' in size:
+ raise Exception("Setting relative size is not supported.")
+ # LVCREATE(8) -l --extents option with percentage
+ elif '%' in size:
+ size_parts = size.split('%', 1)
+ size_percent = int(size_parts[0])
+ if size_percent > 100:
+ raise Exception(
+ "Size percentage cannot be larger than 100%")
+ size_whole = size_parts[1]
+ if size_whole == 'ORIGIN':
+ raise Exception("Snapshot Volumes are not supported")
+ elif size_whole not in ['VG']:
+ raise Exception("Relative sizes are not supported.")
+ size_opt = '--extents'
+ size_unit = ''
+ else:
+ # LVCREATE(8) -L --size option unit
+ if size[-1].lower() in 'bskmgtpe':
+ size_unit = size[-1].lower()
+ size = size[0:-1]
+
+ # when no unit, megabytes by default
+ if size_opt == '--extents':
+ unit = 'm'
+ else:
+ unit = size_unit
+
+ vgs = get_volume_groups_info(unit, vg)
+ this_volume_group = vgs[0]
+ pe = this_volume_group['ext_size']
+
+ lvs = get_logical_volume_info(unit, vg)
+
+ LOG.info("Volume group info: %s", vgs)
+ LOG.info("Logical Volume info: %s", lvs)
+
+ this_volume = [v for v in lvs if v['name'] == part_name][0]
+
+ LOG.info("Using %s for volume: %s", size_opt, this_volume)
+ if size_opt == '--extents':
+ size_free = this_volume_group['free']
+ if size_whole == 'VG':
+ size_requested = normalize_to_pe(
+ size_percent * this_volume_group['size'] / 100, pe)
+
+ LOG.info("Request %s size for volume %s",
+ size_requested, this_volume)
+ if this_volume['size'] > size_requested:
+ raise Exception("Redusing volume size in not supported.")
+ elif this_volume['size'] < size_requested:
+ if (size_free > 0) and (('+' not in size) or
+ (size_free >= (size_requested - this_volume['size']))):
+ cmd = "lvextend "
+ else:
+ raise Exception(
+ ("Logical Volume %s could not be extended. Not "
+ "enough free space left (%s%s required / %s%s "
+ "available)"),
+ this_volume['name'],
+ size_requested - this_volume['size'],
+ unit, size_free, unit
+ )
+ if resizefs:
+ cmd += "--resizefs "
+
+ cmd = "%s -%s %s%s %s/%s" % (
+ cmd, size_opt, size, size_unit, vg, this_volume['name'])
+ try:
+ LOG.debug("Executing command: %s", cmd)
+ output = subprocess.check_output(
+ cmd,
+ shell=True,
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as exc:
+ raise Exception(
+ "Failed to resize volume %s. Exception: %s" % (
+ part_name, exc.output))
+ else:
+ LOG.info("No need to resize volume %s.", part_name)
+ else:
+ cmd = "lvresize "
+ if normalize_to_pe(int(size), pe) > this_volume['size']:
+ if resizefs:
+ cmd += "--resizefs "
+ cmd = "%s -%s %s%s %s/%s" % (
+ cmd, size_opt, size, size_unit, vg, this_volume['name'])
+ try:
+ LOG.debug("Executing command: %s", cmd)
+ output = subprocess.check_output(
+ cmd,
+ shell=True,
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as exc:
+ raise Exception(
+ "Failed to resize volume %s. Exception: %s" % (
+ part_name, exc.output))
+
+ elif normalize_to_pe(int(size), pe) == this_volume['size']:
+ LOG.info("No need to resize volume %s.", part_name)
+ else:
+ raise Exception(
+ "Redusing size in not supported for provided layout.")
+
+
+if __name__ == '__main__':
+ try:
+ main()
+ except Exception as e:
+ LOG.exception("Failed to apply image layout: %s", e)
+ sys.exit(1)
diff --git a/common/ubuntu_bionic_base.sh b/common/ubuntu_bionic_base.sh
new file mode 100644
index 0000000..4aa98c6
--- /dev/null
+++ b/common/ubuntu_bionic_base.sh
@@ -0,0 +1,73 @@
+#!/bin/bash -xe
+
+# Don't use /tmp/ - some templates do node reboot
+if [ -f /done_ubuntu_base ] ; then
+ echo "INFO: ubuntu_base already finished.Skipping.."
+ exit 0
+fi
+#
+UBUNTU_BASEURL="${UBUNTU_BASEURL:-mirror://mirrors.ubuntu.com/mirrors.txt}"
+## Base packages and setup
+export DEBIAN_FRONTEND=noninteractive
+echo -e '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d
+chmod +x /usr/sbin/policy-rc.d
+# Configure apt. Please refer to
+# https://github.com/Mirantis/reclass-system-salt-model/blob/master/linux/system/single/debian.yml
+# and keep those structures with same naming convention - to prevent
+# misconfiguration between base system and salt state.
+echo "Acquire::CompressionTypes::Order gz;" >/etc/apt/apt.conf.d/99compression-workaround-salt
+echo "Acquire::EnableSrvRecords false;" >/etc/apt/apt.conf.d/99enablesrvrecords-false
+echo "Acquire::http::Pipeline-Depth 0;" > /etc/apt/apt.conf.d/99aws-s3-mirrors-workaround-salt
+echo "APT::Install-Recommends false;" > /etc/apt/apt.conf.d/99dont_install_recommends-salt
+echo "APT::Install-Suggests false;" > /etc/apt/apt.conf.d/99dont_install_suggests-salt
+echo "Acquire::Languages none;" > /etc/apt/apt.conf.d/99dont_acquire_all_languages-salt
+echo "APT::Periodic::Update-Package-Lists 0;" > /etc/apt/apt.conf.d/99dont_update_package_list-salt
+echo "APT::Periodic::Download-Upgradeable-Packages 0;" > /etc/apt/apt.conf.d/99dont_update_download_upg_packages-salt
+echo "APT::Periodic::Unattended-Upgrade 0;" > /etc/apt/apt.conf.d/99disable_unattended_upgrade-salt
+
+sysctl -w fs.file-max=100000
+# Overwrite default mirrors
+echo "deb [arch=amd64] ${UBUNTU_BASEURL} bionic main restricted universe" > /etc/apt/sources.list
+echo "deb [arch=amd64] ${UBUNTU_BASEURL} bionic-updates main restricted universe" >> /etc/apt/sources.list
+echo "deb [arch=amd64] ${UBUNTU_BASEURL} bionic-security main restricted universe" >> /etc/apt/sources.list
+#echo "deb [arch=amd64] ${UBUNTU_BASEURL} bionic-backports main restricted universe" >> /etc/apt/sources.list
+
+apt-get clean
+apt-get update
+
+# Useful tools
+EXTRA_PKGS="byobu curl ethtool iputils-ping lsof strace tcpdump traceroute wget iptables"
+# Pretty tools
+EXTRA_PKGS="${EXTRA_PKGS} byobu htop tmux tree vim-nox mc eatmydata"
+# Common prerequisites
+# growlvm.py dependencies
+GROWLVM_PKGS="python3-jsonschema python3-yaml"
+EXTRA_PKGS="$EXTRA_PKGS $GROWLVM_PKGS gnupg2 apt-transport-https libmnl0 python3-apt python3-psutil acpid virt-what dbus bridge-utils vlan ifenslave"
+apt-get -y install ${EXTRA_PKGS}
+
+# Cleanup old kernels, ensure latest is installed via metapackage package
+if [ ! -f /tmp/no_install_kernel ]; then
+ apt-get purge -y linux-image-* linux-headers-* | grep -v 'is not installed, so not removed'
+ #apt-get install -y linux-image-virtual-hwe-18.04 linux-image-extra-virtual-hwe-18.04 linux-headers-generic-hwe-18.04
+ apt-get install -y linux-image-virtual linux-image-extra-virtual linux-headers-generic
+ # Update grub cmdline
+ sed -i 's|GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT="console=tty0 console=ttyS0,115200n8"|g' /etc/default/grub
+ sed -i 's|GRUB_CMDLINE_LINUX=.*|GRUB_CMDLINE_LINUX="console=tty0 console=ttyS0,115200n8"|g' /etc/default/grub
+ update-grub
+fi
+
+apt-get -y upgrade
+apt-get -y dist-upgrade
+
+# Setup cloud-init
+apt-get -y install cloud-init
+
+# FIXME: move to cluster model
+# Disable services
+disable_services="apt-daily.timer apt-daily-upgrade.timer lxc.service snapd.service snapd.socket open-iscsi.service tgt.service iscsid.service"
+for s in ${disable_services}; do
+ systemctl disable ${s} || true
+ systemctl stop ${s} || true
+done
+
+touch /done_ubuntu_base
diff --git a/common/ubuntu_bionic_cleanup.sh b/common/ubuntu_bionic_cleanup.sh
new file mode 100644
index 0000000..e5db691
--- /dev/null
+++ b/common/ubuntu_bionic_cleanup.sh
@@ -0,0 +1,49 @@
+#!/bin/bash -xe
+
+apt-get -y remove --purge unattended-upgrades || true
+apt-get -y autoremove --purge
+apt-get -y clean
+
+rm -rf /var/lib/apt/lists/* || true
+rm -rv /etc/apt/sources.list.d/* || true
+rm -rv /etc/apt/preferences.d/* || true
+echo > /etc/apt/sources.list || true
+rm -vf /usr/sbin/policy-rc.d || true
+
+echo "cleaning up hostname"
+sed -i "/.*ubuntu.*/d" /etc/hosts
+sed -i "/.*salt.*/d" /etc/hosts
+
+echo "cleaning up dhcp leases"
+rm -rf /var/lib/dhcp/* || true
+rm -rfv /var/lib/ntp/ntp.conf.dhcp || true
+
+echo "cleaning up udev rules"
+rm -fv /etc/udev/rules.d/70-persistent-net.rules || true
+rm -rf /dev/.udev/ || true
+rm -fv /lib/udev/rules.d/75-persistent-net-generator.rules || true
+
+echo "cleaning up minion_id for salt"
+rm -vf /etc/salt/minion_id || true
+
+echo "cleaning up /var/cache/{apt,salt}/*"
+rm -rf /var/cache/{apt,salt}/* || true
+
+rm -rf /root/.cache || true
+rm -rf /root/.ssh/known_hosts || true
+
+# Remove flags
+rm -v /done_ubuntu_base || true
+rm -v /done_ubuntu_salt_bootstrap || true
+
+# Force cleanup cloud-init data, if it was
+if [[ -d '/var/lib/cloud/' ]] ; then
+ rm -rf /var/lib/cloud/* || true
+ cloud-init clean || true
+ echo > /var/log/cloud-init-output.log || true
+ echo > /var/log/cloud-init.log || true
+fi
+
+# Clear\drop cache's
+sync
+echo 3 > /proc/sys/vm/drop_caches