Added --type switch for start command

Issue/glitch with kernel panic on first run seems to be fixed on Debian
Bookworm (12). Do not reset VM for that version (and hopefully
upcoming).

README.rst

@@ -2,8 +2,8 @@
 box.py
 ======
 
-Box.py is a simple automation tool meant to run Ubuntu, Fedora or Centos Stream
+Box.py is a simple automation tool meant to run Ubuntu, Fedora, Centos Stream
-cloud images on top of VirtualBox.
+or Debian cloud images on top of VirtualBox.
 
 What it does is simply download official cloud image, set up VM, tweak it up
 and do the initial pre-configuration using generated config drive.
@@ -16,7 +16,7 @@ weird named options for ``vboxmanage`` ;P)
 Requirements
 ------------
 
-- Python >=3.7
+- Python >=3.8
 
   - `pyyaml`_
   - `requests`_
@@ -24,7 +24,7 @@ Requirements
 - Virtualbox (obviously)
 - ``mkisofs`` or ``genisoimage`` command for generating ISO image
 - ``wget`` command for fetching images
-- ``sha256sum`` command for checksum check
+- ``sha256sum`` and ``sha512sum`` commands for checksum check
 - ``qemu-img`` from *qemu-utils* package command for converting between images
   formats
 
@@ -39,7 +39,7 @@ your operating system, or by using virtualenv for Python requirements, i.e.:
 
    $ python -m virtualenv .venv
    $ . .venv/bin/activate
-   (.venv) $ pip install requirements.txt
+   (.venv) $ pip install -r requirements.txt
 
 then you can issue:
 
@@ -55,7 +55,7 @@ or simply link it somewhere in the path:
    $ chmod +x ~/bin/boxpy
 
 and now you can issue some command. For example, to spin up a VM with Ubuntu
-18.04 with one CPU, 2GB of memory and 10GB of disk:
+18.04 with one CPU, 1GB of memory and 6GB of disk:
 
 .. code:: shell-session
 
@@ -91,13 +91,15 @@ use it ad-hoc, or place on your ``.bashrc`` or whatever:
 
 Currently, following commands are available:
 
-- ``list`` - for quickly listing all/running VMs
+- ``completion`` - as described above
-- ``info`` - to get summary about VM
-- ``destroy`` - that is probably obvious one
 - ``create`` - create new VM
+- ``destroy`` - that is probably obvious one
+- ``info`` - to get summary about VM
+- ``list`` - for quickly listing all/running VMs
 - ``rebuild`` - recreate specified VM
 - ``ssh`` - connect to the VM using ssh
-- ``completion`` - as described above
+- ``start`` - stop the running VM
+- ``stop`` - start stopped VM
 
 All of the commands have a range of options, and can be examined by using
 ``--help`` option.

box.py

@@ -18,14 +18,15 @@ import requests
 import yaml
 
 
-__version__ = "1.3"
+__version__ = "1.9"
 
 CACHE_DIR = os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
 CLOUD_IMAGE = "ci.iso"
 FEDORA_RELEASE_MAP = {'32': '1.6', '33': '1.2', '34': '1.2'}
+DEBIAN_CODENAME_MAP = {'12': 'bookworm', '11': 'bullseye', '10': 'buster'}
 TYPE_MAP = {'HardDisk': 'disk', 'DVD': 'dvd', 'Floppy': 'floppy'}
 DISTRO_MAP = {'ubuntu': 'Ubuntu', 'fedora': 'Fedora',
-              'centos': 'Centos Stream'}
+              'centos': 'Centos Stream', 'debian': 'Debian'}
 META_DATA_TPL = string.Template('''\
 instance-id: $instance_id
 local-hostname: $vmhostname
@@ -46,9 +47,9 @@ ssh:
   emit_keys_to_console: false
 boxpy_data:
   cpus: 1
-  disk_size: 10240
+  disk_size: 6144
   key: ~/.ssh/id_rsa
-  memory: 2048
+  memory: 1024
 '''
 COMPLETIONS = {'bash': '''\
 _boxpy() {
@@ -137,12 +138,36 @@ _boxpy() {
                 COMPREPLY=( $(compgen -W "bash" -- ${cur}) )
             fi
             ;;
-        create|rebuild)
+        start)
-            items=(--cpus --disable-nested --disk-size --distro --forwarding
+            items=(--type)
-                --image --key --memory --hostname --port --config --version
-                --type)
             if [[ ${prev} == ${cmd} ]]; then
-                if [[ ${cmd} = "rebuild" ]]; then
+                if [[ ${cmd} = "start" ]]; then
+                    _vms_comp vms
+                else
+                    COMPREPLY=( $(compgen -W "${items[*]}" -- ${cur}) )
+                fi
+            else
+                _get_excluded_items "${items[@]}"
+                COMPREPLY=( $(compgen -W "$result" -- ${cur}) )
+
+                case "${prev}" in
+                    --type)
+                        COMPREPLY=( $(compgen -W "gui headless sdl separate" \
+                            -- ${cur}) )
+                        ;;
+                    --*)
+                        COMPREPLY=( )
+                        ;;
+                esac
+            fi
+
+            ;;
+        create|rebuild)
+            items=(--cpus --disable-nested --disk-size --default-user --distro
+                --forwarding --image --key --memory --hostname --port --config
+                --version --type)
+            if [[ ${prev} == ${cmd} ]]; then
+                if [[ ${cmd} = "rebuild" || ${cmd} == "start" ]]; then
                     _vms_comp vms
                 else
                     COMPREPLY=( $(compgen -W "${items[*]}" -- ${cur}) )
@@ -160,8 +185,8 @@ _boxpy() {
                         _ssh_identityfile
                         ;;
                     --distro)
-                        COMPREPLY=( $(compgen -W "ubuntu fedora centos" \
+                        COMPREPLY=( $(compgen -W "ubuntu fedora centos
-                                -- ${cur}) )
+                            debian" -- ${cur}) )
                         ;;
                     --type)
                         COMPREPLY=( $(compgen -W "gui headless sdl separate" \
@@ -194,11 +219,6 @@ _boxpy() {
                 _vms_comp runningvms
             fi
             ;;
-        start)
-            if [[ ${prev} == ${cmd} ]]; then
-                _vms_comp vms
-            fi
-            ;;
         stop)
             if [[ ${prev} == ${cmd} ]]; then
                 _vms_comp runningvms
@@ -419,6 +439,10 @@ class Config:
                 continue
             setattr(self, attr, str(val))
 
+        # sort out case, where there is image/default-user provided
+        if self.image:
+            self._update_distros_with_custom_image()
+
         # set distribution and version if not specified by user
         if not self.distro:
             self.distro = 'ubuntu'
@@ -552,16 +576,6 @@ class Config:
                 continue
             setattr(self, key, str(val))
 
-        # update distros dict with custom entry if there is at least image
-        if conf.get('boxpy_data') and conf['boxpy_data'].get('image'):
-            custom = {'username': conf['boxpy_data'].get('default_user'),
-                      'realname': 'custom os',
-                      'img_class': CustomImage,
-                      'amd64': 'x86_64',
-                      'image': conf['boxpy_data']['image'],
-                      'default_version': '0'}
-            DISTROS['custom'] = custom
-
         # remove boxpy_data since it will be not needed on the guest side
         if conf.get('boxpy_data'):
             if conf['boxpy_data'].get('advanced'):
@@ -570,6 +584,18 @@ class Config:
 
         self._conf = conf
 
+    def _update_distros_with_custom_image(self):
+        self.image = os.path.abspath(self.image)
+        self.distro = 'custom'
+        if not self.username:
+            self.username = self.default_user
+        DISTROS['custom'] = {'username': self.default_user,
+                             'realname': 'custom os',
+                             'img_class': CustomImage,
+                             'amd64': 'x86_64',
+                             'image': self.image,
+                             'default_version': '0'}
+
     def _update(self, source, update):
         for key, val in update.items():
             if isinstance(val, collections.abc.Mapping):
@@ -579,6 +605,48 @@ class Config:
         return source
 
 
+class OsTypes:
+    def __init__(self, conf):
+        self._conf = conf
+        self._ostypes = []
+
+        self._gather_os_types()
+
+    def _gather_os_types(self):
+        out = Run(['vboxmanage', 'list', 'ostypes']).stdout
+        for line in out.split('\n'):
+            if not line.startswith('ID:'):
+                continue
+            self._ostypes.append(line.split(':')[1].strip())
+
+    def ubuntu(self):
+        lts = ''
+        major, minor = [int(x) for x in self._conf.version.split('.')]
+
+        if major % 2 == 0 and minor == 4:
+            lts = '_LTS'
+        name = "Ubuntu%s%s_64" % (major, lts)
+
+        if name not in self._ostypes:
+            return 'Ubuntu_64'
+
+        return name
+
+    def fedora(self):
+        return "Fedora_64"
+
+    def debian(self):
+        name = "Debian%s_64" % self._conf.version
+        if name not in self._ostypes:
+            return 'Debian_64'
+
+    def get(self):
+        if not hasattr(self, self._conf.distro):
+            return "Linux_64"
+
+        return getattr(self, self._conf.distro)()
+
+
 class VBoxManage:
     """
     Class for dealing with vboxmanage commands
@@ -759,7 +827,11 @@ class VBoxManage:
                '--acpi', 'on',
                '--audio', 'none',
                '--nic1', 'nat',
-               '--natpf1', f'boxpyssh,tcp,,{port},,22']
+               '--natpf1', f'boxpyssh,tcp,,{port},,22',
+               '--graphicscontroller', 'vmsvga',
+               '--vram', '16',
+               '--ostype', OsTypes(conf).get()]
+
         for count, (hostport, vmport) in enumerate(conf.forwarding.items(),
                                                    start=1):
             cmd.extend(['--natpf1', f'custom-pf-{count},tcp,,{hostport},'
@@ -908,6 +980,7 @@ class VBoxManage:
 class Image:
     URL = ""
     IMG = ""
+    CHECKSUMTOOL = 'sha256sum'
 
     def __init__(self, vbox, version, arch, release, fname=None):
         self.vbox = vbox
@@ -958,7 +1031,7 @@ class Image:
             return False
 
         if os.path.exists(os.path.join(CACHE_DIR, self._img_fname)):
-            cmd = ['sha256sum', os.path.join(CACHE_DIR, self._img_fname)]
+            cmd = [self.CHECKSUMTOOL, os.path.join(CACHE_DIR, self._img_fname)]
             calulated_sum = Run(cmd).stdout.split(' ')[0]
             LOG.details('Checksum for image: %s, expected: %s', calulated_sum,
                         expected_sum)
@@ -972,7 +1045,8 @@ class Image:
             return True
 
         fname = os.path.join(CACHE_DIR, self._img_fname)
-        LOG.header('Downloading image %s', self._img_fname)
+        LOG.header('Downloading image %s from %s', self._img_fname,
+                   self._img_url)
         Run(['wget', '-q', self._img_url, '-O', fname])
 
         if not self._checksum():
@@ -1010,6 +1084,30 @@ class Ubuntu(Image):
         return expected_sum
 
 
+class Debian(Image):
+    URL = "https://cloud.debian.org/images/cloud/%s/daily/latest/%s"
+    IMG = "debian-%s-generic-%s-daily.qcow2"
+    CHECKSUMTOOL = 'sha512sum'
+
+    def __init__(self, vbox, version, arch, release, fname=None):
+        super().__init__(vbox, version, arch, release)
+        self._img_fname = self.IMG % (version, arch)
+        self._img_url = self.URL % (release, self._img_fname)
+        self._checksum_file = 'SHA512SUMS'
+        self._checksum_url = self.URL % (release, self._checksum_file)
+
+    def _get_checksum(self, fname):
+        expected_sum = None
+        Run(['wget', self._checksum_url, '-q', '-O', fname])
+        with open(fname) as fobj:
+            for line in fobj.readlines():
+                if self._img_fname in line:
+                    expected_sum = line.split(' ')[0]
+                    break
+
+        return expected_sum
+
+
 class Fedora(Image):
     URL = ("https://download.fedoraproject.org/pub/fedora/linux/releases/%s/"
            "Cloud/%s/images/%s")
@@ -1039,10 +1137,10 @@ class Fedora(Image):
 
 class CentosStream(Image):
     URL = "https://cloud.centos.org/centos/%s-stream/%s/images/%s"
-    IMG = '.*(CentOS-Stream-GenericCloud-%s-[0-9]+\.[0-9].%s.qcow2).*'
+    IMG = '.*(CentOS-Stream-GenericCloud-%s-[0-9]+.[0-9].%s.qcow2).*'
     CHKS = "CHECKSUM"
 
-    def __init__(self, vbox, version, arch, release):
+    def __init__(self, vbox, version, arch, release, fname=None):
         super().__init__(vbox, version, arch, release)
         self._checksum_file = '%s-centos-stream-%s-%s' % (self.CHKS, version,
                                                           arch)
@@ -1098,7 +1196,7 @@ DISTROS = {'ubuntu': {'username': 'ubuntu',
                       'realname': 'ubuntu',
                       'img_class': Ubuntu,
                       'amd64': 'amd64',
-                      'default_version': '20.04'},
+                      'default_version': '22.04'},
            'fedora': {'username': 'fedora',
                       'realname': 'fedora',
                       'img_class': Fedora,
@@ -1108,13 +1206,20 @@ DISTROS = {'ubuntu': {'username': 'ubuntu',
                       'realname': 'centos',
                       'img_class': CentosStream,
                       'amd64': 'x86_64',
-                      'default_version': '8'}}
+                      'default_version': '8'},
+           'debian': {'username': 'debian',
+                      'realname': 'debian',
+                      'img_class': Debian,
+                      'amd64': 'amd64',
+                      'default_version': '11'}}
 
 
 def get_image_object(vbox, version, image='ubuntu', arch='amd64'):
     release = None
     if image == 'fedora':
         release = FEDORA_RELEASE_MAP[version]
+    if image == 'debian':
+        release = DEBIAN_CODENAME_MAP[version]
     return DISTROS[image]['img_class'](vbox, version, DISTROS[image]['amd64'],
                                        release, DISTROS[image].get('image'))
 
@@ -1192,8 +1297,8 @@ def vmcreate(args, conf=None):
     if not vbox.create_controller('SATA', 'sata'):
         return 4
 
-    for key in ('distro', 'hostname', 'key', 'version', 'image'):
+    for key in ('distro', 'hostname', 'key', 'version', 'image', 'username'):
-        if not getattr(conf, key) is None:
+        if getattr(conf, key) is None:
             continue
         if not vbox.setextradata(key, getattr(conf, key)):
             return 5
@@ -1240,6 +1345,7 @@ def vmcreate(args, conf=None):
            f'ssh://{username}@localhost:{vbox.vm_info["port"]}',
            'sudo cloud-init status']
     try:
+        counter = 0
         while True:
             out = Run(cmd)
             LOG.debug('Out: %s', out.stdout)
@@ -1259,6 +1365,22 @@ def vmcreate(args, conf=None):
                                           f'VM with ssh for user {username}. '
                                           f'Check output in debug mode.')
                 time.sleep(3)
+                counter += 1
+
+                # TODO: there is something odd with debian cloud images prior
+                # to 12 (bookworm), as on first run system crashes. In that
+                # case after ~20 seconds there should already be panic, reset
+                # machine as a workaround. Remove this after debian 12
+                # stabilization later this year.
+                if (counter == 8 and conf.distro == 'debian'
+                        and conf.version != '12'):
+                    LOG.debug('Resetting `%s`, due to the issue with kernel '
+                              'panic on Debian %s the first run', conf.name,
+                              conf.version)
+                    counter += 1
+                    vbox.poweroff()
+                    time.sleep(3)
+                    vbox.poweron(args.type)
                 continue
 
             LOG.info(' done')
@@ -1438,6 +1560,13 @@ def vmrebuild(args):
         conf.disk_size = vbox.get_media_size(disk_path)
 
     vmdestroy(args)
+
+    # Wait till VM is gone
+    while True:
+        vbox = VBoxManage(args.name)
+        if not vbox.get_vm_info():
+            break
+
     vmcreate(args, conf)
     return 0
 
@@ -1463,14 +1592,17 @@ def connect(args):
         return 16
 
     username = conf.username or DISTROS[conf.distro]["username"]
-    return Run(['ssh', '-o', 'StrictHostKeyChecking=no',
+    cmd = ['ssh', '-o', 'StrictHostKeyChecking=no',
-                '-o', 'UserKnownHostsFile=/dev/null',
+           '-o', 'UserKnownHostsFile=/dev/null',
-                '-i', conf.ssh_key_path[:-4],
+           '-i', conf.ssh_key_path[:-4],
-                f'ssh://{username}'
+           f'ssh://{username}'
-                f'@localhost:{conf.port}'], False).returncode
+           f'@localhost:{conf.port}']
+    LOG.debug('Connecting to vm `%s` using command:\n%s', args.name,
+              ' '.join(cmd))
+    return Run(cmd, False).returncode
 
 
-def _set_vmstate(name, state):
+def _set_vmstate(name, state, guitype=None):
 
     vbox = VBoxManage(name)
     if not vbox.get_vm_info():
@@ -1486,13 +1618,13 @@ def _set_vmstate(name, state):
         return
 
     if state == "start":
-        vbox.poweron()
+        vbox.poweron(guitype)
     else:
         vbox.acpipowerbutton()
 
 
 def vmstart(args):
-    _set_vmstate(args.name, 'start')
+    _set_vmstate(args.name, 'start', args.type)
 
 
 def vmstop(args):
@@ -1526,10 +1658,16 @@ def main():
                         help="Alternative user-data template filepath")
     create.add_argument('-d', '--distro', help="Image name. 'ubuntu' is "
                         "default")
+    create.add_argument('-e', '--default-user', help="Default cloud-init user "
+                        "to be used with custom image (--image param). "
+                        "Without image it will make no effect.")
     create.add_argument('-f', '--forwarding', action='append', help="expose "
                         "port from VM to the host. It should be in format "
                         "'hostport:vmport'. this option can be used multiple "
                         "times for multiple ports.")
+    create.add_argument('-i', '--image', help="custom qcow2 image filepath. "
+                        "Note, that it requires to provide --default-user as "
+                        "well.")
     create.add_argument('-k', '--key', help="SSH key to be add to the config "
                         "drive. Default ~/.ssh/id_rsa")
     create.add_argument('-m', '--memory', help="amount of memory in "
@@ -1571,10 +1709,16 @@ def main():
     rebuild.add_argument('-c', '--config',
                          help="Alternative user-data template filepath")
     rebuild.add_argument('-d', '--distro', help="Image name.")
+    rebuild.add_argument('-e', '--default-user', help="Default cloud-init "
+                         "user to be used with custom image (--image param). "
+                         "Without image it will make no effect.")
     rebuild.add_argument('-f', '--forwarding', action='append', help="expose "
                          "port from VM to the host. It should be in format "
                          "'hostport:vmport'. this option can be used multiple "
                          "times for multiple ports.")
+    rebuild.add_argument('-i', '--image', help="custom qcow2 image filepath. "
+                         "Note, that it requires to provide --default-user as "
+                         "well.")
     rebuild.add_argument('-k', '--key',
                          help='SSH key to be add to the config drive')
     rebuild.add_argument('-m', '--memory', help='amount of memory in '
@@ -1595,6 +1739,9 @@ def main():
 
     start = subparsers.add_parser('start', help='start VM')
     start.add_argument('name', help='name or UUID of the VM')
+    start.add_argument('-t', '--type', default='headless',
+                       help="VM run type, headless by default.",
+                       choices=['gui', 'headless', 'sdl', 'separate'])
     start.set_defaults(func=vmstart)
 
     stop = subparsers.add_parser('stop', help='stop VM')
@@ -1617,6 +1764,10 @@ def main():
 
     args = parser.parse_args()
 
+    if 'image' in args and 'default_user' not in args:
+        parser.error('Parameter --image requires --default-user')
+        return 22
+
     LOG.set_verbose(args.verbose, args.quiet)
 
     if 'func' not in args and args.version:

examples/devstack-ubuntu-20.04.yaml

@@ -29,7 +29,7 @@ runcmd:
   - [apt, purge, '-y', python3-pyasn1-modules]
   - [apt, purge, '-y', python3-simplejson]
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/vmstrap"]
-  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh"]
+  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh -c"]
   - [rm, -fr, /home/ubuntu/vmstrap]
   - [su, -, ubuntu, -c, "cp /tmp/local.conf /home/ubuntu/devstack/"]
   - [su, -, ubuntu, -c, "echo 'export HOST_IP=10.0.2.15' >> .bashrc"]

examples/kuryr_multinode_master.yaml

@@ -174,7 +174,7 @@ runcmd:
   - [apt, purge, '-y', python3-pyasn1-modules]
   - [apt, purge, '-y', python3-simplejson]
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/vmstrap"]
-  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh"]
+  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh -c"]
   - [rm, -fr, /home/ubuntu/vmstrap]
   - [su, -, ubuntu, -c, "echo 'export HOST_IP=192.168.10.10' >> .bashrc"]
   - [su, -, ubuntu, -c, "cp /tmp/local.conf /home/ubuntu/devstack/"]
@@ -186,3 +186,4 @@ boxpy_data:
   disk_size: 50GB
   advanced:
     nic2: intnet
+  version: 20.04

examples/kuryr_multinode_worker.yaml

@@ -123,7 +123,7 @@ runcmd:
   - [apt, purge, '-y', python3-pyasn1-modules]
   - [apt, purge, '-y', python3-simplejson]
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/vmstrap"]
-  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh"]
+  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh -c"]
   - [rm, -fr, /home/ubuntu/vmstrap]
   - [su, -, ubuntu, -c, "echo 'export HOST_IP=192.168.10.11' >> .bashrc"]
   - [su, -, ubuntu, -c, "cp /tmp/local.conf /home/ubuntu/devstack/"]
@@ -135,3 +135,4 @@ boxpy_data:
   disk_size: 50GB
   advanced:
     nic2: intnet
+  version: 20.04

examples/wmaker.yaml

@@ -1,5 +1,6 @@
 packages:
   - build-essential
+  - exuberant-ctags
   - gettext
   - libfontconfig1-dev
   - libgif-dev
@@ -21,12 +22,16 @@ packages:
   - libxrender-dev
   - libxt-dev
   - make
+  - mc
   - sharutils
+  - silversearcher-ag
+  - tmux
+  - vim-nox
   - xinit
 runcmd:
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/wmaker -b experimental"]
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/vmstrap"]
-  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh"]
+  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh -c"]
   - [rm, -fr, /home/ubuntu/vmstrap]
 boxpy_data:
   key: vm