Added information regarding url for the dl image

Two new commandline options are added: image and default-user. When
image parameter has been add there are implications that:

- default-user is also provided by commandline - regardless it is
  already present in yaml config
- distro parameter is ignored
- custom username, which might be provided by yaml file will become
  default-user if absent.

All of that is the consequence, that by providing custom qcow2 image
there is no easy way to determine what operating system is passed by,
therefore it is purely declarative way of creating VM with such image.

README.rst

@@ -55,7 +55,7 @@ or simply link it somewhere in the path:
    $ chmod +x ~/bin/boxpy
 
 and now you can issue some command. For example, to spin up a VM with Ubuntu
-18.04 with one CPU, 2GB of memory and 10GB of disk:
+18.04 with one CPU, 1GB of memory and 6GB of disk:
 
 .. code:: shell-session
 
@@ -102,6 +102,10 @@ Currently, following commands are available:
 All of the commands have a range of options, and can be examined by using
 ``--help`` option.
 
+
+YAML Configuration
+------------------
+
 What is more interesting though, is the fact, that you can pass your own
 `cloud-init`_ yaml file, so that VM can be provisioned in easy way.
 
@@ -203,6 +207,19 @@ configuration additional NIC for virtual machine, i.e:
      advanced:
        nic2: intnet
 
+To select image from local file system, it is possible to set one by providing
+it under ``boxpy_data.image`` key:
+
+.. code:: yaml
+
+   …
+   boxpy_data:
+     image: /path/to/the/qcow2/image
+     default_user: cloud-user
+
+Note, that default_user is also needed to be provided, as there is no guess,
+what is the default username for cloud-init configured within provided image.
+
 
 License
 -------

box.py

@@ -46,9 +46,9 @@ ssh:
   emit_keys_to_console: false
 boxpy_data:
   cpus: 1
-  disk_size: 10240
+  disk_size: 6144
   key: ~/.ssh/id_rsa
-  memory: 2048
+  memory: 1024
 '''
 COMPLETIONS = {'bash': '''\
 _boxpy() {
@@ -138,8 +138,9 @@ _boxpy() {
             fi
             ;;
         create|rebuild)
-            items=(--cpus --disable-nested --disk-size --distro --forwarding
+            items=(--cpus --disable-nested --disk-size --default-user --distro
-                --key --memory --hostname --port --config --version --type)
+                --forwarding --image --key --memory --hostname --port --config
+                --version --type)
             if [[ ${prev} == ${cmd} ]]; then
                 if [[ ${cmd} = "rebuild" ]]; then
                     _vms_comp vms
@@ -267,6 +268,10 @@ class BoxVBoxFailure(BoxError):
     pass
 
 
+class BoxConfError(BoxError):
+    pass
+
+
 class FakeLogger:
     """
     print based "logger" class. I like to use 'end' parameter of print
@@ -358,18 +363,20 @@ class FakeLogger:
 
 class Config:
     ATTRS = ('cpus', 'config', 'creator', 'disable_nested', 'disk_size',
-             'distro', 'forwarding', 'hostname', 'key', 'memory', 'name',
+             'distro', 'default_user', 'forwarding', 'hostname', 'image',
-             'port', 'version', 'username')
+             'key', 'memory', 'name', 'port', 'version', 'username')
 
     def __init__(self, args, vbox=None):
         self.advanced = None
         self.distro = None
+        self.default_user = None
         self.cpus = None
         self.creator = None
         self.disable_nested = 'False'
         self.disk_size = None
         self.forwarding = {}
         self.hostname = None
+        self.image = None
         self.key = None
         self.memory = None
         self.name = args.name  # this one is not stored anywhere
@@ -412,6 +419,10 @@ class Config:
                 continue
             setattr(self, attr, str(val))
 
+        # sort out case, where there is image/default-user provided
+        if self.image:
+            self._update_distros_with_custom_image()
+
         # set distribution and version if not specified by user
         if not self.distro:
             self.distro = 'ubuntu'
@@ -498,7 +509,7 @@ class Config:
             self.ssh_key_path = os.path.join(os.path.expanduser("~/.ssh"),
                                              self.ssh_key_path)
         if not os.path.exists(self.ssh_key_path):
-            raise BoxNotFound(f'Cannot find ssh public key: {self.key}')
+            raise BoxConfError(f'Cannot find ssh public key: {self.key}')
 
     def _set_defaults(self):
         conf = yaml.safe_load(USER_DATA)
@@ -553,6 +564,18 @@ class Config:
 
         self._conf = conf
 
+    def _update_distros_with_custom_image(self):
+        self.image = os.path.abspath(self.image)
+        self.distro = 'custom'
+        if not self.username:
+            self.username = self.default_user
+        DISTROS['custom'] = {'username': self.default_user,
+                             'realname': 'custom os',
+                             'img_class': CustomImage,
+                             'amd64': 'x86_64',
+                             'image': self.image,
+                             'default_version': '0'}
+
     def _update(self, source, update):
         for key, val in update.items():
             if isinstance(val, collections.abc.Mapping):
@@ -720,6 +743,12 @@ class VBoxManage:
             LOG.fatal('Failed to create VM:\n%s', out.stderr)
             return None
 
+        if out.stdout.startswith('WARNING:'):
+            LOG.fatal('Created crippled VM:\n%s\nFix the issue with '
+                      'VirtualBox, remove the dead VM and start over.',
+                      out.stdout)
+            return None
+
         for line in out.stdout.split('\n'):
             if line.startswith('UUID:'):
                 self.uuid = line.split('UUID:')[1].strip()
@@ -886,10 +915,10 @@ class Image:
     URL = ""
     IMG = ""
 
-    def __init__(self, vbox, version, arch, release):
+    def __init__(self, vbox, version, arch, release, fname=None):
         self.vbox = vbox
         self._tmp = tempfile.mkdtemp(prefix='boxpy_')
-        self._img_fname = None
+        self._img_fname = fname
 
     def convert_to_vdi(self, disk_img, size):
         LOG.info('Converting and resizing "%s", new size: %s', disk_img, size)
@@ -949,7 +978,8 @@ class Image:
             return True
 
         fname = os.path.join(CACHE_DIR, self._img_fname)
-        LOG.header('Downloading image %s', self._img_fname)
+        LOG.header('Downloading image %s from %s', self._img_fname,
+                   self._img_url)
         Run(['wget', '-q', self._img_url, '-O', fname])
 
         if not self._checksum():
@@ -968,7 +998,7 @@ class Ubuntu(Image):
     URL = "https://cloud-images.ubuntu.com/releases/%s/release/%s"
     IMG = "ubuntu-%s-server-cloudimg-%s.img"
 
-    def __init__(self, vbox, version, arch, release):
+    def __init__(self, vbox, version, arch, release, fname=None):
         super().__init__(vbox, version, arch, release)
         self._img_fname = self.IMG % (version, arch)
         self._img_url = self.URL % (version, self._img_fname)
@@ -993,7 +1023,7 @@ class Fedora(Image):
     IMG = "Fedora-Cloud-Base-%s-%s.%s.qcow2"
     CHKS = "Fedora-Cloud-%s-%s-%s-CHECKSUM"
 
-    def __init__(self, vbox, version, arch, release):
+    def __init__(self, vbox, version, arch, release, fname=None):
         super().__init__(vbox, version, arch, release)
         self._img_fname = self.IMG % (version, release, arch)
         self._img_url = self.URL % (version, arch, self._img_fname)
@@ -1016,10 +1046,10 @@ class Fedora(Image):
 
 class CentosStream(Image):
     URL = "https://cloud.centos.org/centos/%s-stream/%s/images/%s"
-    IMG = '.*(CentOS-Stream-GenericCloud-%s-[0-9]+\.[0-9].%s.qcow2).*'
+    IMG = '.*(CentOS-Stream-GenericCloud-%s-[0-9]+.[0-9].%s.qcow2).*'
     CHKS = "CHECKSUM"
 
-    def __init__(self, vbox, version, arch, release):
+    def __init__(self, vbox, version, arch, release, fname=None):
         super().__init__(vbox, version, arch, release)
         self._checksum_file = '%s-centos-stream-%s-%s' % (self.CHKS, version,
                                                           arch)
@@ -1064,11 +1094,18 @@ class CentosStream(Image):
         return expected_sum
 
 
+class CustomImage(Image):
+
+    def _download_image(self):
+        # just use provided image
+        return True
+
+
 DISTROS = {'ubuntu': {'username': 'ubuntu',
                       'realname': 'ubuntu',
                       'img_class': Ubuntu,
                       'amd64': 'amd64',
-                      'default_version': '20.04'},
+                      'default_version': '22.04'},
            'fedora': {'username': 'fedora',
                       'realname': 'fedora',
                       'img_class': Fedora,
@@ -1086,7 +1123,7 @@ def get_image_object(vbox, version, image='ubuntu', arch='amd64'):
     if image == 'fedora':
         release = FEDORA_RELEASE_MAP[version]
     return DISTROS[image]['img_class'](vbox, version, DISTROS[image]['amd64'],
-                                       release)
+                                       release, DISTROS[image].get('image'))
 
 
 class IsoImage:
@@ -1137,7 +1174,8 @@ def vmcreate(args, conf=None):
     if not conf:
         try:
             conf = Config(args)
-        except BoxNotFound:
+        except BoxConfError as err:
+            LOG.fatal(f'Configuration error: {err.args[0]}.')
             return 7
         except yaml.YAMLError:
             LOG.fatal(f'Cannot read or parse file `{args.config}` as YAML '
@@ -1161,7 +1199,9 @@ def vmcreate(args, conf=None):
     if not vbox.create_controller('SATA', 'sata'):
         return 4
 
-    for key in ('distro', 'hostname', 'key', 'version'):
+    for key in ('distro', 'hostname', 'key', 'version', 'image', 'username'):
+        if getattr(conf, key) is None:
+            continue
         if not vbox.setextradata(key, getattr(conf, key)):
             return 5
 
@@ -1493,10 +1533,16 @@ def main():
                         help="Alternative user-data template filepath")
     create.add_argument('-d', '--distro', help="Image name. 'ubuntu' is "
                         "default")
+    create.add_argument('-e', '--default-user', help="Default cloud-init user "
+                        "to be used with custom image (--image param). "
+                        "Without image it will make no effect.")
     create.add_argument('-f', '--forwarding', action='append', help="expose "
                         "port from VM to the host. It should be in format "
                         "'hostport:vmport'. this option can be used multiple "
                         "times for multiple ports.")
+    create.add_argument('-i', '--image', help="custom qcow2 image filepath. "
+                        "Note, that it requires to provide --default-user as "
+                        "well.")
     create.add_argument('-k', '--key', help="SSH key to be add to the config "
                         "drive. Default ~/.ssh/id_rsa")
     create.add_argument('-m', '--memory', help="amount of memory in "
@@ -1538,10 +1584,16 @@ def main():
     rebuild.add_argument('-c', '--config',
                          help="Alternative user-data template filepath")
     rebuild.add_argument('-d', '--distro', help="Image name.")
+    rebuild.add_argument('-e', '--default-user', help="Default cloud-init "
+                         "user to be used with custom image (--image param). "
+                         "Without image it will make no effect.")
     rebuild.add_argument('-f', '--forwarding', action='append', help="expose "
                          "port from VM to the host. It should be in format "
                          "'hostport:vmport'. this option can be used multiple "
                          "times for multiple ports.")
+    rebuild.add_argument('-i', '--image', help="custom qcow2 image filepath. "
+                         "Note, that it requires to provide --default-user as "
+                         "well.")
     rebuild.add_argument('-k', '--key',
                          help='SSH key to be add to the config drive')
     rebuild.add_argument('-m', '--memory', help='amount of memory in '
@@ -1584,6 +1636,10 @@ def main():
 
     args = parser.parse_args()
 
+    if 'image' in args and 'default_user' not in args:
+        parser.error('Parameter --image requires --default-user')
+        return 22
+
     LOG.set_verbose(args.verbose, args.quiet)
 
     if 'func' not in args and args.version:

examples/devstack-ubuntu-20.04.yaml

@@ -29,7 +29,7 @@ runcmd:
   - [apt, purge, '-y', python3-pyasn1-modules]
   - [apt, purge, '-y', python3-simplejson]
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/vmstrap"]
-  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh"]
+  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh -c"]
   - [rm, -fr, /home/ubuntu/vmstrap]
   - [su, -, ubuntu, -c, "cp /tmp/local.conf /home/ubuntu/devstack/"]
   - [su, -, ubuntu, -c, "echo 'export HOST_IP=10.0.2.15' >> .bashrc"]

examples/kuryr_multinode_master.yaml

@@ -34,55 +34,55 @@ write_files:
     content: |
       [[local|localrc]]
       disable_all_services
-      
+
       # Cinder
       disable_service c-api
       disable_service c-bak
       disable_service c-sch
       disable_service c-vol
       disable_service cinder
-      
+
       # Coredns
       disable_service coredns
-      
+
-      # Dstat 
+      # Dstat
       disable_service dstat
-      
+
       # ETCD
       enable_service etcd3
-      
+
       # Glance
       enable_service g-api
       enable_service g-reg
-      
+
       # Horizon
       disable_service horizon
-      
+
       # Keystone
       enable_service key
-      
+
       # kubernetes
       enable_service kubernetes-master
       disable_service kubernetes-worker
-      
+
       # kuryr
       enable_service kuryr-daemon
       enable_service kuryr-kubernetes
-      
+
       # mysql.
       enable_service mysql
-      
+
       # Nova services
       enable_service n-api
       enable_service n-api-meta
       enable_service n-cond
       enable_service n-cpu
       enable_service n-sch
-      
+
       # Neutron
       enable_service neutron
       enable_service neutron-tag-ports-during-bulk-creation
-      
+
       # Octavia
       enable_service o-api
       enable_service o-cw
@@ -90,41 +90,41 @@ write_files:
       enable_service o-hk
       enable_service o-hm
       enable_service octavia
-      
+
       # Neutron ovn services
       enable_service ovn-controller
       enable_service ovn-northd
       enable_service ovs-vswitchd
       enable_service ovsdb-server
-      
+
       # Placement API
       enable_service placement-api
       enable_service placement-client
-      
+
       # Neutron services
       enable_service q-ovn-metadata-agent
       enable_service q-qos
       enable_service q-svc
       enable_service q-trunk
       enable_service rabbit
-      
+
       # Swift
       disable_service s-account
       disable_service s-container
       disable_service s-object
       disable_service s-proxy
-      
+
       # Tempest
       enable_service tempest
-      
+
       # TLS
       disable_service tls-proxy
-      
+
       # Vars
       ADMIN_PASSWORD="secretadmin"
       DATABASE_PASSWORD="secretdatabase"
       ETCD_USE_RAMDISK="True"
-      
+
       KURYR_ENABLED_HANDLERS="vif,endpoints,service,namespace,pod_label,policy,kuryrnetworkpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer"
       KURYR_ENFORCE_SG_RULES="False"
       KURYR_EP_DRIVER_OCTAVIA_PROVIDER="ovn"
@@ -134,30 +134,30 @@ write_files:
       KURYR_NEUTRON_DEFAULT_ROUTER="router1"
       KURYR_SG_DRIVER="policy"
       KURYR_SUBNET_DRIVER="namespace"
-      
+
       LOGFILE="/opt/stack/logs/devstacklog.txt"
       LOG_COLOR="False"
       ML2_L3_PLUGIN="ovn-router,trunk,qos"
       OCTAVIA_AMP_IMAGE_FILE="/tmp/test-only-amphora-x64-haproxy-ubuntu-bionic.qcow2"
       OCTAVIA_AMP_IMAGE_NAME="test-only-amphora-x64-haproxy-ubuntu-bionic"
       OCTAVIA_AMP_IMAGE_SIZE="3"
-      
+
       OVN_BRANCH="v20.06.2"
       OVN_BUILD_FROM_SOURCE="True"
       OVN_DBS_LOG_LEVEL="dbg"
       OVN_L3_CREATE_PUBLIC_NETWORK="True"
-      
+
       VAR_RUN_PATH="/usr/local/var/run"
-      
+
       RABBIT_PASSWORD="secretrabbit"
       RECLONE="no"
       SERVICE_PASSWORD="secretservice"
       SERVICE_TOKEN="password"
-      
+
       LIBS_FROM_GIT=cinder,devstack,devstack-gate,devstack-plugin-container,glance,keystone,kuryr-kubernetes,kuryr-tempest-plugin,neutron,nova,octavia,octavia-tempest-plugin,ovn-octavia-provider,placement,python-octaviaclient,requirements,swift,tempest
-      
+
       TEMPEST_PLUGINS="/opt/stack/kuryr-tempest-plugin /opt/stack/octavia-tempest-plugin"
-      
+
       # enabled plugins
       enable_plugin devstack-plugin-container https://opendev.org/openstack/devstack-plugin-container
       enable_plugin kuryr-kubernetes https://opendev.org/openstack/kuryr-kubernetes
@@ -166,7 +166,7 @@ write_files:
       enable_plugin octavia https://opendev.org/openstack/octavia
       enable_plugin octavia-tempest-plugin https://opendev.org/openstack/octavia-tempest-plugin
       enable_plugin ovn-octavia-provider https://opendev.org/openstack/ovn-octavia-provider
-      
+
       [[post-config|$OCTAVIA_CONF]]
       [api_settings]
       enabled_provider_drivers = amphora:'Octavia Amphora driver',ovn:'Octavia OVN driver'
@@ -174,7 +174,7 @@ runcmd:
   - [apt, purge, '-y', python3-pyasn1-modules]
   - [apt, purge, '-y', python3-simplejson]
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/vmstrap"]
-  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh"]
+  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh -c"]
   - [rm, -fr, /home/ubuntu/vmstrap]
   - [su, -, ubuntu, -c, "echo 'export HOST_IP=192.168.10.10' >> .bashrc"]
   - [su, -, ubuntu, -c, "cp /tmp/local.conf /home/ubuntu/devstack/"]
@@ -186,3 +186,4 @@ boxpy_data:
   disk_size: 50GB
   advanced:
     nic2: intnet
+  version: 20.04

examples/kuryr_multinode_worker.yaml

@@ -34,28 +34,28 @@ write_files:
     content: |
       [[local|localrc]]
       disable_all_services
-      
+
       # Cinder
       disable_service c-bak
       disable_service c-vol
-      
+
       # Dstat (started to fail recently)
       disable_service dstat
-      
+
       # Horizon
       disable_service horizon
-      
+
       # kubernetes
       disable_service kubernetes-master
       enable_service kubernetes-worker
-      
+
       # kuryr
       enable_service kuryr-daemon
       disable_service kuryr-kubernetes
-      
+
       # Nova
       enable_service n-cpu
-      
+
       # Neutron
       enable_service neutron
       enable_service ovn-controller
@@ -63,27 +63,27 @@ write_files:
       enable_service ovn-octavia-provider
       enable_service ovs-vswitchd
       enable_service ovsdb-server
-      
+
       # Placement API
       enable_service placement-client
-      
+
       # Neutron services cd
       enable_service q-ovn-metadata-agent
       disable_service q-svc
-      
+
       # tempest
       disable_service tempest
-      
+
       # tls
       disable_service tls-proxy
-      
+
       # Vars
       ADMIN_PASSWORD="secretadmin"
       DATABASE_HOST="192.168.10.10"
       DATABASE_PASSWORD="secretdatabase"
       ENABLE_CHASSIS_AS_GW="False"
       GLANCE_HOSTPORT="192.168.10.10:9292"
-     
+
       # turn on ovn-provider
       KURYR_ENFORCE_SG_RULES="False"
       KURYR_EP_DRIVER_OCTAVIA_PROVIDER="ovn"
@@ -91,18 +91,18 @@ write_files:
       KURYR_LB_ALGORITHM="SOURCE_IP_PORT"
       KURYR_NEUTRON_DEFAULT_ROUTER="router1"
       VAR_RUN_PATH="/usr/local/var/run"
-      
+
       KURYR_ENABLED_HANDLERS="vif,endpoints,service,namespace,pod_label,policy,kuryrnetworkpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer"
       KURYR_SG_DRIVER="policy"
       KURYR_SUBNET_DRIVER="namespace"
-      
+
-      
+
       OVN_BRANCH="v20.06.2"
       OVN_BUILD_FROM_SOURCE="True"
       OVN_DBS_LOG_LEVEL="dbg"
       OVN_L3_CREATE_PUBLIC_NETWORK="True"
-      
+
-      
+
       LIBVIRT_TYPE="qemu"
       LOGFILE="/opt/stack/logs/devstacklog.txt"
       LOG_COLOR="False"
@@ -113,9 +113,9 @@ write_files:
       SERVICE_HOST="192.168.10.10"
       SERVICE_PASSWORD="secretservice"
       SERVICE_TOKEN="password"
-      
+
       LIBS_FROM_GIT=cinder,devstack,devstack-gate,devstack-plugin-container,glance,keystone,kuryr-kubernetes,kuryr-tempest-plugin,neutron,nova,octavia,placement,python-octaviaclient,requirements,swift,tempest
-      
+
       # enabled plugins
       enable_plugin devstack-plugin-container https://opendev.org/openstack/devstack-plugin-container
       enable_plugin kuryr-kubernetes https://opendev.org/openstack/kuryr
@@ -123,7 +123,7 @@ runcmd:
   - [apt, purge, '-y', python3-pyasn1-modules]
   - [apt, purge, '-y', python3-simplejson]
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/vmstrap"]
-  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh"]
+  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh -c"]
   - [rm, -fr, /home/ubuntu/vmstrap]
   - [su, -, ubuntu, -c, "echo 'export HOST_IP=192.168.10.11' >> .bashrc"]
   - [su, -, ubuntu, -c, "cp /tmp/local.conf /home/ubuntu/devstack/"]
@@ -135,3 +135,4 @@ boxpy_data:
   disk_size: 50GB
   advanced:
     nic2: intnet
+  version: 20.04

examples/wmaker.yaml

@@ -1,5 +1,6 @@
 packages:
   - build-essential
+  - exuberant-ctags
   - gettext
   - libfontconfig1-dev
   - libgif-dev
@@ -21,12 +22,16 @@ packages:
   - libxrender-dev
   - libxt-dev
   - make
+  - mc
   - sharutils
+  - silversearcher-ag
+  - tmux
+  - vim-nox
   - xinit
 runcmd:
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/wmaker -b experimental"]
   - [su, -, ubuntu, -c, "git clone https://github.com/gryf/vmstrap"]
-  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh"]
+  - [su, -, ubuntu, -c, "vmstrap/bootstrap.sh -c"]
   - [rm, -fr, /home/ubuntu/vmstrap]
 boxpy_data:
   key: vm