version 0.3.3 typo bugfix

Proxmox supports tags, so class Proxmox supports tags, too.
It is possible to show and change tags in config and `e status` prints
tags.

The new command `e ct volume_move NAMEORID DISK POOL` "moves" the disk
of a CT to an other RBD-pool.  It only supports RBD, yet.
It uses XFS on the new device.
The CT must be stopped on the same machine, you are calling the command.

The class PVE::Output helps to print status.  Supports info, failed and
progress (with a spinner).  Monitoring tasks used a custom spinner,
which is now part of the new PVE::Output.

The status of CTs/VMs/Nodes are symbols only.  Strings shouldn't be used
anymore.

lib/pve/cli.rb

@@ -11,9 +11,6 @@ require_relative 'cli/qm'
 require_relative 'cli/node'
 require_relative 'cli/storage'
 
-class UsageError <RuntimeError
-end
-
 class DenCli::Sub
   def provide_help name: nil, aliases: nil, min: nil
     base = self
@@ -26,6 +23,73 @@ class DenCli::Sub
 end
 
 class PVE::Cli
+  class RuntimeError <::RuntimeError
+  end
+
+  class UsageError <RuntimeError
+  end
+  def UsageError msg
+    raise UsageError, msg
+  end
+
+  class OperationalError <RuntimeError
+  end
+  def OperationalError msg
+    raise OperationalError, msg
+  end
+
+  class ConditionViolation <RuntimeError
+  end
+  def ConditionViolation msg
+    raise ConditionViolation, msg
+  end
+
+  class Output
+    LINERESET = "\r\e[J"
+    RESET     = "\e[0m"
+    SUCCESS_  = "\e[32;1m✓"
+    SUCCESS   = "#{LINERESET}[%s] #{SUCCESS_}#{RESET} "
+    FAILED_   = "\e[31;1m✗"
+    FAILED    = "#{LINERESET}[%s] #{FAILED_}#{RESET} "
+    PROGRESS_ = "\e[33;1m%s"
+    PROGRESS  = "#{LINERESET}[%s] #{PROGRESS_}#{RESET} "
+    INFO_     = "\e[33;1m•"
+    INFO      = "#{LINERESET}[%s] #{INFO_}#{RESET} "
+
+    attr_accessor :host
+
+    def initialize host
+      @host, @spinners, @spin = host, "▖▘▝▗", 0
+    end
+
+    def progress format, *args, **kwargs
+      STDERR.printf PROGRESS, @host, @spinners[@spin = (@spin + 1) % 4]
+      STDERR.printf( format, *args, **kwargs)
+    end
+
+    def info format, *args, **kwargs
+      STDERR.printf INFO, @host
+      STDERR.printf( format, *args, **kwargs)
+      STDERR.puts
+    end
+
+    def failed format, *args, **kwargs
+      STDERR.printf FAILED, @host
+      STDERR.printf( format, *args, **kwargs)
+      STDERR.puts
+    end
+
+    def success format, *args, **kwargs
+      STDERR.printf SUCCESS, @host
+      STDERR.printf( format, *args, **kwargs)
+      STDERR.puts
+    end
+
+    def reset
+      STDERR.printf LINERESET
+    end
+  end
+
   attr_reader :cfg, :cli
 
   def connect
@@ -39,10 +103,12 @@ class PVE::Cli
   def initialize config_file = nil
     config_file ||= '/etc/pve/pvecli.yml'
     @cfg =
-      YAML.safe_load File.read( config_file), [], [], false,
-        config_file, symbolize_names: true
+      YAML.safe_load File.read( config_file), permitted_classes: [],
+        permitted_symbols: [], aliases: false, filename: config_file,
+        symbolize_names: true
     @cli = DenCli.new File.basename($0), "PVE CLI"
     @interactive = false
+    @out = Output.new "<>"
     prepare
   end
 
@@ -51,38 +117,37 @@ class PVE::Cli
   end
 
   def enter host, *args
+    @out.host = host.name
     r = host.enter *args
-    STDERR.puts "! #{$?.exitstatus}"  unless r
+    operational_error "Exitstatus: #{$?.exitstatus}"  unless r
   end
 
   def wait_state host, state, timeout: nil, lock: nil
-    spinners = %w[- / | \\]
-    spin = 0
+    @out.host = host.name
     r =
       host.wait state, lock: lock, timeout: timeout, secs: 0.2 do |state, lock|
-        lock = " (locked: #{lock})"  if lock
-        STDERR.printf "\r[%s] %s Still %s%s...\e[J\n", host.name, spinners[spin = (spin + 1) % 4], state, lock
+        @out.progress "Still %s%s...", state, lock ? " (locked: #{lock})" : ''
       end
-    STDERR.printf "\r\e[J"
-    exit 1  unless interactive?  and  r
+    @out.reset
+    exit 1  if interactive?  and  !r
   end
 
   def task_log task, logn, limit = 1024
     log = task.log start: logn, limit: limit
     log = []  if [{n: 1, t: 'no content'}] == log
     unless log.empty?
-      STDERR.printf "\r\e[J"
+      @out.reset
       log.each {|l| puts l[:t] }
       logn = log.last[:n]
     end
     logn
   end
 
-  def wait task, secs: nil, text: nil
+  def wait task, secs: nil, text: nil, timeout: nil
     secs ||= 0.1
-    spinners, spin, logn = "▖▘▝▗", 0, 0
-    STDERR.puts task.upid
-    host = task.host&.name
+    logn = 0
+    STDOUT.puts task.upid
+    @out.host = host = task.host&.name || '<>'
     loop do
       s = task.status
       logn = self.task_log task, logn
@@ -92,48 +157,53 @@ class PVE::Cli
           break  if 0 == logn - r
           logn = r
         end
-        STDERR.printf "\r[%s] %s %s %s\e[J\n",
-          host || s.id,
-          s.successfull? ? "\e[32;1m✓\e[0m" : "\e[31;1m✗\e[0m",
-          text && "#{text}:",
-          s.stopped? ? :finished : s.status
+        if s.successfull?
+          @out.success "%s %s", text && "#{text}:", s.stopped? ? :finished : s.status
+        else
+          @out.failed "%s %s", text && "#{text}:", s.stopped? ? :finished : s.status
+        end
         return s
       end
-      STDERR.printf "\r[%s] \e[33;1m%s\e[0m %s...\e[J",
-        host || s[:id],
-        spinners[spin = (spin + 1) % 4],
-        text || "Working"
+      @out.progress text || "Working"
       sleep secs
     end
   end
 
-  def start host, node: nil, timeout: nil, fire: nil, secs: nil
+  def migrate host, node, timeout: nil, fire: nil, secs: nil, online: nil
     timeout ||= 30
-    if node
-      task = host.migrate Proxmox::Node.find_by_name!( node)
-      wait task, text: "Migrating"
-    end
+    task = host.migrate Proxmox::Node.find_by_name!( node), online: online || false
+    wait task, text: "Migrating", timeout: timeout  if fire
+  end
+
+  def start host, node: nil, timeout: nil, fire: nil, secs: nil
+    timeout ||= 60
     if host.running?
-      STDERR.puts "Already running."
+      ConditionViolation "Already running."
       return
     end
     task = host.start
-    wait task, text: "Starting"  unless fire
+    t = Time.now
+    unless fire
+      wait task, text: "Starting", timeout: timeout
+      wait_state host, :running, timeout: timeout-(Time.now-t)  if host.ha.exist? and not host.running?
+    end
   end
 
   def stop host, timeout: nil, fire: nil, secs: nil
     timeout ||= 30
     if host.stopped?
-      STDERR.puts "Already stopped."
+      preset_error "Already stopped."
       return
     end
     task = host.stop
+    t = Time.now
     unless fire
-      wait task, text: "Stopping"
+      wait task, text: "Stopping", timeout: timeout
+      wait_state host, :stopped, timeout: timeout-(Time.now-t)  if host.ha.exist? and not host.stopped?
     end
   end
 
-  def create klass, template, timeout: nil, fire: nil, secs: nil, start: nil, **options
+  def create klass, template, timeout: nil, fire: nil, start: nil, **options
     options[:start] = fire && start
     task = klass.create template, **options
     return  if fire
@@ -153,9 +223,139 @@ class PVE::Cli
     end
   end
 
+  def rbd_device_map image_spec, &exe
+    dev = IO.popen( [*%w[rbd device map], image_spec]) {|io| io.read }.chomp
+    raise OperationalError, "Device mapping #{image_spec} localy failed."  unless $?.success?
+    begin
+      yield dev
+    ensure
+      system *%w[rbd device unmap], image_spec
+      raise OperationalError, "Device unmapping #{image_spec} failed."  unless $?.success?
+    end
+  end
+
+  def rbd_devices_maps spec, *specs, &exe
+    rbd_device_map spec do |dev|
+      if specs.empty?
+        yield dev
+      else
+        rbd_devices_maps *specs do |*devs|
+          yield dev, *devs
+        end
+      end
+    end
+  end
+
+  def mount dev, mp, &exe
+    system 'mount', dev.to_s, mp.to_s
+    begin yield
+    ensure
+      # it could be possible, that a process is working in fs, yet.
+      sleep 1
+      system 'umount', mp.to_s
+    end
+  end
+
+  def ct_volume_move ct, disk, destination
+    @out.host = ct.name
+    host = Proxmox::Node.find_by_name! Socket.gethostname
+    unless host.node == ct.node.node
+      ConditionViolation( "CT is hosted on #{ct.node.node}, not local (#{Socket.gethostname})")
+    end
+    # Lock, we do not want changes, while we check/prepare,
+    # these checks could be obsolete while moving.
+    configupdate = {}
+    ct.lock :migrate do
+      disk =
+        case disk
+        when 'rootfs', /\Amp\d+\z/ then disk.to_sym
+        else UsageError( "Unknown disk #{disk}")
+        end
+      ctcnf = ct.config
+      UsageError( "CT has no disk #{disk}")  if ctcnf[disk].nil?
+      UsageError( "CT not stopped, yet.")    unless ct.stopped?
+      dest = host.storage.find {|x| destination == x.storage }
+      ConditionViolation( "Destination storage #{dest.storage} disabled")    unless dest.enabled?
+      ConditionViolation( "Destination storage #{dest.storage} not active")  unless dest.active?
+
+      unless /\A([^:]+):([^,]+)(,.+)?\z/ =~ ctcnf[disk]
+        OperationalError( "disk-specification cannot be parsed: [#{disk}: #{ctcnf[disk]}]")
+      end
+      source, name, diskopts = $1, $2, $3.to_s
+      src = host.storage.find {|x| source == x.storage }
+
+      ConditionViolation( "Source storage #{dest.storage} disabled")    unless dest.enabled?
+      ConditionViolation( "Source storage #{dest.storage} not active")  unless dest.active?
+      case src.type
+      when 'rbd'
+      else ConditionViolation( "Storage type #{src.type} not supported as source. (supported: rbd only, yet)")
+      end
+      #unless c = src.content.find {|x| x.name == name }
+      #  usage_error "Source storage #{src.storage} has no disk named #{name}"
+      #end
+      case dest.type
+      when 'rbd'
+      else ConditionViolation( "Storage type #{dest.type} not supported as destination. (supported: rbd only, yet)")
+      end
+      #if c = dest.content.find {|x| x.name == name }
+      #  usage_error "Destination storage #{dest.storage} has already a disk named #{name}"
+      #end
+
+      src_image_spec, dest_image_spec = "#{source}/#{name}", "#{destination}/#{name}"
+      mp_path = Pathname.new( "/var/lib/lxc/").join ct.vmid.to_s
+      mp_path.mkdir  unless mp_path.exist?
+      src_mp, dest_mp = mp_path + src_image_spec.gsub('/','-'), mp_path + dest_image_spec.gsub('/','-')
+
+      src_mp.mkdir   unless src_mp.exist?
+      dest_mp.mkdir  unless dest_mp.exist?
+      # We check, if something is mounted already on our mountpoints.
+      # We check it later, too, but later cleaning up would be impossible.
+      # We do not trust only this checks, so we check later, too.
+      ConditionViolation( "Something already mounted at #{src_mp}")   if src_mp.mountpoint?
+      ConditionViolation( "Something already mounted at #{dest_mp}")  if dest_mp.mountpoint?
+
+      rbd_image_info = JSON.parse IO.popen( [*%w[rbd info --format json], src_image_spec]) {|io| io.read }
+      OperationalError( "Couldn't determine size of #{src_image_spec}.")  unless $?.success?
+
+      # checks and preparation done.
+      size = rbd_image_info['size']/1024/1024
+
+      unless system *%w[rbd create -s], size.to_s, dest_image_spec
+        OperationalError( "Creating disk #{dest_image_spec} failed.")
+      end
+      @out.progress "Map devices %s, %s \e[J", ct.name, dest_image_spec, src_image_spec
+      rbd_devices_maps dest_image_spec, src_image_spec do |dest_dev, src_dev|
+        @out.info "Maped source      device %s => %s => %s", src_image_spec, src_dev, src_mp
+        @out.info "Maped destination device %s => %s => %s", dest_image_spec, dest_dev, dest_mp
+        @out.info "Formatting destination disk"
+        unless system *%w[mkfs.xfs -mreflink=1 -bsize=4096 -ssize=4096], dest_dev
+          OperationalError( "Formatting #{dest_image_spec} failed.")
+        end
+        OperationalError( "Something already mounted at #{src_mp}")  if src_mp.mountpoint?
+        @out.progress "Mounting source disk"
+        mount src_dev, src_mp do
+          OperationalError( "Something already mounted at #{src_mp}")  if dest_mp.mountpoint?
+          @out.progress "Mounting destination disk"
+          mount dest_dev, dest_mp do
+            #@out.info "rsyncing..."
+            #system *%w[rsync -aHAX --info=progress2], "#{src_mp}/", "#{dest_mp}/"
+            IO.popen %w[rsync -ahHAX --info=progress2] + ["#{src_mp}/", "#{dest_mp}/"] do |io|
+              io.each_line("\r") {|l| @out.progress 'rsync|%s', l.chomp }
+            end
+            OperationalError( "rsync had an error. [#{$?.exitcode}]")  unless $?.success?
+          end
+        end
+      end
+
+      unusedfield = (0..20).map{|i| "unused#{i}" }.find {|n| ctcnf[n].nil? }
+      configupdate = {unusedfield => "#{source}:#{name}", disk => "#{destination}:#{name}#{diskopts}"}
+    end
+    ct.cnfset **configupdate
+    @out.success "disk moved."
+  end
+
   def node_opt node = nil
-    node &&= /\A#{node}\z/
-    node ? Proxmox::Node.find_by_name( name) : Proxmox::Node.all
+    node ? [Proxmox::Node.find_by_name!( node)] : Proxmox::Node.all
   end
 
   def target_opt target = nil, &exe
@@ -167,29 +367,44 @@ class PVE::Cli
     end
   end
 
-  def hosting_table target:, status:, sort:
+  def hosting_table target:, status:, sort:, tags:
     connect
-    to = TablizedOutput.new %w[Status HA ID Name Host Uptime CPU/% Mem/MiB Mem/% Disk/MiB Disk/%], format: '<<<<<>>>>>>'
+    to = TablizedOutput.new %w[Status HA ID Name Host Uptime CPU/% Mem/MiB Mem/% Disk/MiB Disk/% Tags], format: '<<<<<>>>>>><'
     target &&= /\A#{target}\z/i
+    nottags = nil
+    if tags
+      tags = tags.split /,/
+      nottags = tags.grep /\A-/
+      tags -= nottags
+      nottags.map! {|x| x[1..-1] }
+    end
     status =
       case status
-      when /\Asta(r(t(ed?)?)?)?\z/i, /\Aon(l(i(ne?)?)?)?\z/i, /\Ar(u(n(n(i(ng?)?)?)?)?)?\z/i, '1'
+      when /\Asta(?:r(?:t(?:ed?)?)?)?\z/i, /\Aon(?:l(?:i(?:ne?)?)?)?\z/i, /\Ar(?:u(?:n(?:n(?:i(?:ng?)?)?)?)?)?\z/i, '1'
         %i[started online running]
-      when /\Asto(p(p(ed?)?)?)?\z/i, /\Aof(f(l(i(ne?)?)?)?)?\z/i, '0'
+      when /\Asto(?:p(?:p(?:ed?)?)?)?\z/i, /\Aof(?:f(?:l(?:i(?:ne?)?)?)?)?\z/i, '0'
         %i[stopped offline]
       when nil, '', /\Aa(ll?)?\z/i then nil
       else
-        raise DenKn::UsageError, "Unknown state #{status}"
+        usage_error "Unknown state #{status}"
       end
     push =
-      if target and status
-        lambda {|n| to.virt n  if n === target and status.include?( n.state) }
-      elsif target
-        lambda {|n| to.virt n  if n === target }
-      elsif status
-        lambda {|n| to.virt n  if status.include? n.state }
-      else
-        to.method :virt
+      begin
+        condition = []
+        condition.push lambda {|n| n === target }  if target
+        if status
+          condition.push lambda {|n| !n.respond_to?( :status) or status.include?( n.status) }
+        end
+        if tags
+          condition.push lambda {|n|
+            if n.respond_to?( :tags) and n.tags
+              nt = n.tags
+              [] == tags - nt and nottags == nottags - nt
+            else false
+            end
+          }
+        end
+        lambda {|n| to.push tablized_virt( n)  if condition.all? {|c| c.call n} }
       end
     yield push
     to.print order: sort.each_char.map {|c| (2*c.ord[5]-1) * (' sainhucmd'.index( c.downcase)) }
@@ -250,10 +465,32 @@ class PVE::Cli
   def call *argv
     cli.call *argv
   rescue RestClient::ExceptionWithResponse
-    STDERR.puts "#$! - #{$!.response} (#{$!.class})" #, $!.backtrace.map {|b|"  #{b}"}
-  rescue UsageError, DenCli::UsageError
-    STDERR.puts $!
+    @out.failed "%s: %s - %s (%s)", $!.request, $!, $!.response, $!.class
+    #STDERR.puts $!.backtrace.map {|b|"  #{b}"}
     exit 1
+  rescue DenCli::UsageError, RuntimeError
+    @out.failed "%s", $!
+    exit 1
+  end
+
+  def per_argument arguments, print: nil, &exe
+    arguments.each do |argument|
+      @out.host = argument
+      @out.info "\e[1;34m#{print}\e[0m", argument  if print
+      begin
+        yield argument
+      rescue RestClient::ExceptionWithResponse
+        @out.failed "%s: %s - %s", $!.request, $!, JSON.parse( $!.response.body)
+      rescue DenCli::UsageError, RuntimeError
+        @out.failed "%s", $!
+      rescue RestClient::BadRequest
+        @out.failed "%p", $!.message
+      rescue Interrupt
+        @out.failed "Interrupted by user"
+      rescue SystemExit
+        @out.failed "Exitcode: %d", $!.status  if 0 < $!.status
+      end
+    end
   end
 
   def appliances node, regexp, system, applications
@@ -274,4 +511,49 @@ class PVE::Cli
       end
     to.print order: [1,2]
   end
+
+  COLORS = %w[black red green yellow blue magenta cyan white].each_with_index.to_h
+  def fgcolor color
+    color = color.to_s
+    if /\Abright[-_]?(.*)\z/ =~ color
+      c = COLORS[$1]
+      c.nil? ? nil : "1;3#{c}"
+    else
+      c = COLORS[color]
+      c.nil? ? nil : "3#{c}"
+    end
+  end
+
+  def tablized_virt v
+    ha = v.respond_to?( :ha) ? v.ha : nil
+    unknown = TablizedOutput::V.new 0, '-'
+    node = v.node.is_a?(String) ? v.node : v.node.node
+    if color = @cfg[:hosts]&.[](node.to_sym)&.[](:color)
+      node = ColoredString.new node, fgcolor( color)
+    end
+    [
+      case v.status
+      when :running, :online  then ColoredString.new v.status, "32"
+      when :stopped, :offline then ColoredString.new v.status, "31"
+      else v.status
+      end,
+      ha&.state || '·',
+      case v.t
+      when "nd" then ColoredString.new v.sid, "33"
+      when "qm" then ColoredString.new v.sid, "35"
+      when "ct" then ColoredString.new v.sid, "36"
+      else v.sid
+      end,
+      v.name, node,
+      v.respond_to?(:uptime) ? TablizedOutput::V.new( v.uptime, Measured.seconds( v.uptime)) : unknown,
+      v.respond_to?(:cpu)    ? TablizedOutput::Percentage.new( v.cpu) : unknown,
+      v.respond_to?(:mem)    ? TablizedOutput::V.new( v.mem, Measured.bytes( v.mem)) : unknown,
+      v.respond_to?(:maxmem) ? TablizedOutput::Percentage.new( v.mem/v.maxmem.to_f) : unknown,
+      v.respond_to?(:disk)   ? TablizedOutput::V.new( v.disk.to_i, Measured.bytes( v.disk.to_i)) : unknown,
+      if v.respond_to?(:maxdisk) and 0 < v.maxdisk.to_i
+        TablizedOutput::Percentage.new( v.disk.to_f/v.maxdisk.to_f)
+      else unknown end,
+      v.respond_to?(:tags)   ? v.tags.join(', ') : '',
+    ]
+  end
 end

lib/pve/cli/base.rb

@@ -15,8 +15,8 @@ def cli_base
       each {|c| puts c }
   }
 
-  cli.cmd( :status, "Lists Nodes/VMs/CTs with status", &lambda {|target=nil, sort:, node: nil, status: nil|
-    hosting_table target: target, status: status, sort: sort do |push|
+  cli.cmd( :status, "Lists Nodes/VMs/CTs with status", &lambda {|target=nil, sort:, node: nil, status: nil, tags: nil|
+    hosting_table target: target, status: status, sort: sort, tags: tags do |push|
       node_opt( node).
         each( &push).lazy.
         flat_map {|n| [ Thread.new( n, &:lxc), Thread.new( n, &:qemu) ] }.
@@ -25,7 +25,16 @@ def cli_base
   }).
     opt( :sort, '-s', '--sort=COLUMNS', "Sort by COLUMNs eg hn for host and name ([s]tatus, h[a], [i]d, [n]ame (default), [h]ost, [u]ptime, [c]pu, [m]em, [d]isk)", default: 'n').
     opt( :node, '-n', '--node=NODE', "List only hosted by this NODE").
-    opt( :status, '-S', '--status=STATUS', "Filter for status (running, stopped, ...) (default: no filter)")
+    opt( :status, '-S', '--status=STATUS', "Filter for status (running, stopped, ...) (default: no filter)").
+    opt( :tags, '-t', '--tags=TAGS', "Filter by comma-seperated tags.  All tags must be present for Machine")
+
+  def val2str v
+    case v
+    when true then 1
+    when false then 0
+    else v
+    end
+  end
 
   def prepare_show_config cnf
     r = {}
@@ -37,18 +46,15 @@ def cli_base
             net.
               reject {|k, v| :card == k }.
               sort_by {|k, v| :name == k ? :AAAAAAAAA : k }.
-              map {|k, v| case v when true then [k,1] when false then [k,0] else [k,v] end }.
-              map {|k, v| "#{k}=#{v}" }
-          r[net[:card].to_sym] = s.join(",")
+              map {|k, v| "#{k}=#{val2str v}" }
+          r[net[:card].to_sym] = s.join ','
         end
       when :sshkeys
-        r[k] = CGI.unescape(v).gsub( /^/, " "*14).gsub /\A {14}|\n\z/, ''
+        r[k] = CGI.unescape( v).gsub( /^/, " "*14).gsub /\A {14}|\n\z/, ''
+      when :features
+        r[k] = v.map {|k, v| "#{k}=#{val2str v}" }.join ','
       else
-        case v
-        when true then v = 1
-        when false then v = 0
-        end
-        r[k] = v.to_s.gsub( /$^/, " "*14).gsub /\n\z/, ''
+        r[k] = val2str( v).to_s.gsub( /$^/, " "*14).gsub /\n\z/, ''
       end
     end
     r
@@ -56,20 +62,21 @@ def cli_base
 
   def show_config cnf, old = nil
     cnf = prepare_show_config cnf
+    l = cnf.keys.map( &:length).max
     if old
       old = prepare_show_config old
       (cnf.keys+old.keys).uniq.sort.each do |k|
         v, o = cnf[k], old[k]
         if v == o
-          puts "#{k}:#{' ' * (12-k.length)} #{v}"
+          puts "#{k}:#{' ' * (l-k.length)} #{v}"
         else
-          puts "\e[31m#{k}:#{' ' * (12-k.length)} #{o}\e[0m"  unless o.nil?
-          puts "\e[32m#{k}:#{' ' * (12-k.length)} #{v}\e[0m"  unless v.nil?
+          puts "\e[31m#{k}:#{' ' * (l-k.length)} #{o}\e[0m"  unless o.nil?
+          puts "\e[32m#{k}:#{' ' * (l-k.length)} #{v}\e[0m"  unless v.nil?
         end
       end
     else
-      cnf.sort_by{|k,v|k}.each do |k,v|
-        puts "#{k}:#{' ' * (12-k.length)} #{v}"
+      cnf.sort_by {|k,_| k }.each do |k,v|
+        puts "#{k}:#{' ' * (l-k.length)} #{v}"
       end
     end
   end
@@ -86,17 +93,26 @@ def cli_base
       opts = {}
       until args.empty?
         case arg = args.shift
+        when /\A--no-(\w+)\z/
+          opts[$1.to_sym] = false
         when /\A--(\w+)=(.*)\z/
           opts[$1.to_sym] = $2
+        when /\A--(\w+)!\z/, /\A--del-(\w+)\z/
+          opts[$1.to_sym] = nil
         when /\A--(\w+)\z/
-          opts[$1.to_sym] = args.shift
+          n = $1.to_sym
+          opts[n] =
+            if args.empty? or /\A--/ == args.first
+              true
+            else opts[n] = args.shift
+            end
         else
           raise UsageError, "Expection option to set. What do you mean with: #{arg}"
         end
       end
       opts.each do |k, v|
         opts[k] =
-          case v = opts[k]
+          case v
           when '' then nil
           else v
           end
@@ -121,14 +137,15 @@ def cli_base
         next  unless opts.has_key? k
         opts[k] =
           case v = opts[k]
-          when *%w[1 T TRUE t true True Y YES y yes Yes] then true
-          when *%w[0 F FALSE f false False N NO n no No] then false
+          when true, *%w[1 T TRUE t true True Y YES y yes Yes] then true
+          when false, *%w[0 F FALSE f false False N NO n no No] then false
           when '', 'nil', nil then nil
           else raise UsageError, "Boolean expected, given: #{v.inspect}"
           end
       end
       connect
       th = Proxmox::LXC.find( name_or_id) || Proxmox::Qemu.find_by_name( name_or_id)
+      raise UsageError, "Container or Node not found: #{name_or_id}"  unless th
       old = th.config
       opts[:digest] ||= old[:digest]
       th.cnfset opts
@@ -138,6 +155,7 @@ def cli_base
     ccli.cmd :show, "Show Config of CT/VM", &lambda {|name_or_id|
       connect
       th = Proxmox::LXC.find( name_or_id) || Proxmox::Qemu.find_by_name( name_or_id)
+      raise UsageError, "Container or Node not found: #{name_or_id}"  unless th
       show_config th.config
     }
   end
@@ -154,11 +172,14 @@ def cli_base
       end
     end
 
-  cli.cmd( :run, "Starts CT/VM", aliases: %w[start star], &lambda {|name_or_id|
+  cli.cmd( :run, "Starts CT/VM", aliases: %w[start star], &lambda {|*names_or_ids|
     connect
-    th = Proxmox::LXC.find( name_or_id) || Proxmox::Qemu.find_by_name( name_or_id)
-    raise UsageError, "Container or Node not found: #{name_or_id}"  unless th
-    start th
+    raise UsageError, "Nothing to start?"  if names_or_ids.empty?
+    per_argument names_or_ids, print: "\e[34;1mRunning CT/VM\e[0m %s:\n" do |name_or_id|
+      th = Proxmox::LXC.find( name_or_id) || Proxmox::Qemu.find_by_name( name_or_id)
+      raise UsageError, "Container or Node not found: #{name_or_id}"  unless th
+      start th
+    end
   }).
     completion do |*pre, arg|
       completion_helper *pre, arg do |f|
@@ -166,6 +187,21 @@ def cli_base
       end
     end
 
+  cli.cmd( :migrate, "Migrates (halted) CTs/VMs to an other host", min: 2, &lambda {|target, *names_or_ids, fire:, timeout:, secs:|
+    connect
+    node = Proxmox::Node.find_by_name! target
+    per_argument names_or_ids, print: "\e[1;34mCT/VM(s)\e[0m %s:\n" do |name_or_id|
+      th = Proxmox::LXC.find( name_or_id) || Proxmox::Qemu.find_by_name( name_or_id)
+      raise UsageError, "CT/VM not found: #{name_or_id}"  unless th
+      unless th.stopped?
+        raise UsageError, "VM #{name_or_id} is running.  Shutdown or for VM-online-migration see `help qm migrate`"  if Proxmox::Qemu === th
+        raise UsageError, "CT #{name_or_id} is running.  You have to shutdown it."
+      end
+      task = th.migrate node
+      wait task, text: "Migrating", timeout: timeout  unless fire
+    end
+  }).tap {|c| opts_wait c }
+
   #cli.cmd :reboot, "Reboot CT/VM (not implemented, yet)", min: 6, &lambda {|name_or_id|
   #  connect
   #  th = Proxmox::LXC.find( name_or_id) || Proxmox::Qemu.find_by_name( name_or_id)
@@ -173,11 +209,13 @@ def cli_base
   #  reboot th
   #}
 
-  cli.cmd( :stop, "Stops CT/VM", min: 4, &lambda {|name_or_id|
+  cli.cmd( :stop, "Stops CT/VM", min: 4, &lambda {|*names_or_ids|
     connect
-    th = Proxmox::LXC.find( name_or_id) || Proxmox::Qemu.find_by_name( name_or_id)
-    raise UsageError, "Container or Node not found: #{name_or_id}"  unless th
-    stop th
+    per_argument names_or_ids, print: "\e[34;1mStopping CT/VM\e[0m %s:\n" do |name_or_id|
+      th = Proxmox::LXC.find( name_or_id) || Proxmox::Qemu.find_by_name( name_or_id)
+      raise UsageError, "Container or Node not found: #{name_or_id}"  unless th
+      stop th
+    end
   }).
     completion do |*pre, arg|
       completion_helper *pre, arg do |f|

lib/pve/cli/ct.rb

@@ -9,40 +9,60 @@ def cli_ct
       end.sort.each {|c| puts c }
     }
 
-    ct_cli.cmd( :status, "Lists CTs with status", aliases: [nil], &lambda {|target=nil, sort: nil, node: nil, status: nil|
-      hosting_table target: target, status: status, sort: sort do |push|
-        node_opt( node).
-          each( &push).lazy.
+    ct_cli.cmd( :status, "Lists CTs with status", aliases: [nil], &lambda {|target=nil, sort: nil, node: nil, status: nil, tags: nil|
+      hosting_table target: target, status: status, sort: sort, tags: tags do |push|
+        node_opt( node).lazy.
           map {|n| Thread.new n, &:lxc }.
           each {|n| n.value.each &push }
       end
     }).
       opt( :sort, '-s', '--sort=COLUMNS', "Sort by COLUMNs eg hn for host and name ([s]tatus, h[a], [i]d, [n]ame (default), [h]ost, [u]ptime, [c]pu, [m]em, [d]isk)", default: 'n').
       opt( :node, '-n', '--node=NODE', "List only hosted by this NODE").
-      opt( :status, '-S', '--status=STATUS', "Filter for status (running, stopped, ...) (default: no filter)")
+      opt( :status, '-S', '--status=STATUS', "Filter for status (running, stopped, ...) (default: no filter)").
+      opt( :tags, '-t', '--tags=TAGS', "Filter by comma-seperated tags.  All tags must be present for CT")
 
     ct_cli.cmd :enter, "Enter Console of CT", &lambda {|name_or_id|
       connect
-      STDERR.puts "! #{$?.exitstatus}"  unless Proxmox::LXC.find!( name_or_id).enter
+      @out.host = name_or_id
+      @out.failed "Exitstatus #{$?.exitstatus}"  unless Proxmox::LXC.find!( name_or_id).enter
     }
 
     ct_cli.cmd :exec, "Executes Command in CT", min: 4, &lambda {|name_or_id, *command|
       connect
-      STDERR.puts "! #{$?.exitstatus}"  unless Proxmox::LXC.find!( name_or_id).exec *command
+      @out.host = name_or_id
+      @out.failed "Exitstatus #{$?.exitstatus}"  unless Proxmox::LXC.find!( name_or_id).exec *command
     }
 
-    ct_cli.cmd( :start, "Starts CT", min: 4, &lambda {|name_or_id, node: nil, fire:, timeout:, secs:|
+    ct_cli.cmd( :migrate, "Migrates halted CT(s) to an other host", min: 2, &lambda {|target, *names_or_ids, fire:, timeout:, secs:|
       connect
-      ct = Proxmox::LXC.find! name_or_id
-      start ct, node: node, fire: fire, timeout: timeout, secs: secs
-    }).
-      opt( :node, "-nNODE", "--node=NODE", "On NODE (default, as is, so without migration)").
-      tap {|c| opts_wait c }
+      node = Proxmox::Node.find_by_name! target
+      per_argument names_or_ids, print: "Migrate CT:" do |name_or_id|
+        ct = Proxmox::LXC.find! name_or_id
+        task = ct.migrate node
+        wait task, text: "Migrating", timeout: timeout  unless fire
+      end
+    }).tap {|c| opts_wait c }
 
-    ct_cli.cmd( :stop, "Stops CT", min: 3, &lambda {|name_or_id, fire: nil, timeout:, secs:|
+    ct_cli.cmd( :volume_move, "Moves volume to an other storage/pool and marks old volume as unused.", min: 11, aliases: %i[volmv mv], &lambda {|name_or_id, volume, destination|
       connect
       ct = Proxmox::LXC.find! name_or_id
-      stop ct, fire: fire, timeout: timeout, secs: secs
+      ct_volume_move ct, volume, destination
+    })
+
+    ct_cli.cmd( :start, "Starts CT(s)", min: 4, &lambda {|*names_or_ids, node: nil, fire:, timeout:, secs:|
+      connect
+      per_argument names_or_ids, print: "Start CT:" do |name_or_id|
+        ct = Proxmox::LXC.find! name_or_id
+        start ct, node: node, fire: fire, timeout: timeout, secs: secs
+      end
+    }).tap {|c| opts_wait c }
+
+    ct_cli.cmd( :stop, "Stops CT(s)", min: 3, &lambda {|*names_or_ids, fire: nil, timeout:, secs:|
+      connect
+      per_argument names_or_ids, print: "Start CT:" do |name_or_id|
+        ct = Proxmox::LXC.find! name_or_id
+        stop ct, fire: fire, timeout: timeout, secs: secs
+      end
     }).tap {|c| opts_wait c }
 
     ct_cli.cmd( :wait, "Wait till CT is in state", &lambda {|name_or_id, state, timeout: nil, secs: nil|

lib/pve/cli/qm.rb

@@ -1,7 +1,7 @@
 class PVE::Cli
 def cli_qm
-  cli.sub :qm, "Virtual Machines", aliases: %w[v vm qemu], &lambda {|qm|
-    qm.cmd :list, "List VM-IDs", aliases: ['ls'], &lambda {|node=nil|
+  cli.sub :qm, "Virtual Machines", aliases: %w[v vm qemu], &lambda {|qm_cli|
+    qm_cli.cmd :list, "List VM-IDs", aliases: ['ls'], &lambda {|node=nil|
       connect
       nodes = Proxmox::Node.all
       nodes = nodes.select {|n| node == n.name }  if node
@@ -10,31 +10,72 @@ def cli_qm
       end.sort.each {|c| puts c }
     }
 
-    qm.cmd( :status, "Lists CTs with status", aliases: [nil], &lambda {|target=nil, sort: nil, node: nil, status: nil|
-      hosting_table target: target, state: state, sort: sort do |push|
-        node_opt( node).
-          each( &push).lazy.
-          flat_map {|n| [ Thread.new( n, &:lxc), Thread.new( n, &:qemu) ] }.
+    qm_cli.cmd( :migrate, "Migrates VM(s) to an other host", min: 2, &lambda {|target, *names_or_ids, fire:, timeout:, secs:|#, online:, restart:|
+      #if online and restart
+      #  raise UsageError, "You have to decide for one migration-mode: --restart or --online."
+      #end
+      #online = !restart or online
+      connect
+      node = Proxmox::Node.find_by_name! target
+      per_argument names_or_ids, print: "\e[1;34mVM\e[0m %s:\n" do |name_or_id|
+        qm = Proxmox::Qemu.find! name_or_id
+        task = qm.migrate node #, online: online
+        wait task, text: "Migrating", timeout: timeout  unless fire
+      end
+    }).
+      #opt( :online, '-o', '--online', "Online-migration: Does not shutdown or interrupt running VM. Opposite of --online, Default", default: nil).
+      #opt( :restart, '-r', '--restart', "Restart-migration: Does shutdown or interrupt running VM. Opposite of --restart", default: nil).
+      tap {|c| opts_wait c }
+
+    qm_cli.cmd( :start, "Starts VM(s)", min: 4, &lambda {|*names_or_ids, node: nil, fire:, timeout:, secs:|
+      connect
+      per_argument names_or_ids, print: "\e[1;34mStart VM %s:\e[0m" do |name_or_id|
+        qm = Proxmox::Qemu.find! name_or_id
+        start qm, node: node, fire: fire, timeout: timeout, secs: secs
+      end
+    }).tap {|c| opts_wait c }
+
+    qm_cli.cmd( :stop, "Stops VM(s)", min: 3, &lambda {|*names_or_ids, fire: nil, timeout:, secs:|
+      connect
+      per_argument names_or_ids, print: "\e[1;34mStart VM %s:\e[0m" do |name_or_id|
+        qm = Proxmox::Qemu.find! name_or_id
+        stop qm, fire: fire, timeout: timeout, secs: secs
+      end
+    }).tap {|c| opts_wait c }
+
+    qm_cli.cmd( :wait, "Wait till VM is in state", &lambda {|name_or_id, state, timeout: nil, secs: nil|
+      connect
+      qm = Proxmox::Qemu.find! name_or_id
+      wait qm, state, timeout: timeout, secs: secs
+    }).
+      opt( :timeout, "-tTIMEOUT", "--timeout=TIMEOUT", "Wait for max TIMEOUT seconds (default: endless)", default: nil).
+      opt( :secs, "-sSECONDS", "--seconds=SECONDS", "Check every SECONDS for state (default: 0.2)", default: 0.2)
+
+    qm_cli.cmd( :status, "Lists VMs with status", aliases: [nil], &lambda {|target=nil, sort: nil, node: nil, status: nil, tags: nil|
+      hosting_table target: target, status: status, sort: sort, tags: tags do |push|
+        node_opt( node).lazy.
+          flat_map {|n| [ Thread.new( n, &:qemu) ] }.
           each {|n| n.value.each &push }
       end
     }).
       opt( :sort, '-s', '--sort=COLUMNS', "Sort by COLUMNs eg hn for host and name ([s]tatus, h[a], [i]d, [n]ame (default), [h]ost, [u]ptime, [c]pu, [m]em, [d]isk)", default: 'n').
       opt( :node, '-n', '--node=NODE', "List only hosted by this NODE").
-      opt( :status, '-S', '--status=STATUS', "Filter for status (running, stopped, ...) (default: no filter)")
+      opt( :status, '-S', '--status=STATUS', "Filter for status (running, stopped, ...) (default: no filter)").
+      opt( :tags, '-t', '--tags=TAGS', "Filter by comma-seperated tags.  All tags must be present for CT")
 
-    qm.cmd :exec, "Executes Command in VM via qemu-guest-agent", min: 4, &lambda {|name_or_id, *command|
+    qm_cli.cmd :exec, "Executes Command in VM via qemu-guest-agent", min: 4, &lambda {|name_or_id, *command|
       connect
       STDERR.puts "! #{$?.exitstatus}"  unless Proxmox::Qemu.find!( name_or_id).exec *command
     }
 
-    qm.cmd( :resize, 'Resize a disk', &lambda {|name_or_id, disk, size|
+    qm_cli.cmd( :resize, 'Resize a disk', &lambda {|name_or_id, disk, size|
       connect
       qm = Proxmox::Qemu.find! name_or_id
       task = qm.resize disk, size
       wait task, text: "Resizing #{qm.sid} #{disk} to #{size}"
     })
 
-    qm.cmd 'help', '', aliases: ['-h', '--help'], &lambda {|*args| help qm, *args }
+    qm_cli.cmd 'help', '', aliases: ['-h', '--help'], &lambda {|*args| help qm_cli, *args }
   }
 end
 end

lib/pve/cli/storage.rb

@@ -40,7 +40,7 @@ def cli_storage
         storage = node.storage.select {|sm| storage == sm.storage }.first
         storage.content.each {|c| puts c.to_s }
       }
-      cli_cnt.cmd( :help, '', aliases: ['-h', '--help']) {|*args| help cli_cnt, *args }
+      cli_cnt.cmd( :help, '', aliases: ['-h', '--help', nil]) {|*args| help cli_cnt, *args }
     end
 
     cli_sm.cmd( :help, '', aliases: ['-h', '--help']) {|*args| help cli_sm, *args }

lib/pve/cli/task.rb

@@ -1,4 +1,33 @@
 class PVE::Cli
+  def task_table order: nil, &exe
+    to = TablizedOutput.new %w[S Starttime Node SID Type UPID], format: '<<<<<<'
+    hosted = {}
+    Proxmox::Hosted.all.each {|h| hosted[h.vmid.to_i] = h }
+    hosted.delete nil
+    exe.call lambda {|t|
+      u = t.upid
+      v = u.id ? hosted[u.id.to_i] : t.node
+      to.push [
+        case t.status.state
+        when :running then ColoredString.new '...', '30'
+        when :success then ColoredString.new 'OK', '32'
+        when :failed  then ColoredString.new 'failed', '31'
+        end,
+        u.starttime.strftime( '%Y-%m-%d %H:%M:%S'),
+        u.node,
+        case v&.t
+        when 'nd' then ColoredString.new v.sid, '33'
+        when 'qm' then ColoredString.new v.sid, '35'
+        when 'ct' then ColoredString.new v.sid, '36'
+        when nil then u.id.inspect
+        else v.sid
+        end,
+        u.dtype, u.upid
+      ]
+    }
+    to.print order: order
+  end
+
 def cli_task
   cli.sub :task, "Inspect tasks" do |tcli|
     tcli.cmd :list, "List done tasks", &lambda {|node=nil|
@@ -7,7 +36,7 @@ def cli_task
       nodes = nodes.select {|n| node == n.name }  if node
       nodes.flat_map do |n|
         n.tasks.map &:upid
-      end.sort.each {|upid| puts upid }
+      end.sort_by(&:upid).each {|upid| puts upid }
     }
 
     tcli.cmd :get, "Inspect a task", &lambda {|upid|
@@ -21,6 +50,40 @@ def cli_task
         end
       end
     }
+
+    tcli.cmd( :status, "Lists tasks with status", aliases: [nil], &lambda {|target=nil, sort: nil, node: nil, status: nil|
+      connect
+      task_table order: [2] do |push|
+        Proxmox::Node.all.each {|n| n.tasks.each &push }
+      end
+    })
+
+    tcli.cmd( :monitor, "Monitors running tasks", &lambda {|node: nil|
+      connect
+      nodes = Proxmox::Node.all
+      nodes = nodes.select {|n| node == n.name }  if node
+      tasks = {}
+      nodes.each {|n| n.tasks.each {|t| tasks[t.upid.upid] = true } }
+      begin
+        loop do
+          task_table order: [2] do |push|
+            begin
+              nodes.
+                flat_map {|n| n.tasks }.
+                select {|t| tasks[t.upid.upid] != true or t.running? }.
+                each( &push)
+              STDERR.print "\e[2J\e[1;1H"
+            rescue RestClient::InternalServerError
+            end
+          end
+          sleep 1
+        end
+      rescue Interrupt
+        STDERR.print "\e[2J\e[1;1H"
+      end
+    })
+
+    tcli.cmd( :help, '', aliases: ['-h', '--help']) {|*args| help ct_cli, *args }
   end
 end
 end

lib/pve/helper.rb

@@ -163,7 +163,14 @@ class TablizedOutput
     end
     @stdout.puts \
       @header.each_with_index.map {|s, i|
-        "#{' ' * (@maxs[i] - s.length)}#{s}"
+        #"#{' ' * (@maxs[i] - s.length)}#{s}"
+        pad = ' ' * (@maxs[i] - s.length)
+        case @format[i]
+        when '<'
+          "#{s}#{pad}"
+        else
+          "#{pad}#{s}"
+        end
       }.join( ' | ')
     ls.each_with_index do |l, i|
       @stdout.puts \
@@ -178,32 +185,4 @@ class TablizedOutput
         }.join( "\e[3#{i.even? ? 6 : 3}m | \e[0m")
     end
   end
-
-  def virt v
-    ha = v.respond_to?( :ha) ? v.ha : nil
-    unknown = V.new 0, '-'
-    push [
-      case v.status
-      when "running", "online" then ColoredString.new v.status, "32"
-      when "stopped" then ColoredString.new v.status, "31"
-      else v.status
-      end,
-      ha&.state || '·',
-      case v.t
-      when "nd" then ColoredString.new v.sid, "33"
-      when "qm" then ColoredString.new v.sid, "35"
-      when "ct" then ColoredString.new v.sid, "36"
-      else v.sid
-      end,
-      v.name, v.node.is_a?(String) ? v.node : v.node.node,
-      v.respond_to?(:uptime) ? V.new( v.uptime, Measured.seconds( v.uptime)) : unknown,
-      v.respond_to?(:cpu) ? Percentage.new( v.cpu) : unknown,
-      v.respond_to?(:mem) ? V.new( v.mem, Measured.bytes( v.mem)) : unknown,
-      v.respond_to?(:maxmem) ? Percentage.new( v.mem/v.maxmem.to_f) : unknown,
-      v.respond_to?(:disk) ? V.new( v.disk.to_i, Measured.bytes( v.disk.to_i)) : unknown,
-      if v.respond_to?(:maxdisk) and 0 < v.maxdisk.to_i
-        Percentage.new( v.disk.to_f/v.maxdisk.to_f)
-      else unknown end,
-    ]
-  end
 end

lib/pve/proxmox.rb

@@ -5,6 +5,13 @@ require 'ipaddress'
 require 'shellwords'
 require 'active_support/all'
 
+class RestClient::Exception
+  attr_reader :request
+  def to_s
+    "#{request|"?"}: #{message}: #{response}"
+  end
+end
+
 module Proxmox
   class Exception < ::Exception
   end
@@ -12,6 +19,34 @@ module Proxmox
   end
   class AlreadyExists < Exception
   end
+  class AlreadyLocked < Exception
+  end
+  class UnparsableUPID < Exception
+  end
+
+  def self.cnfstr2hash str
+    str.
+      split( ',').
+      map do |o|
+        k, v = o.split( '=', 2)
+        [k.to_sym, v]
+      end.
+      to_h
+  end
+
+  def self.hash2cnfstr **opts
+    opts.
+      map do |k, v|
+        v =
+          case v
+          when true then 1
+          when false then 0
+          else v.to_s
+          end
+        "#{k}=#{v}"
+      end.
+      join( ',')
+  end
 
   def self.connect username, password, realm: nil, verify_tls: nil, uri: nil
     uri ||= 'https://localhost:8006/api2/json'
@@ -51,7 +86,7 @@ module Proxmox
   end
 
   def self.find_by_vmid vmid
-    Proxmox::LXC.find_by_vmid( vmid) || Proxmox::Qemu.find_by_vmid( vmid) || Proxmox::Node.find_by_vmid( vmid)
+    Proxmox::LXC.find_by_vmid( vmid) || Proxmox::Qemu.find_by_vmid( vmid)
   end
 
   def self.find name_or_id
@@ -95,19 +130,37 @@ module Proxmox
     def rest_get path, **data, &exe
       data = data.delete_if {|k,v|v.nil?}
       path += "#{path.include?( ??) ? ?& : ??}#{data.map{|k,v|"#{CGI.escape k.to_s}=#{CGI.escape v.to_s}"}.join '&'}"  unless data.empty?
+      STDERR.puts "GET #{path} <= #{data}"  if $DEBUG
       __response__ Proxmox.connection[path].get( __headers__( :'Content-Type' => 'application/json'))
+    rescue RestClient::Exception
+      $!.instance_variable_set :@request, "GET #{path}"
+      raise
     end
 
     def rest_put path, **data, &exe
-      __response__ Proxmox.connection[path].put( __data__( data), __headers__( :'Content-Type' => 'application/json'))
+      data = __data__( **data)
+      STDERR.puts "PUT #{path} <= #{data}"  if $DEBUG
+      __response__ Proxmox.connection[path].put( data, __headers__( :'Content-Type' => 'application/json'))
+    rescue RestClient::Exception
+      $!.instance_variable_set :@request, "PUT #{path}"
+      raise
     end
 
     def rest_del path, &exe
+      STDERR.puts "DELETE #{path}"  if $DEBUG
       __response__ Proxmox.connection[path].delete( __headers__( :'Content-Type' => 'application/json'))
+    rescue RestClient::Exception
+      $!.instance_variable_set :@request, "DELETE #{path}"
+      raise
     end
 
     def rest_post path, **data, &exe
-      __response__ Proxmox.connection[path].post( __data__( data), __headers__( :'Content-Type' => 'application/json'))
+      data = __data__( data)
+      STDERR.puts "POST #{path} <= #{data}"  if $DEBUG
+      __response__ Proxmox.connection[path].post( data, __headers__( :'Content-Type' => 'application/json'))
+    rescue RestClient::Exception
+      $!.instance_variable_set :@request, "POST #{path}"
+      raise
     end
   end
 
@@ -178,6 +231,12 @@ module Proxmox
 
     attr_reader :name
 
+    def __update__ **data
+      data[:status] &&= data[:status].to_sym
+      super **data
+    end
+    private :__update__
+
     def === t
       @name =~ t or @vmid.to_s =~ t or @sid =~ t
     end
@@ -192,8 +251,8 @@ module Proxmox
       @name = @node
     end
 
-    def offline?()  @status.nil? or 'offline' == @status  end
-    def online?()  'online' == @status  end
+    def offline?()  @status.nil? or :offline == @status  end
+    def online?()  :online == @status  end
 
     def lxc
       return []  if offline?
@@ -242,8 +301,23 @@ module Proxmox
       def refresh!
         d = rest_get @rest_prefix
         d[:starttime] &&= Time.at d[:starttime]
-        d = {exitstatus: nil}.merge d
-        __update__ d.merge( node: @node, t: 'status', upid: @upid, task: @task)
+        __update__ d.merge( node: @node, upid: @upid, task: @task)
+      end
+
+      def __update__ **data
+        data = data.merge t: 'status'
+        data[:status] = data[:status]&.to_sym
+        data[:exitstatus] = data[:exitstatus]&.to_sym
+        data[:state] =
+          case data[:status]
+          when :running then :running
+          when :stopped
+            case data[:exitstatus]
+            when :OK then :success
+            else :failed
+            end
+          end
+        super **data
       end
 
       def initialize
@@ -257,11 +331,51 @@ module Proxmox
         "#<#{self.class.name}|#{@upid} node=#{@node.node} #{h.join ' '}>"
       end
 
-      def running?() 'running' == @status end
-      def finished?() 'stopped' == @status end
+      def running?()     :running == @status end
+      def finished?()    :stopped == @status end
       alias stopped? finished?
-      def successfull?() stopped? ? 'OK' == @exitstatus : nil end
-      def failed?() stopped? ? 'OK' != @exitstatus : nil end
+      def successfull?() stopped? ? :OK == @exitstatus : nil end
+      def failed?()      stopped? ? :OK != @exitstatus : nil end
+    end
+
+    def running?()     status.running? end
+    def finished?()    status.finished? end
+    alias stopped? finished?
+    def successfull?() status.successfull? end
+    def failed?()      status.failed? end
+
+    class UPID
+      RE =
+        /^
+          UPID :
+          (?<node>[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?) :
+          (?<pid>[0-9A-Fa-f]{8}) :
+          (?<pstart>[0-9A-Fa-f]{8,9}) :
+          (?<starttime>[0-9A-Fa-f]{8}) :
+          (?<dtype>[^:\s]+) :
+          (?<id>[^:\s]*) :
+          (?<user>[^:\s]+) :
+        $/x
+
+      attr_reader :upid, :node, :pid, :pstart, :starttime, :dtype, :id, :user
+      alias to_s upid
+
+      def initialize upid
+        m = RE.match upid
+        raise UnparsableUPID, "UPID cannot be parsed: #{upid.inspect} (#{upid.class})"  unless m
+        @upid      = upid
+        @node      = m[:node]
+        @pid       = m[:pid].to_i 16
+        @pstart    = m[:pstart].to_i 16
+        @starttime = Time.at m[:starttime].to_i( 16)
+        @dtype     = m[:dtype]
+        @id        = case m[:id] when '' then nil else m[:id].to_i end
+        @user      = m[:user]
+      end
+
+      def inspect
+        "#<#{self.class.name} #{@upid}>"
+      end
     end
 
     def rest_prefix
@@ -270,7 +384,7 @@ module Proxmox
 
     def initialize
       rest_prefix
-      @sid = upid
+      @sid, @upid = upid, UPID.new( upid)
     end
 
     def inspect
@@ -287,23 +401,25 @@ module Proxmox
   end
 
   class Hosted < Base
+    def self.all
+      Node.all.flat_map {|n| n.qemu + n.lxc }
+    end
+
     def refresh!
       __update__ rest_get( "#{@rest_prefix}/status/current").merge( node: @node, t: @t)
     end
 
+    def __update__ **data
+      data[:tags] = (data[:tags]||'').split ';'
+      data[:status] &&= data[:status].to_sym
+      super **data
+    end
+    private :__update__
+
     def === t
       @name =~ t or @vmid.to_s =~ t or @sid =~ t
     end
 
-    def migrate node
-      node =
-        case node
-        when Node then node
-        else Node.find!( node.to_s)
-        end
-      Task.send :__new__, node: @node, host: self, upid: rest_post( "#{@rest_prefix}/migrate", target: node.node)
-    end
-
     def start
       Task.send :__new__, node: @node, host: self, upid: rest_post( "#{@rest_prefix}/status/start")
     end
@@ -350,23 +466,23 @@ module Proxmox
           map( &:to_s).
           grep( /\Anet\d+\z/).
           map do |k|
-            nc = {card: k}
-            cnf.delete( k.to_sym).
-              split( ',').
-              each do |f|
-                k, v = f.split( '=', 2)
-                nc[k.to_sym] = v
-              end
-            nc[:ip] &&= IPAddress::IPv4.new nc[:ip]
-            nc[:gw] &&= IPAddress::IPv4.new nc[:gw]
-            nc[:mtu] &&= nc[:mtu].to_i
-            nc[:tag] &&= nc[:tag].to_i
+            nc = Proxmox::cnfstr2hash cnf.delete( k.to_sym)
+            nc[:card] = k
+            nc[:ip]       &&= IPAddress::IPv4.new nc[:ip]
+            nc[:gw]       &&= IPAddress::IPv4.new nc[:gw]
+            nc[:mtu]      &&= nc[:mtu].to_i
+            nc[:tag]      &&= nc[:tag].to_i
             nc[:firewall] &&= 1 == nc[:firewall].to_i
             nc
           end
+      cnf[:features]     &&=
+        Proxmox::cnfstr2hash( cnf[:features]).
+          tap {|f| f[:nesting] &&= '1' == f[:nesting] }
       cnf[:unprivileged] &&= 1 == cnf[:unprivileged]
-      cnf[:memory] &&= cnf[:memory].to_i
-      cnf[:cores] &&= cnf[:cores].to_i
+      cnf[:memory]       &&= cnf[:memory].to_i
+      cnf[:cores]        &&= cnf[:cores].to_i
+      cnf[:tags]         &&= cnf[:tags].split ';'
+      cnf.update cnf.delete( :lxc).map{|k,v|[k.to_sym,v]}.to_h  if cnf[:lxc]
       cnf
     end
 
@@ -377,11 +493,36 @@ module Proxmox
         when true then r[k] = 1
         when false then r[k] = 0
         when nil then r[:delete].push k
+        when Array
+          case k
+          when :tags
+            r[k] = v.join ';'
+          else r[k] = v
+          end
         else r[k] = v
         end
       end
       r.delete :delete  if r[:delete].empty?
-      rest_put "#{@rest_prefix}/config", r
+      rest_put "#{@rest_prefix}/config", **r
+    end
+
+    # it is only allowed to lock local guests, because it is only possible to unlock local guests.
+    def lock reason, &exe
+      cnf = config
+      raise Proxmox::AlreadyLocked, "Machine #{self} Already locked for: #{cnf[:lock].inspect}"  unless cnf[:lock].nil?
+      r = rest_put "#{@rest_prefix}/config", {lock: reason}
+      if block_given?
+        begin return yield
+        ensure unlock
+        end
+      else r
+      end
+    end
+
+    def unlock
+      system 'pct', 'unlock', @vmid.to_s
+      raise Proxmox::UnlockFailed  unless $?.success?
+      #rest_put "#{@rest_prefix}/config", {delete: %w[lock], skiplock: 1}
     end
 
     def resize disk, size
@@ -461,6 +602,15 @@ module Proxmox
       upid = rest_put "#{@rest_prefix}/resize", disk: disk, size: size
       Task.send :__new__, node: @node, host: self, upid: upid
     end
+
+    def migrate target, online: nil
+      target =
+        case target
+        when Node then target
+        else Node.find!( target.to_s)
+        end
+      Task.send :__new__, node: @node, host: self, upid: rest_post( "#{@rest_prefix}/migrate", target: target.node, online: ! !online)
+    end
   end
 
   class LXC < Hosted
@@ -551,6 +701,7 @@ module Proxmox
           #}.map {|k,v| "#{k}=#{v}" }.join( ','),
           swap: tmplt.swap,
           unprivileged: tmplt.unprivileged,
+          features: tmplt.features ? Proxmox::hash2cnfstr( tmplt.features) : nil,
         }.delete_if {|k,v| v.nil? }
 
         temp = LXC.send :__new__, node: node, vmid: options[:vmid], name: name, hostname: options[:hostname]
@@ -579,6 +730,15 @@ module Proxmox
     def enter
       node.enter 'pct', 'enter', vmid
     end
+
+    def migrate target
+      target =
+        case target
+        when Node then target
+        else Node.find!( target.to_s)
+        end
+      Task.send :__new__, node: @node, host: self, upid: rest_post( "#{@rest_prefix}/migrate", target: target.node)
+    end
   end
 
   class HA < Base
@@ -628,7 +788,8 @@ module Proxmox
     end
 
     def state= state
-      rest_put "#{@rest_prefix}", state: state.to_s
+      r = rest_put "#{@rest_prefix}", state: state.to_s
+      pp r: r
       refresh!
     end
 
@@ -651,6 +812,7 @@ module Proxmox
     def stopped?() 'stopped' == self.state end
     def error?() 'error' == self.state end
     def active?() ! ! digest end
+    def exist?() ! ! digest end
   end
 
   class Storage < Base
@@ -665,8 +827,9 @@ module Proxmox
     end
 
     def initialize() rest_prefix end
-
-    def to_s() "#{@node.node}:#{@storage}" end
+    def to_s()     "#{@node.node}:#{@storage}" end
+    def active?()  1 == @active end
+    def enabled?() 1 == @enabled end
 
     class Content < Base
       def rest_prefix

lib/pve/templates.rb

@@ -20,7 +20,18 @@ module PVE::CTTemplate
     def hostname() options.hostname || name end
     def memory() options.memory || 1024 end
     def swap() options.swap || 0 end
-    def unprivileged() options.unprivileged() || 1 end
+    def unprivileged() options.unprivileged || 1 end
+
+    def features
+      r =
+        if options.features
+          Proxmox::cnfstr2hash( options.features).
+            tap {|f| f[:nesting] &&= 1 == f[:nesting] }
+        else {}
+        end
+      r[:nesting] = options.nesting.nil? ? true : options.nesting
+      r
+    end
 
     def ssh_public_keys
       options[:'ssh-public-keys'] ||
@@ -82,6 +93,8 @@ module PVE::CTTemplate
         ipv6:                      [:string,  false, "IPv6-Address with net-size or auto."],
         gateway6:                  [:string,  false, "IPv6-Address of gateway."],
         storage:                   [:string,  false, "Device will be create on this Storage (default: local"],
+        features:                  [:string,  false, "Features"],
+        nesting:                   [:boolean, false, "Support nested containers (->Features)"],
       }
     end
   end
@@ -122,12 +135,14 @@ module PVE::CTTemplate
         gateway6:                  [:string,  false, "IPv6-Address of gateway."],
         storage:                   [:string,  false, "Device will be create on this Storage (default: root)"],
         ostemplate:                [:string,  false, "OS-Template eg. local:vztmpl/superlinux-1.2-amd64.tar.xz"],
+        features:                  [:string,  false, "Features"],
+        nesting:                   [:boolean, false, "Support nested containers (->Features)"],
       }
     end
 
-    def node() options.node || 'svc1' end
-    def ostype() options.ostype || 'debian' end
-    def memory() options.memory || 2048 end
+    def node()    options.node    || 'svc1' end
+    def ostype()  options.ostype  || 'debian' end
+    def memory()  options.memory  || 2048 end
     def storage() options.storage || 'root' end
 
     def network_id

lib/pve/version.rb

@@ -1,3 +1,3 @@
 module PVE
-  VERSION = '0.2.4'
+  VERSION = '0.3.3'
 end

pve.gemspec

@@ -12,8 +12,10 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://git.denkn.at/deac/pve"
   # Ruby3 interpret **opts in another way than before.
   # 2.7.0 should work with both(?) behaviours.
-  # PVE based on debian, so ruby 2.5 is default.
-  spec.required_ruby_version = Gem::Requirement.new "~> 2.5.0"
+  # PVE-7 based on debian-11, so ruby 2.7 is default.
+  # No support for ruby 2.5 anymore (PVE-6 based on debian-10)
+  # Old behaviour will be droped.
+  spec.required_ruby_version = Gem::Requirement.new "~> 2.7"
 
   spec.metadata["homepage_uri"]    = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage