diff --git a/lib/to_lvm_xfs/base.rb b/lib/to_lvm_xfs/base.rb index aaa9b42..70f1c8f 100644 --- a/lib/to_lvm_xfs/base.rb +++ b/lib/to_lvm_xfs/base.rb @@ -11,14 +11,27 @@ end class Pathname def self.which cmd - ENV['PATH'].each do |path| + ENV['PATH'].split(':').each do |path| path = Pathname.new( path) + cmd return path if path.executable? end + nil end end class Base + def task name, &exe + STDERR.puts "\e[30;1m***\e[0m #{name} \e[30;1m#{'*' * [0,80-name.length].max}\e[0m" + if yield + STDERR.puts "\e[32;1m---\e[0m #{name} \e[32;1m#{'-' * [0,80-name.length].max}\e[0m\n" + else + STDERR.puts "\e[35;1m---\e[0m #{name} \e[35;1m#{'-' * [0,80-name.length].max}\e[0m\n" + end + rescue Object + STDERR.puts "\n\e[31;1m|||\e[0m #{name} \e[31;1m#{'|' * [0,80-name.length].max}\e[0m\n" + raise + end + def self.run *args, &exe new( *args).instance_eval &exe end @@ -105,9 +118,10 @@ class Base STDERR.print "\e[0m" end - def check_program cmd + def check_programm cmd path = Pathname.which cmd - raise Error, "#{cmd} not found." unless path&.executable? + raise ProgrammError, "#{cmd} not found." unless path&.executable? + path end attr_reader :sh, :mounted, :looped, :base, :dest, :vgname @@ -118,10 +132,10 @@ class Base end %w[kpartx parted rsync lvm lvs pvs vgs mkfs.xfs mkfs.vfat dmsetup losetup lsblk blkid].each do |cmd| - check_program cmd + check_programm cmd end - @qemu_bin_src = check_program 'qemu-arm-static' - raise Error, "@qemu-arm-static not found. Please install qemu-user-static" unless @qemu_bin_srci&.executable? + @qemu_bin_src = check_programm 'qemu-arm-static' + raise ProgrammError, "qemu-arm-static not found. Please install qemu-user-static" unless @qemu_bin_src&.executable? if :ask == @password v = b = nil @@ -159,7 +173,7 @@ Settings: hostname: #{@hostname || '(default)'} EOF - sh.def_system_commands *%i[echo sed mount umount kpartx sync rsync xz gzip bzip2 zip tar bash dpkg apt] + sh.def_system_commands *%i[echo sed mount umount partx kpartx sync rsync xz gzip bzip2 zip tar bash dpkg apt] sh.def_system_commands *%i[losetup dmsetup lvcreate vgcreate pvcreate vgchange mkswap vgscan] sh.alias_command :losetup_list, *%w[losetup --list --json], return: :json, could_be_empty: true sh.alias_command :pvs, *%w[pvs --noheadings] @@ -182,7 +196,7 @@ EOF exit 1 end - opts.on '-bIMAGE', '--baseimage=IMAGE', 'Write image to IMAGE. Device or file' do |v| + opts.on '-bIMAGE', '--baseimage=IMAGE', 'Read image from IMAGE. Device or file' do |v| @baseimage = v end @@ -364,6 +378,25 @@ EOF dest.root.join( 'etc/hostname').write "#{hostname}\n" end + FSTabMountEntry = + Struct.new :spec, :file, :type, :ops, :freq, :passno do + def to_a() [spec, file, type, ops, freq||0, passno||0] end + def to_s() to_a.join ' ' end + end + + def update_fstab content + cnt = {} + content = content.each do |ent| + cnt[ent.file.to_s] = ent.to_s + end + dest.root.join( 'etc/fstab').replace_i do |f| + f.each_line.flat_map do |l| + mp = l.split( /\s+/)[1] + cnt.delete( mp) || l + end + cnt.values + end + end + def install_packages_from_dir *paths paths.each do |pkgs| next unless pkgs.directory? diff --git a/lib/to_lvm_xfs/exts.rb b/lib/to_lvm_xfs/exts.rb index b86187a..843def0 100644 --- a/lib/to_lvm_xfs/exts.rb +++ b/lib/to_lvm_xfs/exts.rb @@ -15,34 +15,45 @@ class Pathname FileUtils.copy_file src.to_s, self.to_s, **opts end - def symlink to - File.symlink to.to_s, to_s - end + alias link make_link + alias symlink make_symlink - def link to - File.link to.to_s, to_s - end - - def replace_i + def replace_i &e + changed = false open 'r+' do |f| - lines = yield f - f.truncate 0 + old = f.readlines f.pos = 0 - f.puts lines + lines = yield f + changed = lines != old + if changed + f.truncate 0 + f.pos = 0 + f.puts lines + end end + not changed end def sed_i **replaces, &e + changed = false open 'r+' do |f| - lines = f.each_line.flat_map {|l| yield l.chomp, replaces } - f.truncate 0 - f.pos = 0 - f.puts lines - replaces.each {|_,rls| f.puts rls } + lines = + f.each_line.flat_map do |l| + yield( l.chomp, replaces).tap {|r| changed = true unless l == r } + end + changed = true unless replaces.empty? + if changed + f.truncate 0 + f.pos = 0 + f.puts lines + replaces.each {|_,rls| f.puts rls } + end end + not changed end end + class XPathname < Pathname %i[/ + join].each do |meth| define_method meth do |*n| @@ -50,44 +61,53 @@ class XPathname < Pathname end end - %i[unlink read].each do |meth| - define_method meth do |*a, &e| + %i[unlink read mkdir mkpath].each do |meth| + define_method meth do STDERR.puts "\e[1;36m#{meth} \e[1;35m#{self}\e[0m" - super *a, &e + super() end end %i[replace_i sed_i].each do |meth| - define_method meth do |*a, &e| - STDERR.puts "\e[1;36mpatching \e[1;35m#{self}\e[0m" - super *a, &e + define_method meth do |*a, **o, &e| + STDERR.print "\e[1;36mpatching \e[1;35m#{self}\e[0m" + begin + r = super *a, **o, &e + STDERR.puts "\r\e[#{r ? "1;32" : "33"}mpatching \e[1;35m#{self}\e[0m" + rescue Object + STDERR.puts "\r\e[1;31mpatching \e[1;35m#{self}\e[0m" + raise + end + r end end - %i[link rename symlink].each do |meth| - define_method meth do |*a, &e| - STDERR.puts "\e[1;36m#{meth} \e[1;35m#{self}\e[0m -> #{a.map{|x|"\e[1;35m#{x}\e[0m"}.join ', '}" - super *a, &e - end + def rename dest + STDERR.puts "\e[1;36mrename \e[1;35m#{self}\e[0m -> \e[1;35m#{dest}\e[0m" + super dest end - %i[move copy].each do |meth| - define_method meth do |*a, &e| - STDERR.puts "\e[1;36m#{meth} \e[1;35m#{self}\e[0m <- #{a.map{|x|"\e[1;35m#{x}\e[0m"}.join ', '}" - super *a, &e - end + def move src + STDERR.puts "\e[1;36mmove \e[1;35m#{self}\e[0m <- \e[1;35m#{dest}\e[0m" + super src end - %i[make_link make_symlink].each do |meth| - name = meth.to_s.sub /\Amake_/, '' - define_method meth do |*a, &e| - STDERR.puts "\e[1;36m#{name} \e[1;35m#{self}\e[0m -> #{a.map{|x|"\e[1;35m#{x}\e[0m"}.join ', '}" - super *a, &e + def copy src, **opts + STDERR.puts "\e[1;36mcopy \e[1;35m#{self}\e[0m <- \e[1;35m#{src}\e[0m" + super src, **opts + end + + %i[link symlink].each do |meth| + define_method meth do |dest| + STDERR.puts "\e[1;36m#{meth} \e[1;35m#{self}\e[0m -> \e[1;35m#{dest}\e[0m" + super dest end end + alias link make_link + alias symlink make_symlink %i[write].each do |meth| - define_method meth do |*a, &e| + define_method meth do |*a, **o, &e| STDERR.puts "\e[1;36m#{meth} \e[1;35m#{self}\e[0m \e[1;35m#{a[0][0...128].inspect}\e[0m" super *a, &e end diff --git a/lib/to_lvm_xfs/raspbian.rb b/lib/to_lvm_xfs/raspbian.rb index 83d532d..53c5b06 100644 --- a/lib/to_lvm_xfs/raspbian.rb +++ b/lib/to_lvm_xfs/raspbian.rb @@ -1,9 +1,50 @@ require 'to_lvm_xfs' class Raspbian < Base + Sizes = { + '' => 1, + 'b' => 1, + 's' => 512, + 'k' => 1024, + 'm' => 1024*1024, + 'g' => 1024*1024*1024, + 't' => 1024*1024*1024*1024, + 'p' => 1024*1024*1024*1024*1024, + 'e' => 1024*1024*1024*1024*1024*1024, + } + + Volume = Struct.new :sh, :name, :device, :mountpoint, :builder_mp, :size do + def uuid() sh.fs_uuid device end + def type() sh.fs_type device end + + def to_fstab_entry + FSTabMountEntry.new "UUID=#{uuid}", mountpoint, type, 'defaults,noatime', nil, nil + end + end + def initialize *args @vgname = "raspi_#{SecureRandom.urlsafe_base64 5}" + @volumes = { root: ['/', '4.2G'], home: ['/home', '100M'] } super *args + @vgpath = XPathname.new( '/dev') + @vgname + + STDERR.printf "Volumes:\n" + vols = {} + @volumes.each do |name, (mp, size)| + mp = XPathname.new( mp).cleanpath + vols[mp.to_s] = Volume.new sh, name.to_s, @vgpath+name.to_s, mp, dest.root + mp.to_s[1..-1], size + STDERR.printf "%13s: %s (%s)\n", name, mp, size + end + @volumes = vols + end + + def getopts opts + super opts + opts.on '-vNAME', '--volume=NAME', 'Creates additional volume name:mountpoint=size or overwrites defaults.' do |v| + m = %r<\A([0-9a-z_-]+):(/[0-9a-z/_-]*)=([0-9.]+[%a-z]?)\z>i.match v + fail "Volume expected in format: \"name:mountpoint=size\"" unless m + @volumes[m[1].to_sym] = [XPathname.new( m[2]).cleanpath, m[3]] + end end def build @@ -16,182 +57,207 @@ class Raspbian < Base r = sh.losetup_list unless r.empty? or r['loopdevices'] r['loopdevices'].each do |lo| - d "File #{dest.image} used as loop-device back-file", - +dest.image != +XPathname.new(lo['back-file']) + fail "File #{dest.image} used as loop-device back-file" if +dest.image == +XPathname.new( lo['back-file']) end end when dest.image.blockdev?, dest.image.chardev? lsblk( dest.image).each do |l| - d "Device #{l[:name]} mounted at #{l[:mountpoint]}", ! l[:mountpoint] + fail "Device #{l[:name]} mounted at #{l[:mountpoint]}" if l[:mountpoint] end end - sleep 5 + task "Mount base image #{base.image}" do + sh.partx -:u, base.image if base.image.blockdev? - d "Base image does not exist", base.image.exist? - base_parts = kpartx base.image - d "two partitions in base expected, got: #{base_parts.inspect}", 2 == base_parts.length - mount base_parts[1], base.root, -:oro - mount base_parts[0], base.root+'boot', -:oro + fail "Base image does not exist" unless base.image.exist? + base_parts = kpartx base.image + fail "two partitions in base expected, got: #{base_parts.inspect}" unless 2 == base_parts.length + mount base_parts[1], base.root, -:oro + mount base_parts[0], base.root+'boot', -:oro + end dest.image.open 'w' do |f| + size = + @volumes.inject 136*1024*1024 do |s, (_n, vol)| + m = /\A([0-9.]+)([bBsSkKmMgGtTpPeE]?)\z/i.match vol.size + fail "invalid size: #{vol.size}" unless m + s + m[1].to_f * Sizes[m[2].downcase] + end f << 0.chr*4096 - f.pos = 4.8*1024*1024*1024-1 + f.pos = size - 1 f.putc 0.chr end - sh.parted dest.image, *%w[-- - mklabel msdos - mkpart primary fat32 4MB 132MB - mkpart primary ext2 132MB -1s - set 2 LVM on - print] + task "Partitioning destination image #{dest.image}" do + sh.parted dest.image, *%w[-- + mklabel msdos + mkpart primary fat32 4MB 132MB + mkpart primary ext2 132MB -1s + set 2 LVM on + print] + sh.partx -:u, dest.image if dest.image.blockdev? + end *dest_parts = begin lsblk( dest.image).select do |l| - STDERR.puts l sh.dmsetup :remove, File.basename(l[:name]) if 'lvm' == l[:type] l[:name].start_with?( dest.image.to_s) and 'part' == l[:type] end.map {|l| XPathname.new l[:name] }.sort rescue Sh::ProcessError kpartx dest.image end - d "two partitions in destination expected", 2 == dest_parts.length - dest_parts[0].open( 'w') {|f| f << 0.chr*4*1024*1024 } - dest_parts[1].open( 'w') {|f| f << 0.chr*4*1024*1024 } - sh.vgscan '--cache' - vgpath = XPathname.new( '/dev') + vgname - sh.pvcreate -:ff, dest_parts[1] - sh.vgcreate vgname, dest_parts[1] - sh.lvcreate -:nroot, '-L4.2G', vgname - sh.lvcreate -:nhome, '-L100M', vgname - sh.vgchange -:ae, vgname - sh.mkvfat -:nboot, dest_parts[0] - sh.mkxfs -:Lroot, vgpath+'root' - sh.mkxfs -:Lhome, vgpath+'home' - mount vgpath+'root', dest.root + fail "two partitions in destination expected" unless 2 == dest_parts.length + task "Prepare boot-partition #{dest_parts[0]} and lvm #{dest_parts[1]}" do + dest_parts[0].open( 'w') {|f| f << 0.chr*4*1024*1024 } + dest_parts[1].open( 'w') {|f| f << 0.chr*4*1024*1024 } + sh.vgscan '--cache' + sh.pvcreate -:ff, dest_parts[1] + sh.vgcreate vgname, dest_parts[1] + @volumes.each {|_name, vol| sh.lvcreate "-n#{vol.name}", "-L#{vol.size}", vgname } + sh.vgchange -:ae, vgname + sh.mkvfat -:nboot, dest_parts[0] + @volumes.each {|_name, vol| sh.mkxfs "-L#{vol.name}", vol.device } + end + addmp = {run_udev: dest.root+'run/udev'} - %i[home boot dev proc sys].each do |n| - d = addmp[n] = dest.root+n.to_s - d.mkdir - end - mount vgpath+'home', addmp[:home] - mount dest_parts[0], addmp[:boot] + task "Mount all filesystems" do + mount @vgpath+'root', dest.root + (%i[boot dev proc sys]).each do |n| + d = addmp[n] = dest.root+n.to_s + d.mkdir + end + mount dest_parts[0], addmp[:boot] + mount '/dev', addmp[:dev], --:bind + mount 'proc', addmp[:proc], -:tproc + mount 'sysfs', addmp[:sys], -:tsysfs - mount '/dev', addmp[:dev], --:bind - mount 'proc', addmp[:proc], -:tproc - mount 'sysfs', addmp[:sys], -:tsysfs - - sh.rsync_all "#{base.root}/", dest.root - - install_authorized_keys - rename_user - - (dest.root+'etc'+'fstab').replace_i do |f| - replace = { - '/' => "UUID=#{sh.fs_uuid vgpath+'root'} / xfs defaults,noatime 0 0", - '/boot' => "UUID=#{sh.fs_uuid dest_parts[0]} /boot vfat defaults 1 1", - '/home' => "UUID=#{sh.fs_uuid vgpath+'home'} /home xfs defaults,noatime 1 1", - } - f.each_line.flat_map do |l| - mp = l.split( /\s+/)[1] - replace.delete( mp) || l - end + replace.values + @volumes.each do |_name, vol| + path = vol.builder_mp + next if dest.root == path || addmp.values.include?( path) + path.mkdir + mount @vgpath+vol.name, path + end end - (addmp[:boot]+'config.txt').replace_i do |f| - replace = { - 'pi4' => { initramfs: 'initramfs initrd7l.img followkernel', }, - 'pi3' => { initramfs: 'initramfs initrd7.img followkernel', }, - 'pi2' => { initramfs: 'initramfs initrd7.img followkernel', }, - 'pi1' => { initramfs: 'initramfs initrd.img followkernel', }, - 'pi0' => { initramfs: 'initramfs initrd.img followkernel', }, - } - blocks = [nil] - content = Hash.new {|h,block| h[block] = [] } - block = nil - f.each_line do |l| - l.chomp! - case l - when /\A\[([^\]]*)\]\z/ - block = $1 - blocks.push block - when /\Ainitramfs / - l = replace[block].delete :initramfs + task "Copy raspbian from base to dest" do + sh.rsync_all "#{base.root}/", dest.root + end + + task "Prepare users" do + install_authorized_keys + rename_user + end + + task "Prepare /etc and /boot" do + update_fstab @volumes.map { |_name, vol| vol.to_fstab_entry } + + [ FSTabMountEntry.new( "UUID=#{sh.fs_uuid( dest_parts[0])}", '/boot', 'vfat', 'defaults', 1, 1)] + + addmp[:boot].join( 'config.txt').replace_i do |f| + replace = { + 'pi4' => { initramfs: 'initramfs initrd8.img followkernel', arm_64bit: 'arm_64bit=1' }, + 'pi3' => { initramfs: 'initramfs initrd7l.img followkernel', }, + 'pi2' => { initramfs: 'initramfs initrd7.img followkernel', }, + 'pi1' => { initramfs: 'initramfs initrd.img followkernel', }, + 'pi0' => { initramfs: 'initramfs initrd.img followkernel', }, + } + blocks = [nil] + content = Hash.new {|h,block| h[block] = [] } + block = nil + f.each_line do |l| + l.chomp! + case l + when /\A\[([^\]]*)\]\z/ + block = $1 + blocks.push block + when /\Ainitramfs / + l = replace[block].delete :initramfs + end + content[block].push l end - content[block].push l + replace.each {|block, rpl| content[block] += rpl.values + [''] unless rpl.empty? } + blocks.flat_map {|block| content[block] } end - replace.each {|block, rpl| content[block] += rpl.values + [''] unless rpl.empty? } - blocks.flat_map {|block| content[block] } - end - (addmp[:boot]+'ssh').write '' + addmp[:boot].join( 'ssh').write '' - (addmp[:boot]+'cmdline.txt').replace_i do |f| - lines = f.readlines - d "Only one line in cmdline.txt expected", 1 == lines.length - opts = {} - lines[0].split( ' ').each do |line| - /^([^=]*)(?:=(.*))?/ =~ line - opts[$1.to_sym] = $2 + addmp[:boot].join( 'cmdline.txt').replace_i do |f| + lines = f.readlines + fail "Only one line in cmdline.txt expected" unless 1 == lines.length + opts = {} + lines[0].split( ' ').each do |line| + /^([^=]*)(?:=(.*))?/ =~ line + opts[$1.to_sym] = $2 + end + opts[:root] = @vgpath+'root' + opts[:rootfstype] = :xfs + opts.delete :init + opts.map {|k,v| v ? "#{k}=#{v}" : "#{k}" }.join(' ') end - opts[:root] = vgpath+'root' - opts[:rootfstype] = :xfs - opts.delete :init - opts.map {|k,v| v ? "#{k}=#{v}" : "#{k}" }.join(' ') - end - (dest.root+'etc').chdir do - XPathname.glob( 'rc*.d/*resize2fs_once').each do |fn| - fn.unlink + dest.root.join( 'etc').chdir do + XPathname.glob( 'rc*.d/*resize2fs_once').each do |fn| + fn.unlink + end end - end - @qemu_bin.copy @qemu_bin_src, preserve: true + set_hostname + end preload, preload_x = dest.root+'etc/ld.so.preload', dest.root+'etc/ld.so.preload.tp' - preload.rename preload_x + task "Prepare to chroot to raspbian" do + @qemu_bin.copy @qemu_bin_src, preserve: true - ish = sh.chroot( dest.root).chdir( '/') - ish.apt :update - ish.apt :upgrade, -:y - ish.apt :update - - # We mount /run/udev for lvm-scanning - vgs / vgcfgbackup need it to connect to udev. - addmp[:run_udev].mkdir - mount '/run/udev', addmp[:run_udev], --:bind - - # prevent installing exim by installing nullmailer - #ish.apt :install, -:y, :lvm2, :xfsprogs, :nullmailer, :dracut - #dest.root.join( 'etc/dracut.conf.d/10-denkn.conf').open 'w' do |f| - # f.puts 'add_modules+="lvm"' - # f.puts 'add_drivers+="dm-mod xfs"' - # f.puts 'compress="xz"' - #end - - ish.apt :install, -:y, :lvm2, :xfsprogs, 'initramfs-tools' - dest.root.join( 'etc/initramfs-tools/initramfs.conf').replace_i do |f| - replace = { compress: 'COMPRESS=xz', } - f.each_line.flat_map do |l| - case l.chomp! - when /^COMPRESS=/ then replace.delete :compress - when /^# *COMPRESS=/ then [l, replace.delete( :compress)] - else l - end - end + replace.values + preload.rename preload_x end - set_hostname + task "update, upgrade and install" do + ish = sh.chroot( dest.root).chdir( '/') + ish.apt :update + ish.apt :upgrade, -:y + ish.apt :update - install_packages_from_dir( - XPathname.new( $0).expand_path.dirname + 'raspbian-files', - XPathname.new( 'files') - ) + install_packages_from_dir( + XPathname.new( $0).expand_path.dirname + 'raspbian-files', + XPathname.new( 'files') + ) - # generates implicite initramfs - ish.system *%w[dpkg-reconfigure raspberrypi-kernel] + # We mount /run/udev for lvm-scanning - vgs / vgcfgbackup need it to connect to udev. + addmp[:run_udev].mkdir + mount '/run/udev', addmp[:run_udev], --:bind - preload_x.rename preload - @qemu_bin.unlink + # prevent installing exim by installing nullmailer + #ish.apt *%w[install -y lvm2 xfsprogs nullmailer dracut] + #dest.root.join( 'etc/dracut.conf.d/10-denkn.conf').open 'w' do |f| + # f.puts 'add_modules+="lvm"' + # f.puts 'add_drivers+="dm-mod xfs"' + # f.puts 'compress="xz"' + #end + + ish.apt *%w[install -y lvm2 xfsprogs initramfs-tools] + dest.root.join( 'etc/initramfs-tools/initramfs.conf').replace_i do |f| + replace = { compress: 'COMPRESS=xz', } + f.each_line.flat_map do |l| + case l.chomp! + when /^COMPRESS=/ then replace.delete :compress + when /^# *COMPRESS=/ then [l, replace.delete( :compress)] + else l + end + end + replace.values + end + + # strange error while generating initramfs; + # it want to delete this directory, but it does not exist. + #overlays = dest.root + 'usr/share/rpikernelhack/overlays' + #overlays.mkpath + #overlays.join( '.keep').write '' + + # generates implicite initramfs + ish.system *%w[dpkg-reconfigure raspberrypi-kernel] + end + + task "Reverse hacking for chrooting" do + preload_x.rename preload + @qemu_bin.unlink + end end end diff --git a/raspbian-files/raspbian-files.tar b/raspbian-files/raspbian-files.tar index 014902c..19f5c30 100644 Binary files a/raspbian-files/raspbian-files.tar and b/raspbian-files/raspbian-files.tar differ