diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3aa6369 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.*.sw[po] diff --git a/README.adoc b/README.adoc new file mode 100644 index 0000000..672da35 --- /dev/null +++ b/README.adoc @@ -0,0 +1,3 @@ += Dependencies = + + sudo aptitude install git kpartx ruby qemu-system-arm qemu-user-static diff --git a/init.sh b/init.sh new file mode 100644 index 0000000..372cb92 --- /dev/null +++ b/init.sh @@ -0,0 +1,91 @@ +#!/bin/sh -e + +Usage() { + >&2 echo "Usage: $0 hostname [vgname]" + exit 1 +} + +die() { + >&2 echo "died: $*" + exit 1 +} + +msg() { + >&2 printf '\e[34;1m%s\e[0m\n' "$*" +} + +lv_exists() { + true && lvs --noheadings --options lv_name "$1" + case $? in + 0) return 0 ;; + 5) return 1 ;; + *) die "Error while determining, if lv «$1» exists" ;; + esac +} + +alias blkuuid="blkid -ovalue -sUUID" blkfstype="blkid -ovalue -sTYPE" + +[ X = X"$1" ] && Usage +[ X-h = X"$1" ] && Usage +[ X--help = X"$1" ] && Usage + +hostname="$1" +#vgname="${2:-${hostname}sd}" +disc="/dev/mmcblk1" +pvpath="${disc}p2" + +echo "Hostname: «$hostname»" +#echo "VG-name: «$vgname»" +echo + +msg "Install needed tools..." +apt update +apt install -y lvm2 xfsprogs + +msg "Set Hostname «$hostname»..." +oldhostname=`cat /etc/hostname` +sed -i -e "s/$oldhostname/$hostname/g" /etc/hosts +echo "$hostname" > /etc/hostname + +vgname="`pvs --noheadings --options vg_name "$pvpath" | sed -e 's/^ *//'`" +# renaming does not work, because blockdevices will not be created. +# vgknodes, vgscan --mknodes tested. +#if [ "X$oldvgname" = "X$vgname" ] +#then +# msg "VG already named «$vgname»." +#else +# msg "Rename VG in «$vgname»..." +# vgrename "$oldvgname" "$vgname" +#fi + +msg "Expand PV «$pvpath»..." +parted -ms "$disc" -- resizepart 2 -4MB print +pvresize "$pvpath" + +msg "Expand root-LV..." +lvextend -rL4G "$vgname/root" + +if lv_exists "$vgname/swap" +then + msg "LV «swap» already exists." +else + msg "Prepare swap-LV..." + lvcreate -nswap -L2G "$vgname" + mkswap "/dev/$vgname/swap" + echo "UUID=`blkuuid /dev/$vgname/swap` swap swap sw 0 0" >>/etc/fstab +fi + +if lv_exists "$vgname/home" +then + msg "LV «home» already exists" +else + msg "Prepare home-LV..." + lvcreate -nhome -L4G "$vgname" + mkfs.xfs "/dev/$vgname/home" + echo "UUID=`blkuuid /dev/$vgname/home` /home xfs defaults 0 0" >>/etc/fstab +fi + +msg "Install Updates..." +apt dist-upgrade -y + +msg "ok" diff --git a/lib/to_lvm_xfs/base.rb b/lib/to_lvm_xfs/base.rb index 821f277..e0f80b0 100644 --- a/lib/to_lvm_xfs/base.rb +++ b/lib/to_lvm_xfs/base.rb @@ -32,7 +32,7 @@ class Base end def lpartx image - mapper = Pathname.new '/dev/mapper' + mapper = XPathname.new '/dev/mapper' lines = sh.kpartx( -:al, image, return: :lines)[0..-1] if lines.grep( /^loop deleted/).empty? lines.map do |line| @@ -138,16 +138,17 @@ Settings: EOF sh.def_system_commands *%i[echo sed mount umount kpartx sync rsync xz gzip bzip2 zip tar bash dpkg apt] - sh.def_system_commands *%i[dmsetup lvcreate vgcreate pvcreate vgchange mkswap vgscan] + sh.def_system_commands *%i[losetup dmsetup lvcreate vgcreate pvcreate vgchange mkswap vgscan] + sh.alias_command :losetup_list, *%w[losetup --list --json], return: :json, could_be_empty: true sh.alias_command :pvs, *%w[pvs --noheadings] sh.alias_command :blkid, 'blkid', return: :line sh.alias_command :lsblk, *%w[lsblk --noheadings --paths --list], return: :lines - sh.alias_command :parted, *%w[parted -ms] + sh.alias_command :parted, *%w[parted --machine --script] sh.alias_command :mkxfs, *%w[mkfs.xfs -f] sh.alias_command :mkext2fs, *%w[mkfs.ext2] sh.alias_command :mkext4fs, *%w[mkfs.ext4] sh.alias_command :mkvfat, *%w[mkfs.vfat] - sh.alias_command :rsync_all, *%w[rsync -aHAX] + sh.alias_command :rsync_all, *%w[rsync --archive --hard-links --acls --xattrs] sh.alias_command :mount_ro, *%w[mount -oro] sh.alias_command :fs_uuid, *%w[blkid -o value -s UUID], return: :line sh.alias_command :fs_type, *%w[blkid -o value -s TYPE], return: :line @@ -188,19 +189,20 @@ EOF end opts.on '-aAUTHKEYS', '--auth-keys=AUTHKEYS', 'SSH-Keys for user' do |f| - f = Pathname.new f + f = XPathname.new f d "#{f} not found.", f.exist? @authorized_keys = f end opts.on '-AAUTHKEYS', '--root-auth-keys=AUTHKEYS', 'SSH-Keys for root' do |f| - f = Pathname.new f + f = XPathname.new f d "#{f} not found.", f.exist? @root_authorized_keys = f end end def run + nok = false build qemu_bin = dest.root + "usr/bin/qemu-arm-static" @@ -210,19 +212,20 @@ EOF end rescue ProgrammError + nok = true err $! raise ensure - STDERR.puts "="*80 + STDERR.puts "\e[1;36m#{"<"*80}\e[0m" umount_all ignore_exceptions: true - umount dest.root, -:R rescue Object - umount base.root, -:R rescue Object - sh.vgchange -:an, vgname rescue Object - sh.sync rescue Object - departx dest.image rescue Object - departx base.image rescue Object - sh.sync rescue Object - STDERR.puts "="*80 + umount dest.root, -:R rescue Object + umount base.root, -:R rescue Object + sh.vgchange -:an, vgname rescue Object + sh.sync rescue Object + departx dest.image rescue Object + departx base.image rescue Object + sh.sync rescue Object + STDERR.puts "\e[1;#{nok ? 31: 36}m#{">"*80}\e[0m" end def activate_vg vgname @@ -283,66 +286,61 @@ EOF end def ssh_copy_id_local src, home, uid, gid = nil - akf = home+'.ssh/authorized_keys' - akd = akf.dirname - akd.mkdir - akd.chown uid, gid - akd.chmod 0700 - akf.copy src - akf.chown uid, gid - akf.chmod 0600 + akfile = home+'.ssh/authorized_keys' + akdir = akfile.dirname + akdir. mkdir + akdir. chown uid, gid + akdir. chmod 0700 + akfile.copy src + akfile.chown uid, gid + akfile.chmod 0600 end - def rename_user + def install_authorized_keys ssh_copy_id_local @authorized_keys, dest.root+'home/pi', 1000, 1000 if @authorized_keys ssh_copy_id_local @root_authorized_keys, dest.root+'root', 0, 0 if @root_authorized_keys + end - if @username - dest.root.join( 'etc/passwd').replace_i do |f| - f.each_line.flat_map do |l| - user, pwd, uid, gid, name, home, shell = l.split( ':') - user, home = @username, "/home/#{@username}" if 'pi' == user - [user, pwd, uid, gid, name, home, shell].join ':' - end + def rename_user username = nil, password: nil, old: nil + username ||= @username + old ||= 'pi' + password ||= @password + + if username and old != username + dest.root.join( 'etc/passwd').sed_i do |l| + user, pwd, uid, gid, name, home, shell = l.split( ':', 7) + user, home = username, "/home/#{username}" if old == user + [user, pwd, uid, gid, name, home, shell].join ':' end - dest.root.join( 'etc/group').replace_i do |f| - f.each_line.flat_map do |l| - group, pwd, gid, users = l.split( ':') - group = @username if 'pi' == group - users = users.split( ',').map {|user| 'pi' == user ? @username : user } - [group, pwd, gid, users.join( ',')].join ':' - end + dest.root.join( 'etc/group').sed_i do |l| + group, pwd, gid, users = l.split( ':', 4) + group = username if old == group + users = users.split( ',').map {|user| old == user ? username : user } + [group, pwd, gid, users.join( ',')].join ':' end - dest.root.join( 'home/pi').rename dest.root.join( 'home', @username) + dest.root.join( 'home', old).rename dest.root.join( 'home', username) end - if @password or @username - dest.root.join( 'etc/shadow').replace_i do |f| - f.each_line.flat_map do |l| - user, crypt, lastchange, minage, maxage, warn, inact, expiry, reserved = l.split( ':') - if 'pi' == user - crypt = @password if @password - user = @username if @username - end - [user, crypt, lastchange, minage, maxage, warn, inact, expiry, reserved].join ':' + if password or username + dest.root.join( 'etc/shadow').sed_i do |l| + user, crypt, lastchange, minage, maxage, warn, inact, expiry, reserved = l.split( ':', 9) + if old == user + crypt = password if password + user = username if username end + [user, crypt, lastchange, minage, maxage, warn, inact, expiry, reserved].join ':' end end end def set_hostname hostname = nil - hostname = @hostname + hostname ||= @hostname return if hostname.nil? or hostname.empty? - msg :patch, 'etc/hosts', 'set hostname to', hostname - dest.root.join( 'etc/hosts').replace_i do |f| - f.each_line.map {|l| l.gsub /\/, hostname } - end - msg :write, hostname, :to, 'etc/hostname' - dest.root.join( 'etc/hostname').open 'w' do |f| - f.puts hostname - end + msg "setting hostname", hostname + dest.root.join( 'etc/hosts').sed_i {|l| l.gsub /\/, hostname } + dest.root.join( 'etc/hostname').write "#{hostname}\n" end def install_packages_from_dir *paths @@ -368,7 +366,7 @@ EOF def install_deb path #sh.dpkg '--unpack', '--force-architecture', pn, chroot: dest.root - adeb = Pathname.new( 'var/cache/apt/archives')+path.basename.to_s + adeb = XPathname.new( 'var/cache/apt/archives')+path.basename.to_s dest.root.join( adeb.to_s).copy path sh.dpkg -:i, adeb, chroot: dest.root #msg :unpack, path diff --git a/lib/to_lvm_xfs/exts.rb b/lib/to_lvm_xfs/exts.rb index 6203800..b86187a 100644 --- a/lib/to_lvm_xfs/exts.rb +++ b/lib/to_lvm_xfs/exts.rb @@ -1,7 +1,9 @@ require 'pathname' class Pathname - alias +@ to_s + def +@ + expand_path.to_s + end def chdir &exe Dir.chdir self.to_s, &exe @@ -13,6 +15,14 @@ class Pathname FileUtils.copy_file src.to_s, self.to_s, **opts end + def symlink to + File.symlink to.to_s, to_s + end + + def link to + File.link to.to_s, to_s + end + def replace_i open 'r+' do |f| lines = yield f @@ -21,6 +31,67 @@ class Pathname f.puts lines end end + + def sed_i **replaces, &e + open 'r+' do |f| + lines = f.each_line.flat_map {|l| yield l.chomp, replaces } + f.truncate 0 + f.pos = 0 + f.puts lines + replaces.each {|_,rls| f.puts rls } + end + end +end + +class XPathname < Pathname + %i[/ + join].each do |meth| + define_method meth do |*n| + XPathname.new "#{super *n}" + end + end + + %i[unlink read].each do |meth| + define_method meth do |*a, &e| + STDERR.puts "\e[1;36m#{meth} \e[1;35m#{self}\e[0m" + super *a, &e + end + end + + %i[replace_i sed_i].each do |meth| + define_method meth do |*a, &e| + STDERR.puts "\e[1;36mpatching \e[1;35m#{self}\e[0m" + super *a, &e + end + end + + %i[link rename symlink].each do |meth| + define_method meth do |*a, &e| + STDERR.puts "\e[1;36m#{meth} \e[1;35m#{self}\e[0m -> #{a.map{|x|"\e[1;35m#{x}\e[0m"}.join ', '}" + super *a, &e + end + end + + %i[move copy].each do |meth| + define_method meth do |*a, &e| + STDERR.puts "\e[1;36m#{meth} \e[1;35m#{self}\e[0m <- #{a.map{|x|"\e[1;35m#{x}\e[0m"}.join ', '}" + super *a, &e + end + end + + %i[make_link make_symlink].each do |meth| + name = meth.to_s.sub /\Amake_/, '' + define_method meth do |*a, &e| + STDERR.puts "\e[1;36m#{name} \e[1;35m#{self}\e[0m -> #{a.map{|x|"\e[1;35m#{x}\e[0m"}.join ', '}" + super *a, &e + end + end + + %i[write].each do |meth| + define_method meth do |*a, &e| + STDERR.puts "\e[1;36m#{meth} \e[1;35m#{self}\e[0m \e[1;35m#{a[0][0...128].inspect}\e[0m" + super *a, &e + end + end end class Symbol diff --git a/lib/to_lvm_xfs/raspbian.rb b/lib/to_lvm_xfs/raspbian.rb index 508e821..d486284 100644 --- a/lib/to_lvm_xfs/raspbian.rb +++ b/lib/to_lvm_xfs/raspbian.rb @@ -10,10 +10,24 @@ class Raspbian < Base ENV['LANG'] = 'C' mkmppaths - lsblk( dest.image).each do |l| - d "Device #{l[:name]} mounted at #{l[:mountpoint]}", ! l[:mountpoint] + case + when !dest.image.exist? + when dest.image.file? + r = sh.losetup_list + unless r.empty? or r['loopdevices'] + r['loopdevices'].each do |lo| + d "File #{dest.image} used as loop-device back-file", + +dest.image != +XPathname.new(lo['back-file']) + end + end + when dest.image.blockdev?, dest.image.chardev? + lsblk( dest.image).each do |l| + d "Device #{l[:name]} mounted at #{l[:mountpoint]}", ! l[:mountpoint] + end end + sleep 5 + d "Base image does not exist", base.image.exist? base_parts = kpartx base.image d "two partitions in base expected, got: #{base_parts.inspect}", 2 == base_parts.length @@ -38,7 +52,7 @@ class Raspbian < Base STDERR.puts l sh.dmsetup :remove, File.basename(l[:name]) if 'lvm' == l[:type] l[:name].start_with?( dest.image.to_s) and 'part' == l[:type] - end.map {|l| Pathname.new l[:name] }.sort + end.map {|l| XPathname.new l[:name] }.sort rescue Sh::ProcessError kpartx dest.image end @@ -46,8 +60,8 @@ class Raspbian < Base dest_parts[0].open( 'w') {|f| f << 0.chr*4*1024*1024 } dest_parts[1].open( 'w') {|f| f << 0.chr*4*1024*1024 } sh.vgscan '--cache' - vgpath = Pathname.new( '/dev') + vgname - sh.pvcreate '-ff', dest_parts[1] + vgpath = XPathname.new( '/dev') + vgname + sh.pvcreate -:ff, dest_parts[1] sh.vgcreate vgname, dest_parts[1] sh.lvcreate -:nroot, '-L4.2G', vgname sh.lvcreate -:nhome, '-L100M', vgname @@ -56,7 +70,7 @@ class Raspbian < Base sh.mkxfs -:Lroot, vgpath+'root' sh.mkxfs -:Lhome, vgpath+'home' mount vgpath+'root', dest.root - addmp = {} + addmp = {run_udev: dest.root+'run/udev'} %i[home boot dev proc sys].each do |n| d = addmp[n] = dest.root+n.to_s d.mkdir @@ -69,11 +83,10 @@ class Raspbian < Base mount 'sysfs', addmp[:sys], -:tsysfs sh.rsync_all "#{base.root}/", dest.root - #sh.rsync *%w[kernel7.img initrd7.img], addmp[:boot] + install_authorized_keys rename_user - msg :patch, 'etc/fstab' (dest.root+'etc'+'fstab').replace_i do |f| replace = { '/' => "UUID=#{sh.fs_uuid vgpath+'root'} / xfs defaults,noatime 0 0", @@ -86,25 +99,34 @@ class Raspbian < Base end + replace.values end - msg :patch, 'boot/config.txt' (addmp[:boot]+'config.txt').replace_i do |f| replace = { - initramfs: 'initramfs initrd7.img followkernel', + 'pi4' => { initramfs: 'initramfs initrd7l.img followkernel', }, + 'pi3' => { initramfs: 'initramfs initrd7.img followkernel', }, + 'pi2' => { initramfs: 'initramfs initrd7.img followkernel', }, + 'pi1' => { initramfs: 'initramfs initrd.img followkernel', }, + 'pi0' => { initramfs: 'initramfs initrd.img followkernel', }, } - f.each_line.flat_map do |l| + blocks = [nil] + content = Hash.new {|h,block| h[block] = [] } + block = nil + f.each_line do |l| l.chomp! case l - when /^initramfs / - replace.delete :initramfs - else l + when /\A\[([^\]]*)\]\z/ + block = $1 + blocks.push block + when /\Ainitramfs / + l = replace[block].delete :initramfs end - end + replace.values + content[block].push l + end + replace.each {|block, rpl| content[block] += rpl.values + [''] unless rpl.empty? } + blocks.flat_map {|block| content[block] } end - msg :touch, 'boot/ssh' - (addmp[:boot]+'ssh').open('w') {|f|} + (addmp[:boot]+'ssh').write '' - msg :patch, 'boot/cmdline.txt' (addmp[:boot]+'cmdline.txt').replace_i do |f| lines = f.readlines d "Only one line in cmdline.txt expected", 1 == lines.length @@ -119,46 +141,58 @@ class Raspbian < Base opts.map {|k,v| v ? "#{k}=#{v}" : "#{k}" }.join(' ') end - msg :patch, 'etc/initramfs-tools/initramfs.conf' - dest.root.join( 'etc/initramfs-tools/initramfs.conf').replace_i do |f| - f.each_line.flat_map do |l| - l.chomp! - case l - when /^COMPRESS=/ - 'COMPRESS=xz' - when /^# *COMPRESS=/ - [l, 'COMPRESS=xz'] - else - l - end - end - end - - set_hostname - - msg :unlinking, 'etc/rc*.d/*resize2fs_once' (dest.root+'etc').chdir do - Pathname.glob( 'rc*.d/*resize2fs_once').each do |fn| + XPathname.glob( 'rc*.d/*resize2fs_once').each do |fn| fn.unlink end end qemu_bin = dest.root.join 'usr/bin/qemu-arm-static' - msg :copy, "/usr/bin/qemu-arm-static" qemu_bin.copy '/usr/bin/qemu-arm-static', preserve: true + preload, preload_x = dest.root+'etc/ld.so.preload', dest.root+'etc/ld.so.preload.tp' + preload.rename preload_x + ish = sh.chroot( dest.root).chdir( '/') ish.apt :update ish.apt :upgrade, -:y ish.apt :update - ish.apt :install, -:y, :lvm2, :xfsprogs + + # We mount /run/udev for lvm-scanning - vgs / vgcfgbackup need it to connect to udev. + addmp[:run_udev].mkdir + mount '/run/udev', addmp[:run_udev], --:bind + + # prevent installing exim by installing nullmailer + #ish.apt :install, -:y, :lvm2, :xfsprogs, :nullmailer, :dracut + #dest.root.join( 'etc/dracut.conf.d/10-denkn.conf').open 'w' do |f| + # f.puts 'add_modules+="lvm"' + # f.puts 'add_drivers+="dm-mod xfs"' + # f.puts 'compress="xz"' + #end + + ish.apt :install, -:y, :lvm2, :xfsprogs, 'initramfs-tools' + dest.root.join( 'etc/initramfs-tools/initramfs.conf').replace_i do |f| + replace = { compress: 'COMPRESS=xz', } + f.each_line.flat_map do |l| + case l.chomp! + when /^COMPRESS=/ then replace.delete :compress + when /^# *COMPRESS=/ then [l, replace.delete( :compress)] + else l + end + end + replace.values + end + + set_hostname install_packages_from_dir( - Pathname.new($0).expand_path.dirname+'raspbian-files', - Pathname.new('files') + XPathname.new( $0).expand_path.dirname + 'raspbian-files', + XPathname.new( 'files') ) # generates implicite initramfs ish.system *%w[dpkg-reconfigure raspberrypi-kernel] + + preload_x.rename preload + qemu_bin.unlink end end diff --git a/lib/to_lvm_xfs/sh.rb b/lib/to_lvm_xfs/sh.rb index 4651263..5b217a2 100644 --- a/lib/to_lvm_xfs/sh.rb +++ b/lib/to_lvm_xfs/sh.rb @@ -1,6 +1,7 @@ require 'forwardable' require 'shellwords' require 'socket' +require 'json' class Sh class ForkError < RuntimeError @@ -99,13 +100,14 @@ class Sh if opts[:chroot] STDERR.printf "\e[1;34mchroot(%s)\e[0m ", opts[:chroot].to_s Dir.chroot opts[:chroot].to_s + Dir.chdir '/' end - if @shell.opts[:pwd] - STDERR.printf "%s ", @shell.opts[:pwd] - Dir.chdir @shell.opts[:pwd] + if @shell.opts[:dir] + STDERR.printf "%s ", @shell.opts[:dir] + Dir.chdir @shell.opts[:dir] end #STDERR.puts "\e[0m#{opts[:return] ? '<=' : '#'} \e[33m#{cmd.shellescape} \e[35m#{usable_args.shelljoin}\e[0m" - STDERR.printf "\e[0m%s \e[33m%s \e[35m%s\e[0m\n", + STDERR.printf "\e[30;1m%s \e[0;33m%s \e[1;35m%s\e[0m\n", opts[:return] ? '<=' : '#', cmd.shellescape, usable_args.shelljoin @@ -132,7 +134,19 @@ class Sh when :string opts[:mode], exe = :io, lambda( &:read) when :line - opts[:mode], exe = :io, lambda{|f|f.read.chomp} + opts[:mode], exe = :io, lambda {|f| f.read.chomp } + when :json + opts[:mode] = :io + if opts[:could_be_empty] + exe = lambda do |f| + r = f.read + r.empty? ? [] : JSON.parse( r) + end + else + exe = lambda {|f| JSON.parse f.read } + end + when :jsonl + opts[:mode], exe = :io, lambda {|f| f.each_line.map {|l| JSON.parse l } } end r = f = fork **opts begin diff --git a/lib/to_lvm_xfs/structs.rb b/lib/to_lvm_xfs/structs.rb index a517e91..1145c9a 100644 --- a/lib/to_lvm_xfs/structs.rb +++ b/lib/to_lvm_xfs/structs.rb @@ -7,13 +7,13 @@ class Image @dir = case dir when Pathname then dir - when String then Pathname.new dir + when String then XPathname.new dir else raise ArgumentError, "Pathname for dir expected" end @image = case image when Pathname then image - when String then Pathname.new image + when String then XPathname.new image when nil then @dir+'image' else raise ArgumentError, "Pathname for image expected" end @@ -21,7 +21,7 @@ class Image @root = case root when Pathname then root - when String then Pathname.new root + when String then XPathname.new root when nil then @dir+'root' else raise ArgumentError, "Path for root must be Pathname, String or MP, if given" end diff --git a/raspbian-files/raspbian-files.tar b/raspbian-files/raspbian-files.tar index ad38980..014902c 100644 Binary files a/raspbian-files/raspbian-files.tar and b/raspbian-files/raspbian-files.tar differ