balena-os / balenahup

BALENA Host os UPdater
https://balena.io/
36 stars 11 forks source link

Stop reading the storage driver from the boot partition #371

Closed alexgg closed 2 years ago

alexgg commented 2 years ago

This has been removed with the aufs to overlay2 transition.

Needs to check that HUPs between an aufs based OS and an overlay2 based one stills spaws the docker daemon with the correct storage engine.

Resolves: https://github.com/balena-os/balenahup/issues/372

jellyfish-bot commented 2 years ago

[lmbarros] This has attached https://jel.ly.fish/c892bceb-7c31-4f96-9d37-c97e9c2d9845

jaomaloy commented 2 years ago

Listing test cases:

alexgg commented 2 years ago

2021.10-2 -> 2021.10.2

First time (2021.10.2 -> empty ):

+ '[' -S /var/run/balena-host.sock ']'                                                                                                                   
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'                                                                              
+ '[' balena = balena ']'                                                                                                  
+ '[' -d /mnt/sysroot/inactive/docker ']'                                                                                  
++ df /mnt/sysroot/inactive                                                                                                                         
++ grep /mnt/sysroot/inactive                                                                                                                               
++ awk '{ print $3}'                                                                                                       
+ inactive_used=2200                                                                                                       
+ '[' 2200 -gt 5000 ']'                                                                                                    
+ '[' no = yes ']'                                                                                                         
+ '[' no = yes ']'                                                                                                         
+ log 'Starting hostapp-update' 

Second time (2021.10.2 -> 2021.10.2):

+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'
+ '[' -d /mnt/sysroot/inactive/docker ']'
++ df /mnt/sysroot/inactive
++ awk '{ print $3}'
++ grep /mnt/sysroot/inactive
+ inactive_used=288833
+ '[' 288833 -gt 5000 ']'
++ DOCKER_HOST=unix:///var/run/balena-host.sock
++ balena images -q
++ wc -l
+ hostapp_image_count=1q
+ '[' 1 -eq 0 ']'
+ '[' no = yes ']'
+ '[' no = yes ']'
+ log 'Starting hostapp-update'

2021.10.2 -> 2022.4.1

Third time (2021.10.2 -> 2022.4.1)

root@c81defc:~# du -chs /mnt/sysroot/inactive/
288M    /mnt/sysroot/inactive/
288M    total

+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'
+ '[' -d /mnt/sysroot/inactive/docker ']'
++ df /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive
++ awk '{ print $3}'
+ inactive_used=296564
+ '[' 296564 -gt 5000 ']'
++ DOCKER_HOST=unix:///var/run/balena-host.sock
++ balena images -q
++ wc -l
+ hostapp_image_count=1
+ '[' 1 -eq 0 ']'
+ '[' no = yes ']'
+ '[' no = yes ']'
+ log 'Starting hostapp-update'

Fourth time (2022.4.1 (from aufs migration) -> 2022.4.1):

root@c81defc:~# cat /etc/os-release 
ID="balena-os"
NAME="balenaOS"
VERSION="2022.4.1"
VERSION_ID="2022.4.1"
PRETTY_NAME="balenaOS 2022.4.1"
MACHINE="raspberrypi3-64"
META_BALENA_VERSION="2.98.33"
BALENA_BOARD_REV=""
META_BALENA_REV=""
SLUG="raspberrypi3-64"
root@c81defc:~# 
root@c81defc:~# 
root@c81defc:~# du -chs /mnt/sysroot/inactive/
288M    /mnt/sysroot/inactive/
288M    total

+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'
+ '[' -d /mnt/sysroot/inactive/docker ']'
++ df /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive
++ awk '{ print $3}'
+ inactive_used=296641
+ '[' 296641 -gt 5000 ']'
++ DOCKER_HOST=unix:///var/run/balena-host.sock
++ balena images -q
++ wc -l
+ hostapp_image_count=0
+ '[' 0 -eq 0 ']'
+ local target_folder=/mnt/sysroot/inactive/balena/
+ systemctl stop balena-host
Warning: Stopping balena-host.service, but it can still be activated by:
  balena-host.socket
+ log 'Found potential leftover data, cleaning /mnt/sysroot/inactive/balena/'
+ case $1 in
+ loglevel=LOG
++ date +%s
+ endtime=1655717656
+ '[' LOG == ERROR ']'
[000000035][LOG]Found potential leftover data, cleaning /mnt/sysroot/inactive/balena/
+ printf '[%09d%s%s\n' 35 '][LOG]' 'Found potential leftover data, cleaning /mnt/sysroot/inactive/balena/'
+ find /mnt/sysroot/inactive/balena/ -mindepth 1 -maxdepth 1 -exec rm -r '{}' ';'
++ df -h /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive
++ awk '{ print $3}'
+ log 'Inactive partition usage after cleanup: 2.1M'
+ case $1 in
+ loglevel=LOG
++ date +%s
+ endtime=1655717659
+ '[' LOG == ERROR ']'
+ printf '[%09d%s%s\n' 38 '][LOG]' 'Inactive partition usage after cleanup: 2.1M'
[000000038][LOG]Inactive partition usage after cleanup: 2.1M
+ systemctl start balena-host
+ local timeout_iterations=0
+ DOCKER_HOST=unix:///var/run/balena-host.sock
+ balena ps
+ '[' no = yes ']'
+ '[' no = yes ']'
+ log 'Starting hostapp-update'

Fifth time (Third time (2022.4.1 -> empty):

root@c81defc:~# du -chs /mnt/sysroot/inactive/
284M    /mnt/sysroot/inactive/
284M    total

+ storage_driver=overlay2                                                                                                              
+ '[' -f /boot/storage-driver ']'                              
+ case ${SLUG} in                                           
+ log 'Running pre-update fixes for raspberrypi3-64'              
+ case $1 in                                                                                                                           
+ loglevel=LOG                                                 
++ date +%s                                                     
+ endtime=1655718033                                                                                                                   
+ '[' LOG == ERROR ']'                                                                                                                 
+ printf '[%09d%s%s\n' 51 '][LOG]' 'Running pre-update fixes for raspberrypi3-64'                                                      
+ pre_update_pi_bootfiles_removal                                                                                                      
[000000051][LOG]Running pre-update fixes for raspberrypi3-64                                                                           
Removing start_db.elf from boot partition                      
+ boot_files_for_removal=('start_db.elf' 'fixup_db.dat')                                                                               
+ local boot_files_for_removal                                 
+ for f in "${boot_files_for_removal[@]}"                          
+ echo 'Removing start_db.elf from boot partition'                                                                                     
+ rm -f /mnt/boot/start_db.elf                                                                                                         
+ for f in "${boot_files_for_removal[@]}"                                                                                              
+ echo 'Removing fixup_db.dat from boot partition'                                                                                     
Removing fixup_db.dat from boot partition                 
+ rm -f /mnt/boot/fixup_db.dat                                
+ sync /mnt/boot                                             
+ version_gt 2.98.33 2.7.6                                    
++ echo 2.98.33 2.7.6                                        
++ tr ' ' '\n'
++ sort -V
++ head -n 1
+ test 2.7.6 '!=' 2.98.33
+ '[' balena = docker ']'
+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'
+ '[' -d /mnt/sysroot/inactive/docker ']'
++ df /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive
++ awk '{ print $3}'
+ inactive_used=2316
+ '[' 2316 -gt 5000 ']'
+ '[' no = yes ']'
+ '[' no = yes ']'
+ log 'Starting hostapp-update'

Sixth time (Third time (2022.4.1 -> 2022.4.1):

+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'
+ '[' -d /mnt/sysroot/inactive/docker ']'                 
++ df /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive
++ awk '{ print $3}'
+ inactive_used=292495
+ '[' 292495 -gt 5000 ']'
++ DOCKER_HOST=unix:///var/run/balena-host.sock                                                                                        
++ balena images -q                                               
++ wc -l                         
+ hostapp_image_count=1
+ '[' 1 -eq 0 ']'            
+ '[' no = yes ']'                                                                                                                     
+ '[' no = yes ']'                           
+ log 'Starting hostapp-update'          
+ case $1 in     
+ loglevel=LOG                                
++ date +%s                                                                                                                            
+ endtime=1655718519       
+ '[' LOG == ERROR ']'                                                                                                                 
+ printf '[%09d%s%s\n' 29 '][LOG]' 'Starting hostapp-update'   

2021.10.2 -> 2022.4.1

First time:

+ '[' -S /var/run/balena-host.sock ']'                             
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'                                          
+ '[' -d /mnt/sysroot/inactive/docker ']'
++ df /mnt/sysroot/inactive                                        
++ grep /mnt/sysroot/inactive                       
++ awk '{ print $3}'                 
+ inactive_used=2200                                                                                                                   
+ '[' 2200 -gt 5000 ']'                                           
+ '[' no = yes ']'               
+ '[' no = yes ']'                                                 
+ log 'Starting hostapp-update'

Second time:

+ '[' -S /var/run/balena-host.sock ']'                             
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'             
+ '[' balena = balena ']'                                          
+ '[' -d /mnt/sysroot/inactive/docker ']'                          
++ df /mnt/sysroot/inactive                                        
++ grep /mnt/sysroot/inactive                                      
++ awk '{ print $3}'                                               
+ inactive_used=288919                                             
+ '[' 288919 -gt 5000 ']'                                          
++ DOCKER_HOST=unix:///var/run/balena-host.sock                    
++ balena images -q                                                
++ wc -l                                                           
+ hostapp_image_count=0                                            
+ '[' 0 -eq 0 ']'                                                  
+ local target_folder=/mnt/sysroot/inactive/balena/                
+ systemctl stop balena-host                                       
Warning: Stopping balena-host.service, but it can still be activated by:                                                               
  balena-host.socket                                               
+ log 'Found potential leftover data, cleaning /mnt/sysroot/inactive/balena/'                                                          
+ case $1 in                                                       
+ loglevel=LOG             
++ date +%s                                                                                                                            
+ endtime=1655724472                                                                                                                   
+ '[' LOG == ERROR ']'                                                                                                                 
[000000040][LOG]Found potential leftover data, cleaning /mnt/sysroot/inactive/balena/                                                  
+ printf '[%09d%s%s\n' 40 '][LOG]' 'Found potential leftover data, cleaning /mnt/sysroot/inactive/balena/'                             
+ find /mnt/sysroot/inactive/balena/ -mindepth 1 -maxdepth 1 -exec rm -r '{}' ';'                                                      
++ df -h /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive
++ awk '{ print $3}'  
+ log 'Inactive partition usage after cleanup: 2.1M'
+ case $1 in                     
+ loglevel=LOG   
++ date +%s                                         
+ endtime=1655724475
+ '[' LOG == ERROR ']'
+ printf '[%09d%s%s\n' 43 '][LOG]' 'Inactive partition usage after cleanup: 2.1M'                                                      
[000000043][LOG]Inactive partition usage after cleanup: 2.1M
+ systemctl start balena-host                               
+ local timeout_iterations=0
+ DOCKER_HOST=unix:///var/run/balena-host.sock                                                                                         
+ balena ps                      
+ '[' no = yes ']'                                      
+ '[' no = yes ']'            
+ log 'Starting hostapp-update'          
+ case $1 in                                      
+ loglevel=LOG                           
++ date +%s                   
+ endtime=1655724481                     
[000000049][LOG]Starting hostapp-update          

Third time:

+ '[' -S /var/run/balena-host.sock ']'                                                                                                                                                                                                                                         
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'                                                                                                                                                                                                                         
+ '[' balena = balena ']'                                                                                                                                                                                                                                                      
+ '[' -d /mnt/sysroot/inactive/docker ']'                                                                                                                                                                                                                                      
++ df /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive                                                                                                                                                                                                                                                  
++ awk '{ print $3}'                                                                                                                                                                                                                                                           
+ inactive_used=292499                                                                                                                                                                                                                                                         
+ '[' 292499 -gt 5000 ']'
++ DOCKER_HOST=unix:///var/run/balena-host.sock
++ balena images -q        
++ wc -l             
+ hostapp_image_count=0    
+ '[' 0 -eq 0 ']'     
+ local target_folder=/mnt/sysroot/inactive/balena/
+ systemctl stop balena-host     
Warning: Stopping balena-host.service, but it can still be activated by:
  balena-host.socket                                
+ log 'Found potential leftover data, cleaning /mnt/sysroot/inactive/balena/'
+ case $1 in  
+ loglevel=LOG
++ date +%s         
+ endtime=1655724905  
+ '[' LOG == ERROR ']'                                      
+ printf '[%09d%s%s\n' 37 '][LOG]' 'Found potential leftover data, cleaning /mnt/sysroot/inactive/balena/'
[000000037][LOG]Found potential leftover data, cleaning /mnt/sysroot/inactive/balena/
+ find /mnt/sysroot/inactive/balena/ -mindepth 1 -maxdepth 1 -exec rm -r '{}' ';'
++ df -h /mnt/sysroot/inactive                          
++ grep /mnt/sysroot/inactive 
++ awk '{ print $3}'                     
+ log 'Inactive partition usage after cleanup: 2.1M'
+ case $1 in                  
+ loglevel=LOG                           
++ date +%s                              
+ endtime=1655724908                              
+ '[' LOG == ERROR ']'        
+ printf '[%09d%s%s\n' 40 '][LOG]' 'Inactive partition usage after cleanup: 2.1M'
[000000040][LOG]Inactive partition usage after cleanup: 2.1M
+ systemctl start balena-host
+ local timeout_iterations=0
+ DOCKER_HOST=unix:///var/run/balena-host.sock
+ balena ps 
+ '[' no = yes ']'       
+ '[' no = yes ']'       
+ log 'Starting hostapp-update'       

Fourth time:

+ '[' -S /var/run/balena-host.sock ']'   
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'                                                                                 
+ '[' balena = balena ']'        
+ '[' -d /mnt/sysroot/inactive/docker ']'               
++ df /mnt/sysroot/inactive   
++ grep /mnt/sysroot/inactive            
++ awk '{ print $3}'                              
+ inactive_used=292495        
+ '[' 292495 -gt 5000 ']'                
++ wc -l                                 
++ DOCKER_HOST=unix:///var/run/balena-host.sock   
++ balena images -q           
+ hostapp_image_count=1
+ '[' 1 -eq 0 ']'         
+ '[' no = yes ']'   
+ '[' no = yes ']'
+ log 'Starting hostapp-update'
+ case $1 in
+ loglevel=LOG           
++ date +%s              
+ endtime=1655725337                  
[000000032][LOG]Starting hostapp-update 

2022.4.1 -> 2022.4.1

First time:

+ '[' -S /var/run/balena-host.sock ']'                                                                                                                                                                                                                                         
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'                                                                                                                                                                                                                         
+ '[' balena = balena ']'                                                                                                                                                                                                                                                      
+ '[' -d /mnt/sysroot/inactive/docker ']'                                                                                                                                                                                                                                      
++ df /mnt/sysroot/inactive                                                                                                                                                                                                                                                    
++ grep /mnt/sysroot/inactive                                                                                                                                                                                                                                                  
++ awk '{ print $3}'                                                                                                                                                                                                                                                           
+ inactive_used=194                                                                                                                                                                                                                                                            
+ '[' 194 -gt 5000 ']'                                                                                                                                                                                                                                                         
+ '[' no = yes ']'                                                                                                                                                                                                                                                             
+ '[' no = yes ']'                                                                                                                                                                                                                                                             
+ log 'Starting hostapp-update'                                                                                                                                                                                                                                                
+ case $1 in                                                                                                                                                                                                                                                                   
+ loglevel=LOG                                                                                                                                                                                                                                                                 
++ date +%s                                                                                                                                                                                                                                                                    
+ endtime=1655729115                                                                                                                                                                                                                                                           
+ '[' LOG == ERROR ']'                                                                                                                                                                                                                                                         
+ printf '[%09d%s%s\n' 26 '][LOG]' 'Starting hostapp-update'                                                                                                                                                                                                                   
[000000026][LOG]Starting hostapp-update  

Second time:

+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'              
+ '[' balena = balena ']'                       
+ '[' -d /mnt/sysroot/inactive/docker ']'                 
++ df /mnt/sysroot/inactive            
++ grep /mnt/sysroot/inactive      
++ awk '{ print $3}'                   
+ inactive_used=290411                                                                                                                                                                                                                                                         
+ '[' 290411 -gt 5000 ']'                                                                                              
++ DOCKER_HOST=unix:///var/run/balena-host.sock                                        
++ balena images -q                                                                                                                       
++ wc -l                         
+ hostapp_image_count=1               
+ '[' 1 -eq 0 ']'                                                                                                                         
+ '[' no = yes ']'                                                 
+ '[' no = yes ']'                                               
+ log 'Starting hostapp-update'              
+ case $1 in                                                                                                                              
+ loglevel=LOG                                                    
++ date +%s                                                                                                                
+ endtime=1655729850                                                                                                                               
+ '[' LOG == ERROR ']'                                                                                                                              
+ printf '[%09d%s%s\n' 25 '][LOG]' 'Starting hostapp-update'                                                                                                                                                                                                                   
[000000025][LOG]Starting hostapp-update     
alexgg commented 2 years ago

Need to retest after addressing the review comments.

jaomaloy commented 2 years ago

Tested latest:

Second time (2021.10.2 -> 2021.10.2)

+ '[' balena = docker ']'
+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'
+ '[' -d /mnt/sysroot/inactive/docker ']'
++ df /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive
++ awk '{ print $3}'
+ inactive_used=296569
+ '[' 296569 -gt 5000 ']'
++ DOCKER_HOST=unix:///var/run/balena-host.sock
++ balena images -q
++ wc -l
+ hostapp_image_count=1
+ '[' 1 -eq 0 ']'
+ '[' no = yes ']'
+ '[' no = yes ']'
+ log 'Starting hostapp-update'

Fifth time (2021.10.2 -> 2022.4.1)

+ '[' balena = docker ']'
+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'
+ '[' -d /mnt/sysroot/inactive/docker ']'
++ df /mnt/sysroot/inactive
++ awk '{ print $3}'
++ grep /mnt/sysroot/inactive
+ inactive_used=296570
+ '[' 296570 -gt 5000 ']'
++ DOCKER_HOST=unix:///var/run/balena-host.sock
++ balena images -q
++ wc -l
+ hostapp_image_count=1
+ '[' 1 -eq 0 ']'
+ '[' no = yes ']'
+ '[' no = yes ']'
+ log 'Starting hostapp-update'
+ case $1 in
+ loglevel=LOG
++ date +%s
+ endtime=1655805331
+ '[' LOG == ERROR ']'
+ printf '[%09d%s%s\n' 26 '][LOG]' 'Starting hostapp-update'

Eighth time (2022.4.1 -> 2021.10.2)

+ '[' balena = docker ']'
+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'
+ '[' -d /mnt/sysroot/inactive/docker ']'
++ df /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive
++ awk '{ print $3}'
+ inactive_used=2302
+ '[' 2302 -gt 5000 ']'
+ '[' no = yes ']'
+ '[' no = yes ']'
+ log 'Starting hostapp-update'
+ case $1 in
+ loglevel=LOG
++ date +%s
+ endtime=1655806028
+ '[' LOG == ERROR ']'
+ printf '[%09d%s%s\n' 35 '][LOG]' 'Starting hostapp-update'

Seventh time (2022.4.1 -> 2022.4.1)

+ '[' balena = docker ']'
+ '[' -S /var/run/balena-host.sock ']'
+ '[' -f /mnt/sysroot/inactive/resinos.fingerprint ']'
+ '[' balena = balena ']'
+ '[' -d /mnt/sysroot/inactive/docker ']'
++ df /mnt/sysroot/inactive
++ grep /mnt/sysroot/inactive
++ awk '{ print $3}'
+ inactive_used=292425
+ '[' 292425 -gt 5000 ']'
++ DOCKER_HOST=unix:///var/run/balena-host.sock
++ balena images -q
++ wc -l
+ hostapp_image_count=1
+ '[' 1 -eq 0 ']'
+ '[' no = yes ']'
+ '[' no = yes ']'
+ log 'Starting hostapp-update'
+ case $1 in
+ loglevel=LOG
++ date +%s
+ endtime=1655805773
+ '[' LOG == ERROR ']'
+ printf '[%09d%s%s\n' 29 '][LOG]' 'Starting hostapp-update'

LGTM

alexgg commented 2 years ago

Tested manual HUP laddering on RPI3-64:

acostach commented 2 years ago

Tested on Pi3 32bit: 2021.01.0 -> 2021.04.0 -> 2021.07.1 -> 2021.10.2 -> 2022.4.1 -> 2022.4.1 - OK

alexgg commented 2 years ago

@balena-ci republish