Closed junaruga closed 11 months ago
I created a minimal reproducer below for the test test_deflate_chunked
above.
$ cat test/zlib/_test_zlib_test_deflate_chunked.rb
require 'zlib'
z = Zlib::Deflate.new
input = "\x01"
z.deflate(input) do |chunk|
end
final = z.finish
puts "final.length: #{final.length}"
On my local machine.
$ rpm -qf /lib64/libz.so.1
zlib-1.2.13-3.fc38.x86_64
$ ruby -I./lib test/zlib/_test_zlib_test_deflate_chunked.rb
final.length: 9
$ dpkg -S /lib/s390x-linux-gnu/libz.so.1
zlib1g:s390x: /lib/s390x-linux-gnu/libz.so.1
$ dpkg -s zlib1g | grep ^Version
Version: 1:1.2.11.dfsg-2ubuntu9.2
$ ruby -I./lib test/zlib/_test_zlib_test_deflate_chunked.rb
final.length: 10
$ gdb --args ruby -I./lib test/zlib/_test_zlib_test_deflate_chunked.rb
(gdb) b rb_deflate_deflate
On the x86_64, after executing the do_deflate
, the avail_out = 1022, total_out = 2
(gdb) f
#0 rb_deflate_deflate (argc=1, argv=0x7ffff7440060, obj=140737077406920)
at ../../../../ext/zlib/zlib.c:1804
1804 return zstream_detach_buffer(z);
(gdb) p *z
$6 = {flags = 3, buf = 140737077406680, input = 4, mutex = 140737077406880, stream = {
next_in = 0x7fffe7816c19 "", avail_in = 0, total_in = 1, next_out = 0x5654c2 "",
avail_out = 1022, total_out = 2, msg = 0x0, state = 0x4c3880,
zalloc = 0x7ffff7e51771 <zlib_mem_alloc>, zfree = 0x7ffff7e5179e <zlib_mem_free>,
opaque = 0x7fffe7816cc8, data_type = 2, adler = 131074, reserved = 0},
func = 0x7ffff7e5ecf0 <deflate_funcs>}
On the s390x, after executing the do_deflate
, the values are avail_out = 1021, total_out = 3
. These are different.
(gdb) f
#0 rb_deflate_deflate (argc=1, argv=0x3fffceff060, obj=4398000606760)
at ../../../../ext/zlib/zlib.c:1804
1804>- return zstream_detach_buffer(z);
(gdb) p *z
$8 = {flags = 3, buf = 4398000606520, input = 4, mutex = 4398000606720, stream = {
next_in = 0x3fffd438d79 "", avail_in = 0, total_in = 1, next_out = 0x2aa001f4c93 "",-
avail_out = 1021, total_out = 3, msg = 0x0, state = 0x2aa002762a0,-
zalloc = 0x3fffa885b68 <zlib_mem_alloc>, zfree = 0x3fffa885bc8 <zlib_mem_free>,-
opaque = 0x3fffd438e28, data_type = 2, adler = 131074, reserved = 0},-
func = 0x3fffa896d08 <deflate_funcs>}
Then I watched when the z->stream.avail_out
is changed.
(gdb) b zstream_run_try
(gdb) r
(gdb) watch z->stream.avail_out
(gdb) c
(gdb) c
...
(gdb) c
Continuing.
Hardware watchpoint 3: z->stream.avail_out
Old value = 1024
New value = 1022
flush_pending (strm=strm@entry=0x6cf850) at /usr/src/debug/zlib-1.2.13-3.fc38.x86_64/deflate.c:824
824 s->pending -= len;
(gdb) bt
#0 flush_pending (strm=strm@entry=0x6cf850)
at /usr/src/debug/zlib-1.2.13-3.fc38.x86_64/deflate.c:824
#1 0x00007ffff7f9959a in deflate (strm=0x6cf850, flush=0)
at /usr/src/debug/zlib-1.2.13-3.fc38.x86_64/deflate.c:922
#2 0x00007ffff7e525f0 in zstream_run_func (ptr=0x7fffffffc610) at ../../../../ext/zlib/zlib.c:1024
#3 0x00007ffff7aa1684 in rb_nogvl (func=0x7ffff7e52590 <zstream_run_func>, data1=0x7fffffffc610,
ubf=0x7ffff7e5274c <zstream_unblock_func>, data2=0x7fffffffc610, flags=2) at thread.c:1552
#4 0x00007ffff7e52890 in zstream_run_try (value_arg=140737488340496)
at ../../../../ext/zlib/zlib.c:1115
...
(gdb) f
#0 flush_pending (strm=strm@entry=0x6cf850)
at /usr/src/debug/zlib-1.2.13-3.fc38.x86_64/deflate.c:824
824 s->pending -= len;
(gdb) l
819 zmemcpy(strm->next_out, s->pending_out, len);
820 strm->next_out += len;
821 s->pending_out += len;
822 strm->total_out += len;
823 strm->avail_out -= len;
824 s->pending -= len;
825 if (s->pending == 0) {
826 s->pending_out = s->pending_buf;
827 }
828 }
(gdb) c
Continuing.
Watchpoint 3 deleted because the program has left the block in
which its expression is valid.
0x00007ffff78ff3f1 in rb_ensure (b_proc=0x7ffff7e5276a <zstream_run_try>, data1=140737488340496,
e_proc=0x7ffff7e52a37 <zstream_run_ensure>, data2=140737488340496) at eval.c:1009
1009 result = (*b_proc) (data1);
(gdb) c
Continuing.
Hardware watchpoint 2: z->stream.avail_out
Old value = 0
New value = 1024
zstream_expand_buffer_into (z=0x2aa00272e20, size=1024) at ../../../../ext/zlib/zlib.c:714
714 rb_obj_hide(z->buf);
(gdb) c
Continuing.
Hardware watchpoint 2: z->stream.avail_out
Old value = 1024
New value = 1022
0x000003fffd60930e in ?? () from /lib/s390x-linux-gnu/libz.so.1
(gdb) c
Continuing.
Hardware watchpoint 2: z->stream.avail_out
Old value = 1022
New value = 1021
0x000003fffd60b7da in deflate () from /lib/s390x-linux-gnu/libz.so.1
(gdb) bt
#0 0x000003fffd60b7da in deflate () from /lib/s390x-linux-gnu/libz.so.1
#1 0x000003fffa88712e in zstream_run_func (ptr=0x3ffffffd7a0) at ../../../../ext/zlib/zlib.c:1024
#2 0x000003fffdc38202 in rb_nogvl (func=0x3fffa8870b0 <zstream_run_func>, data1=0x3ffffffd7a0,
ubf=0x3fffa887320 <zstream_unblock_func>, data2=0x3ffffffd7a0, flags=2) at thread.c:1552
#3 0x000003fffa8874e4 in zstream_run_try (value_arg=4398046500768)
at ../../../../ext/zlib/zlib.c:1115
#4 0x000003fffda2066c in rb_ensure (b_proc=0x3fffa887368 <zstream_run_try>, data1=4398046500768,
e_proc=0x3fffa887738 <zstream_run_ensure>, data2=4398046500768) at eval.c:1009
...
(gdb) f
#0 0x000003fffd60b7da in deflate () from /lib/s390x-linux-gnu/libz.so.1
(gdb) l
709 /* I uses rb_str_new here not rb_str_buf_new because
710 rb_str_buf_new makes a zero-length string. */
711 z->buf = rb_str_buf_new(size);
712 z->stream.next_out = (Bytef*)RSTRING_PTR(z->buf);
713 z->stream.avail_out = MAX_UINT(size);
714 rb_obj_hide(z->buf);
715 }
716 else if (z->stream.avail_out != size) {
717 rb_str_modify_expand(z->buf, size);
718 z->stream.next_out = (Bytef*)RSTRING_END(z->buf);
Now I want to install a debug symbol of the zlib package to print the lines in the zlib library in Ubuntu jammy. Do you know how to do it?
Now I want to install a debug symbol of the zlib package to print the lines in the zlib library in Ubuntu jammy. Do you know how to do it?
All right. Now I was able to install the zlib package's debug symbol package on the s390x server.
I executed the page's "Getting -dbgsym.ddeb packages" section. https://wiki.ubuntu.com/Debug%20Symbol%20Packages
I was not able to install the zlib1g-dbg
package.
$ sudo apt-get install zlib1g-dbg
...
E: Package 'zlib1g-dbg' has no installation candidate
But I was able to install the zlib1g-dbgsym
package.
$ sudo apt-get install zlib1g-dbgsym
...
The following NEW packages will be installed:
zlib1g-dbgsym
0 upgraded, 1 newly installed, 0 to remove and 22 not upgraded.
...
The zlib debug symbol pacakge's version is same with the zlib package.
$ dpkg -s zlib1g-dbgsym | grep ^Version
Version: 1:1.2.11.dfsg-2ubuntu9.2
$ dpkg -s zlib1g | grep ^Version
Version: 1:1.2.11.dfsg-2ubuntu9.2
First, here are the information about the Ubuntu zlib and upsgream zlib.
Download Source Package zlib:
On the s390x server, I debugged again. I was able to see the code line number now.
$ gdb --args ruby -I./lib test/zlib/_test_zlib_test_deflate_chunked.rb
Reading symbols from ruby...
(gdb) b zstream_run_try
Function "zstream_run_try" not defined.
Breakpoint 1 (zstream_run_try) pending.
(gdb) r
Starting program: /home/jaruga/.local/ruby-05a853c2f2-debug/bin/ruby -I./lib test/zlib/_test_zlib_test_deflate_chunked.rb
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/s390x-linux-gnu/libthread_db.so.1".
Breakpoint 1, zstream_run_try (value_arg=4398046500768) at ../../../../ext/zlib/zlib.c:1082
1082 struct zstream_run_args *args = (struct zstream_run_args *)value_arg;
(gdb) watch z->stream.avail_out
Hardware watchpoint 2: z->stream.avail_out
(gdb) c
Continuing.
Hardware watchpoint 2: z->stream.avail_out
Old value = 0
New value = 1024
zstream_expand_buffer_into (z=0x2aa00272e20, size=1024) at ../../../../ext/zlib/zlib.c:714
714 rb_obj_hide(z->buf);
(gdb) c
Continuing.
Hardware watchpoint 2: z->stream.avail_out
Old value = 1024
New value = 1022
0x000003fffd60930e in flush_pending (strm=strm@entry=0x2aa00272e40) at deflate.c:806
806 deflate.c: No such file or directory.
(gdb) c
Continuing.
Hardware watchpoint 2: z->stream.avail_out
Old value = 1022
New value = 1021
0x000003fffd60b7da in dfltcc_cmpr (strm=0x2aa00272e40) at contrib/s390/dfltcc.c:307
307 contrib/s390/dfltcc.c: No such file or directory.
Hmm, the debug information is still not enough. I wanted to see the actual codes with lines.
(gdb) l
302 in contrib/s390/dfltcc.c
Here is the backtrace.
(gdb) bt
#0 0x000003fffd60b7da in dfltcc_cmpr (strm=0x2aa00272e40) at contrib/s390/dfltcc.c:307
#1 dfltcc_deflate (result=<synthetic pointer>, flush=<optimized out>, strm=0x2aa00272e40)
at contrib/s390/dfltcc.c:475
#2 deflate (flush=<optimized out>, strm=0x2aa00272e40) at deflate.c:1062
#3 deflate (strm=0x2aa00272e40, flush=<optimized out>) at deflate.c:825
#4 0x000003fffa88712e in zstream_run_func (ptr=0x3ffffffd7a0) at ../../../../ext/zlib/zlib.c:1024
#5 0x000003fffdc38202 in rb_nogvl (func=0x3fffa8870b0 <zstream_run_func>, data1=0x3ffffffd7a0,
ubf=0x3fffa887320 <zstream_unblock_func>, data2=0x3ffffffd7a0, flags=2) at thread.c:1552
#6 0x000003fffa8874e4 in zstream_run_try (value_arg=4398046500768)
at ../../../../ext/zlib/zlib.c:1115
...
But I couldn't find contrib/s390/dfltcc.c
printed on the backtrace above, seeing the zlib 1.2.11's upstream source below.
https://github.com/madler/zlib/tree/v1.2.11/contrib
$ git remote -v
origin https://github.com/madler/zlib.git (fetch)
origin https://github.com/madler/zlib.git (push)
$ git branch
* (HEAD detached at v1.2.11)
develop
$ pwd
/home/jaruga/git/zlib
$ grep -r dfltcc.c
=> Empty result
I downloaded the Ubuntu jammy zlib package's patch files.
https://packages.ubuntu.com/jammy-updates/zlib1g
$ wget http://archive.ubuntu.com/ubuntu/pool/main/z/zlib/zlib_1.2.11.dfsg-2ubuntu9.2.debian.tar.xz
$ tar xvf zlib_1.2.11.dfsg-2ubuntu9.2.debian.tar.xz
$ cd debian/
$ grep -r dfltcc.c * | wc -l
49
It seems the patches/410.patch
is the patch file creating the contrib/s390/dfltcc.c
file.
$ cat patches/410.patch
From 992a7afc3edfa511dff0650d1c545b11bf64e655 Mon Sep 17 00:00:00 2001
From: Ilya Leoshkevich <iii@linux.ibm.com>
Date: Wed, 18 Jul 2018 13:14:07 +0200
Subject: [PATCH] Add support for IBM Z hardware-accelerated deflate
...
contrib/s390/dfltcc.c | 996 ++++++++++++++++++++++++++++++++++
...
Hmm, I feel it's harder to debug the Ubuntu package than Debian package. Note Debian zlib package link is here. It seems Ubuntu doesn't have the open opened Git repository to track the package's patch files change. And I am family with Fedora Linux packaging things.
I found one zlib package s390x issue ticket affecting Ubuntu 22.04 (jammy) in the list of the zlib Ubuntu package bugs.
It seems that following ticket is related to DFLTCC, reported from s390x machine (= "z15/LinuxONE III" on the ticket). https://bugs.launchpad.net/ubuntu/+source/zlib/+bug/1990379
And the PR https://github.com/madler/zlib/pull/410 is reported on the ticket. The PR is adding the contrib/s390/dfltcc.c
file newly.
I plan to run the ruby/zlib unit tests and the minimal reproducer (https://github.com/ruby/zlib/issues/60#issuecomment-1721530114), and debug on GDB on Ubuntu focal (the failures didn't happen on Ubuntu focal native s390x server), and newer versions Ubuntu and Fedora rawhide s390x containers on QEMU, and check the difference of the behaviors.
Hi, https://github.com/madler/zlib/pull/410 author here.
Things like
assert_equal 20016, deflated.length
are very likely to break with the DFLTCC hardware accelerator, since it produces a different (but still valid) compressed byte stream than the software implementation. I would suggest to either verify the validity of compressed streams differently (e.g., try to decompress them) or disable DFLTCC by setting the DFLTCC=0
environment variable.
Hi @iii-i, thanks for your help providing the info!
I ran the reproducing script that I mentioned at https://github.com/ruby/zlib/issues/60#issuecomment-1721530114 and the ruby/zlib's unit tests again with DFLTCC=0
in Ubuntu jammy s390x environment above. And I see the result of the reproducer is same with the x86_64 case, and the unit tests passed.
$ uname -m
s390x
$ dpkg -s zlib1g | grep ^Version
Version: 1:1.2.11.dfsg-2ubuntu9.2
$ DFLTCC=0 ruby -I./lib test/zlib/_test_zlib_test_deflate_chunked.rb
final.length: 9
$ DFLTCC=0 bundle exec rake test
...
------------------------------------------------------------------------------------------------------
95 tests, 518 assertions, 0 failures, 0 errors, 1 pendings, 0 omissions, 0 notifications
98.9474% passed
------------------------------------------------------------------------------------------------------
17.45 tests/s, 95.17 assertions/s
ruby01|s390x$ echo $?
0
And I can see the logic below in the https://github.com/madler/zlib/pull/410 - contrib/s390/dfltcc.c
env = secure_getenv("DFLTCC");
env_dfltcc_disabled = env && !strcmp(env, "0");
Comparing the 410.patch
between Ubuntu focal and jammy, the 410.patch
is the same between the versions. However, the build options are different in the rules
file, The --dfltcc
and --crc32-vx
are added in rules
filel in Ubuntu jammy.
$ diff -u ubuntu_focal/zlib1g/debian/rules ubuntu_jammy/zlib1g/debian/rules
--- ubuntu_focal/zlib1g/debian/rules 2020-08-20 01:52:59.000000000 +0200
+++ ubuntu_jammy/zlib1g/debian/rules 2021-08-12 05:28:03.000000000 +0200
@@ -21,6 +21,9 @@
LDFLAGS = `dpkg-buildflags --get LDFLAGS`
EXTRA_MAKE =
+CONFIGURE_COMMON=--shared --prefix=/usr
+CONFIGURE_HOST=--libdir=\$${prefix}/lib/$(DEB_HOST_MULTIARCH)
+
# binutils doesn't supply the prefixed version normally like GCC does so
# we can't just unconditionally use DEB_HOST_GNU_TYPE-ar
ifeq ($(DEB_HOST_GNU_TYPE),$(DEB_BUILD_GNU_TYPE))
@@ -46,8 +49,9 @@
# s390x fails at compatibility.
ifneq (,$(findstring $(DEB_HOST_ARCH), s390x))
m32=-m31
-CFLAGS += -DDFLTCC
-EXTRA_MAKE += OBJA=dfltcc.o PIC_OBJA=dfltcc.lo
+CFLAGS += -DDFLTCC_LEVEL_MASK=0x7e
+CONFIGURE_COMMON += --dfltcc
+CONFIGURE_HOST += --crc32-vx
else
m32=-m32
endif
@@ -95,7 +99,7 @@
if [ ! -f Makefile.stash ]; then cp Makefile Makefile.stash ; fi
- AR=$(AR) CC="$(DEB_HOST_GNU_TYPE)-gcc" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" uname=GNU ./configure --shared --prefix=/usr --libdir=\$${prefix}/lib/$(DEB_HOST_MULTIARCH)
+ AR=$(AR) CC="$(DEB_HOST_GNU_TYPE)-gcc" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" uname=GNU ./configure $(CONFIGURE_COMMON) $(CONFIGURE_HOST)
touch $@
@@ -106,7 +110,7 @@
cp -r $(COPYLIST) debian/64
cd debian/64 && AR=$(AR) CC="$(DEB_HOST_GNU_TYPE)-gcc $(m64)" \
CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" \
- uname=GNU ./configure --shared --prefix=/usr --libdir=\$${prefix}/usr/lib64
+ uname=GNU ./configure $(CONFIGURE_COMMON) --libdir=\$${prefix}/usr/lib64
touch $@
configure32-stamp: configure
@@ -116,7 +120,7 @@
cp -r $(COPYLIST) debian/32
cd debian/32 && AR=$(AR) CC="$(DEB_HOST_GNU_TYPE)-gcc $(m32)" \
CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" \
- uname=GNU ./configure --shared --prefix=/usr --libdir=\$${prefix}/usr/lib32
+ uname=GNU ./configure $(CONFIGURE_COMMON) --libdir=\$${prefix}/usr/lib32
touch $@
configuren32-stamp: configure
@@ -126,7 +130,7 @@
cp -r $(COPYLIST) debian/n32
cd debian/n32 && AR=$(AR) CC="$(DEB_HOST_GNU_TYPE)-gcc $(mn32)" \
CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" \
- uname=GNU ./configure --shared --prefix=/usr --libdir=\$${prefix}/usr/lib32
+ uname=GNU ./configure $(CONFIGURE_COMMON) --libdir=\$${prefix}/usr/lib32
touch $@
configurex32-stamp: configure
@@ -136,7 +140,7 @@
cp -r $(COPYLIST) debian/x32
cd debian/x32 && AR=$(AR) CC="$(DEB_HOST_GNU_TYPE)-gcc $(mx32)" \
CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" \
- uname=GNU ./configure --shared --prefix=/usr --libdir=\$${prefix}/usr/libx32
+ uname=GNU ./configure $(CONFIGURE_COMMON) --libdir=\$${prefix}/usr/libx32
touch $@
build: build-stamp $(EXTRA_BUILD)
I would suggest to either verify the validity of compressed streams differently (e.g., try to decompress them) or disable DFLTCC by setting the DFLTCC=0 environment variable.
Thanks for your suggestion. Right now it seems to me that it's a hard to verify the each test failing with enabled DFLTCC feature, I will send a PR to add a logic to run the ruby/zlib s390x case with DFLTCC=0
.
I sent the PR https://github.com/ruby/zlib/pull/63.
I am seeing the following test failures on RubyCI Ubuntu s390x version Jammy (22.04.3 LTS) on the ruby/zlib latest master branch 85637fa0fc2e5f9f1b9d8fd6652f9d81fd37f4d1 with the latest ruby/ruby master branch. Could you take a look at the failures? I think you can log in to the RubyCI Ubuntu s390x server to debug. Thank you for your help.
Below is the used Zlib deb package version.
ruby/ruby
I also can see the failures in ruby/ruby latest master branch
05a853c2f21f60f9e1c544c2d0709f10de453571
.