Closed chevah-robot closed 7 years ago
Comment by pqm at 2017-03-15 18:16:53Z.
dumol requested the review of this ticket.
needs-review
Comment by pqm at 2017-03-15 20:31:46Z.
adi approved changes. No more reviewers. Ready to merge.
if the test pass... please merge.
Please don't push the change into production just yet.
I want to release 3.20.0 and then after release we can push into production and have a full release to test the new build.
thanks
changes-approved
Comment by dumol at 2017-03-16 08:02:40Z.
ACK, will get this into master, but I won't upload new binaries in production.
Comment by pqm at 2017-03-16 19:11:05Z.
dumol requested the review of this ticket.
While reviewing my changes I have tried a fix for a minor issues and then discovered lots of redundant code in the chevahbs
scripts and got a bit carried away with those clean-ups. Have updated the pull request and have re-run the tests.
Consequently the changes are a bit bigger (lots of removals). For the record, these would be the relevant changes to master (minus the upstream sources and the patches for Python):
diff --git a/chevah_build b/chevah_build
index 735c88f9..e3b27e27 100755
--- a/chevah_build
+++ b/chevah_build
@@ -8,26 +8,32 @@
# publish_staging
#
-LIBFFI_VERSION=3.2.1
-GMP_VERSION=6.0.0
-LIBEDIT_VERSION=20150325-3.1
-PYTHON_BUILD_VERSION=2.7.10
-PYSQLITE_VERSION=2.6.3
-SQLITE_VERSION=3.8.1
+# Beware that older versions might be used for selected platforms below.
+PYTHON_BUILD_VERSION="2.7.13"
+LIBFFI_VERSION="3.2.1"
+# We statically build the BSD libedit on selected platforms to get the
+# readline module available without linking to the GPL-only readline libs.
+LIBEDIT_VERSION="20160903-3.1"
+# PyCrypto needs GMP or MPIR for fast math. GMP has support for AIX/HP-UX/etc.
+GMP_VERSION="6.1.2"
+SQLITE_VERSION="3.17.0"
+PYSQLITE_VERSION="2.6.3"
# Git revision to inject into Python's sys.version string through chevahbs.
PYTHON_PACKAGE_VERSION=$(git rev-parse --short HEAD)
# The ID of the redistributable version used on Windows.
REDISTRIBUTABLE_VERSION="9.00.30729.6161"
-export PYTHON_BUILD_VERSION PYTHON_PACKAGE_VERSION REDISTRIBUTABLE_VERSION
-
# pycparser needs to be explicitly installed to work around setuptools auto
# dependencies.
-PYCPARSER_VERSION=2.14
+PYCPARSER_VERSION="2.14"
-BUILD_LIBFFI="yes"
-BUILD_CFFI="yes"
+# Export the variables needed by the chevahbs scripts and the test phase.
+export PYTHON_BUILD_VERSION PYTHON_PACKAGE_VERSION REDISTRIBUTABLE_VERSION
+export BUILD_LIBEDIT="yes"
+export BUILD_GMP="yes"
+export BUILD_LIBFFI="yes"
+export BUILD_CFFI="yes"
EXTRA_LIBRARIES="\
python-modules/cffi-1.5.2 \
python-modules/pycrypto-2.6.1 \
@@ -103,7 +109,7 @@ INSTALLED_PACKAGES=''
# for example by exporting ARCH in paver.sh as "x86" instead of "x64" or
# "ppc" instead of "ppc64".
# We also use $ARCH when building the statically compiled libffi and GMP.
-# and $OS when building for ArchLinux
+# $OS is used when patching/configuring/building/testing.
export ARCH
export OS
# Explicitly choose the C compiler in order to make it possible to switch
@@ -153,8 +159,10 @@ case $OS in
export CFLAGS="$CFLAGS -qmaxmem=-1 -q64"
fi
fi
+ # There are issues with compiling libedit on AIX.
+ export BUILD_LIBEDIT="no"
# On AIX cffi based modules are not ported yet.
- BUILD_CFFI="no"
+ export BUILD_CFFI="no"
PIP_LIBRARIES=""
EXTRA_LIBRARIES=$EXTRA_LIBRARIES_NO_CFFI
;;
@@ -191,8 +199,10 @@ case $OS in
export LDFLAGS="$LDFLAGS -L/usr/lib/mps/64 -R/usr/lib/mps/64"
fi
fi
+ # readline with libedit is doable in Solaris, but the result is buggy.
+ export BUILD_LIBEDIT="no"
# cffi modules are not ready yet on Solaris.
- BUILD_CFFI="no"
+ export BUILD_CFFI="no"
PIP_LIBRARIES=""
EXTRA_LIBRARIES=$EXTRA_LIBRARIES_NO_CFFI
;;
@@ -202,7 +212,7 @@ case $OS in
export MAKE=gmake
;;
osx*)
- # The extra params needed to fix the minimum target version to 10.8
+ # The extra params needed to set the minimum target version to 10.8
# have the important benefit of getting rid of the libgcc dependency.
export CC="clang"
export CXX="clang++"
@@ -217,14 +227,24 @@ case $OS in
# The location of the Homebrew OpenSSL libs and include directories.
export LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
export CPPFLAGS="-I/usr/local/opt/openssl/include"
+ export BUILD_LIBEDIT="no"
;;
-
freebsd*)
export CC="clang"
export CXX="clang++"
+ export BUILD_LIBEDIT="no"
+ ;;
+ archlinux)
+ export BUILD_LIBEDIT="no"
+ ;;
+ linux*)
+ # We don't compile libedit for generic Linux builds because it links
+ # to local ncurses libs (one of ncurses, ncursesw, tinfo, others?),
+ # and the result is not portable.
+ export BUILD_LIBEDIT="no"
;;
sles10)
- BUILD_CFFI="no"
+ export BUILD_CFFI="no"
PIP_LIBRARIES=""
EXTRA_LIBRARIES=$EXTRA_LIBRARIES_NO_CFFI
;;
@@ -235,8 +255,11 @@ case $OS in
# On windows extra libraries are installed only using PIP.
EXTRA_LIBRARIES=""
PIP_LIBRARIES=$PIP_LIBRARIES_WIN
+ # For Windows we don't build everything from source yet.
+ export BUILD_LIBEDIT="no"
+ export BUILD_GMP="no"
# We don't build libffi, as cffi it is available as a wheel in pypi.
- BUILD_LIBFFI="no"
+ export BUILD_LIBFFI="no"
;;
esac
@@ -394,46 +417,29 @@ command_build() {
# Clean the build dir to avoid contamination from previous builds.
command_clean
- if [ $BUILD_LIBFFI == "yes" ]; then
+ # Build stuff statically on most platforms, copy headers and libs in
+ # the following locations, added to $CPPFLAGS and $LDFLAGS accordingly.
+ mkdir -p $INSTALL_FOLDER/{include,lib}
+ export CPPFLAGS="-I${INSTALL_FOLDER}/include/ ${CPPFLAGS}"
+ export LDFLAGS="-L${INSTALL_FOLDER}/lib/ ${LDFLAGS}"
+
+ if [ "$BUILD_LIBFFI" = "yes" ]; then
build 'libffi' "libffi-$LIBFFI_VERSION" ${PYTHON_BUILD_FOLDER}
+ else
+ (>&2 echo "Skipping building LIBFFI!")
fi
- case $OS in
- windows*)
- # For Windows we don't build everything from source yet.
- echo "Skipping GMP"
- ;;
- *)
- # Build GMP statically on all platforms, copy headers and libs in
- # a temp location, and update $CPPFLAGS and $LDFLAGS accordingly.
- # For any other statically-compiled lib, such as libedit,
- # we'll need these shell vars.
- build 'gmp' "gmp-$GMP_VERSION" ${PYTHON_BUILD_FOLDER}
- ;;
- esac
+ if [ "$BUILD_LIBEDIT" = "yes" ]; then
+ build 'libedit' "libedit-$LIBEDIT_VERSION" ${PYTHON_BUILD_FOLDER}
+ else
+ (>&2 echo "Skipping building LIBEDIT!")
+ fi
- mkdir -p $INSTALL_FOLDER/{include,lib}
- export CPPFLAGS="${CPPFLAGS} -I$INSTALL_FOLDER/include/"
- export LDFLAGS="${LDFLAGS} -L$INSTALL_FOLDER/lib/"
-
- # Statically build the BSD libedit on selected platforms to get readline
- # support without linking to the GPL-only readline. We don't do this for
- # generic Linux builds because the result depends on the local ncurses
- # libs (ncurses, ncursesw, tinfo, others?), and the result is not portable.
- # $CPPFLAGS and $LDFLAGS already point to these 'include' and 'lib' dirs.
- case $OS in
- sles10)
- # Don't build libedit on SLES 10 as this is an old platform and is
- # ok if it runs with limited support.
- true
- ;;
- ubuntu*|raspbian*|rhel*|sles*|solaris*|archlinux)
- build 'libedit' "libedit-$LIBEDIT_VERSION" ${PYTHON_BUILD_FOLDER}
- cp -r $INSTALL_FOLDER/tmp/libedit/{editline/,*.h} \
- $INSTALL_FOLDER/include/
- cp $INSTALL_FOLDER/tmp/libedit/libedit* $INSTALL_FOLDER/lib/
- ;;
- esac
+ if [ "$BUILD_GMP" = "yes" ]; then
+ build 'gmp' "gmp-$GMP_VERSION" ${PYTHON_BUILD_FOLDER}
+ else
+ (>&2 echo "Skipping building GMP!")
+ fi
case $OS in
windows*)
@@ -625,7 +631,6 @@ initialize_python_module(){
help_text_test=\
"Run a quick test for the Python from build."
command_test() {
- export BUILD_CFFI
test_file='test_python_binary_dist.py'
execute mkdir -p build/
execute cp python-modules/chevah-python-test/${test_file} build/
diff --git a/functions.sh b/functions.sh
index e44b4399..48617754 100644
--- a/functions.sh
+++ b/functions.sh
@@ -96,6 +96,11 @@ select_chevahbs_command() {
# Internal function for calling build script on each source.
#
chevahbs_build() {
+ if [ -n "$(type -t chevahbs_patch)" ]; then
+ # Looks like the chevahbs script has patches to apply.
+ echo "Patching..."
+ chevahbs_patch $@
+ fi
echo "Configuring..."
chevahbs_configure $@
echo "Compiling..."
@@ -149,6 +154,11 @@ build() {
echo "Copying source code ${build_folder}..."
execute cp -r ${source_folder} ${build_folder}
execute cp src/${project_folder}/chevahbs ${build_folder}/
+ if [ $(ls src/${project_folder}/*.patch 2>/dev/null | wc -l) -gt 0 ]; then
+ echo "The following patches are to be copied:"
+ ls -1 src/${project_folder}/*.patch
+ execute cp src/${project_folder}/*.patch ${build_folder}/
+ fi
execute cp 'functions.sh' ${build_folder}/
execute pushd ${build_folder}
diff --git a/python-modules/chevah-python-test/test_python_binary_dist.py b/python-modules/chevah-python-test/test_python_binary_dist.py
index 7bd1c4d3..d1078430 100644
--- a/python-modules/chevah-python-test/test_python_binary_dist.py
+++ b/python-modules/chevah-python-test/test_python_binary_dist.py
@@ -7,10 +7,10 @@ import subprocess
script_helper = './get_binaries_deps.sh'
platform_system = platform.system().lower()
-test_for_readline = False
with open('../DEFAULT_VALUES') as default_values_file:
chevah_os = default_values_file.read().split(' ')[2]
BUILD_CFFI = os.environ.get('BUILD_CFFI', 'no').lower() == 'yes'
+BUILD_LIBEDIT = os.environ.get('BUILD_LIBEDIT', 'no').lower() == 'yes'
def get_allowed_deps():
@@ -37,7 +37,6 @@ def get_allowed_deps():
]
# Distro-specific deps to add. Now we may specify major versions too.
if ('rhel' in chevah_os):
- test_for_readline = True
allowed_deps.extend([
'libcom_err.so.2',
'libgssapi_krb5.so.2',
@@ -65,23 +64,19 @@ def get_allowed_deps():
'libpcre.so.1',
])
elif ('sles' in chevah_os):
- test_for_readline = True
sles_version = int(chevah_os[4:])
- if sles_version >= 11:
- allowed_deps.extend([
- 'libncursesw.so.5',
- ])
+ allowed_deps.extend([
+ 'libncursesw.so.5',
+ ])
if sles_version >= 12:
allowed_deps.extend([
'libtinfo.so.5',
])
elif ('ubuntu' in chevah_os):
- test_for_readline = True
allowed_deps.extend([
'libtinfo.so.5',
])
elif ('raspbian' in chevah_os):
- test_for_readline = True
allowed_deps.extend([
'libcofi_rpi.so',
'libgcc_s.so.1',
@@ -117,7 +112,6 @@ def get_allowed_deps():
'libthread.a',
])
elif platform_system == 'sunos':
- test_for_readline = True
# This is the common list of deps for Solaris 10 & 11 builds.
allowed_deps = [
'libc.so.1',
@@ -184,6 +178,7 @@ def get_allowed_deps():
allowed_deps.extend([
'/usr/lib/libcrypto.0.9.8.dylib',
'/usr/lib/libssl.0.9.8.dylib',
+ '/usr/lib/libncurses.5.4.dylib',
])
elif ('macos' in chevah_os):
# Additional deps when using Homebrew's OpenSSL.
@@ -210,6 +205,7 @@ def get_allowed_deps():
'/usr/lib/libc.so',
'/usr/lib/libcrypto.so',
'/usr/lib/libm.so',
+ '/usr/lib/libncursesw.so.14.0',
'/usr/lib/libpthread.so',
'/usr/lib/libssl.so',
'/usr/lib/libutil.so',
@@ -500,17 +496,22 @@ def main():
sys.stderr.write('"spwd" missing.\n')
exit_code = 11
+
# We compile the readline module using libedit only on selected platforms.
- if test_for_readline:
+ if BUILD_LIBEDIT:
try:
import readline
readline.get_history_length()
except:
sys.stderr.write('"readline" missing.\n')
exit_code = 12
+ else:
+ print '"readline" module is present.'
+
exit_code = test_dependencies() | exit_code
+
sys.exit(exit_code)
diff --git a/src/gmp/chevahbs b/src/gmp/chevahbs
index 2610def9..7d738df7 100755
--- a/src/gmp/chevahbs
+++ b/src/gmp/chevahbs
@@ -39,23 +39,7 @@ chevahbs_compile() {
chevahbs_install() {
install_folder=$1
- # As with libffi, we copy the files manually to a temporary location and
- # then we get them from there before compiling PyCrypto. We'll also have
- # to instruct the compiler to link against our GMP by augmenting CPPFLAGS
- # with the include's location and LDFLAGS with the location of the libs.
- local temp_folder=$INSTALL_FOLDER/tmp/gmp/
- execute mkdir -p $temp_folder
- execute cp .libs/* $temp_folder
- ranlib $temp_folder/libgmp.a
- execute cp gmp.h $temp_folder
-
- # We need GMP or MPIR for fast math in PyCrypto. We use GMP because
- # its devs do a better job of supporting exotic platforms such as AIX.
- # $CPPFLAGS and $LDFLAGS already point to these 'include' and 'lib' dirs.
- mkdir -p $INSTALL_FOLDER/include
- mkdir -p $INSTALL_FOLDER/lib
- cp $INSTALL_FOLDER/tmp/gmp/gmp.h $INSTALL_FOLDER/include
- cp $INSTALL_FOLDER/tmp/gmp/libgmp* $INSTALL_FOLDER/lib
+ execute $MAKE install DESTDIR=$INSTALL_FOLDER
}
diff --git a/src/libedit/chevahbs b/src/libedit/chevahbs
index 809a7504..373ab52c 100755
--- a/src/libedit/chevahbs
+++ b/src/libedit/chevahbs
@@ -8,19 +8,8 @@
chevahbs_configure() {
# We want static builds only, so that we don't have to mess with LIBPATH.
- # Support for Unicode (wide-char/UTF-8) is added with "--enable-widec".
- CONF_OPTS="--disable-shared --enable-static --enable-widec"
- execute mkdir m4
- # RHEL 4 and 5 have an older autoconf, and we can't auto-reconfigure.
- case $OS in
- rhel4|rhel5)
- # This redirects the echo output to stderr.
- >&2 echo "This seems to be a RHEL 4/5 system, skipping autoreconf!"
- ;;
- *)
- execute autoreconf --install --force
- ;;
- esac
+ CONF_OPTS="--disable-shared --enable-static"
+ touch *
execute ./configure --prefix="" $CONF_OPTS
}
@@ -32,15 +21,7 @@ chevahbs_compile() {
chevahbs_install() {
install_folder=$1
- # We copy the includes and libs manually to a temporary location, and
- # we instruct the compiler to link against our lib by augmenting CPPFLAGS
- # with the include's location and LDFLAGS with the location of the libs.
- local temp_folder=$INSTALL_FOLDER/tmp/libedit/
- execute mkdir -p $temp_folder
- execute cp src/.libs/* $temp_folder
- execute ranlib $temp_folder/libedit.a
- execute chmod +x $temp_folder/libedit.a
- execute cp -r src/*.h src/editline/ $temp_folder
+ execute $MAKE install DESTDIR=$INSTALL_FOLDER
}
diff --git a/src/libffi/chevahbs b/src/libffi/chevahbs
index 4e87acef..77523f74 100755
--- a/src/libffi/chevahbs
+++ b/src/libffi/chevahbs
@@ -22,39 +22,34 @@ chevahbs_compile() {
chevahbs_install() {
install_folder=$1
- if [ x${ARCH##ppc} != x"$ARCH" ]; then
- # libffi installs its headers in $PREFIX/lib/libffi-$VERSION/include,
- # instead of $PREFIX/include, so we copy them manually to a temporary
- # location.
- # From there we will copy them just in time for building Python sources.
- # For Python's setup.py to pick them up, we will augment CPPFLAGS with
- # the include's location and LDFLAGS with the location of the libs.
- local temp_folder=$INSTALL_FOLDER/tmp/libffi
- execute mkdir -p $temp_folder
-
- execute cp powerpc-ibm-aix*/.libs/* $temp_folder
- execute cp powerpc-ibm-aix*/include/*.h $temp_folder
- elif [ "$OS" = "solaris10" ]; then
- # I don't know why it is skip... but doing the actual install will
- # break the build.
- echo 'Skipping installation of libffi on Solaris 10.'
- elif [ "$OS" = "solaris11" ]; then
- # I don't know why it is skip... but doing the actual install will
- # break the build.
- echo 'Skipping installation of libffi on Solaris 11.'
- else
- execute $MAKE install DESTDIR=$INSTALL_FOLDER
-
- # libffi installs its headers in $PREFIX/lib/libffi-$VERSION/include,
- # instead of $PREFIX/include, so we copy them manually to the default
- # location.
- execute mkdir -p $INSTALL_FOLDER/include
- execute cp $INSTALL_FOLDER/lib/libffi-*/include/* $INSTALL_FOLDER/include
- # On some systems libffi in installed in /lib64 and then cffi is
- # searching for it in /lib.
- # THis affect RHEL, SLES and OpenBSD.
- cp $INSTALL_FOLDER/lib64/* $INSTALL_FOLDER/lib
- echo 'cffi done... here to ignore a possible previous error'
+ # Installing fails on some OS'es.
+ # Following workarounds should cover all 32/64bit cases.
+ case $OS in
+ aix*)
+ execute ln -s ../install-sh powerpc-ibm-aix"$(oslevel)"/install-sh
+ ;;
+ solaris10)
+ case $ARCH in
+ sparc*)
+ execute ln -s ../install-sh sparc-sun-solaris2.10/install-sh
+ ;;
+ *)
+ execute ln -s ../install-sh i386-pc-solaris2.10/install-sh
+ ;;
+ esac
+ ;;
+ esac
+
+ execute $MAKE install DESTDIR=$INSTALL_FOLDER
+
+ # libffi installs its headers in $PREFIX/lib/libffi-$VERSION/include,
+ # instead of $PREFIX/include, so we copy them to the default location.
+ execute mkdir -p $INSTALL_FOLDER/include
+ execute cp $INSTALL_FOLDER/lib*/libffi-*/include/* $INSTALL_FOLDER/include/
+ # On some systems libffi is installed in /lib64 and then cffi is
+ # searching for it in /lib. This affects RHEL 5-7 and SLES 11-12.
+ if [ -d $INSTALL_FOLDER/lib64 ]; then
+ execute cp $INSTALL_FOLDER/lib64/* $INSTALL_FOLDER/lib/
fi
}
diff --git a/src/python/chevahbs b/src/python/chevahbs
index 199599d5..844f5446 100755
--- a/src/python/chevahbs
+++ b/src/python/chevahbs
@@ -5,6 +5,25 @@
# Import shared code.
. ./functions.sh
+chevahbs_patch() {
+ echo "Applying disabled_modules.patch:"
+ patch < disabled_modules.patch
+ if [ "$BUILD_LIBEDIT" = "yes" ]; then
+ echo "Applying readline_libedit.patch:"
+ patch -p0 < readline_libedit.patch
+ fi
+ case $OS in
+ solaris10)
+ echo "Applying solaris10_paths.patch:"
+ patch < solaris10_paths.patch
+ ;;
+ openbsd*)
+ echo "Applying random_openbsd.patch:"
+ patch < random_openbsd.patch
+ esac
+}
+
+
chevahbs_configure() {
CONFIG_ARGS="--disable-shared"
@@ -28,20 +47,14 @@ chevahbs_configure() {
"
;;
aix*)
- # In AIX we build _ctypes with external libffi, but not the system
- # one. We use our libffi files and statically link against its libs.
- execute mkdir -p build/libffi
- execute cp $INSTALL_FOLDER/tmp/libffi/* build/libffi/
- # The following two parameters are picked up by Python's setup.py
- # and will convince it to use our external libffi for _ctypes.
- export CPPFLAGS="${CPPFLAGS} -Ibuild/libffi/"
- export LDFLAGS="${LDFLAGS} -Lbuild/libffi/"
# Workaround for http://bugs.python.org/issue21917
echo "import os; os.__dict__.pop('O_NOFOLLOW', None)" \
>> Lib/site-packages/sitecustomize.py
- # MAXMEM option with a value greater than 8192.
+ # Computed goto's are a problem with Python 2.7.13,
+ # details at https://bugs.python.org/issue9799.
CONFIG_ARGS="${CONFIG_ARGS} \
--with-system-ffi \
+ --without-computed-gotos \
"
;;
solaris10)
@@ -68,6 +81,11 @@ chevahbs_configure() {
;;
esac
+ # This requires the v4 patch from https://bugs.python.org/issue13501.
+ if [ "$BUILD_LIBEDIT" = "yes" ]; then
+ CONFIG_ARGS="${CONFIG_ARGS} --with-readline=editline"
+ fi
+
case $OS in
windows)
echo "Skip configure on Windows."
needs-review
Comment by pqm at 2017-03-16 20:51:52Z.
adi approved changes. No more reviewers. Ready to merge.
thanks. changes-approved
Comment by dumol at 2017-03-17 11:46:41Z.
Merged and updated the following packages:
root@weesp:~# readlink /srv/binary-site/binary.chevah.com/production/python/* | grep 8f291bb
archlinux/x64/python-2.7.13.8f291bb-archlinux-x64.tar.gz
freebsd10/x64/python-2.7.13.8f291bb-freebsd10-x64.tar.gz
linux/x64/python-2.7.13.8f291bb-linux-x64.tar.gz
linux/x86/python-2.7.13.8f291bb-linux-x86.tar.gz
macos1012/x64/python-2.7.13.8f291bb-macos1012-x64.tar.gz
openbsd60/x64/python-2.7.13.8f291bb-openbsd60-x64.tar.gz
osx108/x64/python-2.7.13.8f291bb-osx108-x64.tar.gz
raspbian7/armv7l/python-2.7.13.8f291bb-raspbian7-armv7l.tar.gz
ubuntu1204/x64/python-2.7.13.8f291bb-ubuntu1204-x64.tar.gz
ubuntu1204/x86/python-2.7.13.8f291bb-ubuntu1204-x86.tar.gz
ubuntu1404/x64/python-2.7.13.8f291bb-ubuntu1404-x64.tar.gz
ubuntu1604/x64/python-2.7.13.8f291bb-ubuntu1604-x64.tar.gz
T3931 task was created by dumol on 2017-03-10 15:03:52Z. Last changed on 2017-04-11 09:19:15Z. PR at https://github.com/chevah/python-package/pull/71.
For easier porting of the upstream patches at ftp://hpux.connect.org.uk/hpux/, needed for #367, we have to get the latest stable Python (2.7.13) and GMP (6.1.2) in python-package.
I think a separate branch would make for a clearer repo history.