search

newhopecrypto / newhope

Software of the NIST Post-Quantum submission NewHope
https://newhopecrypto.org
43 stars 9 forks source link

MacOS: code fails to compile under current Xcode #6

Open mouse07410 opened 4 years ago

mouse07410 commented 4 years ago

This happens with the ref/Makefile intact:

$ /usr/bin/gcc -v
Configured with: --prefix=/Applications/Xcode.app/Contents/Developer/usr --with-gxx-include-dir=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/4.2.1
Apple clang version 11.0.3 (clang-1103.0.32.29)
Target: x86_64-apple-darwin19.4.0
Thread model: posix
InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin
$ make
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -o PQCgenKAT_cpakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
rng.c:10:10: fatal error: 'openssl/conf.h' file not found
#include <openssl/conf.h>
         ^~~~~~~~~~~~~~~~
1 error generated.
make: *** [PQCgenKAT_cpakem512] Error 1

After pointing the compiler at the correct location of the OpenSSL-1.1.1 (there's no other usable OpenSSL on this box, except for OpenSSL-3.0 Dev master):

diff --git a/ref/Makefile b/ref/Makefile
index f7d4469..5f6fbba 100644
--- a/ref/Makefile
+++ b/ref/Makefile
@@ -1,6 +1,6 @@
 CC = /usr/bin/gcc
-CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native
-NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie
+CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib
+NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib

 HEADERS = params.h poly.h randombytes.h ntt.h reduce.h verify.h cpapke.h
 SOURCES = poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c

the code in ref/ builds but crashes:

$ make
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_cpakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_cpakem1024 -DNEWHOPE_N=1024 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf ccakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_ccakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c ccakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf ccakem.h api.h
/usr/bin/gcc -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_ccakem1024 -DNEWHOPE_N=1024 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c ccakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 cpakem.c randombytes.c test_newhope.c -o test_cpakem512
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 cpakem.c randombytes.c test_newhope.c -o test_cpakem1024
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 ccakem.c randombytes.c test_newhope.c -o test_ccakem512
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 ccakem.c randombytes.c test_newhope.c -o test_ccakem1024
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 cpakem.c randombytes.c cpucycles.c speed.c -o speed_cpakem512
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 cpakem.c randombytes.c cpucycles.c speed.c -o speed_cpakem1024
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 ccakem.c randombytes.c cpucycles.c speed.c -o speed_ccakem512
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 ccakem.c randombytes.c cpucycles.c speed.c -o speed_ccakem1024
$ ./PQCgenKAT_ccakem512 
Segmentation fault: **11**

The problem appears to be stack alignment that gets screwed up by the Xcode compilers - from the crash report:

Thread 0 Crashed:: Dispatch queue: com.apple.main-thread
0   libdyld.dylib                   0x00007fff6c5d970a stack_not_16_byte_aligned_error + 0
1   ???                             0x00007ffee97bce08 0 + 140732815625736

This workaround (adding -ffrestanding flag) helps:

diff --git a/ref/Makefile b/ref/Makefile
index f7d4469..f7fe9bc 100644
--- a/ref/Makefile
+++ b/ref/Makefile
@@ -1,6 +1,6 @@
 CC = /usr/bin/gcc
-CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native
-NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie
+CFLAGS = -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib
+NISTFLAGS = -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib

 HEADERS = params.h poly.h randombytes.h ntt.h reduce.h verify.h cpapke.h
 SOURCES = poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c

and produces desirable results:

$ make
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_cpakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf cpakem.h api.h
/usr/bin/gcc -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_cpakem1024 -DNEWHOPE_N=1024 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c cpakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf ccakem.h api.h
/usr/bin/gcc -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_ccakem512 -DNEWHOPE_N=512 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c ccakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf ccakem.h api.h
/usr/bin/gcc -O3 -ffreestanding -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib -o PQCgenKAT_ccakem1024 -DNEWHOPE_N=1024 poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c ccakem.c -I. rng.c PQCgenKAT_kem.c -lcrypto
clang: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 cpakem.c randombytes.c test_newhope.c -o test_cpakem512
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 cpakem.c randombytes.c test_newhope.c -o test_cpakem1024
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 ccakem.c randombytes.c test_newhope.c -o test_ccakem512
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 ccakem.c randombytes.c test_newhope.c -o test_ccakem1024
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 cpakem.c randombytes.c cpucycles.c speed.c -o speed_cpakem512
ln -sf cpakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 cpakem.c randombytes.c cpucycles.c speed.c -o speed_cpakem1024
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=512 ccakem.c randombytes.c cpucycles.c speed.c -o speed_ccakem512
ln -sf ccakem.h api.h
/usr/bin/gcc -Wall -Wextra -ffreestanding -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c -DNEWHOPE_N=1024 ccakem.c randombytes.c cpucycles.c speed.c -o speed_ccakem1024
$ ./PQCgenKAT_ccakem512 
$

Switching from Xcode clang to Macports-installed clang-mp-9.0 alleviated the need to add -ffreestanding flag.

Update

Similar problem with avx2/ code, for the same reason ("stack not 16-byte aligned"). Compiling with Macports-installed clang-mp-9.0, or adding -ffreestanding flag resolved it here as well.

mouse07410 commented 4 years ago

Finally, the following patch allows build and tests succeed using Xcode-11.4 clang:

diff --git a/avx2/Makefile b/avx2/Makefile
index 83c0285..41ef314 100644
--- a/avx2/Makefile
+++ b/avx2/Makefile
@@ -1,6 +1,6 @@
-CC = /usr/bin/gcc
-CFLAGS = -no-pie -Wall -Wextra -g -O3 -fomit-frame-pointer -msse2avx -mavx2 -march=native
-NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie
+CC ?= gcc
+CFLAGS = -ffreestanding -fno-stack-check -no-pie -Wall -Wextra -g -O3 -fomit-frame-pointer -msse2 -mavx -mavx2 -march=native -I/opt/local/include -L/opt/local/lib 
+NISTFLAGS = -ffreestanding -fno-stack-check -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib

 HEADERS = params.h poly.h randombytes.h ntt.h reduce.h verify.h cpapke.h
 SOURCES = poly.c reduce.c fips202.c  verify.c cpapke.c ntt_double.s ntt.c precomp.c
@@ -19,7 +19,7 @@ all:  PQCgenKAT_cpakem512 \
            speed_cpakem512 \
            speed_cpakem1024 \
            speed_ccakem512 \
-           speed_ccakem1024 \
+           speed_ccakem1024

 PQCgenKAT_cpakem512: $(SOURCES) $(HEADERS) PQCgenKAT_kem.c rng.c rng.h
    ln -sf cpakem.h api.h
diff --git a/avx2/poly.h b/avx2/poly.h
index 1bfa840..92d4d0a 100644
--- a/avx2/poly.h
+++ b/avx2/poly.h
@@ -2,6 +2,7 @@
 #define POLY_H

 #include <stdint.h>
+#include <stddef.h>
 #include "params.h"

 typedef struct {
diff --git a/avx2/speed.c b/avx2/speed.c
index bb9c225..a3f0d64 100644
--- a/avx2/speed.c
+++ b/avx2/speed.c
@@ -13,7 +13,7 @@ static int cmp_llu(const void *a, const void*b)
   return 0;
 }

-static unsigned long long median(unsigned long long *l, size_t llen)
+static unsigned long long median(unsigned long long *l, const size_t llen)
 {
   qsort(l,llen,sizeof(unsigned long long),cmp_llu);

@@ -21,7 +21,7 @@ static unsigned long long median(unsigned long long *l, size_t llen)
   else return (l[llen/2-1]+l[llen/2])/2;
 }

-static unsigned long long average(unsigned long long *t, size_t tlen)
+static unsigned long long average(unsigned long long *t, const size_t tlen)
 {
   unsigned long long acc=0;
   size_t i;
@@ -30,7 +30,7 @@ static unsigned long long average(unsigned long long *t, size_t tlen)
   return acc/(tlen);
 }

-static void print_results(const char *s, unsigned long long *t, size_t tlen)
+static void print_results(const char *s, unsigned long long *t, const size_t tlen)
 {
   size_t i;
   printf("%s", s);
diff --git a/avx2/test_newhope.c b/avx2/test_newhope.c
index e30ddcd..23b85d1 100644
--- a/avx2/test_newhope.c
+++ b/avx2/test_newhope.c
@@ -1,9 +1,9 @@
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
 #include "api.h"
 #include "poly.h"
 #include "randombytes.h"
-#include <math.h>
-#include <stdio.h>
-#include <string.h>

 #define NTESTS 1000

@@ -24,7 +24,7 @@ int test_keys()
     //Bob derives a secret key and creates a response
     crypto_kem_enc(sendb, key_b, pk);

-    //Alice uses Bobs response to get her secre key
+    //Alice uses Bobs response to get her secret key
     crypto_kem_dec(key_a, sendb, sk_a);

     if(memcmp(key_a, key_b, 32))
@@ -65,7 +65,7 @@ int test_invalid_sk_a()
     //Replace secret key with random values
     fread(sk_a, CRYPTO_SECRETKEYBYTES, 1, urandom); 

-    //Alice uses Bobs response to get her secre key
+    //Alice uses Bobs response to get her secret key
     crypto_kem_dec(key_a, sendb, sk_a);

     if(!memcmp(key_a, key_b, 32))
@@ -99,7 +99,7 @@ int test_invalid_ciphertext()
     //Change some byte in the ciphertext (i.e., encapsulated key)
     sendb[42] ^= 23;

-    //Alice uses Bobs response to get her secre key
+    //Alice uses Bobs response to get her secret key
     crypto_kem_dec(key_a, sendb, sk_a);

     if(!memcmp(key_a, key_b, 32))
diff --git a/avx2/verify.c b/avx2/verify.c
index 7d01a47..b634a24 100644
--- a/avx2/verify.c
+++ b/avx2/verify.c
@@ -2,7 +2,7 @@
 #include <stdint.h>

 /* returns 0 for equal strings, 1 for non-equal strings */
-int verify(const unsigned char *a, const unsigned char *b, size_t len)
+int verify(const unsigned char *a, const unsigned char *b, const size_t len)
 {
   uint64_t r;
   size_t i;
diff --git a/avx2/verify.h b/avx2/verify.h
index 5394316..693af4b 100644
--- a/avx2/verify.h
+++ b/avx2/verify.h
@@ -4,7 +4,7 @@
 #include <stdio.h>

 /* returns 0 for equal strings, 1 for non-equal strings */
-int verify(const unsigned char *a, const unsigned char *b, size_t len);
+int verify(const unsigned char *a, const unsigned char *b, const size_t len);

 /* b = 1 means mov, b = 0 means don't mov*/
 void cmov(unsigned char *r, const unsigned char *x, size_t len, unsigned char b);
diff --git a/ref/Makefile b/ref/Makefile
index f7d4469..a133512 100644
--- a/ref/Makefile
+++ b/ref/Makefile
@@ -1,6 +1,6 @@
-CC = /usr/bin/gcc
-CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native
-NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie
+CC ?= gcc
+CFLAGS = -Wall -Wextra -g -O3 -fomit-frame-pointer -march=native -I/opt/local/include -L/opt/local/lib
+NISTFLAGS = -O3 -fomit-frame-pointer -march=native -fPIC -no-pie -I/opt/local/include -L/opt/local/lib

 HEADERS = params.h poly.h randombytes.h ntt.h reduce.h verify.h cpapke.h
 SOURCES = poly.c reduce.c fips202.c verify.c cpapke.c ntt.c precomp.c
@@ -18,7 +18,7 @@ all:  PQCgenKAT_cpakem512 \
            speed_cpakem512 \
            speed_cpakem1024 \
            speed_ccakem512 \
-           speed_ccakem1024 \
+           speed_ccakem1024 

 PQCgenKAT_cpakem512: $(SOURCES) $(HEADERS) PQCgenKAT_kem.c rng.c rng.h
    ln -sf cpakem.h api.h
diff --git a/ref/poly.h b/ref/poly.h
index efe5f49..4728a30 100644
--- a/ref/poly.h
+++ b/ref/poly.h
@@ -2,6 +2,7 @@
 #define POLY_H

 #include <stdint.h>
+#include <stddef.h>
 #include "params.h"

 /*

I took the liberty of adding const where it belonged and the required header files where they were missing. Also, re-ordered the headers.

Here's the patch file: macosx.patch.zip