Error upon casting load_npy during compilation

[OneRaynyDay]

Reproducible code:

#define XTENSOR_USE_XSIMD

#include <iostream>
#include <xtensor/xarray.hpp>
#include <xtensor/xio.hpp>
#include <xtensor/xview.hpp>
#include <xtensor/xexpression.hpp>
#include <xtensor/xnpy.hpp>
#include <xtensor/xsort.hpp>
#include <xtensor-blas/xlinalg.hpp>
#include <unordered_map>
#include <sys/stat.h>
#include <unistd.h>
#include <chrono>

bool exists (const std::string& name) {
  struct stat buffer;
  return (stat (name.c_str(), &buffer) == 0);
}

template <typename T>
xt::xarray<T> getParameter(const std::string& path){
    if(!exists(path))
        throw std::runtime_error("The parameter on path : " + path + " does not exist.");
    return xt::cast<T>(xt::load_npy<double>(path));
}

extern "C" {
auto&& Block__0Dense__weight = getParameter<double>("weight");
auto&& Block__0Dense__bias = getParameter<double>("bias");
}

My Makefile:

make:
    g++-8 -fopenmp -fPIC -std=c++14 -I$CONDA_PREFIX/include -shared -Wl,-undefined,dynamic_lookup -xc++ -I/usr/local/include -o binary main.cpp

Error:

In file included from /usr/local/Cellar/gcc/8.2.0/include/c++/8.2.0/functional:54,
                 from /Users/ray_zhang/anaconda3/envs/idp3/include/xtensor/xbuffer_adaptor.hpp:13,
                 from /Users/ray_zhang/anaconda3/envs/idp3/include/xtensor/xarray.hpp:18,
                 from main.cpp:4:
/usr/local/Cellar/gcc/8.2.0/include/c++/8.2.0/tuple: In instantiation of 'constexpr std::_Head_base<_Idx, _Head, false>::_Head_base(_UHead&&) [with _UHead = const xt::xarray_adaptor<xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >, (xt::layout_type)0, std::vector<long unsigned int>, xt::xtensor_expression_tag>; long unsigned int _Idx = 0; _Head = const xt::xarray_adaptor<xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >, (xt::layout_type)0, std::vector<long unsigned int>, xt::xtensor_expression_tag>]':
/usr/local/Cellar/gcc/8.2.0/include/c++/8.2.0/tuple:373:49:   required from 'constexpr std::_Tuple_impl<_Idx, _Head>::_Tuple_impl(std::_Tuple_impl<_Idx, _Head>&&) [with long unsigned int _Idx = 0; _Head = const xt::xarray_adaptor<xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >, (xt::layout_type)0, std::vector<long unsigned int>, xt::xtensor_expression_tag>]'
/usr/local/Cellar/gcc/8.2.0/include/c++/8.2.0/tuple:662:17:   required from 'auto xt::detail::make_xfunction(E&& ...) [with F = xt::detail::cast<float>::functor; E = {xt::xarray_adaptor<xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >, (xt::layout_type)0, std::vector<long unsigned int, std::allocator<long unsigned int> >, xt::xtensor_expression_tag>}]'
/Users/ray_zhang/anaconda3/envs/idp3/include/xtensor/xoperation.hpp:836:73:   required from 'xt::detail::xfunction_type_t<xt::detail::cast<R>::template functor, E> xt::cast(E&&) [with R = float; E = xt::xarray_adaptor<xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >, (xt::layout_type)0, std::vector<long unsigned int>, xt::xtensor_expression_tag>; xt::detail::xfunction_type_t<xt::detail::cast<R>::template functor, E> = xt::xfunction<xt::detail::cast<float>::functor<double>, float, const xt::xarray_adaptor<xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >, (xt::layout_type)0, std::vector<long unsigned int, std::allocator<long unsigned int> >, xt::xtensor_expression_tag> >]'
main.cpp:26:23:   required from 'xt::xarray<T> getParameter(const string&) [with T = float; xt::xarray<T> = xt::xarray_container<xt::uvector<float, xsimd::aligned_allocator<float, 16> >, (xt::layout_type)1, xt::svector<long unsigned int, 4, std::allocator<long unsigned int>, true>, xt::xtensor_expression_tag>; typename std::vector<T, xsimd::aligned_allocator<T, 16> >::size_type = long unsigned int; typename xt::uvector<T, xsimd::aligned_allocator<T, 16> >::size_type = long unsigned int; std::__cxx11::string = std::__cxx11::basic_string<char>]'
main.cpp:30:136:   required from here
/usr/local/Cellar/gcc/8.2.0/include/c++/8.2.0/tuple:133:42: error: use of deleted function 'xt::xarray_adaptor<EC, L, SC, Tag>::xarray_adaptor(const xt::xarray_adaptor<EC, L, SC, Tag>&) [with EC = xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >; xt::layout_type L = (xt::layout_type)0; SC = std::vector<long unsigned int>; Tag = xt::xtensor_expression_tag]'
  : _M_head_impl(std::forward<_UHead>(__h)) { }
                                          ^
In file included from main.cpp:4:
/Users/ray_zhang/anaconda3/envs/idp3/include/xtensor/xarray.hpp:196:9: note: 'xt::xarray_adaptor<EC, L, SC, Tag>::xarray_adaptor(const xt::xarray_adaptor<EC, L, SC, Tag>&) [with EC = xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >; xt::layout_type L = (xt::layout_type)0; SC = std::vector<long unsigned int>; Tag = xt::xtensor_expression_tag]' is implicitly deleted because the default definition would be ill-formed:
         xarray_adaptor(const xarray_adaptor&) = default;
         ^~~~~~~~~~~~~~
/Users/ray_zhang/anaconda3/envs/idp3/include/xtensor/xarray.hpp:196:9: error: use of deleted function 'xt::xbuffer_adaptor<CP, O, A>::xbuffer_adaptor(const self_type&) [with CP = double*; O = xt::acquire_ownership; A = std::allocator<double>; xt::xbuffer_adaptor<CP, O, A>::self_type = xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >]'
In file included from /Users/ray_zhang/anaconda3/envs/idp3/include/xtensor/xarray.hpp:18,
                 from main.cpp:4:
/Users/ray_zhang/anaconda3/envs/idp3/include/xtensor/xbuffer_adaptor.hpp:179:9: note: 'xt::xbuffer_adaptor<CP, O, A>::xbuffer_adaptor(const self_type&) [with CP = double*; O = xt::acquire_ownership; A = std::allocator<double>; xt::xbuffer_adaptor<CP, O, A>::self_type = xt::xbuffer_adaptor<double*, xt::acquire_ownership, std::allocator<double> >]' is implicitly deleted because the default definition would be ill-formed:
         xbuffer_adaptor(const self_type&) = default;
         ^~~~~~~~~~~~~~~
/Users/ray_zhang/anaconda3/envs/idp3/include/xtensor/xbuffer_adaptor.hpp:179:9: error: use of deleted function 'xt::detail::xbuffer_owner_storage<CP, A>::xbuffer_owner_storage(const self_type&) [with CP = double*; A = std::allocator<double>; xt::detail::xbuffer_owner_storage<CP, A>::self_type = xt::detail::xbuffer_owner_storage<double*, std::allocator<double> >]'
/Users/ray_zhang/anaconda3/envs/idp3/include/xtensor/xbuffer_adaptor.hpp:109:13: note: declared here
             xbuffer_owner_storage(const self_type&) = delete;
             ^~~~~~~~~~~~~~~~~~~~~
make: *** [make] Error 1

GCC version 8.2.0.

[wolfv]

Thanks for the bug report! I'm going to look into it.

[wolfv]

shoot it looks like we've got a big problem here!

The first problem which is supposed to be easily solved is that one cannot copy an owning buffer, which you're not doing, because you're rvalue constructing it.

However, our xfunction doesn't correctly set rvalues in the std::tuple so the move constructor of the tuple actually doesn't work right! Because we use the closure type in the std::tuple, the tuple of arguments will look like something like std::tuple<xt::xarray_adaptor<...>> (not an rvalue).

However, the move constructor of the tuple calls std::forward<TYPE>(value), which only works if we set the tuple member to an rvalue! In effect it appears to work if I hack my own xfunction type like the following:

using buffer_type = std::decay_t<decltype(load_npy<double>("mnist.npy"))>;
using xfunction_type = xfunction<xt::detail::identity<double>, double, const buffer_type&&>;
return xfunction_type(xt::detail::identity<double>(), load_npy<double>("mnist.npy"));

Note for the sake of the argument I use the identity function, this line crashes the same way (return +load_npy<double>("mnist.npy");

So I guess we need to propagate the rvalue information much better.

@SylvainCorlay @JohanMabille thoughts?

That's the error message:

In file included from /usr/bin/../lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/functional:54:
/usr/bin/../lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/tuple:133:4: error: 
      call to implicitly-deleted copy constructor of 'const
      xt::xarray_adaptor<xt::xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >, xt::layout_type::dynamic, std::vector<unsigned long,
      std::allocator<unsigned long> >, xt::xtensor_expression_tag>'
        : _M_head_impl(std::forward<_UHead>(__h)) { }
          ^            ~~~~~~~~~~~~~~~~~~~~~~~~~
/usr/bin/../lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/tuple:373:9: note: in
      instantiation of function template specialization 'std::_Head_base<0, const
      xt::xarray_adaptor<xt::xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >, xt::layout_type::dynamic, std::vector<unsigned long,
      std::allocator<unsigned long> >, xt::xtensor_expression_tag>, false>::_Head_base<const
      xt::xarray_adaptor<xt::xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >, xt::layout_type::dynamic, std::vector<unsigned long,
      std::allocator<unsigned long> >, xt::xtensor_expression_tag> >' requested here
      : _Base(std::forward<_Head>(_M_head(__in))) { }
        ^
/usr/bin/../lib/gcc/x86_64-redhat-linux/8/../../../../include/c++/8/tuple:662:17: note: in
      instantiation of member function 'std::_Tuple_impl<0, const
      xt::xarray_adaptor<xt::xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >, xt::layout_type::dynamic, std::vector<unsigned long,
      std::allocator<unsigned long> >, xt::xtensor_expression_tag> >::_Tuple_impl' requested
      here
      constexpr tuple(tuple&&) = default;
                ^
/home/wolfv/Programs/xtensor/tutorial/../include/xtensor/xfunction.hpp:342:9: note: in
      implicit move constructor for 'std::tuple<const
      xt::xarray_adaptor<xt::xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >, xt::layout_type::dynamic, std::vector<unsigned long,
      std::allocator<unsigned long> >, xt::xtensor_expression_tag> >' first required here
        xfunction_base(xfunction_base&&) = default;
        ^
/home/wolfv/Programs/xtensor/tutorial/../include/xtensor/xfunction.hpp:573:9: note: in
      implicit move constructor for 'xt::xfunction_base<xt::detail::identity<double>,
      double, const xt::xarray_adaptor<xt::xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >, xt::layout_type::dynamic, std::vector<unsigned long,
      std::allocator<unsigned long> >, xt::xtensor_expression_tag> >' first required here
        xfunction(xfunction&&) = default;
        ^
/home/wolfv/Programs/xtensor/tutorial/../include/xtensor/xoperation.hpp:252:20: note: in
      implicit move constructor for 'xt::xfunction<xt::detail::identity<double>, double,
      const xt::xarray_adaptor<xt::xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >, xt::layout_type::dynamic, std::vector<unsigned long,
      std::allocator<unsigned long> >, xt::xtensor_expression_tag> >' first required here
            return type(functor_type(), std::forward<E>(e)...);
                   ^
/home/wolfv/Programs/xtensor/tutorial/../include/xtensor/xoperation.hpp:299:26: note: in
      instantiation of function template specialization
      'xt::detail::make_xfunction<detail::identity,
      xt::xarray_adaptor<xt::xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >, xt::layout_type::dynamic, std::vector<unsigned long,
      std::allocator<unsigned long> >, xt::xtensor_expression_tag> >' requested here
        auto f = detail::make_xfunction<detail::identity>(std::forward<E>(e));
                         ^
/home/wolfv/Programs/xtensor/tutorial/tutorial.cpp:666:9: note: in instantiation of function
      template specialization 'xt::operator+<xt::xarray_adaptor<xt::xbuffer_adaptor<double
      *, xt::acquire_ownership, std::allocator<double> >, xt::layout_type::dynamic,
      std::vector<unsigned long, std::allocator<unsigned long> >,
      xt::xtensor_expression_tag> >' requested here
        return +load_npy<double>("mnist.npy");
               ^
/home/wolfv/Programs/xtensor/tutorial/tutorial.cpp:681:16: note: in instantiation of
      function template specialization 'load_file<char>' requested here
    auto xxx = load_file<char>();
               ^
/home/wolfv/Programs/xtensor/tutorial/../include/xtensor/xarray.hpp:196:9: note: explicitly
      defaulted function was implicitly deleted here
        xarray_adaptor(const xarray_adaptor&) = default;
        ^
/home/wolfv/Programs/xtensor/tutorial/../include/xtensor/xarray.hpp:208:32: note: copy
      constructor of 'xarray_adaptor<xt::xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >, xt::layout_type::dynamic, std::vector<unsigned long,
      std::allocator<unsigned long> >, xt::xtensor_expression_tag>' is implicitly deleted
      because field 'm_storage' has a deleted copy constructor
        container_closure_type m_storage;
                               ^
/home/wolfv/Programs/xtensor/tutorial/../include/xtensor/xbuffer_adaptor.hpp:179:9: note: 
      explicitly defaulted function was implicitly deleted here
        xbuffer_adaptor(const self_type&) = default;
        ^
/home/wolfv/Programs/xtensor/tutorial/../include/xtensor/xbuffer_adaptor.hpp:150:29: note: 
      copy constructor of 'xbuffer_adaptor<double *, xt::acquire_ownership,
      std::allocator<double> >' is implicitly deleted because base class
      'detail::buffer_storage_t<double *, allocator<double>, acquire_ownership>' (aka
      'xt::detail::xbuffer_owner_storage<double *, std::allocator<double> >') has a deleted
      copy constructor
    class xbuffer_adaptor : private detail::buffer_storage_t<CP, A, O>
                            ^
/home/wolfv/Programs/xtensor/tutorial/../include/xtensor/xbuffer_adaptor.hpp:109:13: note: 
      'xbuffer_owner_storage' has been explicitly marked deleted here
            xbuffer_owner_storage(const self_type&) = delete;

[wolfv]

The problem seems to be that forwarding const types doesn't correctly move them.

[wolfv]

fixed in #1094 and with xtl 0.4.15