search

robertoraggi / cplusplus

A compiler front end for the C++ language
https://robertoraggi.github.io/cplusplus/
MIT License
121 stars 12 forks source link

Testing with examples from https://en.cppreference.com/w/ #314

Open mingodad opened 8 months ago

mingodad commented 8 months ago

Using the Lua script shown bellow to extract the code examples from https://en.cppreference.com/w/ I'm getting this results:

#cxx
Total files scanned     :   5629
Total files with code   :   2889
Total files failed code :   172

#clang-17
Total files scanned     :   5629
Total files with code   :   2889
Total files failed code :   266

#g++-13
Total files scanned     :   5629
Total files with code   :   2889
Total files failed code :   197

#circle-200
Total files scanned     :   5629
Total files with code   :   2889
Total files failed code :   474
local base_dir = "/home/mingo/.local/share/Zeal/Zeal/docsets/C++.docset/Contents/Resources/Documents/en.cppreference.com/w/cpp/";

local cppCodeFName = "/tmp/cppCode.cpp";
--local cxx_cmd = "../build/src/frontend/cxx -fsyntax-only -toolchain linux " .. cppCodeFName;
--local cxx_cmd = "/usr/bin/time ../build-release/src/frontend/cxx -fsyntax-only -toolchain linux " .. cppCodeFName;
local cxx_cmd = "/usr/bin/time ./cxx -fsyntax-only -toolchain linux " .. cppCodeFName;
--local cxx_cmd = "clang-17-env /usr/bin/time clang++ -std=c++23 -fsyntax-only " .. cppCodeFName;
--local cxx_cmd = "clang-17-env /usr/bin/time clang++ -fsyntax-only " .. cppCodeFName;
--local cxx_cmd = "gcc-13-env /usr/bin/time g++ -std=c++23 -fsyntax-only " .. cppCodeFName;
--local cxx_cmd = "gcc-13-env /usr/bin/time g++ -fsyntax-only " .. cppCodeFName;
--local cxx_cmd = "/usr/bin/time timeout 3s circle  -std=c++2b -fsyntax-only " .. cppCodeFName;

local code_re = "<div class=\"t%-example%-live%-link\">.-<div class=\"cpp source%-cpp\">(.-)</pre></div>";

local find_cmd = "find " .. base_dir .. " -name '*.html'";
local start_t = os.time();

local html_list = {};
for fname in io.popen(find_cmd, "r"):lines("l") do
    table.insert(html_list, fname);
end
table.sort(html_list);

local count_with_code = 0;
local fname_with_code_failed = {};
for idx, fname in ipairs(html_list) do
    print(fname);

    local html = io.open(fname, "r"):read("a");
    for code in html:gmatch(code_re) do
        code = code:gsub("%b<>", "");
        code = code:gsub("&nbsp;", " ");
        code = code:gsub("&lt;", "<");
        code = code:gsub("&gt;", ">");
        code = code:gsub("&amp;", "&");

        --print("//====Start");
        --print(code);
        --print("//====End");

        local fd = io.open(cppCodeFName, "w");
        fd:write(code);
        fd:close();
        count_with_code = count_with_code + 1;
        print("==CHECK==")
        print(cxx_cmd);
        io.stdout:flush();
        local rc, rc_type, rc_code = os.execute(cxx_cmd);
        if rc_code == 0 then
            print("==rc:OK==")
        else
            table.insert(fname_with_code_failed, idx);
            print("==rc:FAILED==")
            print("//====Start");
            print(code);
            print("//====End");
        end
    end

end
local end_t = os.time();
print("elapsed time: ", os.difftime(end_t, start_t));
print("Total files scanned     :", #html_list);
print("Total files with code   :", count_with_code);
print("Total files failed code :", #fname_with_code_failed);
for idx, fname_idx in ipairs(fname_with_code_failed) do
    print(html_list[fname_idx]);
end

See attached the output for cxx and clang-17: test-cpp-reference-examples-cxx.output.zip test-cpp-reference-examples-clang-17-std23.output.zip

mingodad commented 8 months ago

Also testing https://github.com/abseil/abseil-cpp with the Lua script shown bellow gives this result:

#cxx
Total files scanned     :   172
Total files with code   :   0
Total files failed code :   47

#clang-17
Total files scanned     :   172
Total files with code   :   0
Total files failed code :   3

#g++-13
Total files scanned     :   172
Total files with code   :   0
Total files failed code :   71

#circle-200
Total files scanned     :   172
Total files with code   :   0
Total files failed code :   59
local base_dir = "/home/mingo/dev/c/A_libs/abseil-cpp/absl/";

--local cxx_cmd = "/usr/bin/time ../build/src/frontend/cxx -fsyntax-only -toolchain linux ";
local cxx_cmd = "/usr/bin/time ../build-release/src/frontend/cxx -fsyntax-only -toolchain linux ";
--local cxx_cmd = "clang-17-env /usr/bin/time clang++ -std=c++14 -fsyntax-only ";
--local cxx_cmd = "clang-17-env /usr/bin/time clang++ -fsyntax-only ";
--local cxx_cmd = "gcc-13-env /usr/bin/time g++ -std=c++14 -fsyntax-only ";
--local cxx_cmd = "gcc-13-env /usr/bin/time g++ -fsyntax-only ";
--local cxx_cmd = "/usr/bin/time timeout 3s circle -fsyntax-only ";

local find_cmd = "find " .. base_dir .. " -name '*.cc'";
local start_t = os.time();

local cpp_list = {};
for fname in io.popen(find_cmd, "r"):lines("l") do
    if not (fname:match("_test.cc$") or fname:match("_benchmark.cc$")) then
        table.insert(cpp_list, fname);
    end
end
table.sort(cpp_list);

local count_with_code = 0;
local fname_with_code_failed = {};
for idx, fname in ipairs(cpp_list) do
    print(fname);

    local cmd = cxx_cmd .. fname .. " -I" .. base_dir .. "..";
    print("==CHECK==")
    print(cmd);
    io.stdout:flush();
    local rc = os.execute(cmd);
    if rc then
        print("==rc:OK==")
    else
        table.insert(fname_with_code_failed, idx);
        print("==rc:FAILED==")
    end

end
local end_t = os.time();
print("elapsed time: ", os.difftime(end_t, start_t));
print("Total files scanned     :", #cpp_list);
print("Total files with code   :", count_with_code);
print("Total files failed code :", #fname_with_code_failed);
for idx, fname_idx in ipairs(fname_with_code_failed) do
    print(cpp_list[fname_idx]);
end

See attached cxx and clang output: test-abseil-cpp-cxx.output.zip test-abseil-cpp-clang17.output.zip

robertoraggi commented 8 months ago

This looks very helpful, thanks for sharing. I will have a look at the failures, do you have a repo for this?

mingodad commented 8 months ago

No repo only the Lua script and you can download the offline cpp-reference from here https://en.cppreference.com/w/Cppreference:Archives .

mingodad commented 4 months ago

Now when trying to test I'm getting errors that was not happening before:

/home/mingo/.local/share/Zeal/Zeal/docsets/C++.docset/Contents/Resources/Documents/en.cppreference.com/w/cpp/algorithm/accumulate.html
==CHECK==
/usr/bin/time ./cxx -fsyntax-only -toolchain linux /tmp/cppCode.cpp
/usr/include/c++/11/ext/numeric_traits.h:72:57: expected '('
    = __is_integer_nonstrict<_Value>::__width - __is_signed;
                                                           ^
/usr/include/c++/11/ext/numeric_traits.h:76:2: expected '('
    ? (((((_Value)1 << (__digits - 1)) - 1) << 1) + 1)
    ^
/usr/include/c++/11/ext/numeric_traits.h:78:47: expected '('
      static const _Value __min = __is_signed ? -__max - 1 : (_Value)0;
                                              ^
/usr/include/c++/11/bits/stl_algobase.h:907:40: expected '('
    __gnu_cxx::__enable_if<!__is_scalar<_Tp>::__value, void>::__type
                                       ^
/usr/include/c++/11/bits/stl_algobase.h:907:54: expected a declaration
    __gnu_cxx::__enable_if<!__is_scalar<_Tp>::__value, void>::__type
                                                     ^
/usr/include/c++/11/bits/stl_algobase.h:907:60: expected a declaration
    __gnu_cxx::__enable_if<!__is_scalar<_Tp>::__value, void>::__type
                                                           ^
/usr/include/c++/11/bits/stl_algobase.h:1055:40: expected '('
    __gnu_cxx::__enable_if<!__is_scalar<_Tp>::__value, _OutputIterator>::__type
                                       ^
/usr/include/c++/11/bits/stl_algobase.h:1055:54: expected a declaration
    __gnu_cxx::__enable_if<!__is_scalar<_Tp>::__value, _OutputIterator>::__type
                                                     ^
/usr/include/c++/11/bits/stl_algobase.h:1055:71: expected a declaration
    __gnu_cxx::__enable_if<!__is_scalar<_Tp>::__value, _OutputIterator>::__type
                                                                      ^
/usr/include/c++/11/bits/stl_algobase.h:1372:18: expected '('
     && __is_pointer<_II1>::__value
                    ^
/usr/include/c++/11/bits/stl_algobase.h:1373:18: expected '('
     && __is_pointer<_II2>::__value
                    ^
Command exited with non-zero status 1
0.22user 0.03system 0:00.26elapsed 98%CPU (0avgtext+0avgdata 51712maxresident)k
104inputs+0outputs (0major+13959minor)pagefaults 0swaps
==rc:FAILED==
robertoraggi commented 4 months ago

I see, I think I broke this when I started reworking the support for the built-in type traits. https://github.com/robertoraggi/cplusplus/pull/359 should fix it, I also added some code to test the linux toolchain in the CI workflow. Thanks for the report.