[INFRA] compare output of snippets

[marehr]

Description

We want to compare the output of snippets.

• [ ] Add basic functionality that compares stdout and stderr

[marehr]

I tinkered a bit and got this to work:

• execute snippet (e.g. <build>my/awesome_snippet)
• redirect output to files
  • stdout: e.g. <build>my/awesome_snippet.out
  • stderr: e.g. <build>my/awesome_snippet.err
• compare output with "gold" standard
  • stdout: e.g. <source>/test/snippet/my/awesome.out
  • stderr: e.g. <source>/test/snippet/my/awesome.err
• warn if "gold" standard and output mismatches
• warn if "gold" standard exists, but output is missing
• warn if output exists, but "gold" standard is missing

From f29e40ea588ee9b40dc1e13683322b406e989608 Mon Sep 17 00:00:00 2001
From: marehr <marehr-github@marehr.dialup.fu-berlin.de>
Date: Fri, 14 May 2021 03:42:48 +0200
Subject: [PATCH] [INFRA] compare snippets with actual output

---
 test/snippet/CMakeLists.txt               |  4 ++
 test/snippet/compare_snippet_output.cmake | 68 +++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 test/snippet/compare_snippet_output.cmake

diff --git a/test/snippet/CMakeLists.txt b/test/snippet/CMakeLists.txt
index e2340bba5..00769588c 100644
--- a/test/snippet/CMakeLists.txt
+++ b/test/snippet/CMakeLists.txt
@@ -25,6 +25,10 @@ macro (seqan3_snippet test_name_prefix snippet snippet_base_path)
     )

     add_test (NAME "${test_name_prefix}/${snippet_test_name}_snippet" COMMAND ${target})
+    add_test (NAME "${test_name_prefix}/${snippet_test_name}_snippet_cmp_output"
+              COMMAND cmake -DTARGET_FILE=$<TARGET_FILE:${target}>
+                            -DSOURCE_FILE=${CMAKE_SOURCE_DIR}/${snippet}
+                            -P "${CMAKE_SOURCE_DIR}/compare_snippet_output.cmake")

     unset (snippet_target_name)
     unset (snippet_test_name)
diff --git a/test/snippet/compare_snippet_output.cmake b/test/snippet/compare_snippet_output.cmake
new file mode 100644
index 000000000..3a555d8d4
--- /dev/null
+++ b/test/snippet/compare_snippet_output.cmake
@@ -0,0 +1,68 @@
+cmake_minimum_required (VERSION 3.7)
+
+message (STATUS "TARGET_FILE: ${TARGET_FILE}")
+message (STATUS "SOURCE_FILE: ${SOURCE_FILE}")
+
+get_filename_component (TARGET_FILE_DIR "${TARGET_FILE}" DIRECTORY)
+get_filename_component (SOURCE_FILE_NAME "${SOURCE_FILE}" NAME_WE)
+get_filename_component (SOURCE_FILE_DIR "${SOURCE_FILE}" DIRECTORY)
+
+set (ACTUAL_OUTPUT_FILE "${TARGET_FILE}.out")
+set (EXPECTED_OUTPUT_FILE "${SOURCE_FILE_DIR}/${SOURCE_FILE_NAME}.out")
+set (ACTUAL_ERROR_FILE "${TARGET_FILE}.err")
+set (EXPECTED_ERROR_FILE "${SOURCE_FILE_DIR}/${SOURCE_FILE_NAME}.err")
+
+message (STATUS "ACTUAL_OUTPUT_FILE: ${ACTUAL_OUTPUT_FILE}")
+message (STATUS "EXPECTED_OUTPUT_FILE: ${EXPECTED_OUTPUT_FILE}")
+
+message (STATUS "ACTUAL_ERROR_FILE: ${ACTUAL_ERROR_FILE}")
+message (STATUS "EXPECTED_ERROR_FILE: ${EXPECTED_ERROR_FILE}")
+
+# execute snippet
+execute_process(COMMAND "${TARGET_FILE}"
+                WORKING_DIRECTORY "${TARGET_FILE_DIR}"
+                OUTPUT_FILE "${ACTUAL_OUTPUT_FILE}"
+                ERROR_FILE "${ACTUAL_ERROR_FILE}"
+                RESULT_VARIABLE error_result)
+
+if (error_result) # != 0 return code
+    message (SEND_ERROR "error: executing snippet exited with '${error_result}'")
+endif ()
+
+function (compare_files actual_file expected_file)
+    message (STATUS "actual_file: ${actual_file}")
+    message (STATUS "expected_file: ${expected_file}")
+
+    file (READ "${actual_file}" actual_output)
+    # if (actual_output)
+    #     message ("configure_file(\"${actual_file}\" \"${expected_file}\" COPYONLY)")
+    #     configure_file("${actual_file}" "${expected_file}" COPYONLY)
+    # endif ()
+
+    if (actual_output AND EXISTS "${expected_file}")
+        execute_process (COMMAND ${CMAKE_COMMAND} -E compare_files --ignore-eol "${actual_file}" "${expected_file}"
+                         RESULT_VARIABLE error_result)
+
+        if (NOT error_result) # == 0 return code => files are identical
+            # if successful move one
+            return ()
+        endif ()
+
+        message (SEND_ERROR "error: `${actual_file}` and `${expected_file}` differ (exited with '${error_result}')")
+
+        find_package (Git)
+        if (Git_FOUND)
+            execute_process(COMMAND "${GIT_EXECUTABLE}" diff --no-index "${expected_file}" "${actual_file}")
+        endif ()
+    elseif (EXISTS "${expected_file}")
+        message (SEND_ERROR "error: `${expected_file}` exists, but `${actual_file}` has no output.")
+    elseif (actual_output)
+        message (SEND_ERROR "error: `${actual_file}` has output, but `${expected_file}` does not exist.")
+    else ()
+        message (STATUS "Output matches ${expected_file}")
+    endif ()
+endfunction ()
+
+# compare output to file
+compare_files("${ACTUAL_OUTPUT_FILE}" "${EXPECTED_OUTPUT_FILE}")
+compare_files("${ACTUAL_ERROR_FILE}" "${EXPECTED_ERROR_FILE}")
-- 
2.31.1

This works pretty good, except:

1. some snippet that don't have a main always execute gtest main:
   • [x] e.g. test/snippet/alphabet/alphabet_base.out

     Running main() from /seqan3-build/snippet-gcc-11-std20-distcc/_deps/gtest_fetch_content-src/googletest/src/gtest_main.cc
     [==========] Running 0 tests from 0 test suites.
     [==========] 0 tests from 0 test suites ran. (0 ms total)
     [  PASSED  ] 0 tests

2. some snippet output in nondeterministic:
   • [x] make it possible to disable output check