From 9c17e765073bf7b337515e5d748c6c71502698d5 Mon Sep 17 00:00:00 2001 From: Gunes Bayir Date: Tue, 6 Jan 2026 14:09:47 +0000 Subject: [PATCH] test: Add warning level logging to the test framework We add another level of logging between ERRORS and INFO/DEBUG because we print skipped tests with INFO level logging and they don't appear in CI. Reporting if they're skipped or not is useful when trying to understand if the CI is running the correct set of tests or not on a given hardware. INFO level debugging is too heavy for CI output because it prints every tensor value mismatch if there are failing tests and this could be miliions, thus it's not a good option. After this commit, we'll also change test skip reports to warning level in various places and also change the CI to log at warning level instead of error level. Change-Id: I4de2859b6987778d9b228fd96c2336c4ce056079 Signed-off-by: Gunes Bayir --- tests/framework/Asserts.h | 17 ++++++++- tests/framework/Exceptions.cpp | 14 +++++-- tests/framework/Exceptions.h | 4 +- tests/framework/Framework.cpp | 38 ++++++++++++++++++- tests/framework/Framework.h | 30 ++++++++++++++- .../framework/command_line/CommonOptions.cpp | 6 +-- tests/framework/printers/JSONPrinter.cpp | 12 +++++- tests/framework/printers/JSONPrinter.h | 4 +- tests/framework/printers/PrettyPrinter.cpp | 7 +++- tests/framework/printers/PrettyPrinter.h | 3 +- tests/framework/printers/Printer.h | 8 +++- 11 files changed, 128 insertions(+), 15 deletions(-) diff --git a/tests/framework/Asserts.h b/tests/framework/Asserts.h index fb578592a3..0b0d787b9f 100644 --- a/tests/framework/Asserts.h +++ b/tests/framework/Asserts.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022, 2025 Arm Limited. + * Copyright (c) 2017-2022, 2025-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -68,6 +68,14 @@ inline void ARM_COMPUTE_PRINT_INFO() arm_compute::test::framework::Framework::get().clear_test_info(); } +inline void ARM_COMPUTE_PRINT_WARNING() +{ + std::stringstream msg; + arm_compute::test::framework::Framework::get().print_test_warning(msg); + arm_compute::test::framework::Framework::get().log_warning(msg.str()); + arm_compute::test::framework::Framework::get().clear_test_warning(); +} + #define ARM_COMPUTE_TEST_INFO(INFO) \ { \ std::stringstream info; \ @@ -75,6 +83,13 @@ inline void ARM_COMPUTE_PRINT_INFO() arm_compute::test::framework::Framework::get().add_test_info(info.str()); \ } +#define ARM_COMPUTE_TEST_WARNING(WARNING) \ + { \ + std::stringstream warning; \ + warning << WARNING; \ + arm_compute::test::framework::Framework::get().add_test_warning(warning.str()); \ + } + namespace detail { #define ARM_COMPUTE_TEST_COMP_FACTORY(SEVERITY, SEVERITY_NAME, COMP, COMP_NAME, ERROR_CALL) \ diff --git a/tests/framework/Exceptions.cpp b/tests/framework/Exceptions.cpp index c0355ed448..83718914ed 100644 --- a/tests/framework/Exceptions.cpp +++ b/tests/framework/Exceptions.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2025 Arm Limited. + * Copyright (c) 2017, 2025-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -36,8 +36,13 @@ namespace framework LogLevel log_level_from_name(const std::string &name) { static const std::map levels = { - {"none", LogLevel::NONE}, {"config", LogLevel::CONFIG}, {"tests", LogLevel::TESTS}, - {"errors", LogLevel::ERRORS}, {"debug", LogLevel::DEBUG}, {"measurements", LogLevel::MEASUREMENTS}, + {"none", LogLevel::NONE}, + {"config", LogLevel::CONFIG}, + {"tests", LogLevel::TESTS}, + {"errors", LogLevel::ERRORS}, + {"warnings", LogLevel::WARNINGS}, + {"debug", LogLevel::DEBUG}, + {"measurements", LogLevel::MEASUREMENTS}, {"all", LogLevel::ALL}, }; @@ -75,6 +80,9 @@ ::std::ostream &operator<<(::std::ostream &stream, LogLevel level) case LogLevel::ERRORS: stream << "ERRORS"; break; + case LogLevel::WARNINGS: + stream << "WARNINGS"; + break; case LogLevel::DEBUG: stream << "DEBUG"; break; diff --git a/tests/framework/Exceptions.h b/tests/framework/Exceptions.h index 1d5e302c57..503f4197a4 100644 --- a/tests/framework/Exceptions.h +++ b/tests/framework/Exceptions.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018, 2025 Arm Limited. + * Copyright (c) 2017-2018, 2025-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -42,6 +42,7 @@ namespace framework * NONE == Only for filtering. Not used to tag information. * CONFIG == Configuration info. * TESTS == Information about the tests. + * WARNINGS == Warnings, such as skipped tests. * ERRORS == Violated assertions/expectations. * DEBUG == More violated assertions/expectations. * MEASUREMENTS == Information about measurements. @@ -53,6 +54,7 @@ enum class LogLevel CONFIG, TESTS, ERRORS, + WARNINGS, DEBUG, MEASUREMENTS, ALL, diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp index 9db3e9e03d..0d5d05af2f 100644 --- a/tests/framework/Framework.cpp +++ b/tests/framework/Framework.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021, 2023-2025 Arm Limited. + * Copyright (c) 2017-2021, 2023-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -230,6 +230,34 @@ void Framework::print_test_info(std::ostream &os) const } } +void Framework::add_test_warning(std::string warning) +{ + _test_warning.emplace_back(std::move(warning)); +} + +void Framework::clear_test_warning() +{ + _test_warning.clear(); +} + +bool Framework::has_test_warning() const +{ + return !_test_warning.empty(); +} + +void Framework::print_test_warning(std::ostream &os) const +{ + if (!_test_warning.empty()) + { + os << "CONTEXT:\n"; + + for (const auto &str : _test_warning) + { + os << " " << str << "\n"; + } + } +} + template void Framework::func_on_all_printers(F &&func) { @@ -290,6 +318,14 @@ void Framework::log_info(const std::string &info) } } +void Framework::log_warning(const std::string &warning) +{ + if (_log_level >= LogLevel::WARNINGS) + { + func_on_all_printers([&](Printer *p) { p->print_warning(warning); }); + } +} + int Framework::num_iterations() const { return _num_iterations; diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h index 4bebc4fb55..96ddf8034d 100644 --- a/tests/framework/Framework.h +++ b/tests/framework/Framework.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021, 2023-2025 Arm Limited. + * Copyright (c) 2017-2021, 2023-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -176,6 +176,27 @@ class Framework final */ void print_test_info(std::ostream &os) const; + /** Add warning string for the next expectation/assertion. + * + * @param[in] info Warning string. + */ + void add_test_warning(std::string warning); + + /** Clear the collected test warning. */ + void clear_test_warning(); + + /** Check if any warning has been registered. + * + * @return True if there is test warning. + */ + bool has_test_warning() const; + + /** Print test warning. + * + * @param[out] os Output stream. + */ + void print_test_warning(std::ostream &os) const; + /** Tell the framework that execution of a test starts. * * @param[in] info Test info. @@ -206,6 +227,12 @@ class Framework final */ void log_info(const std::string &info); + /** Print the warnings that has already been logged + * + * @param[in] info Description of the log warning. + */ + void log_warning(const std::string &warning); + /** Number of iterations per test case. * * @return Number of iterations per test case. @@ -406,6 +433,7 @@ class Framework final const TestInfo *_current_test_info{nullptr}; TestResult *_current_test_result{nullptr}; std::vector _test_info{}; + std::vector _test_warning{}; }; template diff --git a/tests/framework/command_line/CommonOptions.cpp b/tests/framework/command_line/CommonOptions.cpp index 847bbe46dd..0bba5da8c4 100644 --- a/tests/framework/command_line/CommonOptions.cpp +++ b/tests/framework/command_line/CommonOptions.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020,2024-2025 Arm Limited. + * Copyright (c) 2018-2020,2024-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -74,8 +74,8 @@ CommonOptions::CommonOptions(CommandLineParser &parser) }; std::set supported_log_levels{ - LogLevel::NONE, LogLevel::CONFIG, LogLevel::TESTS, LogLevel::ERRORS, - LogLevel::DEBUG, LogLevel::MEASUREMENTS, LogLevel::ALL, + LogLevel::NONE, LogLevel::CONFIG, LogLevel::TESTS, LogLevel::ERRORS, + LogLevel::WARNINGS, LogLevel::DEBUG, LogLevel::MEASUREMENTS, LogLevel::ALL, }; instruments = parser.add_option>( diff --git a/tests/framework/printers/JSONPrinter.cpp b/tests/framework/printers/JSONPrinter.cpp index ec0da78428..6fc7aeb45a 100644 --- a/tests/framework/printers/JSONPrinter.cpp +++ b/tests/framework/printers/JSONPrinter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019,2021, 2025 Arm Limited. + * Copyright (c) 2017-2019,2021, 2025-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -132,6 +132,7 @@ void JSONPrinter::print_errors_header() { _errors.clear(); _expected_errors.clear(); + _warnings.clear(); _infos.clear(); } @@ -147,6 +148,10 @@ void JSONPrinter::print_errors_footer() print_strings(_expected_errors.begin(), _expected_errors.end()); *_stream << "]"; + *_stream << R"(, "warnings" : [)"; + print_strings(_warnings.begin(), _warnings.end()); + *_stream << "]"; + *_stream << R"(, "infos" : [)"; print_strings(_infos.begin(), _infos.end()); *_stream << "]"; @@ -164,6 +169,11 @@ void JSONPrinter::print_error(const std::exception &error, bool expected) } } +void JSONPrinter::print_warning(const std::string &warning) +{ + _warnings.push_back(warning); +} + void JSONPrinter::print_info(const std::string &info) { _infos.push_back(info); diff --git a/tests/framework/printers/JSONPrinter.h b/tests/framework/printers/JSONPrinter.h index 2bdcb37ada..1d5c3c72c4 100644 --- a/tests/framework/printers/JSONPrinter.h +++ b/tests/framework/printers/JSONPrinter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017,2021, 2025 Arm Limited. + * Copyright (c) 2017,2021, 2025-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -49,6 +49,7 @@ class JSONPrinter : public Printer void print_errors_header() override; void print_errors_footer() override; void print_error(const std::exception &error, bool expected) override; + void print_warning(const std::string &warning) override; void print_info(const std::string &info) override; void print_profiler_header(const std::string &header_data) override; void print_measurements(const Profiler::MeasurementsMap &measurements) override; @@ -60,6 +61,7 @@ class JSONPrinter : public Printer void print_strings(T &&first, T &&last); std::list _infos{}; + std::list _warnings{}; std::list _errors{}; std::list _expected_errors{}; diff --git a/tests/framework/printers/PrettyPrinter.cpp b/tests/framework/printers/PrettyPrinter.cpp index a42c2bb20d..5fac661103 100644 --- a/tests/framework/printers/PrettyPrinter.cpp +++ b/tests/framework/printers/PrettyPrinter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019,2021, 2025 Arm Limited. + * Copyright (c) 2017-2019,2021, 2025-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -102,6 +102,11 @@ void PrettyPrinter::print_info(const std::string &info) *_stream << begin_color("1") << "INFO: " << info << end_color() << "\n"; } +void PrettyPrinter::print_warning(const std::string &warning) +{ + *_stream << begin_color("1") << "WARNING: " << warning << end_color() << "\n"; +} + void PrettyPrinter::print_error(const std::exception &error, bool expected) { std::string prefix = expected ? "EXPECTED ERROR: " : "ERROR: "; diff --git a/tests/framework/printers/PrettyPrinter.h b/tests/framework/printers/PrettyPrinter.h index f3b044f765..d182bafb0b 100644 --- a/tests/framework/printers/PrettyPrinter.h +++ b/tests/framework/printers/PrettyPrinter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017,2021, 2025 Arm Limited. + * Copyright (c) 2017,2021, 2025-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -54,6 +54,7 @@ class PrettyPrinter : public Printer void print_errors_header() override; void print_errors_footer() override; void print_error(const std::exception &error, bool expected) override; + void print_warning(const std::string &warning) override; void print_info(const std::string &info) override; void print_profiler_header(const std::string &header_data) override; void print_measurements(const Profiler::MeasurementsMap &measurements) override; diff --git a/tests/framework/printers/Printer.h b/tests/framework/printers/Printer.h index a7e08c5cd2..3647d9520f 100644 --- a/tests/framework/printers/Printer.h +++ b/tests/framework/printers/Printer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018,2021, 2025 Arm Limited. + * Copyright (c) 2017-2018,2021, 2025-2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -119,6 +119,12 @@ class Printer */ virtual void print_error(const std::exception &error, bool expected) = 0; + /** Print test log warning. + * + * @param[in] warning Description of the log. + */ + virtual void print_warning(const std::string &warning) = 0; + /** Print test log info. * * @param[in] info Description of the log.