Re: [PATCH 2/3] kunit: tool: unit tests all check parser errors, standardize formatting a bit

From: David Gow
Date: Thu Nov 03 2022 - 02:38:17 EST

Next message: Jing Leng: "[PATCH v5] usb: gadget: uvc: add bulk transfer support"
Previous message: David Gow: "Re: [PATCH 1/3] kunit: tool: make TestCounts a dataclass"
In reply to: Daniel Latypov: "[PATCH 2/3] kunit: tool: unit tests all check parser errors, standardize formatting a bit"
Next in thread: Daniel Latypov: "Re: [PATCH 2/3] kunit: tool: unit tests all check parser errors, standardize formatting a bit"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

On Thu, Nov 3, 2022 at 12:40 AM 'Daniel Latypov' via KUnit Development
<kunit-dev@xxxxxxxxxxxxxxxx> wrote:
>
> Let's verify that the parser isn't reporting any errors for valid
> inputs.
>
> This change also
> * does result.status checking on one line
> * makes sure we consistently do it outside of the `with` block
>
> Signed-off-by: Daniel Latypov <dlatypov@xxxxxxxxxx>
> ---

Looks good, thanks.

Note that this patch does conflict with "kunit: tool: print summary of
failed tests if a few failed out of a lot":
https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git/commit/?h=kunit&id=f19dd011d8de6f0c1d20abea5158aa4f5d9cea44
It's only a context line issue, though.

Reviewed-by: David Gow <davidgow@xxxxxxxxxx>

Cheers,
-- David

> tools/testing/kunit/kunit_tool_test.py | 93 +++++++++++---------------
> 1 file changed, 38 insertions(+), 55 deletions(-)
>
> diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
> index 9fa4babb2506..0063773c0fc4 100755
> --- a/tools/testing/kunit/kunit_tool_test.py
> +++ b/tools/testing/kunit/kunit_tool_test.py
> @@ -133,33 +133,29 @@ class KUnitParserTest(unittest.TestCase):
> all_passed_log = test_data_path('test_is_test_passed-all_passed.log')
> with open(all_passed_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> + self.assertEqual(result.counts.errors, 0)
>
> def test_parse_successful_nested_tests_log(self):
> all_passed_log = test_data_path('test_is_test_passed-all_passed_nested.log')
> with open(all_passed_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> + self.assertEqual(result.counts.errors, 0)
>
> def test_kselftest_nested(self):
> kselftest_log = test_data_path('test_is_test_passed-kselftest.log')
> with open(kselftest_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> + self.assertEqual(result.counts.errors, 0)
>
> def test_parse_failed_test_log(self):
> failed_log = test_data_path('test_is_test_passed-failure.log')
> with open(failed_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.FAILURE,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.FAILURE, result.status)
> + self.assertEqual(result.counts.errors, 0)
>
> def test_no_header(self):
> empty_log = test_data_path('test_is_test_passed-no_tests_run_no_header.log')
> @@ -167,9 +163,8 @@ class KUnitParserTest(unittest.TestCase):
> result = kunit_parser.parse_run_tests(
> kunit_parser.extract_tap_lines(file.readlines()))
> self.assertEqual(0, len(result.subtests))
> - self.assertEqual(
> - kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS, result.status)
> + self.assertEqual(result.counts.errors, 1)
>
> def test_missing_test_plan(self):
> missing_plan_log = test_data_path('test_is_test_passed-'
> @@ -180,9 +175,7 @@ class KUnitParserTest(unittest.TestCase):
> file.readlines()))
> # A missing test plan is not an error.
> self.assertEqual(result.counts, kunit_parser.TestCounts(passed=10, errors=0))
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
>
> def test_no_tests(self):
> header_log = test_data_path('test_is_test_passed-no_tests_run_with_header.log')
> @@ -190,9 +183,8 @@ class KUnitParserTest(unittest.TestCase):
> result = kunit_parser.parse_run_tests(
> kunit_parser.extract_tap_lines(file.readlines()))
> self.assertEqual(0, len(result.subtests))
> - self.assertEqual(
> - kunit_parser.TestStatus.NO_TESTS,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.NO_TESTS, result.status)
> + self.assertEqual(result.counts.errors, 1)
>
> def test_no_tests_no_plan(self):
> no_plan_log = test_data_path('test_is_test_passed-no_tests_no_plan.log')
> @@ -203,7 +195,7 @@ class KUnitParserTest(unittest.TestCase):
> self.assertEqual(
> kunit_parser.TestStatus.NO_TESTS,
> result.subtests[0].subtests[0].status)
> - self.assertEqual(1, result.counts.errors)
> + self.assertEqual(result.counts, kunit_parser.TestCounts(passed=1, errors=1))
>
>
> def test_no_kunit_output(self):
> @@ -215,6 +207,7 @@ class KUnitParserTest(unittest.TestCase):
> print_mock.assert_any_call(StrContains('could not find any KTAP output!'))
> print_mock.stop()
> self.assertEqual(0, len(result.subtests))
> + self.assertEqual(result.counts.errors, 1)
>
> def test_skipped_test(self):
> skipped_log = test_data_path('test_skip_tests.log')
> @@ -222,18 +215,16 @@ class KUnitParserTest(unittest.TestCase):
> result = kunit_parser.parse_run_tests(file.readlines())
>
> # A skipped test does not fail the whole suite.
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> + self.assertEqual(result.counts, kunit_parser.TestCounts(passed=4, skipped=1))
>
> def test_skipped_all_tests(self):
> skipped_log = test_data_path('test_skip_all_tests.log')
> with open(skipped_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
>
> - self.assertEqual(
> - kunit_parser.TestStatus.SKIPPED,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.SKIPPED, result.status)
> + self.assertEqual(result.counts, kunit_parser.TestCounts(skipped=5))
>
> def test_ignores_hyphen(self):
> hyphen_log = test_data_path('test_strip_hyphen.log')
> @@ -241,9 +232,7 @@ class KUnitParserTest(unittest.TestCase):
> result = kunit_parser.parse_run_tests(file.readlines())
>
> # A skipped test does not fail the whole suite.
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> self.assertEqual(
> "sysctl_test",
> result.subtests[0].name)
> @@ -257,55 +246,49 @@ class KUnitParserTest(unittest.TestCase):
> prefix_log = test_data_path('test_config_printk_time.log')
> with open(prefix_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> - self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> + self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(result.counts.errors, 0)
>
> def test_ignores_multiple_prefixes(self):
> prefix_log = test_data_path('test_multiple_prefixes.log')
> with open(prefix_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> - self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> + self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(result.counts.errors, 0)
>
> def test_prefix_mixed_kernel_output(self):
> mixed_prefix_log = test_data_path('test_interrupted_tap_output.log')
> with open(mixed_prefix_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> - self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> + self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(result.counts.errors, 0)
>
> def test_prefix_poundsign(self):
> pound_log = test_data_path('test_pound_sign.log')
> with open(pound_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> - self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> + self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(result.counts.errors, 0)
>
> def test_kernel_panic_end(self):
> panic_log = test_data_path('test_kernel_panic_interrupt.log')
> with open(panic_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.TEST_CRASHED,
> - result.status)
> - self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(kunit_parser.TestStatus.TEST_CRASHED, result.status)
> + self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertGreaterEqual(result.counts.errors, 1)
>
> def test_pound_no_prefix(self):
> pound_log = test_data_path('test_pound_no_prefix.log')
> with open(pound_log) as file:
> result = kunit_parser.parse_run_tests(file.readlines())
> - self.assertEqual(
> - kunit_parser.TestStatus.SUCCESS,
> - result.status)
> - self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status)
> + self.assertEqual('kunit-resource-test', result.subtests[0].name)
> + self.assertEqual(result.counts.errors, 0)
>
> def line_stream_from_strs(strs: Iterable[str]) -> kunit_parser.LineStream:
> return kunit_parser.LineStream(enumerate(strs, start=1))
> --
> 2.38.1.273.g43a17bfeac-goog
>
> --
> You received this message because you are subscribed to the Google Groups "KUnit Development" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to kunit-dev+unsubscribe@xxxxxxxxxxxxxxxx.
> To view this discussion on the web visit https://groups.google.com/d/msgid/kunit-dev/20221102164005.2516646-2-dlatypov%40google.com.

Attachment: smime.p7s
Description: S/MIME Cryptographic Signature

Next message: Jing Leng: "[PATCH v5] usb: gadget: uvc: add bulk transfer support"
Previous message: David Gow: "Re: [PATCH 1/3] kunit: tool: make TestCounts a dataclass"
In reply to: Daniel Latypov: "[PATCH 2/3] kunit: tool: unit tests all check parser errors, standardize formatting a bit"
Next in thread: Daniel Latypov: "Re: [PATCH 2/3] kunit: tool: unit tests all check parser errors, standardize formatting a bit"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]