From dcafc605f3232f5109e3f86ac8d8281e6ed611d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Tue, 17 May 2022 21:49:17 +0200 Subject: [PATCH] Only reseed the internal RNG when a test is first entered This fixes multiple issues with random generators, with the most important one being that multiple nested generators could return values from the same sequence, due to internal implementation details of `GENERATE`, and how they interact with test case paths. The cost of doing this is that given this simple `TEST_CASE`, ```cpp TEST_CASE("foo") { auto i = GENERATE(take(10, random(0, 100)); SECTION("A") { auto j = GENERATE(take(10, random(0, 100)); } SECTION("B") { auto k = GENERATE(take(10, random(0, 100)); } } ``` `k` will have different values between running the test as a whole, e.g. with `./tests "foo"`, and running only the "B" section with `./tests "foo" -c "B"`. I consider this an acceptable cost, because the only alternative would be very messy to implement, and add a lot of brittle and complex code for relatively little benefit. If this calculation changes, we will need to instead walk the current tracker tree whenever a random generator is being constructed, check for random generators on the path to root, and take a seed from them. --- src/catch2/internal/catch_run_context.cpp | 35 +++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp index 179eaefb..e31f8a3f 100644 --- a/src/catch2/internal/catch_run_context.cpp +++ b/src/catch2/internal/catch_run_context.cpp @@ -193,6 +193,39 @@ namespace Catch { assert(rootTracker.isSectionTracker()); static_cast(rootTracker).addInitialFilters(m_config->getSectionsToRun()); + // We intentionally only seed the internal RNG once per test case, + // before it is first invoked. The reason for that is a complex + // interplay of generator/section implementation details and the + // Random*Generator types. + // + // The issue boils down to us needing to seed the Random*Generators + // with different seed each, so that they return different sequences + // of random numbers. We do this by giving them a number from the + // shared RNG instance as their seed. + // + // However, this runs into an issue if the reseeding happens each + // time the test case is entered (as opposed to first time only), + // because multiple generators could get the same seed, e.g. in + // ```cpp + // TEST_CASE() { + // auto i = GENERATE(take(10, random(0, 100)); + // SECTION("A") { + // auto j = GENERATE(take(10, random(0, 100)); + // } + // SECTION("B") { + // auto k = GENERATE(take(10, random(0, 100)); + // } + // } + // ``` + // `i` and `j` would properly return values from different sequences, + // but `i` and `k` would return the same sequence, because their seed + // would be the same. + // (The reason their seeds would be the same is that the generator + // for k would be initialized when the test case is entered the second + // time, after the shared RNG instance was reset to the same value + // it had when the generator for i was initialized.) + seedRng( *m_config ); + uint64_t testRuns = 0; do { m_trackerContext.startCycle(); @@ -422,8 +455,6 @@ namespace Catch { m_shouldReportUnexpected = true; m_lastAssertionInfo = { "TEST_CASE"_sr, testCaseInfo.lineInfo, StringRef(), ResultDisposition::Normal }; - seedRng(*m_config); - Timer timer; CATCH_TRY { if (m_reporter->getPreferences().shouldRedirectStdOut) {