Make fake io error tests in PDisk more robust (#10091)

Author: va-kuznecov

ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp

@@ -346,7 +346,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
 
     Y_UNIT_TEST(TestFakeErrorPDiskManyLogWrite) {
         TActorTestContext testCtx({ false });
-        testCtx.TestCtx.SectorMap->ImitateIoErrorProbability = 1e-4;
+        testCtx.TestCtx.SectorMap->IoErrorEveryNthRequests = 1000;
 
         const TVDiskID vDiskID(0, 1, 0, 0, 0);
         const auto evInitRes = testCtx.TestResponse<NPDisk::TEvYardInitResult>(
@@ -355,7 +355,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
 
         ui32 errors = 0;
         ui32 lsn = 2;
-        for (ui32 i = 0; i < 100'000; ++i) {
+        for (ui32 i = 0; i < 10'000; ++i) {
             testCtx.Send(new NPDisk::TEvLog(evInitRes->PDiskParams->Owner, evInitRes->PDiskParams->OwnerRound, 0,
                         TRcBuf(TString("abc")), TLsnSeg(lsn, lsn), nullptr));
             ++lsn;
@@ -384,7 +384,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
         vdisk.Init();
 
         // Make sure there will be read error.
-        testCtx.TestCtx.SectorMap->ImitateReadIoErrorProbability = 1;
+        testCtx.TestCtx.SectorMap->ReadIoErrorEveryNthRequests = 1;
 
         auto res = vdisk.ReadLog(true);
 
@@ -404,7 +404,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
         vdisk.InitFull();
 
         // Make sure there will be syslog read error.
-        testCtx.TestCtx.SectorMap->ImitateReadIoErrorProbability = 1;
+        testCtx.TestCtx.SectorMap->ReadIoErrorEveryNthRequests = 1;
 
         testCtx.TestResponse<NPDisk::TEvYardControlResult>(
                 new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStop, nullptr),
@@ -417,7 +417,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
 
     Y_UNIT_TEST(TestFakeErrorPDiskManyChunkRead) {
         TActorTestContext testCtx({ false });
-        testCtx.TestCtx.SectorMap->ImitateReadIoErrorProbability = 1e-4;
+        testCtx.TestCtx.SectorMap->ReadIoErrorEveryNthRequests = 100;
 
         TVDiskMock vdisk(&testCtx);
         vdisk.InitFull();
@@ -434,7 +434,8 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
                 NKikimrProto::OK);
 
         bool printed = false;
-        for (ui32 i = 0; i < 100'000; ++i) {
+        ui32 errors = 0;
+        for (ui32 i = 0; i < 10'000; ++i) {
             testCtx.Send(new NPDisk::TEvChunkRead(vdisk.PDiskParams->Owner, vdisk.PDiskParams->OwnerRound,
                     reservedChunk, 0, 1024, 0, nullptr));
 
@@ -445,30 +446,32 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
                     printed = true;
                     Ctest << res->ToString() << Endl;
                 }
+                ++errors;
             }
         }
+        UNIT_ASSERT(errors > 0);
         // Check that PDisk is in working state now
         vdisk.InitFull();
     }
 
     Y_UNIT_TEST(TestFakeErrorPDiskManyChunkWrite) {
         TActorTestContext testCtx({ false });
-        testCtx.TestCtx.SectorMap->ImitateIoErrorProbability = 1e-4;
+        testCtx.TestCtx.SectorMap->IoErrorEveryNthRequests = 1000;
 
         const TVDiskID vDiskID(0, 1, 0, 0, 0);
         const auto evInitRes = testCtx.TestResponse<NPDisk::TEvYardInitResult>(
                 new NPDisk::TEvYardInit(2, vDiskID, testCtx.TestCtx.PDiskGuid),
                 NKikimrProto::OK);
 
-        ui32 errors = 0;
         const auto evReserveRes = testCtx.TestResponse<NPDisk::TEvChunkReserveResult>(
                 new NPDisk::TEvChunkReserve(evInitRes->PDiskParams->Owner, evInitRes->PDiskParams->OwnerRound, 1),
                 NKikimrProto::OK);
         UNIT_ASSERT(evReserveRes->ChunkIds.size() == 1);
         const ui32 reservedChunk = evReserveRes->ChunkIds.front();
 
+        ui32 errors = 0;
         bool printed = false;
-        for (ui32 i = 0; i < 100'000; ++i) {
+        for (ui32 i = 0; i < 10'000; ++i) {
             TString data = PrepareData(1024);
             testCtx.Send(new NPDisk::TEvChunkWrite(evInitRes->PDiskParams->Owner, evInitRes->PDiskParams->OwnerRound,
                     reservedChunk, 0, new NPDisk::TEvChunkWrite::TStrokaBackedUpParts(data), nullptr, false, 0));

ydb/library/pdisk_io/aio_map.cpp

@@ -218,6 +218,8 @@ class TAsyncIoContextMap : public IAsyncIoContext {
 
     TSpinLock SpinLock;
     IAsyncIoOperation* LastOngoingAsyncIoOperation = nullptr;
+    std::atomic<ui64> GlobalCounter = 0;
+
 public:
 
     TAsyncIoContextMap(const TString &path, ui32 pDiskId, TIntrusivePtr<TSectorMap> sectorMap)
@@ -248,6 +250,25 @@ class TAsyncIoContextMap : public IAsyncIoContext {
         return EIoResult::Ok;
     }
 
+    EIoResult GenerateResultForOperaion(IAsyncIoOperation::EType type) {
+        auto result = EIoResult::Ok;
+
+        auto globalIdx = GlobalCounter++;
+        if (auto everyNth = SectorMap->IoErrorEveryNthRequests.load()) {
+            if (globalIdx % everyNth == everyNth - 1) {
+                result = EIoResult::FakeError;
+            }
+        }
+        if (type == IAsyncIoOperation::EType::PRead) {
+            if (auto everyNth = SectorMap->ReadIoErrorEveryNthRequests.load()) {
+                if (globalIdx % everyNth == everyNth - 1) {
+                    result = EIoResult::FakeError;
+                }
+            }
+        }
+        return result;
+    }
+
     i64 GetEvents(ui64 minEvents, ui64 maxEvents, TAsyncIoOperationResult *events, TDuration timeout) override {
         ui64 outputIdx = 0;
         TInstant startTime = TInstant::Now();
@@ -258,14 +279,8 @@ class TAsyncIoContextMap : public IAsyncIoContext {
                 for (TAtomicBase idx = 0; idx < size; ++idx) {
                     TAsyncIoOperationMap *op = static_cast<TAsyncIoOperationMap*>(CompleteQueue.Pop());
                     events[outputIdx].Operation = op;
-                    events[outputIdx].Result = (RandomNumber<double>() <
-                            SectorMap->ImitateIoErrorProbability.load())
-                        ? EIoResult::FakeError
-                        : EIoResult::Ok;
-                    if (op->GetType() == IAsyncIoOperation::EType::PRead &&
-                            RandomNumber<double>() < SectorMap->ImitateReadIoErrorProbability.load()) {
-                        events[outputIdx].Result = EIoResult::FakeError;
-                    }
+
+                    events[outputIdx].Result = GenerateResultForOperaion(op->GetType());
                     events[outputIdx].Operation->ExecCallback(&events[outputIdx]);
                     ++outputIdx;
                     if (outputIdx == maxEvents) {

ydb/library/pdisk_io/sector_map.h

@@ -191,16 +191,16 @@ class TSectorMap : public TThrRefBase {
     TTicketLock MapLock;
     std::atomic<bool> IsLocked;
     std::optional<std::pair<TDuration, TDuration>> ImitateRandomWait;
-    std::atomic<double> ImitateIoErrorProbability;
-    std::atomic<double> ImitateReadIoErrorProbability;
+    std::atomic<ui64> IoErrorEveryNthRequests;
+    std::atomic<ui64> ReadIoErrorEveryNthRequests;
 
     std::atomic<ui64> AllocatedBytes;
 
     TSectorMap(ui64 deviceSize = 0, NSectorMap::EDiskMode diskMode = NSectorMap::DM_NONE)
       : DeviceSize(deviceSize)
       , IsLocked(false)
-      , ImitateIoErrorProbability(0.0)
-      , ImitateReadIoErrorProbability(0.0)
+      , IoErrorEveryNthRequests(0)
+      , ReadIoErrorEveryNthRequests(0)
       , AllocatedBytes(0)
       , DiskMode(diskMode)
     {
@@ -327,8 +327,8 @@ class TSectorMap : public TThrRefBase {
             str << "ImitateRandomWait# [" << ImitateRandomWait->first << ", "
                 << ImitateRandomWait->first + ImitateRandomWait->second << ")" << "\n";
         }
-        str << "ImitateReadIoErrorProbability# " << ImitateReadIoErrorProbability.load() << "\n";
-        str << "ImitateIoErrorProbability# " << ImitateIoErrorProbability.load() << "\n";
+        str << "ReadIoErrorEveryNthRequests# " << ReadIoErrorEveryNthRequests.load() << "\n";
+        str << "IoErrorEveryNthRequests# " << IoErrorEveryNthRequests.load() << "\n";
         str << "AllocatedBytes (approx.)# " << HumanReadableSize(AllocatedBytes.load(), SF_QUANTITY)  << "\n";
         str << "DataBytes# " << HumanReadableSize(DataBytes(), SF_QUANTITY)  << "\n";
         str << "DiskMode# " << DiskModeToString(DiskMode) << "\n";