Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NBS-4827 Shadow disk actor #598

Merged
merged 4 commits into from
Mar 15, 2024
Merged

NBS-4827 Shadow disk actor #598

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d0d461c
NBS-4826 Fill shadow disk
drbasic Feb 12, 2024
a215722
NBS-4827 fix review issues #1
drbasic Mar 5, 2024
45142c4
NBS-4827 fix review issues #2
drbasic Mar 11, 2024
3094168
NBS-4827 Fix review issues and tests flakiness
drbasic Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions cloud/blockstore/config/storage.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -926,4 +926,26 @@ message TStorageServiceConfig
// caches. Disk agent config has higher priority than storage config.
optional string CachedDiskAgentConfigPath = 350;
optional string CachedDiskAgentSessionsPath = 351;

// Max bandwidth used for shadow disk fill in MiB/s (actual bandwidth is x2
// due to the need to read and write).
drbasic marked this conversation as resolved.
Show resolved Hide resolved
optional uint32 MaxShadowDiskFillBandwidth = 352;
// Minimum delay between shadow disk acquire attempts when writes to the
// source disk are blocked (in ms).
optional uint32 MinAcquireShadowDiskRetryDelayWhenBlocked = 353;
// Maximum delay between shadow disk acquire attempts when writes to the
// source disk are blocked (in ms).
optional uint32 MaxAcquireShadowDiskRetryDelayWhenBlocked = 354;
// Minimum delay between shadow disk acquire attempts when writes to the
// source disk are not blocked (in ms).
optional uint32 MinAcquireShadowDiskRetryDelayWhenNonBlocked = 355;
// Maximum delay between shadow disk acquire attempts when writes to the
// source disk are not blocked (in ms).
optional uint32 MaxAcquireShadowDiskRetryDelayWhenNonBlocked = 356;
// Timeout for attempts to acquire the shadow disk when writes to the source
// disk are blocked (in ms).
optional uint32 MaxAcquireShadowDiskTotalTimeoutWhenBlocked = 357;
// Timeout for attempts to acquire the shadow disk when writes to the source
// disk are not blocked (in ms).
optional uint32 MaxAcquireShadowDiskTotalTimeoutWhenNonBlocked = 358;
}
9 changes: 9 additions & 0 deletions cloud/blockstore/libs/storage/core/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,15 @@ TDuration MSeconds(ui32 value)
xxx(CachedAcquireRequestLifetime, TDuration, Seconds(40) )\
\
xxx(UnconfirmedBlobCountHardLimit, ui32, 1000 )\
\
xxx(MaxShadowDiskFillBandwidth, ui32, 500 )\
xxx(MinAcquireShadowDiskRetryDelayWhenBlocked, TDuration, MSeconds(250) )\
xxx(MaxAcquireShadowDiskRetryDelayWhenBlocked, TDuration, Seconds(1) )\
xxx(MinAcquireShadowDiskRetryDelayWhenNonBlocked, TDuration, Seconds(1) )\
xxx(MaxAcquireShadowDiskRetryDelayWhenNonBlocked, TDuration, Seconds(10) )\
xxx(MaxAcquireShadowDiskTotalTimeoutWhenBlocked, TDuration, Seconds(5) )\
xxx(MaxAcquireShadowDiskTotalTimeoutWhenNonBlocked, TDuration, Seconds(600) )\

// BLOCKSTORE_STORAGE_CONFIG_RW

#define BLOCKSTORE_STORAGE_CONFIG(xxx) \
Expand Down
8 changes: 8 additions & 0 deletions cloud/blockstore/libs/storage/core/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -553,6 +553,14 @@ class TStorageConfig

TString GetCachedDiskAgentConfigPath() const;
TString GetCachedDiskAgentSessionsPath() const;

ui32 GetMaxShadowDiskFillBandwidth() const;
TDuration GetMinAcquireShadowDiskRetryDelayWhenBlocked() const;
TDuration GetMaxAcquireShadowDiskRetryDelayWhenBlocked() const;
TDuration GetMinAcquireShadowDiskRetryDelayWhenNonBlocked() const;
TDuration GetMaxAcquireShadowDiskRetryDelayWhenNonBlocked() const;
TDuration GetMaxAcquireShadowDiskTotalTimeoutWhenBlocked() const;
TDuration GetMaxAcquireShadowDiskTotalTimeoutWhenNonBlocked() const;
};

ui64 GetAllocationUnit(
Expand Down
12 changes: 6 additions & 6 deletions cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,20 @@ namespace NCloud::NBlockStore::NStorage {
///////////////////////////////////////////////////////////////////////////////

TMigrationTimeoutCalculator::TMigrationTimeoutCalculator(
TStorageConfigPtr config,
ui32 maxMigrationBandwidthMiBs,
ui32 expectedDiskAgentSize,
TNonreplicatedPartitionConfigPtr partitionConfig)
: Config(std::move(config))
: MaxMigrationBandwidthMiBs(maxMigrationBandwidthMiBs)
, ExpectedDiskAgentSize(expectedDiskAgentSize)
, PartitionConfig(std::move(partitionConfig))
{}

TDuration TMigrationTimeoutCalculator::CalculateTimeout(
TBlockRange64 nextProcessingRange) const
{
const ui32 maxMigrationBandwidthMiBs = Config->GetMaxMigrationBandwidth();
const ui32 expectedDiskAgentSize = Config->GetExpectedDiskAgentSize();

// migration range is 4_MB
const auto migrationFactorPerAgent = maxMigrationBandwidthMiBs / 4;
const auto migrationFactorPerAgent = MaxMigrationBandwidthMiBs / 4;

if (PartitionConfig->GetUseSimpleMigrationBandwidthLimiter()) {
return TDuration::Seconds(1) / migrationFactorPerAgent;
Expand All @@ -40,7 +40,7 @@ TDuration TMigrationTimeoutCalculator::CalculateTimeout(
}

const auto factor =
Max(migrationFactorPerAgent * agentDeviceCount / expectedDiskAgentSize,
Max(migrationFactorPerAgent * agentDeviceCount / ExpectedDiskAgentSize,
1U);

return TDuration::Seconds(1) / factor;
Expand Down
10 changes: 6 additions & 4 deletions cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,18 @@ namespace NCloud::NBlockStore::NStorage {
class TMigrationTimeoutCalculator
{
private:
const TStorageConfigPtr Config;
const ui32 MaxMigrationBandwidthMiBs = 0;
const ui32 ExpectedDiskAgentSize = 0;
TNonreplicatedPartitionConfigPtr PartitionConfig;

public:
TMigrationTimeoutCalculator(
TStorageConfigPtr config,
ui32 maxMigrationBandwidthMiBs,
ui32 expectedDiskAgentSize,
TNonreplicatedPartitionConfigPtr partitionConfig);

[[nodiscard]] TDuration CalculateTimeout(
TBlockRange64 nextProcessingRange) const;
[[nodiscard]] TDuration
CalculateTimeout(TBlockRange64 nextProcessingRange) const;
};

} // namespace NCloud::NBlockStore::NStorage
15 changes: 4 additions & 11 deletions cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,6 @@ TDevices MakeDevices()
return result;
}

TStorageConfigPtr MakeStorageConfig(ui32 expectedDiskAgentSize)
{
NProto::TStorageServiceConfig storageConfig;
storageConfig.SetMaxMigrationBandwidth(16);
storageConfig.SetExpectedDiskAgentSize(expectedDiskAgentSize);

return std::make_shared<TStorageConfig>(std::move(storageConfig), nullptr);
}

TNonreplicatedPartitionConfigPtr MakePartitionConfig(
TDevices devices,
bool useSimpleMigrationBandwidthLimiter)
Expand Down Expand Up @@ -80,7 +71,8 @@ Y_UNIT_TEST_SUITE(TMigrationCalculatorTest)
Y_UNIT_TEST(ShouldCalculateMigrationTimeout)
{
TMigrationTimeoutCalculator timeoutCalculator(
MakeStorageConfig(4),
16,
4,
MakePartitionConfig(MakeDevices(), false));

// Devices #1, #2, #4 belong to Agent#1, device #3 belong to Agent#2.
Expand Down Expand Up @@ -112,7 +104,8 @@ Y_UNIT_TEST_SUITE(TMigrationCalculatorTest)
Y_UNIT_TEST(ShouldCalculateMigrationTimeoutWithSimpleLimiter)
{
TMigrationTimeoutCalculator timeoutCalculator(
MakeStorageConfig(100500),
16,
100500,
MakePartitionConfig(MakeDevices(), true));

// When UseSimpleMigrationBandwidthLimiter enabled we expect the same
Expand Down
5 changes: 4 additions & 1 deletion cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,10 @@ TNonreplicatedPartitionMigrationActor::TNonreplicatedPartitionMigrationActor(
, SrcConfig(std::move(srcConfig))
, Migrations(std::move(migrations))
, RdmaClient(std::move(rdmaClient))
, TimeoutCalculator(Config, SrcConfig)
, TimeoutCalculator(
Config->GetMaxMigrationBandwidth(),
Config->GetExpectedDiskAgentSize(),
SrcConfig)
{}

void TNonreplicatedPartitionMigrationActor::OnBootstrap(
Expand Down
Loading