Skip to content

Commit 1cc4449

Browse files
stidsborgstidsborg
committed
Added EnableWatchdogs setting
1 parent 32ca2b7 commit 1cc4449

File tree

9 files changed

+43
-35
lines changed

9 files changed

+43
-35
lines changed

‎Core/Cleipnir.ResilientFunctions.Tests/TestTemplates/RFunctionTests/ControlPanelTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -708,7 +708,7 @@ protected async Task LeaseIsUpdatedForExecutingAction(Task<IFunctionStore> store
708708
store, new Settings(
709709
unhandledExceptionCatcher.Catch,
710710
leaseLength: TimeSpan.FromMilliseconds(250),
711-
watchdogCheckFrequency: TimeSpan.MaxValue
711+
enableWatchdogs: false
712712
)
713713
);
714714
var flag = new SyncedFlag();

‎Core/Cleipnir.ResilientFunctions.Tests/TestTemplates/RFunctionTests/CrashedTests.cs

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ protected async Task NonCompletedFuncIsCompletedByWatchDog(Task<IFunctionStore>
2727
store,
2828
new Settings(
2929
unhandledExceptionHandler.Catch,
30-
leaseLength: TimeSpan.Zero,
31-
watchdogCheckFrequency: TimeSpan.Zero
30+
enableWatchdogs: false,
31+
leaseLength: TimeSpan.Zero
3232
)
3333
)
3434
.RegisterFunc(
@@ -43,7 +43,7 @@ protected async Task NonCompletedFuncIsCompletedByWatchDog(Task<IFunctionStore>
4343
store,
4444
new Settings(
4545
unhandledExceptionHandler.Catch,
46-
leaseLength: TimeSpan.FromMilliseconds(1_000)
46+
leaseLength: TimeSpan.FromSeconds(1)
4747
)
4848
);
4949

@@ -56,8 +56,7 @@ protected async Task NonCompletedFuncIsCompletedByWatchDog(Task<IFunctionStore>
5656
await BusyWait.Until(
5757
async () => await store
5858
.GetFunction(functionId)
59-
.Map(f => f?.Status ?? Status.Failed) == Status.Succeeded,
60-
maxWait: TimeSpan.FromSeconds(10)
59+
.Map(f => f?.Status == Status.Succeeded)
6160
);
6261

6362
var status = await store.GetFunction(functionId).Map(f => f?.Status);
@@ -84,7 +83,7 @@ protected async Task NonCompletedFuncWithStateIsCompletedByWatchDog(Task<IFuncti
8483
new Settings(
8584
unhandledExceptionHandler.Catch,
8685
leaseLength: TimeSpan.Zero,
87-
watchdogCheckFrequency: TimeSpan.Zero
86+
enableWatchdogs: false
8887
)
8988
);
9089
var nonCompletingFunctionsRegistry = functionsRegistry
@@ -152,7 +151,7 @@ protected async Task NonCompletedActionIsCompletedByWatchDog(Task<IFunctionStore
152151
new Settings(
153152
unhandledExceptionHandler.Catch,
154153
leaseLength: TimeSpan.Zero,
155-
watchdogCheckFrequency: TimeSpan.Zero
154+
enableWatchdogs: false
156155
)
157156
)
158157
.RegisterAction(
@@ -209,7 +208,7 @@ protected async Task NonCompletedActionWithStateIsCompletedByWatchDog(Task<IFunc
209208
new Settings(
210209
unhandledExceptionHandler.Catch,
211210
leaseLength: TimeSpan.Zero,
212-
watchdogCheckFrequency: TimeSpan.Zero
211+
enableWatchdogs: false
213212
)
214213
)
215214
.RegisterAction(

‎Core/Cleipnir.ResilientFunctions.Tests/TestTemplates/RFunctionTests/DoubleInvocationTests.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ protected async Task SecondInvocationWaitsForAndReturnsSuccessfulResult(Task<IFu
2727
new Settings(
2828
unhandledExceptionHandler.Catch,
2929
leaseLength: TimeSpan.Zero,
30-
watchdogCheckFrequency: TimeSpan.Zero
30+
enableWatchdogs: false
3131
)
3232
);
3333
var rFunc = functionsRegistry.RegisterFunc(
@@ -64,7 +64,7 @@ protected async Task SecondInvocationFailsOnSuspendedFlow(Task<IFunctionStore> s
6464
new Settings(
6565
unhandledExceptionHandler.Catch,
6666
leaseLength: TimeSpan.Zero,
67-
watchdogCheckFrequency: TimeSpan.Zero
67+
enableWatchdogs: false
6868
)
6969
);
7070
var rFunc = functionsRegistry.RegisterFunc(
@@ -96,7 +96,7 @@ protected async Task SecondInvocationFailsOnPostponedFlow(Task<IFunctionStore> s
9696
new Settings(
9797
unhandledExceptionHandler.Catch,
9898
leaseLength: TimeSpan.Zero,
99-
watchdogCheckFrequency: TimeSpan.Zero
99+
enableWatchdogs: false
100100
)
101101
);
102102
var rFunc = functionsRegistry.RegisterFunc(
@@ -128,7 +128,7 @@ protected async Task SecondInvocationFailsOnFailedFlow(Task<IFunctionStore> stor
128128
new Settings(
129129
unhandledExceptionHandler.Catch,
130130
leaseLength: TimeSpan.Zero,
131-
watchdogCheckFrequency: TimeSpan.Zero
131+
enableWatchdogs: false
132132
)
133133
);
134134
var rFunc = functionsRegistry.RegisterFunc(

‎Core/Cleipnir.ResilientFunctions.Tests/TestTemplates/RFunctionTests/FailedTests.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ private async Task ExceptionThrowingFuncIsNotCompletedByWatchDog(
4040
new Settings(
4141
unhandledExceptionHandler.Catch,
4242
leaseLength: TimeSpan.Zero,
43-
watchdogCheckFrequency: TimeSpan.Zero
43+
enableWatchdogs: false
4444
)
4545
);
4646
var rFunc = functionsRegistry
@@ -110,7 +110,7 @@ private async Task ExceptionThrowingFuncWithStateIsNotCompletedByWatchDog(
110110
new Settings(
111111
unhandledExceptionHandler.Catch,
112112
leaseLength: TimeSpan.FromMilliseconds(0),
113-
watchdogCheckFrequency: TimeSpan.FromMilliseconds(0)
113+
enableWatchdogs: false
114114
)
115115
);
116116
var nonCompletingFunctionsRegistry = functionsRegistry
@@ -171,7 +171,7 @@ public async Task ExceptionThrowingActionIsNotCompletedByWatchDog(Task<IFunction
171171
new Settings(
172172
unhandledExceptionHandler.Catch,
173173
leaseLength: TimeSpan.FromMilliseconds(0),
174-
watchdogCheckFrequency: TimeSpan.FromMilliseconds(0)
174+
enableWatchdogs: false
175175
)
176176
);
177177
var nonCompletingFunctionsRegistry = functionsRegistry

‎Core/Cleipnir.ResilientFunctions.Tests/TestTemplates/RFunctionTests/PostponedTests.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ protected async Task PostponedFuncIsCompletedByWatchDog(Task<IFunctionStore> sto
2828
crashableStore,
2929
new Settings(
3030
unhandledExceptionHandler.Catch,
31-
watchdogCheckFrequency: TimeSpan.Zero
31+
enableWatchdogs: false
3232
)
3333
);
3434
var rFunc = functionsRegistry.RegisterFunc<string, string>(
@@ -78,7 +78,7 @@ protected async Task PostponedFuncWithStateIsCompletedByWatchDog(Task<IFunctionS
7878
store,
7979
new Settings(
8080
unhandledExceptionHandler.Catch,
81-
watchdogCheckFrequency: TimeSpan.Zero
81+
enableWatchdogs: false
8282
)
8383
);
8484
var rFunc = functionsRegistry.RegisterFunc<string, string>(
@@ -140,7 +140,7 @@ protected async Task PostponedActionIsCompletedByWatchDog(Task<IFunctionStore> s
140140
new Settings(
141141
unhandledExceptionHandler.Catch,
142142
leaseLength: TimeSpan.Zero,
143-
watchdogCheckFrequency: TimeSpan.Zero
143+
enableWatchdogs: false
144144
)
145145
);
146146
var rAction = functionsRegistry.RegisterAction(
@@ -187,7 +187,7 @@ protected async Task PostponedActionWithStateIsCompletedByWatchDog(Task<IFunctio
187187
store,
188188
new Settings(
189189
unhandledExceptionHandler.Catch,
190-
watchdogCheckFrequency: TimeSpan.Zero
190+
enableWatchdogs: false
191191
)
192192
);
193193
var rAction = functionsRegistry.RegisterAction(

‎Core/Cleipnir.ResilientFunctions.Tests/TestTemplates/RFunctionTests/ReInvocationTests.cs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ protected async Task ActionReInvocationSunshineScenario(Task<IFunctionStore> sto
2626
new Settings(
2727
unhandledExceptionCatcher.Catch,
2828
leaseLength: TimeSpan.Zero,
29-
watchdogCheckFrequency: TimeSpan.Zero
29+
enableWatchdogs: false
3030
)
3131
);
3232
var syncedParameter = new Synced<string>();
@@ -71,7 +71,7 @@ protected async Task ActionWithStateReInvocationSunshineScenario(Task<IFunctionS
7171
new Settings(
7272
unhandledExceptionCatcher.Catch,
7373
leaseLength: TimeSpan.Zero,
74-
watchdogCheckFrequency: TimeSpan.Zero
74+
enableWatchdogs: false
7575
)
7676
);
7777

@@ -128,7 +128,7 @@ protected async Task UpdatedParameterIsPassedInOnReInvocationSunshineScenario(Ta
128128
new Settings(
129129
unhandledExceptionCatcher.Catch,
130130
leaseLength: TimeSpan.Zero,
131-
watchdogCheckFrequency: TimeSpan.Zero
131+
enableWatchdogs: false
132132
)
133133
);
134134

@@ -176,7 +176,7 @@ protected async Task UpdatedParameterAndStateIsPassedInOnReInvocationSunshineSce
176176
new Settings(
177177
unhandledExceptionCatcher.Catch,
178178
leaseLength: TimeSpan.Zero,
179-
watchdogCheckFrequency: TimeSpan.Zero
179+
enableWatchdogs: false
180180
)
181181
);
182182

@@ -225,7 +225,7 @@ protected async Task FuncReInvocationSunshineScenario(Task<IFunctionStore> store
225225
new Settings(
226226
unhandledExceptionCatcher.Catch,
227227
leaseLength: TimeSpan.Zero,
228-
watchdogCheckFrequency: TimeSpan.Zero
228+
enableWatchdogs: false
229229
)
230230
);
231231

@@ -269,7 +269,7 @@ protected async Task FuncWithStateReInvocationSunshineScenario(Task<IFunctionSto
269269
new Settings(
270270
unhandledExceptionCatcher.Catch,
271271
leaseLength: TimeSpan.Zero,
272-
watchdogCheckFrequency: TimeSpan.Zero
272+
enableWatchdogs: false
273273
)
274274
);
275275

@@ -325,7 +325,7 @@ protected async Task ReInvocationFailsWhenTheFunctionDoesNotExist(Task<IFunction
325325
new Settings(
326326
unhandledExceptionCatcher.Catch,
327327
leaseLength: TimeSpan.Zero,
328-
watchdogCheckFrequency: TimeSpan.Zero
328+
enableWatchdogs: false
329329
)
330330
);
331331

‎Core/Cleipnir.ResilientFunctions.Tests/TestTemplates/RFunctionTests/ScheduleReInvocationTests.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ protected async Task ActionReInvocationSunshineScenario(Task<IFunctionStore> sto
2424
new Settings(
2525
unhandledExceptionCatcher.Catch,
2626
leaseLength: TimeSpan.Zero,
27-
watchdogCheckFrequency: TimeSpan.Zero
27+
enableWatchdogs: false
2828
)
2929
);
3030
var syncedParameter = new Synced<string>();
@@ -76,7 +76,7 @@ protected async Task ActionWithStateReInvocationSunshineScenario(Task<IFunctionS
7676
new Settings(
7777
unhandledExceptionCatcher.Catch,
7878
leaseLength: TimeSpan.Zero,
79-
watchdogCheckFrequency: TimeSpan.Zero
79+
enableWatchdogs: false
8080
)
8181
);
8282

@@ -133,7 +133,7 @@ protected async Task FuncReInvocationSunshineScenario(Task<IFunctionStore> store
133133
new Settings(
134134
unhandledExceptionCatcher.Catch,
135135
leaseLength: TimeSpan.Zero,
136-
watchdogCheckFrequency: TimeSpan.Zero
136+
enableWatchdogs: false
137137
)
138138
);
139139

@@ -181,7 +181,7 @@ protected async Task FuncWithStateReInvocationSunshineScenario(Task<IFunctionSto
181181
new Settings(
182182
unhandledExceptionCatcher.Catch,
183183
leaseLength: TimeSpan.Zero,
184-
watchdogCheckFrequency: TimeSpan.Zero
184+
enableWatchdogs: false
185185
)
186186
);
187187

‎Core/Cleipnir.ResilientFunctions/CoreRuntime/Watchdogs/WatchDogsFactory.cs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,14 @@ public static void CreateAndStart(
1818
SettingsWithDefaults settings,
1919
ShutdownCoordinator shutdownCoordinator)
2020
{
21-
if (settings.WatchdogCheckFrequency == TimeSpan.Zero || settings.WatchdogCheckFrequency == TimeSpan.MaxValue)
21+
if (!settings.EnableWatchdogs)
2222
return;
23+
24+
if (settings.WatchdogCheckFrequency == TimeSpan.Zero || settings.WatchdogCheckFrequency == TimeSpan.MaxValue)
25+
throw new InvalidOperationException(nameof(Settings.WatchdogCheckFrequency) + " is invalid");
2326

2427
var asyncSemaphore = new AsyncSemaphore(settings.MaxParallelRetryInvocations);
25-
var restarterFactory = new ReInvokerFactory(
28+
var reInvokerFactory = new ReInvokerFactory(
2629
functionTypeId,
2730
functionStore,
2831
shutdownCoordinator,
@@ -34,8 +37,8 @@ public static void CreateAndStart(
3437
scheduleReInvoke
3538
);
3639

37-
var crashedWatchdog = new CrashedWatchdog(restarterFactory);
38-
var postponedWatchdog = new PostponedWatchdog(restarterFactory);
40+
var crashedWatchdog = new CrashedWatchdog(reInvokerFactory);
41+
var postponedWatchdog = new PostponedWatchdog(reInvokerFactory);
3942

4043
var messagesWriters = new MessageWriters(
4144
functionTypeId,

‎Core/Cleipnir.ResilientFunctions/Domain/Settings.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ public class Settings
1111
internal Action<RFunctionException>? UnhandledExceptionHandler { get; }
1212
internal TimeSpan? RetentionPeriod { get; }
1313
internal TimeSpan? LeaseLength { get; }
14+
internal bool? EnableWatchdogs { get; }
1415
internal TimeSpan? WatchdogCheckFrequency { get; }
1516
internal TimeSpan? DelayStartup { get; }
1617
internal int? MaxParallelRetryInvocations { get; }
@@ -22,6 +23,7 @@ public Settings(
2223
Action<RFunctionException>? unhandledExceptionHandler = null,
2324
TimeSpan? retentionPeriod = null,
2425
TimeSpan? leaseLength = null,
26+
bool? enableWatchdogs = null,
2527
TimeSpan? watchdogCheckFrequency = null,
2628
TimeSpan? messagesPullFrequency = null,
2729
TimeSpan? delayStartup = null,
@@ -32,6 +34,7 @@ public Settings(
3234
UnhandledExceptionHandler = unhandledExceptionHandler;
3335
RetentionPeriod = retentionPeriod;
3436
LeaseLength = leaseLength;
37+
EnableWatchdogs = enableWatchdogs;
3538
WatchdogCheckFrequency = watchdogCheckFrequency;
3639
DelayStartup = delayStartup;
3740
MaxParallelRetryInvocations = maxParallelRetryInvocations;
@@ -45,6 +48,7 @@ public record SettingsWithDefaults(
4548
UnhandledExceptionHandler UnhandledExceptionHandler,
4649
TimeSpan RetentionPeriod,
4750
TimeSpan LeaseLength,
51+
bool EnableWatchdogs,
4852
TimeSpan WatchdogCheckFrequency,
4953
TimeSpan MessagesPullFrequency,
5054
TimeSpan DelayStartup,
@@ -62,6 +66,7 @@ public SettingsWithDefaults Merge(Settings? child)
6266
: new UnhandledExceptionHandler(child.UnhandledExceptionHandler),
6367
child.RetentionPeriod ?? RetentionPeriod,
6468
child.LeaseLength ?? LeaseLength,
69+
child.EnableWatchdogs ?? EnableWatchdogs,
6570
child.WatchdogCheckFrequency ?? WatchdogCheckFrequency,
6671
child.MessagesPullFrequency ?? MessagesPullFrequency,
6772
child.DelayStartup ?? DelayStartup,
@@ -76,6 +81,7 @@ public SettingsWithDefaults Merge(Settings? child)
7681
UnhandledExceptionHandler: new UnhandledExceptionHandler(_ => {}),
7782
RetentionPeriod: TimeSpan.MaxValue,
7883
LeaseLength: TimeSpan.FromSeconds(10),
84+
EnableWatchdogs: true,
7985
WatchdogCheckFrequency: TimeSpan.FromSeconds(1),
8086
MessagesPullFrequency: TimeSpan.FromMilliseconds(250),
8187
DelayStartup: TimeSpan.FromSeconds(0),

0 commit comments

Comments
 (0)