Skip to content

Commit 9145618

Browse files
Honny1Honny1
committed
Configure HealthCheck with podman update
New flags in a `podman update` can change the configuration of HealthCheck when the container is started, without having to restart or recreate the container. This can help determine why a given container suddenly started failing HealthCheck without interfering with the services it provides. For example, reconfigure HealthCheck to keep logs longer than the usual last X results, store logs to other destinations, etc. Fixes: https://issues.redhat.com/browse/RHEL-60561 Signed-off-by: Jan Rodák <[email protected]>
1 parent 77e67e7 commit 9145618

34 files changed

+954
-196
lines changed

‎cmd/podman/common/create.go

+134-117
Original file line numberDiff line numberDiff line change
@@ -168,78 +168,6 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions,
168168
)
169169
_ = cmd.RegisterFlagCompletionFunc(groupAddFlagName, completion.AutocompleteNone)
170170

171-
healthCmdFlagName := "health-cmd"
172-
createFlags.StringVar(
173-
&cf.HealthCmd,
174-
healthCmdFlagName, "",
175-
"set a healthcheck command for the container ('none' disables the existing healthcheck)",
176-
)
177-
_ = cmd.RegisterFlagCompletionFunc(healthCmdFlagName, completion.AutocompleteNone)
178-
179-
healthIntervalFlagName := "health-interval"
180-
createFlags.StringVar(
181-
&cf.HealthInterval,
182-
healthIntervalFlagName, define.DefaultHealthCheckInterval,
183-
"set an interval for the healthcheck (a value of disable results in no automatic timer setup)",
184-
)
185-
_ = cmd.RegisterFlagCompletionFunc(healthIntervalFlagName, completion.AutocompleteNone)
186-
187-
healthLogDestinationFlagName := "health-log-destination"
188-
createFlags.StringVar(
189-
&cf.HealthLogDestination,
190-
healthLogDestinationFlagName, define.DefaultHealthCheckLocalDestination,
191-
"set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file)",
192-
)
193-
_ = cmd.RegisterFlagCompletionFunc(healthLogDestinationFlagName, completion.AutocompleteNone)
194-
195-
healthMaxLogCountFlagName := "health-max-log-count"
196-
createFlags.UintVar(
197-
&cf.HealthMaxLogCount,
198-
healthMaxLogCountFlagName, define.DefaultHealthMaxLogCount,
199-
"set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file)",
200-
)
201-
_ = cmd.RegisterFlagCompletionFunc(healthMaxLogCountFlagName, completion.AutocompleteNone)
202-
203-
healthMaxLogSizeFlagName := "health-max-log-size"
204-
createFlags.UintVar(
205-
&cf.HealthMaxLogSize,
206-
healthMaxLogSizeFlagName, define.DefaultHealthMaxLogSize,
207-
"set maximum length in characters of stored HealthCheck log. ('0' value means an infinite log length)",
208-
)
209-
_ = cmd.RegisterFlagCompletionFunc(healthMaxLogSizeFlagName, completion.AutocompleteNone)
210-
211-
healthRetriesFlagName := "health-retries"
212-
createFlags.UintVar(
213-
&cf.HealthRetries,
214-
healthRetriesFlagName, define.DefaultHealthCheckRetries,
215-
"the number of retries allowed before a healthcheck is considered to be unhealthy",
216-
)
217-
_ = cmd.RegisterFlagCompletionFunc(healthRetriesFlagName, completion.AutocompleteNone)
218-
219-
healthStartPeriodFlagName := "health-start-period"
220-
createFlags.StringVar(
221-
&cf.HealthStartPeriod,
222-
healthStartPeriodFlagName, define.DefaultHealthCheckStartPeriod,
223-
"the initialization time needed for a container to bootstrap",
224-
)
225-
_ = cmd.RegisterFlagCompletionFunc(healthStartPeriodFlagName, completion.AutocompleteNone)
226-
227-
healthTimeoutFlagName := "health-timeout"
228-
createFlags.StringVar(
229-
&cf.HealthTimeout,
230-
healthTimeoutFlagName, define.DefaultHealthCheckTimeout,
231-
"the maximum time allowed to complete the healthcheck before an interval is considered failed",
232-
)
233-
_ = cmd.RegisterFlagCompletionFunc(healthTimeoutFlagName, completion.AutocompleteNone)
234-
235-
healthOnFailureFlagName := "health-on-failure"
236-
createFlags.StringVar(
237-
&cf.HealthOnFailure,
238-
healthOnFailureFlagName, "none",
239-
"action to take once the container turns unhealthy",
240-
)
241-
_ = cmd.RegisterFlagCompletionFunc(healthOnFailureFlagName, AutocompleteHealthOnFailure)
242-
243171
createFlags.BoolVar(
244172
&cf.HTTPProxy,
245173
"http-proxy", podmanConfig.ContainersConfDefaultsRO.Containers.HTTPProxy,
@@ -311,11 +239,6 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions,
311239
)
312240
_ = cmd.RegisterFlagCompletionFunc(logOptFlagName, AutocompleteLogOpt)
313241

314-
createFlags.BoolVar(
315-
&cf.NoHealthCheck,
316-
"no-healthcheck", false,
317-
"Disable healthchecks on container",
318-
)
319242
createFlags.BoolVar(
320243
&cf.OOMKillDisable,
321244
"oom-kill-disable", false,
@@ -452,46 +375,6 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions,
452375
)
453376
_ = cmd.RegisterFlagCompletionFunc(secretFlagName, AutocompleteSecrets)
454377

455-
startupHCCmdFlagName := "health-startup-cmd"
456-
createFlags.StringVar(
457-
&cf.StartupHCCmd,
458-
startupHCCmdFlagName, "",
459-
"Set a startup healthcheck command for the container",
460-
)
461-
_ = cmd.RegisterFlagCompletionFunc(startupHCCmdFlagName, completion.AutocompleteNone)
462-
463-
startupHCIntervalFlagName := "health-startup-interval"
464-
createFlags.StringVar(
465-
&cf.StartupHCInterval,
466-
startupHCIntervalFlagName, define.DefaultHealthCheckInterval,
467-
"Set an interval for the startup healthcheck",
468-
)
469-
_ = cmd.RegisterFlagCompletionFunc(startupHCIntervalFlagName, completion.AutocompleteNone)
470-
471-
startupHCRetriesFlagName := "health-startup-retries"
472-
createFlags.UintVar(
473-
&cf.StartupHCRetries,
474-
startupHCRetriesFlagName, 0,
475-
"Set the maximum number of retries before the startup healthcheck will restart the container",
476-
)
477-
_ = cmd.RegisterFlagCompletionFunc(startupHCRetriesFlagName, completion.AutocompleteNone)
478-
479-
startupHCSuccessesFlagName := "health-startup-success"
480-
createFlags.UintVar(
481-
&cf.StartupHCSuccesses,
482-
startupHCSuccessesFlagName, 0,
483-
"Set the number of consecutive successes before the startup healthcheck is marked as successful and the normal healthcheck begins (0 indicates any success will start the regular healthcheck)",
484-
)
485-
_ = cmd.RegisterFlagCompletionFunc(startupHCSuccessesFlagName, completion.AutocompleteNone)
486-
487-
startupHCTimeoutFlagName := "health-startup-timeout"
488-
createFlags.StringVar(
489-
&cf.StartupHCTimeout,
490-
startupHCTimeoutFlagName, define.DefaultHealthCheckTimeout,
491-
"Set the maximum amount of time that the startup healthcheck may take before it is considered failed",
492-
)
493-
_ = cmd.RegisterFlagCompletionFunc(startupHCTimeoutFlagName, completion.AutocompleteNone)
494-
495378
stopSignalFlagName := "stop-signal"
496379
createFlags.StringVar(
497380
&cf.StopSignal,
@@ -665,6 +548,140 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions,
665548
`If a container with the same name exists, replace it`,
666549
)
667550
}
551+
if mode == entities.CreateMode || mode == entities.UpdateMode {
552+
createFlags.BoolVar(
553+
&cf.NoHealthCheck,
554+
"no-healthcheck", false,
555+
"Disable healthchecks on container",
556+
)
557+
558+
healthCmdFlagName := "health-cmd"
559+
createFlags.StringVar(
560+
&cf.HealthCmd,
561+
healthCmdFlagName, "",
562+
"set a healthcheck command for the container ('none' disables the existing healthcheck)",
563+
)
564+
_ = cmd.RegisterFlagCompletionFunc(healthCmdFlagName, completion.AutocompleteNone)
565+
566+
info := ""
567+
if mode == entities.UpdateMode {
568+
info = "Changing this setting resets timer."
569+
}
570+
healthIntervalFlagName := "health-interval"
571+
createFlags.StringVar(
572+
&cf.HealthInterval,
573+
healthIntervalFlagName, define.DefaultHealthCheckInterval,
574+
"set an interval for the healthcheck. (a value of disable results in no automatic timer setup) "+info,
575+
)
576+
_ = cmd.RegisterFlagCompletionFunc(healthIntervalFlagName, completion.AutocompleteNone)
577+
578+
warning := ""
579+
if mode == entities.UpdateMode {
580+
warning = "Warning: Changing this setting may cause the loss of previous logs!"
581+
}
582+
healthLogDestinationFlagName := "health-log-destination"
583+
createFlags.StringVar(
584+
&cf.HealthLogDestination,
585+
healthLogDestinationFlagName, define.DefaultHealthCheckLocalDestination,
586+
"set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file) "+warning,
587+
)
588+
_ = cmd.RegisterFlagCompletionFunc(healthLogDestinationFlagName, completion.AutocompleteNone)
589+
590+
healthMaxLogCountFlagName := "health-max-log-count"
591+
createFlags.UintVar(
592+
&cf.HealthMaxLogCount,
593+
healthMaxLogCountFlagName, define.DefaultHealthMaxLogCount,
594+
"set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file)",
595+
)
596+
_ = cmd.RegisterFlagCompletionFunc(healthMaxLogCountFlagName, completion.AutocompleteNone)
597+
598+
healthMaxLogSizeFlagName := "health-max-log-size"
599+
createFlags.UintVar(
600+
&cf.HealthMaxLogSize,
601+
healthMaxLogSizeFlagName, define.DefaultHealthMaxLogSize,
602+
"set maximum length in characters of stored HealthCheck log. ('0' value means an infinite log length)",
603+
)
604+
_ = cmd.RegisterFlagCompletionFunc(healthMaxLogSizeFlagName, completion.AutocompleteNone)
605+
606+
healthRetriesFlagName := "health-retries"
607+
createFlags.UintVar(
608+
&cf.HealthRetries,
609+
healthRetriesFlagName, define.DefaultHealthCheckRetries,
610+
"the number of retries allowed before a healthcheck is considered to be unhealthy",
611+
)
612+
_ = cmd.RegisterFlagCompletionFunc(healthRetriesFlagName, completion.AutocompleteNone)
613+
614+
healthStartPeriodFlagName := "health-start-period"
615+
createFlags.StringVar(
616+
&cf.HealthStartPeriod,
617+
healthStartPeriodFlagName, define.DefaultHealthCheckStartPeriod,
618+
"the initialization time needed for a container to bootstrap",
619+
)
620+
_ = cmd.RegisterFlagCompletionFunc(healthStartPeriodFlagName, completion.AutocompleteNone)
621+
622+
healthTimeoutFlagName := "health-timeout"
623+
createFlags.StringVar(
624+
&cf.HealthTimeout,
625+
healthTimeoutFlagName, define.DefaultHealthCheckTimeout,
626+
"the maximum time allowed to complete the healthcheck before an interval is considered failed",
627+
)
628+
_ = cmd.RegisterFlagCompletionFunc(healthTimeoutFlagName, completion.AutocompleteNone)
629+
630+
healthOnFailureFlagName := "health-on-failure"
631+
createFlags.StringVar(
632+
&cf.HealthOnFailure,
633+
healthOnFailureFlagName, "none",
634+
"action to take once the container turns unhealthy",
635+
)
636+
_ = cmd.RegisterFlagCompletionFunc(healthOnFailureFlagName, AutocompleteHealthOnFailure)
637+
638+
// Startup HealthCheck
639+
640+
startupHCCmdFlagName := "health-startup-cmd"
641+
createFlags.StringVar(
642+
&cf.StartupHCCmd,
643+
startupHCCmdFlagName, "",
644+
"Set a startup healthcheck command for the container",
645+
)
646+
_ = cmd.RegisterFlagCompletionFunc(startupHCCmdFlagName, completion.AutocompleteNone)
647+
648+
info = ""
649+
if mode == entities.UpdateMode {
650+
info = "Changing this setting resets the timer, depending on the state of the container."
651+
}
652+
startupHCIntervalFlagName := "health-startup-interval"
653+
createFlags.StringVar(
654+
&cf.StartupHCInterval,
655+
startupHCIntervalFlagName, define.DefaultHealthCheckInterval,
656+
"Set an interval for the startup healthcheck. "+info,
657+
)
658+
_ = cmd.RegisterFlagCompletionFunc(startupHCIntervalFlagName, completion.AutocompleteNone)
659+
660+
startupHCRetriesFlagName := "health-startup-retries"
661+
createFlags.UintVar(
662+
&cf.StartupHCRetries,
663+
startupHCRetriesFlagName, 0,
664+
"Set the maximum number of retries before the startup healthcheck will restart the container",
665+
)
666+
_ = cmd.RegisterFlagCompletionFunc(startupHCRetriesFlagName, completion.AutocompleteNone)
667+
668+
startupHCSuccessesFlagName := "health-startup-success"
669+
createFlags.UintVar(
670+
&cf.StartupHCSuccesses,
671+
startupHCSuccessesFlagName, 0,
672+
"Set the number of consecutive successes before the startup healthcheck is marked as successful and the normal healthcheck begins (0 indicates any success will start the regular healthcheck)",
673+
)
674+
_ = cmd.RegisterFlagCompletionFunc(startupHCSuccessesFlagName, completion.AutocompleteNone)
675+
676+
startupHCTimeoutFlagName := "health-startup-timeout"
677+
createFlags.StringVar(
678+
&cf.StartupHCTimeout,
679+
startupHCTimeoutFlagName, define.DefaultHealthCheckTimeout,
680+
"Set the maximum amount of time that the startup healthcheck may take before it is considered failed",
681+
)
682+
_ = cmd.RegisterFlagCompletionFunc(startupHCTimeoutFlagName, completion.AutocompleteNone)
683+
}
684+
668685
// Restart is allowed for created, updated, and infra ctr
669686
if mode == entities.InfraMode || mode == entities.CreateMode || mode == entities.UpdateMode {
670687
restartFlagName := "restart"

‎cmd/podman/containers/update.go

+61-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import (
1717
)
1818

1919
var (
20-
updateDescription = `Updates the cgroup configuration of a given container`
20+
updateDescription = `Updates the configuration of an existing container, allowing changes to resource limits and healthchecks`
2121

2222
updateCommand = &cobra.Command{
2323
Use: "update [options] CONTAINER",
@@ -61,6 +61,58 @@ func init() {
6161
updateFlags(containerUpdateCommand)
6262
}
6363

64+
func GetChangedHealthCheckConfiguration(cmd *cobra.Command, vals *entities.ContainerCreateOptions) define.UpdateHealthCheckConfig {
65+
updateHealthCheckConfig := define.UpdateHealthCheckConfig{}
66+
67+
if cmd.Flags().Changed("health-log-destination") {
68+
updateHealthCheckConfig.HealthLogDestination = &vals.HealthLogDestination
69+
}
70+
if cmd.Flags().Changed("health-max-log-size") {
71+
updateHealthCheckConfig.HealthMaxLogSize = &vals.HealthMaxLogSize
72+
}
73+
if cmd.Flags().Changed("health-max-log-count") {
74+
updateHealthCheckConfig.HealthMaxLogCount = &vals.HealthMaxLogCount
75+
}
76+
if cmd.Flags().Changed("health-on-failure") {
77+
updateHealthCheckConfig.HealthOnFailure = &vals.HealthOnFailure
78+
}
79+
if cmd.Flags().Changed("no-healthcheck") {
80+
updateHealthCheckConfig.NoHealthCheck = &vals.NoHealthCheck
81+
}
82+
if cmd.Flags().Changed("health-cmd") {
83+
updateHealthCheckConfig.HealthCmd = &vals.HealthCmd
84+
}
85+
if cmd.Flags().Changed("health-interval") {
86+
updateHealthCheckConfig.HealthInterval = &vals.HealthInterval
87+
}
88+
if cmd.Flags().Changed("health-retries") {
89+
updateHealthCheckConfig.HealthRetries = &vals.HealthRetries
90+
}
91+
if cmd.Flags().Changed("health-timeout") {
92+
updateHealthCheckConfig.HealthTimeout = &vals.HealthTimeout
93+
}
94+
if cmd.Flags().Changed("health-start-period") {
95+
updateHealthCheckConfig.HealthStartPeriod = &vals.HealthStartPeriod
96+
}
97+
if cmd.Flags().Changed("health-startup-cmd") {
98+
updateHealthCheckConfig.HealthStartupCmd = &vals.StartupHCCmd
99+
}
100+
if cmd.Flags().Changed("health-startup-interval") {
101+
updateHealthCheckConfig.HealthStartupInterval = &vals.StartupHCInterval
102+
}
103+
if cmd.Flags().Changed("health-startup-retries") {
104+
updateHealthCheckConfig.HealthStartupRetries = &vals.StartupHCRetries
105+
}
106+
if cmd.Flags().Changed("health-startup-timeout") {
107+
updateHealthCheckConfig.HealthStartupTimeout = &vals.StartupHCTimeout
108+
}
109+
if cmd.Flags().Changed("health-startup-success") {
110+
updateHealthCheckConfig.HealthStartupSuccess = &vals.StartupHCSuccesses
111+
}
112+
113+
return updateHealthCheckConfig
114+
}
115+
64116
func update(cmd *cobra.Command, args []string) error {
65117
var err error
66118
// use a specgen since this is the easiest way to hold resource info
@@ -89,9 +141,15 @@ func update(cmd *cobra.Command, args []string) error {
89141
return err
90142
}
91143

144+
healthCheckConfig := GetChangedHealthCheckConfiguration(cmd, &updateOpts)
145+
if err != nil {
146+
return err
147+
}
148+
92149
opts := &entities.ContainerUpdateOptions{
93-
NameOrID: strings.TrimPrefix(args[0], "/"),
94-
Specgen: s,
150+
NameOrID: strings.TrimPrefix(args[0], "/"),
151+
Specgen: s,
152+
ChangedHealthCheckConfiguration: &healthCheckConfig,
95153
}
96154
rep, err := registry.ContainerEngine().ContainerUpdate(context.Background(), opts)
97155
if err != nil {

‎docs/source/markdown/options/health-cmd.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
####> This option file is used in:
2-
####> podman create, run
2+
####> podman create, run, update
33
####> If file is edited, make sure the changes
44
####> are applicable to all of those.
55
#### **--health-cmd**=*"command"* | *'["command", "arg1", ...]'*

‎docs/source/markdown/options/health-interval.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
####> This option file is used in:
2-
####> podman create, run
2+
####> podman create, run, update
33
####> If file is edited, make sure the changes
44
####> are applicable to all of those.
55
#### **--health-interval**=*interval*

‎docs/source/markdown/options/health-log-destination.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
####> This option file is used in:
2-
####> podman create, run
2+
####> podman create, run, update
33
####> If file is edited, make sure the changes
44
####> are applicable to all of those.
55
#### **--health-log-destination**=*directory_path*

‎docs/source/markdown/options/health-max-log-count.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
####> This option file is used in:
2-
####> podman create, run
2+
####> podman create, run, update
33
####> If file is edited, make sure the changes
44
####> are applicable to all of those.
55
#### **--health-max-log-count**=*number of stored logs*

‎docs/source/markdown/options/health-max-log-size.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
####> This option file is used in:
2-
####> podman create, run
2+
####> podman create, run, update
33
####> If file is edited, make sure the changes
44
####> are applicable to all of those.
55
#### **--health-max-log-size**=*size of stored logs*

0 commit comments

Comments
 (0)