[Devel] [PATCH RH7 v2] sysctl: panic only after softlockup_panic seconds of softlockups in raw

[Konstantin Khorenko]

ack,

but as we've beaten softlockup in VM issues already (for now :)),
i won't apply the patch right now.

Let it wait until we need it again.

--
Best regards,

Konstantin Khorenko,
Virtuozzo Linux Kernel Team

On 09/29/2016 10:47 AM, Pavel Tikhomirov wrote:
> sysctl kernel.softlockup_panic (integer >= 0)
>
>   0(default) - do not generate panics on softlockup
>
>   n > 0 - panic if cpu is in softlockup state for a period
>   of n seconds
>
> For instance: If n == 1 sysctl will work same as mainstream one,
> kernel will panic imediately on first softlockup, as kernel
> should lock at least for (2*kernel.watchdog_thresh) seconds,
> where kernel.watchdog_thresh >= 1, to be softlockuped.
>
> If kernel.watchdog_thresh is 10(default) real softlockups will
> come every 20-22 seconds as sample_period is (kernel.watchdog_thresh
> / 5.0) == 2.0. Assume softlockup_panic == 60, there can be situation
> when first softlockup will be printed at time 0s, second at time 22s,
> third at 44s, forth will come at 66s and finaly as 66 > 60 kernel will
> panic.
>
> Actually softlockups are not required to be realy consecuent(processor
> can be unlocked during period) as we allow 4 * kernel.watchdog_thresh
> between the begining of softlockup and begining of next consequent one.
>
> So it mean that: panic can really happen in intervall between
> kernel.softlockup_panic and (kernel.softlockup_panic +
> 4 * kernel.watchdog_thresh + sample_period). So in case of example
> in question between 60 and 102 seconds. Bigger watchdog_thresh means
> bigger error. To get locked time grep in dmesg: "BUG: continious
> softlockup for".
>
> https://jira.sw.ru/browse/PSBM-52199
> Signed-off-by: Pavel Tikhomirov <ptikhomirov at virtuozzo.com>
> ---
>  kernel/sysctl.c   |  3 ++-
>  kernel/watchdog.c | 27 ++++++++++++++++++++++++++-
>  2 files changed, 28 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/sysctl.c b/kernel/sysctl.c
> index c8f7bc3..40dfc98 100644
> --- a/kernel/sysctl.c
> +++ b/kernel/sysctl.c
> @@ -123,6 +123,7 @@ EXPORT_SYMBOL(ve_allow_module_load);
>  #ifdef CONFIG_LOCKUP_DETECTOR
>  static int sixty = 60;
>  static int neg_one = -1;
> +static int int_max = INT_MAX;
>  #endif
>
>  static int zero;
> @@ -910,7 +911,7 @@ static struct ctl_table kern_table[] = {
>  		.mode		= 0644,
>  		.proc_handler	= proc_dointvec_minmax,
>  		.extra1		= &zero,
> -		.extra2		= &one,
> +		.extra2		= &int_max,
>  	},
>  #ifdef CONFIG_SMP
>  	{
> diff --git a/kernel/watchdog.c b/kernel/watchdog.c
> index ba61141..9addd5b 100644
> --- a/kernel/watchdog.c
> +++ b/kernel/watchdog.c
> @@ -54,6 +54,8 @@ static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
>  static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
>  #endif
>  static unsigned long soft_lockup_nmi_warn;
> +static DEFINE_PER_CPU(unsigned long, first_softlockup_ts);
> +static DEFINE_PER_CPU(unsigned long, prev_softlockup_ts);
>
>  /* boot commands */
>  /*
> @@ -304,6 +306,23 @@ static void watchdog_interrupt_count(void)
>  static int watchdog_nmi_enable(unsigned int cpu);
>  static void watchdog_nmi_disable(unsigned int cpu);
>
> +static int need_panic_on_softlockup(void)
> +{
> +	unsigned long first_ts = __this_cpu_read(first_softlockup_ts);
> +	unsigned long now = get_timestamp();
> +	int slp = softlockup_panic;
> +
> +	if (!slp)
> +		return 0;
> +
> +	if (time_after(now, first_ts + slp)) {
> +		printk(KERN_EMERG "BUG: continious softlockup for %lus!\n",
> +		       now - first_ts);
> +		return 1;
> +	}
> +	return 0;
> +}
> +
>  /* watchdog kicker functions */
>  static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
>  {
> @@ -345,6 +364,12 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
>  	 */
>  	duration = is_softlockup(touch_ts);
>  	if (unlikely(duration)) {
> +		unsigned long prev_ts = __this_cpu_read(prev_softlockup_ts);
> +
> +		if (!prev_ts || time_after(touch_ts, prev_ts + 2 * get_softlockup_thresh()))
> +			__this_cpu_write(first_softlockup_ts, touch_ts);
> +		__this_cpu_write(prev_softlockup_ts, touch_ts);
> +
>  		/*
>  		 * If a virtual machine is stopped by the host it can look to
>  		 * the watchdog like a soft lockup, check to see if the host
> @@ -390,7 +415,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
>  		}
>
>  		add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
> -		if (softlockup_panic)
> +		if (need_panic_on_softlockup())
>  			panic("softlockup: hung tasks");
>  		__this_cpu_write(soft_watchdog_warn, true);
>  	} else
>