[Devel] [PATCH rh7 36/39] seqlock: Introduce raw_read_seqcount_latch()

[Andrey Ryabinin]

From: Peter Zijlstra <peterz at infradead.org>

Because with latches there is a strict data dependency on the seq load
we can avoid the rmb in favour of a read_barrier_depends.

Suggested-by: Ingo Molnar <mingo at kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz at infradead.org>
Signed-off-by: Rusty Russell <rusty at rustcorp.com.au>

https://jira.sw.ru/browse/PSBM-69081
(cherry picked from commit 7fc26327b75685f37f58d64bdb061460f834f80d)
Signed-off-by: Andrey Ryabinin <aryabinin at virtuozzo.com>
---
 include/linux/seqlock.h   | 8 ++++++--
 kernel/time/timekeeping.c | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
index ee088ed20a6c..9d8997027263 100644
--- a/include/linux/seqlock.h
+++ b/include/linux/seqlock.h
@@ -34,6 +34,7 @@
 
 #include <linux/spinlock.h>
 #include <linux/preempt.h>
+#include <linux/compiler.h>
 #include <asm/processor.h>
 
 /*
@@ -170,6 +171,10 @@ static inline int read_seqcount_retry(const seqcount_t *s, unsigned start)
 	return __read_seqcount_retry(s, start);
 }
 
+static inline int raw_read_seqcount_latch(seqcount_t *s)
+{
+	return lockless_dereference(s->sequence);
+}
 
 /**
  * raw_write_seqcount_latch - redirect readers to even/odd copy
@@ -222,8 +227,7 @@ static inline int read_seqcount_retry(const seqcount_t *s, unsigned start)
  *	unsigned seq, idx;
  *
  *	do {
- *		seq = latch->seq;
- *		smp_rmb();
+ *		seq = lockless_dereference(latch->seq);
  *
  *		idx = seq & 0x01;
  *		entry = data_query(latch->data[idx], ...);
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 8e5b95064209..d99c89095bfd 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -292,7 +292,7 @@ static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
 	u64 now;
 
 	do {
-		seq = raw_read_seqcount(&tkf->seq);
+		seq = raw_read_seqcount_latch(&tkf->seq);
 		tkr = tkf->base + (seq & 0x01);
 		now = ktime_to_ns(tkr->base) + timekeeping_get_ns(tkr);
 	} while (read_seqcount_retry(&tkf->seq, seq));
-- 
2.13.5