[Devel] [PATCH RHEL7 COMMIT] fs/fuse kio: implement cs statistics accounting
Konstantin Khorenko
khorenko at virtuozzo.com
Wed May 29 17:35:43 MSK 2019
The commit is pushed to "branch-rh7-3.10.0-957.12.2.vz7.96.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh7-3.10.0-957.12.2.vz7.96.7
------>
commit 7800956cc073eef6ed4eed074e9703d8c54a2932
Author: Pavel Butsykin <pbutsykin at virtuozzo.com>
Date: Wed May 29 17:35:41 2019 +0300
fs/fuse kio: implement cs statistics accounting
This is preparation patch that adds the implementation of calculating statistics
for cs objects. Backport from user-mod vstorage client.
Signed-off-by: Pavel Butsykin <pbutsykin at virtuozzo.com>
=====================
Patchset description:
Statistics for vstorage kernel fast-path
Vstorage provides plenty of statistics information via 'vstorage -c cl mnt-top',
but when it comes kernel fast-path, it doesn't work. All because mnt-top
command collects information from .vstorage.info directory, where vstorage-mount
provides a lot of different statistical information in the form of files, but
it was not implemented for for fast-path.
This patch-set is aimed to implementation of the support of some statistics
inforamtion files from .vstorage.info:
cs_stats
fstat
fstat_lat
iostat
requests
Which will be located at "/sys/fs/fuse/connections/*mnt_id*/kio_stat/". This
will be enough to maintain vstorage mnt-top command for fast-path mount points.
https://pmc.acronis.com/browse/VSTOR-20979
Acked-by: Alexey Kuznetsov <kuznet at virtuozzo.com>
Pavel Butsykin (14):
fs/fuse: add conn_ctl to fuse_conn struct
fs/fuse kio: create sysfs stat directory
fs/fuse kio: implement iostat statistics info
fs/fuse kio: make interface pcs_kio_file_list() for listing kio files
fs/fuse kio: make interface pcs_kio_req_list() for listing kio reqs
fs/fuse kio: add retry counter for kio requests
fs/fuse kio: implement pcs_strerror()
fs/fuse kio: implement requests statistics info
fs/fuse kio: implement fstat statistics info
fs/fuse kio: implement fstat_lat statistics info
fs/fuse kio: remove useless pcs_cs initialization
fs/fuse kio: implement cs statistics accounting
fs/fuse kio: convert rpc state id to string
fs/fuse kio: implement cs_stats statistics info
---
fs/fuse/kio/pcs/fuse_stat.c | 2 +-
fs/fuse/kio/pcs/fuse_stat.h | 1 +
fs/fuse/kio/pcs/pcs_cs.c | 72 ++++++++++++++++++++++++++++++---
fs/fuse/kio/pcs/pcs_cs.h | 34 ++++++++++------
fs/fuse/kio/pcs/pcs_perfcounters.h | 17 +++++++-
fs/fuse/kio/pcs/pcs_perfcounters_stub.h | 30 --------------
6 files changed, 104 insertions(+), 52 deletions(-)
diff --git a/fs/fuse/kio/pcs/fuse_stat.c b/fs/fuse/kio/pcs/fuse_stat.c
index 53398514e841..f54f29c5bae8 100644
--- a/fs/fuse/kio/pcs/fuse_stat.c
+++ b/fs/fuse/kio/pcs/fuse_stat.c
@@ -44,7 +44,7 @@ static inline void fuse_val_stat_update(struct fuse_val_stat *s, u64 val)
preempt_enable();
}
-static inline void fuse_latency_update(struct fuse_lat_stat *s, u64 val)
+void fuse_latency_update(struct fuse_lat_stat *s, u64 val)
{
int i;
diff --git a/fs/fuse/kio/pcs/fuse_stat.h b/fs/fuse/kio/pcs/fuse_stat.h
index dae495cd79aa..b6d513bed7ea 100644
--- a/fs/fuse/kio/pcs/fuse_stat.h
+++ b/fs/fuse/kio/pcs/fuse_stat.h
@@ -79,6 +79,7 @@ void pcs_fuse_stat_fini(struct pcs_fuse_stat *stat);
void pcs_fuse_stat_io_count(struct pcs_int_request *ireq, struct pcs_msg *resp,
u32 io_lat, u32 net_lat);
+void fuse_latency_update(struct fuse_lat_stat *s, u64 val);
int pcs_fuse_io_stat_alloc(struct pcs_fuse_io_stat_sync *iostat);
void pcs_fuse_io_stat_free(struct pcs_fuse_io_stat_sync *iostat);
diff --git a/fs/fuse/kio/pcs/pcs_cs.c b/fs/fuse/kio/pcs/pcs_cs.c
index 64b0229e9bee..98f8193c8a99 100644
--- a/fs/fuse/kio/pcs/pcs_cs.c
+++ b/fs/fuse/kio/pcs/pcs_cs.c
@@ -46,6 +46,50 @@ struct pcs_rpc_ops cn_rpc_ops = {
.connect = cs_connect,
};
+static int pcs_cs_percpu_stat_alloc(struct pcs_cs *cs)
+{
+ seqlock_init(&cs->stat.seqlock);
+ cs->stat.iolat = alloc_percpu(struct fuse_lat_stat);
+ if (!cs->stat.iolat)
+ return -ENOMEM;
+
+ cs->stat.netlat = alloc_percpu(struct fuse_lat_stat);
+ if (!cs->stat.netlat)
+ goto fail1;
+
+ cs->stat.read_ops_rate = alloc_percpu(struct pcs_perf_rate_cnt);
+ if (!cs->stat.read_ops_rate)
+ goto fail2;
+
+ cs->stat.write_ops_rate = alloc_percpu(struct pcs_perf_rate_cnt);
+ if (!cs->stat.write_ops_rate)
+ goto fail3;
+
+ cs->stat.sync_ops_rate = alloc_percpu(struct pcs_perf_rate_cnt);
+ if (!cs->stat.sync_ops_rate)
+ goto fail4;
+
+ return 0;
+fail4:
+ free_percpu(cs->stat.write_ops_rate);
+fail3:
+ free_percpu(cs->stat.read_ops_rate);
+fail2:
+ free_percpu(cs->stat.netlat);
+fail1:
+ free_percpu(cs->stat.iolat);
+ return -ENOMEM;
+}
+
+static void pcs_cs_percpu_stat_free(struct pcs_cs *cs)
+{
+ free_percpu(cs->stat.sync_ops_rate);
+ free_percpu(cs->stat.write_ops_rate);
+ free_percpu(cs->stat.read_ops_rate);
+ free_percpu(cs->stat.netlat);
+ free_percpu(cs->stat.iolat);
+}
+
struct pcs_cs *pcs_cs_alloc(struct pcs_cs_set *css,
struct pcs_cluster_core *cc)
{
@@ -71,8 +115,13 @@ struct pcs_cs *pcs_cs_alloc(struct pcs_cs_set *css,
INIT_LIST_HEAD(&cs->flow_lru);
INIT_LIST_HEAD(&cs->bl_link);
+ if (pcs_cs_percpu_stat_alloc(cs)) {
+ kfree(cs);
+ return NULL;
+ }
cs->rpc = pcs_rpc_create(&cc->eng, &cn_rpc_params, &cn_rpc_ops);
if (cs->rpc == NULL) {
+ pcs_cs_percpu_stat_free(cs);
kfree(cs);
return NULL;
}
@@ -297,20 +346,24 @@ void cs_log_io_times(struct pcs_int_request * ireq, struct pcs_msg * resp, unsig
void pcs_cs_update_stat(struct pcs_cs *cs, u32 iolat, u32 netlat, int op_type)
{
- pcs_perfcounter_stat_update(&cs->stat.iolat, iolat);
- pcs_perfcounter_stat_update(&cs->stat.netlat, netlat);
+ write_seqlock(&cs->stat.seqlock);
+ preempt_disable();
+ fuse_latency_update(cs->stat.iolat, iolat);
+ fuse_latency_update(cs->stat.netlat, netlat);
switch (op_type) {
case PCS_CS_WRITE_SYNC_RESP:
case PCS_CS_WRITE_RESP:
- cs->stat.write_ops_rate.total++;
+ this_cpu_inc(cs->stat.write_ops_rate->total);
break;
case PCS_CS_READ_RESP:
- cs->stat.read_ops_rate.total++;
+ this_cpu_inc(cs->stat.read_ops_rate->total);
break;
case PCS_CS_SYNC_RESP:
- cs->stat.sync_ops_rate.total++;
+ this_cpu_inc(cs->stat.sync_ops_rate->total);
break;
}
+ preempt_enable();
+ write_sequnlock(&cs->stat.seqlock);
}
static void cs_response_done(struct pcs_msg *msg)
@@ -762,6 +815,13 @@ static void pcs_cs_isolate(struct pcs_cs *cs, struct list_head *dispose)
BUG_ON(cs->nflows);
}
+static void cs_destroy_rcu(struct rcu_head *head)
+{
+ struct pcs_cs *cs = container_of(head, struct pcs_cs, rcu);
+ pcs_cs_percpu_stat_free(cs);
+ kfree(cs);
+}
+
static void pcs_cs_destroy(struct pcs_cs *cs)
{
BUG_ON(!list_empty(&cs->active_list));
@@ -772,7 +832,7 @@ static void pcs_cs_destroy(struct pcs_cs *cs)
pcs_rpc_close(cs->rpc);
cs->rpc = NULL;
}
- kfree_rcu(cs, rcu);
+ call_rcu(&cs->rcu, cs_destroy_rcu);
}
void cs_aborting(struct pcs_rpc *ep, int error)
diff --git a/fs/fuse/kio/pcs/pcs_cs.h b/fs/fuse/kio/pcs/pcs_cs.h
index 513d53539211..7f124ecf58f3 100644
--- a/fs/fuse/kio/pcs/pcs_cs.h
+++ b/fs/fuse/kio/pcs/pcs_cs.h
@@ -91,11 +91,12 @@ struct pcs_cs {
struct list_head map_list;
struct {
- struct pcs_perf_stat_cnt iolat;
- struct pcs_perf_stat_cnt netlat;
- struct pcs_perf_rate_cnt read_ops_rate;
- struct pcs_perf_rate_cnt write_ops_rate;
- struct pcs_perf_rate_cnt sync_ops_rate;
+ struct fuse_lat_stat __percpu *iolat;
+ struct fuse_lat_stat __percpu *netlat;
+ struct pcs_perf_rate_cnt __percpu *read_ops_rate;
+ struct pcs_perf_rate_cnt __percpu *write_ops_rate;
+ struct pcs_perf_rate_cnt __percpu *sync_ops_rate;
+ seqlock_t seqlock;
} stat;
};
@@ -161,14 +162,21 @@ void pcs_cs_update_stat(struct pcs_cs *cs, u32 iolat, u32 netlat, int op_type);
static inline void pcs_cs_stat_up(struct pcs_cs *cs)
{
-#if 0
- /* TODO: temproraly disable perf counters */
- pcs_perfcounter_stat_up(&cs->stat.iolat);
- pcs_perfcounter_stat_up(&cs->stat.netlat);
- pcs_perfcounter_up_rate(&cs->stat.write_ops_rate);
- pcs_perfcounter_up_rate(&cs->stat.read_ops_rate);
- pcs_perfcounter_up_rate(&cs->stat.sync_ops_rate);
-#endif
+ int cpu;
+
+ write_seqlock(&cs->stat.seqlock);
+ for_each_possible_cpu(cpu) {
+ struct pcs_perf_rate_cnt *rate;
+ rate = per_cpu_ptr(cs->stat.write_ops_rate, cpu);
+ pcs_perfcounter_up_rate(rate);
+
+ rate = per_cpu_ptr(cs->stat.read_ops_rate, cpu);
+ pcs_perfcounter_up_rate(rate);
+
+ rate = per_cpu_ptr(cs->stat.sync_ops_rate, cpu);
+ pcs_perfcounter_up_rate(rate);
+ }
+ write_sequnlock(&cs->stat.seqlock);
}
static inline bool cs_is_blacklisted(struct pcs_cs *cs)
diff --git a/fs/fuse/kio/pcs/pcs_perfcounters.h b/fs/fuse/kio/pcs/pcs_perfcounters.h
index f902ce06d72d..3ae4db2ff049 100644
--- a/fs/fuse/kio/pcs/pcs_perfcounters.h
+++ b/fs/fuse/kio/pcs/pcs_perfcounters.h
@@ -1,7 +1,20 @@
#ifndef _PCS_PERFCOUNTERS_H_
#define _PCS_PERFCOUNTERS_H_ 1
-/* TODO:!!! this is stump for flow_detection */
-#include "pcs_perfcounters_stub.h"
+/* Generic event rate counter */
+struct pcs_perf_rate_cnt {
+ /* Total number of events */
+ u64 total;
+ u64 last_total;
+ /* The number of events for the last 5 sec interval */
+ u64 rate;
+};
+
+static inline void pcs_perfcounter_up_rate(struct pcs_perf_rate_cnt* cnt)
+{
+ BUG_ON(cnt->total < cnt->last_total);
+ cnt->rate = cnt->total - cnt->last_total;
+ cnt->last_total = cnt->total;
+}
#endif /* _PCS_PERFCOUNTERS_H_ */
diff --git a/fs/fuse/kio/pcs/pcs_perfcounters_stub.h b/fs/fuse/kio/pcs/pcs_perfcounters_stub.h
deleted file mode 100644
index 17dae73fcd08..000000000000
--- a/fs/fuse/kio/pcs/pcs_perfcounters_stub.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#ifndef _PCS_PERFCOUNTERS_STUB_H_
-#define _PCS_PERFCOUNTERS_STUB_H_ 1
-
-
-struct pcs_perf_stat_cnt {
- u64 val_total;
- u64 events;
- u64 curr_max;
- u64 events_last;
- u64 avg;
- u64 maximum;
-};
-
-/* Generic event rate counter */
-struct pcs_perf_rate_cnt {
- /* Total number of events */
- u64 total;
- u64 last_total;
- /* The number of events for the last 5 sec interval */
- u64 rate;
- /* The number of events per 5 sec averaged over 1, 5, 15 min and shifted by AV_SHIFT to the left */
- u64 av1;
- u64 av5;
-};
-
-
-static inline void pcs_perfcounter_stat_update(struct pcs_perf_stat_cnt *cnt, u64 val) __attribute__((unused));
-
-static inline void pcs_perfcounter_stat_update(struct pcs_perf_stat_cnt *cnt, u64 val) {}
-#endif //_PCS_PERFCOUNTERS_STUB_H_
More information about the Devel
mailing list