<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<meta name="Generator" content="Microsoft Exchange Server">
<!-- converted from text --><style><!-- .EmailQuote { margin-left: 1pt; padding-left: 4pt; border-left: #800000 2px solid; } --></style>
</head>
<body>
<meta content="text/html; charset=UTF-8">
<style type="text/css" style="">
<!--
p
{margin-top:0;
margin-bottom:0}
-->
</style>
<div dir="ltr">
<div id="x_divtagdefaultwrapper" dir="ltr" style="font-size:12pt; color:#000000; font-family:Calibri,Helvetica,sans-serif">
<p>ack</p>
</div>
<hr tabindex="-1" style="display:inline-block; width:98%">
<div id="x_divRplyFwdMsg" dir="ltr"><font face="Calibri, sans-serif" color="#000000" style="font-size:11pt"><b>From:</b> Yuriy Vasilev <yuriy.vasilev@virtuozzo.com><br>
<b>Sent:</b> Thursday, July 27, 2023 7:50:30 PM<br>
<b>To:</b> Alexey Kuznetsov<br>
<b>Cc:</b> Andrey Zaitsev; Devel; yuriy.vasilev@virtuozzo.com<br>
<b>Subject:</b> [PATCH vz9 v3] fs/fuse kio: implement rpc errors collection</font>
<div> </div>
</div>
</div>
<font size="2"><span style="font-size:10pt;">
<div class="PlainText">Implement collecting rpc errors and exporting them to userspace fuse<br>
daemon via relayfs.<br>
Errors are grouped per IP addresses and exported as a field of<br>
kfuse_metrics struct. This field is updated on each read.<br>
<br>
Also move pcs_net* functions to a separate file.<br>
<br>
Affects: #VSTOR-70161<br>
<a href="https://pmc.acronis.work/browse/VSTOR-70161">https://pmc.acronis.work/browse/VSTOR-70161</a><br>
<br>
Signed-off-by: Yuriy Vasilev <yuriy.vasilev@virtuozzo.com><br>
---<br>
v2: Replace flex array with a single struct containing a 'next' field<br>
that refreshes after each successful read, provided there is a next elem<br>
in the list. Now, after reading, only head of an rpc errors list is<br>
cleaned.<br>
Replace spinlock with mutex. Also, remove rpc errors accounting as it is<br>
not necessary now.<br>
Add more rpc error types.<br>
<br>
v3: Fixed copying. Now, copying and removal occur in one place. First,<br>
under mutex an error record is removed from list, and this record is<br>
copied to stat struct. fuse_error_metrics is now part of fuse_ktrace,<br>
unnecessary memory management code for it has been removed.<br>
---<br>
fs/fuse/Makefile | 3 +-<br>
fs/fuse/kio/pcs/fuse_ktrace.h | 1 +<br>
fs/fuse/kio/pcs/fuse_prometheus.h | 18 +++++<br>
fs/fuse/kio/pcs/fuse_prometheus_prot.h | 19 ++++++<br>
fs/fuse/kio/pcs/pcs_cs.c | 34 +---------<br>
fs/fuse/kio/pcs/pcs_fuse_kdirect.c | 87 ++++++++++++++++++++++--<br>
fs/fuse/kio/pcs/pcs_net_addr.c | 93 ++++++++++++++++++++++++++<br>
fs/fuse/kio/pcs/pcs_net_addr.h | 16 +++++<br>
fs/fuse/kio/pcs/pcs_rdma_conn.c | 4 ++<br>
fs/fuse/kio/pcs/pcs_rpc.c | 27 ++++++--<br>
fs/fuse/kio/pcs/pcs_rpc.h | 3 +<br>
fs/fuse/kio/pcs/pcs_sock_conn.c | 29 ++------<br>
fs/fuse/kio/pcs/pcs_sock_conn.h | 1 -<br>
13 files changed, 264 insertions(+), 71 deletions(-)<br>
create mode 100644 fs/fuse/kio/pcs/pcs_net_addr.c<br>
create mode 100644 fs/fuse/kio/pcs/pcs_net_addr.h<br>
<br>
diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile<br>
index cc0d0c1d63b3..37ef32ffa13d 100644<br>
--- a/fs/fuse/Makefile<br>
+++ b/fs/fuse/Makefile<br>
@@ -31,6 +31,7 @@ fuse_kio_pcs-objs := kio/pcs/pcs_fuse_kdirect.o \<br>
kio/pcs/pcs_auth.o \<br>
kio/pcs/pcs_rdma_io.o \<br>
kio/pcs/pcs_rdma_rw.o \<br>
- kio/pcs/pcs_rdma_conn.o<br>
+ kio/pcs/pcs_rdma_conn.o \<br>
+ kio/pcs/pcs_net_addr.o<br>
<br>
virtiofs-y := virtio_fs.o<br>
diff --git a/fs/fuse/kio/pcs/fuse_ktrace.h b/fs/fuse/kio/pcs/fuse_ktrace.h<br>
index aef368fcacef..72342342b8da 100644<br>
--- a/fs/fuse/kio/pcs/fuse_ktrace.h<br>
+++ b/fs/fuse/kio/pcs/fuse_ktrace.h<br>
@@ -29,6 +29,7 @@ struct fuse_ktrace<br>
struct dentry *prometheus_dentry;<br>
struct kfuse_metrics __percpu *prometheus_metrics;<br>
u8 * __percpu buf;<br>
+ struct fuse_error_metrics error_metrics;<br>
};<br>
<br>
static inline void * fuse_trace_prepare(struct fuse_ktrace * tr, int type, int len)<br>
diff --git a/fs/fuse/kio/pcs/fuse_prometheus.h b/fs/fuse/kio/pcs/fuse_prometheus.h<br>
index 3f1b31c290d6..6dba65409077 100644<br>
--- a/fs/fuse/kio/pcs/fuse_prometheus.h<br>
+++ b/fs/fuse/kio/pcs/fuse_prometheus.h<br>
@@ -15,4 +15,22 @@ struct fuse_prometheus_data<br>
struct kfuse_histogram __percpu *histo;<br>
};<br>
<br>
+struct fuse_rpc_error {<br>
+ PCS_NET_ADDR_T addr;<br>
+ u64 err[PCS_RPC_ERR_MAX];<br>
+};<br>
+<br>
+struct fuse_rpc_error_metric {<br>
+ struct fuse_rpc_error m;<br>
+ struct list_head list;<br>
+};<br>
+<br>
+struct fuse_error_metrics {<br>
+ struct mutex mutex;<br>
+ struct list_head fuse_rpc_error_metric_list;<br>
+};<br>
+<br>
+void fuse_rpc_error_account(struct fuse_error_metrics *metrics,<br>
+ PCS_NET_ADDR_T const *addr, unsigned int err, u64 val);<br>
+<br>
#endif /* __FUSE_PROMETHEUS_H__ */<br>
diff --git a/fs/fuse/kio/pcs/fuse_prometheus_prot.h b/fs/fuse/kio/pcs/fuse_prometheus_prot.h<br>
index e7b6f8a52d26..eeaa2420a063 100644<br>
--- a/fs/fuse/kio/pcs/fuse_prometheus_prot.h<br>
+++ b/fs/fuse/kio/pcs/fuse_prometheus_prot.h<br>
@@ -39,6 +39,22 @@ struct kfuse_counter {<br>
u64 val_total;<br>
};<br>
<br>
+#define PCS_RPC_ERR_CONNECT_TOUT 0<br>
+#define PCS_RPC_ERR_CONNECT_ERROR 1<br>
+#define PCS_RPC_ERR_AUTH_TOUT 2<br>
+#define PCS_RPC_ERR_AUTH_ERR 3<br>
+#define PCS_RPC_ERR_RESPONSE_TOUT 4<br>
+#define PCS_RPC_ERR_ABORTED 5<br>
+#define PCS_RPC_ERR_MAX 6<br>
+<br>
+#define MAX_RPC_ADDR_LEN 46<br>
+<br>
+struct kfuse_rpc_error {<br>
+ char address[MAX_RPC_ADDR_LEN];<br>
+ u64 error[PCS_RPC_ERR_MAX];<br>
+ int has_next;<br>
+};<br>
+<br>
struct kfuse_metrics {<br>
/* Histograms are compatible with old version of proto<br>
* between userspace and kio where the counters were skipped.<br>
@@ -51,6 +67,9 @@ struct kfuse_metrics {<br>
u64 stucked_reqs_cnt_8s;<br>
u64 stucked_reqs_cnt_30s;<br>
u64 stucked_reqs_cnt_120s;<br>
+<br>
+ /* Each read triggers replacement of this field with the next one from a list */<br>
+ struct kfuse_rpc_error rpc_error;<br>
};<br>
<br>
#endif /* __FUSE_PROMETHEUS_PROT__ */<br>
diff --git a/fs/fuse/kio/pcs/pcs_cs.c b/fs/fuse/kio/pcs/pcs_cs.c<br>
index 9d81fe8bf22c..13e2b4e6b50c 100644<br>
--- a/fs/fuse/kio/pcs/pcs_cs.c<br>
+++ b/fs/fuse/kio/pcs/pcs_cs.c<br>
@@ -24,6 +24,7 @@<br>
#include "pcs_ioctl.h"<br>
#include "log.h"<br>
#include "fuse_ktrace.h"<br>
+#include "pcs_net_addr.h"<br>
<br>
/* Lock order: cs->lock -> css->lock (lru, hash, bl_list) */<br>
<br>
@@ -210,38 +211,6 @@ static void add_cs(struct pcs_cs_set *csset, struct pcs_cs *cs)<br>
hlist_add_head_rcu(&cs->hlist, &csset->ht[hash]);<br>
}<br>
<br>
-static inline int netaddr_cmp(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2, int ignore_port)<br>
-{<br>
- unsigned int d;<br>
- size_t sz = 0;<br>
-<br>
- d = addr1->type - addr2->type;<br>
- if (d)<br>
- return d;<br>
- d = addr1->port - addr2->port;<br>
- if (!ignore_port && d)<br>
- return d;<br>
-<br>
- switch (addr1->type) {<br>
- case PCS_ADDRTYPE_IP:<br>
- case PCS_ADDRTYPE_RDMA:<br>
- sz = sizeof(struct in_addr);<br>
- break;<br>
- case PCS_ADDRTYPE_IP6:<br>
- sz = sizeof(struct in6_addr);<br>
- break;<br>
- default:<br>
- BUG();<br>
- }<br>
-<br>
- return memcmp(addr1->address, addr2->address, sz);<br>
-}<br>
-<br>
-static int pcs_netaddr_cmp(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2)<br>
-{<br>
- return netaddr_cmp(addr1, addr2, 0);<br>
-}<br>
-<br>
/* Return locked cs */<br>
struct pcs_cs *pcs_cs_find_create(struct pcs_cs_set *csset, PCS_NODE_ID_T *id, PCS_NET_ADDR_T *addr, int flags)<br>
{<br>
@@ -502,6 +471,7 @@ static void cs_connect(struct pcs_rpc *ep)<br>
connect_start(ep); /* TODO: rewrite to use pcs_netconnect callback */<br>
return;<br>
fail:<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_CONNECT_ERROR);<br>
pcs_rpc_reset(ep);<br>
return;<br>
}<br>
diff --git a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c<br>
index af008e63294c..706da8e536c9 100644<br>
--- a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c<br>
+++ b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c<br>
@@ -34,6 +34,7 @@<br>
#include "pcs_rpc.h"<br>
#include "fuse_ktrace.h"<br>
#include "fuse_prometheus.h"<br>
+#include "pcs_net_addr.h"<br>
<br>
unsigned int pcs_loglevel = LOG_TRACE;<br>
module_param(pcs_loglevel, uint, 0644);<br>
@@ -1243,6 +1244,8 @@ static void kpcs_req_send(struct fuse_req *req, bool bg)<br>
return;<br>
}<br>
<br>
+static void fuse_rpc_error_metrics_clean(struct fuse_error_metrics *metrics);<br>
+<br>
static void fuse_trace_free(struct fuse_ktrace *tr)<br>
{<br>
relay_close(tr->rchan);<br>
@@ -1254,6 +1257,7 @@ static void fuse_trace_free(struct fuse_ktrace *tr)<br>
free_percpu(tr->prometheus_metrics);<br>
free_percpu(tr->buf);<br>
debugfs_remove(tr->dir);<br>
+ fuse_rpc_error_metrics_clean(&tr->error_metrics);<br>
if (tr->fc)<br>
fuse_conn_put(tr->fc);<br>
kfree(tr);<br>
@@ -1359,6 +1363,73 @@ void fuse_stat_account(struct fuse_conn *fc, int op, u64 val)<br>
}<br>
}<br>
<br>
+static void fuse_rpc_error_metrics_clean(struct fuse_error_metrics *metrics)<br>
+{<br>
+ struct fuse_rpc_error_metric *entry, *next;<br>
+<br>
+ mutex_lock(&metrics->mutex);<br>
+ list_for_each_entry_safe(entry, next,<br>
+ &metrics->fuse_rpc_error_metric_list, list) {<br>
+ list_del(&entry->list);<br>
+ kfree(entry);<br>
+ }<br>
+ mutex_unlock(&metrics->mutex);<br>
+}<br>
+<br>
+void fuse_rpc_error_account(struct fuse_error_metrics *metrics,<br>
+ PCS_NET_ADDR_T const *addr, unsigned int err, u64 val)<br>
+{<br>
+ struct fuse_rpc_error_metric *metric, *entry;<br>
+<br>
+ if (!metrics || err >= PCS_RPC_ERR_MAX) {<br>
+ WARN_ON_ONCE(1);<br>
+ return;<br>
+ }<br>
+<br>
+ metric = NULL;<br>
+ mutex_lock(&metrics->mutex);<br>
+ list_for_each_entry(entry, &metrics->fuse_rpc_error_metric_list, list) {<br>
+ if (pcs_netaddr_cmp_ignore_port(&entry->m.addr, addr) == 0) {<br>
+ metric = entry;<br>
+ break;<br>
+ }<br>
+ }<br>
+<br>
+ if (!metric) {<br>
+ metric = kzalloc(sizeof(*metric), GFP_KERNEL);<br>
+ if (!metric)<br>
+ goto out;<br>
+ metric->m.addr = *addr;<br>
+ list_add_tail(&metric->list, &metrics->fuse_rpc_error_metric_list);<br>
+ }<br>
+<br>
+ metric->m.err[err] += val;<br>
+<br>
+out:<br>
+ mutex_unlock(&metrics->mutex);<br>
+}<br>
+<br>
+static void fuse_rpc_error_remove_first_and_copy_to_stat(struct fuse_error_metrics *metrics,<br>
+ struct kfuse_metrics *stats)<br>
+{<br>
+ struct fuse_rpc_error_metric *entry;<br>
+<br>
+ mutex_lock(&metrics->mutex);<br>
+ entry = list_first_entry_or_null(&metrics->fuse_rpc_error_metric_list,<br>
+ struct fuse_rpc_error_metric, list);<br>
+ if (!entry) {<br>
+ mutex_unlock(&metrics->mutex);<br>
+ return;<br>
+ }<br>
+ list_del(&entry->list);<br>
+ mutex_unlock(&metrics->mutex);<br>
+<br>
+ pcs_format_netaddr_ignore_port(stats->rpc_error.address, MAX_RPC_ADDR_LEN, &entry->m.addr);<br>
+ memcpy(stats->rpc_error.error, entry->m.err, sizeof(stats->rpc_error.error));<br>
+ stats->rpc_error.has_next = entry->list.next != NULL;<br>
+ kfree(entry);<br>
+}<br>
+<br>
static int prometheus_file_open(struct inode *inode, struct file *filp)<br>
{<br>
struct fuse_ktrace * tr = inode->i_private;<br>
@@ -1407,19 +1478,18 @@ static ssize_t prometheus_file_read(struct file *filp,<br>
struct kfuse_metrics *stats;<br>
int cpu;<br>
<br>
+ if (!tr->prometheus_metrics)<br>
+ return -EINVAL;<br>
+<br>
if (*ppos >= sizeof(struct kfuse_metrics))<br>
return 0;<br>
if (*ppos + count > sizeof(struct kfuse_metrics))<br>
count = sizeof(struct kfuse_metrics) - *ppos;<br>
<br>
- stats = (void *)get_zeroed_page(GFP_KERNEL);<br>
- BUILD_BUG_ON(sizeof(*stats) > PAGE_SIZE);<br>
+ stats = kzalloc(sizeof(struct kfuse_metrics), GFP_KERNEL);<br>
if (!stats)<br>
return -ENOMEM;<br>
<br>
- if (!tr->prometheus_metrics)<br>
- return -EINVAL;<br>
-<br>
for_each_possible_cpu(cpu) {<br>
struct kfuse_metrics *m;<br>
<br>
@@ -1446,12 +1516,14 @@ static ssize_t prometheus_file_read(struct file *filp,<br>
pcs_kio_req_list(tr->fc, prometheus_req_iter, stats);<br>
spin_unlock(&tr->fc->lock);<br>
<br>
+ fuse_rpc_error_remove_first_and_copy_to_stat(&tr->error_metrics, stats);<br>
+<br>
if (copy_to_user(buffer, (char *)stats + *ppos, count))<br>
count = -EFAULT;<br>
else<br>
*ppos += count;<br>
<br>
- free_page((unsigned long)stats);<br>
+ kfree(stats);<br>
return count;<br>
}<br>
<br>
@@ -1517,6 +1589,9 @@ static int fuse_ktrace_setup(struct fuse_conn * fc)<br>
<br>
tr->buf = __alloc_percpu(KTRACE_LOG_BUF_SIZE, 16);<br>
<br>
+ INIT_LIST_HEAD(&tr->error_metrics.fuse_rpc_error_metric_list);<br>
+ mutex_init(&tr->error_metrics.mutex);<br>
+<br>
atomic_set(&tr->refcnt, 1);<br>
<br>
ret = -EBUSY;<br>
diff --git a/fs/fuse/kio/pcs/pcs_net_addr.c b/fs/fuse/kio/pcs/pcs_net_addr.c<br>
new file mode 100644<br>
index 000000000000..f75b42e1e7c9<br>
--- /dev/null<br>
+++ b/fs/fuse/kio/pcs/pcs_net_addr.c<br>
@@ -0,0 +1,93 @@<br>
+/*<br>
+ * fs/fuse/kio/pcs/pcs_net_addr.c<br>
+ *<br>
+ * Copyright (c) 2023 Virtuozzo International GmbH. All rights reserved.<br>
+ *<br>
+ */<br>
+<br>
+#include <net/sock.h><br>
+<br>
+#include "pcs_types.h"<br>
+<br>
+int pcs_netaddr2sockaddr(PCS_NET_ADDR_T const *addr, struct sockaddr *sa, int *salen)<br>
+{<br>
+ BUG_ON(!sa);<br>
+ if (addr->type == PCS_ADDRTYPE_IP || addr->type == PCS_ADDRTYPE_RDMA) {<br>
+ struct sockaddr_in *saddr4 = (struct sockaddr_in *)sa;<br>
+ *saddr4 = (struct sockaddr_in) {<br>
+ .sin_family = AF_INET,<br>
+ .sin_port = (u16)addr->port,<br>
+ };<br>
+ memcpy(&saddr4->sin_addr, addr->address, sizeof(saddr4->sin_addr));<br>
+ *salen = sizeof(*saddr4);<br>
+ } else if (addr->type == PCS_ADDRTYPE_IP6) {<br>
+ struct sockaddr_in6 *saddr6 = (struct sockaddr_in6 *)sa;<br>
+ *saddr6 = (struct sockaddr_in6) {<br>
+ .sin6_family = AF_INET6,<br>
+ .sin6_port = (u16)addr->port,<br>
+ };<br>
+ memcpy(&saddr6->sin6_addr, addr->address, sizeof(saddr6->sin6_addr));<br>
+ *salen = sizeof(*saddr6);<br>
+ } else<br>
+ return -EINVAL;<br>
+<br>
+ return 0;<br>
+}<br>
+<br>
+static inline int netaddr_cmp(PCS_NET_ADDR_T const *addr1,<br>
+ PCS_NET_ADDR_T const *addr2, int ignore_port)<br>
+{<br>
+ unsigned int d;<br>
+ size_t sz = 0;<br>
+<br>
+ d = addr1->type - addr2->type;<br>
+ if (d)<br>
+ return d;<br>
+ d = addr1->port - addr2->port;<br>
+ if (!ignore_port && d)<br>
+ return d;<br>
+<br>
+ switch (addr1->type) {<br>
+ case PCS_ADDRTYPE_IP:<br>
+ case PCS_ADDRTYPE_RDMA:<br>
+ sz = sizeof(struct in_addr);<br>
+ break;<br>
+ case PCS_ADDRTYPE_IP6:<br>
+ sz = sizeof(struct in6_addr);<br>
+ break;<br>
+ default:<br>
+ BUG();<br>
+ }<br>
+<br>
+ return memcmp(addr1->address, addr2->address, sz);<br>
+}<br>
+<br>
+int pcs_netaddr_cmp(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2)<br>
+{<br>
+ return netaddr_cmp(addr1, addr2, 0);<br>
+}<br>
+<br>
+int pcs_netaddr_cmp_ignore_port(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2)<br>
+{<br>
+ return netaddr_cmp(addr1, addr2, 1);<br>
+}<br>
+<br>
+int pcs_format_netaddr_ignore_port(char *str, int len, PCS_NET_ADDR_T const *addr)<br>
+{<br>
+ int ret;<br>
+<br>
+ switch (addr->type) {<br>
+ case PCS_ADDRTYPE_IP:<br>
+ case PCS_ADDRTYPE_RDMA:<br>
+ ret = snprintf(str, len, "%pI4", addr->address);<br>
+ break;<br>
+ case PCS_ADDRTYPE_IP6:<br>
+ ret = snprintf(str, len, "%pI6", addr->address);<br>
+ break;<br>
+ default:<br>
+ ret = snprintf(str, len, "unknown");<br>
+ break;<br>
+ }<br>
+<br>
+ return ret;<br>
+}<br>
diff --git a/fs/fuse/kio/pcs/pcs_net_addr.h b/fs/fuse/kio/pcs/pcs_net_addr.h<br>
new file mode 100644<br>
index 000000000000..032d265472fd<br>
--- /dev/null<br>
+++ b/fs/fuse/kio/pcs/pcs_net_addr.h<br>
@@ -0,0 +1,16 @@<br>
+/*<br>
+ * fs/fuse/kio/pcs/pcs_net_addr.h<br>
+ *<br>
+ * Copyright (c) 2023 Virtuozzo International GmbH. All rights reserved.<br>
+ *<br>
+ */<br>
+<br>
+#ifndef __PCS_NET_ADDR_H__<br>
+#define __PCS_NET_ADDR_H__ 1<br>
+<br>
+int pcs_netaddr2sockaddr(PCS_NET_ADDR_T const *addr, struct sockaddr *sa, int *salen);<br>
+int pcs_netaddr_cmp(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2);<br>
+int pcs_netaddr_cmp_ignore_port(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2);<br>
+int pcs_format_netaddr_ignore_port(char *str, int len, PCS_NET_ADDR_T const *addr);<br>
+<br>
+#endif /* __PCS_NET_ADDR_H__ */<br>
diff --git a/fs/fuse/kio/pcs/pcs_rdma_conn.c b/fs/fuse/kio/pcs/pcs_rdma_conn.c<br>
index 8d27c9b8f7ec..8ba7c60fb9b2 100644<br>
--- a/fs/fuse/kio/pcs/pcs_rdma_conn.c<br>
+++ b/fs/fuse/kio/pcs/pcs_rdma_conn.c<br>
@@ -130,18 +130,21 @@ void pcs_rdmaconnect_start(struct pcs_rpc *ep)<br>
RDMA_PS_TCP, IB_QPT_RC);<br>
if (IS_ERR(rc.cmid)) {<br>
TRACE("rdma_create_id failed: %ld\n", PTR_ERR(rc.cmid));<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_CONNECT_ERROR);<br>
goto fail;<br>
}<br>
<br>
ret = rdma_resolve_addr(rc.cmid, NULL, sa, RESOLVE_TIMEOUT_MS);<br>
if (ret) {<br>
TRACE("rdma_resolve_addr failed: %d\n", ret);<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_CONNECT_ERROR);<br>
goto fail_cm;<br>
}<br>
<br>
wait_for_completion(&rc.cm_done);<br>
if (rc.cm_event != RDMA_CM_EVENT_ESTABLISHED) {<br>
TRACE("rdma connection failed: %d\n", rc.cm_event);<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_CONNECT_ERROR);<br>
goto fail_cm;<br>
}<br>
<br>
@@ -168,6 +171,7 @@ void pcs_rdmaconnect_start(struct pcs_rpc *ep)<br>
if (ret < 0) {<br>
TRACE("rdma authorization failed: %d, rio: 0x%p",<br>
ret, rc.rio);<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_AUTH_ERR);<br>
goto fail; /* since ep->conn is initialized,<br>
* rio will be freed in pcs_rpc_reset()<br>
*/<br>
diff --git a/fs/fuse/kio/pcs/pcs_rpc.c b/fs/fuse/kio/pcs/pcs_rpc.c<br>
index 7f8d4b250277..4f49b4eb60dd 100644<br>
--- a/fs/fuse/kio/pcs/pcs_rpc.c<br>
+++ b/fs/fuse/kio/pcs/pcs_rpc.c<br>
@@ -102,6 +102,17 @@ static void rpc_del_hash(struct pcs_rpc * ep)<br>
}<br>
}<br>
<br>
+void pcs_rpc_report_error(struct pcs_rpc *ep, unsigned int err)<br>
+{<br>
+ if (!ep->eng || !cc_from_rpc(ep->eng)->fc->ktrace) {<br>
+ WARN_ON_ONCE(1);<br>
+ return;<br>
+ }<br>
+<br>
+ fuse_rpc_error_account(&cc_from_rpc(ep->eng)->fc->ktrace->error_metrics,<br>
+ &ep->addr, err, 1);<br>
+}<br>
+<br>
<br>
struct pcs_msg * pcs_rpc_lookup_xid(struct pcs_rpc * ep, PCS_XID_T * xid)<br>
{<br>
@@ -230,12 +241,15 @@ void rpc_abort(struct pcs_rpc * ep, int fatal, int error)<br>
queue_delayed_work(cc->wq, &ep->timer_work, ep->params.holddown_timeout);<br>
}<br>
<br>
- while (!list_empty(&failed_list)) {<br>
- struct pcs_msg * msg = list_first_entry(&failed_list, struct pcs_msg, list);<br>
- list_del_init(&msg->list);<br>
- pcs_set_rpc_error(&msg->error, error, ep);<br>
- BUG_ON(!hlist_unhashed(&msg->kill_link));<br>
- msg->done(msg);<br>
+ if (!list_empty(&failed_list)) {<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_ABORTED);<br>
+ while (!list_empty(&failed_list)) {<br>
+ struct pcs_msg * msg = list_first_entry(&failed_list, struct pcs_msg, list);<br>
+ list_del_init(&msg->list);<br>
+ pcs_set_rpc_error(&msg->error, error, ep);<br>
+ BUG_ON(!hlist_unhashed(&msg->kill_link));<br>
+ msg->done(msg);<br>
+ }<br>
}<br>
<br>
if (ep->state != PCS_RPC_ABORT)<br>
@@ -1279,6 +1293,7 @@ static void timer_work(struct work_struct *w)<br>
case PCS_RPC_WORK: {<br>
int err = list_empty(&ep->pending_queue) ? PCS_ERR_RESPONSE_TIMEOUT : PCS_ERR_WRITE_TIMEOUT;<br>
<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_RESPONSE_TOUT);<br>
FUSE_KTRACE(cc_from_rpc(ep->eng)->fc, "rpc timer expired, killing connection to " PEER_FMT ", %d",<br>
PEER_ARGS(ep), err);<br>
rpc_abort(ep, 0, err);<br>
diff --git a/fs/fuse/kio/pcs/pcs_rpc.h b/fs/fuse/kio/pcs/pcs_rpc.h<br>
index 7427fe4c4257..6e9f75145bcc 100644<br>
--- a/fs/fuse/kio/pcs/pcs_rpc.h<br>
+++ b/fs/fuse/kio/pcs/pcs_rpc.h<br>
@@ -12,6 +12,7 @@<br>
#include "pcs_rpc_prot.h"<br>
#include "pcs_net.h"<br>
#include "pcs_sock_io.h"<br>
+#include "fuse_prometheus.h"<br>
<br>
struct pcs_msg;<br>
<br>
@@ -316,4 +317,6 @@ static inline struct pcs_rpc *pcs_rpc_from_work(struct work_struct *wr)<br>
<br>
const char* pcs_rpc_state_name(unsigned state);<br>
<br>
+void pcs_rpc_report_error(struct pcs_rpc *ep, unsigned int err);<br>
+<br>
#endif /* _PCS_RPC_H_ */<br>
diff --git a/fs/fuse/kio/pcs/pcs_sock_conn.c b/fs/fuse/kio/pcs/pcs_sock_conn.c<br>
index f463c1ecef9d..13823da22a9d 100644<br>
--- a/fs/fuse/kio/pcs/pcs_sock_conn.c<br>
+++ b/fs/fuse/kio/pcs/pcs_sock_conn.c<br>
@@ -33,31 +33,6 @@ static inline void pcs_sock_cork(struct socket *sock)<br>
tcp_sock_set_cork(sock->sk, true);<br>
}<br>
<br>
-int pcs_netaddr2sockaddr(PCS_NET_ADDR_T const* addr, struct sockaddr *sa, int *salen)<br>
-{<br>
- BUG_ON(!sa);<br>
- if (addr->type == PCS_ADDRTYPE_IP || addr->type == PCS_ADDRTYPE_RDMA) {<br>
- struct sockaddr_in *saddr4 = (struct sockaddr_in *)sa;<br>
- *saddr4 = (struct sockaddr_in) {<br>
- .sin_family = AF_INET,<br>
- .sin_port = (u16)addr->port,<br>
- };<br>
- memcpy(&saddr4->sin_addr, addr->address, sizeof(saddr4->sin_addr));<br>
- *salen = sizeof(*saddr4);<br>
- } else if (addr->type == PCS_ADDRTYPE_IP6) {<br>
- struct sockaddr_in6 *saddr6 = (struct sockaddr_in6 *)sa;<br>
- *saddr6 = (struct sockaddr_in6) {<br>
- .sin6_family = AF_INET6,<br>
- .sin6_port = (u16)addr->port,<br>
- };<br>
- memcpy(&saddr6->sin6_addr, addr->address, sizeof(saddr6->sin6_addr));<br>
- *salen = sizeof(*saddr6);<br>
- } else<br>
- return -EINVAL;<br>
-<br>
- return 0;<br>
-}<br>
-<br>
void pcs_sockconnect_start(struct pcs_rpc *ep)<br>
{<br>
struct pcs_sockio *sio;<br>
@@ -70,6 +45,7 @@ void pcs_sockconnect_start(struct pcs_rpc *ep)<br>
sio = kzalloc(sizeof(struct pcs_sockio) + alloc_max, GFP_NOIO);<br>
if (!sio) {<br>
TRACE("Can't allocate sio\n");<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_CONNECT_ERROR);<br>
goto fail;<br>
}<br>
<br>
@@ -82,6 +58,7 @@ void pcs_sockconnect_start(struct pcs_rpc *ep)<br>
err = sock_create(sa->sa_family, SOCK_STREAM, 0, &sock);<br>
if (err < 0) {<br>
TRACE("Can't create socket: %d\n", err);<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_CONNECT_ERROR);<br>
goto fail2;<br>
}<br>
pcs_clear_error(&sio->error);<br>
@@ -90,6 +67,7 @@ void pcs_sockconnect_start(struct pcs_rpc *ep)<br>
if (err != 0 && err != -EINPROGRESS) {<br>
TRACE("Failed connection: %d\n", err);<br>
sock_release(sock);<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_CONNECT_ERROR);<br>
goto fail2;<br>
}<br>
pcs_sock_keepalive(sock);<br>
@@ -122,6 +100,7 @@ void pcs_sockconnect_start(struct pcs_rpc *ep)<br>
if (err < 0) {<br>
FUSE_KLOG(cc_from_rpc(ep->eng)->fc, LOG_ERR,<br>
"Authorization failed: %d", err);<br>
+ pcs_rpc_report_error(ep, PCS_RPC_ERR_AUTH_ERR);<br>
goto fail; /* since ep->conn is initialized,<br>
* sio will be freed in pcs_rpc_reset()<br>
*/<br>
diff --git a/fs/fuse/kio/pcs/pcs_sock_conn.h b/fs/fuse/kio/pcs/pcs_sock_conn.h<br>
index bf39a29f78e9..554958b419d0 100644<br>
--- a/fs/fuse/kio/pcs/pcs_sock_conn.h<br>
+++ b/fs/fuse/kio/pcs/pcs_sock_conn.h<br>
@@ -9,6 +9,5 @@<br>
#define _PCS_SOCK_CONN_H_ 1<br>
<br>
void pcs_sockconnect_start(struct pcs_rpc *ep);<br>
-int pcs_netaddr2sockaddr(PCS_NET_ADDR_T const* addr, struct sockaddr *sa, int *salen);<br>
<br>
#endif /* _PCS_SOCK_CONN_H_ */<br>
-- <br>
2.34.1<br>
<br>
</div>
</span></font>
</body>
</html>