[Devel] [PATCH vz9] fs/fuse kio: implement rpc errors collection
Yuriy Vasilev
yuriy.vasilev at virtuozzo.com
Mon Jul 24 11:39:34 MSK 2023
Implement collecting rpc errors and exporting them to userspace fuse
daemon via relayfs.
Errors are grouped per IP addresses and exported as an array.
Also move pcs_net* functions to a separate file.
Affects: #VSTOR-70161
https://pmc.acronis.work/browse/VSTOR-70161
Signed-off-by: Yuriy Vasilev <yuriy.vasilev at virtuozzo.com>
---
fs/fuse/Makefile | 3 +-
fs/fuse/kio/pcs/fuse_ktrace.h | 1 +
fs/fuse/kio/pcs/fuse_prometheus.h | 19 ++++
fs/fuse/kio/pcs/fuse_prometheus_prot.h | 15 ++++
fs/fuse/kio/pcs/pcs_cs.c | 33 +------
fs/fuse/kio/pcs/pcs_fuse_kdirect.c | 120 +++++++++++++++++++++++--
fs/fuse/kio/pcs/pcs_net_addr.c | 93 +++++++++++++++++++
fs/fuse/kio/pcs/pcs_net_addr.h | 16 ++++
fs/fuse/kio/pcs/pcs_rpc.c | 28 ++++--
fs/fuse/kio/pcs/pcs_sock_conn.c | 25 ------
fs/fuse/kio/pcs/pcs_sock_conn.h | 1 -
11 files changed, 280 insertions(+), 74 deletions(-)
create mode 100644 fs/fuse/kio/pcs/pcs_net_addr.c
create mode 100644 fs/fuse/kio/pcs/pcs_net_addr.h
diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile
index cc0d0c1d63b3..37ef32ffa13d 100644
--- a/fs/fuse/Makefile
+++ b/fs/fuse/Makefile
@@ -31,6 +31,7 @@ fuse_kio_pcs-objs := kio/pcs/pcs_fuse_kdirect.o \
kio/pcs/pcs_auth.o \
kio/pcs/pcs_rdma_io.o \
kio/pcs/pcs_rdma_rw.o \
- kio/pcs/pcs_rdma_conn.o
+ kio/pcs/pcs_rdma_conn.o \
+ kio/pcs/pcs_net_addr.o
virtiofs-y := virtio_fs.o
diff --git a/fs/fuse/kio/pcs/fuse_ktrace.h b/fs/fuse/kio/pcs/fuse_ktrace.h
index aef368fcacef..bfee4e831a70 100644
--- a/fs/fuse/kio/pcs/fuse_ktrace.h
+++ b/fs/fuse/kio/pcs/fuse_ktrace.h
@@ -29,6 +29,7 @@ struct fuse_ktrace
struct dentry *prometheus_dentry;
struct kfuse_metrics __percpu *prometheus_metrics;
u8 * __percpu buf;
+ struct fuse_error_metrics *error_metrics;
};
static inline void * fuse_trace_prepare(struct fuse_ktrace * tr, int type, int len)
diff --git a/fs/fuse/kio/pcs/fuse_prometheus.h b/fs/fuse/kio/pcs/fuse_prometheus.h
index 3f1b31c290d6..01c700bcc343 100644
--- a/fs/fuse/kio/pcs/fuse_prometheus.h
+++ b/fs/fuse/kio/pcs/fuse_prometheus.h
@@ -15,4 +15,23 @@ struct fuse_prometheus_data
struct kfuse_histogram __percpu *histo;
};
+struct fuse_rpc_error {
+ PCS_NET_ADDR_T addr;
+ u64 err[RPC_ERROR_MAX];
+};
+
+struct fuse_rpc_error_metric {
+ struct fuse_rpc_error m;
+ struct list_head list;
+};
+
+struct fuse_error_metrics {
+ spinlock_t lock;
+ u64 rpc_size;
+ struct list_head fuse_rpc_error_metric_list;
+};
+
+void fuse_rpc_error_account(struct fuse_error_metrics *metrics,
+ PCS_NET_ADDR_T const *addr, int err, u64 val);
+
#endif /* __FUSE_PROMETHEUS_H__ */
diff --git a/fs/fuse/kio/pcs/fuse_prometheus_prot.h b/fs/fuse/kio/pcs/fuse_prometheus_prot.h
index e7b6f8a52d26..d760d6fdb7d4 100644
--- a/fs/fuse/kio/pcs/fuse_prometheus_prot.h
+++ b/fs/fuse/kio/pcs/fuse_prometheus_prot.h
@@ -39,6 +39,17 @@ struct kfuse_counter {
u64 val_total;
};
+#define RPC_ERROR_ABORT 0
+#define RPC_ERROR_TOUT 1
+#define RPC_ERROR_MAX 2
+
+#define MAX_RPC_ADDR_LEN 46
+
+struct kfuse_rpc_error {
+ char address[MAX_RPC_ADDR_LEN];
+ u64 error[RPC_ERROR_MAX];
+};
+
struct kfuse_metrics {
/* Histograms are compatible with old version of proto
* between userspace and kio where the counters were skipped.
@@ -51,6 +62,10 @@ struct kfuse_metrics {
u64 stucked_reqs_cnt_8s;
u64 stucked_reqs_cnt_30s;
u64 stucked_reqs_cnt_120s;
+
+ /* All rpc errors are cleaned after full reading */
+ u32 rpc_errors_size;
+ struct kfuse_rpc_error rpc_errors[];
};
#endif /* __FUSE_PROMETHEUS_PROT__ */
diff --git a/fs/fuse/kio/pcs/pcs_cs.c b/fs/fuse/kio/pcs/pcs_cs.c
index 9d81fe8bf22c..2e7c37113178 100644
--- a/fs/fuse/kio/pcs/pcs_cs.c
+++ b/fs/fuse/kio/pcs/pcs_cs.c
@@ -24,6 +24,7 @@
#include "pcs_ioctl.h"
#include "log.h"
#include "fuse_ktrace.h"
+#include "pcs_net_addr.h"
/* Lock order: cs->lock -> css->lock (lru, hash, bl_list) */
@@ -210,38 +211,6 @@ static void add_cs(struct pcs_cs_set *csset, struct pcs_cs *cs)
hlist_add_head_rcu(&cs->hlist, &csset->ht[hash]);
}
-static inline int netaddr_cmp(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2, int ignore_port)
-{
- unsigned int d;
- size_t sz = 0;
-
- d = addr1->type - addr2->type;
- if (d)
- return d;
- d = addr1->port - addr2->port;
- if (!ignore_port && d)
- return d;
-
- switch (addr1->type) {
- case PCS_ADDRTYPE_IP:
- case PCS_ADDRTYPE_RDMA:
- sz = sizeof(struct in_addr);
- break;
- case PCS_ADDRTYPE_IP6:
- sz = sizeof(struct in6_addr);
- break;
- default:
- BUG();
- }
-
- return memcmp(addr1->address, addr2->address, sz);
-}
-
-static int pcs_netaddr_cmp(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2)
-{
- return netaddr_cmp(addr1, addr2, 0);
-}
-
/* Return locked cs */
struct pcs_cs *pcs_cs_find_create(struct pcs_cs_set *csset, PCS_NODE_ID_T *id, PCS_NET_ADDR_T *addr, int flags)
{
diff --git a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
index af008e63294c..031d6f823bad 100644
--- a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
+++ b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
@@ -34,6 +34,7 @@
#include "pcs_rpc.h"
#include "fuse_ktrace.h"
#include "fuse_prometheus.h"
+#include "pcs_net_addr.h"
unsigned int pcs_loglevel = LOG_TRACE;
module_param(pcs_loglevel, uint, 0644);
@@ -1243,6 +1244,8 @@ static void kpcs_req_send(struct fuse_req *req, bool bg)
return;
}
+static void fuse_rpc_error_metrics_clean(struct fuse_error_metrics *metrics);
+
static void fuse_trace_free(struct fuse_ktrace *tr)
{
relay_close(tr->rchan);
@@ -1256,6 +1259,11 @@ static void fuse_trace_free(struct fuse_ktrace *tr)
debugfs_remove(tr->dir);
if (tr->fc)
fuse_conn_put(tr->fc);
+
+ if (tr->error_metrics) {
+ fuse_rpc_error_metrics_clean(tr->error_metrics);
+ kfree(tr->error_metrics);
+ }
kfree(tr);
}
@@ -1359,6 +1367,60 @@ void fuse_stat_account(struct fuse_conn *fc, int op, u64 val)
}
}
+struct fuse_rpc_error_metric *fuse_rpc_error_metric_get_or_create(
+ struct fuse_error_metrics *metrics, PCS_NET_ADDR_T const *addr)
+{
+ struct fuse_rpc_error_metric *metric = NULL;
+
+ spin_lock(&metrics->lock);
+
+ list_for_each_entry(metric, &metrics->fuse_rpc_error_metric_list, list)
+ if (pcs_netaddr_cmp_ignore_port(&metric->m.addr, addr) == 0)
+ goto out;
+
+ metric = kzalloc(sizeof(*metric), GFP_KERNEL);
+ if (!metric)
+ goto out;
+
+ metric->m.addr = *addr;
+ metrics->rpc_size++;
+
+ list_add_tail(&metric->list, &metrics->fuse_rpc_error_metric_list);
+
+out:
+ spin_unlock(&metrics->lock);
+ return metric;
+}
+
+static void fuse_rpc_error_metrics_clean(struct fuse_error_metrics *metrics)
+{
+ struct fuse_rpc_error_metric *entry, *next;
+
+ spin_lock(&metrics->lock);
+ list_for_each_entry_safe(entry, next,
+ &metrics->fuse_rpc_error_metric_list, list) {
+ list_del(&entry->list);
+ kfree(entry);
+ }
+ metrics->rpc_size = 0;
+ spin_unlock(&metrics->lock);
+}
+
+void fuse_rpc_error_account(struct fuse_error_metrics *metrics,
+ PCS_NET_ADDR_T const *addr, int err, u64 val)
+{
+ struct fuse_rpc_error_metric *metric;
+
+ if (!metrics || err >= RPC_ERROR_MAX) {
+ WARN_ON_ONCE(1);
+ return;
+ }
+
+ metric = fuse_rpc_error_metric_get_or_create(metrics, addr);
+ if (metric)
+ metric->m.err[err] += val;
+}
+
static int prometheus_file_open(struct inode *inode, struct file *filp)
{
struct fuse_ktrace * tr = inode->i_private;
@@ -1397,6 +1459,26 @@ static void prometheus_req_iter(struct fuse_file *ff, struct fuse_req *req,
stats->stucked_reqs_cnt_120s++;
}
+static void fuse_rpc_collect_errors(struct fuse_error_metrics *metrics, u64 size, struct kfuse_metrics *stats)
+{
+ struct fuse_rpc_error_metric *entry;
+ u64 i;
+
+ spin_lock(&metrics->lock);
+ stats->rpc_errors_size = size;
+ i = 0;
+ list_for_each_entry(entry, &metrics->fuse_rpc_error_metric_list, list) {
+ /* If the rpc errors size was changed after allocation */
+ if (i > stats->rpc_errors_size - 1)
+ break;
+ pcs_format_netaddr_ignore_port(stats->rpc_errors[i].address,
+ MAX_RPC_ADDR_LEN, &entry->m.addr);
+ memcpy(stats->rpc_errors[i].error, entry->m.err, sizeof(stats->rpc_errors[i].error));
+ i++;
+ }
+ spin_unlock(&metrics->lock);
+}
+
/* NOTE: old versions of userspace could read only histograms */
static ssize_t prometheus_file_read(struct file *filp,
char __user *buffer,
@@ -1405,21 +1487,27 @@ static ssize_t prometheus_file_read(struct file *filp,
{
struct fuse_ktrace *tr = filp->private_data;
struct kfuse_metrics *stats;
+ u64 sz, alloc_rpc_size;
int cpu;
- if (*ppos >= sizeof(struct kfuse_metrics))
+ if (!tr->prometheus_metrics)
+ return -EINVAL;
+
+ if (!tr->error_metrics)
+ return -EINVAL;
+
+ alloc_rpc_size = tr->error_metrics->rpc_size;
+ sz = sizeof(struct kfuse_metrics) +
+ sizeof(struct kfuse_rpc_error) * alloc_rpc_size;
+ if (*ppos >= sz)
return 0;
- if (*ppos + count > sizeof(struct kfuse_metrics))
- count = sizeof(struct kfuse_metrics) - *ppos;
+ if (*ppos + count > sz)
+ count = sz - *ppos;
- stats = (void *)get_zeroed_page(GFP_KERNEL);
- BUILD_BUG_ON(sizeof(*stats) > PAGE_SIZE);
+ stats = kzalloc(sz, GFP_KERNEL);
if (!stats)
return -ENOMEM;
- if (!tr->prometheus_metrics)
- return -EINVAL;
-
for_each_possible_cpu(cpu) {
struct kfuse_metrics *m;
@@ -1446,12 +1534,18 @@ static ssize_t prometheus_file_read(struct file *filp,
pcs_kio_req_list(tr->fc, prometheus_req_iter, stats);
spin_unlock(&tr->fc->lock);
+ fuse_rpc_collect_errors(tr->error_metrics, alloc_rpc_size, stats);
+
if (copy_to_user(buffer, (char *)stats + *ppos, count))
count = -EFAULT;
else
*ppos += count;
- free_page((unsigned long)stats);
+ /* clean all rpc errors as they are fully read */
+ if (*ppos == sz && tr->error_metrics->rpc_size == alloc_rpc_size)
+ fuse_rpc_error_metrics_clean(tr->error_metrics);
+
+ kfree(stats);
return count;
}
@@ -1468,6 +1562,7 @@ static int fuse_ktrace_setup(struct fuse_conn * fc)
struct fuse_ktrace * old_tr;
struct dentry * dir;
struct kfuse_metrics __percpu * metrics;
+ struct fuse_error_metrics * error_metrics;
int cpu;
char name[16];
@@ -1517,6 +1612,13 @@ static int fuse_ktrace_setup(struct fuse_conn * fc)
tr->buf = __alloc_percpu(KTRACE_LOG_BUF_SIZE, 16);
+ error_metrics = kzalloc(sizeof(struct fuse_error_metrics), GFP_KERNEL);
+ if (!error_metrics)
+ goto err;
+ INIT_LIST_HEAD(&error_metrics->fuse_rpc_error_metric_list);
+ spin_lock_init(&error_metrics->lock);
+ tr->error_metrics = error_metrics;
+
atomic_set(&tr->refcnt, 1);
ret = -EBUSY;
diff --git a/fs/fuse/kio/pcs/pcs_net_addr.c b/fs/fuse/kio/pcs/pcs_net_addr.c
new file mode 100644
index 000000000000..f75b42e1e7c9
--- /dev/null
+++ b/fs/fuse/kio/pcs/pcs_net_addr.c
@@ -0,0 +1,93 @@
+/*
+ * fs/fuse/kio/pcs/pcs_net_addr.c
+ *
+ * Copyright (c) 2023 Virtuozzo International GmbH. All rights reserved.
+ *
+ */
+
+#include <net/sock.h>
+
+#include "pcs_types.h"
+
+int pcs_netaddr2sockaddr(PCS_NET_ADDR_T const *addr, struct sockaddr *sa, int *salen)
+{
+ BUG_ON(!sa);
+ if (addr->type == PCS_ADDRTYPE_IP || addr->type == PCS_ADDRTYPE_RDMA) {
+ struct sockaddr_in *saddr4 = (struct sockaddr_in *)sa;
+ *saddr4 = (struct sockaddr_in) {
+ .sin_family = AF_INET,
+ .sin_port = (u16)addr->port,
+ };
+ memcpy(&saddr4->sin_addr, addr->address, sizeof(saddr4->sin_addr));
+ *salen = sizeof(*saddr4);
+ } else if (addr->type == PCS_ADDRTYPE_IP6) {
+ struct sockaddr_in6 *saddr6 = (struct sockaddr_in6 *)sa;
+ *saddr6 = (struct sockaddr_in6) {
+ .sin6_family = AF_INET6,
+ .sin6_port = (u16)addr->port,
+ };
+ memcpy(&saddr6->sin6_addr, addr->address, sizeof(saddr6->sin6_addr));
+ *salen = sizeof(*saddr6);
+ } else
+ return -EINVAL;
+
+ return 0;
+}
+
+static inline int netaddr_cmp(PCS_NET_ADDR_T const *addr1,
+ PCS_NET_ADDR_T const *addr2, int ignore_port)
+{
+ unsigned int d;
+ size_t sz = 0;
+
+ d = addr1->type - addr2->type;
+ if (d)
+ return d;
+ d = addr1->port - addr2->port;
+ if (!ignore_port && d)
+ return d;
+
+ switch (addr1->type) {
+ case PCS_ADDRTYPE_IP:
+ case PCS_ADDRTYPE_RDMA:
+ sz = sizeof(struct in_addr);
+ break;
+ case PCS_ADDRTYPE_IP6:
+ sz = sizeof(struct in6_addr);
+ break;
+ default:
+ BUG();
+ }
+
+ return memcmp(addr1->address, addr2->address, sz);
+}
+
+int pcs_netaddr_cmp(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2)
+{
+ return netaddr_cmp(addr1, addr2, 0);
+}
+
+int pcs_netaddr_cmp_ignore_port(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2)
+{
+ return netaddr_cmp(addr1, addr2, 1);
+}
+
+int pcs_format_netaddr_ignore_port(char *str, int len, PCS_NET_ADDR_T const *addr)
+{
+ int ret;
+
+ switch (addr->type) {
+ case PCS_ADDRTYPE_IP:
+ case PCS_ADDRTYPE_RDMA:
+ ret = snprintf(str, len, "%pI4", addr->address);
+ break;
+ case PCS_ADDRTYPE_IP6:
+ ret = snprintf(str, len, "%pI6", addr->address);
+ break;
+ default:
+ ret = snprintf(str, len, "unknown");
+ break;
+ }
+
+ return ret;
+}
diff --git a/fs/fuse/kio/pcs/pcs_net_addr.h b/fs/fuse/kio/pcs/pcs_net_addr.h
new file mode 100644
index 000000000000..032d265472fd
--- /dev/null
+++ b/fs/fuse/kio/pcs/pcs_net_addr.h
@@ -0,0 +1,16 @@
+/*
+ * fs/fuse/kio/pcs/pcs_net_addr.h
+ *
+ * Copyright (c) 2023 Virtuozzo International GmbH. All rights reserved.
+ *
+ */
+
+#ifndef __PCS_NET_ADDR_H__
+#define __PCS_NET_ADDR_H__ 1
+
+int pcs_netaddr2sockaddr(PCS_NET_ADDR_T const *addr, struct sockaddr *sa, int *salen);
+int pcs_netaddr_cmp(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2);
+int pcs_netaddr_cmp_ignore_port(PCS_NET_ADDR_T const *addr1, PCS_NET_ADDR_T const *addr2);
+int pcs_format_netaddr_ignore_port(char *str, int len, PCS_NET_ADDR_T const *addr);
+
+#endif /* __PCS_NET_ADDR_H__ */
diff --git a/fs/fuse/kio/pcs/pcs_rpc.c b/fs/fuse/kio/pcs/pcs_rpc.c
index 7f8d4b250277..223962c75941 100644
--- a/fs/fuse/kio/pcs/pcs_rpc.c
+++ b/fs/fuse/kio/pcs/pcs_rpc.c
@@ -34,6 +34,7 @@
#include "pcs_cluster.h"
#include "log.h"
#include "fuse_ktrace.h"
+#include "fuse_prometheus.h"
static unsigned int rpc_affinity_mode = RPC_AFFINITY_RETENT;
@@ -102,6 +103,17 @@ static void rpc_del_hash(struct pcs_rpc * ep)
}
}
+static void rpc_report_error(struct pcs_rpc *ep, int err)
+{
+ if (!ep->eng || !cc_from_rpc(ep->eng)->fc->ktrace) {
+ WARN_ON_ONCE(1);
+ return;
+ }
+
+ fuse_rpc_error_account(cc_from_rpc(ep->eng)->fc->ktrace->error_metrics,
+ &ep->addr, err, 1);
+}
+
struct pcs_msg * pcs_rpc_lookup_xid(struct pcs_rpc * ep, PCS_XID_T * xid)
{
@@ -230,12 +242,15 @@ void rpc_abort(struct pcs_rpc * ep, int fatal, int error)
queue_delayed_work(cc->wq, &ep->timer_work, ep->params.holddown_timeout);
}
- while (!list_empty(&failed_list)) {
- struct pcs_msg * msg = list_first_entry(&failed_list, struct pcs_msg, list);
- list_del_init(&msg->list);
- pcs_set_rpc_error(&msg->error, error, ep);
- BUG_ON(!hlist_unhashed(&msg->kill_link));
- msg->done(msg);
+ if (!list_empty(&failed_list)) {
+ rpc_report_error(ep, RPC_ERROR_ABORT);
+ while (!list_empty(&failed_list)) {
+ struct pcs_msg * msg = list_first_entry(&failed_list, struct pcs_msg, list);
+ list_del_init(&msg->list);
+ pcs_set_rpc_error(&msg->error, error, ep);
+ BUG_ON(!hlist_unhashed(&msg->kill_link));
+ msg->done(msg);
+ }
}
if (ep->state != PCS_RPC_ABORT)
@@ -1279,6 +1294,7 @@ static void timer_work(struct work_struct *w)
case PCS_RPC_WORK: {
int err = list_empty(&ep->pending_queue) ? PCS_ERR_RESPONSE_TIMEOUT : PCS_ERR_WRITE_TIMEOUT;
+ rpc_report_error(ep, RPC_ERROR_TOUT);
FUSE_KTRACE(cc_from_rpc(ep->eng)->fc, "rpc timer expired, killing connection to " PEER_FMT ", %d",
PEER_ARGS(ep), err);
rpc_abort(ep, 0, err);
diff --git a/fs/fuse/kio/pcs/pcs_sock_conn.c b/fs/fuse/kio/pcs/pcs_sock_conn.c
index f463c1ecef9d..6acbd2bc6d2b 100644
--- a/fs/fuse/kio/pcs/pcs_sock_conn.c
+++ b/fs/fuse/kio/pcs/pcs_sock_conn.c
@@ -33,31 +33,6 @@ static inline void pcs_sock_cork(struct socket *sock)
tcp_sock_set_cork(sock->sk, true);
}
-int pcs_netaddr2sockaddr(PCS_NET_ADDR_T const* addr, struct sockaddr *sa, int *salen)
-{
- BUG_ON(!sa);
- if (addr->type == PCS_ADDRTYPE_IP || addr->type == PCS_ADDRTYPE_RDMA) {
- struct sockaddr_in *saddr4 = (struct sockaddr_in *)sa;
- *saddr4 = (struct sockaddr_in) {
- .sin_family = AF_INET,
- .sin_port = (u16)addr->port,
- };
- memcpy(&saddr4->sin_addr, addr->address, sizeof(saddr4->sin_addr));
- *salen = sizeof(*saddr4);
- } else if (addr->type == PCS_ADDRTYPE_IP6) {
- struct sockaddr_in6 *saddr6 = (struct sockaddr_in6 *)sa;
- *saddr6 = (struct sockaddr_in6) {
- .sin6_family = AF_INET6,
- .sin6_port = (u16)addr->port,
- };
- memcpy(&saddr6->sin6_addr, addr->address, sizeof(saddr6->sin6_addr));
- *salen = sizeof(*saddr6);
- } else
- return -EINVAL;
-
- return 0;
-}
-
void pcs_sockconnect_start(struct pcs_rpc *ep)
{
struct pcs_sockio *sio;
diff --git a/fs/fuse/kio/pcs/pcs_sock_conn.h b/fs/fuse/kio/pcs/pcs_sock_conn.h
index bf39a29f78e9..554958b419d0 100644
--- a/fs/fuse/kio/pcs/pcs_sock_conn.h
+++ b/fs/fuse/kio/pcs/pcs_sock_conn.h
@@ -9,6 +9,5 @@
#define _PCS_SOCK_CONN_H_ 1
void pcs_sockconnect_start(struct pcs_rpc *ep);
-int pcs_netaddr2sockaddr(PCS_NET_ADDR_T const* addr, struct sockaddr *sa, int *salen);
#endif /* _PCS_SOCK_CONN_H_ */
--
2.34.1
More information about the Devel
mailing list