[Devel] [PATCH RFC 2/2] fs/fuse kio: drop external cs connection
Pavel Butsykin
pbutsykin at virtuozzo.com
Mon Jul 8 17:57:25 MSK 2019
Remove the old implementation of creating cs connection through usermode.
Signed-off-by: Pavel Butsykin <pbutsykin at virtuozzo.com>
---
fs/fuse/kio/pcs/pcs_cluster.h | 1 -
fs/fuse/kio/pcs/pcs_cs.c | 17 +----
fs/fuse/kio/pcs/pcs_fuse_kdirect.c | 127 -------------------------------------
fs/fuse/kio/pcs/pcs_ioctl.h | 11 ----
fs/fuse/kio/pcs/pcs_rpc.c | 35 ----------
fs/fuse/kio/pcs/pcs_rpc.h | 1 -
fs/fuse/kio/pcs/pcs_sock_conn.c | 2 +-
fs/fuse/kio/pcs/pcs_sock_io.c | 81 ++---------------------
fs/fuse/kio/pcs/pcs_sock_io.h | 4 +-
9 files changed, 10 insertions(+), 269 deletions(-)
diff --git a/fs/fuse/kio/pcs/pcs_cluster.h b/fs/fuse/kio/pcs/pcs_cluster.h
index ba41271c78f2..b7e3de8f1a25 100644
--- a/fs/fuse/kio/pcs/pcs_cluster.h
+++ b/fs/fuse/kio/pcs/pcs_cluster.h
@@ -119,7 +119,6 @@ void pcs_cc_fini(struct pcs_cluster_core *cc);
void pcs_fuse_prep_io(struct pcs_fuse_req *r, unsigned short type, off_t offset, size_t size, u64 aux);
void pcs_fuse_prep_fallocate(struct pcs_fuse_req *r);
-int fuse_pcs_csconn_send(struct fuse_conn *fc, struct pcs_rpc *ep, int flags);
static inline void pcs_cc_set_abort_timeout(struct pcs_cluster_core *cc, int timeout)
diff --git a/fs/fuse/kio/pcs/pcs_cs.c b/fs/fuse/kio/pcs/pcs_cs.c
index 8150b3390691..9ac5246cfa63 100644
--- a/fs/fuse/kio/pcs/pcs_cs.c
+++ b/fs/fuse/kio/pcs/pcs_cs.c
@@ -35,8 +35,7 @@ static void cs_aborting(struct pcs_rpc *ep, int error);
static struct pcs_msg *cs_get_hdr(struct pcs_rpc *ep, struct pcs_rpc_hdr *h);
static int cs_input(struct pcs_rpc *ep, struct pcs_msg *msg);
static void cs_keep_waiting(struct pcs_rpc *ep, struct pcs_msg *req, struct pcs_msg *msg);
-static void cs_user_connect(struct pcs_rpc *ep);
-static void cs_kernel_connect(struct pcs_rpc *ep);
+static void cs_connect(struct pcs_rpc *ep);
static void pcs_cs_isolate(struct pcs_cs *cs, struct list_head *dispose);
static void pcs_cs_destroy(struct pcs_cs *cs);
@@ -45,7 +44,7 @@ struct pcs_rpc_ops cn_rpc_ops = {
.get_hdr = cs_get_hdr,
.state_change = cs_aborting,
.keep_waiting = cs_keep_waiting,
- .connect = cs_kernel_connect,
+ .connect = cs_connect,
};
static int pcs_cs_percpu_stat_alloc(struct pcs_cs *cs)
@@ -434,7 +433,7 @@ static void cs_get_read_response_iter(struct pcs_msg *msg, int offset, struct io
}
}
-static void cs_kernel_connect(struct pcs_rpc *ep)
+static void cs_connect(struct pcs_rpc *ep)
{
if (ep->flags & PCS_RPC_F_LOCAL) {
char path[128];
@@ -473,16 +472,6 @@ fail:
return;
}
-__maybe_unused static void cs_user_connect(struct pcs_rpc *ep)
-{
- struct pcs_cluster_core *cc = cc_from_rpc(ep->eng);
- struct pcs_fuse_cluster *pfc = pcs_cluster_from_cc(cc);
-
- ep->state = PCS_RPC_CONNECT;
- if (fuse_pcs_csconn_send(pfc->fc, ep, PCS_IOC_CS_OPEN))
- pcs_rpc_reset(ep);
-}
-
static struct pcs_msg *cs_get_hdr(struct pcs_rpc *ep, struct pcs_rpc_hdr *h)
{
struct pcs_msg *msg, *resp;
diff --git a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
index 07aa683f2245..d227a1586ee9 100644
--- a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
+++ b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
@@ -565,132 +565,6 @@ int fuse_map_resolve(struct pcs_map_entry *m, int direction)
return 0;
}
-static void pfocess_pcs_csconn_work(struct work_struct *w)
-{
- struct pcs_fuse_work *work = container_of(w, struct pcs_fuse_work, work);
- struct pcs_rpc *ep = (struct pcs_rpc *)work->ctx;
- struct socket *sock = (struct socket *)work->ctx2;
- BUG_ON(!ep);
-
- if (pcs_if_error(&work->status)) {
- mutex_lock(&ep->mutex);
- pcs_rpc_reset(ep);
- mutex_unlock(&ep->mutex);
- TRACE(PEER_FMT" fail with %d\n", PEER_ARGS(ep), work->status.value);
- } else {
- if (sock)
- rpc_connect_done(ep, sock);
- }
- pcs_rpc_put(ep);
- kfree(work);
-}
-
-static void process_pcs_csconn_reply(struct fuse_conn *fc, struct fuse_req *req)
-{
- struct pcs_ioc_csconn *csconn = (struct pcs_ioc_csconn *)req->in.args[1].value;
- struct fuse_ioctl_out *arg = &req->misc.ioctl.out;
- struct pcs_fuse_work *work = (struct pcs_fuse_work*) req->misc.ioctl.ctx;
- int is_open = csconn->flags & PCS_IOC_CS_OPEN;
-
- if (req->out.h.error || arg->result < 0) {
- pcs_set_local_error(&work->status, PCS_ERR_PROTOCOL);
- goto out;
- }
- /* Grab socket from caller's context (fuse-evloop) and do the rest in kwork */
- if (is_open) {
- struct socket *sock;
- struct file* filp;
- int err;
-
- filp = fget((unsigned int)arg->result);
- arg->result = 0;
- if (!filp) {
- pcs_set_local_error(&work->status, PCS_ERR_PROTOCOL);
- goto out;
- }
- sock = sock_from_file(filp, &err);
- if (!sock) {
- fput(filp);
- pcs_set_local_error(&work->status, PCS_ERR_PROTOCOL);
- } else
- TRACE("id: "NODE_FMT" sock:%p\n", NODE_ARGS(csconn->id), sock);
- work->ctx2 = sock;
- }
-out:
- kfree(csconn);
- pcs_fuse_reply_handle(fc, req);
-
-}
-
-int fuse_pcs_csconn_send(struct fuse_conn *fc, struct pcs_rpc *ep, int flags)
-{
- struct fuse_req *req;
- struct fuse_ioctl_in *inarg;
- struct fuse_ioctl_out *outarg;
- struct pcs_ioc_csconn *csconn;
- struct pcs_fuse_work *reply_work;
-
- /* Socket must being freed from kernelspace before requesting new one*/
- BUG_ON(!(flags & PCS_IOC_CS_REOPEN));
-
- TRACE("start %s cmd:%ld id:%lld flags:%x\n", __FUNCTION__,
- PCS_IOC_CSCONN, ep->peer_id.val, flags);
-
- csconn = kzalloc(sizeof(*csconn), GFP_NOIO);
- if (!csconn)
- return -ENOMEM;
-
- reply_work = kzalloc(sizeof(*reply_work), GFP_NOIO);
- if (!reply_work) {
- kfree(csconn);
- return -ENOMEM;
- }
-
- req = fuse_get_nonblock_req_for_background(fc, 0);
- if (IS_ERR(req)) {
- kfree(csconn);
- kfree(reply_work);
- return PTR_ERR(req);
- }
-
- __set_bit(FR_KIO_INTERNAL, &req->flags);
- memset(&req->misc.ioctl, 0, sizeof(req->misc.ioctl));
- inarg = &req->misc.ioctl.in;
- outarg = &req->misc.ioctl.out;
-
- inarg->cmd = PCS_IOC_CSCONN;
- inarg->fh = 0;
- inarg->arg = 0;
- inarg->flags = 0;
-
- csconn->id.val = ep->peer_id.val;
- memcpy(&csconn->address, &ep->addr, sizeof(ep->addr));
- csconn->flags = flags;
-
- if (ep->flags & PCS_RPC_F_LOCAL)
- csconn->address.type = PCS_ADDRTYPE_UNIX;
-
- req->in.h.opcode = FUSE_IOCTL;
- req->in.numargs = 2;
- req->in.args[0].size = sizeof(*inarg);
- req->in.args[0].value = inarg;
- req->in.args[1].size = sizeof(*csconn);
- req->in.args[1].value = csconn;
-
- req->out.numargs = 1;
- req->out.args[0].size = sizeof(*outarg);
- req->out.args[0].value = outarg;
-
- INIT_WORK(&reply_work->work, pfocess_pcs_csconn_work);
- reply_work->ctx = pcs_rpc_get(ep);
- reply_work->ctx2 = NULL; /* return socket should be here */
- req->misc.ioctl.ctx = reply_work;
-
- req->end = process_pcs_csconn_reply;
- fuse_request_send_background(fc, req);
-
- return 0;
-}
struct fuse_req *kpcs_req_alloc(struct fuse_conn *fc,
unsigned npages, gfp_t flags)
@@ -1252,7 +1126,6 @@ static int pcs_kio_classify_req(struct fuse_conn *fc, struct fuse_req *req, bool
case PCS_IOC_NOWRITEDELAY:
return -EOPNOTSUPP;
case PCS_IOC_INIT_KDIRECT:
- case PCS_IOC_CSCONN:
case PCS_IOC_GETFILEINFO:
case PCS_IOC_KDIRECT_CLAIM:
case PCS_IOC_KDIRECT_RELEASE:
diff --git a/fs/fuse/kio/pcs/pcs_ioctl.h b/fs/fuse/kio/pcs/pcs_ioctl.h
index b86d670466cc..3c646b1df489 100644
--- a/fs/fuse/kio/pcs/pcs_ioctl.h
+++ b/fs/fuse/kio/pcs/pcs_ioctl.h
@@ -69,23 +69,12 @@ struct pcs_ioc_getmap
struct pcs_cs_info cs[0]; /* Array of CS including root */
};
-struct pcs_ioc_csconn
-{
- PCS_NODE_ID_T id;
- PCS_NET_ADDR_T address;
- u32 flags;
-#define PCS_IOC_CS_OPEN 0x1
-#define PCS_IOC_CS_CLOSE 0x2
-#define PCS_IOC_CS_REOPEN (PCS_IOC_CS_OPEN|PCS_IOC_CS_CLOSE)
-};
-
#define PCS_IOC_NOCSUMONREAD _IOW('V',3,u32)
# define PCS_IOC_NOCSUMONREAD_DATA 1
# define PCS_IOC_NOCSUMONREAD_METADATA 2
#define PCS_IOC_NOWRITEDELAY _IOW('V',4,u32)
#define PCS_IOC_INIT_KDIRECT _IOR('V',32, struct pcs_ioc_init_kdirect)
-#define PCS_IOC_CSCONN _IOR('V',33, struct pcs_ioc_csconn)
#define PCS_IOC_GETFILEINFO _IOR('V',34, struct pcs_ioc_fileinfo)
#define PCS_IOC_KDIRECT_CLAIM _IO('V',35)
#define PCS_IOC_KDIRECT_RELEASE _IO('V',36)
diff --git a/fs/fuse/kio/pcs/pcs_rpc.c b/fs/fuse/kio/pcs/pcs_rpc.c
index 71bc7e0a806d..e047e4aec079 100644
--- a/fs/fuse/kio/pcs/pcs_rpc.c
+++ b/fs/fuse/kio/pcs/pcs_rpc.c
@@ -1398,41 +1398,6 @@ void pcs_rpc_set_memlimits(struct pcs_rpc_engine * eng, u64 thresh, u64 limit)
eng->mem_limit = limit;
}
-void rpc_connect_done(struct pcs_rpc *ep, struct socket *sock)
-{
- struct pcs_sockio * sio;
-
- mutex_lock(&ep->mutex);
-
- TRACE(PEER_FMT " ->state:%d sock:%p\n", PEER_ARGS(ep), ep->state, sock);
- cancel_delayed_work(&ep->timer_work);
-
- if (ep->state != PCS_RPC_CONNECT) {
- FUSE_KLOG(cc_from_rpc(ep->eng)->fc, LOG_ERR, "Invalid state: %u", ep->state);
- BUG();
- }
-
- sio = pcs_sockio_init(sock, ep->params.alloc_hdr_size,
- sizeof(struct pcs_rpc_hdr));
- if (sio == NULL)
- BUG();
-
- ep->conn = &sio->ioconn;
- sio->parent = pcs_rpc_get(ep);
- sio->get_msg = rpc_get_hdr;
- sio->eof = rpc_eof_cb;
- //pcs_ioconn_register(ep->conn);
- if (ep->gc)
- list_lru_add(&ep->gc->lru, &ep->lru_link);
-
- if (ep->flags & PCS_RPC_F_CLNT_PEER_ID)
- ep->flags |= PCS_RPC_F_PEER_ID;
- ep->state = PCS_RPC_APPWAIT;
- pcs_rpc_enable(ep, 0);
- mutex_unlock(&ep->mutex);
-
-}
-
static const char *s_rpc_state_names[] = {
[PCS_RPC_UNCONN] = "UNCONN", /* Not connected */
[PCS_RPC_CONNECT] = "CONNECT", /* Connect in progress */
diff --git a/fs/fuse/kio/pcs/pcs_rpc.h b/fs/fuse/kio/pcs/pcs_rpc.h
index 2699657fac14..64563b4e1a83 100644
--- a/fs/fuse/kio/pcs/pcs_rpc.h
+++ b/fs/fuse/kio/pcs/pcs_rpc.h
@@ -299,7 +299,6 @@ int pcs_cluster_id_eq(PCS_CLUSTER_ID_T *id1, PCS_CLUSTER_ID_T *id2);
void rpc_trace_health(struct pcs_rpc * ep);
void pcs_rpc_enumerate_rpc(struct pcs_rpc_engine *eng, void (*cb)(struct pcs_rpc *ep, void *arg), void *arg);
void pcs_rpc_set_sock(struct pcs_rpc *ep, struct pcs_sockio * sio);
-void rpc_connect_done(struct pcs_rpc *ep, struct socket *sock);
void pcs_rpc_enable(struct pcs_rpc * ep, int error);
void rpc_eof_cb(struct pcs_sockio *sio);
diff --git a/fs/fuse/kio/pcs/pcs_sock_conn.c b/fs/fuse/kio/pcs/pcs_sock_conn.c
index c924ed439a27..6f6dd5408147 100644
--- a/fs/fuse/kio/pcs/pcs_sock_conn.c
+++ b/fs/fuse/kio/pcs/pcs_sock_conn.c
@@ -607,7 +607,7 @@ void pcs_sockconnect_start(struct pcs_rpc *ep)
sio->eof = rpc_eof_cb;
sio->send_timeout = PCS_SIO_TIMEOUT;
sio->ioconn.socket = sock;
- sio->ioconn.destruct = pcs_sock_internal_ioconn_destruct;
+ sio->ioconn.destruct = pcs_sock_ioconn_destruct;
if (ep->gc)
list_lru_add(&ep->gc->lru, &ep->lru_link);
diff --git a/fs/fuse/kio/pcs/pcs_sock_io.c b/fs/fuse/kio/pcs/pcs_sock_io.c
index 007df3b77bdd..a4b8d72fe244 100644
--- a/fs/fuse/kio/pcs/pcs_sock_io.c
+++ b/fs/fuse/kio/pcs/pcs_sock_io.c
@@ -36,11 +36,6 @@ void sio_push(struct pcs_sockio * sio)
}
}
-//// TODO:dmonakhov@ implement unregister and close,
-//// socket close must being synchronized with userspace THINK
-//// caseA: userspace close socket and wait for kernelspace
-//// caseB: kernelspace want to close socket and have to somehow
-//// notify about this to userspace (NEW API REQUIRED)
static void pcs_ioconn_unregister(struct pcs_ioconn *ioconn)
{
if (!test_bit(PCS_IOCONN_BF_DEAD, &ioconn->flags))
@@ -499,11 +494,11 @@ static void sio_destroy_rcu(struct rcu_head *head)
kfree(sio);
}
-static void pcs_sock_ioconn_destruct(struct pcs_ioconn *ioconn, bool internal)
+void pcs_sock_ioconn_destruct(struct pcs_ioconn *ioconn)
{
struct pcs_sockio * sio = sio_from_ioconn(ioconn);
- TRACE("Sock destruct_cb, sio: %p, internal: %d", sio, internal);
+ TRACE("Sock destruct_cb, sio: %p", sio);
BUG_ON(sio->current_msg);
BUG_ON(!list_empty(&sio->write_queue));
@@ -511,10 +506,7 @@ static void pcs_sock_ioconn_destruct(struct pcs_ioconn *ioconn, bool internal)
if (ioconn->socket) {
pcs_restore_sockets(ioconn);
- if (internal)
- sock_release(ioconn->socket);
- else
- fput(ioconn->socket->file);
+ sock_release(ioconn->socket);
ioconn->socket = NULL;
}
@@ -522,21 +514,11 @@ static void pcs_sock_ioconn_destruct(struct pcs_ioconn *ioconn, bool internal)
call_rcu(&sio->rcu, sio_destroy_rcu);
}
-void pcs_sock_internal_ioconn_destruct(struct pcs_ioconn *ioconn)
-{
- pcs_sock_ioconn_destruct(ioconn, true);
-}
-
-void pcs_sock_external_ioconn_destruct(struct pcs_ioconn *ioconn)
-{
- pcs_sock_ioconn_destruct(ioconn, false);
-}
-
static void pcs_sk_kick_queue(struct sock *sk)
{
struct pcs_sockio *sio;
- smp_rmb(); /* Pairs with smp_wmb() in pcs_sockio_init() */
+ smp_rmb(); /* Pairs with smp_wmb() in pcs_sockconnect_start() */
rcu_read_lock();
sio = rcu_dereference_sk_user_data(sk);
@@ -562,7 +544,7 @@ void pcs_sk_error_report(struct sock *sk)
{
struct pcs_sockio *sio;
- smp_rmb(); /* Pairs with smp_wmb() in pcs_sockio_init() */
+ smp_rmb(); /* Pairs with smp_wmb() in pcs_sockconnect_start() */
rcu_read_lock();
sio = rcu_dereference_sk_user_data(sk);
@@ -580,59 +562,6 @@ unlock:
rcu_read_unlock();
}
-struct pcs_sockio * pcs_sockio_init(struct socket *sock,
- int alloc_max, int hdr_max)
-{
- struct pcs_sockio * sio;
- struct sock *sk;
-
- sio = kzalloc(sizeof(struct pcs_sockio) + alloc_max, GFP_NOIO);
- if (!sio)
- return NULL;
-
- INIT_LIST_HEAD(&sio->write_queue);
- iov_iter_init_bad(&sio->read_iter);
- iov_iter_init_bad(&sio->write_iter);
- sio->hdr_max = hdr_max;
- sio->flags = sock->sk->sk_family != AF_UNIX ? PCS_SOCK_F_CORK : 0;
-
- //// TODO:dmonakhov init ioconn here
- INIT_LIST_HEAD(&sio->ioconn.list);
- sk = sock->sk;
- write_lock_bh(&sk->sk_callback_lock);
-
- /*
- * Backup original callbaks.
- * TCP and unix sockets do not have sk_user_data set.
- * So we avoid taking sk_callback_lock in callbacks,
- * since this seems to be able to result in performance.
- */
- WARN_ON_ONCE(sk->sk_user_data);
- sio->ioconn.orig.user_data = sk->sk_user_data;
- sio->ioconn.orig.data_ready = sk->sk_data_ready;
- sio->ioconn.orig.write_space = sk->sk_write_space;
- sio->ioconn.orig.error_report = sk->sk_error_report;
- //sio->ioconn.orig_state_change = sk->sk_state_change;
-
- //sock->sk->sk_state_change = pcs_state_chage;
-
- sk->sk_sndtimeo = PCS_SIO_TIMEOUT;
- sk->sk_allocation = GFP_NOFS;
- sio->send_timeout = PCS_SIO_TIMEOUT;
- sio->ioconn.socket = sock;
- sio->ioconn.destruct = pcs_sock_external_ioconn_destruct;
-
- rcu_assign_sk_user_data(sk, sio);
- smp_wmb(); /* Pairs with smp_rmb() in callbacks */
- sk->sk_data_ready = pcs_sk_data_ready;
- sk->sk_write_space = pcs_sk_write_space;
- sk->sk_error_report = pcs_sk_error_report;
- write_unlock_bh(&sock->sk->sk_callback_lock);
-
- pcs_clear_error(&sio->error);
- return sio;
-}
-
void pcs_sockio_start(struct pcs_sockio * sio)
{
//// TODO: dmonakhov
diff --git a/fs/fuse/kio/pcs/pcs_sock_io.h b/fs/fuse/kio/pcs/pcs_sock_io.h
index b50c784ee3b9..52c6f315effd 100644
--- a/fs/fuse/kio/pcs/pcs_sock_io.h
+++ b/fs/fuse/kio/pcs/pcs_sock_io.h
@@ -147,7 +147,6 @@ struct pcs_sockio
#define sio_from_ioconn(conn) container_of(conn, struct pcs_sockio, ioconn)
-struct pcs_sockio * pcs_sockio_init(struct socket* sock, int alloc_max, int hdr_max);
void pcs_sockio_start(struct pcs_sockio * sio);
void pcs_sock_sendmsg(struct pcs_sockio * sio, struct pcs_msg *msg);
int pcs_sock_cancel_msg(struct pcs_msg * msg);
@@ -171,8 +170,7 @@ struct pcs_msg * pcs_cow_msg(struct pcs_msg * msg, int data_len);
void pcs_clone_done(struct pcs_msg * msg);
void pcs_free_msg(struct pcs_msg * msg);
void pcs_get_iter_inline(struct pcs_msg * msg, int offset, struct iov_iter*it);
-void pcs_sock_internal_ioconn_destruct(struct pcs_ioconn *ioconn);
-void pcs_sock_external_ioconn_destruct(struct pcs_ioconn *ioconn);
+void pcs_sock_ioconn_destruct(struct pcs_ioconn *ioconn);
static inline void * msg_inline_head(struct pcs_msg * msg)
{
--
2.15.1
More information about the Devel
mailing list