[Devel] [PATCH] target/iscsi: add an ability to set io limits for iscsi targets
Andrei Vagin
avagin at virtuozzo.com
Wed Mar 21 23:49:55 MSK 2018
On Wed, Mar 14, 2018 at 11:01:43AM +0300, Vasily Averin wrote:
> See small comment below
>
> On 2018-03-14 01:48, Andrei Vagin wrote:
> > This patch add an ability to set a blkio cgroup for an iscsi target.
> >
> > When a new client is connected, the kernel creates two kernel threads and
> > run all io requests from them, this means that we can set a blkio group
> > for these threads and set io limits via this group.
> >
> > Here is an exampe how this works:
> >
> > $ TGT_PATH=/sys/kernel/config/target/iscsi/iqn.2014-06.com.vstorage\:test
> > $ CG_PATH=/sys/fs/cgroup/blkio/system.slice/vstorage-iscsi/
> >
> > # create a group
> > $ mkdir -p $CG_PATH
> >
> > # set cgroup for iscsi target
> > $ bash -c 'echo $$ > $CG_PATH/tasks &&
> > echo 1 > $TGT_PATH/tpgt_1/param/BlkioCgroup'
> > $ cat $TGT_PATH/tpgt_1/param/BlkioCgroup
> > /system.slice/vstorage-iscsi
> >
> > # attach iscsi target
> > $ IQN=iqn.2014-06.com.vstorage:test
> > $ iscsiadm -m node -T $IQN -l
> > Logging in to [iface: default, target: iqn.2014-06.com.vstorage:test,
> > portal: 10.94.120.187,3260] (multiple)
> > Login to [iface: default, target: iqn.2014-06.com.vstorage:test, portal:
> > 10.94.120.187,3260] successful.
> >
> > # check that iscsi threads in the required cgroup
> > $ ps -C iscsi_ttx
> > PID TTY TIME CMD
> > 4097 ? 00:00:00 iscsi_ttx
> > $ cat /proc/4097/cgroup | grep blkio
> > 1:blkio:/system.slice/vstorage-iscsi
> >
> > # set io limits for a target backing store device
> > $ ploop list
> > ploop15810 /mnt/vstorage/vols/iscsi/iqn.2014-06.com.vstorage:test/lun1/ploop
> > $ ls -l /dev/ploop15810
> > brw-rw---- 1 root disk 182, 252960 Mar 14 00:59 /dev/ploop15810
> > $ echo "182:252960 6291456" > $CG_PATH/blkio.throttle.read_bps_device
> >
> > # check that limits work as expected
> > $ dd if=/dev/sda of=/dev/null bs=10M count=10
> > 10+0 records in
> > 10+0 records out
> > 104857600 bytes (105 MB) copied, 17.3534 s, 6.0 MB/s
> >
> > Signed-off-by: Andrei Vagin <avagin at openvz.org>
> > ---
> > drivers/target/iscsi/iscsi_target_configfs.c | 46 ++++++++++++++++++++++++++++
> > drivers/target/iscsi/iscsi_target_login.c | 34 +++++++++++++++++++-
> > drivers/target/iscsi/iscsi_target_tpg.c | 36 ++++++++++++++++++++++
> > drivers/target/iscsi/iscsi_target_tpg.h | 2 ++
> > include/target/iscsi/iscsi_target_core.h | 2 ++
> > kernel/cgroup.c | 1 +
> > 6 files changed, 120 insertions(+), 1 deletion(-)
> >
> > diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c
> > index e6631ab..6d754cf 100644
> > --- a/drivers/target/iscsi/iscsi_target_configfs.c
> > +++ b/drivers/target/iscsi/iscsi_target_configfs.c
> > @@ -1412,6 +1412,51 @@ TPG_PARAM_ATTR(IFMarkInt, S_IRUGO | S_IWUSR);
> > DEF_TPG_PARAM(OFMarkInt);
> > TPG_PARAM_ATTR(OFMarkInt, S_IRUGO | S_IWUSR);
> >
> > +static ssize_t iscsi_tpg_param_show_BlkioCgroup(
> > + struct se_portal_group *se_tpg,
> > + char *page)
> > +{
> > + struct iscsi_portal_group *tpg = container_of(se_tpg,
> > + struct iscsi_portal_group, tpg_se_tpg);
> > + ssize_t rb;
> > +
> > + if (iscsit_get_tpg(tpg) < 0)
> > + return -EINVAL;
> > +
> > + rb = iscsit_ta_tpg_show_blkcg(tpg, page);
> > + iscsit_put_tpg(tpg);
> > + return rb;
> > +}
> > +
> > +static ssize_t iscsi_tpg_param_store_BlkioCgroup(
> > + struct se_portal_group *se_tpg,
> > + const char *page,
> > + size_t count)
> > +{
> > + struct iscsi_portal_group *tpg = container_of(se_tpg,
> > + struct iscsi_portal_group, tpg_se_tpg);
> > + u32 val;
> > + int ret;
> > +
> > + if (iscsit_get_tpg(tpg) < 0)
> > + return -EINVAL;
> > +
> > + ret = kstrtou32(page, 0, &val);
> > + if (ret)
> > + goto out;
> > + ret = iscsit_ta_tpg_set_blkcg(tpg, val);
> > + if (ret < 0)
> > + goto out;
> > +
> > + iscsit_put_tpg(tpg);
> > + return count;
> > +out:
> > + iscsit_put_tpg(tpg);
> > + return ret;
> > +}
> > +
> > +TPG_PARAM_ATTR(BlkioCgroup, S_IRUGO | S_IWUSR);
> > +
> > static struct configfs_attribute *lio_target_tpg_param_attrs[] = {
> > &iscsi_tpg_param_AuthMethod.attr,
> > &iscsi_tpg_param_HeaderDigest.attr,
> > @@ -1434,6 +1479,7 @@ static struct configfs_attribute *lio_target_tpg_param_attrs[] = {
> > &iscsi_tpg_param_OFMarker.attr,
> > &iscsi_tpg_param_IFMarkInt.attr,
> > &iscsi_tpg_param_OFMarkInt.attr,
> > + &iscsi_tpg_param_BlkioCgroup.attr,
> > NULL,
> > };
> >
> > diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
> > index c20b561..4f59416 100644
> > --- a/drivers/target/iscsi/iscsi_target_login.c
> > +++ b/drivers/target/iscsi/iscsi_target_login.c
> > @@ -20,6 +20,7 @@
> > #include <linux/string.h>
> > #include <linux/kthread.h>
> > #include <linux/idr.h>
> > +#include <linux/cgroup.h>
> > #include <scsi/iscsi_proto.h>
> > #include <target/target_core_base.h>
> > #include <target/target_core_fabric.h>
> > @@ -712,8 +713,18 @@ static void iscsi_post_login_start_timers(struct iscsi_conn *conn)
> >
> > int iscsit_start_kthreads(struct iscsi_conn *conn)
> > {
> > + struct iscsi_portal_group *tpg = conn->tpg;
> > + struct cgroup_subsys_state *blk_css = NULL;
> > int ret = 0;
> >
> > + if (iscsit_get_tpg(tpg) < 0)
> > + return -EINVAL;
> > + if (tpg->blk_css) {
> > + blk_css = tpg->blk_css;
> > + css_get(blk_css);
> > + }
> > + iscsit_put_tpg(tpg);
> > +
> > spin_lock(&iscsit_global->ts_bitmap_lock);
> > conn->bitmap_id = bitmap_find_free_region(iscsit_global->ts_bitmap,
> > ISCSIT_BITMAP_BITS, get_order(1));
> > @@ -722,7 +733,8 @@ int iscsit_start_kthreads(struct iscsi_conn *conn)
> > if (conn->bitmap_id < 0) {
> > pr_err("bitmap_find_free_region() failed for"
> > " iscsit_start_kthreads()\n");
> > - return -ENOMEM;
> > + ret = -ENOMEM;
> > + goto put_blk_css;
> > }
> >
> > conn->tx_thread = kthread_run(iscsi_target_tx_thread, conn,
> > @@ -732,6 +744,11 @@ int iscsit_start_kthreads(struct iscsi_conn *conn)
> > ret = PTR_ERR(conn->tx_thread);
> > goto out_bitmap;
> > }
> > + if (blk_css) {
> > + ret = cgroup_kernel_attach(blk_css->cgroup, conn->tx_thread);
> > + if (ret < 0)
> > + goto out_tx;
> > + }
> > conn->tx_thread_active = true;
> >
> > conn->rx_thread = kthread_run(iscsi_target_rx_thread, conn,
> > @@ -741,9 +758,21 @@ int iscsit_start_kthreads(struct iscsi_conn *conn)
> > ret = PTR_ERR(conn->rx_thread);
> > goto out_tx;
> > }
> > + if (blk_css) {
> > + ret = cgroup_kernel_attach(blk_css->cgroup, conn->rx_thread);
> > + if (ret < 0)
> > + goto out_rx;
> > + }
> > conn->rx_thread_active = true;
> >
> > + if (blk_css)
> > + css_put(blk_css);
> > +
> > return 0;
> > +out_rx:
> > + send_sig(SIGINT, conn->rx_thread, 1);
> > + kthread_stop(conn->rx_thread);
> > + conn->rx_thread_active = false;
>
> Seems rx_thread_active = false can be skipped here,
> it is set to true in success case only,
> after goto to this label.
If in a future we will add one more out label, we will need
rx_thread_active = false, so I added it in this patch to reduce a chance
to forget about it next time.
>
> > out_tx:
> > send_sig(SIGINT, conn->tx_thread, 1);
> > kthread_stop(conn->tx_thread);
> > @@ -753,6 +782,9 @@ out_bitmap:
> > bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
> > get_order(1));
> > spin_unlock(&iscsit_global->ts_bitmap_lock);
> > +put_blk_css:
> > + if (blk_css)
> > + css_put(blk_css);
> > return ret;
> > }
> >
> > diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c
> > index 2426b0b..792d3ee 100644
> > --- a/drivers/target/iscsi/iscsi_target_tpg.c
> > +++ b/drivers/target/iscsi/iscsi_target_tpg.c
> > @@ -16,6 +16,7 @@
> > * GNU General Public License for more details.
> > ******************************************************************************/
> >
> > +#include <linux/cgroup.h>
> > #include <target/target_core_base.h>
> > #include <target/target_core_fabric.h>
> > #include <target/target_core_configfs.h>
> > @@ -303,6 +304,9 @@ int iscsit_tpg_del_portal_group(
> > list_del(&tpg->tpg_list);
> > spin_unlock(&tiqn->tiqn_tpg_lock);
> >
> > + if (tpg->blk_css)
> > + css_put(tpg->blk_css);
> > +
> > pr_debug("CORE[%s]_TPG[%hu] - Deleted iSCSI Target Portal Group\n",
> > tiqn->tiqn, tpg->tpgt);
> >
> > @@ -896,3 +900,35 @@ int iscsit_ta_tpg_enabled_sendtargets(
> >
> > return 0;
> > }
> > +
> > +int iscsit_ta_tpg_set_blkcg(struct iscsi_portal_group *tpg, u32 flag)
> > +{
> > + struct cgroup_subsys_state *css;
> > +
> > + if (flag != 1)
> > + return -EINVAL;
> > +
> > + css = task_get_css(current, blkio_subsys_id);
> > + if (tpg->blk_css)
> > + css_put(tpg->blk_css);
> > + tpg->blk_css = css;
> > +
> > + return 0;
> > +}
> > +
> > +int iscsit_ta_tpg_show_blkcg(struct iscsi_portal_group *tpg, char *page)
> > +{
> > + int rb;
> > +
> > + if (tpg->blk_css) {
> > + rb = cgroup_path(tpg->blk_css->cgroup, page, PAGE_SIZE - 1);
> > + if (rb == 0)
> > + rb = strlen(page);
> > + page[rb] = '\n';
> > + page[rb + 1] = 0;
> > + rb++;
> > + } else
> > + rb = 0;
> > +
> > + return rb;
> > +}
> > diff --git a/drivers/target/iscsi/iscsi_target_tpg.h b/drivers/target/iscsi/iscsi_target_tpg.h
> > index b2f49c0..416b929 100644
> > --- a/drivers/target/iscsi/iscsi_target_tpg.h
> > +++ b/drivers/target/iscsi/iscsi_target_tpg.h
> > @@ -40,5 +40,7 @@ extern int iscsit_ta_demo_mode_discovery(struct iscsi_portal_group *, u32);
> > extern int iscsit_ta_default_erl(struct iscsi_portal_group *, u32);
> > extern int iscsit_ta_t10_pi(struct iscsi_portal_group *, u32);
> > extern int iscsit_ta_tpg_enabled_sendtargets(struct iscsi_portal_group *, u32);
> > +extern int iscsit_ta_tpg_set_blkcg(struct iscsi_portal_group *tpg, u32 flag);
> > +extern int iscsit_ta_tpg_show_blkcg(struct iscsi_portal_group *tpg, char *page);
> >
> > #endif /* ISCSI_TARGET_TPG_H */
> > diff --git a/include/target/iscsi/iscsi_target_core.h b/include/target/iscsi/iscsi_target_core.h
> > index 3e3b2f9..83eb63b 100644
> > --- a/include/target/iscsi/iscsi_target_core.h
> > +++ b/include/target/iscsi/iscsi_target_core.h
> > @@ -838,6 +838,8 @@ struct iscsi_portal_group {
> > struct iscsi_tiqn *tpg_tiqn;
> > struct list_head tpg_gnp_list;
> > struct list_head tpg_list;
> > +
> > + struct cgroup_subsys_state *blk_css;
> > } ____cacheline_aligned;
> >
> > struct iscsi_wwn_stat_grps {
> > diff --git a/kernel/cgroup.c b/kernel/cgroup.c
> > index 11c547f..0fbb37e 100644
> > --- a/kernel/cgroup.c
> > +++ b/kernel/cgroup.c
> > @@ -5924,6 +5924,7 @@ int cgroup_kernel_attach(struct cgroup *cgrp, struct task_struct *tsk)
> > mutex_unlock(&cgroup_mutex);
> > return ret;
> > }
> > +EXPORT_SYMBOL_GPL(cgroup_kernel_attach);
> >
> > void cgroup_kernel_close(struct cgroup *cgrp)
> > {
> >
More information about the Devel
mailing list