[Devel] [PATCH 3/4] target: add histogram for LUN statistics
Andrei Vagin
avagin at virtuozzo.com
Thu Dec 28 05:06:18 MSK 2017
On Wed, Dec 20, 2017 at 01:10:13PM +0300, Andrey Grafin wrote:
> This patch adds histogram statistics to scsi_ports_stats.
> Histogram can be obtained and configured via config_fs.
> Histogram measurement unit is usec.
>
> Histogram usage:
> 1. Configure histogram '| 1 ms | 10 ms | 15 ms | largest values |'
> `echo "1000 10000 15000" > target/iscsi/iqn.2003-01.org.linux-iscsi.localhost.x8664\:sn.fdee138936b9/tpgt_1/lun/lun_3/statistics/scsi_tgt_port/write_hist`
>
> 2. Obtain histogram stats
> `cat /target/iscsi/iqn.2003-01.org.linux-iscsi.localhost.x8664\:sn.fdee138936b9/tpgt_1/lun/lun_3/statistics/scsi_tgt_port/write_hist`
>
> 3. Stop histogram stats
> `echo "" > /target/iscsi/iqn.2003-01.org.linux-iscsi.localhost.x8664\:sn.fdee138936b9/tpgt_1/lun/lun_3/statistics/scsi_tgt_port/write_hist`
>
> Signed-off-by: Andrey Grafin <Andrey.Grafin at acronis.com>
> ---
> drivers/target/target_core_stat.c | 207 +++++++++++++++++++++++++++++++++
> drivers/target/target_core_tpg.c | 27 +++++
> drivers/target/target_core_transport.c | 23 ++++
> include/target/target_core_base.h | 16 +++
> 4 files changed, 273 insertions(+)
>
> diff --git a/drivers/target/target_core_stat.c b/drivers/target/target_core_stat.c
> index e88e29612db..6445869a22b 100644
> --- a/drivers/target/target_core_stat.c
> +++ b/drivers/target/target_core_stat.c
> @@ -33,6 +33,7 @@
> #include <linux/proc_fs.h>
> #include <linux/seq_file.h>
> #include <linux/configfs.h>
> +#include <linux/ctype.h>
> #include <scsi/scsi.h>
> #include <scsi/scsi_device.h>
> #include <scsi/scsi_host.h>
> @@ -699,6 +700,209 @@ static ssize_t target_stat_scsi_tgt_port_show_attr_##_name( \
> return ret; \
> }
>
> +#define DEV_STAT_SCSI_TGT_PORT_SHOW_HIST(_name) \
> +static ssize_t target_stat_scsi_tgt_port_show_attr_##_name( \
> + struct se_port_stat_grps *pgrps, char *page) \
> +{ \
> + ssize_t size = -ENODEV; \
> + struct se_lun *lun = container_of(pgrps, \
> + struct se_lun, port_stat_grps); \
> + \
> + spin_lock(&lun->lun_sep_lock); \
> + if (lun->lun_sep) { \
> + rcu_read_lock(); \
> + size = snprintf_histogram(page, PAGE_SIZE, \
> + rcu_dereference(lun->lun_stats._name)); \
> + rcu_read_unlock(); \
> + } \
> + spin_unlock(&lun->lun_sep_lock); \
> + return size; \
> +}
> +
> +#define DEV_STAT_SCSI_TGT_PORT_STORE_HIST(_name) \
> +static ssize_t target_stat_scsi_tgt_port_store_attr_##_name( \
> + struct se_port_stat_grps *pgrps, const char *page, size_t size) \
> +{ \
> + struct se_lun *lun = container_of(pgrps, \
> + struct se_lun, port_stat_grps); \
> + struct scsi_port_stats_hist *old, *new; \
> + ssize_t ret; \
> + \
> + new = kzalloc(sizeof(*new), GFP_KERNEL); \
> + if (!new) \
> + return -ENOMEM; \
> + \
> + ret = read_histogram_items(page, \
> + size, new->items, TCM_SE_PORT_STATS_HIST_MAX - 1); \
> + \
> + if (ret < 0) \
> + goto err; \
> + \
> + if (ret == 0) { \
> + kfree(new); \
> + new = NULL; \
> + } else { \
> + new->items[ret] = U64_MAX; \
> + new->count = ret + 1; \
> + } \
> + \
> + spin_lock(&lun->lun_sep_lock); \
> + \
> + if (!lun->lun_sep) { \
> + spin_unlock(&lun->lun_sep_lock); \
> + ret = -ENODEV; \
> + goto err; \
> + } \
> + \
> + old = rcu_dereference_protected(lun->lun_stats._name, \
> + lockdep_is_held(&lun->lun_sep_lock)); \
> + rcu_assign_pointer(lun->lun_stats._name, new); \
> + \
> + spin_unlock(&lun->lun_sep_lock); \
> + \
> + if (old) \
> + kfree_rcu(old, rcu_head); \
> + \
> + return size; \
> + \
> +err: \
> + if (new) \
> + kfree(new); \
> + return ret; \
> +}
> +
> +static void scsi_port_stats_hist_observe_bsearch(
> + struct scsi_port_stats_hist *hist, u64 val)
> +{
> + size_t start = 0, end = hist->count - 1;
> +
> + while (start < end) {
> + size_t mid = start + (end - start) / 2;
> +
> + if (val < hist->items[mid])
> + end = mid;
> + else
> + start = mid + 1;
> + }
> +
> + atomic64_inc(&hist->counters[start]);
> +}
> +
> +void scsi_port_stats_hist_observe(
> + struct scsi_port_stats_hist *hist, u64 val)
> +{
> + if (!hist)
> + return;
> +
> + scsi_port_stats_hist_observe_bsearch(hist, val);
> +}
> +
> +static ssize_t find_token(const char *page, size_t size,
> + size_t offset, size_t *token_len)
> +{
> + size_t i;
> + ssize_t pos = -1;
> + size_t len = 0;
> +
> + BUG_ON(offset > size);
> +
> + for (i = offset; i < size; ++i) {
> + if (isspace(page[i])) {
> + if (len)
> + break;
> + } else if (!len++) {
> + pos = i;
> + }
> + }
> +
> + if (pos > -1)
> + *token_len = len;
> +
> + return pos;
> +}
> +
> +static ssize_t read_histogram_items(const char *page, size_t size,
> + u64 *items, u8 items_max)
> +{
> + u8 total = 0;
> + size_t token_len = 0;
> + ssize_t pos;
> +
> + if (size == 0)
> + return 0;
> +
> + pos = find_token(page, size, 0, &token_len);
> + if (pos == -1)
> + return 0;
> +
> + while (pos != -1) {
> + int ret;
> + char buf[64];
> + u64 item;
> +
> + if (token_len >= sizeof(buf))
> + return -EINVAL;
> +
> + if (total + 1 == items_max) {
if (total == items_max)
> + pr_err("items count can't be greater than %d: %d",
> + items_max, -EPERM);
> + return -EPERM;
return -EINVAL;
> + }
> +
> + memcpy(buf, page + pos, token_len);
> + buf[token_len] = 0;
> +
> + ret = kstrtou64(buf, 10, &item);
> + if (ret < 0) {
> + pr_err("kstrtou64() failed for an item '%s': %d",
> + buf, ret);
> + return ret;
> + }
> +
> + if ((item <= 0) || (total && item <= items[total - 1])) {
> + pr_err("items must be positive, unique and sorted: %d",
> + -EINVAL);
> + return -EINVAL;
> + }
> +
> + items[total++] = item;
> + pos = find_token(page, size, pos + token_len, &token_len);
> + }
> +
> + return total;
> +}
> +
> +static ssize_t snprintf_histogram(char *page, size_t size,
> + struct scsi_port_stats_hist *hist)
> +{
> + ssize_t ret = 0;
> + u8 i;
> +
> + if (!hist)
> + return 0;
> +
> + for (i = 0; i < hist->count - 1; ++i)
> + ret += snprintf(page + ret, PAGE_SIZE - ret,
> + "%llu ", (u64)atomic64_read(&hist->counters[i]));
> +
> + ret += snprintf(page + ret, PAGE_SIZE - ret,
> + "%llu\n", (u64)atomic64_read(&hist->counters[i]));
> +
> + return ret;
> +}
> +
> +DEV_STAT_SCSI_TGT_PORT_STORE_HIST(read_hist);
> +DEV_STAT_SCSI_TGT_PORT_SHOW_HIST(read_hist);
> +DEV_STAT_SCSI_TGT_PORT_ATTR(read_hist, 0644);
> +
> +DEV_STAT_SCSI_TGT_PORT_STORE_HIST(write_hist);
> +DEV_STAT_SCSI_TGT_PORT_SHOW_HIST(write_hist);
> +DEV_STAT_SCSI_TGT_PORT_ATTR(write_hist, 0644);
> +
> +DEV_STAT_SCSI_TGT_PORT_STORE_HIST(sync_hist);
> +DEV_STAT_SCSI_TGT_PORT_SHOW_HIST(sync_hist);
> +DEV_STAT_SCSI_TGT_PORT_ATTR(sync_hist, 0644);
> +
> DEV_STAT_SCSI_TGT_PORT_STATS_SHOW_SIMPLE(read_bytes, tx_data_octets);
> DEV_STAT_SCSI_TGT_PORT_ATTR_RO(read_bytes);
>
> @@ -926,6 +1130,9 @@ static struct configfs_attribute *target_stat_scsi_tgt_port_attrs[] = {
> &target_stat_scsi_tgt_port_bidi_errors.attr,
> &target_stat_scsi_tgt_port_aborts.attr,
> &target_stat_scsi_tgt_port_queue_cmds.attr,
> + &target_stat_scsi_tgt_port_read_hist.attr,
> + &target_stat_scsi_tgt_port_write_hist.attr,
> + &target_stat_scsi_tgt_port_sync_hist.attr,
> NULL,
> };
>
> diff --git a/drivers/target/target_core_tpg.c b/drivers/target/target_core_tpg.c
> index 17442d5b686..eb1e9a323cf 100644
> --- a/drivers/target/target_core_tpg.c
> +++ b/drivers/target/target_core_tpg.c
> @@ -854,6 +854,8 @@ void core_tpg_remove_lun(
> struct se_portal_group *tpg,
> struct se_lun *lun)
> {
> + struct scsi_port_stats_hist *write_hist, *read_hist, *sync_hist;
> +
> core_clear_lun_from_tpg(lun, tpg);
> transport_clear_lun_ref(lun);
>
> @@ -863,5 +865,30 @@ void core_tpg_remove_lun(
> lun->lun_status = TRANSPORT_LUN_STATUS_FREE;
> spin_unlock(&tpg->tpg_lun_lock);
>
> + spin_lock(&lun->lun_sep_lock);
> +
> + write_hist = rcu_dereference_protected(lun->lun_stats.write_hist,
> + lockdep_is_held(&lun->lun_sep_lock));
> + rcu_assign_pointer(lun->lun_stats.write_hist, NULL);
> +
> + read_hist = rcu_dereference_protected(lun->lun_stats.read_hist,
> + lockdep_is_held(&lun->lun_sep_lock));
> + rcu_assign_pointer(lun->lun_stats.read_hist, NULL);
> +
> + sync_hist = rcu_dereference_protected(lun->lun_stats.sync_hist,
> + lockdep_is_held(&lun->lun_sep_lock));
> + rcu_assign_pointer(lun->lun_stats.sync_hist, NULL);
> +
> + spin_unlock(&lun->lun_sep_lock);
> +
> + if (write_hist)
> + kfree_rcu(write_hist, rcu_head);
> +
> + if (read_hist)
> + kfree_rcu(read_hist, rcu_head);
> +
> + if (sync_hist)
> + kfree_rcu(sync_hist, rcu_head);
> +
> percpu_ref_exit(&lun->lun_ref);
> }
> diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
> index f9d1491b08f..b27c17bda01 100644
> --- a/drivers/target/target_core_transport.c
> +++ b/drivers/target/target_core_transport.c
> @@ -712,6 +712,7 @@ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
> } else if (!success) {
> INIT_WORK(&cmd->work, target_complete_failure_work);
> } else {
> + cmd->finished_exec_at = ktime_get_ns() / NSEC_PER_USEC;
> INIT_WORK(&cmd->work, target_complete_ok_work);
> }
>
> @@ -1709,6 +1710,7 @@ void __target_execute_cmd(struct se_cmd *cmd)
> sense_reason_t ret;
>
> if (cmd->execute_cmd) {
> + cmd->started_exec_at = ktime_get_ns() / NSEC_PER_USEC;
> ret = cmd->execute_cmd(cmd);
> if (ret) {
> spin_lock_irq(&cmd->t_state_lock);
> @@ -2039,6 +2041,15 @@ static void target_complete_ok_work(struct work_struct *work)
> }
>
> queue_rsp:
> + if (cmd->t_task_cdb[0] == SYNCHRONIZE_CACHE ||
> + cmd->t_task_cdb[0] == SYNCHRONIZE_CACHE_16) {
> + rcu_read_lock();
> + scsi_port_stats_hist_observe(
> + rcu_dereference(cmd->se_lun->lun_stats.sync_hist),
> + cmd->finished_exec_at - cmd->started_exec_at);
> + rcu_read_unlock();
> + }
> +
> switch (cmd->data_direction) {
> case DMA_FROM_DEVICE:
> atomic_long_add(cmd->data_length,
> @@ -2046,6 +2057,12 @@ queue_rsp:
>
> atomic_long_inc(&cmd->se_lun->lun_stats.read_cmds);
>
> + rcu_read_lock();
> + scsi_port_stats_hist_observe(
> + rcu_dereference(cmd->se_lun->lun_stats.read_hist),
> + cmd->finished_exec_at - cmd->started_exec_at);
> + rcu_read_unlock();
> +
> /*
> * Perform READ_STRIP of PI using software emulation when
> * backend had PI enabled, if the transport will not be
> @@ -2076,6 +2093,12 @@ queue_rsp:
> atomic_long_inc(&cmd->se_lun->lun_stats.bidi_cmds) :
> atomic_long_inc(&cmd->se_lun->lun_stats.write_cmds);
>
> + rcu_read_lock();
> + scsi_port_stats_hist_observe(
> + rcu_dereference(cmd->se_lun->lun_stats.write_hist),
> + cmd->finished_exec_at - cmd->started_exec_at);
> + rcu_read_unlock();
> +
> /*
> * Check if we need to send READ payload for BIDI-COMMAND
> */
> diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
> index 83538ca9f7c..8cfc2c11287 100644
> --- a/include/target/target_core_base.h
> +++ b/include/target/target_core_base.h
> @@ -573,6 +573,8 @@ struct se_cmd {
> sense_reason_t pi_err;
> sector_t bad_sector;
> bool prot_pto;
> + u64 started_exec_at;
> + u64 finished_exec_at;
> };
>
> struct se_ua {
> @@ -710,6 +712,17 @@ struct se_port_stat_grps {
> struct config_group scsi_transport_group;
> };
>
> +#define TCM_SE_PORT_STATS_HIST_MAX 40
> +
> +struct scsi_port_stats_hist {
> + u64 items[TCM_SE_PORT_STATS_HIST_MAX];
> + atomic64_t counters[TCM_SE_PORT_STATS_HIST_MAX];
> + u8 count;
> + struct rcu_head rcu_head;
> +};
> +
> +void scsi_port_stats_hist_observe(struct scsi_port_stats_hist *hist, u64 val);
> +
> struct scsi_port_stats {
> atomic_long_t cmd_pdus;
> atomic_long_t tx_data_octets;
> @@ -722,6 +735,9 @@ struct scsi_port_stats {
> atomic_long_t bidi_errors;
> atomic_long_t queue_cmds;
> atomic_long_t aborts;
> + struct scsi_port_stats_hist __rcu *read_hist;
> + struct scsi_port_stats_hist __rcu *write_hist;
> + struct scsi_port_stats_hist __rcu *sync_hist;
> };
>
> struct se_lun {
> --
> 2.11.0
>
More information about the Devel
mailing list