[Devel] [PATCH RHEL10 COMMIT] drivers/md/dm-stats: introduce I/O classes for histogram dispatch
Konstantin Khorenko
khorenko at virtuozzo.com
Fri Jul 31 15:11:50 MSK 2026
The commit is pushed to "branch-rh10-6.12.0-211.39.1.16.x.vz10-ovz" and will appear at git at bitbucket.org:openvz/vzkernel.git
after rh10-6.12.0-211.39.1.16.1.vz10
------>
commit 8f46920173496603a1aa2c169f4e979bdf0ffd77
Author: Andrey Zhadchenko <andrey.zhadchenko at virtuozzo.com>
Date: Mon Jul 27 19:32:49 2026 +0300
drivers/md/dm-stats: introduce I/O classes for histogram dispatch
Currently every histogram of a region accounts every I/O. To later
support histograms that only account a particular kind of I/O,
classify each once per accounting call into a bitmask of classes
and give each histogram a class_mask of the classes it accounts.
The per-class dispatch is precomputed at region creation time into a
small lookup table indexed by the class bitmask, so the hot path
selects the histograms to update with a single table load instead of
per-histogram filter comparisons. dm_stats_bin_duration() then walks
only the set bits of the resulting histogram mask.
This commit does not introduce any histogram types yet, but let's
imagine we have 'write', 'all' and 'read' histograms.
Then the table dm_stat->hist_mask will look like that:
[0b00] = 0b000
[0b01] (read) = 0b110
[0b10] (write) = 0b011
[0b11] (read & write) = 0b111
For WRITE request (class 0b10) during the binning we will iterate over
__ffs(0b011) and add the duration value into hist#0 (write) and
hist#1 (all).
https://virtuozzo.atlassian.net/browse/VSTOR-103846
Feature: dm-stats: latency and histogram enhancements
Signed-off-by: Andrey Zhadchenko <andrey.zhadchenko at virtuozzo.com>
Reviewed-by: Vasileios Almpanis <vasileios.almpanis at virtuozzo.com>
Reviewed-by: Pavel Tikhomirov <ptikhomirov at virtuozzo.com>
---
drivers/md/dm-stats.c | 73 +++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 59 insertions(+), 14 deletions(-)
diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c
index b0b6124124e2a..e55b821fb9a34 100644
--- a/drivers/md/dm-stats.c
+++ b/drivers/md/dm-stats.c
@@ -39,9 +39,23 @@ struct dm_stat_shared {
struct dm_stat_percpu tmp;
};
+/*
+ * Histograms account I/O classes independently. An I/O is classified once
+ * per dm_stats_account_io() and may belong to several classes at once.
+ */
+enum {
+ DM_STAT_CLASS_READ,
+ DM_STAT_CLASS_WRITE,
+ DM_STAT_NR_CLASSES,
+};
+
+#define DM_STAT_CLASSES_DATA ((1 << DM_STAT_CLASS_READ) | \
+ (1 << DM_STAT_CLASS_WRITE))
+
#define DM_STAT_MAX_HISTOGRAMS 1
struct dm_stat_histogram {
+ unsigned int class_mask; /* classes this histogram accounts */
unsigned int n_entries;
unsigned int offset;
unsigned long long *boundaries;
@@ -57,6 +71,8 @@ struct dm_stat {
sector_t step;
unsigned int n_histograms;
unsigned int histogram_buckets;
+ /* class bitmask -> bitmask of histograms accounting these classes */
+ u8 hist_mask[1 << DM_STAT_NR_CLASSES];
struct dm_stat_histogram histograms[DM_STAT_MAX_HISTOGRAMS];
const char *program_id;
const char *aux_data;
@@ -352,6 +368,12 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end,
s->n_histograms = n_histograms;
s->histogram_buckets = histogram_buckets;
for (hn = 0; hn < n_histograms; hn++) {
+ unsigned int cm;
+
+ s->histograms[hn].class_mask = histograms[hn].class_mask;
+ for (cm = 1; cm < ARRAY_SIZE(s->hist_mask); cm++)
+ if (histograms[hn].class_mask & cm)
+ s->hist_mask[cm] |= 1 << hn;
s->histograms[hn].n_entries = histograms[hn].n_entries;
s->histograms[hn].offset = (hn ? s->histograms[hn - 1].offset +
s->histograms[hn - 1].n_entries + 1 : 0);
@@ -597,16 +619,27 @@ static void dm_stat_round(struct dm_stat *s, struct dm_stat_shared *shared,
shared->stamp = now;
}
+static unsigned int dm_stat_io_classes(blk_opf_t bi_opf, unsigned int bi_sectors)
+{
+ unsigned int classes = 0;
+
+ if (bi_sectors)
+ classes |= 1 << (op_is_write(bi_opf) ? DM_STAT_CLASS_WRITE :
+ DM_STAT_CLASS_READ);
+
+ return classes;
+}
static void dm_stats_bin_duration(struct dm_stat *s, struct dm_stat_percpu *p,
+ unsigned int hmask,
unsigned long long duration)
{
- unsigned int hn;
-
- for (hn = 0; hn < s->n_histograms; hn++) {
- struct dm_stat_histogram *h = &s->histograms[hn];
+ do {
+ struct dm_stat_histogram *h = &s->histograms[__ffs(hmask)];
unsigned int lo = 0, hi = h->n_entries + 1;
+ hmask &= hmask - 1;
+
while (lo + 1 < hi) {
unsigned int mid = (lo + hi) / 2;
@@ -616,11 +649,12 @@ static void dm_stats_bin_duration(struct dm_stat *s, struct dm_stat_percpu *p,
lo = mid;
}
p->histogram[h->offset + lo]++;
- }
+ } while (hmask);
}
static void dm_stat_for_entry(struct dm_stat *s, size_t entry,
- blk_opf_t bi_opf, sector_t len,
+ blk_opf_t bi_opf, unsigned int classes,
+ sector_t len,
struct dm_stats_aux *stats_aux, bool end,
unsigned long duration_jiffies)
{
@@ -657,6 +691,7 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry,
atomic_inc(&shared->in_flight[idx]);
} else {
unsigned long long duration;
+ unsigned int hmask = s->hist_mask[classes];
dm_stat_round(s, shared, p);
atomic_dec(&shared->in_flight[idx]);
@@ -674,8 +709,8 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry,
else
duration = stats_aux->duration_ns;
}
- if (s->n_histograms)
- dm_stats_bin_duration(s, p, duration);
+ if (hmask)
+ dm_stats_bin_duration(s, p, hmask, duration);
}
#if BITS_PER_LONG == 32
@@ -686,6 +721,7 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry,
}
static void __dm_stat_bio(struct dm_stat *s, blk_opf_t bi_opf,
+ unsigned int classes,
sector_t bi_sector, sector_t end_sector,
bool end, unsigned long duration_jiffies,
struct dm_stats_aux *stats_aux)
@@ -715,7 +751,7 @@ static void __dm_stat_bio(struct dm_stat *s, blk_opf_t bi_opf,
fragment_len = todo;
if (fragment_len > s->step - offset)
fragment_len = s->step - offset;
- dm_stat_for_entry(s, entry, bi_opf, fragment_len,
+ dm_stat_for_entry(s, entry, bi_opf, classes, fragment_len,
stats_aux, end, duration_jiffies);
todo -= fragment_len;
entry++;
@@ -734,8 +770,9 @@ void dm_stats_account_io(struct dm_stats *stats, blk_opf_t bi_opf,
bool got_precise_time;
unsigned long duration_jiffies = 0;
unsigned long bi_rw = op_is_write(bi_opf);
+ unsigned int classes = dm_stat_io_classes(bi_opf, bi_sectors);
- if (unlikely(!bi_sectors))
+ if (unlikely(!classes))
return;
end_sector = bi_sector + bi_sectors;
@@ -766,7 +803,8 @@ void dm_stats_account_io(struct dm_stats *stats, blk_opf_t bi_opf,
stats_aux->duration_ns = ktime_to_ns(ktime_get()) - stats_aux->duration_ns;
got_precise_time = true;
}
- __dm_stat_bio(s, bi_opf, bi_sector, end_sector, end, duration_jiffies, stats_aux);
+ __dm_stat_bio(s, bi_opf, classes, bi_sector, end_sector, end,
+ duration_jiffies, stats_aux);
}
rcu_read_unlock();
@@ -1057,14 +1095,20 @@ static int parse_histogram(const char *h, unsigned int *n_histogram_entries,
}
static int add_histogram(struct dm_stat_histogram *histograms,
- unsigned int *n_histograms, const char *h)
+ unsigned int *n_histograms, unsigned int class_mask,
+ const char *h)
{
- unsigned int idx;
+ unsigned int i, idx;
+
+ for (i = 0; i < *n_histograms; i++)
+ if (histograms[i].class_mask == class_mask)
+ return -EINVAL;
if (*n_histograms >= DM_STAT_MAX_HISTOGRAMS)
return -EINVAL;
idx = (*n_histograms)++;
+ histograms[idx].class_mask = class_mask;
histograms[idx].boundaries = NULL;
return parse_histogram(h, &histograms[idx].n_entries,
@@ -1140,7 +1184,8 @@ static int message_stats_create(struct mapped_device *md,
else if (!strcasecmp(a, "hist_total_latency"))
stat_flags |= STAT_HIST_TOTAL_LATENCY;
else if (!strncasecmp(a, "histogram:", 10)) {
- r = add_histogram(histograms, &n_histograms, a + 10);
+ r = add_histogram(histograms, &n_histograms,
+ DM_STAT_CLASSES_DATA, a + 10);
if (r)
goto ret;
} else
More information about the Devel
mailing list