From 353120b48d4f61288e4745b0c8a191784b11c0f4 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:09 +0200 Subject: libperf: Add perf_mmap__init() function Add perf_mmap__init() function to initialize 'struct perf_mmap' objects. Add it to a new mmap.c source file, that will carry all the mmap related functions. Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-2-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tools/perf/lib/mmap.c (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c new file mode 100644 index 000000000000..3da6177510e6 --- /dev/null +++ b/tools/perf/lib/mmap.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +void perf_mmap__init(struct perf_mmap *map, bool overwrite) +{ + map->fd = -1; + map->overwrite = overwrite; + refcount_set(&map->refcnt, 0); +} -- cgit From bf59b3053e63783520c2810fc3f676553bc7eedd Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:11 +0200 Subject: libperf: Adopt perf_mmap__mmap_len() function from tools/perf Move perf_mmap__mmap_len() from tools/perf wto libperf, it will be used in the following patches. And rename the existing perf's function to mmap__mmap_len(). Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-4-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index 3da6177510e6..cc4284da4d99 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include void perf_mmap__init(struct perf_mmap *map, bool overwrite) { @@ -7,3 +8,8 @@ void perf_mmap__init(struct perf_mmap *map, bool overwrite) map->overwrite = overwrite; refcount_set(&map->refcnt, 0); } + +size_t perf_mmap__mmap_len(struct perf_mmap *map) +{ + return map->mask + 1 + page_size; +} -- cgit From 32c261c070c222858148c2171698d2954242ddd9 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:12 +0200 Subject: libperf: Adopt perf_mmap__mmap() function from tools/perf Move perf_mmap__mmap() from tools/perf to libperf, it will be used in the following patches. And rename the existing perf's function to mmap__mmap(). Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-5-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index cc4284da4d99..b216a7db857f 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include @@ -13,3 +14,20 @@ size_t perf_mmap__mmap_len(struct perf_mmap *map) { return map->mask + 1 + page_size; } + +int perf_mmap__mmap(struct perf_mmap *map, struct perf_mmap_param *mp, + int fd, int cpu) +{ + map->prev = 0; + map->mask = mp->mask; + map->base = mmap(NULL, perf_mmap__mmap_len(map), mp->prot, + MAP_SHARED, fd, 0); + if (map->base == MAP_FAILED) { + map->base = NULL; + return -1; + } + + map->fd = fd; + map->cpu = cpu; + return 0; +} -- cgit From e75710f063e29ae7715c57b45eb27c2d504b32ca Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:13 +0200 Subject: libperf: Adopt perf_mmap__get() function from tools/perf Move perf_mmap__get() from tools/perf to libperf in the internal header internal/mmap.h. Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-6-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index b216a7db857f..b765e0505bb6 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -31,3 +31,8 @@ int perf_mmap__mmap(struct perf_mmap *map, struct perf_mmap_param *mp, map->cpu = cpu; return 0; } + +void perf_mmap__get(struct perf_mmap *map) +{ + refcount_inc(&map->refcnt); +} -- cgit From 59d7ea620b58fa7d107834a81528e3098f1c27b0 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:14 +0200 Subject: libperf: Adopt perf_mmap__unmap() function from tools/perf Move perf_mmap__unmap() from tools/perf to libperf, to internal header internal/mmap.h. It will be used in the following patches. And rename the existing perf's function to mmap__munmap(). Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-7-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index b765e0505bb6..6eb228d89206 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -32,6 +32,16 @@ int perf_mmap__mmap(struct perf_mmap *map, struct perf_mmap_param *mp, return 0; } +void perf_mmap__munmap(struct perf_mmap *map) +{ + if (map && map->base != NULL) { + munmap(map->base, perf_mmap__mmap_len(map)); + map->base = NULL; + map->fd = -1; + refcount_set(&map->refcnt, 0); + } +} + void perf_mmap__get(struct perf_mmap *map) { refcount_inc(&map->refcnt); -- cgit From 80e53d1148231d7d4fdc4cd89e5393616b33bf82 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:15 +0200 Subject: libperf: Adopt perf_mmap__put() function from tools/perf Move perf_mmap__put() from tools/perf to libperf. Once perf_mmap__put() is moved, we need a way to call application related unmap code (AIO and aux related code for eprf), when the map goes away. Add the perf_mmap::unmap callback to do that. The unmap path from perf is: perf_mmap__put (libperf) perf_mmap__munmap (libperf) map->unmap_cb -> perf_mmap__unmap_cb (perf) mmap__munmap (perf) Committer notes: Add missing linux/kernel.h to tools/perf/lib/mmap.c to get the BUG_ON definition. Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-8-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index 6eb228d89206..89c1e0e8b897 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -2,11 +2,14 @@ #include #include #include +#include -void perf_mmap__init(struct perf_mmap *map, bool overwrite) +void perf_mmap__init(struct perf_mmap *map, bool overwrite, + libperf_unmap_cb_t unmap_cb) { map->fd = -1; map->overwrite = overwrite; + map->unmap_cb = unmap_cb; refcount_set(&map->refcnt, 0); } @@ -40,9 +43,19 @@ void perf_mmap__munmap(struct perf_mmap *map) map->fd = -1; refcount_set(&map->refcnt, 0); } + if (map && map->unmap_cb) + map->unmap_cb(map); } void perf_mmap__get(struct perf_mmap *map) { refcount_inc(&map->refcnt); } + +void perf_mmap__put(struct perf_mmap *map) +{ + BUG_ON(map->base && refcount_read(&map->refcnt) == 0); + + if (refcount_dec_and_test(&map->refcnt)) + perf_mmap__munmap(map); +} -- cgit From 7728fa0cfaeb7d25b12c8865c733359cc8e5fb13 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:17 +0200 Subject: libperf: Adopt perf_mmap__consume() function from tools/perf Move perf_mmap__consume() vrom tools/perf to libperf and export it in the perf/mmap.h header. Move also the needed helpers perf_mmap__write_tail(), perf_mmap__read_head() and perf_mmap__empty(). Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-10-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index 89c1e0e8b897..4cada1c89fdb 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include +#include +#include #include #include #include @@ -59,3 +62,32 @@ void perf_mmap__put(struct perf_mmap *map) if (refcount_dec_and_test(&map->refcnt)) perf_mmap__munmap(map); } + +static inline void perf_mmap__write_tail(struct perf_mmap *md, u64 tail) +{ + ring_buffer_write_tail(md->base, tail); +} + +u64 perf_mmap__read_head(struct perf_mmap *map) +{ + return ring_buffer_read_head(map->base); +} + +static bool perf_mmap__empty(struct perf_mmap *map) +{ + struct perf_event_mmap_page *pc = map->base; + + return perf_mmap__read_head(map) == map->prev && !pc->aux_size; +} + +void perf_mmap__consume(struct perf_mmap *map) +{ + if (!map->overwrite) { + u64 old = map->prev; + + perf_mmap__write_tail(map, old); + } + + if (refcount_read(&map->refcnt) == 1 && perf_mmap__empty(map)) + perf_mmap__put(map); +} -- cgit From 7c4d41824f9afc659ba425a41018546531cffd72 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:18 +0200 Subject: libperf: Adopt perf_mmap__read_init() from tools/perf Move perf_mmap__read_init() from tools/perf to libperf and export it in perf/mmap.h header. And add pr_debug2()/pr_debug3() macros support, because the code is using them. Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-11-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index 4cada1c89fdb..fdbc6c550dea 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -1,11 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include +#include +#include #include #include #include #include #include #include +#include "internal.h" void perf_mmap__init(struct perf_mmap *map, bool overwrite, libperf_unmap_cb_t unmap_cb) @@ -91,3 +95,83 @@ void perf_mmap__consume(struct perf_mmap *map) if (refcount_read(&map->refcnt) == 1 && perf_mmap__empty(map)) perf_mmap__put(map); } + +static int overwrite_rb_find_range(void *buf, int mask, u64 *start, u64 *end) +{ + struct perf_event_header *pheader; + u64 evt_head = *start; + int size = mask + 1; + + pr_debug2("%s: buf=%p, start=%"PRIx64"\n", __func__, buf, *start); + pheader = (struct perf_event_header *)(buf + (*start & mask)); + while (true) { + if (evt_head - *start >= (unsigned int)size) { + pr_debug("Finished reading overwrite ring buffer: rewind\n"); + if (evt_head - *start > (unsigned int)size) + evt_head -= pheader->size; + *end = evt_head; + return 0; + } + + pheader = (struct perf_event_header *)(buf + (evt_head & mask)); + + if (pheader->size == 0) { + pr_debug("Finished reading overwrite ring buffer: get start\n"); + *end = evt_head; + return 0; + } + + evt_head += pheader->size; + pr_debug3("move evt_head: %"PRIx64"\n", evt_head); + } + WARN_ONCE(1, "Shouldn't get here\n"); + return -1; +} + +/* + * Report the start and end of the available data in ringbuffer + */ +static int __perf_mmap__read_init(struct perf_mmap *md) +{ + u64 head = perf_mmap__read_head(md); + u64 old = md->prev; + unsigned char *data = md->base + page_size; + unsigned long size; + + md->start = md->overwrite ? head : old; + md->end = md->overwrite ? old : head; + + if ((md->end - md->start) < md->flush) + return -EAGAIN; + + size = md->end - md->start; + if (size > (unsigned long)(md->mask) + 1) { + if (!md->overwrite) { + WARN_ONCE(1, "failed to keep up with mmap data. (warn only once)\n"); + + md->prev = head; + perf_mmap__consume(md); + return -EAGAIN; + } + + /* + * Backward ring buffer is full. We still have a chance to read + * most of data from it. + */ + if (overwrite_rb_find_range(data, md->mask, &md->start, &md->end)) + return -EINVAL; + } + + return 0; +} + +int perf_mmap__read_init(struct perf_mmap *map) +{ + /* + * Check if event was unmapped due to a POLLHUP/POLLERR. + */ + if (!refcount_read(&map->refcnt)) + return -ENOENT; + + return __perf_mmap__read_init(map); +} -- cgit From 32fdc2ca7e2ae8ae5d0ff660ca7783acd8ee6396 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:19 +0200 Subject: libperf: Adopt perf_mmap__read_done() from tools/perf Move perf_mmap__read_init() from tools/perf to libperf and export it in the perf/mmap.h header. Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-12-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index fdbc6c550dea..97297cba44e3 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -175,3 +175,20 @@ int perf_mmap__read_init(struct perf_mmap *map) return __perf_mmap__read_init(map); } + +/* + * Mandatory for overwrite mode + * The direction of overwrite mode is backward. + * The last perf_mmap__read() will set tail to map->core.prev. + * Need to correct the map->core.prev to head which is the end of next read. + */ +void perf_mmap__read_done(struct perf_mmap *map) +{ + /* + * Check if event was unmapped due to a POLLHUP/POLLERR. + */ + if (!refcount_read(&map->refcnt)) + return; + + map->prev = perf_mmap__read_head(map); +} -- cgit From 151ed5d70da87720022e4171227733a008b3c719 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 7 Oct 2019 14:53:20 +0200 Subject: libperf: Adopt perf_mmap__read_event() from tools/perf Move perf_mmap__read_event() from tools/perf to libperf and export it in the perf/mmap.h header. Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Link: http://lore.kernel.org/lkml/20191007125344.14268-13-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index 97297cba44e3..0752c193b0fb 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -3,9 +3,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -192,3 +194,80 @@ void perf_mmap__read_done(struct perf_mmap *map) map->prev = perf_mmap__read_head(map); } + +/* When check_messup is true, 'end' must points to a good entry */ +static union perf_event *perf_mmap__read(struct perf_mmap *map, + u64 *startp, u64 end) +{ + unsigned char *data = map->base + page_size; + union perf_event *event = NULL; + int diff = end - *startp; + + if (diff >= (int)sizeof(event->header)) { + size_t size; + + event = (union perf_event *)&data[*startp & map->mask]; + size = event->header.size; + + if (size < sizeof(event->header) || diff < (int)size) + return NULL; + + /* + * Event straddles the mmap boundary -- header should always + * be inside due to u64 alignment of output. + */ + if ((*startp & map->mask) + size != ((*startp + size) & map->mask)) { + unsigned int offset = *startp; + unsigned int len = min(sizeof(*event), size), cpy; + void *dst = map->event_copy; + + do { + cpy = min(map->mask + 1 - (offset & map->mask), len); + memcpy(dst, &data[offset & map->mask], cpy); + offset += cpy; + dst += cpy; + len -= cpy; + } while (len); + + event = (union perf_event *)map->event_copy; + } + + *startp += size; + } + + return event; +} + +/* + * Read event from ring buffer one by one. + * Return one event for each call. + * + * Usage: + * perf_mmap__read_init() + * while(event = perf_mmap__read_event()) { + * //process the event + * perf_mmap__consume() + * } + * perf_mmap__read_done() + */ +union perf_event *perf_mmap__read_event(struct perf_mmap *map) +{ + union perf_event *event; + + /* + * Check if event was unmapped due to a POLLHUP/POLLERR. + */ + if (!refcount_read(&map->refcnt)) + return NULL; + + /* non-overwirte doesn't pause the ringbuffer */ + if (!map->overwrite) + map->end = perf_mmap__read_head(map); + + event = perf_mmap__read(map, &map->start, map->end); + + if (!map->overwrite) + map->prev = map->start; + + return event; +} -- cgit From 6eb65f7a5cc553f5dffb5cea3a874f1087524d99 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 17 Oct 2019 12:59:09 +0200 Subject: libperf: Introduce perf_evlist__for_each_mmap() Add the perf_evlist__for_each_mmap() function and export it in the perf/evlist.h header, so that the user can iterate through 'struct perf_mmap' objects. Add a internal perf_mmap__link() function to do the actual linking. Signed-off-by: Jiri Olsa Cc: Alexander Shishkin Cc: Alexey Budankov Cc: Andi Kleen Cc: Jin Yao Cc: Michael Petlan Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Stephane Eranian Link: http://lore.kernel.org/lkml/20191017105918.20873-2-jolsa@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/lib/mmap.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tools/perf/lib/mmap.c') diff --git a/tools/perf/lib/mmap.c b/tools/perf/lib/mmap.c index 0752c193b0fb..79d5ed6c38cc 100644 --- a/tools/perf/lib/mmap.c +++ b/tools/perf/lib/mmap.c @@ -13,13 +13,15 @@ #include #include "internal.h" -void perf_mmap__init(struct perf_mmap *map, bool overwrite, - libperf_unmap_cb_t unmap_cb) +void perf_mmap__init(struct perf_mmap *map, struct perf_mmap *prev, + bool overwrite, libperf_unmap_cb_t unmap_cb) { map->fd = -1; map->overwrite = overwrite; map->unmap_cb = unmap_cb; refcount_set(&map->refcnt, 0); + if (prev) + prev->next = map; } size_t perf_mmap__mmap_len(struct perf_mmap *map) -- cgit