|
@@ -13,6 +13,7 @@
|
|
|
#include "../util/cloexec.h"
|
|
|
#include "bench.h"
|
|
|
#include "mem-memcpy-arch.h"
|
|
|
+#include "mem-memset-arch.h"
|
|
|
|
|
|
#include <stdio.h>
|
|
|
#include <stdlib.h>
|
|
@@ -48,20 +49,24 @@ static const struct option options[] = {
|
|
|
};
|
|
|
|
|
|
typedef void *(*memcpy_t)(void *, const void *, size_t);
|
|
|
+typedef void *(*memset_t)(void *, int, size_t);
|
|
|
|
|
|
struct routine {
|
|
|
const char *name;
|
|
|
const char *desc;
|
|
|
- memcpy_t fn;
|
|
|
+ union {
|
|
|
+ memcpy_t memcpy;
|
|
|
+ memset_t memset;
|
|
|
+ } fn;
|
|
|
};
|
|
|
|
|
|
-struct routine routines[] = {
|
|
|
- { "default",
|
|
|
- "Default memcpy() provided by glibc",
|
|
|
- memcpy },
|
|
|
+struct routine memcpy_routines[] = {
|
|
|
+ { .name = "default",
|
|
|
+ .desc = "Default memcpy() provided by glibc",
|
|
|
+ .fn.memcpy = memcpy },
|
|
|
#ifdef HAVE_ARCH_X86_64_SUPPORT
|
|
|
|
|
|
-#define MEMCPY_FN(fn, name, desc) { name, desc, fn },
|
|
|
+#define MEMCPY_FN(_fn, _name, _desc) {.name = _name, .desc = _desc, .fn.memcpy = _fn},
|
|
|
#include "mem-memcpy-x86-64-asm-def.h"
|
|
|
#undef MEMCPY_FN
|
|
|
|
|
@@ -69,7 +74,7 @@ struct routine routines[] = {
|
|
|
|
|
|
{ NULL,
|
|
|
NULL,
|
|
|
- NULL }
|
|
|
+ {NULL} }
|
|
|
};
|
|
|
|
|
|
static const char * const bench_mem_memcpy_usage[] = {
|
|
@@ -110,63 +115,6 @@ static double timeval2double(struct timeval *ts)
|
|
|
(double)ts->tv_usec / (double)1000000;
|
|
|
}
|
|
|
|
|
|
-static void alloc_mem(void **dst, void **src, size_t length)
|
|
|
-{
|
|
|
- *dst = zalloc(length);
|
|
|
- if (!*dst)
|
|
|
- die("memory allocation failed - maybe length is too large?\n");
|
|
|
-
|
|
|
- *src = zalloc(length);
|
|
|
- if (!*src)
|
|
|
- die("memory allocation failed - maybe length is too large?\n");
|
|
|
- /* Make sure to always replace the zero pages even if MMAP_THRESH is crossed */
|
|
|
- memset(*src, 0, length);
|
|
|
-}
|
|
|
-
|
|
|
-static u64 do_memcpy_cycle(memcpy_t fn, size_t len, bool prefault)
|
|
|
-{
|
|
|
- u64 cycle_start = 0ULL, cycle_end = 0ULL;
|
|
|
- void *src = NULL, *dst = NULL;
|
|
|
- int i;
|
|
|
-
|
|
|
- alloc_mem(&src, &dst, len);
|
|
|
-
|
|
|
- if (prefault)
|
|
|
- fn(dst, src, len);
|
|
|
-
|
|
|
- cycle_start = get_cycle();
|
|
|
- for (i = 0; i < iterations; ++i)
|
|
|
- fn(dst, src, len);
|
|
|
- cycle_end = get_cycle();
|
|
|
-
|
|
|
- free(src);
|
|
|
- free(dst);
|
|
|
- return cycle_end - cycle_start;
|
|
|
-}
|
|
|
-
|
|
|
-static double do_memcpy_gettimeofday(memcpy_t fn, size_t len, bool prefault)
|
|
|
-{
|
|
|
- struct timeval tv_start, tv_end, tv_diff;
|
|
|
- void *src = NULL, *dst = NULL;
|
|
|
- int i;
|
|
|
-
|
|
|
- alloc_mem(&src, &dst, len);
|
|
|
-
|
|
|
- if (prefault)
|
|
|
- fn(dst, src, len);
|
|
|
-
|
|
|
- BUG_ON(gettimeofday(&tv_start, NULL));
|
|
|
- for (i = 0; i < iterations; ++i)
|
|
|
- fn(dst, src, len);
|
|
|
- BUG_ON(gettimeofday(&tv_end, NULL));
|
|
|
-
|
|
|
- timersub(&tv_end, &tv_start, &tv_diff);
|
|
|
-
|
|
|
- free(src);
|
|
|
- free(dst);
|
|
|
- return (double)((double)len / timeval2double(&tv_diff));
|
|
|
-}
|
|
|
-
|
|
|
#define pf (no_prefault ? 0 : 1)
|
|
|
|
|
|
#define print_bps(x) do { \
|
|
@@ -180,16 +128,25 @@ static double do_memcpy_gettimeofday(memcpy_t fn, size_t len, bool prefault)
|
|
|
printf(" %14lf GB/Sec", x / K / K / K); \
|
|
|
} while (0)
|
|
|
|
|
|
-int bench_mem_memcpy(int argc, const char **argv,
|
|
|
- const char *prefix __maybe_unused)
|
|
|
+struct bench_mem_info {
|
|
|
+ const struct routine *routines;
|
|
|
+ u64 (*do_cycle)(const struct routine *r, size_t len, bool prefault);
|
|
|
+ double (*do_gettimeofday)(const struct routine *r, size_t len, bool prefault);
|
|
|
+ const char *const *usage;
|
|
|
+};
|
|
|
+
|
|
|
+static int bench_mem_common(int argc, const char **argv,
|
|
|
+ const char *prefix __maybe_unused,
|
|
|
+ struct bench_mem_info *info)
|
|
|
{
|
|
|
int i;
|
|
|
size_t len;
|
|
|
+ double totallen;
|
|
|
double result_bps[2];
|
|
|
u64 result_cycle[2];
|
|
|
|
|
|
argc = parse_options(argc, argv, options,
|
|
|
- bench_mem_memcpy_usage, 0);
|
|
|
+ info->usage, 0);
|
|
|
|
|
|
if (no_prefault && only_prefault) {
|
|
|
fprintf(stderr, "Invalid options: -o and -n are mutually exclusive\n");
|
|
@@ -200,6 +157,7 @@ int bench_mem_memcpy(int argc, const char **argv,
|
|
|
init_cycle();
|
|
|
|
|
|
len = (size_t)perf_atoll((char *)length_str);
|
|
|
+ totallen = (double)len * iterations;
|
|
|
|
|
|
result_cycle[0] = result_cycle[1] = 0ULL;
|
|
|
result_bps[0] = result_bps[1] = 0.0;
|
|
@@ -213,16 +171,16 @@ int bench_mem_memcpy(int argc, const char **argv,
|
|
|
if (only_prefault && no_prefault)
|
|
|
only_prefault = no_prefault = false;
|
|
|
|
|
|
- for (i = 0; routines[i].name; i++) {
|
|
|
- if (!strcmp(routines[i].name, routine))
|
|
|
+ for (i = 0; info->routines[i].name; i++) {
|
|
|
+ if (!strcmp(info->routines[i].name, routine))
|
|
|
break;
|
|
|
}
|
|
|
- if (!routines[i].name) {
|
|
|
+ if (!info->routines[i].name) {
|
|
|
printf("Unknown routine:%s\n", routine);
|
|
|
printf("Available routines...\n");
|
|
|
- for (i = 0; routines[i].name; i++) {
|
|
|
+ for (i = 0; info->routines[i].name; i++) {
|
|
|
printf("\t%s ... %s\n",
|
|
|
- routines[i].name, routines[i].desc);
|
|
|
+ info->routines[i].name, info->routines[i].desc);
|
|
|
}
|
|
|
return 1;
|
|
|
}
|
|
@@ -234,25 +192,25 @@ int bench_mem_memcpy(int argc, const char **argv,
|
|
|
/* show both of results */
|
|
|
if (use_cycle) {
|
|
|
result_cycle[0] =
|
|
|
- do_memcpy_cycle(routines[i].fn, len, false);
|
|
|
+ info->do_cycle(&info->routines[i], len, false);
|
|
|
result_cycle[1] =
|
|
|
- do_memcpy_cycle(routines[i].fn, len, true);
|
|
|
+ info->do_cycle(&info->routines[i], len, true);
|
|
|
} else {
|
|
|
result_bps[0] =
|
|
|
- do_memcpy_gettimeofday(routines[i].fn,
|
|
|
+ info->do_gettimeofday(&info->routines[i],
|
|
|
len, false);
|
|
|
result_bps[1] =
|
|
|
- do_memcpy_gettimeofday(routines[i].fn,
|
|
|
+ info->do_gettimeofday(&info->routines[i],
|
|
|
len, true);
|
|
|
}
|
|
|
} else {
|
|
|
if (use_cycle) {
|
|
|
result_cycle[pf] =
|
|
|
- do_memcpy_cycle(routines[i].fn,
|
|
|
+ info->do_cycle(&info->routines[i],
|
|
|
len, only_prefault);
|
|
|
} else {
|
|
|
result_bps[pf] =
|
|
|
- do_memcpy_gettimeofday(routines[i].fn,
|
|
|
+ info->do_gettimeofday(&info->routines[i],
|
|
|
len, only_prefault);
|
|
|
}
|
|
|
}
|
|
@@ -263,10 +221,10 @@ int bench_mem_memcpy(int argc, const char **argv,
|
|
|
if (use_cycle) {
|
|
|
printf(" %14lf Cycle/Byte\n",
|
|
|
(double)result_cycle[0]
|
|
|
- / (double)len);
|
|
|
+ / totallen);
|
|
|
printf(" %14lf Cycle/Byte (with prefault)\n",
|
|
|
(double)result_cycle[1]
|
|
|
- / (double)len);
|
|
|
+ / totallen);
|
|
|
} else {
|
|
|
print_bps(result_bps[0]);
|
|
|
printf("\n");
|
|
@@ -277,7 +235,7 @@ int bench_mem_memcpy(int argc, const char **argv,
|
|
|
if (use_cycle) {
|
|
|
printf(" %14lf Cycle/Byte",
|
|
|
(double)result_cycle[pf]
|
|
|
- / (double)len);
|
|
|
+ / totallen);
|
|
|
} else
|
|
|
print_bps(result_bps[pf]);
|
|
|
|
|
@@ -288,8 +246,8 @@ int bench_mem_memcpy(int argc, const char **argv,
|
|
|
if (!only_prefault && !no_prefault) {
|
|
|
if (use_cycle) {
|
|
|
printf("%lf %lf\n",
|
|
|
- (double)result_cycle[0] / (double)len,
|
|
|
- (double)result_cycle[1] / (double)len);
|
|
|
+ (double)result_cycle[0] / totallen,
|
|
|
+ (double)result_cycle[1] / totallen);
|
|
|
} else {
|
|
|
printf("%lf %lf\n",
|
|
|
result_bps[0], result_bps[1]);
|
|
@@ -297,7 +255,7 @@ int bench_mem_memcpy(int argc, const char **argv,
|
|
|
} else {
|
|
|
if (use_cycle) {
|
|
|
printf("%lf\n", (double)result_cycle[pf]
|
|
|
- / (double)len);
|
|
|
+ / totallen);
|
|
|
} else
|
|
|
printf("%lf\n", result_bps[pf]);
|
|
|
}
|
|
@@ -310,3 +268,163 @@ int bench_mem_memcpy(int argc, const char **argv,
|
|
|
|
|
|
return 0;
|
|
|
}
|
|
|
+
|
|
|
+static void memcpy_alloc_mem(void **dst, void **src, size_t length)
|
|
|
+{
|
|
|
+ *dst = zalloc(length);
|
|
|
+ if (!*dst)
|
|
|
+ die("memory allocation failed - maybe length is too large?\n");
|
|
|
+
|
|
|
+ *src = zalloc(length);
|
|
|
+ if (!*src)
|
|
|
+ die("memory allocation failed - maybe length is too large?\n");
|
|
|
+ /* Make sure to always replace the zero pages even if MMAP_THRESH is crossed */
|
|
|
+ memset(*src, 0, length);
|
|
|
+}
|
|
|
+
|
|
|
+static u64 do_memcpy_cycle(const struct routine *r, size_t len, bool prefault)
|
|
|
+{
|
|
|
+ u64 cycle_start = 0ULL, cycle_end = 0ULL;
|
|
|
+ void *src = NULL, *dst = NULL;
|
|
|
+ memcpy_t fn = r->fn.memcpy;
|
|
|
+ int i;
|
|
|
+
|
|
|
+ memcpy_alloc_mem(&src, &dst, len);
|
|
|
+
|
|
|
+ if (prefault)
|
|
|
+ fn(dst, src, len);
|
|
|
+
|
|
|
+ cycle_start = get_cycle();
|
|
|
+ for (i = 0; i < iterations; ++i)
|
|
|
+ fn(dst, src, len);
|
|
|
+ cycle_end = get_cycle();
|
|
|
+
|
|
|
+ free(src);
|
|
|
+ free(dst);
|
|
|
+ return cycle_end - cycle_start;
|
|
|
+}
|
|
|
+
|
|
|
+static double do_memcpy_gettimeofday(const struct routine *r, size_t len,
|
|
|
+ bool prefault)
|
|
|
+{
|
|
|
+ struct timeval tv_start, tv_end, tv_diff;
|
|
|
+ memcpy_t fn = r->fn.memcpy;
|
|
|
+ void *src = NULL, *dst = NULL;
|
|
|
+ int i;
|
|
|
+
|
|
|
+ memcpy_alloc_mem(&src, &dst, len);
|
|
|
+
|
|
|
+ if (prefault)
|
|
|
+ fn(dst, src, len);
|
|
|
+
|
|
|
+ BUG_ON(gettimeofday(&tv_start, NULL));
|
|
|
+ for (i = 0; i < iterations; ++i)
|
|
|
+ fn(dst, src, len);
|
|
|
+ BUG_ON(gettimeofday(&tv_end, NULL));
|
|
|
+
|
|
|
+ timersub(&tv_end, &tv_start, &tv_diff);
|
|
|
+
|
|
|
+ free(src);
|
|
|
+ free(dst);
|
|
|
+ return (double)(((double)len * iterations) / timeval2double(&tv_diff));
|
|
|
+}
|
|
|
+
|
|
|
+int bench_mem_memcpy(int argc, const char **argv,
|
|
|
+ const char *prefix __maybe_unused)
|
|
|
+{
|
|
|
+ struct bench_mem_info info = {
|
|
|
+ .routines = memcpy_routines,
|
|
|
+ .do_cycle = do_memcpy_cycle,
|
|
|
+ .do_gettimeofday = do_memcpy_gettimeofday,
|
|
|
+ .usage = bench_mem_memcpy_usage,
|
|
|
+ };
|
|
|
+
|
|
|
+ return bench_mem_common(argc, argv, prefix, &info);
|
|
|
+}
|
|
|
+
|
|
|
+static void memset_alloc_mem(void **dst, size_t length)
|
|
|
+{
|
|
|
+ *dst = zalloc(length);
|
|
|
+ if (!*dst)
|
|
|
+ die("memory allocation failed - maybe length is too large?\n");
|
|
|
+}
|
|
|
+
|
|
|
+static u64 do_memset_cycle(const struct routine *r, size_t len, bool prefault)
|
|
|
+{
|
|
|
+ u64 cycle_start = 0ULL, cycle_end = 0ULL;
|
|
|
+ memset_t fn = r->fn.memset;
|
|
|
+ void *dst = NULL;
|
|
|
+ int i;
|
|
|
+
|
|
|
+ memset_alloc_mem(&dst, len);
|
|
|
+
|
|
|
+ if (prefault)
|
|
|
+ fn(dst, -1, len);
|
|
|
+
|
|
|
+ cycle_start = get_cycle();
|
|
|
+ for (i = 0; i < iterations; ++i)
|
|
|
+ fn(dst, i, len);
|
|
|
+ cycle_end = get_cycle();
|
|
|
+
|
|
|
+ free(dst);
|
|
|
+ return cycle_end - cycle_start;
|
|
|
+}
|
|
|
+
|
|
|
+static double do_memset_gettimeofday(const struct routine *r, size_t len,
|
|
|
+ bool prefault)
|
|
|
+{
|
|
|
+ struct timeval tv_start, tv_end, tv_diff;
|
|
|
+ memset_t fn = r->fn.memset;
|
|
|
+ void *dst = NULL;
|
|
|
+ int i;
|
|
|
+
|
|
|
+ memset_alloc_mem(&dst, len);
|
|
|
+
|
|
|
+ if (prefault)
|
|
|
+ fn(dst, -1, len);
|
|
|
+
|
|
|
+ BUG_ON(gettimeofday(&tv_start, NULL));
|
|
|
+ for (i = 0; i < iterations; ++i)
|
|
|
+ fn(dst, i, len);
|
|
|
+ BUG_ON(gettimeofday(&tv_end, NULL));
|
|
|
+
|
|
|
+ timersub(&tv_end, &tv_start, &tv_diff);
|
|
|
+
|
|
|
+ free(dst);
|
|
|
+ return (double)(((double)len * iterations) / timeval2double(&tv_diff));
|
|
|
+}
|
|
|
+
|
|
|
+static const char * const bench_mem_memset_usage[] = {
|
|
|
+ "perf bench mem memset <options>",
|
|
|
+ NULL
|
|
|
+};
|
|
|
+
|
|
|
+static const struct routine memset_routines[] = {
|
|
|
+ { .name ="default",
|
|
|
+ .desc = "Default memset() provided by glibc",
|
|
|
+ .fn.memset = memset },
|
|
|
+#ifdef HAVE_ARCH_X86_64_SUPPORT
|
|
|
+
|
|
|
+#define MEMSET_FN(_fn, _name, _desc) { .name = _name, .desc = _desc, .fn.memset = _fn },
|
|
|
+#include "mem-memset-x86-64-asm-def.h"
|
|
|
+#undef MEMSET_FN
|
|
|
+
|
|
|
+#endif
|
|
|
+
|
|
|
+ { .name = NULL,
|
|
|
+ .desc = NULL,
|
|
|
+ .fn.memset = NULL }
|
|
|
+};
|
|
|
+
|
|
|
+int bench_mem_memset(int argc, const char **argv,
|
|
|
+ const char *prefix __maybe_unused)
|
|
|
+{
|
|
|
+ struct bench_mem_info info = {
|
|
|
+ .routines = memset_routines,
|
|
|
+ .do_cycle = do_memset_cycle,
|
|
|
+ .do_gettimeofday = do_memset_gettimeofday,
|
|
|
+ .usage = bench_mem_memset_usage,
|
|
|
+ };
|
|
|
+
|
|
|
+ return bench_mem_common(argc, argv, prefix, &info);
|
|
|
+}
|