The implementation is based on libcgroup. It can be optionally configured by specifying --with-cgroup.
Building instruction and an example are given in doc/tutorial.txt.
Tested on x86_64 and arm64.
Known issue: The build shows some warnings that some of the functions in libcgroup.a (e.g. cgroup_change_cgroup_flags()) are calling functions (e.g. getgrgid()) which require at run-time the same version of shared libraries from the glibc version used for linking.
Signed-off-by: Dietmar Eggemann dietmar.eggemann@arm.com --- configure.ac | 17 +++ doc/tutorial.txt | 50 +++++++- src/Makefile.am | 2 +- src/rt-app.c | 284 +++++++++++++++++++++++++++++++++++++++++++++- src/rt-app.h | 4 + src/rt-app_parse_config.c | 39 ++++++- src/rt-app_types.h | 23 ++++ 7 files changed, 407 insertions(+), 12 deletions(-)
diff --git a/configure.ac b/configure.ac index 27b4311d04df..8b122b37f796 100644 --- a/configure.ac +++ b/configure.ac @@ -18,6 +18,23 @@ AC_CHECK_LIB([rt], [clock_gettime]) AC_CHECK_LIB([json-c], [json_object_from_file], [], [AC_MSG_ERROR([json-c libraries required])]) AC_CHECK_FUNCS(sched_setattr, [], [])
+AC_ARG_WITH([cgroup], + [AS_HELP_STRING([--with-cgroup], + [Add support for cpu cgroups])], + [], + [with_cgroup=no]) + +LIBCGROUP= +AS_IF([test "x$with_cgroup" != xno], + AC_CHECK_LIB([cgroup], [cgroup_init], + [AC_SUBST([LIBCGROUP], ["-lcgroup"]) + AC_DEFINE([HAVE_LIBCGROUP], [1], [Define if you have CGroup support]) + ], + [AC_MSG_FAILURE([libcgroup test failed (use --without-cgroup to disable or install cgroup)])], + [-lcgroup] + ) + ) + AC_ARG_WITH([deadline], [AS_HELP_STRING([--with-deadline], [Add support for SCHED_DEADLINE])], diff --git a/doc/tutorial.txt b/doc/tutorial.txt index d65a8435d0c9..6595ea87e0f1 100644 --- a/doc/tutorial.txt +++ b/doc/tutorial.txt @@ -36,11 +36,29 @@ export ac_cv_func_realloc_0_nonnull=yes ./configure --host=aarch64-linux-gnu --disable-shared --enable-static make
+For libcgroup: (tested with v0.41) + +./git clone https://github.com/matsumoto-r/libcgroup.git + +export ac_cv_func_malloc_0_nonnull=yes +export ac_cv_func_realloc_0_nonnull=yes +autoreconf -v --install +./configure --host=aarch64-linux-gnu --disable-shared --enable-static --disable-pam --disable-daemon --disable-tools +make + For rt-app:
export ac_cv_lib_json_c_json_object_from_file=yes ./autogen.sh -./configure --host=aarch64-linux-gnu LDFLAGS=" --static -L<path to parent of json repo>/json-c/." CFLAGS="-I<path to parent of json repo>" --with-deadline + +w/o cgroup support: + +./configure --host=aarch64-linux-gnu LDFLAGS="--static -L<path to parent of json repo>/json-c/." CFLAGS="-I<path to parent of json repo>" --with-deadline + +w/ cgroup support: + +./configure --host=aarch64-linux-gnu LDFLAGS="--static -L<path to parent of json repo>/json-c/. -L<path to parent of libcgroup repo>/src/.libs" CFLAGS="-I<path to parent of libcgroup repo>/include --I<path to parent of json repo>" --with-deadline --with-cgroup + make
e.g, with a directory structure like the following: @@ -304,6 +322,36 @@ affinity to CPU 2, and the third phase with affinity to CPU 4, 5, and 6 (it will } }
+* cgroups: String. Can be specified at task level or phase level. Only the cpu +cgroup controller is supported so a cgroup is equal to a task group. The +example below sets up a task 'thread1' with three phases. The task will run in +taskgroup '/tg1/tg11' in the phase1, in '/tg1' in phase2 and in '/' (root +task group) in phase3. Phase 2 does not specify a cgroup so the phase will +inherit from the task. Currently only SCHED_OTHER tasks are supported. + +"task" : { + "thread1" : { + "policy": "SCHED_OTHER", + "cgroup": "/tg1", + "phases" : { + "phase1" : { + "cgroup": "/tg1/tg11", + "run" : 10, + "sleep" : 10 + }, + "phase2" : { + "run" : 10, + "sleep" : 10 + }, + "phase3" : { + "cgroup": "/", + "run" : 10, + "sleep" : 10 + } + } + } +} + *** events ***
events are simple action that will be performed by the thread or on the diff --git a/src/Makefile.am b/src/Makefile.am index 1fe362d1ab4e..0afd24a8d2ec 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -3,7 +3,7 @@ AM_CPPFLAGS = -I$(srcdir)/../libdl/ bin_PROGRAMS = rt-app rt_app_SOURCES= rt-app_types.h rt-app_args.h rt-app_utils.h rt-app_utils.c rt-app_args.c rt-app.h rt-app.c rt_app_SOURCES += rt-app_parse_config.h rt-app_parse_config.c -rt_app_LDADD = $(QRESLIB) +rt_app_LDADD = $(QRESLIB) $(LIBCGROUP) if SET_DLSCHED rt_app_LDADD += ../libdl/libdl.a endif diff --git a/src/rt-app.c b/src/rt-app.c index f3d231722a35..43654620da74 100644 --- a/src/rt-app.c +++ b/src/rt-app.c @@ -37,6 +37,50 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "rt-app_utils.h" #include "rt-app_args.h"
+#ifdef HAVE_LIBCGROUP +/* + * Save and restore the following rt-app defines from src/config.h. libcgroup + * defines its own set and its public header libcgroup.h includes libcgroup's + * config.h. + */ +#define _VERSION VERSION +#define _PACKAGE PACKAGE +#define _PACKAGE_NAME PACKAGE_NAME +#define _PACKAGE_VERSION PACKAGE_VERSION +#define _PACKAGE_TARNAME PACKAGE_TARNAME +#define _PACKAGE_STRING PACKAGE_STRING +#define _PACKAGE_BUGREPORT PACKAGE_BUGREPORT +#undef VERSION +#undef PACKAGE +#undef PACKAGE_NAME +#undef PACKAGE_VERSION +#undef PACKAGE_TARNAME +#undef PACKAGE_STRING +#undef PACKAGE_BUGREPORT +#include <libcgroup.h> +#undef VERSION +#undef PACKAGE +#undef PACKAGE_NAME +#undef PACKAGE_VERSION +#undef PACKAGE_TARNAME +#undef PACKAGE_STRING +#undef PACKAGE_BUGREPORT +#define VERSION _VERSION +#define PACKAGE _PACKAGE +#define PACKAGE_NAME _PACKAGE_NAME +#define PACKAGE_VERSION _PACKAGE_VERSION +#define PACKAGE_TARNAME _PACKAGE_TARNAME +#define PACKAGE_STRING _PACKAGE_STRING +#define PACKAGE_BUGREPORT _PACKAGE_BUGREPORT +#undef _VERSION +#undef _PACKAGE +#undef _PACKAGE_NAME +#undef _PACKAGE_VERSION +#undef _PACKAGE_TARNAME +#undef _PACKAGE_STRING +#undef _PACKAGE_BUGREPORT +#endif + static int errno; static volatile sig_atomic_t continue_running; static pthread_t *threads; @@ -539,6 +583,226 @@ static void create_cpuset_str(cpuset_data_t *cpu_data) strncat(cpu_data->cpuset_str, " ]", size_needed - idx - 1); }
+#ifdef HAVE_LIBCGROUP +static void add_cgroup(rtapp_options_t *opts, const char *name) +{ + cgroup_t *cgroup, *new; + + new = malloc(sizeof *cgroup); + + if (!new) { + log_error("Cannot allocate cgroup"); + exit(EXIT_FAILURE); + } + + strcpy(new->name, name); + + if (TAILQ_EMPTY(&opts->head)) { + TAILQ_INSERT_HEAD(&opts->head, new, cgroups); + } else { + TAILQ_FOREACH(cgroup, &opts->head, cgroups) { + int n = strcmp(new->name, cgroup->name); + + if (n < 0) { + TAILQ_INSERT_BEFORE(cgroup, new, cgroups); + return; + } + + /* Do not add cgroup twice */ + if (!n) + return; + } + TAILQ_INSERT_TAIL(&opts->head, new, cgroups); + } +} + +void split_cgroup_data(rtapp_options_t *opts, char *name) +{ + char *tmp, *p = name; + + if (!strcmp(name, ROOT_CGROUP)) { + log_notice("Cannot split root cgroup, continue ..."); + return; + } + + if (strlen(p) >= PATH_LENGTH) { + log_error("Cgroup path length %lu not supported", strlen(p)); + exit(EXIT_FAILURE); + } + + tmp = malloc(PATH_LENGTH); + + if (!tmp) { + log_error("Cannot allocate tmp buffer"); + exit(EXIT_FAILURE); + } + + strcpy(tmp, p); + + /* remove trailing slash */ + p = &tmp[strlen(tmp)-1]; + + if (*p == '/') { *p = '\0'; } + + for (p = tmp; p;) { + p = strchr(p, '/'); + + if (p) { *p = '\0'; } + + if (strlen(tmp)) + add_cgroup(opts, tmp); + + if (p) { *p++ = '/'; } + } + + free(tmp); +} + +static void create_cgroup(cgroup_t *cgroup) +{ + struct cgroup *group; + struct cgroup_controller *controller; + int res; + + if (!strcmp(cgroup->name, ROOT_CGROUP)) + return; + + log_notice("Create cgroup [%s]", cgroup->name); + + group = cgroup_new_cgroup(cgroup->name); + + if (!group) { + log_critical("Cannot add new cgroup [%s] (libcgroup error: %s)", + cgroup->name, cgroup_strerror(ECGFAIL)); + exit(EXIT_FAILURE); + } + + controller = cgroup_add_controller(group, "cpu"); + + if (!controller) { + log_critical("Cannot add cpu controller to cgroup [%s]", cgroup->name); + exit(EXIT_FAILURE); + } + + res = cgroup_create_cgroup(group, 1); + + if (res) { + log_critical("Cannot create cgroup [%s] (libcgroup error: %s)", + cgroup->name, cgroup_strerror(res)); + exit(EXIT_FAILURE); + } + + cgroup->obj = group; +} + +static void create_cgroups(rtapp_options_t *opts) +{ + cgroup_t *cgroup; + + TAILQ_FOREACH(cgroup, &opts->head, cgroups) { + create_cgroup(cgroup); + } +} + +static void destroy_cgroup(cgroup_t *cgroup) +{ + int res; + + if (!strcmp(cgroup->name, ROOT_CGROUP)) + return; + + log_notice("Destroy cgroup [%s]", cgroup->name); + + res = cgroup_delete_cgroup(cgroup->obj, CGFLAG_DELETE_IGNORE_MIGRATION); + + if (res) { + log_notice("Cannot destroy cgroup (libcgroup error: %s), continue ...", + cgroup_strerror(res)); + return; + } + + cgroup_free(&cgroup->obj); +} + +static void destroy_cgroups(rtapp_options_t *opts) +{ + cgroup_t *cgroup; + + while ((cgroup = TAILQ_LAST(&opts->head, cgrouplist))) { + destroy_cgroup(cgroup); + TAILQ_REMOVE(&opts->head, cgroup, cgroups); + free(cgroup); + } +} + +cgroup_t* find_cgroup(rtapp_options_t *opts, char *name) +{ + cgroup_t *cgroup = NULL, *tmp; + + TAILQ_FOREACH(tmp, &opts->head, cgroups) { + if (!strcmp(tmp->name, name)) { + cgroup = tmp; + break; + } + } + + return cgroup; +} + +static void set_cgroup(cgroup_t *cgroup) +{ + int res; + + log_info("Set cgroup [%s]", cgroup->name); + + res = cgroup_attach_task(cgroup->obj); + + if (res) { + log_critical("Cannot attach task to cgroup [%s] (libcgroup error: %s)", + cgroup->name, cgroup_strerror(res)); + exit(EXIT_FAILURE); + } +} + +static void preinit_cgroup(void) +{ + /* Elements are already added during cmd line parsing */ + TAILQ_INIT(&opts.head); + + add_cgroup(&opts, ROOT_CGROUP); +} + +static void init_cgroup(void) +{ + int res; + char *mount; + + res = cgroup_init(); + + if (res) { + log_error("Cannot initialize cgroup library (libcgroup error: %s)", + cgroup_strerror(res)); + exit(EXIT_FAILURE); + } + + res = cgroup_get_subsys_mount_point("cpu", &mount); + + if (res) { + log_error("Cannot get 'cpu' cgroup controller's moint point (libcgroup error: %s)", + cgroup_strerror(res)); + exit(EXIT_FAILURE); + } + + create_cgroups(&opts); +} + +#else /* HAVE_LIBCGROUP */ +static void preinit_cgroup(void) {} +static void init_cgroup(void) {} +static void set_cgroup(void *cgroup) {} +static void destroy_cgroups(rtapp_options_t *opts) {} +#endif /* HAVE_LIBCGROUP */ + static void set_thread_affinity(thread_data_t *data, cpuset_data_t *cpu_data) { int ret; @@ -807,6 +1071,8 @@ void *thread_body(void *arg) set_thread_affinity(data, &pdata->cpu_data); set_thread_priority(data, pdata->sched_data);
+ set_cgroup(GET_CGROUP(pdata)); + if (opts.ftrace) log_ftrace(ft_data.marker_fd, "[%d] begins thread_loop %d phase %d phase_loop %d", @@ -917,6 +1183,9 @@ int main(int argc, char* argv[]) rtapp_resource_t *rdata; char tmp[PATH_LENGTH]; static cpu_set_t orig_set; + int status = EXIT_SUCCESS; + + preinit_cgroup();
parse_command_line(argc, argv, &opts);
@@ -1109,6 +1378,8 @@ int main(int argc, char* argv[]) /* Sync timer resources with start time */ clock_gettime(CLOCK_MONOTONIC, &t_start);
+ init_cgroup(); + /* Start the use case */ for (i = 0; i < nthreads; i++) { tdata = &opts.threads_data[i]; @@ -1116,8 +1387,10 @@ int main(int argc, char* argv[]) if (pthread_create(&threads[i], NULL, thread_body, - (void*) tdata)) - goto exit_err; + (void*) tdata)) { + status = EXIT_FAILURE; + goto out; + } } running_threads = nthreads;
@@ -1136,9 +1409,8 @@ int main(int argc, char* argv[]) log_ftrace(ft_data.marker_fd, "main ends\n"); close(ft_data.marker_fd); } - exit(EXIT_SUCCESS); - +out: + destroy_cgroups(&opts);
-exit_err: - exit(EXIT_FAILURE); + exit(status); } diff --git a/src/rt-app.h b/src/rt-app.h index 85eccb1fb716..72df4be25e50 100644 --- a/src/rt-app.h +++ b/src/rt-app.h @@ -23,6 +23,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #define _RT_APP_H_
void *thread_body(void *arg); +#ifdef HAVE_LIBCGROUP +void split_cgroup_data(rtapp_options_t *opts, char *name); +cgroup_t* find_cgroup(rtapp_options_t *opts, char *name); +#endif
#endif /* _RT_APP_H_ */
diff --git a/src/rt-app_parse_config.c b/src/rt-app_parse_config.c index 25c3f1bfcc94..497d8ad66fe0 100644 --- a/src/rt-app_parse_config.c +++ b/src/rt-app_parse_config.c @@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "rt-app_utils.h" #include "rt-app_parse_config.h" +#include "rt-app.h"
#define PFX "[json] " #define PFL " "PFX @@ -160,6 +161,7 @@ get_string_value_from(struct json_object *where, { struct json_object *value; char *s_value; + value = get_in_object(where, key, have_def); set_default_if_needed_str(key, value, have_def, def_value); if (json_object_is_type(value, json_type_null)) { @@ -751,9 +753,35 @@ static sched_data_t *parse_sched_data(struct json_object *obj, int def_policy) return NULL; }
+#ifdef HAVE_LIBCGROUP +static void +parse_cgroup_data(struct json_object *obj, phase_data_t *pdata, thread_data_t *tdata, rtapp_options_t *opts, cgroup_t **cgroup) +{ + char *name = get_string_value_from(obj, "cgroup", TRUE, ""); + sched_data_t *sched_data = pdata ? pdata->sched_data : tdata->sched_data; + + if (sched_data && (sched_data->policy != other)) { + log_critical(PIN2 "No cgroup support for policy %s", policy_to_string(sched_data->policy)); + exit(EXIT_INV_CONFIG); + } + + if (strlen(name)) { + if (strcmp(name, ROOT_CGROUP)) + split_cgroup_data(opts, name); + } else { + name = pdata ? tdata->cgroup->name : ROOT_CGROUP; + } + + *cgroup = find_cgroup(opts, name); +} +#else +static void +parse_cgroup_data(struct json_object *obj, phase_data_t *pdata, thread_data_t *tdata, rtapp_options_t *opts, void **cgroup) {} +#endif + static void -parse_thread_phase_data(struct json_object *obj, - phase_data_t *data, rtapp_options_t *opts, long tag) +parse_thread_phase_data(struct json_object *obj, phase_data_t *data, thread_data_t *tdata, + rtapp_options_t *opts, long tag) { /* used in the foreach macro */ struct lh_entry *entry; char *key; struct json_object *val; int idx; @@ -787,6 +815,7 @@ parse_thread_phase_data(struct json_object *obj, parse_cpuset_data(obj, &data->cpu_data); data->sched_data = parse_sched_data(obj, -1);
+ parse_cgroup_data(obj, data, tdata, opts, GET_CGROUP(&data)); }
static void @@ -816,6 +845,8 @@ parse_thread_data(char *name, struct json_object *obj, int index, /* Scheduling policy */ data->sched_data = parse_sched_data(obj, opts->policy);
+ parse_cgroup_data(obj, NULL, data, opts, GET_CGROUP(&data)); + /* initial delay */ data->delay = get_int_value_from(obj, "delay", TRUE, 0);
@@ -839,7 +870,7 @@ parse_thread_data(char *name, struct json_object *obj, int index, foreach(phases_obj, entry, key, val, idx) { log_info(PIN "Parsing phase %s", key); assure_type_is(val, phases_obj, key, json_type_object); - parse_thread_phase_data(val, &data->phases[idx], opts, (long)data); + parse_thread_phase_data(val, &data->phases[idx], data, opts, (long)data); }
/* Get loop number */ @@ -848,7 +879,7 @@ parse_thread_data(char *name, struct json_object *obj, int index, } else { data->nphases = 1; data->phases = malloc(sizeof(phase_data_t) * data->nphases); - parse_thread_phase_data(obj, &data->phases[0], opts, (long)data); + parse_thread_phase_data(obj, &data->phases[0], data, opts, (long)data);
/* There is no "phases" object which means that thread and phase will * use same scheduling parameters. But thread object looks for default diff --git a/src/rt-app_types.h b/src/rt-app_types.h index 7e0686b03db7..900ee3bc7cb7 100644 --- a/src/rt-app_types.h +++ b/src/rt-app_types.h @@ -26,6 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "config.h" #include "dl_syscalls.h" +#include <sys/queue.h>
#define RTAPP_POLICY_DESCR_LENGTH 16 #define RTAPP_RESOURCE_DESCR_LENGTH 16 @@ -134,6 +135,18 @@ typedef struct _rtapp_resource_t { char *name; } rtapp_resource_t;
+#ifdef HAVE_LIBCGROUP +typedef struct _cgroup_t { + char name[PATH_LENGTH]; + struct cgroup *obj; + TAILQ_ENTRY(_cgroup_t) cgroups; +} cgroup_t; +#define ROOT_CGROUP "/" +#define GET_CGROUP(p) (p->cgroup) +#else +#define GET_CGROUP(p) (NULL) +#endif + typedef struct _event_data_t { resource_t type; int res; @@ -163,6 +176,9 @@ typedef struct _phase_data_t { int sched_prio; cpuset_data_t cpu_data; sched_data_t *sched_data; +#ifdef HAVE_LIBCGROUP + cgroup_t *cgroup; +#endif } phase_data_t;
typedef struct _thread_data_t { @@ -188,6 +204,9 @@ typedef struct _thread_data_t { FILE *log_handler;
unsigned long delay; +#ifdef HAVE_LIBCGROUP + cgroup_t *cgroup; +#endif } thread_data_t;
typedef struct _ftrace_data_t { @@ -231,6 +250,10 @@ typedef struct _rtapp_options_t { char *io_device;
int cumulative_slack; + +#ifdef HAVE_LIBCGROUP + TAILQ_HEAD(cgrouplist ,_cgroup_t) head; +#endif } rtapp_options_t;
typedef struct _timing_point_t {