- Linaro-mm-sig - lists.linaro.org

[PATCH v2 5/7] async: Add support for explicit fine-grained barriers

by Chris Wilson

The current async-domain model supports running a multitude of independent tasks with a coarse synchronisation point. This is sufficient for its original purpose of allowing independent drivers to run concurrently during various phases (booting, early resume, late resume etc), and keep the asynchronous domain out of the synchronous kernel domains. However, for greater exploitation, drivers themselves want to schedule multiple tasks within a phase (or between phases) and control the order of execution within those tasks relative to each other. To enable this, we extend the synchronisation scheme based upon kfences and back every task with one. Any task may now wait upon the kfence before being scheduled, and equally the kfence may be used to wait on the task itself (rather than waiting on the cookie for all previous tasks to be completed). Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/async.h | 60 ++++++++- kernel/async.c | 234 ++++++++++++++++++++-------------- lib/test-async-domain.c | 324 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 515 insertions(+), 103 deletions(-) diff --git a/include/linux/async.h b/include/linux/async.h index 6b0226bdaadc..e7d7289a9889 100644 --- a/include/linux/async.h +++ b/include/linux/async.h @@ -13,38 +13,88 @@ #define __ASYNC_H__ #include <linux/types.h> +#include <linux/kfence.h> #include <linux/list.h> typedef u64 async_cookie_t; typedef void (*async_func_t) (void *data, async_cookie_t cookie); + +struct async_work { + struct kfence fence; + /* private */ +}; + struct async_domain { struct list_head pending; unsigned registered:1; }; +#define ASYNC_DOMAIN_INIT(_name, _r) { \ + .pending = LIST_HEAD_INIT(_name.pending), \ + .registered = _r \ +} + /* * domain participates in global async_synchronize_full */ #define ASYNC_DOMAIN(_name) \ - struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \ - .registered = 1 } + struct async_domain _name = ASYNC_DOMAIN_INIT(_name, 1) /* * domain is free to go out of scope as soon as all pending work is * complete, this domain does not participate in async_synchronize_full */ #define ASYNC_DOMAIN_EXCLUSIVE(_name) \ - struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \ - .registered = 0 } + struct async_domain _name = ASYNC_DOMAIN_INIT(_name, 0) + +extern void init_async_domain(struct async_domain *domain, bool registered); extern async_cookie_t async_schedule(async_func_t func, void *data); extern async_cookie_t async_schedule_domain(async_func_t func, void *data, struct async_domain *domain); -void async_unregister_domain(struct async_domain *domain); +extern void async_unregister_domain(struct async_domain *domain); extern void async_synchronize_full(void); extern void async_synchronize_full_domain(struct async_domain *domain); extern void async_synchronize_cookie(async_cookie_t cookie); extern void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain); + extern bool current_is_async(void); + +extern struct async_work * +async_work_create(async_func_t func, void *data, gfp_t gfp); + +static inline struct async_work *async_work_get(struct async_work *work) +{ + kfence_get(&work->fence); + return work; +} + +static inline int +async_work_after(struct async_work *work, struct kfence *fence) +{ + return kfence_await_kfence(&work->fence, fence, GFP_KERNEL); +} + +static inline int +async_work_before(struct async_work *work, struct kfence *fence) +{ + return kfence_await_kfence(fence, &work->fence, GFP_KERNEL); +} + +static inline void async_work_wait(struct async_work *work) +{ + kfence_wait(&work->fence); +} + +static inline void async_work_put(struct async_work *work) +{ + kfence_put(&work->fence); +} + +extern async_cookie_t queue_async_work(struct async_domain *domain, + struct async_work *work, + gfp_t gfp); +extern async_cookie_t schedule_async_work(struct async_work *work); + #endif diff --git a/kernel/async.c b/kernel/async.c index d2edd6efec56..0d695919a60d 100644 --- a/kernel/async.c +++ b/kernel/async.c @@ -2,6 +2,7 @@ * async.c: Asynchronous function calls for boot performance * * (C) Copyright 2009 Intel Corporation + * (C) Copyright 2016 Intel Corporation * Author: Arjan van de Ven <arjan(a)linux.intel.com> * * This program is free software; you can redistribute it and/or @@ -59,59 +60,39 @@ asynchronous and synchronous parts of the kernel. #include "workqueue_internal.h" -static async_cookie_t next_cookie = 1; - +#define ASYNC_QUEUED_BIT KFENCE_PRIVATE_BIT #define MAX_WORK 32768 -#define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */ - -static LIST_HEAD(async_global_pending); /* pending from all registered doms */ -static ASYNC_DOMAIN(async_dfl_domain); -static DEFINE_SPINLOCK(async_lock); struct async_entry { - struct list_head domain_list; - struct list_head global_list; - struct work_struct work; - async_cookie_t cookie; - async_func_t func; - void *data; - struct async_domain *domain; -}; - -static DECLARE_WAIT_QUEUE_HEAD(async_done); + struct async_work base; + struct work_struct work; -static atomic_t entry_count; + struct list_head pending_link[2]; -static async_cookie_t lowest_in_progress(struct async_domain *domain) -{ - struct list_head *pending; - async_cookie_t ret = ASYNC_COOKIE_MAX; - unsigned long flags; + async_cookie_t cookie; + async_func_t func; + void *data; +}; - spin_lock_irqsave(&async_lock, flags); +static LIST_HEAD(async_global_pending); /* pending from all registered doms */ +static ASYNC_DOMAIN(async_dfl_domain); +static DEFINE_SPINLOCK(async_lock); +static unsigned int async_pending_count; - if (domain) - pending = &domain->pending; - else - pending = &async_global_pending; +static async_cookie_t assign_cookie(void) +{ + static async_cookie_t next_cookie; - if (!list_empty(pending)) - ret = list_first_entry(pending, struct async_entry, - domain_list)->cookie; + if (++next_cookie == 0) + next_cookie = 1; - spin_unlock_irqrestore(&async_lock, flags); - return ret; + return next_cookie; } -/* - * pick the first pending entry and run it - */ static void async_run_entry_fn(struct work_struct *work) { - struct async_entry *entry = - container_of(work, struct async_entry, work); - unsigned long flags; - ktime_t uninitialized_var(calltime), delta, rettime; + struct async_entry *entry = container_of(work, typeof(*entry), work); + ktime_t uninitialized_var(calltime); /* 1) run (and print duration) */ if (initcall_debug && system_state == SYSTEM_BOOTING) { @@ -122,8 +103,7 @@ static void async_run_entry_fn(struct work_struct *work) } entry->func(entry->data, entry->cookie); if (initcall_debug && system_state == SYSTEM_BOOTING) { - rettime = ktime_get(); - delta = ktime_sub(rettime, calltime); + ktime_t delta = ktime_sub(ktime_get(), calltime); pr_debug("initcall %lli_%pF returned 0 after %lld usecs\n", (long long)entry->cookie, entry->func, @@ -131,69 +111,81 @@ static void async_run_entry_fn(struct work_struct *work) } /* 2) remove self from the pending queues */ - spin_lock_irqsave(&async_lock, flags); - list_del_init(&entry->domain_list); - list_del_init(&entry->global_list); + spin_lock_irq(&async_lock); + list_del(&entry->pending_link[0]); + list_del(&entry->pending_link[1]); + async_pending_count--; + spin_unlock_irq(&async_lock); - /* 3) free the entry */ - kfree(entry); - atomic_dec(&entry_count); + /* 3) wake up any waiters */ + kfence_wake_up_all(&entry->base.fence); + kfence_put(&entry->base.fence); +} - spin_unlock_irqrestore(&async_lock, flags); +__kfence_call static int async_work_notify(struct kfence *fence) +{ + struct async_entry *entry = + container_of(fence, typeof(*entry), base.fence); + + if (kfence_done(fence)) { + kfree(entry); + return NOTIFY_DONE; + } - /* 4) wake up any waiters */ - wake_up(&async_done); + queue_work(system_unbound_wq, &entry->work); + return NOTIFY_OK; } -static async_cookie_t __async_schedule(async_func_t func, void *data, struct async_domain *domain) +struct async_work *async_work_create(async_func_t func, void *data, gfp_t gfp) { struct async_entry *entry; - unsigned long flags; - async_cookie_t newcookie; - /* allow irq-off callers */ - entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC); + entry = kmalloc(sizeof(*entry), gfp); + if (!entry) + return NULL; - /* - * If we're out of memory or if there's too much work - * pending already, we execute synchronously. - */ - if (!entry || atomic_read(&entry_count) > MAX_WORK) { - kfree(entry); - spin_lock_irqsave(&async_lock, flags); - newcookie = next_cookie++; - spin_unlock_irqrestore(&async_lock, flags); + kfence_init(&entry->base.fence, async_work_notify); - /* low on memory.. run synchronously */ - func(data, newcookie); - return newcookie; - } - INIT_LIST_HEAD(&entry->domain_list); - INIT_LIST_HEAD(&entry->global_list); INIT_WORK(&entry->work, async_run_entry_fn); entry->func = func; entry->data = data; - entry->domain = domain; - spin_lock_irqsave(&async_lock, flags); + return &entry->base; +} +EXPORT_SYMBOL_GPL(async_work_create); - /* allocate cookie and queue */ - newcookie = entry->cookie = next_cookie++; +async_cookie_t queue_async_work(struct async_domain *domain, + struct async_work *work, + gfp_t gfp) +{ + struct async_entry *entry = container_of(work, typeof(*entry), base); + unsigned long flags; - list_add_tail(&entry->domain_list, &domain->pending); - if (domain->registered) - list_add_tail(&entry->global_list, &async_global_pending); + if (WARN_ON(test_and_set_bit(ASYNC_QUEUED_BIT, + &entry->base.fence.flags))) + return 0; - atomic_inc(&entry_count); + spin_lock_irqsave(&async_lock, flags); + entry->cookie = assign_cookie(); + list_add_tail(&entry->pending_link[0], &domain->pending); + INIT_LIST_HEAD(&entry->pending_link[1]); + if (domain->registered) + list_add_tail(&entry->pending_link[1], &async_global_pending); + async_pending_count++; spin_unlock_irqrestore(&async_lock, flags); /* mark that this task has queued an async job, used by module init */ current->flags |= PF_USED_ASYNC; - /* schedule for execution */ - queue_work(system_unbound_wq, &entry->work); + kfence_complete(kfence_get(&entry->base.fence)); + + return entry->cookie; +} +EXPORT_SYMBOL_GPL(queue_async_work); - return newcookie; +async_cookie_t schedule_async_work(struct async_work *work) +{ + return queue_async_work(&async_dfl_domain, work, GFP_KERNEL); } /** @@ -206,7 +198,7 @@ static async_cookie_t __async_schedule(async_func_t func, void *data, struct asy */ async_cookie_t async_schedule(async_func_t func, void *data) { - return __async_schedule(func, data, &async_dfl_domain); + return async_schedule_domain(func, data, &async_dfl_domain); } EXPORT_SYMBOL_GPL(async_schedule); @@ -225,7 +217,27 @@ EXPORT_SYMBOL_GPL(async_schedule); async_cookie_t async_schedule_domain(async_func_t func, void *data, struct async_domain *domain) { - return __async_schedule(func, data, domain); + struct async_work *work; + async_cookie_t cookie = 0; + + work = NULL; + if (READ_ONCE(async_pending_count) < MAX_WORK) + work = async_work_create(func, data, GFP_ATOMIC); + if (work) { + cookie = queue_async_work(domain, work, GFP_ATOMIC); + async_work_put(work); + } + if (!cookie) { + unsigned long flags; + + spin_lock_irqsave(&async_lock, flags); + cookie = assign_cookie(); + spin_unlock_irqrestore(&async_lock, flags); + + func(data, cookie); + } + + return cookie; } EXPORT_SYMBOL_GPL(async_schedule_domain); @@ -251,10 +263,8 @@ EXPORT_SYMBOL_GPL(async_synchronize_full); */ void async_unregister_domain(struct async_domain *domain) { - spin_lock_irq(&async_lock); - WARN_ON(!domain->registered || !list_empty(&domain->pending)); + WARN_ON(!list_empty(&domain->pending)); domain->registered = 0; - spin_unlock_irq(&async_lock); } EXPORT_SYMBOL_GPL(async_unregister_domain); @@ -267,7 +277,7 @@ EXPORT_SYMBOL_GPL(async_unregister_domain); */ void async_synchronize_full_domain(struct async_domain *domain) { - async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain); + async_synchronize_cookie_domain(0, domain); } EXPORT_SYMBOL_GPL(async_synchronize_full_domain); @@ -282,19 +292,49 @@ EXPORT_SYMBOL_GPL(async_synchronize_full_domain); */ void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain) { - ktime_t uninitialized_var(starttime), delta, endtime; + ktime_t uninitialized_var(starttime); + struct list_head *pending; + + pending = domain ? &domain->pending : &async_global_pending; if (initcall_debug && system_state == SYSTEM_BOOTING) { pr_debug("async_waiting @ %i\n", task_pid_nr(current)); starttime = ktime_get(); } - wait_event(async_done, lowest_in_progress(domain) >= cookie); + do { + struct kfence *fence = NULL; + unsigned long flags; - if (initcall_debug && system_state == SYSTEM_BOOTING) { - endtime = ktime_get(); - delta = ktime_sub(endtime, starttime); + spin_lock_irqsave(&async_lock, flags); + if (!list_empty(pending)) { + struct async_entry *entry; + + if (cookie) { + entry = list_first_entry(pending, + struct async_entry, + pending_link[!domain]); + if ((s64)(cookie - entry->cookie) > 0) + fence = kfence_get(&entry->base.fence); + } else { + entry = list_last_entry(pending, + struct async_entry, + pending_link[!domain]); + cookie = entry->cookie; + fence = kfence_get(&entry->base.fence); + } + } + spin_unlock_irqrestore(&async_lock, flags); + + if (!fence) + break; + + kfence_wait(fence); + kfence_put(fence); + } while (1); + if (initcall_debug && system_state == SYSTEM_BOOTING) { + ktime_t delta = ktime_sub(ktime_get(), starttime); pr_debug("async_continuing @ %i after %lli usec\n", task_pid_nr(current), (long long)ktime_to_ns(delta) >> 10); @@ -327,3 +367,11 @@ bool current_is_async(void) return worker && worker->current_func == async_run_entry_fn; } EXPORT_SYMBOL_GPL(current_is_async); + +void init_async_domain(struct async_domain *domain, bool registered) +{ + memset(domain, 0, sizeof(*domain)); + INIT_LIST_HEAD(&domain->pending); + domain->registered = registered; +} +EXPORT_SYMBOL_GPL(init_async_domain); diff --git a/lib/test-async-domain.c b/lib/test-async-domain.c index 558a71414fb6..ecbeba9cd65b 100644 --- a/lib/test-async-domain.c +++ b/lib/test-async-domain.c @@ -7,6 +7,19 @@ #include <linux/async.h> #include <linux/module.h> #include <linux/delay.h> +#include <linux/slab.h> + +static struct kfence *alloc_kfence(void) +{ + struct kfence *fence; + + fence = kmalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return NULL; + + kfence_init(fence, NULL); + return fence; +} static void task_A(void *data, async_cookie_t cookie) { @@ -21,6 +34,269 @@ static void task_B(void *data, async_cookie_t cookie) smp_store_mb(*result, 'B'); } +static int __init test_x(const char *name, + struct async_domain *domain, + async_func_t func, + const long expected) +{ + struct async_work *A; + long result = 0; + + A = async_work_create(func, &result, GFP_KERNEL); + if (!A) + return -ENOMEM; + + queue_async_work(domain, A, GFP_KERNEL); + async_work_wait(A); + async_work_put(A); + + if (READ_ONCE(result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + name, (char)expected, expected, result); + return -EINVAL; + } + + return 0; +} + +static int __init test_A(struct async_domain *domain) +{ + return test_x(__func__, domain, task_A, 'A'); +} + +static int __init test_B(struct async_domain *domain) +{ + return test_x(__func__, domain, task_B, 'B'); +} + +static int __init test_x_fence(const char *name, + struct async_domain *domain, + async_func_t func, + const long expected) +{ + struct async_work *A; + struct kfence *fence; + long result = 0; + + A = async_work_create(func, &result, GFP_KERNEL); + if (!A) + return -ENOMEM; + + fence = alloc_kfence(); + if (!fence) + return -ENOMEM; + + queue_async_work(domain, A, GFP_KERNEL); + + kfence_await_kfence(fence, &A->fence, GFP_KERNEL); + kfence_complete(fence); + + kfence_wait(fence); + + async_work_put(A); + kfence_put(fence); + + if (READ_ONCE(result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + name, (char)expected, expected, result); + return -EINVAL; + } + + return 0; +} + +static int __init test_A_fence(struct async_domain *domain) +{ + return test_x_fence(__func__, domain, task_A, 'A'); +} + +static int __init test_B_fence(struct async_domain *domain) +{ + return test_x_fence(__func__, domain, task_B, 'B'); +} + +static int __init test_x_fence_y(const char *name, + struct async_domain *domain, + async_func_t x, + async_func_t y, + const long expected) +{ + struct async_work *A, *B; + struct kfence *fence; + long result = 0; + + A = async_work_create(x, &result, GFP_KERNEL); + if (!A) + return -ENOMEM; + + B = async_work_create(y, &result, GFP_KERNEL); + if (!B) + return -ENOMEM; + + fence = alloc_kfence(); + if (!fence) + return -ENOMEM; + + kfence_await_kfence(fence, &A->fence, GFP_KERNEL); + kfence_complete(fence); + + queue_async_work(domain, A, GFP_KERNEL); + async_work_put(A); + + async_work_after(B, fence); + queue_async_work(domain, B, GFP_KERNEL); + kfence_put(fence); + + async_work_wait(B); + async_work_put(B); + + if (READ_ONCE(result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + name, (char)expected, expected, result); + return -EINVAL; + } + + return 0; +} + +static int __init test_A_fence_B(struct async_domain *domain) +{ + return test_x_fence_y(__func__, domain, task_A, task_B, 'B'); +} + +static int __init test_B_fence_A(struct async_domain *domain) +{ + return test_x_fence_y(__func__, domain, task_B, task_A, 'A'); +} + +struct long_context { + struct kfence *barrier; + long *src; + long result; +}; + +static void task_wait(void *data, async_cookie_t cookie) +{ + struct long_context *ctx = data; + + kfence_wait(ctx->barrier); + smp_store_mb(ctx->result, READ_ONCE(*ctx->src)); +} + +static int __init test_pause(struct async_domain *domain) +{ + struct long_context ctx; + struct async_work *A, *B; + const long expected = 'B'; + long out_B = 'A'; + + ctx.result = 0; + ctx.src = &out_B; + + A = async_work_create(task_wait, &ctx, GFP_KERNEL); + if (!A) + return -ENOMEM; + + B = async_work_create(task_B, &out_B, GFP_KERNEL); + if (!B) + return -ENOMEM; + + ctx.barrier = kfence_get(&B->fence); + + queue_async_work(domain, A, GFP_KERNEL); + queue_async_work(domain, B, GFP_KERNEL); + async_work_put(B); + + async_work_wait(A); + async_work_put(A); + + if (READ_ONCE(ctx.result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + __func__, (char)expected, expected, ctx.result); + return -EINVAL; + } + + kfence_put(ctx.barrier); + + return 0; +} + +static void task_signal(void *data, async_cookie_t cookie) +{ + struct long_context *ctx = data; + + kfence_complete(ctx->barrier); +} + +static int __init test_manual(struct async_domain *domain) +{ + struct long_context ctx; + struct async_work *A, *B, *C; + const long expected = 'B'; + long out_B = 'A'; + + ctx.result = 0; + ctx.src = &out_B; + ctx.barrier = alloc_kfence(); + + A = async_work_create(task_wait, &ctx, GFP_KERNEL); + if (!A) + return -ENOMEM; + + B = async_work_create(task_B, &out_B, GFP_KERNEL); + if (!B) + return -ENOMEM; + + C = async_work_create(task_signal, &ctx, GFP_KERNEL); + if (!B) + return -ENOMEM; + + async_work_after(C, &B->fence); + + queue_async_work(domain, A, GFP_KERNEL); + queue_async_work(domain, B, GFP_KERNEL); + queue_async_work(domain, C, GFP_KERNEL); + + async_work_wait(A); + + async_work_put(C); + async_work_put(B); + async_work_put(A); + kfence_put(ctx.barrier); + + if (READ_ONCE(ctx.result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + __func__, (char)expected, expected, ctx.result); + return -EINVAL; + } + + return 0; +} + +static int __init test_sync(struct async_domain *domain) +{ + struct async_work *B; + const long expected = 'B'; + long result = 0; + + B = async_work_create(task_B, &result, GFP_KERNEL); + if (!B) + return -ENOMEM; + + queue_async_work(domain, B, GFP_KERNEL); + async_work_put(B); + + async_synchronize_full_domain(domain); + + if (READ_ONCE(result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + __func__, (char)expected, expected, result); + return -EINVAL; + } + + return 0; +} + static int __init test_implicit(struct async_domain *domain) { const long expected = 'B'; @@ -99,24 +375,62 @@ static int __init test_async_domain_init(void) pr_info("Testing async-domains\n"); - ret = test_implicit(&domain); + ret = test_A(&domain); if (ret) return ret; + ret = test_A_fence(&domain); + if (ret) + goto err; + + ret = test_A_fence_B(&domain); + if (ret) + goto err; + + ret = test_B(&domain); + if (ret) + goto err; + + ret = test_B_fence(&domain); + if (ret) + goto err; + + ret = test_B_fence_A(&domain); + if (ret) + goto err; + + ret = test_pause(&domain); + if (ret) + goto err; + + ret = test_manual(&domain); + if (ret) + goto err; + + ret = test_sync(&domain); + if (ret) + goto err; + + ret = test_implicit(&domain); + if (ret) + goto err; + ret = test_registered(&domain); if (ret) - return ret; + goto err; ret = perf_nop(1, 100); if (ret) - return ret; + goto err; ret = perf_nop(128, 1000); if (ret) - return ret; + goto err; +err: + async_synchronize_full_domain(&domain); async_unregister_domain(&domain); - return 0; + return ret; } static void __exit test_async_domain_cleanup(void) -- 2.8.1

9 years, 7 months

1
0
0 0

[PATCH v2 3/7] kfence: Extend kfences for listening on DMA fences

by Chris Wilson

dma-buf provides an interfaces for receiving notifications from DMA hardware, and for implicitly tracking fences used for rendering into dma-buf. We want to be able to use these event sources along with kfence for easy collection and combining with other events. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- drivers/dma-buf/fence.c | 58 +++++++++++++++++++++++++++++++++++++++++++ drivers/dma-buf/reservation.c | 48 +++++++++++++++++++++++++++++++++++ include/linux/fence.h | 6 +++++ include/linux/kfence.h | 2 ++ include/linux/reservation.h | 7 ++++++ kernel/kfence.c | 8 ++++++ 6 files changed, 129 insertions(+) diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c index 7b05dbe9b296..3f06b3b1b4cc 100644 --- a/drivers/dma-buf/fence.c +++ b/drivers/dma-buf/fence.c @@ -22,6 +22,7 @@ #include <linux/export.h> #include <linux/atomic.h> #include <linux/fence.h> +#include <linux/kfence.h> #define CREATE_TRACE_POINTS #include <trace/events/fence.h> @@ -530,3 +531,60 @@ fence_init(struct fence *fence, const struct fence_ops *ops, trace_fence_init(fence); } EXPORT_SYMBOL(fence_init); + +struct dma_fence_cb { + struct fence_cb base; + struct kfence *fence; +}; + +static void dma_kfence_wake(struct fence *dma, struct fence_cb *data) +{ + struct dma_fence_cb *cb = container_of(data, typeof(*cb), base); + + kfence_complete(cb->fence); + kfence_put(cb->fence); + kfree(cb); +} + +/** + * kfence_await_dma_fence - set the fence to wait upon a DMA fence + * @fence: this kfence + * @dma: target DMA fence to wait upon + * @gfp: the allowed allocation type + * + * kfence_add_dma() causes the @fence to wait upon completion of a DMA fence. + * + * Returns 1 if the @fence was successfully to the waitqueue of @dma, 0 + * if @dma was already signaled (and so not added), or a negative error code. + */ +int kfence_await_dma_fence(struct kfence *fence, struct fence *dma, gfp_t gfp) +{ + struct dma_fence_cb *cb; + int ret; + + if (fence_is_signaled(dma)) + return 0; + + cb = kmalloc(sizeof(*cb), gfp); + if (!cb) { + if (!gfpflags_allow_blocking(gfp)) + return -ENOMEM; + + return fence_wait(dma, false); + } + + cb->fence = kfence_get(fence); + kfence_await(fence); + + ret = fence_add_callback(dma, &cb->base, dma_kfence_wake); + if (ret == 0) { + ret = 1; + } else { + dma_kfence_wake(dma, &cb->base); + if (ret == -ENOENT) /* fence already signaled */ + ret = 0; + } + + return ret; +} +EXPORT_SYMBOL_GPL(kfence_await_dma_fence); diff --git a/drivers/dma-buf/reservation.c b/drivers/dma-buf/reservation.c index 9566a62ad8e3..138b792af0c3 100644 --- a/drivers/dma-buf/reservation.c +++ b/drivers/dma-buf/reservation.c @@ -543,3 +543,51 @@ unlock_retry: goto retry; } EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu); + +/** + * kfence_add_reservation - set the fence to wait upon a reservation_object + * @fence: this kfence + * @resv: target reservation_object (collection of DMA fences) to wait upon + * @write: Wait for read or read/write access + * @gfp: the allowed allocation type + * + * kfence_add_reservation() causes the @fence to wait upon completion of the + * reservation object (a collection of DMA fences), either for read access + * or for read/write access. + * + * Returns 1 if the @fence was successfully to the waitqueues of @resv, 0 + * if @resev was already signaled (and so not added), or a negative error code. + */ +int kfence_await_reservation(struct kfence *fence, + struct reservation_object *resv, + bool write, + gfp_t gfp) +{ + struct fence *excl, **shared; + unsigned int count, i; + int ret; + + ret = reservation_object_get_fences_rcu(resv, &excl, &count, &shared); + if (ret) + return ret; + + if (write) { + for (i = 0; i < count; i++) { + ret |= kfence_await_dma_fence(fence, shared[i], gfp); + if (ret < 0) + goto out; + } + } + + if (excl) + ret |= kfence_await_dma_fence(fence, excl, gfp); + +out: + fence_put(excl); + for (i = 0; i < count; i++) + fence_put(shared[i]); + kfree(shared); + + return ret; +} +EXPORT_SYMBOL_GPL(kfence_await_reservation); diff --git a/include/linux/fence.h b/include/linux/fence.h index 2056e9fd0138..3c3bc318e826 100644 --- a/include/linux/fence.h +++ b/include/linux/fence.h @@ -34,6 +34,8 @@ struct fence; struct fence_ops; struct fence_cb; +struct kfence; + /** * struct fence - software synchronization primitive * @refcount: refcount for this fence @@ -378,4 +380,8 @@ unsigned fence_context_alloc(unsigned num); ##args); \ } while (0) +int kfence_await_dma_fence(struct kfence *fence, + struct fence *dma, + gfp_t gfp); + #endif /* __LINUX_FENCE_H */ diff --git a/include/linux/kfence.h b/include/linux/kfence.h index 76a2f95dfb70..acbfc2ea7c49 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -16,6 +16,8 @@ #include <linux/wait.h> struct completion; +struct fence; +struct reservation_object; enum hrtimer_mode; struct kfence { diff --git a/include/linux/reservation.h b/include/linux/reservation.h index b0f305e77b7f..1954bab95db9 100644 --- a/include/linux/reservation.h +++ b/include/linux/reservation.h @@ -49,6 +49,8 @@ extern struct ww_class reservation_ww_class; extern struct lock_class_key reservation_seqcount_class; extern const char reservation_seqcount_string[]; +struct kfence; + /** * struct reservation_object_list - a list of shared fences * @rcu: for internal use @@ -210,4 +212,9 @@ long reservation_object_wait_timeout_rcu(struct reservation_object *obj, bool reservation_object_test_signaled_rcu(struct reservation_object *obj, bool test_all); +int kfence_await_reservation(struct kfence *fence, + struct reservation_object *resv, + bool write, + gfp_t gfp); + #endif /* _LINUX_RESERVATION_H */ diff --git a/kernel/kfence.c b/kernel/kfence.c index 59c27910a749..4605eabc2c1b 100644 --- a/kernel/kfence.c +++ b/kernel/kfence.c @@ -7,7 +7,9 @@ * of the License. */ +#include <linux/fence.h> #include <linux/kfence.h> +#include <linux/reservation.h> #include <linux/slab.h> /** @@ -51,6 +53,12 @@ * - kfence_await_hrtimer(): the kfence asynchronously wait for an expiration * of a timer * + * - kfence_await_dma_fence(): the kfence asynchronously waits for a DMA + * (hardware signaled) fence + * + * - kfence_await_reservation(): the kfence asynchronously waits for a DMA + * reservation object + * * A kfence is initialised using kfence_init(), and starts off awaiting an * event. Once you have finished setting up the fence, including adding * all of its asynchronous waits, call kfence_complete(). -- 2.8.1

9 years, 7 months

1
0
0 0

[PATCH v2 2/7] kfence: Wrap hrtimer to provide a time source for a kfence

by Chris Wilson

A common requirement when scheduling a task is that it should be not be begun until a certain point in time is passed (e.g. queue_delayed_work()). kfence_await_hrtimer() causes the kfence to asynchronously wait until after the appropriate time before being woken. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/kfence.h | 5 +++++ kernel/kfence.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ lib/test-kfence.c | 44 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+) diff --git a/include/linux/kfence.h b/include/linux/kfence.h index 6e32385b3b8c..76a2f95dfb70 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -16,6 +16,7 @@ #include <linux/wait.h> struct completion; +enum hrtimer_mode; struct kfence { wait_queue_head_t wait; @@ -43,6 +44,10 @@ int kfence_await_kfence(struct kfence *fence, int kfence_await_completion(struct kfence *fence, struct completion *x, gfp_t gfp); +int kfence_await_hrtimer(struct kfence *fence, + clockid_t clock, enum hrtimer_mode mode, + ktime_t delay, u64 slack, + gfp_t gfp); void kfence_complete(struct kfence *fence); void kfence_wake_up_all(struct kfence *fence); void kfence_wait(struct kfence *fence); diff --git a/kernel/kfence.c b/kernel/kfence.c index 693af9da545a..59c27910a749 100644 --- a/kernel/kfence.c +++ b/kernel/kfence.c @@ -48,6 +48,9 @@ * - kfence_await_completion(): the kfence asynchronously waits upon a * completion * + * - kfence_await_hrtimer(): the kfence asynchronously wait for an expiration + * of a timer + * * A kfence is initialised using kfence_init(), and starts off awaiting an * event. Once you have finished setting up the fence, including adding * all of its asynchronous waits, call kfence_complete(). @@ -429,3 +432,58 @@ int kfence_await_completion(struct kfence *fence, return pending; } EXPORT_SYMBOL_GPL(kfence_await_completion); + +struct timer_cb { + struct hrtimer timer; + struct kfence *fence; +}; + +static enum hrtimer_restart +timer_kfence_wake(struct hrtimer *timer) +{ + struct timer_cb *cb = container_of(timer, typeof(*cb), timer); + + kfence_complete(cb->fence); + kfence_put(cb->fence); + kfree(cb); + + return HRTIMER_NORESTART; +} + +/** + * kfence_await_hrtimer - set the fence to wait for a period of time + * @fence: this kfence + * @clock: which clock to program + * @mode: delay given as relative or absolute + * @delay: how long or until what time to wait + * @slack: how much slack that may be applied to the delay + * + * kfence_await_hrtimer() causes the @fence to wait for a a period of time, or + * until a certain point in time. It is a convenience wrapper around + * hrtimer_start_range_ns(). For more details on @clock, @mode, @delay and + * @slack please consult the hrtimer documentation. + * + * Returns 1 if the delay was sucessfuly added to the @fence, or a negative + * error code on failure. + */ +int kfence_await_hrtimer(struct kfence *fence, + clockid_t clock, enum hrtimer_mode mode, + ktime_t delay, u64 slack, + gfp_t gfp) +{ + struct timer_cb *cb; + + cb = kmalloc(sizeof(*cb), gfp); + if (!cb) + return -ENOMEM; + + cb->fence = kfence_get(fence); + kfence_await(fence); + + hrtimer_init(&cb->timer, clock, mode); + cb->timer.function = timer_kfence_wake; + + hrtimer_start_range_ns(&cb->timer, delay, slack, mode); + return 1; +} +EXPORT_SYMBOL_GPL(kfence_await_hrtimer); diff --git a/lib/test-kfence.c b/lib/test-kfence.c index b40719fce967..1b0853fda7c3 100644 --- a/lib/test-kfence.c +++ b/lib/test-kfence.c @@ -352,6 +352,44 @@ static int __init test_completion(void) return 0; } +static int __init test_delay(void) +{ + struct kfence *fence; + ktime_t delay; + int ret; + + /* Test use of a hrtimer as an event source for kfences */ + pr_debug("%s\n", __func__); + + fence = alloc_kfence(); + if (!fence) + return -ENOMEM; + + delay = ktime_get(); + + ret = kfence_await_hrtimer(fence, CLOCK_MONOTONIC, HRTIMER_MODE_REL, + ms_to_ktime(1), 1 << 10, + GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_complete(fence); + kfence_wait(fence); + + delay = ktime_sub(ktime_get(), delay); + kfence_put(fence); + + if (!ktime_to_ms(delay)) { + pr_err("kfence woke too early, delay was only %lldns\n", + (long long)ktime_to_ns(delay)); + return -EINVAL; + } + + return 0; +} + struct task_ipc { struct work_struct work; struct completion started; @@ -522,6 +560,12 @@ static int __init test_kfence_init(void) return ret; } + ret = test_delay(); + if (ret < 0) { + pr_err("delay failed\n"); + return ret; + } + return 0; } -- 2.8.1

9 years, 7 months

1
0
0 0

[PATCH v2 1/7] kfence: Introduce kfence, a N:M completion mechanism

by Chris Wilson

Completions are a simple synchronization mechanism, suitable for 1:M barriers where many waiters maybe waiting for a single event. However, some event driven mechanisms require a graph of events where one event may depend upon several earlier events. The kfence extends the struct completion to be able to asynchronously wait upon several event sources, including completions and other fences forming the basis on which an acyclic dependency graph can be built. Most often this is used to create a set of interdependent tasks that can be run concurrently but yet serialised where necessary. For example, the kernel init sequence has many tasks that could be run in parallel so long as their dependencies on previous tasks have been completed. Similarly we have the problem of assigning interdependent tasks to multiple hardware execution engines, be they used for rendering or for display. kfences provides a building block which can be used for determining an order in which tasks can execute. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/kfence.h | 64 ++++ kernel/Makefile | 2 +- kernel/kfence.c | 431 +++++++++++++++++++++++++++ lib/Kconfig.debug | 23 ++ lib/Makefile | 1 + lib/test-kfence.c | 536 ++++++++++++++++++++++++++++++++++ tools/testing/selftests/lib/kfence.sh | 10 + 7 files changed, 1066 insertions(+), 1 deletion(-) create mode 100644 include/linux/kfence.h create mode 100644 kernel/kfence.c create mode 100644 lib/test-kfence.c create mode 100755 tools/testing/selftests/lib/kfence.sh diff --git a/include/linux/kfence.h b/include/linux/kfence.h new file mode 100644 index 000000000000..6e32385b3b8c --- /dev/null +++ b/include/linux/kfence.h @@ -0,0 +1,64 @@ +/* + * kfence.h - library routines for N:M synchronisation points + * + * Copyright (C) 2016 Intel Corporation + * + * This file is released under the GPLv2. + * + */ + +#ifndef _KFENCE_H_ +#define _KFENCE_H_ + +#include <linux/gfp.h> +#include <linux/kref.h> +#include <linux/notifier.h> /* for NOTIFY_DONE */ +#include <linux/wait.h> + +struct completion; + +struct kfence { + wait_queue_head_t wait; + unsigned long flags; + struct kref kref; + atomic_t pending; +}; + +#define KFENCE_CHECKED_BIT 0 /* used internally for DAG checking */ +#define KFENCE_PRIVATE_BIT 1 /* available for use by owner */ +#define KFENCE_MASK (~3) + +#define __kfence_call __aligned(4) +typedef int (*kfence_notify_t)(struct kfence *); + +void kfence_init(struct kfence *fence, kfence_notify_t fn); + +struct kfence *kfence_get(struct kfence *fence); +void kfence_put(struct kfence *fence); + +void kfence_await(struct kfence *fence); +int kfence_await_kfence(struct kfence *fence, + struct kfence *after, + gfp_t gfp); +int kfence_await_completion(struct kfence *fence, + struct completion *x, + gfp_t gfp); +void kfence_complete(struct kfence *fence); +void kfence_wake_up_all(struct kfence *fence); +void kfence_wait(struct kfence *fence); + +/** + * kfence_done - report when the fence has been passed + * @fence: the kfence to query + * + * kfence_done() reports true when the fence is no longer waiting for any + * events and has completed its fence-complete notification. + * + * Returns true when the fence has been passed, false otherwise. + */ +static inline bool kfence_done(const struct kfence *fence) +{ + return atomic_read(&fence->pending) < 0; +} + +#endif /* _KFENCE_H_ */ diff --git a/kernel/Makefile b/kernel/Makefile index e2ec54e2b952..ff11f31b7ec9 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -9,7 +9,7 @@ obj-y = fork.o exec_domain.o panic.o \ extable.o params.o \ kthread.o sys_ni.o nsproxy.o \ notifier.o ksysfs.o cred.o reboot.o \ - async.o range.o smpboot.o + async.o kfence.o range.o smpboot.o obj-$(CONFIG_MULTIUSER) += groups.o diff --git a/kernel/kfence.c b/kernel/kfence.c new file mode 100644 index 000000000000..693af9da545a --- /dev/null +++ b/kernel/kfence.c @@ -0,0 +1,431 @@ +/* + * (C) Copyright 2016 Intel Corporation + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + */ + +#include <linux/kfence.h> +#include <linux/slab.h> + +/** + * DOC: kfence overview + * + * kfences provide synchronisation barriers between multiple tasks. They are + * very similar to completions, or the OpenGL fence synchronisation object. + * Where kfences differ from completions is their ability to track multiple + * event sources rather than being a singular "completion event". Similar to + * completions multiple processes can wait upon a kfence. However, unlike + * completions, a kfence can wait upon other kfences allowing for a graph + * of interdependent events. + * + * Each kfence is a one-shot flag, signaling that work has progressed past + * a certain point (as measured by completion of all events the kfence is + * listening for) and the waiters upon that kfence may proceed. + * + * kfences provide both signaling and waiting routines: + * + * - kfence_await(): indicates that the kfence is asynchronously waiting for + * another event. + * + * - kfence_complete(): undoes the earlier await and marks the fence as done + * if all of its pending events have been completed. + * + * - kfence_done(): reports whether or not the kfence has been passed. + * + * - kfence_wait(): allows the caller to sleep (uninterruptibly) until the + * fence is passed. + * + * This interface is very similar to completions, with the exception of + * allowing the fence to await multiple events. kfences can wait upon other + * fences or other hardware events, building an ordered dependency graph: + * + * - kfence_await_kfence(): the kfence asynchronously waits upon completion + * of another kfence + * + * - kfence_await_completion(): the kfence asynchronously waits upon a + * completion + * + * A kfence is initialised using kfence_init(), and starts off awaiting an + * event. Once you have finished setting up the fence, including adding + * all of its asynchronous waits, call kfence_complete(). + * + * Unlike completions, kfences are expected to live inside more complex graphs + * and form the basis for parallel execution of interdependent tasks and so are + * reference counted. Use kfence_get() and kfence_put() to acquire or release + * a reference to the kfence respectively. + * + * The kfence can be embedded inside a larger structure and be used as part + * of its event driven mechanism. As such kfence_init() can be passed a + * callback function that will be called first when the kfence is completed, + * and again when the kfence is to be freed. If no callback is provided, the + * kfence will be freed using kfree() when its reference count hits zero - + * if it is embedded inside another structure and no callback is provided, + * it must be the first member of its parent struct. + * + * The fence-completed notification is called before any listeners upon the + * fence are signaled, or any waiters woken. You can defer their wake up by + * returning NOTIFY_OK from the fence-completed notification and calling + * kfence_wake_up_all() later when ready. + */ + +static DEFINE_SPINLOCK(kfence_lock); + +static int __kfence_notify(struct kfence *fence) +{ + kfence_notify_t fn; + + fn = (kfence_notify_t)(fence->flags & KFENCE_MASK); + return fn(fence); +} + +static void kfence_free(struct kref *kref) +{ + struct kfence *fence = container_of(kref, typeof(*fence), kref); + + WARN_ON(atomic_read(&fence->pending) > 0); + + if (fence->flags & KFENCE_MASK) + WARN_ON(__kfence_notify(fence) != NOTIFY_DONE); + else + kfree(fence); +} + +/** + * kfence_put - release a reference to a kfence + * @fence: the kfence being disposed of + * + * kfence_put() decrements the reference count on the @fence, and when + * it hits zero the fence will be freed. + */ +void kfence_put(struct kfence *fence) +{ + kref_put(&fence->kref, kfence_free); +} +EXPORT_SYMBOL_GPL(kfence_put); + +/** + * kfence_get - acquire a reference to a kfence + * @fence: the kfence being used + * + * Returns the pointer to the kfence, with its reference count incremented. + */ +struct kfence *kfence_get(struct kfence *fence) +{ + kref_get(&fence->kref); + return fence; +} +EXPORT_SYMBOL_GPL(kfence_get); + +static void __kfence_wake_up_all(struct kfence *fence, + struct list_head *continuation) +{ + wait_queue_head_t *x = &fence->wait; + unsigned long flags; + + atomic_dec(&fence->pending); + + /* + * To prevent unbounded recursion as we traverse the graph of kfences, + * we move the task_list from this the next ready fence to the tail of + * the original fence's task_list (and so added to the list to be + * woken). + */ + smp_mb__before_spinlock(); + spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); + if (continuation) { + list_splice_tail_init(&x->task_list, continuation); + } else { + while (!list_empty(&x->task_list)) + __wake_up_locked_key(x, TASK_NORMAL, &x->task_list); + } + spin_unlock_irqrestore(&x->lock, flags); +} + +/** + * kfence_wake_up_all - wake all waiters upon a fence + * @fence: the kfence to signal + * + * If the fence-complete notification is deferred, when the callback is + * complete it should call kfence_wake_up_all() to wake up all waiters + * upon the fence. + * + * It is invalid to call kfence_wake_up_all() at any time other than from + * inside a deferred fence-complete notification. + */ +void kfence_wake_up_all(struct kfence *fence) +{ + WARN_ON(atomic_read(&fence->pending) != 0); + __kfence_wake_up_all(fence, NULL); +} + +static void __kfence_complete(struct kfence *fence, + struct list_head *continuation) +{ + if (!atomic_dec_and_test(&fence->pending)) + return; + + if (fence->flags & KFENCE_MASK && __kfence_notify(fence) != NOTIFY_DONE) + return; + + __kfence_wake_up_all(fence, continuation); +} + +/** + * kfence_await - increment the count of events being asynchronously waited upon + * @fence: the kfence + * + * kfence_await() indicates that the @fence is waiting upon the completion + * of an event. The @fence may wait upon multiple events concurrently. + * When that event is complete, a corresponding call to kfence_complete() + * should be made. + */ +void kfence_await(struct kfence *fence) +{ + WARN_ON(atomic_inc_return(&fence->pending) <= 1); +} +EXPORT_SYMBOL_GPL(kfence_await); + +/** + * kfence_complete - decrement the count of events waited upon + * @fence: the kfence + * + * When all event sources for the @fence are completed, i.e. the event count + * hits zero, all waiters upon the @fence are woken up. + */ +void kfence_complete(struct kfence *fence) +{ + if (WARN_ON(kfence_done(fence))) + return; + + __kfence_complete(fence, NULL); +} +EXPORT_SYMBOL_GPL(kfence_complete); + +/** + * kfence_wait - wait upon a fence to be completed + * @fence: the kfence to wait upon + * + * Blocks (uninterruptibly waits) until the @fence event counter reaches zero + * and then also waits for the fence-completed notification to finish. + */ +void kfence_wait(struct kfence *fence) +{ + wait_event(fence->wait, kfence_done(fence)); +} +EXPORT_SYMBOL_GPL(kfence_wait); + +/** + * kfence_init - initialize a fence for embedded use within a struct + * @fence: this kfence + * @fn: a callback function for when the fence is complete, and when the + * fence is released + * + * This function initialises the @fence for use embedded within a parent + * structure. The optional @fn hook is first called when the fence is completed + * (when all its pending event count hits 0) and again when the fence is + * to be freed. Note that the @fn will be called from atomic context. The @fn + * is stored inside the fence mixed with some flags, and so the @fn must + * be aligned using the __kfence_call function attribute. + * + * If the @fn is not provided, the kfence must be the first member in its + * parent struct as it will be freed using kfree(). + * + * fence-complete notification: @fn will be called when the pending event + * count hits 0, however the fence is not completed unless the callback + * returns NOTIFY_DONE. During this notification callback fence_done() reports + * false. You can suspend completion of the fence by returning + * NOTIFY_OK instead and then later calling kfence_wake_up_all(). + * + * fence-release notification: @fn will be called when the reference count + * hits 0, fence_done() will report true. + */ +void kfence_init(struct kfence *fence, kfence_notify_t fn) +{ + BUG_ON((unsigned long)fn & ~KFENCE_MASK); + + init_waitqueue_head(&fence->wait); + kref_init(&fence->kref); + atomic_set(&fence->pending, 1); + fence->flags = (unsigned long)fn; +} +EXPORT_SYMBOL_GPL(kfence_init); + +static int kfence_wake(wait_queue_t *wq, unsigned mode, int flags, void *key) +{ + list_del(&wq->task_list); + __kfence_complete(wq->private, key); + kfence_put(wq->private); + kfree(wq); + return 0; +} + +static bool __kfence_check_if_after(struct kfence *fence, + const struct kfence * const signaler) +{ + wait_queue_t *wq; + + if (__test_and_set_bit(KFENCE_CHECKED_BIT, &fence->flags)) + return false; + + if (fence == signaler) + return true; + + list_for_each_entry(wq, &fence->wait.task_list, task_list) { + if (wq->func != kfence_wake) + continue; + + if (__kfence_check_if_after(wq->private, signaler)) + return true; + } + + return false; +} + +static void __kfence_clear_checked_bit(struct kfence *fence) +{ + wait_queue_t *wq; + + if (!__test_and_clear_bit(KFENCE_CHECKED_BIT, &fence->flags)) + return; + + list_for_each_entry(wq, &fence->wait.task_list, task_list) { + if (wq->func != kfence_wake) + continue; + + __kfence_clear_checked_bit(wq->private); + } +} + +static bool kfence_check_if_after(struct kfence *fence, + const struct kfence * const signaler) +{ + unsigned long flags; + bool err; + + if (!config_enabled(CONFIG_KFENCE_CHECK_DAG)) + return false; + + spin_lock_irqsave(&kfence_lock, flags); + err = __kfence_check_if_after(fence, signaler); + __kfence_clear_checked_bit(fence); + spin_unlock_irqrestore(&kfence_lock, flags); + + return err; +} + +static wait_queue_t *__kfence_create_wq(struct kfence *fence, gfp_t gfp) +{ + wait_queue_t *wq; + + wq = kmalloc(sizeof(*wq), gfp); + if (unlikely(!wq)) + return NULL; + + INIT_LIST_HEAD(&wq->task_list); + wq->flags = 0; + wq->func = kfence_wake; + wq->private = kfence_get(fence); + + kfence_await(fence); + + return wq; +} + +/** + * kfence_await_kfence - set one fence to wait upon another + * @fence: this kfence + * @signaler: target kfence to wait upon + * @gfp: the allowed allocation mask + * + * kfence_await_kfence() causes the @fence to asynchronously wait upon the + * completion of @signaler. + * + * Returns 1 if the @fence was added to the waitqueue of @signaler, 0 + * if @signaler was already complete, or a negative error code. + */ +int kfence_await_kfence(struct kfence *fence, + struct kfence *signaler, + gfp_t gfp) +{ + wait_queue_t *wq; + unsigned long flags; + int pending; + + if (kfence_done(signaler)) + return 0; + + /* The dependency graph must be acyclic. */ + if (unlikely(kfence_check_if_after(fence, signaler))) + return -EINVAL; + + wq = __kfence_create_wq(fence, gfp); + if (unlikely(!wq)) { + if (!gfpflags_allow_blocking(gfp)) + return -ENOMEM; + + kfence_wait(signaler); + return 0; + } + + spin_lock_irqsave(&signaler->wait.lock, flags); + if (likely(!kfence_done(signaler))) { + __add_wait_queue_tail(&signaler->wait, wq); + pending = 1; + } else { + kfence_wake(wq, 0, 0, NULL); + pending = 0; + } + spin_unlock_irqrestore(&signaler->wait.lock, flags); + + return pending; +} +EXPORT_SYMBOL_GPL(kfence_await_kfence); + +/** + * kfence_await_completion - set the fence to wait upon a completion + * @fence: this kfence + * @x: target completion to wait upon + * @gfp: the allowed allocation mask + * + * kfence_await_completion() causes the @fence to asynchronously wait upon + * the completion. + * + * Returns 1 if the @fence was added to the waitqueue of @x, 0 + * if @x was already complete, or a negative error code. + */ +int kfence_await_completion(struct kfence *fence, + struct completion *x, + gfp_t gfp) +{ + wait_queue_t *wq; + unsigned long flags; + int pending; + + if (completion_done(x)) + return 0; + + wq = __kfence_create_wq(fence, gfp); + if (unlikely(!wq)) { + if (!gfpflags_allow_blocking(gfp)) + return -ENOMEM; + + wait_for_completion(x); + return 0; + } + + spin_lock_irqsave(&x->wait.lock, flags); + if (likely(!READ_ONCE(x->done))) { + __add_wait_queue_tail(&x->wait, wq); + pending = 1; + } else { + kfence_wake(wq, 0, 0, NULL); + pending = 0; + } + spin_unlock_irqrestore(&x->wait.lock, flags); + + return pending; +} +EXPORT_SYMBOL_GPL(kfence_await_completion); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index b9cfdbfae9aa..df1182d41f06 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1763,6 +1763,29 @@ config KPROBES_SANITY_TEST Say N if you are unsure. +config KFENCE_SELFTEST + tristate "Kfence self tests" + depends on DEBUG_KERNEL + default n + help + This option provides a kernel modules that can be used to test + the kfence handling. This option is not useful for distributions + or general kernels, but only for kernel developers working on the + kfence and async_domain facility. + + Say N if you are unsure. + +config KFENCE_CHECK_DAG + bool "Check that kfence are only used with directed acyclic graphs" + depends on DEBUG_KERNEL + default n + help + This option enforces that kfences are only used with directed acyclic + graphs (DAG), as otherwise the cycles in the graph means that they + will never be signaled (or the corresponding task executed). + + Say N if you are unsure. + config BACKTRACE_SELF_TEST tristate "Self test for the backtrace code" depends on DEBUG_KERNEL diff --git a/lib/Makefile b/lib/Makefile index ff6a7a6c6395..943781cfe8d1 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -28,6 +28,7 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ earlycpio.o seq_buf.o nmi_backtrace.o nodemask.o obj-$(CONFIG_ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS) += usercopy.o +obj-$(CONFIG_KFENCE_SELFTEST) += test-kfence.o lib-$(CONFIG_MMU) += ioremap.o lib-$(CONFIG_SMP) += cpumask.o lib-$(CONFIG_HAS_DMA) += dma-noop.o diff --git a/lib/test-kfence.c b/lib/test-kfence.c new file mode 100644 index 000000000000..b40719fce967 --- /dev/null +++ b/lib/test-kfence.c @@ -0,0 +1,536 @@ +/* + * Test cases for kfence facility. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/delay.h> +#include <linux/kfence.h> +#include <linux/module.h> +#include <linux/slab.h> + +static struct kfence *alloc_kfence(void) +{ + struct kfence *fence; + + fence = kmalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return NULL; + + kfence_init(fence, NULL); + return fence; +} + +static int __init __test_self(struct kfence *fence) +{ + if (kfence_done(fence)) + return -EINVAL; + + kfence_complete(fence); + if (!kfence_done(fence)) + return -EINVAL; + + kfence_wait(fence); + if (!kfence_done(fence)) + return -EINVAL; + + return 0; +} + +static int __init test_self(void) +{ + struct kfence *fence; + int ret; + + /* Test kfence signaling and completion testing */ + pr_debug("%s\n", __func__); + + fence = alloc_kfence(); + if (!fence) + return -ENOMEM; + + ret = __test_self(fence); + + kfence_put(fence); + return ret; +} + +struct test_stack { + struct kfence fence; + bool seen; +}; + +static int __init __kfence_call fence_callback(struct kfence *fence) +{ + container_of(fence, typeof(struct test_stack), fence)->seen = true; + return NOTIFY_DONE; +} + +static int __init test_stack(void) +{ + struct test_stack ts; + int ret; + + /* Test kfence signaling and completion testing (on stack) */ + pr_debug("%s\n", __func__); + + ts.seen = false; + kfence_init(&ts.fence, fence_callback); + + ret = __test_self(&ts.fence); + if (ret < 0) + return ret; + + if (!ts.seen) { + pr_err("fence callback not executed\n"); + return -EINVAL; + } + + return 0; +} + +static int __init test_dag(void) +{ + struct kfence *A, *B, *C; + + /* Test detection of cycles within the kfence graphs */ + pr_debug("%s\n", __func__); + + if (!config_enabled(CONFIG_KFENCE_CHECK_DAG)) + return 0; + + A = alloc_kfence(); + if (kfence_await_kfence(A, A, GFP_KERNEL) != -EINVAL) { + pr_err("recursive cycle not detected (AA)\n"); + return -EINVAL; + } + + B = alloc_kfence(); + + kfence_await_kfence(A, B, GFP_KERNEL); + if (kfence_await_kfence(B, A, GFP_KERNEL) != -EINVAL) { + pr_err("single depth cycle not detected (BAB)\n"); + return -EINVAL; + } + + C = alloc_kfence(); + kfence_await_kfence(B, C, GFP_KERNEL); + if (kfence_await_kfence(C, A, GFP_KERNEL) != -EINVAL) { + pr_err("cycle not detected (BA, CB, AC)\n"); + return -EINVAL; + } + + kfence_complete(A); + kfence_put(A); + + kfence_complete(B); + kfence_put(B); + + kfence_complete(C); + kfence_put(C); + + return 0; +} + +static int __init test_AB(void) +{ + struct kfence *A, *B; + int ret; + + /* Test kfence (A) waiting on an event source (B) */ + pr_debug("%s\n", __func__); + + A = alloc_kfence(); + B = alloc_kfence(); + if (!A || !B) + return -ENOMEM; + + ret = kfence_await_kfence(A, B, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_complete(A); + if (kfence_done(A)) + return -EINVAL; + + kfence_complete(B); + if (!kfence_done(B)) + return -EINVAL; + + if (!kfence_done(A)) + return -EINVAL; + + kfence_put(B); + kfence_put(A); + return 0; +} + +static int __init test_ABC(void) +{ + struct kfence *A, *B, *C; + int ret; + + /* Test a chain of fences, A waits on B who waits on C */ + pr_debug("%s\n", __func__); + + A = alloc_kfence(); + B = alloc_kfence(); + C = alloc_kfence(); + if (!A || !B || !C) + return -ENOMEM; + + ret = kfence_await_kfence(A, B, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + ret = kfence_await_kfence(B, C, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_complete(A); + if (kfence_done(A)) + return -EINVAL; + + kfence_complete(B); + if (kfence_done(B)) + return -EINVAL; + + if (kfence_done(A)) + return -EINVAL; + + kfence_complete(C); + if (!kfence_done(C)) + return -EINVAL; + + if (!kfence_done(B)) + return -EINVAL; + + if (!kfence_done(A)) + return -EINVAL; + + kfence_put(C); + kfence_put(B); + kfence_put(A); + return 0; +} + +static int __init test_AB_C(void) +{ + struct kfence *A, *B, *C; + int ret; + + /* Test multiple fences (AB) waiting on a single event (C) */ + pr_debug("%s\n", __func__); + + A = alloc_kfence(); + B = alloc_kfence(); + C = alloc_kfence(); + if (!A || !B || !C) + return -ENOMEM; + + ret = kfence_await_kfence(A, C, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + ret = kfence_await_kfence(B, C, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_complete(A); + kfence_complete(B); + + if (kfence_done(A)) + return -EINVAL; + + if (kfence_done(B)) + return -EINVAL; + + kfence_complete(C); + if (!kfence_done(C)) + return -EINVAL; + + if (!kfence_done(B)) + return -EINVAL; + + if (!kfence_done(A)) + return -EINVAL; + + kfence_put(C); + kfence_put(B); + kfence_put(A); + return 0; +} + +static int __init test_C_AB(void) +{ + struct kfence *A, *B, *C; + int ret; + + /* Test multiple event sources (A,B) for a single fence (C) */ + pr_debug("%s\n", __func__); + + A = alloc_kfence(); + B = alloc_kfence(); + C = alloc_kfence(); + if (!A || !B || !C) + return -ENOMEM; + + ret = kfence_await_kfence(C, A, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + ret = kfence_await_kfence(C, B, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_complete(C); + if (kfence_done(C)) + return -EINVAL; + + kfence_complete(A); + kfence_complete(B); + + if (!kfence_done(A)) + return -EINVAL; + + if (!kfence_done(B)) + return -EINVAL; + + if (!kfence_done(C)) + return -EINVAL; + + kfence_put(C); + kfence_put(B); + kfence_put(A); + return 0; +} + +static int __init test_completion(void) +{ + struct kfence *fence; + struct completion x; + int ret; + + /* Test use of a completion as an event source for kfences */ + pr_debug("%s\n", __func__); + + init_completion(&x); + + fence = alloc_kfence(); + if (!fence) + return -ENOMEM; + + ret = kfence_await_completion(fence, &x, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_complete(fence); + if (kfence_done(fence)) + return -EINVAL; + + complete_all(&x); + if (!kfence_done(fence)) + return -EINVAL; + + kfence_put(fence); + return 0; +} + +struct task_ipc { + struct work_struct work; + struct completion started; + struct kfence *in, *out; + int value; +}; + +static void __init task_ipc(struct work_struct *work) +{ + struct task_ipc *ipc = container_of(work, typeof(*ipc), work); + + complete(&ipc->started); + + kfence_wait(ipc->in); + smp_store_mb(ipc->value, 1); + kfence_complete(ipc->out); +} + +static int __init test_chain(void) +{ + const int nfences = 4096; + struct kfence **fences; + int ret, i; + + /* Test a long chain of fences */ + pr_debug("%s\n", __func__); + + fences = kmalloc_array(nfences, sizeof(*fences), GFP_KERNEL); + if (!fences) + return -ENOMEM; + + for (i = 0; i < nfences; i++) { + fences[i] = alloc_kfence(); + if (!fences[i]) + return -ENOMEM; + + if (i > 0) { + ret = kfence_await_kfence(fences[i], + fences[i - 1], + GFP_KERNEL); + if (ret < 0) + return ret; + } + } + + for (i = nfences; --i; ) { + kfence_complete(fences[i]); + if (kfence_done(fences[i])) + return -EINVAL; + } + + kfence_complete(fences[0]); + for (i = 0; i < nfences; i++) { + if (!kfence_done(fences[i])) + return -EINVAL; + + kfence_put(fences[i]); + } + kfree(fences); + return 0; +} + +static int __init test_ipc(void) +{ + struct task_ipc ipc; + int ret = 0; + + /* Test use of kfence as an interprocess signaling mechanism */ + pr_debug("%s\n", __func__); + + ipc.in = alloc_kfence(); + ipc.out = alloc_kfence(); + if (!ipc.in || !ipc.out) + return -ENOMEM; + + /* use a completion to avoid chicken-and-egg testing for kfence */ + init_completion(&ipc.started); + + ipc.value = 0; + INIT_WORK(&ipc.work, task_ipc); + schedule_work(&ipc.work); + + wait_for_completion(&ipc.started); + + usleep_range(1000, 2000); + if (READ_ONCE(ipc.value)) { + pr_err("worker updated value before kfence was signaled\n"); + ret = -EINVAL; + } + + kfence_complete(ipc.in); + kfence_wait(ipc.out); + + if (!READ_ONCE(ipc.value)) { + pr_err("worker signaled kfence before value was posted\n"); + ret = -EINVAL; + } + + flush_work(&ipc.work); + kfence_put(ipc.in); + kfence_put(ipc.out); + return ret; +} + +static int __init test_kfence_init(void) +{ + int ret; + + pr_info("Testing kfences\n"); + + ret = test_self(); + if (ret < 0) { + pr_err("self failed\n"); + return ret; + } + + ret = test_stack(); + if (ret < 0) { + pr_err("stack failed\n"); + return ret; + } + + ret = test_dag(); + if (ret < 0) { + pr_err("DAG checker failed\n"); + return ret; + } + + ret = test_AB(); + if (ret < 0) { + pr_err("AB failed\n"); + return ret; + } + + ret = test_ABC(); + if (ret < 0) { + pr_err("ABC failed\n"); + return ret; + } + + ret = test_AB_C(); + if (ret < 0) { + pr_err("AB_C failed\n"); + return ret; + } + + ret = test_C_AB(); + if (ret < 0) { + pr_err("C_AB failed\n"); + return ret; + } + + ret = test_chain(); + if (ret < 0) { + pr_err("chain failed\n"); + return ret; + } + + ret = test_ipc(); + if (ret < 0) { + pr_err("ipc failed\n"); + return ret; + } + + ret = test_completion(); + if (ret < 0) { + pr_err("completion failed\n"); + return ret; + } + + return 0; +} + +static void __exit test_kfence_cleanup(void) +{ +} + +module_init(test_kfence_init); +module_exit(test_kfence_cleanup); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); diff --git a/tools/testing/selftests/lib/kfence.sh b/tools/testing/selftests/lib/kfence.sh new file mode 100755 index 000000000000..487320c70ed1 --- /dev/null +++ b/tools/testing/selftests/lib/kfence.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Runs infrastructure tests using test-kfence kernel module + +if /sbin/modprobe -q test-kfence; then + /sbin/modprobe -q -r test-kfence + echo "kfence: ok" +else + echo "kfence: [FAIL]" + exit 1 +fi -- 2.8.1

9 years, 7 months

1
0
0 0

[PATCH 2/9] async: Introduce kfence, a N:M completion mechanism

by Chris Wilson

Completions are a simple synchronization mechanism, suitable for 1:M barriers where many waiters maybe waiting for a single event. In situations where a single waiter needs to wait for multiple events they could wait on a list of individual completions. If many waiters need the same set of completion events, they each need to wait upon their own list. The kfence abstracts this by itself being able to wait upon many events before signaling its own completion. A kfence may wait upon completions or other kfences, so long as there are no cycles in the dependency graph. The kfence is a one-shot flag, signaling that work has progressed passed a certain point and the waiters may proceed. Unlike completions, kfences are expected to live inside more complex graphs and form the basis for parallel execution of interdependent tasks and so are referenced counted. kfences provide both signaling and waiting routines: kfence_pending() indicates that the kfence should itself wait for another signal. A kfence created by kfence_create() starts in the pending state. kfence_signal() undoes the earlier pending notification and allows the fence to complete if all pending events have then been signaled. kfence_wait() allows the caller to sleep (uninterruptibly) until the fence is complete. This interface is very similar to completions, with the exception of allowing multiple pending / signals to be sent before the kfence is complete. kfence_add() / kfence_add_completion() sets the kfence to wait upon another fence, or completion respectively. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/kfence.h | 41 +++ kernel/async.c | 358 +++++++++++++++++++++++++ lib/Kconfig.debug | 23 ++ lib/Makefile | 1 + lib/test-kfence.c | 475 ++++++++++++++++++++++++++++++++++ tools/testing/selftests/lib/kfence.sh | 10 + 6 files changed, 908 insertions(+) create mode 100644 include/linux/kfence.h create mode 100644 lib/test-kfence.c create mode 100755 tools/testing/selftests/lib/kfence.sh diff --git a/include/linux/kfence.h b/include/linux/kfence.h new file mode 100644 index 000000000000..82eba8aacd02 --- /dev/null +++ b/include/linux/kfence.h @@ -0,0 +1,41 @@ +/* + * kfence.h - library routines for N:M synchronisation points + * + * Copyright (C) 2016 Intel Corporation + * + * This file is released under the GPLv2. + * + */ + +#ifndef _KFENCE_H_ +#define _KFENCE_H_ + +#include <linux/gfp.h> +#include <linux/kref.h> +#include <linux/wait.h> + +struct completion; + +struct kfence { + wait_queue_head_t wait; + unsigned long flags; + struct kref kref; + atomic_t pending; +}; + +extern struct kfence *kfence_create(gfp_t gfp); +extern struct kfence *kfence_get(struct kfence *fence); +extern int kfence_add(struct kfence *fence, struct kfence *after, gfp_t gfp); +extern int kfence_add_completion(struct kfence *fence, + struct completion *x, + gfp_t gfp); +extern void kfence_pending(struct kfence *fence); +extern void kfence_signal(struct kfence *fence); +extern void kfence_wait(struct kfence *fence); +static inline bool kfence_complete(struct kfence *fence) +{ + return atomic_read(&fence->pending) < 0; +} +extern void kfence_put(struct kfence *fence); + +#endif /* _KFENCE_H_ */ diff --git a/kernel/async.c b/kernel/async.c index d2edd6efec56..d0bcb7cc4884 100644 --- a/kernel/async.c +++ b/kernel/async.c @@ -50,6 +50,7 @@ asynchronous and synchronous parts of the kernel. #include <linux/async.h> #include <linux/atomic.h> +#include <linux/kfence.h> #include <linux/ktime.h> #include <linux/export.h> #include <linux/wait.h> @@ -82,6 +83,363 @@ static DECLARE_WAIT_QUEUE_HEAD(async_done); static atomic_t entry_count; +/** + * DOC: kfence overview + * + * kfences provide synchronisation barriers between multiple processes. + * They are very similar to completions, or a pthread_barrier. Where + * kfence differs from completions is their ability to track multiple + * event sources rather than being a singular "completion event". Similar + * to completions, multiple processes or other kfences can listen or wait + * upon a kfence to signal its completion. + * + * The kfence is a one-shot flag, signaling that work has progressed passed + * a certain point (as measured by completion of all events the kfence is + * listening for) and the waiters upon kfence may proceed. + * + * kfences provide both signaling and waiting routines: + * + * kfence_pending() + * + * indicates that the kfence should itself wait for another signal. A + * kfence created by kfence_create() starts in the pending state. + * + * kfence_signal() + * + * undoes the earlier pending notification and allows the fence to complete + * if all pending events have then been signaled. + * + * kfence_wait() + * + * allows the caller to sleep (uninterruptibly) until the fence is complete. + * Meanwhile, + * + * kfence_complete() + * + * reports whether or not the kfence has been passed. + * + * This interface is very similar to completions, with the exception of + * allowing multiple pending / signals to be sent before the kfence is + * complete. + * + * kfence_add() / kfence_add_completion() + * + * sets the kfence to wait upon another fence, or completion respectively. + * + * Unlike completions, kfences are expected to live inside more complex graphs + * and form the basis for parallel execution of interdependent and so are + * reference counted. A kfence may be created using, + * + * kfence_create() + * + * The fence starts off pending a single signal. Once you have finished + * setting up the fence, use kfence_signal() to allow it to wait upon + * its event sources. + * + * Use + * + * kfence_get() / kfence_put + * + * to acquire or release a reference on kfence respectively. + */ + +#define KFENCE_CHECKED_BIT 0 + +static void kfence_free(struct kref *kref) +{ + struct kfence *fence = container_of(kref, typeof(*fence), kref); + + WARN_ON(atomic_read(&fence->pending) > 0); + + kfree(fence); +} + +/** + * kfence_put - release a reference to a kfence + * @fence: the kfence being disposed of + */ +void kfence_put(struct kfence *fence) +{ + if (fence) + kref_put(&fence->kref, kfence_free); +} +EXPORT_SYMBOL_GPL(kfence_put); + +/** + * kfence_get - acquire a reference to a kfence + * @fence: the kfence being used + * + * Returns the pointer to the kfence, with its reference count incremented. + */ +struct kfence *kfence_get(struct kfence *fence) +{ + if (fence) + kref_get(&fence->kref); + return fence; +} +EXPORT_SYMBOL_GPL(kfence_get); + +static void __kfence_wake_up_all(struct kfence *fence, + struct list_head *continuation) +{ + wait_queue_head_t *x = &fence->wait; + unsigned long flags; + + /* To prevent unbounded recursion as we traverse the graph + * of kfences, we move the task_list from this ready fence + * to the tail of the current fence we are signaling. + */ + spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); + if (continuation) + list_splice_tail_init(&x->task_list, continuation); + else while (!list_empty(&x->task_list)) + __wake_up_locked_key(x, TASK_NORMAL, &x->task_list); + spin_unlock_irqrestore(&x->lock, flags); +} + +static void __kfence_signal(struct kfence *fence, + struct list_head *continuation) +{ + if (!atomic_dec_and_test(&fence->pending)) + return; + + atomic_dec(&fence->pending); + __kfence_wake_up_all(fence, continuation); +} + +/** + * kfence_pending - mark the fence as pending a signal + * @fence: the kfence to be signaled + * + */ +void kfence_pending(struct kfence *fence) +{ + WARN_ON(atomic_inc_return(&fence->pending) <= 1); +} +EXPORT_SYMBOL_GPL(kfence_pending); + +/** + * kfence_signal - signal the fence + * @fence: the kfence to be signaled + * + */ +void kfence_signal(struct kfence *fence) +{ + __kfence_signal(fence, NULL); +} +EXPORT_SYMBOL_GPL(kfence_signal); + +/** + * kfence_wait - wait upon a fence to be completed + * @fence: the kfence to wait upon + * + */ +void kfence_wait(struct kfence *fence) +{ + wait_event(fence->wait, kfence_complete(fence)); +} +EXPORT_SYMBOL_GPL(kfence_wait); + +static void kfence_init(struct kfence *fence) +{ + init_waitqueue_head(&fence->wait); + kref_init(&fence->kref); + atomic_set(&fence->pending, 1); + fence->flags = 0; +} + +/** + * kfence_create - create a fence + * @gfp: the allowed allocation type + * + * A fence is created with a reference count of one, and pending a signal. + * After you have completed setting up the fence for use, call kfence_signal() + * to signal completion. + * + * Returns the newly allocated fence, or NULL on error. + */ +struct kfence *kfence_create(gfp_t gfp) +{ + struct kfence *fence; + + fence = kmalloc(sizeof(*fence), gfp); + if (!fence) + return NULL; + + kfence_init(fence); + return fence; +} +EXPORT_SYMBOL_GPL(kfence_create); + +static int kfence_wake(wait_queue_t *wq, unsigned mode, int flags, void *key) +{ + list_del(&wq->task_list); + __kfence_signal(wq->private, key); + kfence_put(wq->private); + kfree(wq); + return 0; +} + +static bool __kfence_check_if_after(struct kfence *fence, + const struct kfence * const signaler) +{ + wait_queue_t *wq; + + if (__test_and_set_bit(KFENCE_CHECKED_BIT, &fence->flags)) + return false; + + if (fence == signaler) + return true; + + list_for_each_entry(wq, &fence->wait.task_list, task_list) { + if (wq->func != kfence_wake) + continue; + + if (__kfence_check_if_after(wq->private, signaler)) + return true; + } + + return false; +} + +static void __kfence_clear_checked_bit(struct kfence *fence) +{ + wait_queue_t *wq; + + if (!__test_and_clear_bit(KFENCE_CHECKED_BIT, &fence->flags)) + return; + + list_for_each_entry(wq, &fence->wait.task_list, task_list) { + if (wq->func != kfence_wake) + continue; + + __kfence_clear_checked_bit(wq->private); + } +} + +static bool kfence_check_if_after(struct kfence *fence, + const struct kfence * const signaler) +{ + unsigned long flags; + bool err; + + if (!config_enabled(CONFIG_KFENCE_CHECK_DAG)) + return false; + + spin_lock_irqsave(&async_lock, flags); + err = __kfence_check_if_after(fence, signaler); + __kfence_clear_checked_bit(fence); + spin_unlock_irqrestore(&async_lock, flags); + + return err; +} + +/** + * kfence_add - set one fence to wait upon another + * @fence: this kfence + * @signaler: target kfence to wait upon + * @gfp: the allowed allocation type + * + * kfence_add() causes the @fence to wait upon completion of @signaler. + * Internally the @fence is marked as pending a signal from @signaler. + * + * Returns 1 if the @fence was added to the waiqueue of @signaler, 0 + * if @signaler was already complete, or a negative error code. + */ +int kfence_add(struct kfence *fence, struct kfence *signaler, gfp_t gfp) +{ + wait_queue_t *wq; + unsigned long flags; + int pending; + + if (!signaler || kfence_complete(signaler)) + return 0; + + /* The dependency graph must be acyclic */ + if (unlikely(kfence_check_if_after(fence, signaler))) + return -EINVAL; + + wq = kmalloc(sizeof(*wq), gfp); + if (unlikely(!wq)) { + if (!gfpflags_allow_blocking(gfp)) + return -ENOMEM; + + kfence_wait(signaler); + return 0; + } + + wq->flags = 0; + wq->func = kfence_wake; + wq->private = kfence_get(fence); + + kfence_pending(fence); + + spin_lock_irqsave(&signaler->wait.lock, flags); + if (likely(!kfence_complete(signaler))) { + __add_wait_queue_tail(&signaler->wait, wq); + pending = 1; + } else { + INIT_LIST_HEAD(&wq->task_list); + kfence_wake(wq, 0, 0, NULL); + pending = 0; + } + spin_unlock_irqrestore(&signaler->wait.lock, flags); + + return pending; +} +EXPORT_SYMBOL_GPL(kfence_add); + +/** + * kfence_add_completion - set the fence to wait upon a completion + * @fence: this kfence + * @x: target completion to wait upon + * @gfp: the allowed allocation type + * + * kfence_add_completiond() causes the @fence to wait upon a completion. + * Internally the @fence is marked as pending a signal from @x. + * + * Returns 1 if the @fence was added to the waiqueue of @x, 0 + * if @x was already complete, or a negative error code. + */ +int kfence_add_completion(struct kfence *fence, struct completion *x, gfp_t gfp) +{ + wait_queue_t *wq; + unsigned long flags; + int pending; + + if (!x || completion_done(x)) + return 0; + + wq = kmalloc(sizeof(*wq), gfp); + if (unlikely(!wq)) { + if (!gfpflags_allow_blocking(gfp)) + return -ENOMEM; + + wait_for_completion(x); + return 0; + } + + wq->flags = 0; + wq->func = kfence_wake; + wq->private = kfence_get(fence); + + kfence_pending(fence); + + spin_lock_irqsave(&x->wait.lock, flags); + if (likely(!READ_ONCE(x->done))) { + __add_wait_queue_tail(&x->wait, wq); + pending = 1; + } else { + INIT_LIST_HEAD(&wq->task_list); + kfence_wake(wq, 0, 0, NULL); + pending = 0; + } + spin_unlock_irqrestore(&x->wait.lock, flags); + + return pending; +} +EXPORT_SYMBOL_GPL(kfence_add_completion); + static async_cookie_t lowest_in_progress(struct async_domain *domain) { struct list_head *pending; diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index f7b17daf8e4d..47319f501954 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1763,6 +1763,29 @@ config KPROBES_SANITY_TEST Say N if you are unsure. +config KFENCE_SELFTEST + tristate "Kfence self tests" + depends on DEBUG_KERNEL + default n + help + This option provides a kernel modules that can be used to test + the kfence handling. This option is not useful for distributions + or general kernels, but only for kernel developers working on the + kfence and async_domain facility. + + Say N if you are unsure. + +config KFENCE_CHECK_DAG + bool "Check that kfence are only used with directed acyclic graphs" + depends on DEBUG_KERNEL + default n + help + This option enforces that kfences are only used with directed acyclic + graphs (DAG), as otherwise the cycles in the graph means that they + will never be signaled (or the corresponding task executed). + + Say N if you are unsure. + config ASYNC_DOMAIN_SELFTEST tristate "Asynchronous domain self tests" depends on DEBUG_KERNEL diff --git a/lib/Makefile b/lib/Makefile index 23cdb9885e45..82e8b5f77c44 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -28,6 +28,7 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ earlycpio.o seq_buf.o nmi_backtrace.o nodemask.o obj-$(CONFIG_ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS) += usercopy.o +obj-$(CONFIG_KFENCE_SELFTEST) += test-kfence.o obj-$(CONFIG_ASYNC_DOMAIN_SELFTEST) += test-async-domain.o lib-$(CONFIG_MMU) += ioremap.o lib-$(CONFIG_SMP) += cpumask.o diff --git a/lib/test-kfence.c b/lib/test-kfence.c new file mode 100644 index 000000000000..1d9e4a5a2184 --- /dev/null +++ b/lib/test-kfence.c @@ -0,0 +1,475 @@ +/* + * Test cases for kfence facility. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/delay.h> +#include <linux/kfence.h> +#include <linux/module.h> +#include <linux/slab.h> + +static int __init test_self(void) +{ + struct kfence *fence; + + /* Test kfence signaling and completion testing */ + pr_debug("%s\n", __func__); + + fence = kfence_create(GFP_KERNEL); + if (!fence) + return -ENOMEM; + + if (kfence_complete(fence)) + return -EINVAL; + + kfence_signal(fence); + + if (!kfence_complete(fence)) + return -EINVAL; + + kfence_wait(fence); + if (!kfence_complete(fence)) + return -EINVAL; + + kfence_put(fence); + return 0; +} + +static int __init test_dag(void) +{ + struct kfence *A, *B, *C; + + /* Test detection of cycles within the kfence graphs */ + pr_debug("%s\n", __func__); + + if (!config_enabled(CONFIG_KFENCE_CHECK_DAG)) + return 0; + + A = kfence_create(GFP_KERNEL); + if (kfence_add(A, A, GFP_KERNEL) != -EINVAL) { + pr_err("recursive cycle not detected (AA)\n"); + return -EINVAL; + } + + B = kfence_create(GFP_KERNEL); + + kfence_add(A, B, GFP_KERNEL); + if (kfence_add(B, A, GFP_KERNEL) != -EINVAL) { + pr_err("single depth cycle not detected (BAB)\n"); + return -EINVAL; + } + + C = kfence_create(GFP_KERNEL); + kfence_add(B, C, GFP_KERNEL); + if (kfence_add(C, A, GFP_KERNEL) != -EINVAL) { + pr_err("cycle not detected (BA, CB, AC)\n"); + return -EINVAL; + } + + kfence_signal(A); + kfence_put(A); + + kfence_signal(B); + kfence_put(B); + + kfence_signal(C); + kfence_put(C); + + return 0; +} + +static int __init test_AB(void) +{ + struct kfence *A, *B; + int ret; + + /* Test kfence (A) waiting on an event source (B) */ + pr_debug("%s\n", __func__); + + A = kfence_create(GFP_KERNEL); + B = kfence_create(GFP_KERNEL); + if (!A || !B) + return -ENOMEM; + + ret = kfence_add(A, B, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_signal(A); + if (kfence_complete(A)) + return -EINVAL; + + kfence_signal(B); + if (!kfence_complete(B)) + return -EINVAL; + + if (!kfence_complete(A)) + return -EINVAL; + + kfence_put(B); + kfence_put(A); + return 0; +} + +static int __init test_ABC(void) +{ + struct kfence *A, *B, *C; + int ret; + + /* Test a chain of fences, A waits on B who waits on C */ + pr_debug("%s\n", __func__); + + A = kfence_create(GFP_KERNEL); + B = kfence_create(GFP_KERNEL); + C = kfence_create(GFP_KERNEL); + if (!A || !B || !C) + return -ENOMEM; + + ret = kfence_add(A, B, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + ret = kfence_add(B, C, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_signal(A); + if (kfence_complete(A)) + return -EINVAL; + + kfence_signal(B); + if (kfence_complete(B)) + return -EINVAL; + + if (kfence_complete(A)) + return -EINVAL; + + kfence_signal(C); + if (!kfence_complete(C)) + return -EINVAL; + + if (!kfence_complete(B)) + return -EINVAL; + + if (!kfence_complete(A)) + return -EINVAL; + + kfence_put(C); + kfence_put(B); + kfence_put(A); + return 0; +} + +static int __init test_AB_C(void) +{ + struct kfence *A, *B, *C; + int ret; + + /* Test multiple fences (AB) waiting on a single event (C) */ + pr_debug("%s\n", __func__); + + A = kfence_create(GFP_KERNEL); + B = kfence_create(GFP_KERNEL); + C = kfence_create(GFP_KERNEL); + if (!A || !B || !C) + return -ENOMEM; + + ret = kfence_add(A, C, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + ret = kfence_add(B, C, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_signal(A); + kfence_signal(B); + + if (kfence_complete(A)) + return -EINVAL; + + if (kfence_complete(B)) + return -EINVAL; + + kfence_signal(C); + if (!kfence_complete(C)) + return -EINVAL; + + if (!kfence_complete(B)) + return -EINVAL; + + if (!kfence_complete(A)) + return -EINVAL; + + kfence_put(C); + kfence_put(B); + kfence_put(A); + return 0; +} + +static int __init test_C_AB(void) +{ + struct kfence *A, *B, *C; + int ret; + + /* Test multiple event sources (A,B) for a single fence (C) */ + pr_debug("%s\n", __func__); + + A = kfence_create(GFP_KERNEL); + B = kfence_create(GFP_KERNEL); + C = kfence_create(GFP_KERNEL); + if (!A || !B || !C) + return -ENOMEM; + + ret = kfence_add(C, A, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + ret = kfence_add(C, B, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_signal(C); + if (kfence_complete(C)) + return -EINVAL; + + kfence_signal(A); + kfence_signal(B); + + if (!kfence_complete(A)) + return -EINVAL; + + if (!kfence_complete(B)) + return -EINVAL; + + if (!kfence_complete(C)) + return -EINVAL; + + kfence_put(C); + kfence_put(B); + kfence_put(A); + return 0; +} + +static int __init test_completion(void) +{ + struct kfence *fence; + struct completion x; + int ret; + + /* Test use of a completion as an event source for kfences */ + pr_debug("%s\n", __func__); + + init_completion(&x); + + fence = kfence_create(GFP_KERNEL); + if (!fence) + return -ENOMEM; + + ret = kfence_add_completion(fence, &x, GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_signal(fence); + if (kfence_complete(fence)) + return -EINVAL; + + complete_all(&x); + if (!kfence_complete(fence)) + return -EINVAL; + + kfence_put(fence); + return 0; +} + +struct task_ipc { + struct work_struct work; + struct completion started; + struct kfence *in, *out; + int value; +}; + +static void __init task_ipc(struct work_struct *work) +{ + struct task_ipc *ipc = container_of(work, typeof(*ipc), work); + + complete(&ipc->started); + + kfence_wait(ipc->in); + smp_store_mb(ipc->value, 1); + kfence_signal(ipc->out); +} + +static int __init test_chain(void) +{ + const int nfences = 4096; + struct kfence **fences; + int ret, i; + + /* Test a long chain of fences */ + pr_debug("%s\n", __func__); + + fences = kmalloc(sizeof(*fences)*nfences, GFP_KERNEL); + if (!fences) + return -ENOMEM; + + for (i = 0; i < nfences; i++) { + fences[i] = kfence_create(GFP_KERNEL); + if (!fences[i]) + return -ENOMEM; + + if (i > 0) { + ret = kfence_add(fences[i], fences[i-1], GFP_KERNEL); + if (ret < 0) + return ret; + } + } + + for (i = nfences; --i; ) { + kfence_signal(fences[i]); + if (kfence_complete(fences[i])) + return -EINVAL; + } + + kfence_signal(fences[0]); + for (i = 0; i < nfences; i++) { + if (!kfence_complete(fences[i])) + return -EINVAL; + + kfence_put(fences[i]); + } + kfree(fences); + return 0; +} + +static int __init test_ipc(void) +{ + struct task_ipc ipc; + int ret = 0; + + /* Test use of kfence as an interprocess signaling mechanism */ + pr_debug("%s\n", __func__); + + ipc.in = kfence_create(GFP_KERNEL); + ipc.out = kfence_create(GFP_KERNEL); + if (!ipc.in || !ipc.out) + return -ENOMEM; + + /* use a completion to avoid chicken-and-egg testing for kfence */ + init_completion(&ipc.started); + + ipc.value = 0; + INIT_WORK(&ipc.work, task_ipc); + schedule_work(&ipc.work); + + wait_for_completion(&ipc.started); + + usleep_range(1000, 2000); + if (READ_ONCE(ipc.value)) { + pr_err("worker updated value before kfence was signaled\n"); + ret = -EINVAL; + } + + kfence_signal(ipc.in); + kfence_wait(ipc.out); + + if (!READ_ONCE(ipc.value)) { + pr_err("worker signaled kfence before value was posted\n"); + ret = -EINVAL; + } + + flush_work(&ipc.work); + kfence_put(ipc.in); + kfence_put(ipc.out); + return ret; +} + +static int __init test_kfence_init(void) +{ + int ret; + + pr_info("Testing kfences\n"); + + ret = test_self(); + if (ret < 0) { + pr_err("self failed\n"); + return ret; + } + + ret = test_dag(); + if (ret < 0) { + pr_err("DAG checker failed\n"); + return ret; + } + + ret = test_AB(); + if (ret < 0) { + pr_err("AB failed\n"); + return ret; + } + + ret = test_ABC(); + if (ret < 0) { + pr_err("ABC failed\n"); + return ret; + } + + ret = test_AB_C(); + if (ret < 0) { + pr_err("AB_C failed\n"); + return ret; + } + + ret = test_C_AB(); + if (ret < 0) { + pr_err("C_AB failed\n"); + return ret; + } + + ret = test_chain(); + if (ret < 0) { + pr_err("chain failed\n"); + return ret; + } + + ret = test_ipc(); + if (ret < 0) { + pr_err("ipc failed\n"); + return ret; + } + + ret = test_completion(); + if (ret < 0) { + pr_err("completion failed\n"); + return ret; + } + + return 0; +} + +static void __exit test_kfence_cleanup(void) +{ +} + +module_init(test_kfence_init); +module_exit(test_kfence_cleanup); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); diff --git a/tools/testing/selftests/lib/kfence.sh b/tools/testing/selftests/lib/kfence.sh new file mode 100755 index 000000000000..487320c70ed1 --- /dev/null +++ b/tools/testing/selftests/lib/kfence.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Runs infrastructure tests using test-kfence kernel module + +if /sbin/modprobe -q test-kfence; then + /sbin/modprobe -q -r test-kfence + echo "kfence: ok" +else + echo "kfence: [FAIL]" + exit 1 +fi -- 2.8.1

9 years, 7 months

3
4
0 0

[PATCH 5/9] async: Wrap hrtimer to provide a time source for a kfence

by Chris Wilson

kfence_add_delay() is a convenience wrapper around hrtimer_start_range_ns() to provide a time source for a kfence graph. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/kfence.h | 5 +++++ kernel/async.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++- lib/test-kfence.c | 44 ++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) diff --git a/include/linux/kfence.h b/include/linux/kfence.h index d71f30c626ae..1abec5e6b23c 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -16,6 +16,7 @@ struct completion; struct fence; +enum hrtimer_mode; struct kfence { wait_queue_head_t wait; @@ -33,6 +34,10 @@ extern int kfence_add_completion(struct kfence *fence, extern int kfence_add_dma(struct kfence *fence, struct fence *dma, gfp_t gfp); +extern int kfence_add_delay(struct kfence *fence, + clockid_t clock, enum hrtimer_mode mode, + ktime_t delay, u64 slack, + gfp_t gfp); extern void kfence_pending(struct kfence *fence); extern void kfence_signal(struct kfence *fence); extern void kfence_wait(struct kfence *fence); diff --git a/kernel/async.c b/kernel/async.c index 01552d23a916..5d02445e36b7 100644 --- a/kernel/async.c +++ b/kernel/async.c @@ -126,7 +126,10 @@ static atomic_t entry_count; * kfence_add() / kfence_add_completion() / kfence_add_dma() * * sets the kfence to wait upon another fence, completion, or DMA fence - * respectively. + * respectively. To set the fence to wait for at least a certain period of + * time, or until after a certain point in time, use + * + * kfence_add_timer() * * Unlike completions, kfences are expected to live inside more complex graphs * and form the basis for parallel execution of interdependent and so are @@ -543,6 +546,61 @@ int kfence_add_dma(struct kfence *fence, struct fence *dma, gfp_t gfp) } EXPORT_SYMBOL_GPL(kfence_add_dma); +/** + * kfence_add_delay - set the fence to wait for a period of time + * @fence: this kfence + * @clock: which clock to program + * @mode: delay given as relative or absolute + * @delay: how long or until what time to wait + * @slack: how much slack that may be applied to the delay + * + * kfence_add_delay() causes the @fence to wait for a a period of time, or + * until a certain point in time. It is a convenience wrapper around + * hrtimer_start_range_ns(). For more details on @clock, @mode, @delay and + * @slack please consult the hrtimer documentation. + * + * Returns 1 if the delay was sucessfuly added to the @fence, or a negative + * error code on failure. + */ +struct timer_cb { + struct hrtimer timer; + struct kfence *fence; +}; + +static enum hrtimer_restart +timer_kfence_wake(struct hrtimer *timer) +{ + struct timer_cb *cb = container_of(timer, typeof(*cb), timer); + + kfence_signal(cb->fence); + kfence_put(cb->fence); + kfree(cb); + + return HRTIMER_NORESTART; +} + +int kfence_add_delay(struct kfence *fence, + clockid_t clock, enum hrtimer_mode mode, + ktime_t delay, u64 slack, + gfp_t gfp) +{ + struct timer_cb *cb; + + cb = kmalloc(sizeof(*cb), gfp); + if (!cb) + return -ENOMEM; + + cb->fence = kfence_get(fence); + kfence_pending(fence); + + hrtimer_init(&cb->timer, clock, mode); + cb->timer.function = timer_kfence_wake; + + hrtimer_start_range_ns(&cb->timer, delay, slack, mode); + return 1; +} +EXPORT_SYMBOL_GPL(kfence_add_delay); + static async_cookie_t lowest_in_progress(struct async_domain *domain) { struct list_head *pending; diff --git a/lib/test-kfence.c b/lib/test-kfence.c index 0604a27f68b2..782a2cca19c5 100644 --- a/lib/test-kfence.c +++ b/lib/test-kfence.c @@ -341,6 +341,44 @@ static int __init test_completion(void) return 0; } +static int __init test_delay(void) +{ + struct kfence *fence; + ktime_t delay; + int ret; + + /* Test use of a hrtimer as an event source for kfences */ + pr_debug("%s\n", __func__); + + fence = kfence_create(GFP_KERNEL); + if (!fence) + return -ENOMEM; + + delay = ktime_get(); + + ret = kfence_add_delay(fence, CLOCK_MONOTONIC, HRTIMER_MODE_REL, + ms_to_ktime(1), 1 << 10, + GFP_KERNEL); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + + kfence_signal(fence); + kfence_wait(fence); + + delay = ktime_sub(ktime_get(), delay); + kfence_put(fence); + + if (!ktime_to_ms(delay)) { + pr_err("kfence woke too early, delay was only %lldns\n", + (long long)ktime_to_ns(delay)); + return -EINVAL; + } + + return 0; +} + struct task_ipc { struct work_struct work; struct completion started; @@ -509,6 +547,12 @@ static int __init test_kfence_init(void) return ret; } + ret = test_delay(); + if (ret < 0) { + pr_err("delay failed\n"); + return ret; + } + return 0; } -- 2.8.1

9 years, 7 months

2
1
0 0

[PATCH 3/9] async: Extend kfence to allow struct embedding

by Chris Wilson

Provide a kfence_init() function for use for embedding the kfence into a parent structure. kfence_init() takes an optional function pointer argument should the caller wish to be notified when the kfence is complete. This is useful for allowing the kfences to drive other state machinery. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/kfence.h | 5 ++++ kernel/async.c | 52 ++++++++++++++++++++++++++++++++++++---- lib/test-kfence.c | 64 +++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/include/linux/kfence.h b/include/linux/kfence.h index 82eba8aacd02..82096bfafaa1 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -38,4 +38,9 @@ static inline bool kfence_complete(struct kfence *fence) } extern void kfence_put(struct kfence *fence); +typedef void (*kfence_notify_t)(struct kfence *); +#define __kfence_call __attribute__((aligned(4))) + +extern void kfence_init(struct kfence *fence, kfence_notify_t fn); + #endif /* _KFENCE_H_ */ diff --git a/kernel/async.c b/kernel/async.c index d0bcb7cc4884..db22b890711e 100644 --- a/kernel/async.c +++ b/kernel/async.c @@ -134,7 +134,17 @@ static atomic_t entry_count; * * The fence starts off pending a single signal. Once you have finished * setting up the fence, use kfence_signal() to allow it to wait upon - * its event sources. + * its event sources. To embed a kfence within another struct, use + * + * kfence_init() + * + * This can also be used to receive a callback when the kfence is completed + * (pending signal count hits 0). The callback is also called when the fence + * is released (the reference count hits 0, and the fence will be complete). + * Note that since the kfence is embedded inside another, its initial reference + * must not be dropped unless a callback is provided, or the kfence is the + * first member in the parent, and the parent was allocated by kmalloc (i.e. + * valid to be freed by kfree()). * * Use * @@ -151,7 +161,11 @@ static void kfence_free(struct kref *kref) WARN_ON(atomic_read(&fence->pending) > 0); - kfree(fence); + if (fence->flags) { + kfence_notify_t fn = (kfence_notify_t)fence->flags; + fn(fence); + } else + kfree(fence); } /** @@ -203,6 +217,11 @@ static void __kfence_signal(struct kfence *fence, if (!atomic_dec_and_test(&fence->pending)) return; + if (fence->flags) { + kfence_notify_t fn = (kfence_notify_t)fence->flags; + fn(fence); + } + atomic_dec(&fence->pending); __kfence_wake_up_all(fence, continuation); } @@ -240,7 +259,7 @@ void kfence_wait(struct kfence *fence) } EXPORT_SYMBOL_GPL(kfence_wait); -static void kfence_init(struct kfence *fence) +static void __kfence_init(struct kfence *fence) { init_waitqueue_head(&fence->wait); kref_init(&fence->kref); @@ -266,11 +285,36 @@ struct kfence *kfence_create(gfp_t gfp) if (!fence) return NULL; - kfence_init(fence); + __kfence_init(fence); return fence; } EXPORT_SYMBOL_GPL(kfence_create); +/** + * kfence_init - initialize a fence for use embedded within a struct + * @fence: this kfence + * @fn: a callback function for when the fence is complete, and when the + * fence is released + * + * This function initialises the @fence for use embedded within a parent + * structure. The optional @fn hook is called when the fence is completed + * (when all of its events sources have signaled their completion, i.e. + * pending signal count hits 0, fence_complete() however reports false as the + * fence is not considered complete until after the callback returns) and when + * the fence is subsequently released (the reference count hits 0, + * fence_complete() is then true). Note carefully that @fn will be called from + * atomic context. Also the lower 2 bits of the function pointer are used for + * storing flags, so the function must be aligned to at least 4 bytes - use + * __kfence_call as a function attribute to ensure correct alignment. + */ +void kfence_init(struct kfence *fence, kfence_notify_t fn) +{ + __kfence_init(fence); + BUG_ON((unsigned long)fn & KFENCE_CHECKED_BIT); + fence->flags = (unsigned long)fn; +} +EXPORT_SYMBOL_GPL(kfence_init); + static int kfence_wake(wait_queue_t *wq, unsigned mode, int flags, void *key) { list_del(&wq->task_list); diff --git a/lib/test-kfence.c b/lib/test-kfence.c index 1d9e4a5a2184..0604a27f68b2 100644 --- a/lib/test-kfence.c +++ b/lib/test-kfence.c @@ -9,9 +9,27 @@ #include <linux/module.h> #include <linux/slab.h> +static int __init __test_self(struct kfence *fence) +{ + if (kfence_complete(fence)) + return -EINVAL; + + kfence_signal(fence); + + if (!kfence_complete(fence)) + return -EINVAL; + + kfence_wait(fence); + if (!kfence_complete(fence)) + return -EINVAL; + + return 0; +} + static int __init test_self(void) { struct kfence *fence; + int ret; /* Test kfence signaling and completion testing */ pr_debug("%s\n", __func__); @@ -20,19 +38,43 @@ static int __init test_self(void) if (!fence) return -ENOMEM; - if (kfence_complete(fence)) - return -EINVAL; + ret = __test_self(fence); - kfence_signal(fence); + kfence_put(fence); + return ret; +} - if (!kfence_complete(fence)) - return -EINVAL; +struct test_stack { + struct kfence fence; + bool seen; +}; - kfence_wait(fence); - if (!kfence_complete(fence)) +static void __init __kfence_call fence_callback(struct kfence *fence) +{ + struct test_stack *ts = container_of(fence, typeof(*ts), fence); + ts->seen = true; +} + +static int __init test_stack(void) +{ + struct test_stack ts; + int ret; + + /* Test kfence signaling and completion testing (on stack) */ + pr_debug("%s\n", __func__); + + ts.seen = false; + kfence_init(&ts.fence, fence_callback); + + ret = __test_self(&ts.fence); + if (ret < 0) + return ret; + + if (!ts.seen) { + pr_err("fence callback not executed\n"); return -EINVAL; + } - kfence_put(fence); return 0; } @@ -413,6 +455,12 @@ static int __init test_kfence_init(void) return ret; } + ret = test_stack(); + if (ret < 0) { + pr_err("stack failed\n"); + return ret; + } + ret = test_dag(); if (ret < 0) { pr_err("DAG checker failed\n"); -- 2.8.1

9 years, 7 months

2
1
0 0

Re: [Linaro-mm-sig] [PATCH v5 0/4] Generic zpos property

by Benjamin Gaignard

htmldocs get information directly from the comments in .h and .c files 2016-07-07 18:09 GMT+02:00 Tobias Jakobi <tjakobi(a)math.uni-bielefeld.de>: > Benjamin Gaignard wrote: >> zpos new fields are correctly documented in drm-kms.html after running >> make htmldocs. > I'm not sure I understand. Where does htmldocs get the information from > then? > > - Tobias > > >> Benjamin >> >> 2016-07-07 16:01 GMT+02:00 Tobias Jakobi <tjakobi(a)math.uni-bielefeld.de>: >>> Hello Benjamin, >>> >>> >>> Benjamin Gaignard wrote: >>>> version 5: >>>> rebased on drm-next where Documentation/DocBook/gpu.tmpl doesn't >>>> exist anymore. >>> I think the documentation has just moved to Documentation/gpu, so the >>> zpos property should be documented there then. >>> >>> >>> With best wishes, >>> Tobias >>> >>> >>>> rework sti patch because some plane functions have changed since v4 >>>> >>>> version 4: >>>> make sure that normalized zpos value is stay in the defined property >>>> range and warn user if not. >>>> Fix NULL pointer bug in rcar-du while setting zpos value. >>>> No changes in the other drivers. >>>> >>>> version 3: >>>> use kmalloc_array instead of kmalloc. >>>> Correct normalize_zpos computation (comeback to Mareck original code) >>>> >>>> version 2: >>>> add a zpos property into drm_plane structure to simplify code. >>>> This allow to get/set zpos value in core and not in drivers code. >>>> Fix various remarks. >>>> >>>> version 1: >>>> refactor Marek's patches to have per plane zpos property instead of only >>>> one in core. >>>> >>>> Benjamin Gaignard (2): >>>> drm: sti: use generic zpos for plane >>>> drm: rcar: use generic code for managing zpos plane property >>>> >>>> Marek Szyprowski (2): >>>> drm: add generic zpos property >>>> drm/exynos: use generic code for managing zpos plane property >>>> >>>> drivers/gpu/drm/Makefile | 2 +- >>>> drivers/gpu/drm/drm_atomic.c | 4 + >>>> drivers/gpu/drm/drm_atomic_helper.c | 6 + >>>> drivers/gpu/drm/drm_blend.c | 227 ++++++++++++++++++++++++++++++ >>>> drivers/gpu/drm/drm_crtc_internal.h | 4 + >>>> drivers/gpu/drm/exynos/exynos_drm_drv.h | 2 - >>>> drivers/gpu/drm/exynos/exynos_drm_plane.c | 67 ++------- >>>> drivers/gpu/drm/exynos/exynos_mixer.c | 6 +- >>>> drivers/gpu/drm/rcar-du/rcar_du_crtc.c | 2 +- >>>> drivers/gpu/drm/rcar-du/rcar_du_drv.h | 1 - >>>> drivers/gpu/drm/rcar-du/rcar_du_kms.c | 5 - >>>> drivers/gpu/drm/rcar-du/rcar_du_plane.c | 9 +- >>>> drivers/gpu/drm/rcar-du/rcar_du_plane.h | 2 - >>>> drivers/gpu/drm/sti/sti_cursor.c | 4 +- >>>> drivers/gpu/drm/sti/sti_gdp.c | 4 +- >>>> drivers/gpu/drm/sti/sti_hqvdp.c | 4 +- >>>> drivers/gpu/drm/sti/sti_mixer.c | 9 +- >>>> drivers/gpu/drm/sti/sti_plane.c | 76 ++++------ >>>> drivers/gpu/drm/sti/sti_plane.h | 7 +- >>>> include/drm/drm_crtc.h | 30 ++++ >>>> 20 files changed, 324 insertions(+), 147 deletions(-) >>>> create mode 100644 drivers/gpu/drm/drm_blend.c >>>> >>>> Cc: Inki Dae <inki.dae(a)samsung.com> >>>> Cc: Daniel Vetter <daniel(a)ffwll.ch> >>>> Cc: Ville Syrjala <ville.syrjala(a)linux.intel.com> >>>> Cc: Joonyoung Shim <jy0922.shim(a)samsung.com> >>>> Cc: Seung-Woo Kim <sw0312.kim(a)samsung.com> >>>> Cc: Andrzej Hajda <a.hajda(a)samsung.com> >>>> Cc: Krzysztof Kozlowski <k.kozlowski(a)samsung.com> >>>> Cc: Bartlomiej Zolnierkiewicz <b.zolnierkie(a)samsung.com> >>>> Cc: Tobias Jakobi <tjakobi(a)math.uni-bielefeld.de> >>>> Cc: Gustavo Padovan <gustavo(a)padovan.org> >>>> Cc: vincent.abriou(a)st.com >>>> Cc: fabien.dessenne(a)st.com >>>> Cc: Laurent Pinchart <laurent.pinchart(a)ideasonboard.com> >>>> >>> >> >> >> > -- Benjamin Gaignard Graphic Working Group Linaro.org │ Open source software for ARM SoCs Follow Linaro: Facebook | Twitter | Blog

9 years, 7 months

2
3
0 0

Re: [Linaro-mm-sig] [PATCH v5 0/4] Generic zpos property

by Benjamin Gaignard

zpos new fields are correctly documented in drm-kms.html after running make htmldocs. Benjamin 2016-07-07 16:01 GMT+02:00 Tobias Jakobi <tjakobi(a)math.uni-bielefeld.de>: > Hello Benjamin, > > > Benjamin Gaignard wrote: >> version 5: >> rebased on drm-next where Documentation/DocBook/gpu.tmpl doesn't >> exist anymore. > I think the documentation has just moved to Documentation/gpu, so the > zpos property should be documented there then. > > > With best wishes, > Tobias > > >> rework sti patch because some plane functions have changed since v4 >> >> version 4: >> make sure that normalized zpos value is stay in the defined property >> range and warn user if not. >> Fix NULL pointer bug in rcar-du while setting zpos value. >> No changes in the other drivers. >> >> version 3: >> use kmalloc_array instead of kmalloc. >> Correct normalize_zpos computation (comeback to Mareck original code) >> >> version 2: >> add a zpos property into drm_plane structure to simplify code. >> This allow to get/set zpos value in core and not in drivers code. >> Fix various remarks. >> >> version 1: >> refactor Marek's patches to have per plane zpos property instead of only >> one in core. >> >> Benjamin Gaignard (2): >> drm: sti: use generic zpos for plane >> drm: rcar: use generic code for managing zpos plane property >> >> Marek Szyprowski (2): >> drm: add generic zpos property >> drm/exynos: use generic code for managing zpos plane property >> >> drivers/gpu/drm/Makefile | 2 +- >> drivers/gpu/drm/drm_atomic.c | 4 + >> drivers/gpu/drm/drm_atomic_helper.c | 6 + >> drivers/gpu/drm/drm_blend.c | 227 ++++++++++++++++++++++++++++++ >> drivers/gpu/drm/drm_crtc_internal.h | 4 + >> drivers/gpu/drm/exynos/exynos_drm_drv.h | 2 - >> drivers/gpu/drm/exynos/exynos_drm_plane.c | 67 ++------- >> drivers/gpu/drm/exynos/exynos_mixer.c | 6 +- >> drivers/gpu/drm/rcar-du/rcar_du_crtc.c | 2 +- >> drivers/gpu/drm/rcar-du/rcar_du_drv.h | 1 - >> drivers/gpu/drm/rcar-du/rcar_du_kms.c | 5 - >> drivers/gpu/drm/rcar-du/rcar_du_plane.c | 9 +- >> drivers/gpu/drm/rcar-du/rcar_du_plane.h | 2 - >> drivers/gpu/drm/sti/sti_cursor.c | 4 +- >> drivers/gpu/drm/sti/sti_gdp.c | 4 +- >> drivers/gpu/drm/sti/sti_hqvdp.c | 4 +- >> drivers/gpu/drm/sti/sti_mixer.c | 9 +- >> drivers/gpu/drm/sti/sti_plane.c | 76 ++++------ >> drivers/gpu/drm/sti/sti_plane.h | 7 +- >> include/drm/drm_crtc.h | 30 ++++ >> 20 files changed, 324 insertions(+), 147 deletions(-) >> create mode 100644 drivers/gpu/drm/drm_blend.c >> >> Cc: Inki Dae <inki.dae(a)samsung.com> >> Cc: Daniel Vetter <daniel(a)ffwll.ch> >> Cc: Ville Syrjala <ville.syrjala(a)linux.intel.com> >> Cc: Joonyoung Shim <jy0922.shim(a)samsung.com> >> Cc: Seung-Woo Kim <sw0312.kim(a)samsung.com> >> Cc: Andrzej Hajda <a.hajda(a)samsung.com> >> Cc: Krzysztof Kozlowski <k.kozlowski(a)samsung.com> >> Cc: Bartlomiej Zolnierkiewicz <b.zolnierkie(a)samsung.com> >> Cc: Tobias Jakobi <tjakobi(a)math.uni-bielefeld.de> >> Cc: Gustavo Padovan <gustavo(a)padovan.org> >> Cc: vincent.abriou(a)st.com >> Cc: fabien.dessenne(a)st.com >> Cc: Laurent Pinchart <laurent.pinchart(a)ideasonboard.com> >> > -- Benjamin Gaignard Graphic Working Group Linaro.org │ Open source software for ARM SoCs Follow Linaro: Facebook | Twitter | Blog

9 years, 7 months

1
0
0 0

[PATCH 9/9] async: Introduce a dependency resolver for parallel execution

by Chris Wilson

A challenge in driver initialisation is the coordination of many small sometimes independent, sometimes interdependent tasks. We would like to schedule the independent tasks for execution in parallel across as many cores as possible for rapid initialisation, and then schedule all the dependent tasks once they have completed, again running as many of those in parallel as is possible. Resolving the interdependencies by hand is time consuming and error prone. Instead, we want to declare what dependencies a particular task has, and what that task provides, and let a runtime dependency solver work out what tasks to run and when, and which in parallel. To this end, we introduce the struct async_dependency_graph building upon the kfence and async_work from the previous patches to allow for the runtime computation of the topological task ordering. The graph is constructed with async_dependency_graph_build(), which takes the task, its dependencies and what it provides, and builds the graph of kfences required for ordering. Additional kfences can be inserted through async_dependency_depends() and async_dependency_provides() for manual control of the execution order, and async_dependency_get() retrieves a kfence for inspection or waiting upon. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/async.h | 37 +++ kernel/async.c | 250 ++++++++++++++++ lib/Kconfig.debug | 12 + lib/Makefile | 1 + lib/test-async-dependency-graph.c | 317 +++++++++++++++++++++ .../selftests/lib/async-dependency-graph.sh | 10 + 6 files changed, 627 insertions(+) create mode 100644 lib/test-async-dependency-graph.c create mode 100755 tools/testing/selftests/lib/async-dependency-graph.sh diff --git a/include/linux/async.h b/include/linux/async.h index 64a090e3f24f..c9cadb383813 100644 --- a/include/linux/async.h +++ b/include/linux/async.h @@ -15,6 +15,7 @@ #include <linux/types.h> #include <linux/kfence.h> #include <linux/list.h> +#include <linux/rbtree.h> typedef u64 async_cookie_t; typedef void (*async_func_t) (void *data, async_cookie_t cookie); @@ -101,4 +102,40 @@ extern async_cookie_t queue_async_work(struct async_domain *domain, gfp_t gfp); extern async_cookie_t schedule_async_work(struct async_work *work); +/* Build a graph of work based on dependencies generated by keywords. + * The graph must be acyclic. Can be used to both generate a topological + * ordering of tasks, and to execute independent chains of tasks in + * parallel. + */ + +struct async_dependency_graph { + struct rb_root root; + struct list_head list; +}; + +#define ASYNC_DEPENDENCY_GRAPH_INIT(_name) { \ + .root = RB_ROOT, \ + .list = LIST_HEAD_INIT(_name.list), \ +} +#define ASYNC_DEPENDENCY_GRAPH(_name) \ + struct async_dependency_graph _name = ASYNC_DEPENDENCY_GRAPH_INIT(_name) + +extern int async_dependency_graph_build(struct async_dependency_graph *adg, + async_func_t fn, void *data, + const char *depends, + const char *provides); + +extern int async_dependency_depends(struct async_dependency_graph *adg, + struct kfence *fence, + const char *depends); + +extern int async_dependency_provides(struct async_dependency_graph *adg, + struct kfence *fence, + const char *provides); + +extern struct kfence *async_dependency_get(struct async_dependency_graph *adg, + const char *name); + +extern void async_dependency_graph_execute(struct async_dependency_graph *adg); + #endif diff --git a/kernel/async.c b/kernel/async.c index a22945f4b4c4..8330d719074b 100644 --- a/kernel/async.c +++ b/kernel/async.c @@ -1005,3 +1005,253 @@ void init_async_domain(struct async_domain *domain, bool registered) domain->registered = registered; } EXPORT_SYMBOL_GPL(init_async_domain); + +struct async_dependency { + struct kfence fence; + struct rb_node node; + struct list_head link; + char name[0]; +}; + +static struct async_dependency * +__lookup_dependency(struct async_dependency_graph *adg, const char *name) +{ + struct rb_node **p, *parent; + struct async_dependency *d; + int len; + + parent = NULL; + p = &adg->root.rb_node; + while (*p) { + int cmp; + + parent = *p; + d = container_of(parent, typeof(*d), node); + + cmp = strcmp(name, d->name); + if (cmp < 0) + p = &parent->rb_left; + else if (cmp > 0) + p = &parent->rb_right; + else + return d; + } + + len = strlen(name) + 1; + d = kmalloc(sizeof(*d) + len, GFP_KERNEL); + if (!d) + return ERR_PTR(-ENOMEM); + + __kfence_init(&d->fence); + memcpy(d->name, name, len); + + rb_link_node(&d->node, parent, p); + rb_insert_color(&d->node, &adg->root); + list_add_tail(&d->link, &adg->list); + + return d; +} + +/** + * async_dependency_depends - declare a prerequisite fence for a named stage + * @adg: the async_dependency_graph for tracking the named stages + * @fence: the kfence to add that depends upon the named stage completing + * @depends: the named stage + * + * This function appends @fence into the async_dependency_graph @adg after + * the @depends stage is completed. That is the @fence is signaled once + * the chain of dependencies upto and including @depends is complete. + * + * Returns: 0 on success, negative error code on failure. + * In particular, note that if CONFIG_KFENCE_CHECK_DAG is enabled, the + * dependency graph will be checked for cycles, and -EINVAL reported + * in such cases. A dependency cycle leads to unexecutable code. + */ +int async_dependency_depends(struct async_dependency_graph *adg, + struct kfence *fence, + const char *depends) +{ + struct async_dependency *d; + + d = __lookup_dependency(adg, depends); + if (IS_ERR(d)) + return PTR_ERR(d); + + return kfence_add(fence, &d->fence, GFP_KERNEL); +} +EXPORT_SYMBOL_GPL(async_dependency_depends); + +/** + * async_dependency_provides - declare a named stage that should follow + * @adg: the async_dependency_graph for tracking the named stages + * @fence: the kfence to add that provides the named stage with a signal + * @depends: the named stage + * + * This function inserts @fence into the async_dependency_graph @adg before + * the @provides stage is signaled. That is the @fence signals the + * @provides stage once completed (and once all providers have completed, + * work from the @provides commences). + * + * Returns: 0 on success, negative error code on failure. + * In particular, note that if CONFIG_KFENCE_CHECK_DAG is enabled, the + * dependency graph will be checked for cycles, and -EINVAL reported + * in such cases. A dependency cycle leads to unexecutable code. + */ +int async_dependency_provides(struct async_dependency_graph *adg, + struct kfence *fence, + const char *provides) +{ + struct async_dependency *d; + + d = __lookup_dependency(adg, provides); + if (IS_ERR(d)) + return PTR_ERR(d); + + return kfence_add(&d->fence, fence, GFP_KERNEL); +} +EXPORT_SYMBOL_GPL(async_dependency_provides); + +/** + * async_dependency_get - lookup the kfence for a named stage + * @adg: the async_dependency_graph for tracking the named stages + * @name: the named stage + * + * This function lookups the kfence associated with the named stage. This + * fence will be signaled once the named stage is ready. For example, + * waiting on that fence will wait until all prior dependencies of that + * named stage have been completed. + * + * Returns: a new reference on the kfence. The caller must release the + * reference with kfence_put() when finished. + */ +struct kfence *async_dependency_get(struct async_dependency_graph *adg, + const char *name) +{ + struct async_dependency *d; + + d = __lookup_dependency(adg, name); + if (IS_ERR(d)) + return ERR_CAST(d); + + return kfence_get(&d->fence); +} +EXPORT_SYMBOL_GPL(async_dependency_get); + +static int __adg_for_each_token(struct async_dependency_graph *adg, + struct kfence *fence, + const char *string, + int (*fn)(struct async_dependency_graph *, + struct kfence *, + const char *)) +{ + char *tmp, *s, *t; + int ret = 0; + + if (!string) + return 0; + + tmp = kstrdup(string, GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + for (s = tmp; (t = strsep(&s, ",")); ) { + if (*t == '\0') + continue; + + ret |= fn(adg, fence, t); + if (ret < 0) + break; + } + + kfree(tmp); + return ret; +} + +/** + * async_dependency_graph_build - insert a task into the dependency graph + * @adg: the async_dependency_graph for tracking the named stages + * @fn: the async_func_t to execute + * @data: the data to pass to the @fn + * @depends: a comma-separated list of named stages that must complete + * before the task can execute + * @provides: a comma-separated list of named stages that will be signaled + * when this task completes + * + * This function inserts the task @fn into the async_dependency_graph @adg + * after all the named stages in @depends have completed. Upon completion + * of the task, all the named stages in @provides are signaled (and once all + * their dependent tasks have also finished, the tasks afterwards will + * execute). + * + * If a task has no dependency (@depends is NULL or an empty string), it will + * be scheduled for execution as soon as it is inserted into the graph @adg. + * + * Returns: 0 on success, negative error code on failure. + * In particular, note that if CONFIG_KFENCE_CHECK_DAG is enabled, the + * dependency graph will be checked for cycles, and -EINVAL reported + * in such cases. A dependency cycle leads to unexecutable code. + */ +int +async_dependency_graph_build(struct async_dependency_graph *adg, + async_func_t fn, void *data, + const char *depends, const char *provides) +{ + struct async_work *work; + int ret; + + work = async_work_create(fn, data, GFP_KERNEL); + if (!work) + return -ENOMEM; + + ret = __adg_for_each_token(adg, &work->fence, depends, + async_dependency_depends); + if (ret < 0) + goto err; + + ret = __adg_for_each_token(adg, &work->fence, provides, + async_dependency_provides); + if (ret < 0) + goto err; + + if (!schedule_async_work(work)) { + ret = -ENOMEM; + goto err; + } + + ret = 0; +out: + async_work_put(work); + return ret; + +err: + clear_bit(ASYNC_WORK_BIT, &work->fence.flags); + kfence_signal(&work->fence); + goto out; +} +EXPORT_SYMBOL_GPL(async_dependency_graph_build); + +/** + * async_dependency_graph_execute - execute the dependency graph + * @adg: the async_dependency_graph + * + * This function marks the @adg as ready for execution. As soon as the + * dependencies of a task have been completed (in their entirety), that + * task is executed. Once completed, it signals the tasks that have listed + * its @provides as one of their @depends, and once ready (all @provides are + * complete) those tasks are scheduled for execution. + * + * Tasks are executed in the topological order of their dependencies. If two, + * or more, tasks are not dependent on each other they may be run concurrently. + * + * The graph @adg is freed upon execution. + */ +void async_dependency_graph_execute(struct async_dependency_graph *adg) +{ + struct async_dependency *d, *next; + + list_for_each_entry_safe(d, next, &adg->list, link) { + kfence_signal(&d->fence); + kfence_put(&d->fence); + } +} +EXPORT_SYMBOL_GPL(async_dependency_graph_execute); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 47319f501954..4943b8dbcdf1 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1798,6 +1798,18 @@ config ASYNC_DOMAIN_SELFTEST Say N if you are unsure. +config ASYNC_DEPENDENCY_GRAPH_SELFTEST + tristate "Asynchronous dependency graph self tests" + depends on DEBUG_KERNEL + default n + help + This option provides a kernel modules that can be used to test + the asynchronous dependency graph. This option is not useful for + distributions or general kernels, but only for kernel developers + working on the async_dependency_graph facility. + + Say N if you are unsure. + config BACKTRACE_SELF_TEST tristate "Self test for the backtrace code" depends on DEBUG_KERNEL diff --git a/lib/Makefile b/lib/Makefile index 82e8b5f77c44..fa7da38d4383 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -30,6 +30,7 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ obj-$(CONFIG_ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS) += usercopy.o obj-$(CONFIG_KFENCE_SELFTEST) += test-kfence.o obj-$(CONFIG_ASYNC_DOMAIN_SELFTEST) += test-async-domain.o +obj-$(CONFIG_ASYNC_DEPENDENCY_GRAPH_SELFTEST) += test-async-dependency-graph.o lib-$(CONFIG_MMU) += ioremap.o lib-$(CONFIG_SMP) += cpumask.o lib-$(CONFIG_HAS_DMA) += dma-noop.o diff --git a/lib/test-async-dependency-graph.c b/lib/test-async-dependency-graph.c new file mode 100644 index 000000000000..ebee26d7b99e --- /dev/null +++ b/lib/test-async-dependency-graph.c @@ -0,0 +1,317 @@ +/* + * Test cases for async-dependency-graph facility. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/async.h> +#include <linux/delay.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/string.h> + +struct chain { + atomic_t idx; + unsigned long values[0]; +}; + +struct task_write { + struct chain *chain; + unsigned long value; +}; + +static void __init task_write(void *arg, async_cookie_t cookie) +{ + struct task_write *t = arg; + int idx = atomic_inc_return(&t->chain->idx) - 1; + WRITE_ONCE(t->chain->values[idx], t->value); +} + +static void __init task_nop(void *data, async_cookie_t cookie) +{ +} + +static int __init test_ordering(int nchain, int nwide) +{ + ASYNC_DEPENDENCY_GRAPH(adg); + struct chain **chains; + struct task_write *tests, *t; + int c, w, ret; + + /* Test implementation of simple chains within the dependency graphs */ + pr_debug("%s(nchain=%d, nwide=%d)\n", __func__, nchain, nwide); + + chains = kmalloc(sizeof(struct chain *)*nwide, GFP_KERNEL); + tests = kmalloc(sizeof(struct task_write)*nwide*nchain, GFP_KERNEL); + + if (!chains || !tests) + return -ENOMEM; + + t = tests; + for (w = 0; w < nwide; w++) { + char *depends = NULL, *provides; + + chains[w] = kzalloc(sizeof(struct chain) + + nchain*sizeof(unsigned long), + GFP_KERNEL); + + for (c = 0; c < nchain; c++) { + t->chain = chains[w]; + t->value = c; + provides = kasprintf(GFP_KERNEL, "%d.%d", c, w); + async_dependency_graph_build(&adg, task_write, t, + depends, provides); + kfree(depends); + depends = provides; + t++; + } + + kfree(depends); + } + async_dependency_graph_execute(&adg); + async_synchronize_full(); + + ret = 0; + kfree(tests); + for (w = 0; w < nwide; w++) { + for (c = 0; c < nchain; c++) { + if (chains[w]->values[c] != c) { + pr_err("%s(%d, %d): Invalid execution order (chain %d, position %d): found %d\n", + __func__, nchain, nwide, + w, c, (int)chains[w]->values[c]); + + ret = -EINVAL; + } + } + kfree(chains[w]); + } + kfree(chains); + + return ret; +} + +static int __init test_barrier(int nwide) +{ + ASYNC_DEPENDENCY_GRAPH(adg); + struct chain **chains; + struct task_write *tests, *t; + int c, w, ret; + + /* Test implementation of barriers within the dependency graphs */ + pr_debug("%s(nwide=%d)\n", __func__, nwide); + + chains = kmalloc(sizeof(struct chain *)*nwide, GFP_KERNEL); + tests = kmalloc(sizeof(struct task_write)*2*nwide, GFP_KERNEL); + if (!chains || !tests) + return -ENOMEM; + + t = tests; + + /* A,B act as a barrier running between the nops */ + for (w = 0; w < nwide; w++) { + char *provides, *depends; + + chains[w] = kzalloc(sizeof(struct chain) + + 2*sizeof(unsigned long), + GFP_KERNEL); + + depends = NULL; + + provides = kasprintf(GFP_KERNEL, "nop1.%d", w); + async_dependency_graph_build(&adg, task_nop, NULL, + depends, provides); + async_dependency_graph_build(&adg, task_nop, NULL, + depends, provides); + + kfree(depends); + depends = provides; + + provides = kasprintf(GFP_KERNEL, "A.%d", w); + t->chain = chains[w]; + t->value = 0; + async_dependency_graph_build(&adg, task_write, t, + depends, provides); + t++; + + kfree(depends); + depends = provides; + + provides = kasprintf(GFP_KERNEL, "nop2.%d", w); + async_dependency_graph_build(&adg, task_nop, NULL, + depends, provides); + kfree(provides); + + provides = kasprintf(GFP_KERNEL, "nop3.%d", w); + async_dependency_graph_build(&adg, task_nop, NULL, + depends, provides); + kfree(provides); + + kfree(depends); + depends = kasprintf(GFP_KERNEL, "nop2.%d,nop3.%d", w, w); + t->chain = chains[w]; + t->value = 1; + async_dependency_graph_build(&adg, task_write, t, + depends, NULL); + kfree(depends); + t++; + } + async_dependency_graph_execute(&adg); + async_synchronize_full(); + + ret = 0; + kfree(tests); + for (w = 0; w < nwide; w++) { + for (c = 0; c < 2; c++) { + if (chains[w]->values[c] != c) { + pr_err("%s(%d): Invalid execution order (chain %d, position %d): found %d\n", + __func__, nwide, + w, c, (int)chains[w]->values[c]); + + ret = -EINVAL; + } + } + kfree(chains[w]); + } + kfree(chains); + + return ret; +} + +static int __init test_dag(void) +{ + ASYNC_DEPENDENCY_GRAPH(adg); + + /* Test detection of cycles within the dependency graphs */ + pr_debug("%s\n", __func__); + + if (!config_enabled(CONFIG_KFENCE_CHECK_DAG)) + return 0; + + async_dependency_graph_build(&adg, task_nop, NULL, "__start__", "A"); + if (async_dependency_graph_build(&adg, task_nop, NULL, "A", "A") != -EINVAL) { + pr_err("Failed to detect AA cycle\n"); + return -EINVAL; + } + + async_dependency_graph_build(&adg, task_nop, NULL, "A", "B"); + if (async_dependency_graph_build(&adg, task_nop, NULL, "B", "A") != -EINVAL) { + pr_err("Failed to detect ABA cycle\n"); + return -EINVAL; + } + + async_dependency_graph_build(&adg, task_nop, NULL, "B", "C"); + if (async_dependency_graph_build(&adg, task_nop, NULL, "C", "A") != -EINVAL) { + pr_err("Failed to detect ABCA cycle\n"); + return -EINVAL; + } + + async_dependency_graph_execute(&adg); + async_synchronize_full(); + + return 0; +} + +static int __init perf_nop(int chain, int width, long timeout_us) +{ + ktime_t start; + long count, delay; + + count = 0; + start = ktime_get(); + do { + ASYNC_DEPENDENCY_GRAPH(adg); + ktime_t delta; + int c, w; + + for (w = 0; w < width; w++) { + char *depends = NULL, *provides; + + for (c = 0; c < chain; c++) { + provides = kasprintf(GFP_KERNEL, "%d.%d", c, w); + async_dependency_graph_build(&adg, + task_nop, NULL, + depends, provides); + kfree(depends); + depends = provides; + } + + kfree(depends); + } + async_dependency_graph_execute(&adg); + async_synchronize_full(); + delta = ktime_sub(ktime_get(), start); + delay = ktime_to_ns(delta) >> 10; + count += width * chain; + } while (delay < timeout_us); + + pr_info("%ld nop tasks (in chains of %d, %d chains in parallel) completed in %ldus\n", + count, chain, width, delay); + return 0; +} + +static int __init test_async_dependency_graph_init(void) +{ + int ret; + + pr_info("Testing async-dependency-graph\n"); + + ret = test_ordering(1, 1); + if (ret) + return ret; + + ret = test_ordering(2, 1); + if (ret) + return ret; + + ret = test_ordering(1, 2); + if (ret) + return ret; + + ret = test_ordering(2, 2); + if (ret) + return ret; + + ret = test_ordering(26, 26); + if (ret) + return ret; + + ret = test_dag(); + if (ret) + return ret; + + ret = test_barrier(1); + if (ret) + return ret; + + ret = test_barrier(16); + if (ret) + return ret; + + ret = perf_nop(1, 1, 100); + if (ret) + return ret; + + ret = perf_nop(256, 1, 2000); + if (ret) + return ret; + + ret = perf_nop(128, 2, 2000); + if (ret) + return ret; + + ret = perf_nop(16, 16, 2000); + if (ret) + return ret; + + return 0; +} + +static void __exit test_async_dependency_graph_cleanup(void) +{ +} + +module_init(test_async_dependency_graph_init); +module_exit(test_async_dependency_graph_cleanup); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); diff --git a/tools/testing/selftests/lib/async-dependency-graph.sh b/tools/testing/selftests/lib/async-dependency-graph.sh new file mode 100755 index 000000000000..ea4bbc76f60f --- /dev/null +++ b/tools/testing/selftests/lib/async-dependency-graph.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Runs infrastructure tests using test-async-dependency-graph kernel module + +if /sbin/modprobe -q test-async-dependency-graph; then + /sbin/modprobe -q -r test-async-dependency-graph + echo "async-dependency-graph: ok" +else + echo "async-dependency-graph: [FAIL]" + exit 1 +fi -- 2.8.1

9 years, 7 months

1
0
0 0

[PATCH 8/9] async: Add execution barriers

by Chris Wilson

A frequent mode of operation is fanning out N tasks to execute in parallel, collating results, fanning out M tasks, rinse and repeat. This is also common to the notion of the async/sync kernel domain split. A barrier provides a mechanism by which all work queued after the barrier must wait (i.e. not be scheduled) until all work queued before the barrier is completed. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/async.h | 4 ++++ kernel/async.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/include/linux/async.h b/include/linux/async.h index 45d6c8323b60..64a090e3f24f 100644 --- a/include/linux/async.h +++ b/include/linux/async.h @@ -26,6 +26,7 @@ struct async_work { struct async_domain { struct list_head pending; + struct kfence *barrier; unsigned registered:1; }; @@ -59,6 +60,9 @@ extern void async_synchronize_cookie(async_cookie_t cookie); extern void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain); +extern void async_barrier(void); +extern void async_barrier_domain(struct async_domain *domain); + extern bool current_is_async(void); extern struct async_work * diff --git a/kernel/async.c b/kernel/async.c index 83007c39a113..a22945f4b4c4 100644 --- a/kernel/async.c +++ b/kernel/async.c @@ -724,6 +724,12 @@ struct async_work *async_work_create(async_func_t func, void *data, gfp_t gfp) } EXPORT_SYMBOL_GPL(async_work_create); +static void async_barrier_delete(struct async_domain *domain) +{ + kfence_put(domain->barrier); + domain->barrier = NULL; +} + async_cookie_t queue_async_work(struct async_domain *domain, struct async_work *work, gfp_t gfp) @@ -744,6 +750,9 @@ async_cookie_t queue_async_work(struct async_domain *domain, async_pending_count++; spin_unlock_irqrestore(&async_lock, flags); + if (!kfence_add(&entry->base.fence, domain->barrier, gfp)) + async_barrier_delete(domain); + /* mark that this task has queued an async job, used by module init */ current->flags |= PF_USED_ASYNC; @@ -811,6 +820,55 @@ async_cookie_t async_schedule_domain(async_func_t func, void *data, } EXPORT_SYMBOL_GPL(async_schedule_domain); +static struct kfence * __async_barrier_create(struct async_domain *domain) +{ + struct kfence *fence; + struct async_entry *entry; + unsigned long flags; + int ret; + + fence = kfence_create(GFP_KERNEL); + if (!fence) + goto out_sync; + + ret = 0; + spin_lock_irqsave(&async_lock, flags); + list_for_each_entry(entry, &domain->pending, pending_link[0]) { + ret |= kfence_add(fence, &entry->base.fence, GFP_ATOMIC); + if (ret < 0) + break; + } + spin_unlock_irqrestore(&async_lock, flags); + if (ret <= 0) + goto out_put; + + kfence_add(fence, domain->barrier, GFP_KERNEL); + kfence_signal(fence); + return fence; + +out_put: + kfence_signal(fence); + kfence_put(fence); +out_sync: + async_synchronize_full_domain(domain); + return NULL; +} + +void async_barrier(void) +{ + async_barrier_domain(&async_dfl_domain); +} +EXPORT_SYMBOL_GPL(async_barrier); + +void async_barrier_domain(struct async_domain *domain) +{ + struct kfence *barrier = __async_barrier_create(domain); + + kfence_put(domain->barrier); + domain->barrier = barrier; +} +EXPORT_SYMBOL_GPL(async_barrier_domain); + /** * async_synchronize_full - synchronize all asynchronous function calls * @@ -834,6 +892,8 @@ EXPORT_SYMBOL_GPL(async_synchronize_full); void async_unregister_domain(struct async_domain *domain) { WARN_ON(!list_empty(&domain->pending)); + + async_barrier_delete(domain); domain->registered = 0; } EXPORT_SYMBOL_GPL(async_unregister_domain); -- 2.8.1

9 years, 7 months

1
0
0 0

[PATCH 7/9] async: Add support for explicit fine-grained barriers

by Chris Wilson

The current async-domain model supports running a multitude of independent tasks with a coarse synchronisation point. This is sufficient for its original purpose of allowing independent drivers to run concurrently during various phases (booting, early resume, late resume etc), and keep the asynchronous domain out of the synchronous kernel domains. However, for greater exploitation, drivers themselves want to schedule multiple tasks within a phase (or between phases) and control the order of execution within those tasks relative to each other. To enable this, we extend the synchronisation scheme based upon kfences and back every task with one. Any task may now wait upon the kfence before being scheduled, and equally the kfence may be used to wait on the task itself (rather than waiting on the cookie for all previous tasks to be completed). Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/async.h | 60 +++++++++- kernel/async.c | 244 +++++++++++++++++++++---------------- lib/test-async-domain.c | 311 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 505 insertions(+), 110 deletions(-) diff --git a/include/linux/async.h b/include/linux/async.h index 6b0226bdaadc..45d6c8323b60 100644 --- a/include/linux/async.h +++ b/include/linux/async.h @@ -13,38 +13,88 @@ #define __ASYNC_H__ #include <linux/types.h> +#include <linux/kfence.h> #include <linux/list.h> typedef u64 async_cookie_t; typedef void (*async_func_t) (void *data, async_cookie_t cookie); + +struct async_work { + struct kfence fence; + /* private */ +}; + struct async_domain { struct list_head pending; unsigned registered:1; }; +#define ASYNC_DOMAIN_INIT(_name, _r) { \ + .pending = LIST_HEAD_INIT(_name.pending), \ + .registered = _r \ +} + /* * domain participates in global async_synchronize_full */ #define ASYNC_DOMAIN(_name) \ - struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \ - .registered = 1 } + struct async_domain _name = ASYNC_DOMAIN_INIT(_name, 1) /* * domain is free to go out of scope as soon as all pending work is * complete, this domain does not participate in async_synchronize_full */ #define ASYNC_DOMAIN_EXCLUSIVE(_name) \ - struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \ - .registered = 0 } + struct async_domain _name = ASYNC_DOMAIN_INIT(_name, 0) + +extern void init_async_domain(struct async_domain *domain, bool registered); extern async_cookie_t async_schedule(async_func_t func, void *data); extern async_cookie_t async_schedule_domain(async_func_t func, void *data, struct async_domain *domain); -void async_unregister_domain(struct async_domain *domain); +extern void async_unregister_domain(struct async_domain *domain); extern void async_synchronize_full(void); extern void async_synchronize_full_domain(struct async_domain *domain); extern void async_synchronize_cookie(async_cookie_t cookie); extern void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain); + extern bool current_is_async(void); + +extern struct async_work * +async_work_create(async_func_t func, void *data, gfp_t gfp); + +static inline struct async_work *async_work_get(struct async_work *work) +{ + kfence_get(&work->fence); + return work; +} + +static inline int +async_work_after(struct async_work *work, struct kfence *fence) +{ + return kfence_add(&work->fence, fence, GFP_KERNEL); +} + +static inline int +async_work_before(struct async_work *work, struct kfence *fence) +{ + return kfence_add(fence, &work->fence, GFP_KERNEL); +} + +static inline void async_work_wait(struct async_work *work) +{ + kfence_wait(&work->fence); +} + +static inline void async_work_put(struct async_work *work) +{ + kfence_put(&work->fence); +} + +extern async_cookie_t queue_async_work(struct async_domain *domain, + struct async_work *work, + gfp_t gfp); +extern async_cookie_t schedule_async_work(struct async_work *work); + #endif diff --git a/kernel/async.c b/kernel/async.c index 1fa1f39b5a74..83007c39a113 100644 --- a/kernel/async.c +++ b/kernel/async.c @@ -2,6 +2,7 @@ * async.c: Asynchronous function calls for boot performance * * (C) Copyright 2009 Intel Corporation + * (C) Copyright 2016 Intel Corporation * Author: Arjan van de Ven <arjan(a)linux.intel.com> * * This program is free software; you can redistribute it and/or @@ -62,28 +63,31 @@ asynchronous and synchronous parts of the kernel. #include "workqueue_internal.h" -static async_cookie_t next_cookie = 1; - #define MAX_WORK 32768 -#define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */ - -static LIST_HEAD(async_global_pending); /* pending from all registered doms */ -static ASYNC_DOMAIN(async_dfl_domain); -static DEFINE_SPINLOCK(async_lock); struct async_entry { - struct list_head domain_list; - struct list_head global_list; - struct work_struct work; - async_cookie_t cookie; - async_func_t func; - void *data; - struct async_domain *domain; + struct async_work base; + struct work_struct work; + + struct list_head pending_link[2]; + + async_cookie_t cookie; + async_func_t func; + void *data; }; -static DECLARE_WAIT_QUEUE_HEAD(async_done); +static LIST_HEAD(async_global_pending); /* pending from all registered doms */ +static ASYNC_DOMAIN(async_dfl_domain); +static DEFINE_SPINLOCK(async_lock); +static unsigned async_pending_count; -static atomic_t entry_count; +static async_cookie_t assign_cookie(void) +{ + static async_cookie_t next_cookie; + if (++next_cookie == 0) + next_cookie = 1; + return next_cookie; +} /** * DOC: kfence overview @@ -166,6 +170,8 @@ static atomic_t entry_count; */ #define KFENCE_CHECKED_BIT 0 +#define ASYNC_WORK_BIT 1 +#define ASYNC_QUEUED_BIT 2 static void kfence_free(struct kref *kref) { @@ -173,7 +179,7 @@ static void kfence_free(struct kref *kref) WARN_ON(atomic_read(&fence->pending) > 0); - if (fence->flags) { + if (fence->flags && !test_bit(ASYNC_WORK_BIT, &fence->flags)) { kfence_notify_t fn = (kfence_notify_t)fence->flags; fn(fence); } else @@ -229,6 +235,13 @@ static void __kfence_signal(struct kfence *fence, if (!atomic_dec_and_test(&fence->pending)) return; + if (test_bit(ASYNC_WORK_BIT, &fence->flags)) { + struct async_entry *entry = + container_of(fence, typeof(*entry), base.fence); + queue_work(system_unbound_wq, &entry->work); + return; + } + if (fence->flags) { kfence_notify_t fn = (kfence_notify_t)fence->flags; fn(fence); @@ -322,7 +335,7 @@ EXPORT_SYMBOL_GPL(kfence_create); void kfence_init(struct kfence *fence, kfence_notify_t fn) { __kfence_init(fence); - BUG_ON((unsigned long)fn & KFENCE_CHECKED_BIT); + BUG_ON((unsigned long)fn & (KFENCE_CHECKED_BIT | ASYNC_WORK_BIT)); fence->flags = (unsigned long)fn; } EXPORT_SYMBOL_GPL(kfence_init); @@ -656,36 +669,10 @@ int kfence_add_delay(struct kfence *fence, } EXPORT_SYMBOL_GPL(kfence_add_delay); -static async_cookie_t lowest_in_progress(struct async_domain *domain) -{ - struct list_head *pending; - async_cookie_t ret = ASYNC_COOKIE_MAX; - unsigned long flags; - - spin_lock_irqsave(&async_lock, flags); - - if (domain) - pending = &domain->pending; - else - pending = &async_global_pending; - - if (!list_empty(pending)) - ret = list_first_entry(pending, struct async_entry, - domain_list)->cookie; - - spin_unlock_irqrestore(&async_lock, flags); - return ret; -} - -/* - * pick the first pending entry and run it - */ static void async_run_entry_fn(struct work_struct *work) { - struct async_entry *entry = - container_of(work, struct async_entry, work); - unsigned long flags; - ktime_t uninitialized_var(calltime), delta, rettime; + struct async_entry *entry = container_of(work, typeof(*entry), work); + ktime_t uninitialized_var(calltime); /* 1) run (and print duration) */ if (initcall_debug && system_state == SYSTEM_BOOTING) { @@ -696,8 +683,7 @@ static void async_run_entry_fn(struct work_struct *work) } entry->func(entry->data, entry->cookie); if (initcall_debug && system_state == SYSTEM_BOOTING) { - rettime = ktime_get(); - delta = ktime_sub(rettime, calltime); + ktime_t delta = ktime_sub(ktime_get(), calltime); pr_debug("initcall %lli_%pF returned 0 after %lld usecs\n", (long long)entry->cookie, entry->func, @@ -705,69 +691,71 @@ static void async_run_entry_fn(struct work_struct *work) } /* 2) remove self from the pending queues */ - spin_lock_irqsave(&async_lock, flags); - list_del_init(&entry->domain_list); - list_del_init(&entry->global_list); - - /* 3) free the entry */ - kfree(entry); - atomic_dec(&entry_count); + spin_lock_irq(&async_lock); + list_del(&entry->pending_link[0]); + list_del(&entry->pending_link[1]); + async_pending_count--; + spin_unlock_irq(&async_lock); - spin_unlock_irqrestore(&async_lock, flags); + /* 3) wake up any waiters */ + atomic_dec(&entry->base.fence.pending); + __kfence_wake_up_all(&entry->base.fence, NULL); - /* 4) wake up any waiters */ - wake_up(&async_done); + kfence_put(&entry->base.fence); } -static async_cookie_t __async_schedule(async_func_t func, void *data, struct async_domain *domain) +struct async_work *async_work_create(async_func_t func, void *data, gfp_t gfp) { struct async_entry *entry; - unsigned long flags; - async_cookie_t newcookie; - /* allow irq-off callers */ - entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC); + entry = kmalloc(sizeof(*entry), gfp); + if (!entry) + return NULL; - /* - * If we're out of memory or if there's too much work - * pending already, we execute synchronously. - */ - if (!entry || atomic_read(&entry_count) > MAX_WORK) { - kfree(entry); - spin_lock_irqsave(&async_lock, flags); - newcookie = next_cookie++; - spin_unlock_irqrestore(&async_lock, flags); + __kfence_init(&entry->base.fence); - /* low on memory.. run synchronously */ - func(data, newcookie); - return newcookie; - } - INIT_LIST_HEAD(&entry->domain_list); - INIT_LIST_HEAD(&entry->global_list); INIT_WORK(&entry->work, async_run_entry_fn); entry->func = func; entry->data = data; - entry->domain = domain; - spin_lock_irqsave(&async_lock, flags); + __set_bit(ASYNC_WORK_BIT, &entry->base.fence.flags); - /* allocate cookie and queue */ - newcookie = entry->cookie = next_cookie++; + return &entry->base; +} +EXPORT_SYMBOL_GPL(async_work_create); - list_add_tail(&entry->domain_list, &domain->pending); - if (domain->registered) - list_add_tail(&entry->global_list, &async_global_pending); +async_cookie_t queue_async_work(struct async_domain *domain, + struct async_work *work, + gfp_t gfp) +{ + struct async_entry *entry = container_of(work, typeof(*entry), base); + unsigned long flags; - atomic_inc(&entry_count); + if (WARN_ON(test_and_set_bit(ASYNC_QUEUED_BIT, + &entry->base.fence.flags))) + return 0; + + spin_lock_irqsave(&async_lock, flags); + entry->cookie = assign_cookie(); + list_add_tail(&entry->pending_link[0], &domain->pending); + INIT_LIST_HEAD(&entry->pending_link[1]); + if (domain->registered) + list_add_tail(&entry->pending_link[1], &async_global_pending); + async_pending_count++; spin_unlock_irqrestore(&async_lock, flags); /* mark that this task has queued an async job, used by module init */ current->flags |= PF_USED_ASYNC; - /* schedule for execution */ - queue_work(system_unbound_wq, &entry->work); + kfence_signal(kfence_get(&entry->base.fence)); - return newcookie; + return entry->cookie; +} +EXPORT_SYMBOL_GPL(queue_async_work); + +async_cookie_t schedule_async_work(struct async_work *work) +{ + return queue_async_work(&async_dfl_domain, work, GFP_KERNEL); } /** @@ -780,7 +768,7 @@ static async_cookie_t __async_schedule(async_func_t func, void *data, struct asy */ async_cookie_t async_schedule(async_func_t func, void *data) { - return __async_schedule(func, data, &async_dfl_domain); + return async_schedule_domain(func, data, &async_dfl_domain); } EXPORT_SYMBOL_GPL(async_schedule); @@ -799,7 +787,27 @@ EXPORT_SYMBOL_GPL(async_schedule); async_cookie_t async_schedule_domain(async_func_t func, void *data, struct async_domain *domain) { - return __async_schedule(func, data, domain); + struct async_work *work; + async_cookie_t cookie = 0; + + work = NULL; + if (READ_ONCE(async_pending_count) < MAX_WORK) + work = async_work_create(func, data, GFP_ATOMIC); + if (work) { + cookie = queue_async_work(domain, work, GFP_ATOMIC); + async_work_put(work); + } + if (!cookie) { + unsigned long flags; + + spin_lock_irqsave(&async_lock, flags); + cookie = assign_cookie(); + spin_unlock_irqrestore(&async_lock, flags); + + func(data, cookie); + } + + return cookie; } EXPORT_SYMBOL_GPL(async_schedule_domain); @@ -825,10 +833,8 @@ EXPORT_SYMBOL_GPL(async_synchronize_full); */ void async_unregister_domain(struct async_domain *domain) { - spin_lock_irq(&async_lock); - WARN_ON(!domain->registered || !list_empty(&domain->pending)); + WARN_ON(!list_empty(&domain->pending)); domain->registered = 0; - spin_unlock_irq(&async_lock); } EXPORT_SYMBOL_GPL(async_unregister_domain); @@ -841,7 +847,7 @@ EXPORT_SYMBOL_GPL(async_unregister_domain); */ void async_synchronize_full_domain(struct async_domain *domain) { - async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain); + async_synchronize_cookie_domain(0, domain); } EXPORT_SYMBOL_GPL(async_synchronize_full_domain); @@ -856,19 +862,49 @@ EXPORT_SYMBOL_GPL(async_synchronize_full_domain); */ void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain) { - ktime_t uninitialized_var(starttime), delta, endtime; + ktime_t uninitialized_var(starttime); + struct list_head *pending; + + pending = domain ? &domain->pending : &async_global_pending; if (initcall_debug && system_state == SYSTEM_BOOTING) { pr_debug("async_waiting @ %i\n", task_pid_nr(current)); starttime = ktime_get(); } - wait_event(async_done, lowest_in_progress(domain) >= cookie); + do { + struct kfence *fence = NULL; + unsigned long flags; - if (initcall_debug && system_state == SYSTEM_BOOTING) { - endtime = ktime_get(); - delta = ktime_sub(endtime, starttime); + spin_lock_irqsave(&async_lock, flags); + if (!list_empty(pending)) { + struct async_entry *entry; + + if (cookie) { + entry = list_first_entry(pending, + struct async_entry, + pending_link[!domain]); + if ((s64)(cookie - entry->cookie) > 0) + fence = kfence_get(&entry->base.fence); + } else { + entry = list_last_entry(pending, + struct async_entry, + pending_link[!domain]); + cookie = entry->cookie; + fence = kfence_get(&entry->base.fence); + } + } + spin_unlock_irqrestore(&async_lock, flags); + + if (!fence) + break; + + kfence_wait(fence); + kfence_put(fence); + } while (1); + if (initcall_debug && system_state == SYSTEM_BOOTING) { + ktime_t delta = ktime_sub(ktime_get(), starttime); pr_debug("async_continuing @ %i after %lli usec\n", task_pid_nr(current), (long long)ktime_to_ns(delta) >> 10); @@ -901,3 +937,11 @@ bool current_is_async(void) return worker && worker->current_func == async_run_entry_fn; } EXPORT_SYMBOL_GPL(current_is_async); + +void init_async_domain(struct async_domain *domain, bool registered) +{ + memset(domain, 0, sizeof(*domain)); + INIT_LIST_HEAD(&domain->pending); + domain->registered = registered; +} +EXPORT_SYMBOL_GPL(init_async_domain); diff --git a/lib/test-async-domain.c b/lib/test-async-domain.c index 558a71414fb6..24c55537c678 100644 --- a/lib/test-async-domain.c +++ b/lib/test-async-domain.c @@ -21,6 +21,269 @@ static void task_B(void *data, async_cookie_t cookie) smp_store_mb(*result, 'B'); } +static int __init test_x(const char *name, + struct async_domain *domain, + async_func_t func, + const long expected) +{ + struct async_work *A; + long result = 0; + + A = async_work_create(func, &result, GFP_KERNEL); + if (!A) + return -ENOMEM; + + queue_async_work(domain, A, GFP_KERNEL); + async_work_wait(A); + async_work_put(A); + + if (READ_ONCE(result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + name, (char)expected, expected, result); + return -EINVAL; + } + + return 0; +} + +static int __init test_A(struct async_domain *domain) +{ + return test_x(__func__, domain, task_A, 'A'); +} + +static int __init test_B(struct async_domain *domain) +{ + return test_x(__func__, domain, task_B, 'B'); +} + +static int __init test_x_fence(const char *name, + struct async_domain *domain, + async_func_t func, + const long expected) +{ + struct async_work *A; + struct kfence *fence; + long result = 0; + + A = async_work_create(func, &result, GFP_KERNEL); + if (!A) + return -ENOMEM; + + fence = kfence_create(GFP_KERNEL); + if (!fence) + return -ENOMEM; + + queue_async_work(domain, A, GFP_KERNEL); + + kfence_add(fence, &A->fence, GFP_KERNEL); + kfence_signal(fence); + + kfence_wait(fence); + + async_work_put(A); + kfence_put(fence); + + if (READ_ONCE(result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + name, (char)expected, expected, result); + return -EINVAL; + } + + return 0; +} + +static int __init test_A_fence(struct async_domain *domain) +{ + return test_x_fence(__func__, domain, task_A, 'A'); +} + +static int __init test_B_fence(struct async_domain *domain) +{ + return test_x_fence(__func__, domain, task_B, 'B'); +} + +static int __init test_x_fence_y(const char *name, + struct async_domain *domain, + async_func_t x, + async_func_t y, + const long expected) +{ + struct async_work *A, *B; + struct kfence *fence; + long result = 0; + + A = async_work_create(x, &result, GFP_KERNEL); + if (!A) + return -ENOMEM; + + B = async_work_create(y, &result, GFP_KERNEL); + if (!B) + return -ENOMEM; + + fence = kfence_create(GFP_KERNEL); + if (!fence) + return -ENOMEM; + + kfence_add(fence, &A->fence, GFP_KERNEL); + kfence_signal(fence); + + queue_async_work(domain, A, GFP_KERNEL); + async_work_put(A); + + async_work_after(B, fence); + queue_async_work(domain, B, GFP_KERNEL); + kfence_put(fence); + + async_work_wait(B); + async_work_put(B); + + if (READ_ONCE(result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + name, (char)expected, expected, result); + return -EINVAL; + } + + return 0; +} + +static int __init test_A_fence_B(struct async_domain *domain) +{ + return test_x_fence_y(__func__, domain, task_A, task_B, 'B'); +} + +static int __init test_B_fence_A(struct async_domain *domain) +{ + return test_x_fence_y(__func__, domain, task_B, task_A, 'A'); +} + +struct long_context { + struct kfence *barrier; + long *src; + long result; +}; + +static void task_wait(void *data, async_cookie_t cookie) +{ + struct long_context *ctx = data; + + kfence_wait(ctx->barrier); + smp_store_mb(ctx->result, READ_ONCE(*ctx->src)); +} + +static int __init test_pause(struct async_domain *domain) +{ + struct long_context ctx; + struct async_work *A, *B; + const long expected = 'B'; + long out_B = 'A'; + + ctx.result = 0; + ctx.src = &out_B; + + A = async_work_create(task_wait, &ctx, GFP_KERNEL); + if (!A) + return -ENOMEM; + + B = async_work_create(task_B, &out_B, GFP_KERNEL); + if (!B) + return -ENOMEM; + + ctx.barrier = kfence_get(&B->fence); + + queue_async_work(domain, A, GFP_KERNEL); + queue_async_work(domain, B, GFP_KERNEL); + async_work_put(B); + + async_work_wait(A); + async_work_put(A); + + if (READ_ONCE(ctx.result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + __func__, (char)expected, expected, ctx.result); + return -EINVAL; + } + + kfence_put(ctx.barrier); + + return 0; +} + +static void task_signal(void *data, async_cookie_t cookie) +{ + struct long_context *ctx = data; + + kfence_signal(ctx->barrier); +} + +static int __init test_manual(struct async_domain *domain) +{ + struct long_context ctx; + struct async_work *A, *B, *C; + const long expected = 'B'; + long out_B = 'A'; + + ctx.result = 0; + ctx.src = &out_B; + ctx.barrier = kfence_create(GFP_KERNEL); + + A = async_work_create(task_wait, &ctx, GFP_KERNEL); + if (!A) + return -ENOMEM; + + B = async_work_create(task_B, &out_B, GFP_KERNEL); + if (!B) + return -ENOMEM; + + C = async_work_create(task_signal, &ctx, GFP_KERNEL); + if (!B) + return -ENOMEM; + + async_work_after(C, &B->fence); + + queue_async_work(domain, A, GFP_KERNEL); + queue_async_work(domain, B, GFP_KERNEL); + queue_async_work(domain, C, GFP_KERNEL); + + async_work_wait(A); + + async_work_put(C); + async_work_put(B); + async_work_put(A); + kfence_put(ctx.barrier); + + if (READ_ONCE(ctx.result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + __func__, (char)expected, expected, ctx.result); + return -EINVAL; + } + + return 0; +} + +static int __init test_sync(struct async_domain *domain) +{ + struct async_work *B; + const long expected = 'B'; + long result = 0; + + B = async_work_create(task_B, &result, GFP_KERNEL); + if (!B) + return -ENOMEM; + + queue_async_work(domain, B, GFP_KERNEL); + async_work_put(B); + + async_synchronize_full_domain(domain); + + if (READ_ONCE(result) != expected) { + pr_warn("%s expected %c [%ld], got %ld\n", + __func__, (char)expected, expected, result); + return -EINVAL; + } + + return 0; +} + static int __init test_implicit(struct async_domain *domain) { const long expected = 'B'; @@ -99,24 +362,62 @@ static int __init test_async_domain_init(void) pr_info("Testing async-domains\n"); - ret = test_implicit(&domain); + ret = test_A(&domain); if (ret) return ret; + ret = test_A_fence(&domain); + if (ret) + goto err; + + ret = test_A_fence_B(&domain); + if (ret) + goto err; + + ret = test_B(&domain); + if (ret) + goto err; + + ret = test_B_fence(&domain); + if (ret) + goto err; + + ret = test_B_fence_A(&domain); + if (ret) + goto err; + + ret = test_pause(&domain); + if (ret) + goto err; + + ret = test_manual(&domain); + if (ret) + goto err; + + ret = test_sync(&domain); + if (ret) + goto err; + + ret = test_implicit(&domain); + if (ret) + goto err; + ret = test_registered(&domain); if (ret) - return ret; + goto err; ret = perf_nop(1, 100); if (ret) - return ret; + goto err; ret = perf_nop(128, 1000); if (ret) - return ret; + goto err; +err: + async_synchronize_full_domain(&domain); async_unregister_domain(&domain); - return 0; + return ret; } static void __exit test_async_domain_cleanup(void) -- 2.8.1

9 years, 7 months

1
0
0 0

[PATCH 6/9] async: Add a convenience wrapper for waiting on implicit dma-buf

by Chris Wilson

dma-buf implicitly track their (DMA) rendering using a reservation_object, which tracks ether the last write (in an exclusive fence) or the current renders (with a set of shared fences). To wait upon a reservation object in conjunction with other sources, kfence_add_reservation() extracts the DMA fences from the object and adds the individual waits for the kfence. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/kfence.h | 5 ++++ kernel/async.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/include/linux/kfence.h b/include/linux/kfence.h index 1abec5e6b23c..2f01eb052e4d 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -17,6 +17,7 @@ struct completion; struct fence; enum hrtimer_mode; +struct reservation_object; struct kfence { wait_queue_head_t wait; @@ -34,6 +35,10 @@ extern int kfence_add_completion(struct kfence *fence, extern int kfence_add_dma(struct kfence *fence, struct fence *dma, gfp_t gfp); +extern int kfence_add_reservation(struct kfence *fence, + struct reservation_object *resv, + bool write, + gfp_t gfp); extern int kfence_add_delay(struct kfence *fence, clockid_t clock, enum hrtimer_mode mode, ktime_t delay, u64 slack, diff --git a/kernel/async.c b/kernel/async.c index 5d02445e36b7..1fa1f39b5a74 100644 --- a/kernel/async.c +++ b/kernel/async.c @@ -54,9 +54,10 @@ asynchronous and synchronous parts of the kernel. #include <linux/kfence.h> #include <linux/ktime.h> #include <linux/export.h> -#include <linux/wait.h> +#include <linux/reservation.h> #include <linux/sched.h> #include <linux/slab.h> +#include <linux/wait.h> #include <linux/workqueue.h> #include "workqueue_internal.h" @@ -123,11 +124,17 @@ static atomic_t entry_count; * allowing multiple pending / signals to be sent before the kfence is * complete. * - * kfence_add() / kfence_add_completion() / kfence_add_dma() + * kfence_add() / kfence_add_completion() + * / kfence_add_dma() + * + * sets the kfence to wait upon another fence or completion respectively. To + * wait upon DMA activity, either use * - * sets the kfence to wait upon another fence, completion, or DMA fence - * respectively. To set the fence to wait for at least a certain period of - * time, or until after a certain point in time, use + * kfence_add_dma() or kfence_add_reservation() + * + * depending on the manner of DMA activity tracking. To set the fence to wait + * for at least a certain period of time, or until after a certain point in + * time, use * * kfence_add_timer() * @@ -547,6 +554,54 @@ int kfence_add_dma(struct kfence *fence, struct fence *dma, gfp_t gfp) EXPORT_SYMBOL_GPL(kfence_add_dma); /** + * kfence_add_reservation - set the fence to wait upon a reservation_object + * @fence: this kfence + * @resv: target reservation_object (collection of DMA fences) to wait upon + * @write: Wait for read or read/write access + * @gfp: the allowed allocation type + * + * kfence_add_reservation() causes the @fence to wait upon completion of the + * reservation object (a collection of DMA fences), either for read access + * or for read/write access. + * + * Returns 1 if the @fence was successfully to the waitqueues of @resv, 0 + * if @resev was already signaled (and so not added), or a negative error code. + */ +int kfence_add_reservation(struct kfence *fence, + struct reservation_object *resv, + bool write, + gfp_t gfp) +{ + struct fence *excl, **shared; + unsigned count, i; + int ret; + + ret = reservation_object_get_fences_rcu(resv, &excl, &count, &shared); + if (ret) + return ret; + + if (write) { + for (i = 0; i < count; i++) { + ret |= kfence_add_dma(fence, shared[i], gfp); + if (ret < 0) + goto out; + } + } + + if (excl) + ret |= kfence_add_dma(fence, excl, gfp); + +out: + fence_put(excl); + for (i = 0; i < count; i++) + fence_put(shared[i]); + kfree(shared); + + return ret; +} +EXPORT_SYMBOL_GPL(kfence_add_reservation); + +/** * kfence_add_delay - set the fence to wait for a period of time * @fence: this kfence * @clock: which clock to program -- 2.8.1

9 years, 7 months

1
0
0 0

[PATCH 4/9] async: Extend kfences for listening on DMA fences

by Chris Wilson

dma-buf provides an interfaces for receiving notifications from DMA hardware. kfence provides a useful interface for collecting such fences and combining them with other events. Signed-off-by: Chris Wilson <chris(a)chris-wilson.co.uk> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Shuah Khan <shuahkh(a)osg.samsung.com> Cc: Tejun Heo <tj(a)kernel.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: Andrew Morton <akpm(a)linux-foundation.org> Cc: Ingo Molnar <mingo(a)kernel.org> Cc: Kees Cook <keescook(a)chromium.org> Cc: Thomas Gleixner <tglx(a)linutronix.de> Cc: "Paul E. McKenney" <paulmck(a)linux.vnet.ibm.com> Cc: Dan Williams <dan.j.williams(a)intel.com> Cc: Andrey Ryabinin <aryabinin(a)virtuozzo.com> Cc: Davidlohr Bueso <dave(a)stgolabs.net> Cc: Nikolay Aleksandrov <nikolay(a)cumulusnetworks.com> Cc: "David S. Miller" <davem(a)davemloft.net> Cc: "Peter Zijlstra (Intel)" <peterz(a)infradead.org> Cc: Rasmus Villemoes <linux(a)rasmusvillemoes.dk> Cc: Andy Shevchenko <andriy.shevchenko(a)linux.intel.com> Cc: Dmitry Vyukov <dvyukov(a)google.com> Cc: Alexander Potapenko <glider(a)google.com> Cc: linux-kernel(a)vger.kernel.org Cc: linux-media(a)vger.kernel.org Cc: dri-devel(a)lists.freedesktop.org Cc: linaro-mm-sig(a)lists.linaro.org --- include/linux/kfence.h | 4 ++++ kernel/async.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/include/linux/kfence.h b/include/linux/kfence.h index 82096bfafaa1..d71f30c626ae 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -15,6 +15,7 @@ #include <linux/wait.h> struct completion; +struct fence; struct kfence { wait_queue_head_t wait; @@ -29,6 +30,9 @@ extern int kfence_add(struct kfence *fence, struct kfence *after, gfp_t gfp); extern int kfence_add_completion(struct kfence *fence, struct completion *x, gfp_t gfp); +extern int kfence_add_dma(struct kfence *fence, + struct fence *dma, + gfp_t gfp); extern void kfence_pending(struct kfence *fence); extern void kfence_signal(struct kfence *fence); extern void kfence_wait(struct kfence *fence); diff --git a/kernel/async.c b/kernel/async.c index db22b890711e..01552d23a916 100644 --- a/kernel/async.c +++ b/kernel/async.c @@ -50,6 +50,7 @@ asynchronous and synchronous parts of the kernel. #include <linux/async.h> #include <linux/atomic.h> +#include <linux/fence.h> #include <linux/kfence.h> #include <linux/ktime.h> #include <linux/export.h> @@ -122,9 +123,10 @@ static atomic_t entry_count; * allowing multiple pending / signals to be sent before the kfence is * complete. * - * kfence_add() / kfence_add_completion() + * kfence_add() / kfence_add_completion() / kfence_add_dma() * - * sets the kfence to wait upon another fence, or completion respectively. + * sets the kfence to wait upon another fence, completion, or DMA fence + * respectively. * * Unlike completions, kfences are expected to live inside more complex graphs * and form the basis for parallel execution of interdependent and so are @@ -484,6 +486,63 @@ int kfence_add_completion(struct kfence *fence, struct completion *x, gfp_t gfp) } EXPORT_SYMBOL_GPL(kfence_add_completion); +struct dma_fence_cb { + struct fence_cb base; + struct kfence *fence; +}; + +static void dma_kfence_wake(struct fence *dma, struct fence_cb *data) +{ + struct dma_fence_cb *cb = container_of(data, typeof(*cb), base); + kfence_signal(cb->fence); + kfence_put(cb->fence); + kfree(cb); +} + +/** + * kfence_add_dma - set the fence to wait upon a DMA fence + * @fence: this kfence + * @dma: target DMA fence to wait upon + * @gfp: the allowed allocation type + * + * kfence_add_dma() causes the @fence to wait upon completion of a DMA fence. + * + * Returns 1 if the @fence was successfully to the waitqueue of @dma, 0 + * if @dma was already signaled (and so not added), or a negative error code. + */ +int kfence_add_dma(struct kfence *fence, struct fence *dma, gfp_t gfp) +{ + struct dma_fence_cb *cb; + int ret; + + if (!dma || fence_is_signaled(dma)) + return 0; + + cb = kmalloc(sizeof(*cb), gfp); + if (!cb) { + if (!gfpflags_allow_blocking(gfp)) + return -ENOMEM; + + return fence_wait(dma, false); + } + + cb->fence = kfence_get(fence); + kfence_pending(fence); + + ret = fence_add_callback(dma, &cb->base, dma_kfence_wake); + if (ret == 0) { + /* signal fence add and is pending */ + ret = 1; + } else { + dma_kfence_wake(dma, &cb->base); + if (ret == -ENOENT) /* fence already signaled */ + ret = 0; + } + + return ret; +} +EXPORT_SYMBOL_GPL(kfence_add_dma); + static async_cookie_t lowest_in_progress(struct async_domain *domain) { struct list_head *pending; -- 2.8.1

9 years, 7 months

1
0
0 0

[PATCH v4 0/4] Generic zpos property

by Benjamin Gaignard

version 4: make sure that normalized zpos value is stay in the defined property range and warn user if not. Fix NULL pointer bug in rcar-du while setting zpos value. No changes in the other drivers. version 3: use kmalloc_array instead of kmalloc. Correct normalize_zpos computation (comeback to Mareck original code) version 2: add a zpos property into drm_plane structure to simplify code. This allow to get/set zpos value in core and not in drivers code. Fix various remarks. version 1: refactor Marek's patches to have per plane zpos property instead of only one in core. Benjamin Gaignard (3): drm: add generic zpos property drm: sti: use generic zpos for plane drm: rcar: use generic code for managing zpos plane property Marek Szyprowski (1): drm/exynos: use generic code for managing zpos plane property Documentation/DocBook/gpu.tmpl | 10 ++ drivers/gpu/drm/Makefile | 2 +- drivers/gpu/drm/drm_atomic.c | 4 + drivers/gpu/drm/drm_atomic_helper.c | 6 + drivers/gpu/drm/drm_blend.c | 230 ++++++++++++++++++++++++++++++ drivers/gpu/drm/drm_crtc_internal.h | 3 + drivers/gpu/drm/exynos/exynos_drm_drv.h | 2 - drivers/gpu/drm/exynos/exynos_drm_plane.c | 67 ++------- drivers/gpu/drm/exynos/exynos_mixer.c | 6 +- drivers/gpu/drm/rcar-du/rcar_du_crtc.c | 2 +- drivers/gpu/drm/rcar-du/rcar_du_drv.h | 1 - drivers/gpu/drm/rcar-du/rcar_du_kms.c | 5 - drivers/gpu/drm/rcar-du/rcar_du_plane.c | 9 +- drivers/gpu/drm/rcar-du/rcar_du_plane.h | 2 - drivers/gpu/drm/sti/sti_mixer.c | 9 +- drivers/gpu/drm/sti/sti_plane.c | 80 ++++------- drivers/gpu/drm/sti/sti_plane.h | 2 - include/drm/drm_crtc.h | 30 ++++ 18 files changed, 331 insertions(+), 139 deletions(-) create mode 100644 drivers/gpu/drm/drm_blend.c Cc: Inki Dae <inki.dae(a)samsung.com> Cc: Daniel Vetter <daniel(a)ffwll.ch> Cc: Ville Syrjala <ville.syrjala(a)linux.intel.com> Cc: Joonyoung Shim <jy0922.shim(a)samsung.com> Cc: Seung-Woo Kim <sw0312.kim(a)samsung.com> Cc: Andrzej Hajda <a.hajda(a)samsung.com> Cc: Krzysztof Kozlowski <k.kozlowski(a)samsung.com> Cc: Bartlomiej Zolnierkiewicz <b.zolnierkie(a)samsung.com> Cc: Tobias Jakobi <tjakobi(a)math.uni-bielefeld.de> Cc: Gustavo Padovan <gustavo(a)padovan.org> Cc: vincent.abriou(a)st.com Cc: fabien.dessenne(a)st.com Cc: Laurent Pinchart <laurent.pinchart(a)ideasonboard.com> -- 1.9.1

9 years, 7 months

2
6
0 0

[PATCH 0/3] dma-buf: debugfs fixes

by Mathias Krause

This small series is the v2 of the patch posted initially here: http://www.spinics.net/lists/linux-media/msg101347.html It not only fixes the type mix-up and addresses Daniel's remark (patch 1), it also smoothes out the error handling in dma_buf_init_debugfs() (patch 2) and removes the then unneeded function dma_buf_debugfs_create_file() (patch 3). Please apply! Mathias Krause (3): dma-buf: propagate errors from dma_buf_describe() on debugfs read dma-buf: remove dma_buf directory on bufinfo file creation errors dma-buf: remove dma_buf_debugfs_create_file() drivers/dma-buf/dma-buf.c | 44 ++++++++++++++------------------------------ include/linux/dma-buf.h | 2 -- 2 files changed, 14 insertions(+), 32 deletions(-) -- 1.7.10.4

9 years, 7 months

3
7
0 0

Re: [Linaro-mm-sig] [Mesa-dev] [RFC] New dma_buf -> EGLImage EGL extension - Final spec published!

by Rob Clark

So, if we wanted to extend this to support the fourcc-modifiers that we have on the kernel side for compressed/tiled/etc formats, what would be the right approach? A new version of the existing extension or a new EGL_EXT_image_dma_buf_import2 extension, or ?? BR, -R On Mon, Feb 25, 2013 at 6:54 AM, Tom Cooksey <tom.cooksey(a)arm.com> wrote: > Hi All, > > The final spec has had enum values assigned and been published on Khronos: > > http://www.khronos.org/registry/egl/extensions/EXT/EGL_EXT_image_dma_buf_im… > > Thanks to all who've provided input. > > > Cheers, > > Tom > > > >> -----Original Message----- >> From: mesa-dev-bounces+tom.cooksey=arm.com(a)lists.freedesktop.org [mailto:mesa-dev- >> bounces+tom.cooksey=arm.com(a)lists.freedesktop.org] On Behalf Of Tom Cooksey >> Sent: 04 October 2012 13:10 >> To: mesa-dev(a)lists.freedesktop.org; linaro-mm-sig(a)lists.linaro.org; dri- >> devel(a)lists.freedesktop.org; linux-media(a)vger.kernel.org >> Subject: [Mesa-dev] [RFC] New dma_buf -> EGLImage EGL extension - New draft! >> >> Hi All, >> >> After receiving a fair bit of feedback (thanks!), I've updated the >> EGL_EXT_image_dma_buf_import spec >> and expanded it to resolve a number of the issues. Please find the latest draft below and let >> me >> know any additional feedback you might have, either on the lists or by private e-mail - I >> don't mind >> which. >> >> I think the only remaining issue now is if we need a mechanism whereby an application can >> query >> which drm_fourcc.h formats EGL supports or if just failing with EGL_BAD_MATCH when the >> application >> has use one EGL doesn't support is sufficient. Any thoughts? >> >> >> Cheers, >> >> Tom >> >> >> --------------------8<-------------------- >> >> >> Name >> >> EXT_image_dma_buf_import >> >> Name Strings >> >> EGL_EXT_image_dma_buf_import >> >> Contributors >> >> Jesse Barker >> Rob Clark >> Tom Cooksey >> >> Contacts >> >> Jesse Barker (jesse 'dot' barker 'at' linaro 'dot' org) >> Tom Cooksey (tom 'dot' cooksey 'at' arm 'dot' com) >> >> Status >> >> DRAFT >> >> Version >> >> Version 4, October 04, 2012 >> >> Number >> >> EGL Extension ??? >> >> Dependencies >> >> EGL 1.2 is required. >> >> EGL_KHR_image_base is required. >> >> The EGL implementation must be running on a Linux kernel supporting the >> dma_buf buffer sharing mechanism. >> >> This extension is written against the wording of the EGL 1.2 Specification. >> >> Overview >> >> This extension allows creating an EGLImage from a Linux dma_buf file >> descriptor or multiple file descriptors in the case of multi-plane YUV >> images. >> >> New Types >> >> None >> >> New Procedures and Functions >> >> None >> >> New Tokens >> >> Accepted by the <target> parameter of eglCreateImageKHR: >> >> EGL_LINUX_DMA_BUF_EXT >> >> Accepted as an attribute in the <attrib_list> parameter of >> eglCreateImageKHR: >> >> EGL_LINUX_DRM_FOURCC_EXT >> EGL_DMA_BUF_PLANE0_FD_EXT >> EGL_DMA_BUF_PLANE0_OFFSET_EXT >> EGL_DMA_BUF_PLANE0_PITCH_EXT >> EGL_DMA_BUF_PLANE1_FD_EXT >> EGL_DMA_BUF_PLANE1_OFFSET_EXT >> EGL_DMA_BUF_PLANE1_PITCH_EXT >> EGL_DMA_BUF_PLANE2_FD_EXT >> EGL_DMA_BUF_PLANE2_OFFSET_EXT >> EGL_DMA_BUF_PLANE2_PITCH_EXT >> EGL_YUV_COLOR_SPACE_HINT_EXT >> EGL_SAMPLE_RANGE_HINT_EXT >> EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT >> EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT >> >> Accepted as the value for the EGL_YUV_COLOR_SPACE_HINT_EXT attribute: >> >> EGL_ITU_REC601_EXT >> EGL_ITU_REC709_EXT >> EGL_ITU_REC2020_EXT >> >> Accepted as the value for the EGL_SAMPLE_RANGE_HINT_EXT attribute: >> >> EGL_YUV_FULL_RANGE_EXT >> EGL_YUV_NARROW_RANGE_EXT >> >> Accepted as the value for the EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT & >> EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT attributes: >> >> EGL_YUV_CHROMA_SITING_0_EXT >> EGL_YUV_CHROMA_SITING_0_5_EXT >> >> >> Additions to Chapter 2 of the EGL 1.2 Specification (EGL Operation) >> >> Add to section 2.5.1 "EGLImage Specification" (as defined by the >> EGL_KHR_image_base specification), in the description of >> eglCreateImageKHR: >> >> "Values accepted for <target> are listed in Table aaa, below. >> >> +-------------------------+--------------------------------------------+ >> | <target> | Notes | >> +-------------------------+--------------------------------------------+ >> | EGL_LINUX_DMA_BUF_EXT | Used for EGLImages imported from Linux | >> | | dma_buf file descriptors | >> +-------------------------+--------------------------------------------+ >> Table aaa. Legal values for eglCreateImageKHR <target> parameter >> >> ... >> >> If <target> is EGL_LINUX_DMA_BUF_EXT, <dpy> must be a valid display, <ctx> >> must be EGL_NO_CONTEXT, and <buffer> must be NULL, cast into the type >> EGLClientBuffer. The details of the image is specified by the attributes >> passed into eglCreateImageKHR. Required attributes and their values are as >> follows: >> >> * EGL_WIDTH & EGL_HEIGHT: The logical dimensions of the buffer in pixels >> >> * EGL_LINUX_DRM_FOURCC_EXT: The pixel format of the buffer, as specified >> by drm_fourcc.h and used as the pixel_format parameter of the >> drm_mode_fb_cmd2 ioctl. >> >> * EGL_DMA_BUF_PLANE0_FD_EXT: The dma_buf file descriptor of plane 0 of >> the image. >> >> * EGL_DMA_BUF_PLANE0_OFFSET_EXT: The offset from the start of the >> dma_buf of the first sample in plane 0, in bytes. >> >> * EGL_DMA_BUF_PLANE0_PITCH_EXT: The number of bytes between the start of >> subsequent rows of samples in plane 0. May have special meaning for >> non-linear formats. >> >> For images in an RGB color-space or those using a single-plane YUV format, >> only the first plane's file descriptor, offset & pitch should be specified. >> For semi-planar YUV formats, the chroma samples are stored in plane 1 and >> for fully planar formats, U-samples are stored in plane 1 and V-samples are >> stored in plane 2. Planes 1 & 2 are specified by the following attributes, >> which have the same meanings as defined above for plane 0: >> >> * EGL_DMA_BUF_PLANE1_FD_EXT >> * EGL_DMA_BUF_PLANE1_OFFSET_EXT >> * EGL_DMA_BUF_PLANE1_PITCH_EXT >> * EGL_DMA_BUF_PLANE2_FD_EXT >> * EGL_DMA_BUF_PLANE2_OFFSET_EXT >> * EGL_DMA_BUF_PLANE2_PITCH_EXT >> >> In addition to the above required attributes, the application may also >> provide hints as to how the data should be interpreted by the GL. If any of >> these hints are not specified, the GL will guess based on the pixel format >> passed as the EGL_LINUX_DRM_FOURCC_EXT attribute or may fall-back to some >> default value. Not all GLs will be able to support all combinations of >> these hints and are free to use whatever settings they choose to achieve >> the closest possible match. >> >> * EGL_YUV_COLOR_SPACE_HINT_EXT: The color-space the data is in. Only >> relevant for images in a YUV format, ignored when specified for an >> image in an RGB format. Accepted values are: >> EGL_ITU_REC601_EXT, EGL_ITU_REC709_EXT & EGL_ITU_REC2020_EXT. >> >> * EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT & >> EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT: Where chroma samples are >> sited relative to luma samples when the image is in a sub-sampled >> format. When the image is not using chroma sub-sampling, the luma and >> chroma samples are assumed to be co-sited. Siting is split into the >> vertical and horizontal and is in a fixed range. A siting of zero >> means the first luma sample is taken from the same position in that >> dimension as the chroma sample. This is best illustrated in the >> diagram below: >> >> (0.5, 0.5) (0.0, 0.5) (0.0, 0.0) >> + + + + + + + + * + * + >> x x x x >> + + + + + + + + + + + + >> >> + + + + + + + + * + * + >> x x x x >> + + + + + + + + + + + + >> >> Luma samples (+), Chroma samples (x) Chrome & Luma samples (*) >> >> Note this attribute is ignored for RGB images and non sub-sampled >> YUV images. Accepted values are: EGL_YUV_CHROMA_SITING_0_EXT (0.0) >> & EGL_YUV_CHROMA_SITING_0_5_EXT (0.5) >> >> * EGL_SAMPLE_RANGE_HINT_EXT: The numerical range of samples. Only >> relevant for images in a YUV format, ignored when specified for >> images in an RGB format. Accepted values are: EGL_YUV_FULL_RANGE_EXT >> (0-256) & EGL_YUV_NARROW_RANGE_EXT (16-235). >> >> >> If eglCreateImageKHR is successful for a EGL_LINUX_DMA_BUF_EXT target, the >> EGL takes ownership of the file descriptor and is responsible for closing >> it, which it may do at any time while the EGLDisplay is initialized." >> >> >> Add to the list of error conditions for eglCreateImageKHR: >> >> "* If <target> is EGL_LINUX_DMA_BUF_EXT and <buffer> is not NULL, the >> error EGL_BAD_PARAMETER is generated. >> >> * If <target> is EGL_LINUX_DMA_BUF_EXT, and the list of attributes is >> incomplete, EGL_BAD_PARAMETER is generated. >> >> * If <target> is EGL_LINUX_DMA_BUF_EXT, and the EGL_LINUX_DRM_FOURCC_EXT >> attribute is set to a format not supported by the EGL, EGL_BAD_MATCH >> is generated. >> >> * If <target> is EGL_LINUX_DMA_BUF_EXT, and the EGL_LINUX_DRM_FOURCC_EXT >> attribute indicates a single-plane format, EGL_BAD_ATTRIBUTE is >> generated if any of the EGL_DMA_BUF_PLANE1_* or EGL_DMA_BUF_PLANE2_* >> attributes are specified. >> >> * If <target> is EGL_LINUX_DMA_BUF_EXT and the value specified for >> EGL_YUV_COLOR_SPACE_HINT_EXT is not EGL_ITU_REC601_EXT, >> EGL_ITU_REC709_EXT or EGL_ITU_REC2020_EXT, EGL_BAD_ATTRIBUTE is >> generated. >> >> * If <target> is EGL_LINUX_DMA_BUF_EXT and the value specified for >> EGL_SAMPLE_RANGE_HINT_EXT is not EGL_YUV_FULL_RANGE_EXT or >> EGL_YUV_NARROW_RANGE_EXT, EGL_BAD_ATTRIBUTE is generated. >> >> * If <target> is EGL_LINUX_DMA_BUF_EXT and the value specified for >> EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT or >> EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT is not >> EGL_YUV_CHROMA_SITING_0_EXT or EGL_YUV_CHROMA_SITING_0_5_EXT, >> EGL_BAD_ATTRIBUTE is generated. >> >> * If <target> is EGL_LINUX_DMA_BUF_EXT and one or more of the values >> specified for a plane's pitch or offset isn't supported by EGL, >> EGL_BAD_ACCESS is generated. >> >> * If <target> is EGL_LINUX_DMA_BUF_EXT and eglCreateImageKHR fails, >> EGL does not retain ownership of the file descriptor and it is the >> responsibility of the application to close it." >> >> >> Issues >> >> 1. Should this be a KHR or EXT extension? >> >> ANSWER: EXT. Khronos EGL working group not keen on this extension as it is >> seen as contradicting the EGLStream direction the specification is going in. >> The working group recommends creating additional specs to allow an EGLStream >> producer/consumer connected to v4l2/DRM or any other Linux interface. >> >> 2. Should this be a generic any platform extension, or a Linux-only >> extension which explicitly states the handles are dma_buf fds? >> >> ANSWER: There's currently no intention to port this extension to any OS not >> based on the Linux kernel. Consequently, this spec can be explicitly written >> against Linux and the dma_buf API. >> >> 3. Does ownership of the file descriptor pass to the EGL library? >> >> ANSWER: If eglCreateImageKHR is successful, EGL assumes ownership of the >> file descriptors and is responsible for closing them. >> >> 4. How are the different YUV color spaces handled (BT.709/BT.601)? >> >> ANSWER: The pixel formats defined in drm_fourcc.h only specify how the data >> is laid out in memory. It does not define how that data should be >> interpreted. Added a new EGL_YUV_COLOR_SPACE_HINT_EXT attribute to allow the >> application to specify which color space the data is in to allow the GL to >> choose an appropriate set of co-efficients if it needs to convert that data >> to RGB for example. >> >> 5. What chroma-siting is used for sub-sampled YUV formats? >> >> ANSWER: The chroma siting is not specified by either the v4l2 or DRM APIs. >> This is similar to the color-space issue (4) in that the chroma siting >> doesn't affect how the data is stored in memory. However, the GL will need >> to know the siting in order to filter the image correctly. While the visual >> impact of getting the siting wrong is minor, provision should be made to >> allow an application to specify the siting if desired. Added additional >> EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT & >> EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT attributes to allow the siting to >> be specified using a set of pre-defined values (0 or 0.5). >> >> 6. How can an application query which formats the EGL implementation >> supports? >> >> PROPOSAL: Don't provide a query mechanism but instead add an error condition >> that EGL_BAD_MATCH is raised if the EGL implementation doesn't support that >> particular format. >> >> 7. Which image formats should be supported and how is format specified? >> >> Seem to be two options 1) specify a new enum in this specification and >> enumerate all possible formats. 2) Use an existing enum already in Linux, >> either v4l2_mbus_pixelcode and/or those formats listed in drm_fourcc.h? >> >> ANSWER: Go for option 2) and just use values defined in drm_fourcc.h. >> >> 8. How can AYUV images be handled? >> >> ANSWER: At least on fourcc.org and in drm_fourcc.h, there only seems to be >> a single AYUV format and that is a packed format, so everything, including >> the alpha component would be in the first plane. >> >> 9. How can you import interlaced images? >> >> ANSWER: Interlaced frames are usually stored with the top & bottom fields >> interleaved in a single buffer. As the fields would need to be displayed as >> at different times, the application would create two EGLImages from the same >> buffer, one for the top field and another for the bottom. Both EGLImages >> would set the pitch to 2x the buffer width and the second EGLImage would use >> a suitable offset to indicate it started on the second line of the buffer. >> This should work regardless of whether the data is packed in a single plane, >> semi-planar or multi-planar. >> >> If each interlaced field is stored in a separate buffer then it should be >> trivial to create two EGLImages, one for each field's buffer. >> >> 10. How are semi-planar/planar formats handled that have a different >> width/height for Y' and CbCr such as YUV420? >> >> ANSWER: The spec says EGL_WIDTH & EGL_HEIGHT specify the *logical* width and >> height of the buffer in pixels. For pixel formats with sub-sampled Chroma >> values, it should be trivial for the EGL implementation to calculate the >> width/height of the Chroma sample buffers using the logical width & height >> and by inspecting the pixel format passed as the EGL_LINUX_DRM_FOURCC_EXT >> attribute. I.e. If the pixel format says it's YUV420, the Chroma buffer's >> width = EGL_WIDTH/2 & height =EGL_HEIGHT/2. >> >> 11. How are Bayer formats handled? >> >> ANSWER: As of Linux 2.6.34, drm_fourcc.h does not include any Bayer formats. >> However, future kernel versions may add such formats in which case they >> would be handled in the same way as any other format. >> >> 12. Should the spec support buffers which have samples in a "narrow range"? >> >> Content sampled from older analogue sources typically don't use the full >> (0-256) range of the data type storing the sample and instead use a narrow >> (16-235) range to allow some headroom & toeroom in the signals to avoid >> clipping signals which overshoot slightly during processing. This is >> sometimes known as signals using "studio swing". >> >> ANSWER: Add a new attribute to define if the samples use a narrow 16-235 >> range or the full 0-256 range. >> >> 13. Specifying the color space and range seems cumbersome, why not just >> allow the application to specify the full YUV->RGB color conversion matrix? >> >> ANSWER: Some hardware may not be able to use an arbitrary conversion matrix >> and needs to select an appropriate pre-defined matrix based on the color >> space and the sample range. >> >> 14. How do you handle EGL implementations which have restrictions on pitch >> and/or offset? >> >> ANSWER: Buffers being imported using dma_buf pretty much have to be >> allocated by a kernel-space driver. As such, it is expected that a system >> integrator would make sure all devices which allocate buffers suitable for >> exporting make sure they use a pitch supported by all possible importers. >> However, it is still possible eglCreateImageKHR can fail due to an >> unsupported pitch. Added a new error to the list indicating this. >> >> 15. Should this specification also describe how to export an existing >> EGLImage as a dma_buf file descriptor? >> >> ANSWER: No. Importing and exporting buffers are two separate operations and >> importing an existing dma_buf fd into an EGLImage is useful functionality in >> itself. Agree that exporting an EGLImage as a dma_buf fd is useful, E.g. it >> could be used by an OpenMAX IL implementation's OMX_UseEGLImage function to >> give access to the buffer backing an EGLImage to video hardware. However, >> exporting can be split into a separate extension specification. >> >> >> Revision History >> >> #4 (Tom Cooksey, October 04, 2012) >> - Fixed issue numbering! >> - Added issues 8 - 15. >> - Promoted proposal for Issue 3 to be the answer. >> - Added an additional attribute to allow an application to specify the color >> space as a hint which should address issue 4. >> - Added an additional attribute to allow an application to specify the chroma >> siting as a hint which should address issue 5. >> - Added an additional attribute to allow an application to specify the sample >> range as a hint which should address the new issue 12. >> - Added language to end of error section clarifying who owns the fd passed >> to eglCreateImageKHR if an error is generated. >> >> #3 (Tom Cooksey, August 16, 2012) >> - Changed name from EGL_EXT_image_external and re-written language to >> explicitly state this for use with Linux & dma_buf. >> - Added a list of issues, including some still open ones. >> >> #2 (Jesse Barker, May 30, 2012) >> - Revision to split eglCreateImageKHR functionality from export >> Functionality. >> - Update definition of EGLNativeBufferType to be a struct containing a list >> of handles to support multi-buffer/multi-planar formats. >> >> #1 (Jesse Barker, March 20, 2012) >> - Initial draft. >> >> >> >> >> _______________________________________________ >> mesa-dev mailing list >> mesa-dev(a)lists.freedesktop.org >> http://lists.freedesktop.org/mailman/listinfo/mesa-dev > > > > > _______________________________________________ > mesa-dev mailing list > mesa-dev(a)lists.freedesktop.org > http://lists.freedesktop.org/mailman/listinfo/mesa-dev

9 years, 7 months

2
8
0 0

[PATCH] dma-buf: propagate errors from dma_buf_describe() on debugfs read

by Mathias Krause

The callback function dma_buf_describe() returns an int not void so the function pointer cast in dma_buf_show() is wrong. dma_buf_describe() can also fail when acquiring the mutex gets interrupted so always returning 0 in dma_buf_show() is wrong, too. Fix both issues by casting the function pointer to the correct type and propagate its return value. This type mismatch was caught by the PaX RAP plugin. Signed-off-by: Mathias Krause <minipli(a)googlemail.com> Cc: Sumit Semwal <sumit.semwal(a)linaro.org> Cc: Daniel Vetter <daniel.vetter(a)ffwll.ch> Cc: PaX Team <pageexec(a)freemail.hu> --- drivers/dma-buf/dma-buf.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 6355ab38d630..0f2a4592fdd2 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -881,10 +881,9 @@ static int dma_buf_describe(struct seq_file *s) static int dma_buf_show(struct seq_file *s, void *unused) { - void (*func)(struct seq_file *) = s->private; + int (*func)(struct seq_file *) = s->private; - func(s); - return 0; + return func(s); } static int dma_buf_debug_open(struct inode *inode, struct file *file) -- 1.7.10.4

9 years, 8 months

2
2
0 0

[PATCH 0/5] Ion cleanup

by Laura Abbott

Hi, This is some clean up of old Ion interfaces and APIs. These are interfaces that mostly existed before dma_buf was well integrated into the kernel along with reservations for board files. If there are objections to deletion I expect it to turn into a discussion about what other APIs need to be extended. Thanks, Laura Laura Abbott (5): staging: android: ion: Get rid of ion_sg_table staging: android: ion: Drop ion_phys interface staging: android: ion: Get rid of map_dma/unmap_dma staging: android: ion: Drop ion_carveout_allocate definitions staging: android: ion: Get rid of ion_reserve drivers/staging/android/ion/ion.c | 103 ++---------------------- drivers/staging/android/ion/ion.h | 41 ---------- drivers/staging/android/ion/ion_carveout_heap.c | 33 +------- drivers/staging/android/ion/ion_chunk_heap.c | 17 +--- drivers/staging/android/ion/ion_cma_heap.c | 34 +------- drivers/staging/android/ion/ion_priv.h | 30 +------ drivers/staging/android/ion/ion_system_heap.c | 44 +--------- 7 files changed, 19 insertions(+), 283 deletions(-) -- 2.5.5

9 years, 8 months

1
5
0 0

Re: [Linaro-mm-sig] [RFC][PATCH 6/6] staging: android: ion: Introduce new ioctls for dynamic heaps

by Laura Abbott

On 06/08/2016 08:34 AM, Brian Starkey wrote: > Hi, > > I'm finding "usage_id" a bit confusing - there's not a very clear > distinction between usage_id and heap ID. > > For instance, ION_IOC_USAGE_CNT claims to return the number of usage > IDs, but seems to return the number of heaps (i.e. number heap IDs, some > of which might be usage_ids). > > Similarly, alloc2 claims to allocate by usage_id, but will just as > happily allocate by heap ID. > > Maybe this should just be described as a single heap ID namespace, where > some IDs represent groups of heap IDs. > For now I'm just going to focus on comments not about the heap ID mapping because I'm leaning towards dropping the heap ID mapping. > I've a few inline comments below. > > -Brian > > On Mon, Jun 06, 2016 at 11:23:33AM -0700, Laura Abbott wrote: >> From: Laura Abbott <labbott(a)fedoraproject.org> >> >> >> The Ion ABI for heaps is limiting to work with for more complex systems. >> Heaps have to be registered at boot time with known ids available to >> userspace. This becomes a tight ABI which is prone to breakage. >> >> Introduce a new mechanism for registering heap ids dynamically. A base >> set of heap ids are registered at boot time but there is no knowledge >> of fallbacks. Fallback information can be remapped and changed >> dynamically. Information about available heaps can also be queried with >> an ioctl to avoid the need to have heap ids be an ABI with userspace. >> >> Signed-off-by: Laura Abbott <labbott(a)redhat.com> >> --- >> drivers/staging/android/ion/ion-ioctl.c | 109 +++++++++++++++++-- >> drivers/staging/android/ion/ion.c | 184 ++++++++++++++++++++++++++++++-- >> drivers/staging/android/ion/ion_priv.h | 15 +++ >> drivers/staging/android/uapi/ion.h | 135 +++++++++++++++++++++++ >> 4 files changed, 426 insertions(+), 17 deletions(-) >> >> diff --git a/drivers/staging/android/ion/ion-ioctl.c b/drivers/staging/android/ion/ion-ioctl.c >> index 45b89e8..169cad8 100644 >> --- a/drivers/staging/android/ion/ion-ioctl.c >> +++ b/drivers/staging/android/ion/ion-ioctl.c >> @@ -22,6 +22,49 @@ >> #include "ion_priv.h" >> #include "compat_ion.h" >> >> +union ion_ioctl_arg { >> + struct ion_fd_data fd; >> + struct ion_allocation_data allocation; >> + struct ion_handle_data handle; >> + struct ion_custom_data custom; >> + struct ion_abi_version abi_version; >> + struct ion_new_alloc_data allocation2; >> + struct ion_usage_id_map id_map; >> + struct ion_usage_cnt usage_cnt; >> + struct ion_heap_query query; >> +}; >> + >> +static int validate_ioctl_arg(unsigned int cmd, union ion_ioctl_arg *arg) >> +{ >> + int ret = 0; >> + >> + switch (cmd) { >> + case ION_IOC_ABI_VERSION: >> + ret = arg->abi_version.reserved != 0; >> + break; >> + case ION_IOC_ALLOC2: >> + ret = arg->allocation2.reserved0 != 0; >> + ret |= arg->allocation2.reserved1 != 0; >> + ret |= arg->allocation2.reserved2 != 0; >> + break; >> + case ION_IOC_ID_MAP: >> + ret = arg->id_map.reserved0 != 0; >> + ret |= arg->id_map.reserved1 != 0; >> + break; >> + case ION_IOC_USAGE_CNT: >> + ret = arg->usage_cnt.reserved != 0; >> + break; >> + case ION_IOC_HEAP_QUERY: >> + ret = arg->query.reserved0 != 0; >> + ret |= arg->query.reserved1 != 0; >> + ret |= arg->query.reserved2 != 0; >> + break; >> + default: >> + break; >> + } >> + return ret ? -EINVAL : 0; >> +} >> + >> /* fix up the cases where the ioctl direction bits are incorrect */ >> static unsigned int ion_ioctl_dir(unsigned int cmd) >> { >> @@ -42,14 +85,7 @@ long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) >> struct ion_handle *cleanup_handle = NULL; >> int ret = 0; >> unsigned int dir; >> - >> - union { >> - struct ion_fd_data fd; >> - struct ion_allocation_data allocation; >> - struct ion_handle_data handle; >> - struct ion_custom_data custom; >> - struct ion_abi_version abi_version; >> - } data; >> + union ion_ioctl_arg data; >> >> dir = ion_ioctl_dir(cmd); >> >> @@ -60,7 +96,12 @@ long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) >> if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd))) >> return -EFAULT; >> >> + ret = validate_ioctl_arg(cmd, &data); >> + if (ret) >> + return ret; >> + >> switch (cmd) { >> + /* Old ioctl */ >> case ION_IOC_ALLOC: >> { >> struct ion_handle *handle; >> @@ -77,6 +118,7 @@ long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) >> cleanup_handle = handle; >> break; >> } >> + /* Old ioctl */ >> case ION_IOC_FREE: >> { >> struct ion_handle *handle; >> @@ -92,6 +134,7 @@ long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) >> mutex_unlock(&client->lock); >> break; >> } >> + /* Old ioctl */ >> case ION_IOC_SHARE: >> case ION_IOC_MAP: >> { >> @@ -106,6 +149,7 @@ long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) >> ret = data.fd.fd; >> break; >> } >> + /* Old ioctl */ >> case ION_IOC_IMPORT: >> { >> struct ion_handle *handle; >> @@ -117,11 +161,13 @@ long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) >> data.handle.handle = handle->id; >> break; >> } >> + /* Old ioctl */ >> case ION_IOC_SYNC: >> { >> ret = ion_sync_for_device(client, data.fd.fd); >> break; >> } >> + /* Old ioctl */ >> case ION_IOC_CUSTOM: >> { >> if (!dev->custom_ioctl) >> @@ -135,6 +181,53 @@ long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) >> data.abi_version.abi_version = ION_ABI_VERSION; >> break; >> } >> + case ION_IOC_ALLOC2: >> + { >> + struct ion_handle *handle; >> + >> + handle = ion_alloc2(client, data.allocation2.len, >> + data.allocation2.align, >> + data.allocation2.usage_id, >> + data.allocation2.flags); >> + if (IS_ERR(handle)) >> + return PTR_ERR(handle); >> + >> + if (data.allocation2.flags & ION_FLAG_NO_HANDLE) { >> + data.allocation2.fd = ion_share_dma_buf_fd(client, >> + handle); >> + ion_handle_put(handle); >> + if (data.allocation2.fd < 0) >> + ret = data.allocation2.fd; >> + } else { >> + data.allocation2.handle = handle->id; >> + >> + cleanup_handle = handle; >> + } >> + break; >> + } >> + case ION_IOC_ID_MAP: >> + { >> + ret = ion_map_usage_ids(client, >> + (unsigned int __user *)data.id_map.usage_ids, >> + data.id_map.cnt); >> + if (ret > 0) >> + data.id_map.new_id = ret; >> + break; >> + } >> + case ION_IOC_USAGE_CNT: >> + { >> + down_read(&client->dev->lock); >> + data.usage_cnt.cnt = client->dev->heap_cnt; >> + up_read(&client->dev->lock); >> + break; >> + } >> + case ION_IOC_HEAP_QUERY: >> + { >> + ret = ion_query_heaps(client, >> + (struct ion_heap_data __user *)data.query.heaps, >> + data.query.cnt); >> + break; >> + } >> default: >> return -ENOTTY; >> } >> diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c >> index 129707f..c096cb9 100644 >> --- a/drivers/staging/android/ion/ion.c >> +++ b/drivers/staging/android/ion/ion.c >> @@ -459,6 +459,48 @@ static struct ion_handle *__ion_alloc(struct ion_client *client, size_t len, >> return handle; >> } >> >> +struct ion_handle *ion_alloc2(struct ion_client *client, size_t len, >> + size_t align, unsigned int usage_id, >> + unsigned int flags) >> +{ >> + struct ion_device *dev = client->dev; >> + struct ion_heap *heap; >> + struct ion_handle *handle = ERR_PTR(-ENODEV); >> + >> + down_read(&dev->lock); >> + heap = idr_find(&dev->idr, usage_id); >> + if (!heap) { >> + handle = ERR_PTR(-EINVAL); >> + goto out; >> + } >> + >> + if (heap->type == ION_USAGE_ID_MAP) { >> + int i; >> + >> + for (i = 0; i < heap->fallback_cnt; i++){ >> + heap = idr_find(&dev->idr, heap->fallbacks[i]); >> + if (!heap) >> + continue; >> + >> + /* Don't recurse for now? */ >> + if (heap->type == ION_USAGE_ID_MAP) >> + continue; >> + >> + handle = __ion_alloc(client, len, align, heap, flags); >> + if (IS_ERR(handle)) >> + continue; >> + else >> + break; >> + } >> + } else { >> + handle = __ion_alloc(client, len, align, heap, flags); >> + } >> +out: >> + up_read(&dev->lock); >> + return handle; >> +} >> +EXPORT_SYMBOL(ion_alloc2); >> + >> struct ion_handle *ion_alloc(struct ion_client *client, size_t len, >> size_t align, unsigned int heap_mask, >> unsigned int flags) >> @@ -1243,6 +1285,55 @@ int ion_sync_for_device(struct ion_client *client, int fd) >> return 0; >> } >> >> +struct ion_query_data { >> + struct ion_heap_data __user *buffer; >> + int cnt; >> + int max_cnt; >> +}; >> + >> +int __ion_query(int id, void *p, void *data) >> +{ >> + struct ion_heap *heap = p; >> + struct ion_query_data *query = data; >> + struct ion_heap_data hdata = {0}; >> + >> + if (query->cnt >= query->max_cnt) >> + return -ENOSPC; >> + >> + strncpy(hdata.name, heap->name, 20); > > Why 20? > Arbitrary limit for a fixed size buffer. As noted below I need to make this a consistent #define. >> + hdata.name[sizeof(hdata.name) - 1] = '\0'; >> + hdata.type = heap->type; >> + hdata.usage_id = heap->id; >> + >> + return copy_to_user(&query->buffer[query->cnt++], &hdata, sizeof(hdata)); >> +} >> + >> +int ion_query_heaps(struct ion_client *client, >> + struct ion_heap_data __user *buffer, >> + int cnt) >> +{ >> + struct ion_device *dev = client->dev; >> + struct ion_query_data data; >> + int ret; >> + >> + data.buffer = buffer; >> + data.cnt = 0; >> + data.max_cnt = cnt; >> + >> + down_read(&dev->lock); >> + if (data.max_cnt < 0 || data.max_cnt > dev->heap_cnt) { > > Is the check against heap_cnt necessary? You could update the cnt > field in ion_heap_query to say how many were actually copied. > This was mostly a paranoid check to reduce risk of accidentally giving userspace access to something it shouldn't. Updating the count is still a good idea in case userspace wants fewer than are available. > I think you could probably drop ION_IOC_USAGE_COUNT by giving > ION_IOC_HEAP_QUERY similar semantics to the DRM ioctls: > - First call with buffer pointer = NULL, kernel sets cnt to space > required > - Userspace allocates buffer and calls the ioctl again, kernel fills > the buffer > I'm not that familiar with the DRM ioctls but this sounds like a good suggestion. One less ioctl to worry about. >> + ret = -EINVAL; >> + goto out; >> + } >> + ret = idr_for_each(&dev->idr, __ion_query, &data); >> +out: >> + up_read(&dev->lock); >> + >> + return ret; >> +} >> + >> + >> + >> static int ion_release(struct inode *inode, struct file *file) >> { >> struct ion_client *client = file->private_data; >> @@ -1407,13 +1498,69 @@ static int debug_shrink_get(void *data, u64 *val) >> DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get, >> debug_shrink_set, "%llu\n"); >> >> +int ion_map_usage_ids(struct ion_client *client, >> + unsigned int __user *usage_ids, >> + int cnt) >> +{ >> + struct ion_heap *heap = kzalloc(sizeof(*heap), GFP_KERNEL); >> + unsigned int *fallbacks; >> + int i; >> + int ret; >> + >> + if (!heap) >> + return -ENOMEM; >> + >> + fallbacks = kzalloc(sizeof(unsigned int)*cnt, GFP_KERNEL); >> + if (!fallbacks) { >> + ret = -ENOMEM; >> + goto out1; >> + } >> + >> + ret = copy_from_user(fallbacks, usage_ids, sizeof(unsigned int)*cnt); >> + if (ret) >> + goto out2; >> + >> + down_read(&client->dev->lock); >> + for (i = 0; i < cnt; i++) { >> + if (idr_find(&client->dev->idr, fallbacks[i]) == NULL) { >> + ret = -EINVAL; >> + goto out3; >> + } >> + } >> + up_read(&client->dev->lock); >> + >> + /* >> + * This is a racy check since the lock is dropped before the heap >> + * is actually added. It's okay though because ids are never actually >> + * deleted. Worst case some user gets an error back and an indication >> + * to fix races in their code. >> + */ >> + >> + heap->fallbacks = fallbacks; >> + heap->fallback_cnt = cnt; >> + heap->type = ION_USAGE_ID_MAP; >> + heap->id = ION_DYNAMIC_HEAP_ASSIGN; > > It would be nice for userspace to be able to give these usage heaps > meaningful names - That could be their intended usage ("camera", > "display") or their properties ("contiguous", "sram"). > >> + ion_device_add_heap(client->dev, heap); >> + return heap->id; >> +out3: >> + up_read(&client->dev->lock); >> +out2: >> + kfree(fallbacks); >> +out1: >> + kfree(heap); >> + return ret; >> +} >> + >> int ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap) >> { >> struct dentry *debug_file; >> int ret; >> >> - if (!heap->ops->allocate || !heap->ops->free || !heap->ops->map_dma || >> - !heap->ops->unmap_dma) { >> + if (heap->type != ION_USAGE_ID_MAP && >> + (!heap->ops->allocate || >> + !heap->ops->free || >> + !heap->ops->map_dma || >> + !heap->ops->unmap_dma)) { >> pr_err("%s: can not add heap with invalid ops struct.\n", >> __func__); >> return -EINVAL; >> @@ -1425,18 +1572,36 @@ int ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap) >> if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) >> ion_heap_init_deferred_free(heap); >> >> - if ((heap->flags & ION_HEAP_FLAG_DEFER_FREE) || heap->ops->shrink) >> + if ((heap->flags & ION_HEAP_FLAG_DEFER_FREE) || (heap->ops && heap->ops->shrink)) >> ion_heap_init_shrinker(heap); >> >> heap->dev = dev; >> down_write(&dev->lock); >> >> - ret = idr_alloc(&dev->idr, heap, heap->id, heap->id + 1, GFP_KERNEL); >> - if (ret < 0 || ret != heap->id) { >> - pr_info("%s: Failed to add heap id, expected %d got %d\n", >> - __func__, heap->id, ret); >> - up_write(&dev->lock); >> - return ret < 0 ? ret : -EINVAL; >> + if (heap->id == ION_DYNAMIC_HEAP_ASSIGN) { >> + ret = idr_alloc(&dev->idr, heap, >> + ION_DYNAMIC_HEAP_ASSIGN + 1, 0, GFP_KERNEL); >> + if (ret < 0) >> + goto out_unlock; >> + >> + heap->id = ret; >> + } else { >> + ret = idr_alloc(&dev->idr, heap, heap->id, heap->id + 1, >> + GFP_KERNEL); >> + if (ret < 0 || ret != heap->id) { >> + pr_info("%s: Failed to add heap id, expected %d got %d\n", >> + __func__, heap->id, ret); >> + ret = ret < 0 ? ret : -EINVAL; >> + goto out_unlock; >> + } >> + } >> + >> + if (!heap->name) { >> + heap->name = kasprintf(GFP_KERNEL, "heap%d", heap->id); >> + if (!heap->name) { >> + ret = -ENOMEM; >> + goto out_unlock; >> + } >> } >> >> debug_file = debugfs_create_file(heap->name, 0664, >> @@ -1467,6 +1632,7 @@ int ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap) >> } >> } >> dev->heap_cnt++; >> +out_unlock: >> up_write(&dev->lock); >> return 0; >> } >> diff --git a/drivers/staging/android/ion/ion_priv.h b/drivers/staging/android/ion/ion_priv.h >> index b09d751..e03e940 100644 >> --- a/drivers/staging/android/ion/ion_priv.h >> +++ b/drivers/staging/android/ion/ion_priv.h >> @@ -255,6 +255,9 @@ struct ion_heap { >> wait_queue_head_t waitqueue; >> struct task_struct *task; >> >> + unsigned int *fallbacks; >> + int fallback_cnt; >> + >> int (*debug_show)(struct ion_heap *heap, struct seq_file *, void *); >> }; >> >> @@ -381,6 +384,10 @@ size_t ion_heap_freelist_shrink(struct ion_heap *heap, >> size_t ion_heap_freelist_size(struct ion_heap *heap); >> >> >> +int ion_map_usage_ids(struct ion_client *client, >> + unsigned int __user *usage_ids, >> + int cnt); >> + >> /** >> * functions for creating and destroying the built in ion heaps. >> * architectures can add their own custom architecture specific >> @@ -495,4 +502,12 @@ struct ion_handle *ion_handle_get_by_id(struct ion_client *client, >> >> int ion_handle_put(struct ion_handle *handle); >> >> +struct ion_handle *ion_alloc2(struct ion_client *client, size_t len, >> + size_t align, unsigned int usage_id, >> + unsigned int flags); >> + >> +int ion_query_heaps(struct ion_client *client, >> + struct ion_heap_data __user *buffer, >> + int cnt); >> + >> #endif /* _ION_PRIV_H */ >> diff --git a/drivers/staging/android/uapi/ion.h b/drivers/staging/android/uapi/ion.h >> index 145005f..d36b4e4 100644 >> --- a/drivers/staging/android/uapi/ion.h >> +++ b/drivers/staging/android/uapi/ion.h >> @@ -45,10 +45,17 @@ enum ion_heap_type { >> * must be last so device specific heaps always >> * are at the end of this enum >> */ >> + ION_USAGE_ID_MAP, > > A short description for this would be nice > >> }; >> >> #define ION_NUM_HEAP_IDS (sizeof(unsigned int) * 8) >> >> +/* >> + * This isn't completely random. The old Ion heap ID namespace goes up to >> + * 32 bits so take the first one after that to use as a dynamic setting >> + */ >> +#define ION_DYNAMIC_HEAP_ASSIGN 33 >> + > > The old comment says 32, but the implementation looks like 16. > > Either way, shouldn't that be 32? If the previous max was 31, > 32 is next available - but in ion_device_add_heap() you use > ION_DYNAMIC_HEAP_ASSIGN + 1, which makes the first dynamically-assigned > ID 34. > >> /** >> * allocation flags - the lower 16 bits are used by core ion, the upper 16 >> * bits are reserved for use by the heaps themselves. >> @@ -65,6 +72,11 @@ enum ion_heap_type { >> * caches must be managed >> * manually >> */ >> +#define ION_FLAG_NO_HANDLE 3 /* >> + * Return only an fd associated with >> + * this buffer and not a handle >> + */ >> + >> >> /** >> * DOC: Ion Userspace API >> @@ -142,6 +154,96 @@ struct ion_abi_version { >> __u32 reserved; >> }; >> >> +/** >> + * struct ion_new_alloc_data - metadata to/from usersapce allocation v2 >> + * @len: size of the allocation >> + * @align: required alignment of the allocation >> + * @usage_id: mapping to heap id to allocate. Will try fallbacks >> + * if specified in the heap mapping >> + * @flags: flags passed to heap >> + * @handle: pointer that will be populated with a cookie to use to >> + * refer to this allocation >> + * @fd: optionally, may return just an fd with no handle >> + * reference >> + */ >> +struct ion_new_alloc_data { >> + __u64 len; >> + __u64 align; >> + __u32 usage_id; >> + __u32 flags; >> + /* >> + * I'd like to add a flag to just return the fd instead of just >> + * a handle for those who want to skip the next step. >> + */ >> + union { >> + __u32 fd; >> + __u32 handle; >> + }; >> + /* >> + * Allocation has a definite problem of 'creep' so give more than >> + * enough extra bits for expansion >> + */ >> + __u32 reserved0; >> + __u32 reserved1; >> + __u32 reserved2; >> +}; >> + >> +/** >> + * struct ion_usage_id_map - metadata to create a new usage map >> + * @usage_id - userspace allocated array of existing usage ids >> + * @cnt - number of ids to be mapped >> + * @new_id - will be populated with the new usage id >> + */ >> +struct ion_usage_id_map { >> + /* Array of usage ids provided by user */ >> + __u64 usage_ids; >> + __u32 cnt; >> + >> + /* Returned on success */ >> + __u32 new_id; >> + /* Fields for future growth */ >> + __u32 reserved0; >> + __u32 reserved1; >> +}; >> + >> +/** >> + * struct ion_usage_cnt - meta data for the count of heaps >> + * @cnt - returned count of number of heaps present >> + */ >> +struct ion_usage_cnt { >> + __u32 cnt; /* cnt returned */ >> + __u32 reserved; >> +}; >> + >> +/** >> + * struct ion_heap_data - data about a heap >> + * @name - first 32 characters of the heap name >> + * @type - heap type >> + * @usage_id - usage id for the heap >> + */ >> +struct ion_heap_data { >> + char name[32]; I realized I upped this to 32 and didn't update the code. I'm going to make this consistent. >> + __u32 type; >> + __u32 usage_id; > > For heaps with type ION_USAGE_ID_MAP I think userspace would want a way > to turn a usage_id back into a list of fallbacks to figure out that > usage_id == 34 means "only contiguous buffers" for instance. > Otherwise only the process that made the mapping can ever know what its > for. > >> + __u32 reserved0; >> + __u32 reserved1; >> + __u32 reserved2; >> +}; >> + >> +/** >> + * struct ion_heap_query - collection of data about all heaps >> + * @cnt - total number of heaps to be copied >> + * @heaps - buffer to copy heap data >> + */ >> +struct ion_heap_query { >> + __u32 cnt; /* Total number of heaps to be copied */ >> + __u64 heaps; /* buffer to be populated */ > > I guess this field needs explicit alignment to 64 bits > Yes, I was going by the suggestion in botching-up-ioctls.txt https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Docume… >> + __u32 reserved0; >> + __u32 reserved1; >> + __u32 reserved2; >> +}; >> + >> + >> #define ION_IOC_MAGIC 'I' >> >> /** >> @@ -216,5 +318,38 @@ struct ion_abi_version { >> #define ION_IOC_ABI_VERSION _IOR(ION_IOC_MAGIC, 8, \ >> struct ion_abi_version) >> >> +/** >> + * DOC: ION_IOC_ALLOC2 - allocate memory via new API >> + * >> + * Same concept as ION_IOC_ALLOC except with updated ABI. Can return an fd >> + * directly in addition to a handle >> + */ >> +#define ION_IOC_ALLOC2 _IOWR(ION_IOC_MAGIC, 9, \ >> + struct ion_new_alloc_data) >> + >> +/** >> + * DOC: ION_IOC_ID_MAP - remap an array of heap IDs >> + * >> + * Takes an ion_usage_id_map structure populated with information about >> + * fallback information. Returns a new usage id for use in allocation. > > Can you mention the implied priority of the fallbacks here? > >> + */ >> +#define ION_IOC_ID_MAP _IOWR(ION_IOC_MAGIC, 10, \ >> + struct ion_usage_id_map) >> +/** >> + * DOC: ION_IOC_USAGE_CNT - returns a count of the number of usage ids >> + * >> + * Takes an ion_usage_cnt structure and returns the total number of usage >> + * ids available. > > Number of usage IDs or number of heaps? > >> + */ >> +#define ION_IOC_USAGE_CNT _IOR(ION_IOC_MAGIC, 11, \ >> + struct ion_usage_cnt) >> +/** >> + * DOC: ION_IOC_HEAP_QUERY - information about available heaps >> + * >> + * Takes an ion_heap_query structure and populates information about >> + * availble Ion heaps. >> + */ >> +#define ION_IOC_HEAP_QUERY _IOWR(ION_IOC_MAGIC, 12, \ >> + struct ion_heap_query) >> >> #endif /* _UAPI_LINUX_ION_H */ >> -- >> 2.5.5 >>

9 years, 8 months

2
1
0 0

Re: [Linaro-mm-sig] [RFC][PATCH 0/6] ion: improved ABI

by Laura Abbott

On 06/08/2016 08:15 AM, Brian Starkey wrote: > Hi Laura, > > On Mon, Jun 06, 2016 at 11:23:27AM -0700, Laura Abbott wrote: >> The ABI for Ion's ioctl interface are a pain to work with. The heap IDs >> are a 32-bit non-discoverable namespace that form part of the ABI. There's >> no way to determine what ABI version is in use which leads to problems >> if the ABI changes or needs to be updated. >> >> This series is a first approach to give a better ABI for Ion. This includes: >> >> - Following the advice in botching-up-ioctls.txt >> - Ioctl for ABI version >> - Dynamic assignment of heap ids >> - queryable heap ids >> - Runtime mapping of heap ids, including fallbacks. This avoids the need to >> encode the fallbacks as an ABI. >> >> I'm most interested in feedback if this ABI is actually an improvement and >> usable. The heap id map/query interface seems error prone but I didn't have >> a cleaner solution. There aren't any kernel APIs for the new features as the >> focus was on a userspace API but I anticipate that following easily once >> the userspace API is established. >> > > If I understand it right, the big improvement here is that userspace can > find out what heap IDs are available, and what type of heap they > correspond to? This seems good. > > I'm not sure about how userspace is meant to use the usage mappings > though. Who calls ION_IOC_ID_MAP? > The thought was daemons (surfaceflinger, mediaserver etc.) would set this up. > (also, map/mapping is pretty overloaded. How about groups/groupings?) > That's a good suggestion. > If I assume that the thing calling ION_IOC_ID_MAP is the same thing > doing the allocating, then there doesn't seem to be much need for > creating mappings. The combined mapper/allocator would necessarily need > some knowledge about which types can satisfy which usage, and so could > follow something like this: > 1. The heaps can be queried, finding their IDs and types > 2. A mask of heap IDs can be created which satisfies a "usage", based > on the queried types > 3. Allocation operations can then simply use this constructed mask > > On the other hand, if I assume that the thing doing the allocating is > different to the thing doing the usage mapping (i.e. the allocator > doesn't need to know about heap _types_, only usages); then I can't see > a way for the allocator to figure out which usage_id it's meant to > allocate with - which puts it right back to the old problem of opaque > heap IDs (-> opaque usage_ids), except it's worse because they can > change dynamically. > I see your point about the mapping. My thought was that whoever was doing the mapping would have a way to pass information to the allocator or the allocator could do a query. Relying on the passing of information may not be plausible and I realize there isn't enough information from a query to determine what usage id to use. I like the suggestion of just using the query. One of the goals I initially had with this series was to get rid of the heap mask. The 32-bit heap mask became a name space that needed to be controlled across all targets and the fallbacks were difficult to change. The problem that actually wants to be solved is giving userspace enough information to determine what heap masks to allocate from. Just a query ioctl should be able to give that information as long as the requirement is that userspace clients match on the heap name + type only and not rely on the heap ids being constant (I don't think I made that clear with this version of the query ioctl so I'll make sure to clarify). The platform registration can adjust the priority order of the heap ids as necessary. Different names should be able to take care of any changes to the platform configuration. I'll think about this more when I work on v2. Thanks, Laura > Thanks, > Brian > >> >> Thanks, >> Laura >> >> P.S. Not to turn this into a bike shedding session but if you have suggestions >> for a name for this framework other than Ion I would be interested to hear >> them. Too many other things are already named Ion. >> >> Laura Abbott (6): >> staging: android: ion: return error value for ion_device_add_heap >> staging: android: ion: Switch to using an idr to manage heaps >> staging: android: ion: Drop heap type masks >> staging: android: ion: Pull out ion ioctls to a separate file >> staging: android: ion: Add an ioctl for ABI checking >> staging: android: ion: Introduce new ioctls for dynamic heaps >> >> drivers/staging/android/ion/Makefile | 3 +- >> drivers/staging/android/ion/ion-ioctl.c | 243 ++++++++++++++++++ >> drivers/staging/android/ion/ion.c | 438 ++++++++++++++++---------------- >> drivers/staging/android/ion/ion_priv.h | 109 +++++++- >> drivers/staging/android/uapi/ion.h | 164 +++++++++++- >> 5 files changed, 728 insertions(+), 229 deletions(-) >> create mode 100644 drivers/staging/android/ion/ion-ioctl.c >> >> -- 2.5.5

9 years, 8 months

1
0
0 0

[RFC][PATCH 0/6] ion: improved ABI

by Laura Abbott

The ABI for Ion's ioctl interface are a pain to work with. The heap IDs are a 32-bit non-discoverable namespace that form part of the ABI. There's no way to determine what ABI version is in use which leads to problems if the ABI changes or needs to be updated. This series is a first approach to give a better ABI for Ion. This includes: - Following the advice in botching-up-ioctls.txt - Ioctl for ABI version - Dynamic assignment of heap ids - queryable heap ids - Runtime mapping of heap ids, including fallbacks. This avoids the need to encode the fallbacks as an ABI. I'm most interested in feedback if this ABI is actually an improvement and usable. The heap id map/query interface seems error prone but I didn't have a cleaner solution. There aren't any kernel APIs for the new features as the focus was on a userspace API but I anticipate that following easily once the userspace API is established. Thanks, Laura P.S. Not to turn this into a bike shedding session but if you have suggestions for a name for this framework other than Ion I would be interested to hear them. Too many other things are already named Ion. Laura Abbott (6): staging: android: ion: return error value for ion_device_add_heap staging: android: ion: Switch to using an idr to manage heaps staging: android: ion: Drop heap type masks staging: android: ion: Pull out ion ioctls to a separate file staging: android: ion: Add an ioctl for ABI checking staging: android: ion: Introduce new ioctls for dynamic heaps drivers/staging/android/ion/Makefile | 3 +- drivers/staging/android/ion/ion-ioctl.c | 243 ++++++++++++++++++ drivers/staging/android/ion/ion.c | 438 ++++++++++++++++---------------- drivers/staging/android/ion/ion_priv.h | 109 +++++++- drivers/staging/android/uapi/ion.h | 164 +++++++++++- 5 files changed, 728 insertions(+), 229 deletions(-) create mode 100644 drivers/staging/android/ion/ion-ioctl.c -- 2.5.5

9 years, 8 months

2
11
0 0

Re: [Linaro-mm-sig] [RFC][PATCH 0/6] ion: improved ABI

by Laura Abbott

On 06/06/2016 11:59 PM, Chen Feng wrote: > The idea is good, define the heap ids in header file is inconvenient. > > But if we query the heaps information from user-space. > > It need to maintain this ids and name userspace one by one. The code may > be complicated in different module user-space. > > In android, the gralloc and other lib will all use ion to alloc memory. > > This will make it more difficult to maintain user-space code. > This was a concern I had had as well. Anything that involves dynamic probing is going to involve more tracking. Do you think some library code in libion would help? Thanks, Laura > > But beyond this, The new alloc2 with not-handle flag is good. > And the pull out of ioctl interface is also a good cleanup. > > On 2016/6/7 2:23, Laura Abbott wrote: >> >> The ABI for Ion's ioctl interface are a pain to work with. The heap IDs >> are a 32-bit non-discoverable namespace that form part of the ABI. There's >> no way to determine what ABI version is in use which leads to problems >> if the ABI changes or needs to be updated. >> >> This series is a first approach to give a better ABI for Ion. This includes: >> >> - Following the advice in botching-up-ioctls.txt >> - Ioctl for ABI version >> - Dynamic assignment of heap ids >> - queryable heap ids >> - Runtime mapping of heap ids, including fallbacks. This avoids the need to >> encode the fallbacks as an ABI. >> >> I'm most interested in feedback if this ABI is actually an improvement and >> usable. The heap id map/query interface seems error prone but I didn't have >> a cleaner solution. There aren't any kernel APIs for the new features as the >> focus was on a userspace API but I anticipate that following easily once >> the userspace API is established. >> >> >> Thanks, >> Laura >> >> P.S. Not to turn this into a bike shedding session but if you have suggestions >> for a name for this framework other than Ion I would be interested to hear >> them. Too many other things are already named Ion. >> >> Laura Abbott (6): >> staging: android: ion: return error value for ion_device_add_heap >> staging: android: ion: Switch to using an idr to manage heaps >> staging: android: ion: Drop heap type masks >> staging: android: ion: Pull out ion ioctls to a separate file >> staging: android: ion: Add an ioctl for ABI checking >> staging: android: ion: Introduce new ioctls for dynamic heaps >> >> drivers/staging/android/ion/Makefile | 3 +- >> drivers/staging/android/ion/ion-ioctl.c | 243 ++++++++++++++++++ >> drivers/staging/android/ion/ion.c | 438 ++++++++++++++++---------------- >> drivers/staging/android/ion/ion_priv.h | 109 +++++++- >> drivers/staging/android/uapi/ion.h | 164 +++++++++++- >> 5 files changed, 728 insertions(+), 229 deletions(-) >> create mode 100644 drivers/staging/android/ion/ion-ioctl.c >> >

9 years, 8 months

1
0
0 0

[RFC][PATCH 0/3] Clean up Ion mapping/caching

by Laura Abbott

Hi, This series cleans up Ion a bit to be more in line with existing standards for caching and dma mapping. The most controversial part of this is probably going to be the first patch. Ion takes quite a few liberties with how the DMA APIs are used for cache syncing. dma_sync_sg is used without calling dma_map first. There isn't a good way to get the cache synchronization with the DMA APIs. The behavior of Ion is closer to the DRM framework which uses its own private cache APIs for synchronization so I took that approach. Assuming the approach of the first patch is appropriate, the next two patches are fairly simple. dma_buf added support for a sync ioctl. Ion has a similar ioctl already so this fixes the Ion APIs to be compatible with the dma_buf ioctl. My plan would be to put a timeline on deprecation for the old Ion sync ioctl. The map_dma_buf calls were also missing calls to the underlying DMA APIs so the final patch in the series adds the appropriate calls. Feedback is appreciated. Thanks, Laura Laura Abbott (3): staging: ion: Move away from the DMA APIs for cache flushing staging: ion: Add support for syncing with DMA_BUF_IOCTL_SYNC staging: ion: Add dma_map/dma_unmap calls to dma_buf calls drivers/staging/android/ion/Kconfig | 14 ++++- drivers/staging/android/ion/Makefile | 3 + drivers/staging/android/ion/ion-arm.c | 83 ++++++++++++++++++++++++ drivers/staging/android/ion/ion-arm64.c | 46 ++++++++++++++ drivers/staging/android/ion/ion-x86.c | 34 ++++++++++ drivers/staging/android/ion/ion.c | 84 +++++++++++++------------ drivers/staging/android/ion/ion_carveout_heap.c | 5 +- drivers/staging/android/ion/ion_chunk_heap.c | 7 +-- drivers/staging/android/ion/ion_page_pool.c | 3 +- drivers/staging/android/ion/ion_priv.h | 14 ++--- drivers/staging/android/ion/ion_system_heap.c | 5 +- 11 files changed, 235 insertions(+), 63 deletions(-) create mode 100644 drivers/staging/android/ion/ion-arm.c create mode 100644 drivers/staging/android/ion/ion-arm64.c create mode 100644 drivers/staging/android/ion/ion-x86.c -- 2.5.5

9 years, 8 months

5
7
0 0

[GIT PULL]: minor dma-buf updates for 4.7

by Sumit Semwal

Hi Linus, A tiny pull request with mostly cosmetic changes in dma-buf for 4.7; may I request you to please pull? The following changes since commit 852f42a69b93dc71507adedeed876d57b8c2c2fa: Merge branch 'uuid' (lib/uuid fixes from Andy) (2016-05-30 15:27:07 -0700) are available in the git repository at: git://git.kernel.org/pub/scm/linux/kernel/git/sumits/dma-buf.git tags/dma-buf-for-4.7 for you to fetch changes up to b02da6f8236148009c22167cd7013d5ce04a2d37: dma-buf: use vma_pages() (2016-05-31 22:17:05 +0530) ---------------------------------------------------------------- Minor dma-buf updates for 4.7: - use of vma_pages instead of explicit computation. - DocBook and headerdoc updates for dma-buf. ---------------------------------------------------------------- Luis de Bethencourt (1): fence: add missing descriptions for fence Muhammad Falak R Wani (1): dma-buf: use vma_pages() Rob Clark (3): dma-buf: headerdoc fixes reservation: add headerdoc comments doc: update/fixup dma-buf related DocBook Documentation/DocBook/device-drivers.tmpl | 36 ++++++++++++++-- drivers/dma-buf/dma-buf.c | 7 +-- drivers/dma-buf/reservation.c | 72 +++++++++++++++++++++++++++++-- include/linux/dma-buf.h | 13 ++++-- include/linux/fence.h | 2 + include/linux/reservation.h | 53 +++++++++++++++++++++++ 6 files changed, 169 insertions(+), 14 deletions(-) Thanks and best regards, Sumit.

9 years, 8 months

1
0
0 0

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

Linaro-mm-sig