Linaro-mm-sig

linaro-mm-sig@lists.linaro.org

15 participants
3025 discussions

Re: [PATCH 4/5] udmabuf: add get_sg_table helper function

by Christian König

Am 01.08.24 um 12:45 schrieb Huan Yang: > Currently, there are three duplicate pieces of code that retrieve > sg_table and update uduf->sg. > > Since the sgt is used to populate the page in both mmap and vmap.It is > necessary to ensure that ubuf->sg is set correctly. That is a really bad idea. Why are sg tables used to populated the page tables? Regards, Christian. > > This patch add a helper function, if ubuf->sg exist, just return it. > Or else, try alloc a new sgt, and cmpxchg to set it. > > When the swap fails, it means that another process has set sg correctly. > Therefore, we reuse the new sg. If trigger by device, need invoke map to > sync it. > > Signed-off-by: Huan Yang <link(a)vivo.com> > --- > drivers/dma-buf/udmabuf.c | 60 ++++++++++++++++++++++++++++----------- > 1 file changed, 43 insertions(+), 17 deletions(-) > > diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c > index 7ed532342d7f..677ebb2d462f 100644 > --- a/drivers/dma-buf/udmabuf.c > +++ b/drivers/dma-buf/udmabuf.c > @@ -38,8 +38,9 @@ struct udmabuf_folio { > struct list_head list; > }; > > -static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, > - enum dma_data_direction direction); > +static struct sg_table *udmabuf_get_sg_table(struct device *dev, > + struct dma_buf *buf, > + enum dma_data_direction direction); > > static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma) > { > @@ -52,12 +53,9 @@ static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma) > if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) > return -EINVAL; > > - if (!table) { > - table = get_sg_table(NULL, buf, 0); > - if (IS_ERR(table)) > - return PTR_ERR(table); > - ubuf->sg = table; > - } > + table = udmabuf_get_sg_table(NULL, buf, 0); > + if (IS_ERR(table)) > + return PTR_ERR(table); > > for_each_sgtable_page(table, &piter, vma->vm_pgoff) { > struct page *page = sg_page_iter_page(&piter); > @@ -84,12 +82,9 @@ static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) > > dma_resv_assert_held(buf->resv); > > - if (!sg) { > - sg = get_sg_table(NULL, buf, 0); > - if (IS_ERR(sg)) > - return PTR_ERR(sg); > - ubuf->sg = sg; > - } > + sg = udmabuf_get_sg_table(NULL, buf, 0); > + if (IS_ERR(sg)) > + return PTR_ERR(sg); > > pages = kvmalloc_array(ubuf->pagecount, sizeof(*pages), GFP_KERNEL); > if (!pages) > @@ -154,6 +149,39 @@ static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, > return ERR_PTR(ret); > } > > +static struct sg_table *udmabuf_get_sg_table(struct device *dev, > + struct dma_buf *buf, > + enum dma_data_direction direction) > +{ > + struct udmabuf *ubuf = buf->priv; > + struct sg_table *sg = READ_ONCE(ubuf->sg); > + int ret = 0; > + > + if (sg) > + return sg; > + > + sg = get_sg_table(dev, buf, direction); > + if (IS_ERR(sg)) > + return sg; > + > + // Success update ubuf's sg, just return. > + if (!cmpxchg(&ubuf->sg, NULL, sg)) > + return sg; > + > + // use the new sg table. > + sg_free_table(sg); > + kfree(sg); > + sg = READ_ONCE(ubuf->sg); > + > + if (dev) > + ret = dma_map_sgtable(dev, sg, direction, 0); > + > + if (ret < 0) > + return ERR_PTR(ret); > + > + return sg; > +} > + > static void put_sg_table(struct device *dev, struct sg_table *sg, > enum dma_data_direction direction) > { > @@ -230,12 +258,10 @@ static int begin_cpu_udmabuf(struct dma_buf *buf, > return 0; > } > > - sg = get_sg_table(dev, buf, direction); > + sg = udmabuf_get_sg_table(dev, buf, direction); > if (IS_ERR(sg)) > return PTR_ERR(sg); > > - ubuf->sg = sg; > - > return 0; > } >

1 year, 1 month

Re: [PATCH 3/5] udmabuf: fix vmap_udmabuf error page set

by Christian König

Am 01.08.24 um 12:45 schrieb Huan Yang: > Currently vmap_udmabuf set page's array by each folio. > But, ubuf->folios is only contain's the folio's head page. > > That mean we repeatedly mapped the folio head page to the vmalloc area. > > This patch fix it, set each folio's page correct, so that pages array > contains right page, and then map into vmalloc area > > Signed-off-by: Huan Yang <link(a)vivo.com> > --- > drivers/dma-buf/udmabuf.c | 17 +++++++++++++---- > 1 file changed, 13 insertions(+), 4 deletions(-) > > diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c > index a915714c5dce..7ed532342d7f 100644 > --- a/drivers/dma-buf/udmabuf.c > +++ b/drivers/dma-buf/udmabuf.c > @@ -77,18 +77,27 @@ static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma) > static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) > { > struct udmabuf *ubuf = buf->priv; > - struct page **pages; > + struct page **pages, **tmp; > + struct sg_table *sg = ubuf->sg; > + struct sg_page_iter piter; > void *vaddr; > - pgoff_t pg; > > dma_resv_assert_held(buf->resv); > > + if (!sg) { > + sg = get_sg_table(NULL, buf, 0); > + if (IS_ERR(sg)) > + return PTR_ERR(sg); > + ubuf->sg = sg; > + } > + > pages = kvmalloc_array(ubuf->pagecount, sizeof(*pages), GFP_KERNEL); > if (!pages) > return -ENOMEM; > + tmp = pages; > > - for (pg = 0; pg < ubuf->pagecount; pg++) > - pages[pg] = &ubuf->folios[pg]->page; > + for_each_sgtable_page(sg, &piter, 0) > + *tmp++ = sg_page_iter_page(&piter); Again don't abuse the sg table for that! Regards, Christian. > > vaddr = vm_map_ram(pages, ubuf->pagecount, -1); > kvfree(pages);

1 year, 1 month

Re: [PATCH 2/5] udmabuf: change folios array from kmalloc to kvmalloc

by Christian König

Am 01.08.24 um 12:45 schrieb Huan Yang: > When PAGE_SIZE 4096, MAX_PAGE_ORDER 10, 64bit machine, > page_alloc only support 4MB. > If above this, trigger this warn and return NULL. > > udmabuf can change size limit, if change it to 3072(3GB), and then alloc > 3GB udmabuf, will fail create. > > [ 4080.876581] ------------[ cut here ]------------ > [ 4080.876843] WARNING: CPU: 3 PID: 2015 at mm/page_alloc.c:4556 __alloc_pages+0x2c8/0x350 > [ 4080.878839] RIP: 0010:__alloc_pages+0x2c8/0x350 > [ 4080.879470] Call Trace: > [ 4080.879473] <TASK> > [ 4080.879473] ? __alloc_pages+0x2c8/0x350 > [ 4080.879475] ? __warn.cold+0x8e/0xe8 > [ 4080.880647] ? __alloc_pages+0x2c8/0x350 > [ 4080.880909] ? report_bug+0xff/0x140 > [ 4080.881175] ? handle_bug+0x3c/0x80 > [ 4080.881556] ? exc_invalid_op+0x17/0x70 > [ 4080.881559] ? asm_exc_invalid_op+0x1a/0x20 > [ 4080.882077] ? udmabuf_create+0x131/0x400 > > Because MAX_PAGE_ORDER, kmalloc can max alloc 4096 * (1 << 10), 4MB > memory, each array entry is pointer(8byte), so can save 524288 pages(2GB). > > Further more, costly order(order 3) may not be guaranteed that it can be > applied for, due to fragmentation. > > This patch change udmabuf array use kvmalloc_array, this can fallback > alloc into vmalloc, which can guarantee allocation for any size and does > not affect the performance of kmalloc allocations. > > Signed-off-by: Huan Yang <link(a)vivo.com> Acked-by: Christian König <christian.koenig(a)amd.com> > --- > drivers/dma-buf/udmabuf.c | 26 +++++++++++++------------- > 1 file changed, 13 insertions(+), 13 deletions(-) > > diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c > index d69aeada7367..a915714c5dce 100644 > --- a/drivers/dma-buf/udmabuf.c > +++ b/drivers/dma-buf/udmabuf.c > @@ -83,7 +83,7 @@ static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) > > dma_resv_assert_held(buf->resv); > > - pages = kmalloc_array(ubuf->pagecount, sizeof(*pages), GFP_KERNEL); > + pages = kvmalloc_array(ubuf->pagecount, sizeof(*pages), GFP_KERNEL); > if (!pages) > return -ENOMEM; > > @@ -91,7 +91,7 @@ static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) > pages[pg] = &ubuf->folios[pg]->page; > > vaddr = vm_map_ram(pages, ubuf->pagecount, -1); > - kfree(pages); > + kvfree(pages); > if (!vaddr) > return -EINVAL; > > @@ -203,8 +203,8 @@ static void release_udmabuf(struct dma_buf *buf) > put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL); > > unpin_all_folios(&ubuf->unpin_list); > - kfree(ubuf->offsets); > - kfree(ubuf->folios); > + kvfree(ubuf->offsets); > + kvfree(ubuf->folios); > kfree(ubuf); > } > > @@ -330,14 +330,14 @@ static long udmabuf_create(struct miscdevice *device, > if (!ubuf->pagecount) > goto err; > > - ubuf->folios = kmalloc_array(ubuf->pagecount, sizeof(*ubuf->folios), > - GFP_KERNEL); > + ubuf->folios = kvmalloc_array(ubuf->pagecount, sizeof(*ubuf->folios), > + GFP_KERNEL); > if (!ubuf->folios) { > ret = -ENOMEM; > goto err; > } > - ubuf->offsets = kcalloc(ubuf->pagecount, sizeof(*ubuf->offsets), > - GFP_KERNEL); > + ubuf->offsets = > + kvcalloc(ubuf->pagecount, sizeof(*ubuf->offsets), GFP_KERNEL); > if (!ubuf->offsets) { > ret = -ENOMEM; > goto err; > @@ -351,7 +351,7 @@ static long udmabuf_create(struct miscdevice *device, > goto err; > > pgcnt = list[i].size >> PAGE_SHIFT; > - folios = kmalloc_array(pgcnt, sizeof(*folios), GFP_KERNEL); > + folios = kvmalloc_array(pgcnt, sizeof(*folios), GFP_KERNEL); > if (!folios) { > ret = -ENOMEM; > goto err; > @@ -361,7 +361,7 @@ static long udmabuf_create(struct miscdevice *device, > ret = memfd_pin_folios(memfd, list[i].offset, end, > folios, pgcnt, &pgoff); > if (ret <= 0) { > - kfree(folios); > + kvfree(folios); > if (!ret) > ret = -EINVAL; > goto err; > @@ -390,7 +390,7 @@ static long udmabuf_create(struct miscdevice *device, > } > } > > - kfree(folios); > + kvfree(folios); > fput(memfd); > memfd = NULL; > } > @@ -406,8 +406,8 @@ static long udmabuf_create(struct miscdevice *device, > if (memfd) > fput(memfd); > unpin_all_folios(&ubuf->unpin_list); > - kfree(ubuf->offsets); > - kfree(ubuf->folios); > + kvfree(ubuf->offsets); > + kvfree(ubuf->folios); > kfree(ubuf); > return ret; > }

1 year, 1 month

Re: [PATCH 1/5] udmabuf: cancel mmap page fault, direct map it

by Christian König

Am 01.08.24 um 12:45 schrieb Huan Yang: > The current udmabuf mmap uses a page fault mechanism to populate the vma. > > However, the current udmabuf has already obtained and pinned the folio > upon completion of the creation.This means that the physical memory has > already been acquired, rather than being accessed dynamically. The > current page fault method only saves some page table memory. > > As a result, the page fault mechanism has lost its purpose as a demanding > page. Due to the fact that page fault requires trapping into kernel mode > and filling in when accessing the corresponding virtual address in mmap, > this means that user mode access to virtual addresses needs to trap into > kernel mode. > > Therefore, when creating a large size udmabuf, this represents a > considerable overhead. > > Therefore, the current patch removes the page fault method of mmap and > instead fills it directly when mmap is triggered. > > Signed-off-by: Huan Yang <link(a)vivo.com> > --- > drivers/dma-buf/udmabuf.c | 70 ++++++++++++++++++++++----------------- > 1 file changed, 39 insertions(+), 31 deletions(-) > > diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c > index 047c3cd2ceff..d69aeada7367 100644 > --- a/drivers/dma-buf/udmabuf.c > +++ b/drivers/dma-buf/udmabuf.c > @@ -38,36 +38,39 @@ struct udmabuf_folio { > struct list_head list; > }; > > -static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) > -{ > - struct vm_area_struct *vma = vmf->vma; > - struct udmabuf *ubuf = vma->vm_private_data; > - pgoff_t pgoff = vmf->pgoff; > - unsigned long pfn; > - > - if (pgoff >= ubuf->pagecount) > - return VM_FAULT_SIGBUS; > - > - pfn = folio_pfn(ubuf->folios[pgoff]); > - pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT; > - > - return vmf_insert_pfn(vma, vmf->address, pfn); > -} > - > -static const struct vm_operations_struct udmabuf_vm_ops = { > - .fault = udmabuf_vm_fault, > -}; > +static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, > + enum dma_data_direction direction); > > static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma) > { > struct udmabuf *ubuf = buf->priv; > + struct sg_table *table = ubuf->sg; > + unsigned long addr = vma->vm_start; > + struct sg_page_iter piter; > + int ret; > > if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) > return -EINVAL; > > - vma->vm_ops = &udmabuf_vm_ops; > - vma->vm_private_data = ubuf; > - vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); > + if (!table) { > + table = get_sg_table(NULL, buf, 0); > + if (IS_ERR(table)) > + return PTR_ERR(table); > + ubuf->sg = table; > + } > + > + for_each_sgtable_page(table, &piter, vma->vm_pgoff) { That might not work correctly. We intentionally remove the pages from the sgtable when it is shared between devices. Additional to that the sgtable is *not* a page container, but rather a DMA address container. So that here is also a rather bad idea from the design side. Regards, Christian. > + struct page *page = sg_page_iter_page(&piter); > + > + ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE, > + vma->vm_page_prot); > + if (ret) > + return ret; > + addr += PAGE_SIZE; > + if (addr >= vma->vm_end) > + return 0; > + } > + > return 0; > } > > @@ -126,6 +129,10 @@ static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, > sg_set_folio(sgl, ubuf->folios[i], PAGE_SIZE, > ubuf->offsets[i]); > > + // if dev is NULL, no need to sync. > + if (!dev) > + return sg; > + > ret = dma_map_sgtable(dev, sg, direction, 0); > if (ret < 0) > goto err_map; > @@ -206,20 +213,21 @@ static int begin_cpu_udmabuf(struct dma_buf *buf, > { > struct udmabuf *ubuf = buf->priv; > struct device *dev = ubuf->device->this_device; > - int ret = 0; > + struct sg_table *sg; > > - if (!ubuf->sg) { > - ubuf->sg = get_sg_table(dev, buf, direction); > - if (IS_ERR(ubuf->sg)) { > - ret = PTR_ERR(ubuf->sg); > - ubuf->sg = NULL; > - } > - } else { > + if (ubuf->sg) { > dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents, > direction); > + return 0; > } > > - return ret; > + sg = get_sg_table(dev, buf, direction); > + if (IS_ERR(sg)) > + return PTR_ERR(sg); > + > + ubuf->sg = sg; > + > + return 0; > } > > static int end_cpu_udmabuf(struct dma_buf *buf,

1 year, 1 month

Re: [PATCH] udmabuf: use kmem_cache to alloc udmabuf folio

by Daniel Vetter

On Wed, Jul 31, 2024 at 11:34:49AM +0800, Huan Yang wrote: > The current udmabuf_folio contains a list_head and the corresponding > folio pointer, with a size of 24 bytes. udmabuf_folio uses kmalloc to > allocate memory. > > However, kmalloc is a public pool, starting from 64 bytes. This means > that each udmabuf_folio allocation will waste 40 bytes. > > Considering that each udmabuf creates a folio corresponding to a > udmabuf_folio, the wasted memory can be significant in the case of > memory fragmentation. > > Furthermore, if udmabuf is frequently used, the allocation and > deallocation of udmabuf_folio will also be frequent. > > Therefore, this patch adds a kmem_cache dedicated to the allocation and > deallocation of udmabuf_folio.This is expected to improve the > performance of allocation and deallocation within the expected range, > while also avoiding memory waste. > > Signed-off-by: Huan Yang <link(a)vivo.com> > --- > drivers/dma-buf/udmabuf.c | 18 +++++++++++++++--- > 1 file changed, 15 insertions(+), 3 deletions(-) > > diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c > index 047c3cd2ceff..db4de8c745ce 100644 > --- a/drivers/dma-buf/udmabuf.c > +++ b/drivers/dma-buf/udmabuf.c > @@ -24,6 +24,8 @@ static int size_limit_mb = 64; > module_param(size_limit_mb, int, 0644); > MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64."); > > +static struct kmem_cache *udmabuf_folio_cachep; > + > struct udmabuf { > pgoff_t pagecount; > struct folio **folios; > @@ -169,7 +171,7 @@ static void unpin_all_folios(struct list_head *unpin_list) > unpin_folio(ubuf_folio->folio); > > list_del(&ubuf_folio->list); > - kfree(ubuf_folio); > + kmem_cache_free(udmabuf_folio_cachep, ubuf_folio); > } > } > > @@ -178,7 +180,7 @@ static int add_to_unpin_list(struct list_head *unpin_list, > { > struct udmabuf_folio *ubuf_folio; > > - ubuf_folio = kzalloc(sizeof(*ubuf_folio), GFP_KERNEL); > + ubuf_folio = kmem_cache_alloc(udmabuf_folio_cachep, GFP_KERNEL); > if (!ubuf_folio) > return -ENOMEM; > > @@ -492,10 +494,20 @@ static int __init udmabuf_dev_init(void) > if (ret < 0) { > pr_err("Could not setup DMA mask for udmabuf device\n"); > misc_deregister(&udmabuf_misc); misc_deregister() is now called twice in this error path, I think you've forgotten to delete this line too? Otherwise lgtm. -Sima > - return ret; > + goto err; > + } > + > + udmabuf_folio_cachep = KMEM_CACHE(udmabuf_folio, 0); > + if (unlikely(!udmabuf_folio_cachep)) { > + ret = -ENOMEM; > + goto err; > } > > return 0; > + > +err: > + misc_deregister(&udmabuf_misc); > + return ret; > } > > static void __exit udmabuf_dev_exit(void) > > base-commit: cd19ac2f903276b820f5d0d89de0c896c27036ed > -- > 2.45.2 > -- Daniel Vetter Software Engineer, Intel Corporation http://blog.ffwll.ch

1 year, 1 month

Re: [PATCH v2 0/5] Introduce DMA_HEAP_ALLOC_AND_READ_FILE heap flag

by Daniel Vetter

On Tue, Jul 30, 2024 at 08:04:04PM +0800, Huan Yang wrote: > > 在 2024/7/30 17:05, Huan Yang 写道: > > > > 在 2024/7/30 16:56, Daniel Vetter 写道: > > > [????????? daniel.vetter(a)ffwll.ch ????????? > > > https://aka.ms/LearnAboutSenderIdentification?????????????] > > > > > > On Tue, Jul 30, 2024 at 03:57:44PM +0800, Huan Yang wrote: > > > > UDMA-BUF step: > > > > 1. memfd_create > > > > 2. open file(buffer/direct) > > > > 3. udmabuf create > > > > 4. mmap memfd > > > > 5. read file into memfd vaddr > > > Yeah this is really slow and the worst way to do it. You absolutely want > > > to start _all_ the io before you start creating the dma-buf, ideally > > > with > > > everything running in parallel. But just starting the direct I/O with > > > async and then creating the umdabuf should be a lot faster and avoid > > That's greate, Let me rephrase that, and please correct me if I'm wrong. > > > > UDMA-BUF step: > > 1. memfd_create > > 2. mmap memfd > > 3. open file(buffer/direct) > > 4. start thread to async read > > 3. udmabuf create > > > > With this, can improve > > I just test with it. Step is: > > UDMA-BUF step: > 1. memfd_create > 2. mmap memfd > 3. open file(buffer/direct) > 4. start thread to async read > 5. udmabuf create > > 6 . join wait > > 3G file read all step cost 1,527,103,431ns, it's greate. Ok that's almost the throughput of your patch set, which I think is close enough. The remaining difference is probably just the mmap overhead, not sure whether/how we can do direct i/o to an fd directly ... in principle it's possible for any file that uses the standard pagecache. -Sima -- Daniel Vetter Software Engineer, Intel Corporation http://blog.ffwll.ch

1 year, 1 month

Re: [PATCH v3] udmabuf: use kmem_cache to alloc udmabuf folio

by Christophe JAILLET

Le 31/07/2024 à 09:37, Huan Yang a écrit : > The current udmabuf_folio contains a list_head and the corresponding > folio pointer, with a size of 24 bytes. udmabuf_folio uses kmalloc to > allocate memory. > > However, kmalloc is a public pool, starting from 8,16,32 bytes. > Additionally, if the size is not aligned with the kmalloc size, it will > be rounded up to the corresponding size. > This means that each udmabuf_folio allocation will get 32 bytes, and > waste 8 bytes. > > Considering that each udmabuf creates a folio corresponding to a > udmabuf_folio, the wasted memory can be significant in the case of > memory fragmentation. > > Furthermore, if udmabuf is frequently used, the allocation and > deallocation of udmabuf_folio will also be frequent. > > Therefore, this patch adds a kmem_cache dedicated to the allocation and > deallocation of udmabuf_folio.This is expected to improve the > performance of allocation and deallocation within the expected range, > while also avoiding memory waste. > > Signed-off-by: Huan Yang <link(a)vivo.com> > --- > v3 -> v2: fix error description. > v2 -> v1: fix double unregister, remove unlikely. > drivers/dma-buf/udmabuf.c | 19 +++++++++++++++---- > 1 file changed, 15 insertions(+), 4 deletions(-) > > diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c > index 047c3cd2ceff..c112c58ef09a 100644 > --- a/drivers/dma-buf/udmabuf.c > +++ b/drivers/dma-buf/udmabuf.c > @@ -24,6 +24,8 @@ static int size_limit_mb = 64; > module_param(size_limit_mb, int, 0644); > MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64."); > > +static struct kmem_cache *udmabuf_folio_cachep; > + > struct udmabuf { > pgoff_t pagecount; > struct folio **folios; > @@ -169,7 +171,7 @@ static void unpin_all_folios(struct list_head *unpin_list) > unpin_folio(ubuf_folio->folio); > > list_del(&ubuf_folio->list); > - kfree(ubuf_folio); > + kmem_cache_free(udmabuf_folio_cachep, ubuf_folio); > } > } > > @@ -178,7 +180,7 @@ static int add_to_unpin_list(struct list_head *unpin_list, > { > struct udmabuf_folio *ubuf_folio; > > - ubuf_folio = kzalloc(sizeof(*ubuf_folio), GFP_KERNEL); > + ubuf_folio = kmem_cache_alloc(udmabuf_folio_cachep, GFP_KERNEL); > if (!ubuf_folio) > return -ENOMEM; > > @@ -491,11 +493,20 @@ static int __init udmabuf_dev_init(void) > DMA_BIT_MASK(64)); > if (ret < 0) { > pr_err("Could not setup DMA mask for udmabuf device\n"); > - misc_deregister(&udmabuf_misc); > - return ret; > + goto err; > + } > + > + udmabuf_folio_cachep = KMEM_CACHE(udmabuf_folio, 0); > + if (!udmabuf_folio_cachep) { > + ret = -ENOMEM; > + goto err; > } > > return 0; > + > +err: > + misc_deregister(&udmabuf_misc); > + return ret; > } > > static void __exit udmabuf_dev_exit(void) Hi, should a kmem_cache_destroy() be also added in udmabuf_dev_exit()? CJ > > base-commit: cd19ac2f903276b820f5d0d89de0c896c27036ed

1 year, 1 month

Re: [PATCH v2 3/5] dma-buf: heaps: support alloc async read file

by kernel test robot

Hi Huan, kernel test robot noticed the following build warnings: [auto build test WARNING on 931a3b3bccc96e7708c82b30b2b5fa82dfd04890] url: https://github.com/intel-lab-lkp/linux/commits/Huan-Yang/dma-buf-heaps-Intr… base: 931a3b3bccc96e7708c82b30b2b5fa82dfd04890 patch link: https://lore.kernel.org/r/20240730075755.10941-4-link%40vivo.com patch subject: [PATCH v2 3/5] dma-buf: heaps: support alloc async read file config: xtensa-allyesconfig (https://download.01.org/0day-ci/archive/20240731/202407312202.LhLTLEhX-lkp@…) compiler: xtensa-linux-gcc (GCC) 14.1.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240731/202407312202.LhLTLEhX-lkp@…) If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags | Reported-by: kernel test robot <lkp(a)intel.com> | Closes: https://lore.kernel.org/oe-kbuild-all/202407312202.LhLTLEhX-lkp@intel.com/ All warnings (new ones prefixed by >>): drivers/dma-buf/dma-heap.c:45: warning: Function parameter or struct member 'priv' not described in 'dma_heap' drivers/dma-buf/dma-heap.c:45: warning: Function parameter or struct member 'heap_devt' not described in 'dma_heap' drivers/dma-buf/dma-heap.c:45: warning: Function parameter or struct member 'list' not described in 'dma_heap' drivers/dma-buf/dma-heap.c:45: warning: Function parameter or struct member 'heap_cdev' not described in 'dma_heap' >> drivers/dma-buf/dma-heap.c:158: warning: Function parameter or struct member 'lock' not described in 'dma_heap_file_control' drivers/dma-buf/dma-heap.c:482: warning: expecting prototype for Trigger sync file read, read into dma(). Prototype was for dma_heap_read_file_sync() instead vim +158 drivers/dma-buf/dma-heap.c 133 134 /** 135 * struct dma_heap_file_control - global control of dma_heap file read. 136 * @works: @dma_heap_file_work's list head. 137 * 138 * @threadwq: wait queue for @work_thread, if commit work, @work_thread 139 * wakeup and read this work's file contains. 140 * 141 * @workwq: used for main thread wait for file read end, if allocation 142 * end before file read. @dma_heap_file_task ref effect this. 143 * 144 * @work_thread: file read kthread. the dma_heap_file_task work's consumer. 145 * 146 * @heap_fwork_cachep: @dma_heap_file_work's cachep, it's alloc/free frequently. 147 * 148 * @nr_work: global number of how many work committed. 149 */ 150 struct dma_heap_file_control { 151 struct list_head works; 152 spinlock_t lock; // only lock for @works. 153 wait_queue_head_t threadwq; 154 wait_queue_head_t workwq; 155 struct task_struct *work_thread; 156 struct kmem_cache *heap_fwork_cachep; 157 atomic_t nr_work; > 158 }; 159 -- 0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki

1 year, 1 month

Re: [PATCH v2 1/5] dma-buf: heaps: Introduce DMA_HEAP_ALLOC_AND_READ_FILE heap flag

by kernel test robot

Hi Huan, kernel test robot noticed the following build warnings: [auto build test WARNING on 931a3b3bccc96e7708c82b30b2b5fa82dfd04890] url: https://github.com/intel-lab-lkp/linux/commits/Huan-Yang/dma-buf-heaps-Intr… base: 931a3b3bccc96e7708c82b30b2b5fa82dfd04890 patch link: https://lore.kernel.org/r/20240730075755.10941-2-link%40vivo.com patch subject: [PATCH v2 1/5] dma-buf: heaps: Introduce DMA_HEAP_ALLOC_AND_READ_FILE heap flag config: xtensa-allyesconfig (https://download.01.org/0day-ci/archive/20240731/202407311822.ZneNMq5I-lkp@…) compiler: xtensa-linux-gcc (GCC) 14.1.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240731/202407311822.ZneNMq5I-lkp@…) If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags | Reported-by: kernel test robot <lkp(a)intel.com> | Closes: https://lore.kernel.org/oe-kbuild-all/202407311822.ZneNMq5I-lkp@intel.com/ All warnings (new ones prefixed by >>): drivers/dma-buf/dma-heap.c:44: warning: Function parameter or struct member 'priv' not described in 'dma_heap' drivers/dma-buf/dma-heap.c:44: warning: Function parameter or struct member 'heap_devt' not described in 'dma_heap' drivers/dma-buf/dma-heap.c:44: warning: Function parameter or struct member 'list' not described in 'dma_heap' drivers/dma-buf/dma-heap.c:44: warning: Function parameter or struct member 'heap_cdev' not described in 'dma_heap' >> drivers/dma-buf/dma-heap.c:104: warning: expecting prototype for Trigger sync file read, read into dma(). Prototype was for dma_heap_read_file_sync() instead vim +104 drivers/dma-buf/dma-heap.c 86 87 /** 88 * Trigger sync file read, read into dma-buf. 89 * 90 * @dmabuf: which we done alloced and export. 91 * @heap_file: file info wrapper to read from. 92 * 93 * Whether to use buffer I/O or direct I/O depends on the mode when the 94 * file is opened. 95 * Remember, if use direct I/O, file must be page aligned. 96 * Since the buffer used for file reading is provided by dma-buf, when 97 * using direct I/O, the file content will be directly filled into 98 * dma-buf without the need for additional CPU copying. 99 * 100 * 0 on success, negative if anything wrong. 101 */ 102 static int dma_heap_read_file_sync(struct dma_buf *dmabuf, 103 struct dma_heap_file *heap_file) > 104 { 105 struct iosys_map map; 106 ssize_t bytes; 107 int ret; 108 109 ret = dma_buf_vmap(dmabuf, &map); 110 if (ret) 111 return ret; 112 113 /** 114 * The kernel_read_file function can handle file reading effectively, 115 * and if the return value does not match the file size, 116 * then it indicates an error. 117 */ 118 bytes = kernel_read_file(heap_file->file, 0, &map.vaddr, dmabuf->size, 119 &heap_file->fsize, READING_POLICY); 120 if (bytes != heap_file->fsize) 121 ret = -EIO; 122 123 dma_buf_vunmap(dmabuf, &map); 124 125 return ret; 126 } 127 -- 0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki

1 year, 1 month

Re: [PATCH v2 0/5] Introduce DMA_HEAP_ALLOC_AND_READ_FILE heap flag

by T.J. Mercier

On Tue, Jul 30, 2024 at 1:14 AM Huan Yang <link(a)vivo.com> wrote: > > > 在 2024/7/30 16:03, Christian König 写道: > > Am 30.07.24 um 09:57 schrieb Huan Yang: > >> Background > >> ==== > >> Some user may need load file into dma-buf, current way is: > >> 1. allocate a dma-buf, get dma-buf fd > >> 2. mmap dma-buf fd into user vaddr > >> 3. read(file_fd, vaddr, fsz) > >> Due to dma-buf user map can't support direct I/O[1], the file read > >> must be buffer I/O. > >> > >> This means that during the process of reading the file into dma-buf, > >> page cache needs to be generated, and the corresponding content needs to > >> be first copied to the page cache before being copied to the dma-buf. > >> > >> This way worked well when reading relatively small files before, as > >> the page cache can cache the file content, thus improving performance. > >> > >> However, there are new challenges currently, especially as AI models are > >> becoming larger and need to be shared between DMA devices and the CPU > >> via dma-buf. > >> > >> For example, our 7B model file size is around 3.4GB. Using the > >> previous would mean generating a total of 3.4GB of page cache > >> (even if it will be reclaimed), and also requiring the copying of 3.4GB > >> of content between page cache and dma-buf. > >> > >> Due to the limited resources of system memory, files in the gigabyte > >> range > >> cannot persist in memory indefinitely, so this portion of page cache may > >> not provide much assistance for subsequent reads. Additionally, the > >> existence of page cache will consume additional system resources due to > >> the extra copying required by the CPU. > >> > >> Therefore, I think it is necessary for dma-buf to support direct I/O. > >> > >> However, direct I/O file reads cannot be performed using the buffer > >> mmaped by the user space for the dma-buf.[1] > >> > >> Here are some discussions on implementing direct I/O using dma-buf: > >> > >> mmap[1] > >> --- > >> dma-buf never support user map vaddr use of direct I/O. > >> > >> udmabuf[2] > >> --- > >> Currently, udmabuf can use the memfd method to read files into > >> dma-buf in direct I/O mode. > >> > >> However, if the size is large, the current udmabuf needs to adjust the > >> corresponding size_limit(default 64MB). > >> But using udmabuf for files at the 3GB level is not a very good > >> approach. > >> It needs to make some adjustments internally to handle this.[3] Or else, > >> fail create. > >> > >> But, it is indeed a viable way to enable dma-buf to support direct I/O. > >> However, it is necessary to initiate the file read after the memory > >> allocation > >> is completed, and handle race conditions carefully. > >> > >> sendfile/splice[4] > >> --- > >> Another way to enable dma-buf to support direct I/O is by implementing > >> splice_write/write_iter in the dma-buf file operations (fops) to adapt > >> to the sendfile method. > >> However, the current sendfile/splice calls are based on pipe. When using > >> direct I/O to read a file, the content needs to be copied to the buffer > >> allocated by the pipe (default 64KB), and then the dma-buf fops' > >> splice_write needs to be called to write the content into the dma-buf. > >> This approach requires serially reading the content of file pipe size > >> into the pipe buffer and then waiting for the dma-buf to be written > >> before reading the next one.(The I/O performance is relatively weak > >> under direct I/O.) > >> Moreover, due to the existence of the pipe buffer, even when using > >> direct I/O and not needing to generate additional page cache, > >> there still needs to be a CPU copy. > >> > >> copy_file_range[5] > >> --- > >> Consider of copy_file_range, It only supports copying files within the > >> same file system. Similarly, it is not very practical. > >> > >> > >> So, currently, there is no particularly suitable solution on VFS to > >> allow dma-buf to support direct I/O for large file reads. > >> > >> This patchset provides an idea to complete file reads when requesting a > >> dma-buf. > >> > >> Introduce DMA_HEAP_ALLOC_AND_READ_FILE heap flag > >> === > >> This patch provides a method to immediately read the file content after > >> the dma-buf is allocated, and only returns the dma-buf file descriptor > >> after the file is fully read. > >> > >> Since the dma-buf file descriptor is not returned, no other thread can > >> access it except for the current thread, so we don't need to worry about > >> race conditions. > > > > That is a completely false assumption. > Can you provide a detailed explanation as to why this assumption is > incorrect? thanks. > > > >> > >> Map the dma-buf to the vmalloc area and initiate file reads in kernel > >> space, supporting both buffer I/O and direct I/O. > >> > >> This patch adds the DMA_HEAP_ALLOC_AND_READ heap_flag for user. > >> When a user needs to allocate a dma-buf and read a file, they should > >> pass this heap flag. As the size of the file being read is fixed, > >> there is no > >> need to pass the 'len' parameter. Instead, The file_fd needs to be > >> passed to > >> indicate to the kernel the file that needs to be read. > >> > >> The file open flag determines the mode of file reading. > >> But, please note that if direct I/O(O_DIRECT) is needed to read the > >> file, > >> the file size must be page aligned. (with patch 2-5, no need) > >> > >> Therefore, for the user, len and file_fd are mutually exclusive, > >> and they are combined using a union. > >> > >> Once the user obtains the dma-buf fd, the dma-buf directly contains the > >> file content. > > > > And I'm repeating myself, but this is a complete NAK from my side to > > this approach. > > > > We pointed out multiple ways of how to implement this cleanly and not > > by hacking functionality into the kernel which absolutely doesn't > > belong there. > In this patchset, I have provided performance comparisons of each of > these methods. Can you please provide more opinions? > > > > Regards, > > Christian. > > > >> > >> Patch 1 implement it. > >> > >> Patch 2-5 provides an approach for performance improvement. > >> > >> The DMA_HEAP_ALLOC_AND_READ_FILE heap flag patch enables us to > >> synchronously read files using direct I/O. > >> > >> This approach helps to save CPU copying and avoid a certain degree of > >> memory thrashing (page cache generation and reclamation) > >> > >> When dealing with large file sizes, the benefits of this approach become > >> particularly significant. > >> > >> However, there are currently some methods that can improve performance, > >> not just save system resources: > >> > >> Due to the large file size, for example, a AI 7B model of around > >> 3.4GB, the > >> time taken to allocate DMA-BUF memory will be relatively long. Waiting > >> for the allocation to complete before reading the file will add to the > >> overall time consumption. Therefore, the total time for DMA-BUF > >> allocation and file read can be calculated using the formula > >> T(total) = T(alloc) + T(I/O) > >> > >> However, if we change our approach, we don't necessarily need to wait > >> for the DMA-BUF allocation to complete before initiating I/O. In fact, > >> during the allocation process, we already hold a portion of the page, > >> which means that waiting for subsequent page allocations to complete > >> before carrying out file reads is actually unfair to the pages that have > >> already been allocated. > >> > >> The allocation of pages is sequential, and the reading of the file is > >> also sequential, with the content and size corresponding to the file. > >> This means that the memory location for each page, which holds the > >> content of a specific position in the file, can be determined at the > >> time of allocation. > >> > >> However, to fully leverage I/O performance, it is best to wait and > >> gather a certain number of pages before initiating batch processing. > >> > >> The default gather size is 128MB. So, ever gathered can see as a file > >> read > >> work, it maps the gather page to the vmalloc area to obtain a continuous > >> virtual address, which is used as a buffer to store the contents of the > >> corresponding file. So, if using direct I/O to read a file, the file > >> content will be written directly to the corresponding dma-buf buffer > >> memory > >> without any additional copying.(compare to pipe buffer.) > >> > >> Consider other ways to read into dma-buf. If we assume reading after > >> mmap > >> dma-buf, we need to map the pages of the dma-buf to the user virtual > >> address space. Also, udmabuf memfd need do this operations too. > >> Even if we support sendfile, the file copy also need buffer, you must > >> setup it. > >> So, mapping pages to the vmalloc area does not incur any additional > >> performance overhead compared to other methods.[6] > >> > >> Certainly, the administrator can also modify the gather size through > >> patch5. > >> > >> The formula for the time taken for system_heap buffer allocation and > >> file reading through async_read is as follows: > >> > >> T(total) = T(first gather page) + Max(T(remain alloc), T(I/O)) > >> > >> Compared to the synchronous read: > >> T(total) = T(alloc) + T(I/O) > >> > >> If the allocation time or I/O time is long, the time difference will be > >> covered by the maximum value between the allocation and I/O. The other > >> party will be concealed. > >> > >> Therefore, the larger the size of the file that needs to be read, the > >> greater the corresponding benefits will be. > >> > >> How to use > >> === > >> Consider the current pathway for loading model files into DMA-BUF: > >> 1. open dma-heap, get heap fd > >> 2. open file, get file_fd(can't use O_DIRECT) > >> 3. use file len to allocate dma-buf, get dma-buf fd > >> 4. mmap dma-buf fd, get vaddr > >> 5. read(file_fd, vaddr, file_size) into dma-buf pages > >> 6. share, attach, whatever you want > >> > >> Use DMA_HEAP_ALLOC_AND_READ_FILE JUST a little change: > >> 1. open dma-heap, get heap fd > >> 2. open file, get file_fd(buffer/direct) > >> 3. allocate dma-buf with DMA_HEAP_ALLOC_AND_READ_FILE heap flag, > >> set file_fd > >> instead of len. get dma-buf fd(contains file content) > >> 4. share, attach, whatever you want > >> > >> So, test it is easy. > >> > >> How to test > >> === > >> The performance comparison will be conducted for the following > >> scenarios: > >> 1. normal > >> 2. udmabuf with [3] patch > >> 3. sendfile > >> 4. only patch 1 > >> 5. patch1 - patch4. > >> > >> normal: > >> 1. open dma-heap, get heap fd > >> 2. open file, get file_fd(can't use O_DIRECT) > >> 3. use file len to allocate dma-buf, get dma-buf fd > >> 4. mmap dma-buf fd, get vaddr > >> 5. read(file_fd, vaddr, file_size) into dma-buf pages > >> 6. share, attach, whatever you want > >> > >> UDMA-BUF step: > >> 1. memfd_create > >> 2. open file(buffer/direct) > >> 3. udmabuf create > >> 4. mmap memfd > >> 5. read file into memfd vaddr > >> > >> Sendfile step(need suit splice_write/write_iter, just use to compare): > >> 1. open dma-heap, get heap fd > >> 2. open file, get file_fd(buffer/direct) > >> 3. use file len to allocate dma-buf, get dma-buf fd > >> 4. sendfile file_fd to dma-buf fd > >> 6. share, attach, whatever you want > >> > >> patch1/patch1-4: > >> 1. open dma-heap, get heap fd > >> 2. open file, get file_fd(buffer/direct) > >> 3. allocate dma-buf with DMA_HEAP_ALLOC_AND_READ_FILE heap flag, > >> set file_fd > >> instead of len. get dma-buf fd(contains file content) > >> 4. share, attach, whatever you want > >> > >> You can create a file to test it. Compare the performance gap between > >> the two. > >> It is best to compare the differences in file size from KB to MB to GB. > >> > >> The following test data will compare the performance differences > >> between 512KB, > >> 8MB, 1GB, and 3GB under various scenarios. > >> > >> Performance Test > >> === > >> 12G RAM phone > >> UFS4.0(the maximum speed is 4GB/s. ), > >> f2fs > >> kernel 6.1 with patch[7] (or else, can't support kvec direct I/O > >> read.) > >> no memory pressure. > >> drop_cache is used for each test. > >> > >> The average of 5 test results: > >> | scheme-size | 512KB(ns) | 8MB(ns) | 1GB(ns) | > >> 3GB(ns) | > >> | ------------------- | ---------- | ---------- | ------------- | > >> ------------- | > >> | normal | 2,790,861 | 14,535,784 | 1,520,790,492 | > >> 3,332,438,754 | > >> | udmabuf buffer I/O | 1,704,046 | 11,313,476 | 821,348,000 | > >> 2,108,419,923 | > >> | sendfile buffer I/O | 3,261,261 | 12,112,292 | 1,565,939,938 | > >> 3,062,052,984 | > >> | patch1-4 buffer I/O | 2,064,538 | 10,771,474 | 986,338,800 | > >> 2,187,570,861 | > >> | sendfile direct I/O | 12,844,231 | 37,883,938 | 5,110,299,184 | > >> 9,777,661,077 | > >> | patch1 direct I/O | 813,215 | 6,962,092 | 2,364,211,877 | > >> 5,648,897,554 | > >> | udmabuf direct I/O | 1,289,554 | 8,968,138 | 921,480,784 | > >> 2,158,305,738 | > >> | patch1-4 direct I/O | 1,957,661 | 6,581,999 | 520,003,538 | > >> 1,400,006,107 | > > With this test, sendfile can't give a good help base on pipe buffer. > > udmabuf is good, but I think our oem driver can't suit it. (And, AOSP do > not open this feature) Hi Huan, We should be able to turn on udmabuf for the Android kernels. We don't have CONFIG_UDMABUF because nobody has wanted it so far. It's encouraging to see your latest results! -T.J. > > Anyway, I am sending this patchset in the hope of further discussion. > > Thanks. > > >> > >> So, based on the test results: > >> > >> When the file is large, the patchset has the highest performance. > >> Compared to normal, patchset is a 50% improvement; > >> Compared to normal, patch1 only showed a degradation of 41%. > >> patch1 typical performance breakdown is as follows: > >> 1. alloc cost 188,802,693 ns > >> 2. vmap cost 42,491,385 ns > >> 3. file read cost 4,180,876,702 ns > >> Therefore, directly performing a single direct I/O read on a large file > >> may not be the most optimal way for performance. > >> > >> The performance of direct I/O implemented by the sendfile method is > >> the worst. > >> > >> When file size is small, The difference in performance is not > >> significant. This is consistent with expectations. > >> > >> > >> > >> Suggested use cases > >> === > >> 1. When there is a need to read large files and system resources > >> are scarce, > >> especially when the size of memory is limited.(GB level) In this > >> scenario, using direct I/O for file reading can even bring > >> performance > >> improvements.(may need patch2-3) > >> 2. For embedded devices with limited RAM, using direct I/O can > >> save system > >> resources and avoid unnecessary data copying. Therefore, even > >> if the > >> performance is lower when read small file, it can still be used > >> effectively. > >> 3. If there is sufficient memory, pinning the page cache of the > >> model files > >> in memory and placing file in the EROFS file system for > >> read-only access > >> maybe better.(EROFS do not support direct I/O) > >> > >> > >> Changlog > >> === > >> v1 [8] > >> v1->v2: > >> Uses the heap flag method for alloc and read instead of adding a new > >> DMA-buf ioctl command. [9] > >> Split the patchset to facilitate review and test. > >> patch 1 implement alloc and read, offer heap flag into it. > >> patch 2-4 offer async read > >> patch 5 can change gather limit. > >> > >> Reference > >> === > >> [1] > >> https://lore.kernel.org/all/0393cf47-3fa2-4e32-8b3d-d5d5bdece298@amd.com/ > >> [2] > >> https://lore.kernel.org/all/ZpTnzkdolpEwFbtu@phenom.ffwll.local/ > >> [3] > >> https://lore.kernel.org/all/20240725021349.580574-1-link@vivo.com/ > >> [4] > >> https://lore.kernel.org/all/Zpf5R7fRZZmEwVuR@infradead.org/ > >> [5] > >> https://lore.kernel.org/all/ZpiHKY2pGiBuEq4z@infradead.org/ > >> [6] > >> https://lore.kernel.org/all/9b70db2e-e562-4771-be6b-1fa8df19e356@amd.com/ > >> [7] > >> https://patchew.org/linux/20230209102954.528942-1-dhowells@redhat.com/20230… > >> [8] > >> https://lore.kernel.org/all/20240711074221.459589-1-link@vivo.com/ > >> [9] > >> https://lore.kernel.org/all/5ccbe705-883c-4651-9e66-6b452c414c74@amd.com/ > >> > >> Huan Yang (5): > >> dma-buf: heaps: Introduce DMA_HEAP_ALLOC_AND_READ_FILE heap flag > >> dma-buf: heaps: Introduce async alloc read ops > >> dma-buf: heaps: support alloc async read file > >> dma-buf: heaps: system_heap alloc support async read > >> dma-buf: heaps: configurable async read gather limit > >> > >> drivers/dma-buf/dma-heap.c | 552 +++++++++++++++++++++++++++- > >> drivers/dma-buf/heaps/system_heap.c | 70 +++- > >> include/linux/dma-heap.h | 53 ++- > >> include/uapi/linux/dma-heap.h | 11 +- > >> 4 files changed, 673 insertions(+), 13 deletions(-) > >> > >> > >> base-commit: 931a3b3bccc96e7708c82b30b2b5fa82dfd04890 > >

1 year, 1 month

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

Linaro-mm-sig