Re: [Tee-dev] [PATCH v2 09/13] optee: add support for arbitrary shared memory

Hi,

On 03/09/18 17:54, Volodymyr Babchuk wrote:

Shared memory is widely used by NW to communicate with TAs in OP-TEE. NW can share part of own memory with TA or OP-TEE core, by registering it OP-TEE, or by providing a temporal refernce. Anyways, information about such memory buffers are sent to OP-TEE as a list of pages. This mechanism is descripted optee_msg.h.

Mediator should step in when NW tries to share memory with OP-TEE for two reasons:

1. Do address translation from IPA to PA.
2. Pin domain pages till they are mapped into OP-TEE or TA address space, so domain can't transfer this pages to other domain or baloon out them.

s/baloon/balloon/

Address translation is done by translate_noncontig(...) function. It allocates new buffer from xenheap and then walks on guest provided list of pages, translates addresses and stores PAs into newly allocated buffer. This buffer will be provided to OP-TEE instead of original buffer from the guest. This buffer will be free at the end of sdandard call.

In the same time this function pins pages and stores them in struct shm_buf object. This object will live all the time, when given SHM buffer is known to OP-TEE. It will be freed after guest unregisters shared buffer. At this time pages will be unpinned.

Signed-off-by: Volodymyr Babchuk volodymyr_babchuk@epam.com

xen/arch/arm/tee/optee.c | 245 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 244 insertions(+), 1 deletion(-)

diff --git a/xen/arch/arm/tee/optee.c b/xen/arch/arm/tee/optee.c index 6d6b51d..8bfcfdc 100644 --- a/xen/arch/arm/tee/optee.c +++ b/xen/arch/arm/tee/optee.c @@ -22,6 +22,8 @@ #define MAX_STD_CALLS 16 #define MAX_RPC_SHMS 16 +#define MAX_TOTAL_SMH_BUF_PG 16384

So that's 64MB worth of guest memory. Do we expect them to be mapped in Xen? Or just pinned?

+#define MAX_NONCONTIG_ENTRIES 5 /*

• Call context. OP-TEE can issue multiple RPC returns during one call.

@@ -31,6 +33,9 @@ struct std_call_ctx { struct list_head list; struct optee_msg_arg *guest_arg; struct optee_msg_arg *xen_arg;

• /* Buffer for translated page addresses, shared with OP-TEE */
• void *non_contig[MAX_NONCONTIG_ENTRIES];
• int non_contig_order[MAX_NONCONTIG_ENTRIES];

Can you please introduce a structure with the order and mapping?

  mfn_t guest_arg_mfn;
  int optee_thread_id;
  int rpc_op;

@@ -45,13 +50,24 @@ struct shm_rpc { uint64_t cookie; }; +/* Shared memory buffer for arbitrary data */ +struct shm_buf {

• struct list_head list;
• uint64_t cookie;
• int max_page_cnt;
• int page_cnt;

AFAICT, max_page_cnt and page_cnt should never but negative. If so, then they should be unsigned.

• struct page_info *pages[];

+};

• struct domain_ctx { struct list_head list; struct list_head call_ctx_list; struct list_head shm_rpc_list;
• struct list_head shm_buf_list; struct domain *domain; atomic_t call_ctx_count; atomic_t shm_rpc_count;
• atomic_t shm_buf_pages; spinlock_t lock; };

@@ -158,9 +174,12 @@ static int optee_enable(struct domain *d) ctx->domain = d; INIT_LIST_HEAD(&ctx->call_ctx_list); INIT_LIST_HEAD(&ctx->shm_rpc_list);

• INIT_LIST_HEAD(&ctx->shm_buf_list);

atomic_set(&ctx->call_ctx_count, 0); atomic_set(&ctx->shm_rpc_count, 0);

• atomic_set(&ctx->shm_buf_pages, 0);

• spin_lock_init(&ctx->lock);
  

spin_lock(&domain_ctx_list_lock); @@ -339,12 +358,76 @@ static void free_shm_rpc(struct domain_ctx *ctx, uint64_t cookie) xfree(shm_rpc); } +static struct shm_buf *allocate_shm_buf(struct domain_ctx *ctx,

•                                    uint64_t cookie,
  

•                                    int pages_cnt)
  

Ditto.

+{

• struct shm_buf *shm_buf;
• while(1)
• {

•    int old = atomic_read(&ctx->shm_buf_pages);
  

•    int new = old + pages_cnt;
  

•    if ( new >= MAX_TOTAL_SMH_BUF_PG )
  

•        return NULL;
  

•    if ( likely(old == atomic_cmpxchg(&ctx->shm_buf_pages, old, new)) )
  

•        break;
  

• }
• shm_buf = xzalloc_bytes(sizeof(struct shm_buf) +

•                        pages_cnt * sizeof(struct page *));
  

• if ( !shm_buf ) {

Coding style:

if ( ... ) {

•    atomic_sub(pages_cnt, &ctx->shm_buf_pages);
  

•    return NULL;
  

• }
• shm_buf->cookie = cookie;
• shm_buf->max_page_cnt = pages_cnt;
• spin_lock(&ctx->lock);
• list_add_tail(&shm_buf->list, &ctx->shm_buf_list);
• spin_unlock(&ctx->lock);
• return shm_buf;

+}

+static void free_shm_buf(struct domain_ctx *ctx, uint64_t cookie) +{

• struct shm_buf *shm_buf; > + bool found = false;
• spin_lock(&ctx->lock);
• list_for_each_entry( shm_buf, &ctx->shm_buf_list, list )
• {

•    if ( shm_buf->cookie == cookie )
  

What does guarantee you the cookie will be uniq?

•    {
  

•        found = true;
  

•        list_del(&shm_buf->list);
  

•        break;
  

•    }
  

• }
• spin_unlock(&ctx->lock);

At this point you have a shm_buf in hand to free. But what does guarantee you no-one will use it?

• if ( !found ) {

•    return;
  

• }
• for ( int i = 0; i < shm_buf->page_cnt; i++ )

Please define int i before hand.

•    if ( shm_buf->pages[i] )
  

•        put_page(shm_buf->pages[i]);
  

• atomic_sub(shm_buf->max_page_cnt, &ctx->shm_buf_pages);
• xfree(shm_buf);

+}

• static void optee_domain_destroy(struct domain *d) { struct arm_smccc_res resp; struct domain_ctx *ctx; struct std_call_ctx *call, *call_tmp; struct shm_rpc *shm_rpc, *shm_rpc_tmp;
• struct shm_buf *shm_buf, *shm_buf_tmp; bool found = false;

/* At this time all domain VCPUs should be stopped */ @@ -377,12 +460,163 @@ static void optee_domain_destroy(struct domain *d) list_for_each_entry_safe( shm_rpc, shm_rpc_tmp, &ctx->shm_rpc_list, list ) free_shm_rpc(ctx, shm_rpc->cookie);

• list_for_each_entry_safe( shm_buf, shm_buf_tmp, &ctx->shm_buf_list, list )

•    free_shm_buf(ctx, shm_buf->cookie);
  

• ASSERT(!atomic_read(&ctx->call_ctx_count));
  ASSERT(!atomic_read(&ctx->shm_rpc_count));
  

• ASSERT(!atomic_read(&ctx->shm_buf_pages));

xfree(ctx); } +#define PAGELIST_ENTRIES_PER_PAGE \

• ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)

+static size_t get_pages_list_size(size_t num_entries) +{

• int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
• return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;

+}

+static bool translate_noncontig(struct domain_ctx *ctx,

•                            struct std_call_ctx *call,
  

•                            struct optee_msg_param *param,
  

•                            int idx)
  

Most likely this should be unsigned.

+{

• /*

• * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h for details.
  

• */
  

• uint64_t size;
• int page_offset;
• int num_pages;
• int order;
• int entries_on_page = 0;
• paddr_t gaddr;
• mfn_t guest_mfn;

I don't like the terminology guest_mfn. This is misleading because of the past usage in Xen. It would be better if you call this just mfn.

• struct {

•    uint64_t pages_list[PAGELIST_ENTRIES_PER_PAGE];
  

•    uint64_t next_page_data;
  

• } *pages_data_guest, *pages_data_xen, *pages_data_xen_start;
• struct shm_buf *shm_buf;
• page_offset = param->u.tmem.buf_ptr & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1);
• size = ROUNDUP(param->u.tmem.size + page_offset,

•               OPTEE_MSG_NONCONTIG_PAGE_SIZE);
  

• num_pages = DIV_ROUND_UP(size, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
• order = get_order_from_bytes(get_pages_list_size(num_pages));
• pages_data_xen_start = alloc_xenheap_pages(order, 0);

This could be replaced by a _xmalloc and would avoid to allocate more memory than necessary when the order is getting bigger.

• if ( !pages_data_xen_start )

•    return false;
  

• shm_buf = allocate_shm_buf(ctx, param->u.tmem.shm_ref, num_pages);

In alocate_shm_buf you are now globally limiting the number of pages ( (16384) to pin. However, this does not limit per call.

With the current limit, you would could call up to 16384 times lookup_and_pin_guest_ram_addr(...). On Arm, for p2m related operation, we limit to 512 iterations in one go before checking the preemption. So I think 16384 times is far too much.

• if ( !shm_buf )

•    goto err_free;
  

• gaddr = param->u.tmem.buf_ptr & ~(OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1);
• guest_mfn = lookup_and_pin_guest_ram_addr(gaddr, NULL);
• if ( mfn_eq(guest_mfn, INVALID_MFN) )

•    goto err_free;
  

• pages_data_guest = map_domain_page(guest_mfn);
• if ( !pages_data_guest )

•    goto err_free;
  

• pages_data_xen = pages_data_xen_start;
• while ( num_pages ) {

•    struct page_info *page;
  

•    mfn_t entry_mfn = lookup_and_pin_guest_ram_addr(
  

•        pages_data_guest->pages_list[entries_on_page], &page);
  

•    if ( mfn_eq(entry_mfn, INVALID_MFN) )
  

•        goto err_unmap;
  

•    shm_buf->pages[shm_buf->page_cnt++] = page;
  

•    pages_data_xen->pages_list[entries_on_page] = mfn_to_maddr(entry_mfn);
  

•    entries_on_page++;
  

•    if ( entries_on_page == PAGELIST_ENTRIES_PER_PAGE ) {
  

•        pages_data_xen->next_page_data = virt_to_maddr(pages_data_xen + 1);
  

•        pages_data_xen++;
  

•        gaddr = pages_data_guest->next_page_data;
  

next_page_data is not a guest address but a machine address. For anything related to address, the variable should be named accordingly to avoid confusion.

•        unmap_domain_page(pages_data_guest);
  

•        unpin_guest_ram_addr(guest_mfn);
  

•        guest_mfn = lookup_and_pin_guest_ram_addr(gaddr, NULL);
  

•        if ( mfn_eq(guest_mfn, INVALID_MFN) )
  

•            goto err_free;
  

•        pages_data_guest = map_domain_page(guest_mfn);
  

•        if ( !pages_data_guest )
  

•            goto err_free;
  

•        /* Roll over to the next page */
  

•        entries_on_page = 0;
  

•    }
  

•    num_pages--;
  

• }
• param->u.tmem.buf_ptr = virt_to_maddr(pages_data_xen_start) | page_offset;
• call->non_contig[idx] = pages_data_xen_start;
• call->non_contig_order[idx] = order;

If you use _xmalloc, then you don't need to store the order. But, who is going to free non_contig?

• unmap_domain_page(pages_data_guest);
• unpin_guest_ram_addr(guest_mfn);
• return true;

+err_unmap:

• unmap_domain_page(pages_data_guest);
• unpin_guest_ram_addr(guest_mfn);
• free_shm_buf(ctx, shm_buf->cookie);

+err_free:

• free_xenheap_pages(pages_data_xen_start, order);
• return false;

+}

+static bool translate_params(struct domain_ctx *ctx,

•                         struct std_call_ctx *call)
  

+{

• unsigned int i;
• uint32_t attr;
• for ( i = 0; i < call->xen_arg->num_params; i++ ) {

Please pay attention to Xen coding style. I haven't pointed out everywhere, but I would all of them to be fixed in the next version.

•    attr = call->xen_arg->params[i].attr;
  

•    switch ( attr & OPTEE_MSG_ATTR_TYPE_MASK ) {
  

•    case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
  

•    case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
  

•    case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
  

•        if ( attr & OPTEE_MSG_ATTR_NONCONTIG ) {
  

•            if ( !translate_noncontig(ctx, call,
  

•                                      call->xen_arg->params + i, i) )
  

•                return false;
  

•        }
  

•        else {
  

•            gprintk(XENLOG_WARNING, "Guest tries to use old tmem arg\n");
  

•            return false;
  

•        }
  

•        break;
  

•    case OPTEE_MSG_ATTR_TYPE_NONE:
  

•    case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
  

•    case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
  

•    case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
  

•    case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
  

•    case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
  

•    case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
  

•        continue;
  

•    }
  

• }
• return true;

+}

• /*
  • Copy command buffer into xen memory to:
  • 1. Hide translated addresses from guest

@@ -488,6 +722,15 @@ static bool execute_std_call(struct domain_ctx *ctx, copy_std_request_back(ctx, regs, call);

• /*

• * If guest successfully unregistered own shared memory,
  

• * then we can unpin it's pages
  

• */
  

• if ( call->xen_arg->cmd == OPTEE_MSG_CMD_UNREGISTER_SHM &&

•     call->xen_arg->ret == 0 ) {
  

•    free_shm_buf(ctx, call->xen_arg->params[0].u.rmem.shm_ref);
  

• }

• free_std_call_ctx(ctx, call);
  

return true; @@ -522,7 +765,7 @@ static bool handle_std_call(struct domain_ctx *ctx, struct cpu_user_regs *regs) case OPTEE_MSG_CMD_CANCEL: case OPTEE_MSG_CMD_REGISTER_SHM: case OPTEE_MSG_CMD_UNREGISTER_SHM:

•    ret = true;
  

•    ret = translate_params(ctx, call);
      break;
  default:
      ret = false;
  
  

Cheers,