git-lng-odp

git-lng-odp@lists.linaro.org

1203 discussions

[COMMIT] branch master updated. v1.15.0.0-206-ge1c46f8e

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via e1c46f8e296a730ed27141a33189185bb7dfd1b1 (commit) from 054306373ed6aa7a65a160d11ca339b24cf9c662 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit e1c46f8e296a730ed27141a33189185bb7dfd1b1 Author: Kelvin Cheung <keguang.zhang(a)gmail.com> Date: Wed Aug 23 17:00:58 2017 +0800 linux-gen: system: correctly set CPU frequency When CPU frequency info under /sys is available, use the real value instead of default value. Signed-off-by: Kelvin Cheung <keguang.zhang(a)gmail.com> Reviewed-by: Ilias Apalodimas <ilias.apalodimas(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c index 1e9db418..1e2c729a 100644 --- a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c @@ -23,11 +23,6 @@ int cpuinfo_parser(FILE *file ODP_UNUSED, system_info_t *sysinfo) return 0; } -uint64_t odp_cpu_hz_current(int id ODP_UNUSED) -{ - return 0; -} - void sys_info_print_arch(void) { } diff --git a/platform/linux-generic/arch/default/odp_sysinfo_parse.c b/platform/linux-generic/arch/default/odp_sysinfo_parse.c index 1e9db418..1e2c729a 100644 --- a/platform/linux-generic/arch/default/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/default/odp_sysinfo_parse.c @@ -23,11 +23,6 @@ int cpuinfo_parser(FILE *file ODP_UNUSED, system_info_t *sysinfo) return 0; } -uint64_t odp_cpu_hz_current(int id ODP_UNUSED) -{ - return 0; -} - void sys_info_print_arch(void) { } diff --git a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c index 2a37ae70..5ed7a398 100644 --- a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c @@ -60,11 +60,6 @@ int cpuinfo_parser(FILE *file, system_info_t *sysinfo) return 0; } -uint64_t odp_cpu_hz_current(int id ODP_UNUSED) -{ - return 0; -} - void sys_info_print_arch(void) { } diff --git a/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c index 29c97613..02a83e51 100644 --- a/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c @@ -59,11 +59,6 @@ int cpuinfo_parser(FILE *file, system_info_t *sysinfo) return 0; } -uint64_t odp_cpu_hz_current(int id ODP_UNUSED) -{ - return 0; -} - void sys_info_print_arch(void) { } diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c index f7918d40..55bbda8c 100644 --- a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -41,42 +41,6 @@ int cpuinfo_parser(FILE *file, system_info_t *sysinfo) return 0; } -uint64_t odp_cpu_hz_current(int id) -{ - char str[1024]; - FILE *file; - int cpu; - char *pos; - double mhz = 0.0; - - file = fopen("/proc/cpuinfo", "rt"); - - /* find the correct processor instance */ - while (fgets(str, sizeof(str), file) != NULL) { - pos = strstr(str, "processor"); - if (pos) { - if (sscanf(pos, "processor : %d", &cpu) == 1) - if (cpu == id) - break; - } - } - - /* extract the cpu current speed */ - while (fgets(str, sizeof(str), file) != NULL) { - pos = strstr(str, "cpu MHz"); - if (pos) { - if (sscanf(pos, "cpu MHz : %lf", &mhz) == 1) - break; - } - } - - fclose(file); - if (mhz) - return (uint64_t)(mhz * 1000000.0); - - return 0; -} - void sys_info_print_arch(void) { cpu_flags_print_all(); diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index 355e25f7..e0cc6dd4 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -131,6 +131,7 @@ int _odp_ishm_term_global(void); int _odp_ishm_term_local(void); int cpuinfo_parser(FILE *file, system_info_t *sysinfo); +uint64_t odp_cpufreq_id(const char *filename, int id); uint64_t odp_cpu_hz_current(int id); void sys_info_print_arch(void); diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index d6eeb786..d310d779 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -261,6 +261,30 @@ static char *get_hugepage_dir(uint64_t hugepage_sz) } /* + * Analysis of /sys/devices/system/cpu/cpu%d/cpufreq/ files + */ +uint64_t odp_cpufreq_id(const char *filename, int id) +{ + char path[256], buffer[256], *endptr = NULL; + FILE *file; + uint64_t ret = 0; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/%s", id, filename); + + file = fopen(path, "r"); + if (file == NULL) + return ret; + + if (fgets(buffer, sizeof(buffer), file) != NULL) + ret = strtoull(buffer, &endptr, 0) * 1000; + + fclose(file); + + return ret; +} + +/* * Analysis of /sys/devices/system/cpu/ files */ static int systemcpu(system_info_t *sysinfo) @@ -310,6 +334,7 @@ static int system_hp(hugepage_info_t *hugeinfo) */ int odp_system_info_init(void) { + int i; FILE *file; memset(&odp_global_data.system_info, 0, sizeof(system_info_t)); @@ -326,6 +351,13 @@ int odp_system_info_init(void) fclose(file); + for (i = 0; i < MAX_CPU_NUMBER; i++) { + uint64_t cpu_hz_max = odp_cpufreq_id("cpuinfo_max_freq", i); + + if (cpu_hz_max) + odp_global_data.system_info.cpu_hz_max[i] = cpu_hz_max; + } + if (systemcpu(&odp_global_data.system_info)) { ODP_ERR("systemcpu failed\n"); return -1; @@ -351,6 +383,11 @@ int odp_system_info_term(void) * Public access functions ************************* */ +uint64_t odp_cpu_hz_current(int id) +{ + return odp_cpufreq_id("cpuinfo_cur_freq", id); +} + uint64_t odp_cpu_hz(void) { int id = sched_getcpu(); ----------------------------------------------------------------------- Summary of changes: .../linux-generic/arch/arm/odp_sysinfo_parse.c | 5 --- .../linux-generic/arch/default/odp_sysinfo_parse.c | 5 --- .../linux-generic/arch/mips64/odp_sysinfo_parse.c | 5 --- .../linux-generic/arch/powerpc/odp_sysinfo_parse.c | 5 --- .../linux-generic/arch/x86/odp_sysinfo_parse.c | 36 --------------------- platform/linux-generic/include/odp_internal.h | 1 + platform/linux-generic/odp_system_info.c | 37 ++++++++++++++++++++++ 7 files changed, 38 insertions(+), 56 deletions(-) hooks/post-receive --

7 years, 9 months

[COMMIT] branch master updated. v1.15.0.0-205-g05430637

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 054306373ed6aa7a65a160d11ca339b24cf9c662 (commit) from a38ea82cf1ae45b7c807164dde5783d099efd39d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 054306373ed6aa7a65a160d11ca339b24cf9c662 Author: Bill Fischofer <bill.fischofer(a)linaro.org> Date: Thu Sep 14 11:40:45 2017 -0500 doc: helper: add missing doxygen to avoid warnings The latest version of Doxygen requires every struct and field to be documented, so add missing documentation to helper files to avoid superfluous warnings. This resolves Bug https://bugs.linaro.org/show_bug.cgi?id=3238 Signed-off-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/helper/include/odp/helper/ip.h b/helper/include/odp/helper/ip.h index fb7e5ddc..c6eb9d76 100644 --- a/helper/include/odp/helper/ip.h +++ b/helper/include/odp/helper/ip.h @@ -95,6 +95,17 @@ typedef struct ODP_PACKED { ODP_STATIC_ASSERT(sizeof(odph_ipv4hdr_t) == ODPH_IPV4HDR_LEN, "ODPH_IPV4HDR_T__SIZE_ERROR"); +/** + * Calculate IPv4 header checksum + * + * @param pkt The packet to be checksummed + * @param offset Offset into pkt of start of IP header + * @param ip Pointer to IPv4 header to be checksummed + * @param[out] chksum Field to receive checksum results + * + * @retval 0 On success + * @retval <0 On failure + */ static inline int odph_ipv4_csum(odp_packet_t pkt, uint32_t offset, odph_ipv4hdr_t *ip, diff --git a/helper/include/odph_list_internal.h b/helper/include/odph_list_internal.h index 9e532b08..f870ee30 100644 --- a/helper/include/odph_list_internal.h +++ b/helper/include/odph_list_internal.h @@ -18,18 +18,30 @@ extern "C" { #endif +/** List object */ typedef struct odph_list_object { - struct odph_list_object *next, *prev; + /** Next element in list */ + struct odph_list_object *next; + + /** Previous element in list */ + struct odph_list_object *prev; } odph_list_object; +/** Head of list */ typedef odph_list_object odph_list_head; +/** + * @internal Intiailize list head element + * + * @param list List object to be initialized + */ static inline void ODPH_INIT_LIST_HEAD(odph_list_object *list) { list->next = list; list->prev = list; } +/** @internal Inline function @param new @param prev @param next */ static inline void __odph_list_add(odph_list_object *new, odph_list_object *prev, odph_list_object *next) @@ -40,17 +52,20 @@ static inline void __odph_list_add(odph_list_object *new, prev->next = new; } +/** @internal Inline function @param new @param head */ static inline void odph_list_add(odph_list_object *new, odph_list_object *head) { __odph_list_add(new, head, head->next); } +/** @internal Inline function @param new @param head */ static inline void odph_list_add_tail(struct odph_list_object *new, odph_list_object *head) { __odph_list_add(new, head->prev, head); } +/** @internal Inline function @param prev @param next */ static inline void __odph_list_del(struct odph_list_object *prev, odph_list_object *next) { @@ -58,20 +73,24 @@ static inline void __odph_list_del(struct odph_list_object *prev, prev->next = next; } +/** @internal Inline function @param entry */ static inline void odph_list_del(struct odph_list_object *entry) { __odph_list_del(entry->prev, entry->next); ODPH_INIT_LIST_HEAD(entry); } +/** @internal Inline function @param head @return */ static inline int odph_list_empty(const struct odph_list_object *head) { return head->next == head; } +/** @internal */ #define container_of(ptr, type, list_node) \ ((type *)(void *)((char *)ptr - offsetof(type, list_node))) +/** @internal */ #define ODPH_LIST_FOR_EACH(pos, list_head, type, list_node) \ for (pos = container_of((list_head)->next, type, list_node); \ &pos->list_node != (list_head); \ @@ -82,4 +101,3 @@ static inline int odph_list_empty(const struct odph_list_object *head) #endif #endif - ----------------------------------------------------------------------- Summary of changes: helper/include/odp/helper/ip.h | 11 +++++++++++ helper/include/odph_list_internal.h | 22 ++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) hooks/post-receive --

7 years, 9 months

[COMMIT] branch api-next updated. v1.15.0.0-675-g8c5516bb

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 8c5516bba62ee7250cbf64d1f4b89ee4f0b12824 (commit) from 32f5f3ff343b516cb0dbe98e81479e6c748f0ebf (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 8c5516bba62ee7250cbf64d1f4b89ee4f0b12824 Author: Ola Liljedahl <ola.liljedahl(a)arm.com> Date: Thu Aug 31 21:46:47 2017 +0200 linux-gen: scheduled pktin polling for ssched Pktin polling is handled by the scalable scheduler the same way as queues. Non-empty pktin queues stay on the schedq of their designated priority level. Empty pktin queues are pushed to a lower-than-lowest priority schedq so to be polled only when all other (highest to lowest priority) schedq's have been processed. Signed-off-by: Ola Liljedahl <ola.liljedahl(a)arm.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h index f15314b2..ddc65f60 100644 --- a/platform/linux-generic/include/odp_queue_scalable_internal.h +++ b/platform/linux-generic/include/odp_queue_scalable_internal.h @@ -58,6 +58,8 @@ union queue_entry_u { int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num); int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num); int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num); +int _odp_queue_enq_sp(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num); +queue_entry_t *qentry_from_ext(odp_queue_t handle); /* Round up memory size to next cache line size to * align all memory addresses on cache line boundary. diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 657993b1..71538e62 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -14,9 +14,15 @@ extern "C" { #include <odp/api/queue.h> #include <odp_queue_if.h> #include <odp/api/schedule.h> +#include <odp_forward_typedefs_internal.h> -typedef void (*schedule_pktio_start_fn_t)(int pktio_index, int num_in_queue, - int in_queue_idx[]); +/* Number of ordered locks per queue */ +#define SCHEDULE_ORDERED_LOCKS_PER_QUEUE 2 + +typedef void (*schedule_pktio_start_fn_t)(int pktio_index, + int num_in_queue, + int in_queue_idx[], + odp_queue_t odpq[]); typedef int (*schedule_thr_add_fn_t)(odp_schedule_group_t group, int thr); typedef int (*schedule_thr_rem_fn_t)(odp_schedule_group_t group, int thr); typedef int (*schedule_num_grps_fn_t)(void); @@ -66,6 +72,7 @@ extern const schedule_fn_t *sched_fn; /* Interface for the scheduler */ int sched_cb_pktin_poll(int pktio_index, int num_queue, int index[]); +int sched_cb_pktin_poll_one(int pktio_index, int rx_queue, odp_event_t evts[]); void sched_cb_pktio_stop_finalize(int pktio_index); odp_queue_t sched_cb_queue_handle(uint32_t queue_index); void sched_cb_queue_destroy_finalize(uint32_t queue_index); diff --git a/platform/linux-generic/include/odp_schedule_scalable.h b/platform/linux-generic/include/odp_schedule_scalable.h index 8a2d70da..ac357b01 100644 --- a/platform/linux-generic/include/odp_schedule_scalable.h +++ b/platform/linux-generic/include/odp_schedule_scalable.h @@ -22,8 +22,12 @@ * constants, but not ODP_SCHED_PRIO_NUM. The current API for this * is odp_schedule_num_prio(). The other schedulers also define * this internally as NUM_PRIO. + * + * One additional priority level for idle pktin queues. + * This is only for internal use and not visible to the user. */ -#define ODP_SCHED_PRIO_NUM 8 +#define ODP_SCHED_PRIO_PKTIN 8 +#define ODP_SCHED_PRIO_NUM 9 typedef struct { union { @@ -55,14 +59,18 @@ typedef uint32_t ringidx_t; #define ODP_NO_SCHED_QUEUE (ODP_SCHED_SYNC_ORDERED + 1) typedef struct { - struct llnode node; /* must be first */ + struct llnode node; sched_queue_t *schedq; #ifdef CONFIG_QSCHST_LOCK odp_ticketlock_t qschlock; #endif qschedstate_t qschst; - uint16_t pop_deficit; - uint16_t qschst_type; + uint8_t pop_deficit; + uint8_t qschst_type; + uint8_t pktio_idx; + uint8_t rx_queue; + uint16_t xoffset; + uint8_t sched_prio; ringidx_t prod_read SPLIT_PC; ringidx_t prod_write; ringidx_t prod_mask; @@ -80,6 +88,7 @@ typedef struct { #define cons_ring prod_ring #define cons_type qschst_type #endif + odp_schedule_group_t sched_grp; } sched_elem_t ODP_ALIGNED_CACHE; /* Number of scheduling groups */ @@ -106,6 +115,8 @@ typedef struct { typedef struct { /* Atomic queue currently being processed or NULL */ sched_elem_t *atomq; + /* Schedq the currently processed queue was popped from */ + sched_queue_t *src_schedq; /* Current reorder context or NULL */ reorder_context_t *rctx; uint8_t pause; @@ -113,8 +124,6 @@ typedef struct { uint8_t tidx; uint8_t pad; uint32_t dequeued; /* Number of events dequeued from atomic queue */ - uint16_t pktin_next; /* Next pktin tag to poll */ - uint16_t pktin_poll_cnts; uint16_t ticket; /* Ticket for atomic queue or TICKET_INVALID */ uint16_t num_schedq; uint16_t sg_sem; /* Set when sg_wanted is modified by other thread */ @@ -133,7 +142,7 @@ typedef struct { void sched_update_enq(sched_elem_t *q, uint32_t actual); void sched_update_enq_sp(sched_elem_t *q, uint32_t actual); -sched_queue_t *schedq_from_sched_group(odp_schedule_group_t grp, uint32_t prio); -void sched_group_xcount_dec(odp_schedule_group_t grp, uint32_t prio); +sched_queue_t *sched_queue_add(odp_schedule_group_t grp, uint32_t prio); +void sched_queue_rem(odp_schedule_group_t grp, uint32_t prio); #endif /* ODP_SCHEDULE_SCALABLE_H */ diff --git a/platform/linux-generic/include/odp_schedule_scalable_ordered.h b/platform/linux-generic/include/odp_schedule_scalable_ordered.h index 1c365a2b..ca49bbf4 100644 --- a/platform/linux-generic/include/odp_schedule_scalable_ordered.h +++ b/platform/linux-generic/include/odp_schedule_scalable_ordered.h @@ -115,6 +115,8 @@ reorder_window_t *rwin_alloc(_odp_ishm_pool_t *pool, unsigned lock_count); int rwin_free(_odp_ishm_pool_t *pool, reorder_window_t *rwin); bool rwin_reserve(reorder_window_t *rwin, uint32_t *sn); +bool rwin_reserve_sc(reorder_window_t *rwin, uint32_t *sn); +void rwin_unreserve_sc(reorder_window_t *rwin, uint32_t sn); void rctx_init(reorder_context_t *rctx, uint16_t idx, reorder_window_t *rwin, uint32_t sn); void rctx_release(reorder_context_t *rctx); diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 732e0c15..bf792a34 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -469,9 +469,11 @@ int odp_pktio_start(odp_pktio_t hdl) unsigned i; unsigned num = entry->s.num_in_queue; int index[num]; + odp_queue_t odpq[num]; for (i = 0; i < num; i++) { index[i] = i; + odpq[i] = entry->s.in_queue[i].queue; if (entry->s.in_queue[i].queue == ODP_QUEUE_INVALID) { ODP_ERR("No input queue\n"); @@ -479,7 +481,7 @@ int odp_pktio_start(odp_pktio_t hdl) } } - sched_fn->pktio_start(pktio_to_id(hdl), num, index); + sched_fn->pktio_start(pktio_to_id(hdl), num, index, odpq); } return res; @@ -668,6 +670,51 @@ static int pktin_deq_multi(queue_t q_int, odp_buffer_hdr_t *buf_hdr[], int num) return nbr; } +int sched_cb_pktin_poll_one(int pktio_index, + int rx_queue, + odp_event_t evt_tbl[QUEUE_MULTI_MAX]) +{ + int num_rx, num_pkts, i; + pktio_entry_t *entry = pktio_entry_by_index(pktio_index); + odp_packet_t pkt; + odp_packet_hdr_t *pkt_hdr; + odp_buffer_hdr_t *buf_hdr; + odp_packet_t packets[QUEUE_MULTI_MAX]; + queue_t queue; + + if (odp_unlikely(entry->s.state != PKTIO_STATE_STARTED)) { + if (entry->s.state < PKTIO_STATE_ACTIVE || + entry->s.state == PKTIO_STATE_STOP_PENDING) + return -1; + + ODP_DBG("interface not started\n"); + return 0; + } + + ODP_ASSERT((unsigned)rx_queue < entry->s.num_in_queue); + num_pkts = entry->s.ops->recv(entry, rx_queue, + packets, QUEUE_MULTI_MAX); + + num_rx = 0; + for (i = 0; i < num_pkts; i++) { + pkt = packets[i]; + pkt_hdr = odp_packet_hdr(pkt); + if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) { + queue = pkt_hdr->dst_queue; + buf_hdr = packet_to_buf_hdr(pkt); + if (queue_fn->enq_multi(queue, &buf_hdr, 1) < 0) + /* Queue full? */ + odp_packet_free(pkt); + __atomic_fetch_add(&entry->s.stats.in_discards, + 1, + __ATOMIC_RELAXED); + } else { + evt_tbl[num_rx++] = odp_packet_to_event(pkt); + } + } + return num_rx; +} + int sched_cb_pktin_poll(int pktio_index, int num_queue, int index[]) { odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index bcda2195..cef3236d 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -70,6 +70,11 @@ static queue_entry_t *get_qentry(uint32_t queue_id) return &queue_tbl->queue[queue_id]; } +queue_entry_t *qentry_from_ext(odp_queue_t handle) +{ + return get_qentry(queue_to_id(handle)); +} + static int _odp_queue_disable_enq(sched_elem_t *q) { ringidx_t old_read, old_write, new_write; @@ -170,9 +175,12 @@ static int queue_init(queue_entry_t *queue, const char *name, goto rwin_create_failed; } } + sched_elem->sched_grp = param->sched.group; + sched_elem->sched_prio = param->sched.prio; sched_elem->schedq = - schedq_from_sched_group(param->sched.group, - param->sched.prio); + sched_queue_add(param->sched.group, param->sched.prio); + ODP_ASSERT(sched_elem->schedq != NULL); + } return 0; @@ -433,19 +441,20 @@ static int queue_destroy(odp_queue_t handle) doze(); } - /* Adjust the spread factor for the queues in the schedule group */ - if (queue->s.type == ODP_QUEUE_TYPE_SCHED) - sched_group_xcount_dec(queue->s.param.sched.group, - queue->s.param.sched.prio); + if (q->schedq != NULL) { + sched_queue_rem(q->sched_grp, q->sched_prio); + q->schedq = NULL; + } _odp_ishm_pool_free(queue_shm_pool, q->prod_ring); - if (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED) { + if (q->rwin != NULL) { if (rwin_free(queue_shm_pool, q->rwin) < 0) { ODP_ERR("Failed to free reorder window\n"); UNLOCK(&queue->s.lock); return -1; } + q->rwin = NULL; } queue->s.status = QUEUE_STATUS_FREE; UNLOCK(&queue->s.lock); @@ -554,11 +563,11 @@ static inline int _odp_queue_enq(sched_elem_t *q, return actual; } -#else +#endif -static inline int _odp_queue_enq_sp(sched_elem_t *q, - odp_buffer_hdr_t *buf_hdr[], - int num) +int _odp_queue_enq_sp(sched_elem_t *q, + odp_buffer_hdr_t *buf_hdr[], + int num) { ringidx_t old_read; ringidx_t old_write; @@ -602,7 +611,6 @@ static inline int _odp_queue_enq_sp(sched_elem_t *q, return actual; } -#endif static int _queue_enq_multi(queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) diff --git a/platform/linux-generic/odp_schedule.c b/platform/linux-generic/odp_schedule.c index 5b940762..8c444f4d 100644 --- a/platform/linux-generic/odp_schedule.c +++ b/platform/linux-generic/odp_schedule.c @@ -592,7 +592,7 @@ static inline void free_pktio_cmd(pktio_cmd_t *cmd) } static void schedule_pktio_start(int pktio_index, int num_pktin, - int pktin_idx[]) + int pktin_idx[], odp_queue_t odpq[] ODP_UNUSED) { int i, idx; pktio_cmd_t *cmd; diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index b81e5dab..afe8ee1a 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -574,7 +574,10 @@ static inline void free_pktio_cmd(pktio_cmd_t *cmd) odp_rwlock_write_unlock(&sched->pktio_poll.lock); } -static void schedule_pktio_start(int pktio, int count, int pktin[]) +static void schedule_pktio_start(int pktio, + int count, + int pktin[], + odp_queue_t odpq[] ODP_UNUSED) { int i, index; pktio_cmd_t *cmd; diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 765326e8..676b3419 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -40,42 +40,17 @@ #define LOCK(a) _odp_ticketlock_lock((a)) #define UNLOCK(a) _odp_ticketlock_unlock((a)) -#define TAG_EMPTY 0U -#define TAG_USED (1U << 15) -#define TAG_BUSY (1U << 31) -#define PKTIO_QUEUE_2_TAG(p, q) ((p) << 16 | (q) | TAG_USED) -#define TAG_2_PKTIO(t) (((t) >> 16) & 0x7FFF) -#define TAG_2_QUEUE(t) ((t) & 0x7FFF) -#define TAG_IS_READY(t) (((t) & (TAG_USED | TAG_BUSY)) == TAG_USED) -#define PKTIN_MAX (ODP_CONFIG_PKTIO_ENTRIES * PKTIO_MAX_QUEUES) #define MAXTHREADS ATOM_BITSET_SIZE +#define FLAG_PKTIN 0x80 + static _odp_ishm_pool_t *sched_shm_pool; -static uint32_t pktin_num; -static uint32_t pktin_hi; -static uint16_t pktin_count[ODP_CONFIG_PKTIO_ENTRIES]; -static uint32_t pktin_tags[PKTIN_MAX] ODP_ALIGNED_CACHE; - -#define __atomic_fetch_max(var, v, mo) do { \ - /* Evalulate 'v' once */ \ - __typeof__(v) tmp_v = (v); \ - __typeof__(*var) old_var = \ - __atomic_load_n((var), __ATOMIC_RELAXED); \ - while (tmp_v > old_var) { \ - /* Attempt to store 'v' in '*var' */ \ - if (__atomic_compare_exchange_n((var), &old_var, \ - tmp_v, true, (mo), \ - (mo))) \ - break; \ - } \ - /* v <= old_var, nothing to do */ \ - } while (0) - -ODP_STATIC_ASSERT(ODP_SCHED_PRIO_LOWEST == (ODP_SCHED_PRIO_NUM - 1), + +ODP_STATIC_ASSERT(ODP_SCHED_PRIO_LOWEST == (ODP_SCHED_PRIO_NUM - 2), "lowest_prio_does_not_match_with_num_prios"); ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && - (ODP_SCHED_PRIO_NORMAL < (ODP_SCHED_PRIO_NUM - 1)), + (ODP_SCHED_PRIO_NORMAL < (ODP_SCHED_PRIO_NUM - 2)), "normal_prio_is_not_between_highest_and_lowest"); ODP_STATIC_ASSERT(CHECK_IS_POWER2(ODP_CONFIG_QUEUES), @@ -99,9 +74,6 @@ static odp_spinlock_t sched_grp_lock; static sched_scalable_thread_state_t thread_state[MAXTHREADS]; __thread sched_scalable_thread_state_t *sched_ts; -/* - * Forward declarations. - */ static int thread_state_init(int tidx) { sched_scalable_thread_state_t *ts; @@ -110,13 +82,12 @@ static int thread_state_init(int tidx) ODP_ASSERT(tidx < MAXTHREADS); ts = &thread_state[tidx]; ts->atomq = NULL; + ts->src_schedq = NULL; ts->rctx = NULL; ts->pause = false; ts->out_of_order = false; ts->tidx = tidx; ts->dequeued = 0; - ts->pktin_next = 0; - ts->pktin_poll_cnts = 0; ts->ticket = TICKET_INVALID; ts->priv_rvec_free = 0; ts->rvec_free = (1ULL << TS_RVEC_SIZE) - 1; @@ -539,7 +510,25 @@ static inline void sched_update_popd(sched_elem_t *elem) } #endif -sched_queue_t *schedq_from_sched_group(odp_schedule_group_t grp, uint32_t prio) +static void signal_threads_add(sched_group_t *sg, uint32_t sgi, uint32_t prio) +{ + sched_group_mask_t thrds = sg->thr_wanted; + uint32_t thr; + + while (!bitset_is_null(thrds)) { + thr = bitset_ffs(thrds) - 1; + thrds = bitset_clr(thrds, thr); + /* Notify the thread about membership in this + * group/priority. + */ + atom_bitset_set(&thread_state[thr].sg_wanted[prio], + sgi, __ATOMIC_RELEASE); + __atomic_store_n(&thread_state[thr].sg_sem, 1, + __ATOMIC_RELEASE); + } +} + +sched_queue_t *sched_queue_add(odp_schedule_group_t grp, uint32_t prio) { uint32_t sgi; sched_group_t *sg; @@ -561,26 +550,46 @@ sched_queue_t *schedq_from_sched_group(odp_schedule_group_t grp, uint32_t prio) * Notify all threads in sg->thr_wanted that they * should join. */ - sched_group_mask_t thrds = sg->thr_wanted; + signal_threads_add(sg, sgi, prio); + } + return &sg->schedq[prio * sg->xfactor + x % sg->xfactor]; +} - while (!bitset_is_null(thrds)) { - uint32_t thr; +static uint32_t sched_pktin_add(odp_schedule_group_t grp, uint32_t prio) +{ + uint32_t sgi; + sched_group_t *sg; - thr = bitset_ffs(thrds) - 1; - thrds = bitset_clr(thrds, thr); - /* Notify the thread about membership in this - * group/priority. - */ - atom_bitset_set(&thread_state[thr].sg_wanted[prio], - sgi, __ATOMIC_RELEASE); - __atomic_store_n(&thread_state[thr].sg_sem, 1, - __ATOMIC_RELEASE); - } + ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); + ODP_ASSERT((sg_free & (1ULL << grp)) == 0); + ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); + + sgi = grp; + sg = sg_vec[sgi]; + + (void)sched_queue_add(grp, ODP_SCHED_PRIO_PKTIN); + return (ODP_SCHED_PRIO_PKTIN - prio) * sg->xfactor; +} + +static void signal_threads_rem(sched_group_t *sg, uint32_t sgi, uint32_t prio) +{ + sched_group_mask_t thrds = sg->thr_wanted; + uint32_t thr; + + while (!bitset_is_null(thrds)) { + thr = bitset_ffs(thrds) - 1; + thrds = bitset_clr(thrds, thr); + /* Notify the thread about membership in this + * group/priority. + */ + atom_bitset_clr(&thread_state[thr].sg_wanted[prio], + sgi, __ATOMIC_RELEASE); + __atomic_store_n(&thread_state[thr].sg_sem, 1, + __ATOMIC_RELEASE); } - return &sg->schedq[prio * sg->xfactor + x % sg->xfactor]; } -void sched_group_xcount_dec(odp_schedule_group_t grp, uint32_t prio) +void sched_queue_rem(odp_schedule_group_t grp, uint32_t prio) { uint32_t sgi; sched_group_t *sg; @@ -592,77 +601,83 @@ void sched_group_xcount_dec(odp_schedule_group_t grp, uint32_t prio) sgi = grp; sg = sg_vec[sgi]; - x = __atomic_sub_fetch(&sg->xcount[prio], 1, __ATOMIC_RELAXED); + x = __atomic_sub_fetch(&sg->xcount[prio], 1, __ATOMIC_RELAXED); if (x == 0) { /* Last ODP queue for this priority * Notify all threads in sg->thr_wanted that they * should leave. */ - sched_group_mask_t thrds = sg->thr_wanted; + signal_threads_rem(sg, sgi, prio); + } +} - while (!bitset_is_null(thrds)) { - uint32_t thr; +static void sched_pktin_rem(odp_schedule_group_t grp) +{ + sched_queue_rem(grp, ODP_SCHED_PRIO_PKTIN); +} - thr = bitset_ffs(thrds) - 1; - thrds = bitset_clr(thrds, thr); - /* Notify the thread about membership in this - * group/priority. +static void update_sg_add(sched_scalable_thread_state_t *ts, + uint32_t p, + sched_group_mask_t sg_wanted) +{ + sched_group_mask_t added; + uint32_t sgi; + sched_group_t *sg; + uint32_t x; + + added = bitset_andn(sg_wanted, ts->sg_actual[p]); + while (!bitset_is_null(added)) { + sgi = bitset_ffs(added) - 1; + sg = sg_vec[sgi]; + for (x = 0; x < sg->xfactor; x++) { + /* Include our thread index to shift + * (rotate) the order of schedq's */ - atom_bitset_clr(&thread_state[thr].sg_wanted[prio], - sgi, __ATOMIC_RELEASE); - __atomic_store_n(&thread_state[thr].sg_sem, 1, - __ATOMIC_RELEASE); + insert_schedq_in_list(ts, + &sg->schedq[p * sg->xfactor + + (x + ts->tidx) % sg->xfactor]); } + atom_bitset_set(&sg->thr_actual[p], ts->tidx, __ATOMIC_RELAXED); + added = bitset_clr(added, sgi); } } -static void update_sg_membership(sched_scalable_thread_state_t *ts) +static void update_sg_rem(sched_scalable_thread_state_t *ts, + uint32_t p, + sched_group_mask_t sg_wanted) { - uint32_t p; - sched_group_mask_t sg_wanted; - sched_group_mask_t added; sched_group_mask_t removed; uint32_t sgi; sched_group_t *sg; uint32_t x; + removed = bitset_andn(ts->sg_actual[p], sg_wanted); + while (!bitset_is_null(removed)) { + sgi = bitset_ffs(removed) - 1; + sg = sg_vec[sgi]; + for (x = 0; x < sg->xfactor; x++) { + remove_schedq_from_list(ts, + &sg->schedq[p * + sg->xfactor + x]); + } + atom_bitset_clr(&sg->thr_actual[p], ts->tidx, __ATOMIC_RELAXED); + removed = bitset_clr(removed, sgi); + } +} + +static void update_sg_membership(sched_scalable_thread_state_t *ts) +{ + uint32_t p; + sched_group_mask_t sg_wanted; + for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { sg_wanted = atom_bitset_load(&ts->sg_wanted[p], __ATOMIC_ACQUIRE); if (!bitset_is_eql(ts->sg_actual[p], sg_wanted)) { /* Our sched_group membership has changed */ - added = bitset_andn(sg_wanted, ts->sg_actual[p]); - while (!bitset_is_null(added)) { - sgi = bitset_ffs(added) - 1; - sg = sg_vec[sgi]; - for (x = 0; x < sg->xfactor; x++) { - /* Include our thread index to shift - * (rotate) the order of schedq's - */ - insert_schedq_in_list - (ts, - &sg->schedq[p * sg->xfactor + - (x + ts->tidx) % sg->xfactor]); - } - atom_bitset_set(&sg->thr_actual[p], ts->tidx, - __ATOMIC_RELAXED); - added = bitset_clr(added, sgi); - } - removed = bitset_andn(ts->sg_actual[p], sg_wanted); - while (!bitset_is_null(removed)) { - sgi = bitset_ffs(removed) - 1; - sg = sg_vec[sgi]; - for (x = 0; x < sg->xfactor; x++) { - remove_schedq_from_list - (ts, - &sg->schedq[p * - sg->xfactor + x]); - } - atom_bitset_clr(&sg->thr_actual[p], ts->tidx, - __ATOMIC_RELAXED); - removed = bitset_clr(removed, sgi); - } + update_sg_add(ts, p, sg_wanted); + update_sg_rem(ts, p, sg_wanted); ts->sg_actual[p] = sg_wanted; } } @@ -693,61 +708,180 @@ static inline void _schedule_release_ordered(sched_scalable_thread_state_t *ts) ts->rctx = NULL; } -static void pktin_poll(sched_scalable_thread_state_t *ts) +static uint16_t poll_count[ODP_CONFIG_PKTIO_ENTRIES]; + +static void pktio_start(int pktio_idx, + int num_in_queue, + int in_queue_idx[], + odp_queue_t odpq[]) { - uint32_t i, tag, hi, npolls = 0; - int pktio_index, queue_index; + int i, rxq; + queue_entry_t *qentry; + sched_elem_t *elem; - hi = __atomic_load_n(&pktin_hi, __ATOMIC_RELAXED); - if (hi == 0) - return; + ODP_ASSERT(pktio_idx < ODP_CONFIG_PKTIO_ENTRIES); + for (i = 0; i < num_in_queue; i++) { + rxq = in_queue_idx[i]; + ODP_ASSERT(rxq < PKTIO_MAX_QUEUES); + __atomic_fetch_add(&poll_count[pktio_idx], 1, __ATOMIC_RELAXED); + qentry = qentry_from_ext(odpq[i]); + elem = &qentry->s.sched_elem; + elem->cons_type |= FLAG_PKTIN; /* Set pktin queue flag */ + elem->pktio_idx = pktio_idx; + elem->rx_queue = rxq; + elem->xoffset = sched_pktin_add(elem->sched_grp, + elem->sched_prio); + ODP_ASSERT(elem->schedq != NULL); + schedq_push(elem->schedq, elem); + } +} - for (i = ts->pktin_next; npolls != hi; i = (i + 1) % hi, npolls++) { - tag = __atomic_load_n(&pktin_tags[i], __ATOMIC_RELAXED); - if (!TAG_IS_READY(tag)) - continue; - if (!__atomic_compare_exchange_n(&pktin_tags[i], &tag, - tag | TAG_BUSY, - true, - __ATOMIC_ACQUIRE, - __ATOMIC_RELAXED)) - continue; - /* Tag grabbed */ - pktio_index = TAG_2_PKTIO(tag); - queue_index = TAG_2_QUEUE(tag); - if (odp_unlikely(sched_cb_pktin_poll(pktio_index, - 1, &queue_index))) { - /* Pktio stopped or closed - * Remove tag from pktin_tags - */ - __atomic_store_n(&pktin_tags[i], - TAG_EMPTY, __ATOMIC_RELAXED); - __atomic_fetch_sub(&pktin_num, - 1, __ATOMIC_RELEASE); - /* Call stop_finalize when all queues - * of the pktio have been removed - */ - if (__atomic_sub_fetch(&pktin_count[pktio_index], 1, - __ATOMIC_RELAXED) == 0) - sched_cb_pktio_stop_finalize(pktio_index); - } else { - /* We don't know whether any packets were found and enqueued - * Write back original tag value to release pktin queue - */ - __atomic_store_n(&pktin_tags[i], tag, __ATOMIC_RELAXED); - /* Do not iterate through all pktin queues every time */ - if ((ts->pktin_poll_cnts & 0xf) != 0) - break; +static void pktio_stop(sched_elem_t *elem) +{ + elem->cons_type &= ~FLAG_PKTIN; /* Clear pktin queue flag */ + sched_pktin_rem(elem->sched_grp); + if (__atomic_sub_fetch(&poll_count[elem->pktio_idx], + 1, __ATOMIC_RELAXED) == 0) { + /* Call stop_finalize when all queues + * of the pktio have been removed */ + sched_cb_pktio_stop_finalize(elem->pktio_idx); + } +} + +static bool have_reorder_ctx(sched_scalable_thread_state_t *ts) +{ + if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) { + ts->priv_rvec_free = atom_bitset_xchg(&ts->rvec_free, 0, + __ATOMIC_RELAXED); + if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) { + /* No free reorder contexts for this thread */ + return false; } } - ODP_ASSERT(i < hi); - ts->pktin_poll_cnts++; - ts->pktin_next = i; + return true; +} + +static inline bool is_pktin(sched_elem_t *elem) +{ + return (elem->cons_type & FLAG_PKTIN) != 0; +} + +static inline bool is_atomic(sched_elem_t *elem) +{ + return elem->cons_type == (ODP_SCHED_SYNC_ATOMIC | FLAG_PKTIN); +} + +static inline bool is_ordered(sched_elem_t *elem) +{ + return elem->cons_type == (ODP_SCHED_SYNC_ORDERED | FLAG_PKTIN); +} + +static int poll_pktin(sched_elem_t *elem, odp_event_t ev[], int num_evts) +{ + sched_scalable_thread_state_t *ts = sched_ts; + int num, i; + /* For ordered queues only */ + reorder_context_t *rctx; + reorder_window_t *rwin = NULL; + uint32_t sn; + uint32_t idx; + + if (is_ordered(elem)) { + /* Need reorder context and slot in reorder window */ + rwin = queue_get_rwin((queue_entry_t *)elem); + ODP_ASSERT(rwin != NULL); + if (odp_unlikely(!have_reorder_ctx(ts) || + !rwin_reserve_sc(rwin, &sn))) { + /* Put back queue on source schedq */ + schedq_push(ts->src_schedq, elem); + return 0; + } + /* Slot in reorder window reserved! */ + } + + /* Try to dequeue events from the ingress queue itself */ + num = _odp_queue_deq_sc(elem, ev, num_evts); + if (odp_likely(num > 0)) { +events_dequeued: + if (is_atomic(elem)) { + ts->atomq = elem; /* Remember */ + ts->dequeued += num; + /* Don't push atomic queue on schedq */ + } else /* Parallel or ordered */ { + if (is_ordered(elem)) { + /* Find and initialise an unused reorder + * context. */ + idx = bitset_ffs(ts->priv_rvec_free) - 1; + ts->priv_rvec_free = + bitset_clr(ts->priv_rvec_free, idx); + rctx = &ts->rvec[idx]; + rctx_init(rctx, idx, rwin, sn); + /* Are we in-order or out-of-order? */ + ts->out_of_order = sn != rwin->hc.head; + ts->rctx = rctx; + } + schedq_push(elem->schedq, elem); + } + return num; + } + + /* Ingress queue empty => poll pktio RX queue */ + odp_event_t rx_evts[QUEUE_MULTI_MAX]; + int num_rx = sched_cb_pktin_poll_one(elem->pktio_idx, + elem->rx_queue, + rx_evts); + if (odp_likely(num_rx > 0)) { + num = num_rx < num_evts ? num_rx : num_evts; + for (i = 0; i < num; i++) { + /* Return events directly to caller */ + ev[i] = rx_evts[i]; + } + if (num_rx > num) { + /* Events remain, enqueue them */ + odp_buffer_hdr_t *bufs[QUEUE_MULTI_MAX]; + + for (i = num; i < num_rx; i++) + bufs[i] = + (odp_buffer_hdr_t *)(void *)rx_evts[i]; + i = _odp_queue_enq_sp(elem, &bufs[num], num_rx - num); + /* Enqueue must succeed as the queue was empty */ + ODP_ASSERT(i == num_rx - num); + } + goto events_dequeued; + } + /* No packets received, reset state and undo side effects */ + if (is_atomic(elem)) + ts->atomq = NULL; + else if (is_ordered(elem)) + rwin_unreserve_sc(rwin, sn); + + if (odp_likely(num_rx == 0)) { + /* RX queue empty, push it to pktin priority schedq */ + sched_queue_t *schedq = ts->src_schedq; + /* Check if queue came from the designated schedq */ + if (schedq == elem->schedq) { + /* Yes, add offset to the pktin priority level + * in order to get alternate schedq */ + schedq += elem->xoffset; + } + /* Else no, queue must have come from alternate schedq */ + schedq_push(schedq, elem); + } else /* num_rx < 0 => pktio stopped or closed */ { + /* Remove queue */ + pktio_stop(elem); + /* Don't push queue to schedq */ + } + + ODP_ASSERT(ts->atomq == NULL); + ODP_ASSERT(!ts->out_of_order); + ODP_ASSERT(ts->rctx == NULL); + return 0; } static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) { sched_scalable_thread_state_t *ts; + sched_elem_t *first; sched_elem_t *atomq; int num; uint32_t i; @@ -758,7 +892,23 @@ static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) /* Once an atomic queue has been scheduled to a thread, it will stay * on that thread until empty or 'rotated' by WRR */ - if (atomq != NULL) { + if (atomq != NULL && is_pktin(atomq)) { + /* Atomic pktin queue */ + if (ts->dequeued < atomq->qschst.wrr_budget) { + ODP_ASSERT(ts->src_schedq != NULL); + num = poll_pktin(atomq, ev, num_evts); + if (odp_likely(num != 0)) { + if (from) + *from = queue_get_handle( + (queue_entry_t *)atomq); + return num; + } + } else { + /* WRR budget exhausted, move queue to end of schedq */ + schedq_push(atomq->schedq, atomq); + } + ts->atomq = NULL; + } else if (atomq != NULL) { ODP_ASSERT(ts->ticket != TICKET_INVALID); #ifdef CONFIG_QSCHST_LOCK LOCK(&atomq->qschlock); @@ -794,10 +944,6 @@ dequeue_atomic: #endif } - /* Release any previous reorder context. */ - if (ts->rctx != NULL) - _schedule_release_ordered(ts); - /* Check for and perform any scheduler group updates. */ if (odp_unlikely(__atomic_load_n(&ts->sg_sem, __ATOMIC_RELAXED) != 0)) { (void)__atomic_load_n(&ts->sg_sem, __ATOMIC_ACQUIRE); @@ -806,7 +952,7 @@ dequeue_atomic: } /* Scan our schedq list from beginning to end */ - for (i = 0; i < ts->num_schedq; i++) { + for (i = 0, first = NULL; i < ts->num_schedq; i++, first = NULL) { sched_queue_t *schedq = ts->schedq_list[i]; sched_elem_t *elem; restart_same: @@ -815,8 +961,24 @@ restart_same: /* Schedq empty, look at next one. */ continue; } + if (is_pktin(elem)) { + /* Pktio ingress queue */ + if (first == NULL) + first = elem; + else if (elem == first) /* Wrapped around */ + continue; /* Go to next schedq */ + + if (odp_unlikely(!schedq_cond_pop(schedq, elem))) + goto restart_same; - if (elem->cons_type == ODP_SCHED_SYNC_ATOMIC) { + ts->src_schedq = schedq; /* Remember source schedq */ + num = poll_pktin(elem, ev, num_evts); + if (odp_unlikely(num <= 0)) + goto restart_same; + if (from) + *from = queue_get_handle((queue_entry_t *)elem); + return num; + } else if (elem->cons_type == ODP_SCHED_SYNC_ATOMIC) { /* Dequeue element only if it is still at head * of schedq. */ @@ -885,19 +1047,9 @@ restart_same: * collect all outgoing events. Ensure there is at least * one available reorder context. */ - if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) { - ts->priv_rvec_free = atom_bitset_xchg( - &ts->rvec_free, 0, - __ATOMIC_RELAXED); - if (odp_unlikely(bitset_is_null( - ts->priv_rvec_free))) { - /* No free reorder contexts for - * this thread. Look at next schedq, - * hope we find non-ordered queue. - */ - continue; - } - } + if (odp_unlikely(!have_reorder_ctx(ts))) + continue; + /* rwin_reserve and odp_queue_deq must be atomic or * there will be a potential race condition. * Allocate a slot in the reorder window. @@ -984,7 +1136,6 @@ restart_same: } } - pktin_poll(ts); return 0; } @@ -1066,6 +1217,10 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t ev[], odp_time_t deadline; ts = sched_ts; + /* Release any previous reorder context. */ + if (ts->rctx != NULL) + _schedule_release_ordered(ts); + if (odp_unlikely(ts->pause)) { if (ts->atomq != NULL) { #ifdef CONFIG_QSCHST_LOCK @@ -1078,8 +1233,6 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t ev[], #ifdef CONFIG_QSCHST_LOCK UNLOCK(&atomq->qschlock); #endif - } else if (ts->rctx != NULL) { - _schedule_release_ordered(ts); } return 0; } @@ -1124,6 +1277,10 @@ static odp_event_t schedule(odp_queue_t *from, uint64_t wait) odp_time_t deadline; ts = sched_ts; + /* Release any previous reorder context. */ + if (ts->rctx != NULL) + _schedule_release_ordered(ts); + if (odp_unlikely(ts->pause)) { if (ts->atomq != NULL) { #ifdef CONFIG_QSCHST_LOCK @@ -1136,8 +1293,6 @@ static odp_event_t schedule(odp_queue_t *from, uint64_t wait) #ifdef CONFIG_QSCHST_LOCK UNLOCK(&atomq->qschlock); #endif - } else if (ts->rctx != NULL) { - _schedule_release_ordered(ts); } return ev; } @@ -1190,7 +1345,7 @@ static uint64_t schedule_wait_time(uint64_t ns) static int schedule_num_prio(void) { - return ODP_SCHED_PRIO_NUM; + return ODP_SCHED_PRIO_NUM - 1; /* Discount the pktin priority level */ } static int schedule_group_update(sched_group_t *sg, @@ -1387,7 +1542,7 @@ static int schedule_group_destroy(odp_schedule_group_t group) ret = -1; goto thrd_q_present_in_group; } - if (sg->xcount[p] != 0) { + if (p != ODP_SCHED_PRIO_PKTIN && sg->xcount[p] != 0) { ODP_ERR("Group has queues\n"); ret = -1; goto thrd_q_present_in_group; @@ -1783,51 +1938,6 @@ static int schedule_term_local(void) return rc; } -static void pktio_start(int pktio_index, int num_in_queue, int in_queue_idx[]) -{ - int i; - uint32_t old, tag, j; - - for (i = 0; i < num_in_queue; i++) { - /* Try to reserve a slot */ - if (__atomic_fetch_add(&pktin_num, - 1, __ATOMIC_RELAXED) >= PKTIN_MAX) { - __atomic_fetch_sub(&pktin_num, 1, __ATOMIC_RELAXED); - ODP_ABORT("Too many pktio in queues for scheduler\n"); - } - /* A slot has been reserved, now we need to find an empty one */ - for (j = 0; ; j = (j + 1) % PKTIN_MAX) { - if (__atomic_load_n(&pktin_tags[j], - __ATOMIC_RELAXED) != TAG_EMPTY) - /* Slot used, continue with next */ - continue; - /* Empty slot found */ - old = TAG_EMPTY; - tag = PKTIO_QUEUE_2_TAG(pktio_index, in_queue_idx[i]); - if (__atomic_compare_exchange_n(&pktin_tags[j], - &old, - tag, - true, - __ATOMIC_RELEASE, - __ATOMIC_RELAXED)) { - /* Success grabbing slot,update high - * watermark - */ - __atomic_fetch_max(&pktin_hi, - j + 1, __ATOMIC_RELAXED); - /* One more tag (queue) for this pktio - * instance - */ - __atomic_fetch_add(&pktin_count[pktio_index], - 1, __ATOMIC_RELAXED); - /* Continue with next RX queue */ - break; - } - /* Failed to grab slot */ - } - } -} - static int num_grps(void) { return MAX_SCHED_GROUP; diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c index 80ff2490..7261b446 100644 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -72,6 +72,32 @@ bool rwin_reserve(reorder_window_t *rwin, uint32_t *sn) return true; } +bool rwin_reserve_sc(reorder_window_t *rwin, uint32_t *sn) +{ + uint32_t head; + uint32_t oldt; + uint32_t newt; + uint32_t winmask; + + /* Read head and tail separately */ + oldt = rwin->tail; + winmask = rwin->winmask; + head = rwin->hc.head; + if (odp_unlikely(oldt - head >= winmask)) + return false; + newt = oldt + 1; + rwin->tail = newt; + *sn = oldt; + + return true; +} + +void rwin_unreserve_sc(reorder_window_t *rwin, uint32_t sn) +{ + ODP_ASSERT(rwin->tail == sn + 1); + rwin->tail = sn; +} + static void rwin_insert(reorder_window_t *rwin, reorder_context_t *rctx, uint32_t sn, diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 05241275..47e6ddd3 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -424,7 +424,10 @@ static int ord_enq_multi(queue_t q_int, void *buf_hdr[], int num, return 0; } -static void pktio_start(int pktio_index, int num, int pktin_idx[]) +static void pktio_start(int pktio_index, + int num, + int pktin_idx[], + odp_queue_t odpq[] ODP_UNUSED) { int i; sched_cmd_t *cmd; ----------------------------------------------------------------------- Summary of changes: .../include/odp_queue_scalable_internal.h | 2 + platform/linux-generic/include/odp_schedule_if.h | 11 +- .../linux-generic/include/odp_schedule_scalable.h | 25 +- .../include/odp_schedule_scalable_ordered.h | 2 + platform/linux-generic/odp_packet_io.c | 49 +- platform/linux-generic/odp_queue_scalable.c | 32 +- platform/linux-generic/odp_schedule.c | 2 +- platform/linux-generic/odp_schedule_iquery.c | 5 +- platform/linux-generic/odp_schedule_scalable.c | 540 +++++++++++++-------- .../linux-generic/odp_schedule_scalable_ordered.c | 26 + platform/linux-generic/odp_schedule_sp.c | 5 +- 11 files changed, 458 insertions(+), 241 deletions(-) hooks/post-receive --

7 years, 9 months

[COMMIT] branch api-next updated. v1.15.0.0-674-g32f5f3ff

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 32f5f3ff343b516cb0dbe98e81479e6c748f0ebf (commit) via 3c9248f5c98b194d91712fafed4bf3b21327f2a8 (commit) via 91ef9f7e224056af351cbaf99ccfa98ee815460d (commit) via ba97c3174d0de3e08cffaad414bf2a55de8853df (commit) via 69a1179a05344b59961aaf997a1b406698b6840b (commit) via 6efe7d971a3a4b8f7eaa42660b48eee31493924c (commit) via ef4e4d4c4ef1f9f569c236494cb178feb7c90343 (commit) via e4289ab4f73221a9d20ecfb8eac6b79a26df06a3 (commit) via a787abf96b17d9714a6b892091d19c1be2bc5e6c (commit) via 20c75b764af6fd15e5e1d4df969ac33d62525405 (commit) via 04029613fbdc85221d1a20354a49ada912302fc0 (commit) via 88d36e6811de494708c520cb12e9e5f97628e9e4 (commit) via a820d8a84e132652b4cef295756ccf135e3bd54b (commit) via f7c5f3e973814e0fc9211e008a37080b1c7d4a76 (commit) via d445395c66edb38abedb918afdba37ec9f7f95af (commit) via b38117eeed5d7bd5a334ef2387bf83d5cb8b9188 (commit) via d0821c9c43fb89e4c0021b174bbc12bef543ffa0 (commit) via 17e829a3b78d6513d22496734c6edbec955cbfd4 (commit) via eccc78540c05dc71095179b613f014648385f3df (commit) via f8c6360667380f6b3fb917ecbe5b941b23dbadf1 (commit) from 6b6253c30f88c80bf632436ff06c1b000860a2f1 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 32f5f3ff343b516cb0dbe98e81479e6c748f0ebf Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Sep 14 22:29:44 2017 +0300 linux-gen: move packet_to_buffer to packet_inlines.h odp_crypto.c is also uses packet_to_buffer() so move this function to common place. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 6874d149..be7e18ec 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -157,6 +157,11 @@ static inline void _odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, (void)pkt; (void)offset; (void)len; } +static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) +{ + return (odp_buffer_t)pkt; +} + /* Include inlined versions of API functions */ #include <odp/api/plat/static_inline.h> #if ODP_ABI_COMPAT == 0 diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index 081fe178..3174feee 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -19,6 +19,7 @@ #include <odp_debug_internal.h> #include <odp/api/hints.h> #include <odp/api/random.h> +#include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <string.h> diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 5dbe65d3..ff0cb8fe 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -69,11 +69,6 @@ odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)buf_hdr; } -static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) -{ - return (odp_buffer_t)pkt; -} - static inline seg_entry_t *seg_entry(odp_packet_hdr_t *hdr, uint32_t seg_idx) { commit 3c9248f5c98b194d91712fafed4bf3b21327f2a8 Merge: 6b6253c3 91ef9f7e Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Sep 14 17:56:51 2017 +0300 Merge branch 'master' into api-next Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --cc .travis.yml index 0b5fb87d,43141f9f..8cf6ed22 --- a/.travis.yml +++ b/.travis.yml @@@ -67,8 -66,11 +67,12 @@@ env - CONF="--disable-abi-compat" - CONF="--enable-schedule-sp" - CONF="--enable-schedule-iquery" + - CONF="--enable-schedule-scalable" - CONF="--enable-dpdk-zero-copy" + - CROSS_ARCH="arm64" + - CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" + - CROSS_ARCH="powerpc" + - CROSS_ARCH="i386" before_install: diff --cc platform/linux-generic/include/odp_buffer_internal.h index fe0c26b9,5d40303b..358c0e44 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@@ -80,15 -107,9 +107,12 @@@ struct odp_buffer_hdr_t /* User area size */ uint32_t uarea_size; - /* Event type. Maybe different than pool type (crypto compl event) */ - int8_t event_type; + /* Max data size */ + uint32_t size; + /* Event subtype. Should be ODP_EVENT_NO_SUBTYPE except packets. */ + int8_t event_subtype; + - /* Burst table */ - struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE]; - /* ipc mapped process can not walk over pointers, * offset has to be used */ uint64_t ipc_data_offset; diff --cc platform/linux-generic/include/odp_packet_internal.h index 91fba1ea,d76d7bf0..09726423 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@@ -272,11 -230,6 +230,8 @@@ static inline void packet_init(odp_pack CONFIG_PACKET_TAILROOM; pkt_hdr->input = ODP_PKTIO_INVALID; + pkt_hdr->buf_hdr.event_subtype = ODP_EVENT_PACKET_BASIC; + - /* By default packet has no references */ - pkt_hdr->unshared_len = len; - pkt_hdr->ref_hdr = NULL; } static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, ----------------------------------------------------------------------- Summary of changes: .travis.yml | 134 +- include/odp/arch/default/api/abi/packet.h | 2 +- .../include/odp/api/plat/packet_inlines.h | 49 +- .../include/odp/api/plat/packet_inlines_api.h | 16 + .../include/odp/api/plat/packet_types.h | 38 +- .../linux-generic/include/odp_buffer_internal.h | 71 +- .../linux-generic/include/odp_packet_internal.h | 133 +- platform/linux-generic/odp_crypto.c | 1 + platform/linux-generic/odp_packet.c | 1514 +++++++++----------- platform/linux-generic/odp_pool.c | 8 + test/common_plat/validation/api/packet/packet.c | 109 +- 11 files changed, 979 insertions(+), 1096 deletions(-) hooks/post-receive --

7 years, 9 months

[COMMIT] branch master updated. v1.15.0.0-204-ga38ea82c

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via a38ea82cf1ae45b7c807164dde5783d099efd39d (commit) from 91ef9f7e224056af351cbaf99ccfa98ee815460d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit a38ea82cf1ae45b7c807164dde5783d099efd39d Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Thu Sep 14 22:29:44 2017 +0300 linux-gen: move packet_to_buffer to packet_inlines.h odp_crypto.c is also uses packet_to_buffer() so move this function to common place. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 6874d149..be7e18ec 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -157,6 +157,11 @@ static inline void _odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, (void)pkt; (void)offset; (void)len; } +static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) +{ + return (odp_buffer_t)pkt; +} + /* Include inlined versions of API functions */ #include <odp/api/plat/static_inline.h> #if ODP_ABI_COMPAT == 0 diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index c7da548a..40b6d74b 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -19,6 +19,7 @@ #include <odp_debug_internal.h> #include <odp/api/hints.h> #include <odp/api/random.h> +#include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <string.h> diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index d2a87f0f..f4a0e0ec 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -69,11 +69,6 @@ odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)buf_hdr; } -static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) -{ - return (odp_buffer_t)pkt; -} - static inline seg_entry_t *seg_entry(odp_packet_hdr_t *hdr, uint32_t seg_idx) { ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/include/odp/api/plat/packet_inlines.h | 5 +++++ platform/linux-generic/odp_crypto.c | 1 + platform/linux-generic/odp_packet.c | 5 ----- 3 files changed, 6 insertions(+), 5 deletions(-) hooks/post-receive --

7 years, 9 months

[COMMIT] branch master updated. v1.15.0.0-203-g91ef9f7e

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 91ef9f7e224056af351cbaf99ccfa98ee815460d (commit) via ba97c3174d0de3e08cffaad414bf2a55de8853df (commit) via 69a1179a05344b59961aaf997a1b406698b6840b (commit) via 6efe7d971a3a4b8f7eaa42660b48eee31493924c (commit) via ef4e4d4c4ef1f9f569c236494cb178feb7c90343 (commit) via e4289ab4f73221a9d20ecfb8eac6b79a26df06a3 (commit) via a787abf96b17d9714a6b892091d19c1be2bc5e6c (commit) via 20c75b764af6fd15e5e1d4df969ac33d62525405 (commit) via 04029613fbdc85221d1a20354a49ada912302fc0 (commit) via 88d36e6811de494708c520cb12e9e5f97628e9e4 (commit) via a820d8a84e132652b4cef295756ccf135e3bd54b (commit) from f7c5f3e973814e0fc9211e008a37080b1c7d4a76 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 91ef9f7e224056af351cbaf99ccfa98ee815460d Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Tue Sep 5 09:40:59 2017 +0300 linux-gen: packet: optimize header layout Pack most often used data into the first and the last cache line of buffer header, and the first cache line of packet header. Reduce offsets and head-/tailroom fields to 16 bits, since those are in maximum as large as a segment, usually much less. Pack header fields into correct alignments (when possible), so that holes are avoided. These changes reduce packet header size by one cache line. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/include/odp/arch/default/api/abi/packet.h b/include/odp/arch/default/api/abi/packet.h index 4aac75b9..15cf081b 100644 --- a/include/odp/arch/default/api/abi/packet.h +++ b/include/odp/arch/default/api/abi/packet.h @@ -28,7 +28,7 @@ typedef _odp_abi_packet_seg_t *odp_packet_seg_t; #define ODP_PACKET_INVALID ((odp_packet_t)0xffffffff) #define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)0xffffffff) -#define ODP_PACKET_OFFSET_INVALID (0x0fffffff) +#define ODP_PACKET_OFFSET_INVALID 0xffff typedef enum { ODP_PACKET_GREEN = 0, diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index d0cf1390..6874d149 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -56,13 +56,13 @@ static inline uint32_t _odp_packet_len(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_headroom(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint32_t, headroom); + return _odp_pkt_get(pkt, uint16_t, headroom); } /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_tailroom(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint32_t, tailroom); + return _odp_pkt_get(pkt, uint16_t, tailroom); } /** @internal Inline function @param pkt @return */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 96379ebf..68c66312 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -36,7 +36,7 @@ typedef ODP_HANDLE_T(odp_packet_t); #define ODP_PACKET_INVALID _odp_cast_scalar(odp_packet_t, 0) -#define ODP_PACKET_OFFSET_INVALID (0x0fffffff) +#define ODP_PACKET_OFFSET_INVALID 0xffff typedef uint8_t odp_packet_seg_t; diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index cd067a08..5d40303b 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -65,13 +65,10 @@ struct odp_buffer_hdr_t { /* Initial buffer data pointer */ uint8_t *base_data; - /* Reference count */ - odp_atomic_u32_t ref_cnt; - - /* Event type. Maybe different than pool type (crypto compl event) */ - int8_t event_type; + /* Pool pointer */ + void *pool_ptr; - /* --- 37 bytes --- */ + /* --- 40 bytes --- */ /* Segments */ seg_entry_t seg[CONFIG_PACKET_MAX_SEGS]; @@ -95,8 +92,11 @@ struct odp_buffer_hdr_t { const void *buf_cctx; /* const alias for ctx */ }; - /* Pool pointer */ - void *pool_ptr; + /* Reference count */ + odp_atomic_u32_t ref_cnt; + + /* Event type. Maybe different than pool type (crypto compl event) */ + int8_t event_type; /* Initial buffer tail pointer */ uint8_t *buf_end; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index d8d4584a..d76d7bf0 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -88,9 +88,14 @@ typedef struct { error_flags_t error_flags; output_flags_t output_flags; - uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ - uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ - uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ + /* offset to L2 hdr, e.g. Eth */ + uint16_t l2_offset; + + /* offset to L3 hdr, e.g. IPv4, IPv6 */ + uint16_t l3_offset; + + /* offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ + uint16_t l4_offset; } packet_parser_t; /* Packet extra data length */ @@ -116,14 +121,14 @@ typedef struct { packet_parser_t p; - uint32_t frame_len; - odp_pktio_t input; - uint32_t headroom; - uint32_t tailroom; + uint32_t frame_len; uint32_t shared_len; + uint16_t headroom; + uint16_t tailroom; + /* * Members below are not initialized by packet_init() */ diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 2f65cb20..2a0a35e4 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -41,6 +41,9 @@ ODP_STATIC_ASSERT(CONFIG_POOL_CACHE_SIZE > (2 * CACHE_BURST), ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_LEN_MIN >= 256, "ODP Segment size must be a minimum of 256 bytes"); +ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_SIZE < 0xffff, + "Segment size must be less than 64k (16 bit offsets)"); + /* Thread local variables */ typedef struct pool_local_t { pool_cache_t *cache[ODP_CONFIG_POOLS]; commit ba97c3174d0de3e08cffaad414bf2a55de8853df Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Aug 31 15:35:12 2017 +0300 linux-gen: packet: implement dynamic references Use link headers to implement dynamic references. These are segment headers which first segment (seg[0]) does not point to itself but to a referenced segment. Link headers enable long chains of references (new dynamic references from previous references). Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 5990f878..d2a87f0f 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -130,6 +130,43 @@ static inline seg_entry_t *seg_entry_next(odp_packet_hdr_t **cur_hdr, return &hdr->buf_hdr.seg[idx]; } +static inline void seg_entry_find_offset(odp_packet_hdr_t **p_hdr, + uint8_t *p_idx, + uint32_t *seg_offset, + uint32_t *seg_idx, + uint32_t offset) +{ + int i; + odp_packet_hdr_t *hdr, *cur_hdr; + uint8_t idx, cur_idx; + seg_entry_t *seg = NULL; + uint32_t seg_start = 0, seg_end = 0; + int seg_count; + + hdr = *p_hdr; + cur_hdr = hdr; + idx = 0; + cur_idx = 0; + seg_count = hdr->buf_hdr.segcount; + + for (i = 0; i < seg_count; i++) { + cur_hdr = hdr; + cur_idx = idx; + seg = seg_entry_next(&hdr, &idx); + seg_end += seg->len; + + if (odp_likely(offset < seg_end)) + break; + + seg_start = seg_end; + } + + *p_hdr = cur_hdr; + *p_idx = cur_idx; + *seg_offset = offset - seg_start; + *seg_idx = i; +} + static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { @@ -459,6 +496,7 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, new_hdr->frame_len = pkt_hdr->frame_len + len; new_hdr->headroom = pool->headroom + offset; new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->shared_len = pkt_hdr->shared_len; pkt_hdr = new_hdr; } else { @@ -478,20 +516,11 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, return pkt_hdr; } -static inline void copy_buf_hdr(odp_packet_hdr_t *pkt_hdr, int first, int num, - odp_buffer_hdr_t *buf_hdr[]) +static inline int seg_is_link(void *hdr) { - seg_entry_t *seg; - int i; - uint8_t idx; - odp_packet_hdr_t *hdr = pkt_hdr; - - seg_entry_find_idx(&hdr, &idx, first); + odp_packet_hdr_t *pkt_hdr = hdr; - for (i = 0; i < num; i++) { - seg = seg_entry_next(&hdr, &idx); - buf_hdr[i] = seg->hdr; - } + return pkt_hdr != pkt_hdr->buf_hdr.seg[0].hdr; } static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) @@ -553,9 +582,14 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) } } + /* Reset link header back to normal header */ + if (odp_unlikely(seg_is_link(hdr[i]))) + hdr[i]->seg[0].hdr = hdr[i]; + /* Skip references and pack to be freed headers to array head */ if (odp_unlikely(num_ref)) hdr[i - num_ref] = hdr[i]; + } num -= num_ref; @@ -566,14 +600,36 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { - seg_entry_t *seg; int i; - odp_buffer_hdr_t *buf_hdr[num]; - uint8_t idx = 0; + odp_buffer_hdr_t *buf_hdr[num + 1]; - for (i = 0; i < num; i++) { - seg = seg_entry_next(&pkt_hdr, &idx); - buf_hdr[i] = seg->hdr; + if (odp_likely(pkt_hdr->buf_hdr.num_seg == num)) { + for (i = 0; i < num; i++) + buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr; + + if (odp_unlikely(seg_is_link(pkt_hdr))) { + buf_hdr[num] = &pkt_hdr->buf_hdr; + num++; + } + } else { + seg_entry_t *seg; + odp_buffer_hdr_t *link_hdr[num]; + uint8_t idx = 0; + int links = 0; + + for (i = 0; i < num; i++) { + /* Free also link headers */ + if (odp_unlikely(idx == 0 && seg_is_link(pkt_hdr))) { + link_hdr[links] = &pkt_hdr->buf_hdr; + links++; + } + + seg = seg_entry_next(&pkt_hdr, &idx); + buf_hdr[i] = seg->hdr; + } + + if (odp_unlikely(links)) + packet_free_multi(link_hdr, links); } packet_free_multi(buf_hdr, num); @@ -591,14 +647,24 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, uint8_t idx; uint8_t num_seg; odp_buffer_hdr_t *buf_hdr[num]; + odp_buffer_hdr_t *link_hdr[num]; + odp_packet_hdr_t *tmp_hdr; + int links = 0; if (head) { odp_packet_hdr_t *new_hdr; idx = 0; for (i = 0; i < num; i++) { + tmp_hdr = hdr; seg = seg_entry_next(&hdr, &idx); buf_hdr[i] = seg->hdr; + + /* Free link headers, if those become empty */ + if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { + link_hdr[links] = &tmp_hdr->buf_hdr; + links++; + } } /* The first remaining header is the new packet descriptor. @@ -626,13 +692,23 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, /* Tailroom not changed */ new_hdr->tailroom = pkt_hdr->tailroom; - new_hdr->headroom = seg_headroom(new_hdr, 0); - new_hdr->frame_len = pkt_hdr->frame_len - free_len; + + /* Link header does not have headroom */ + if (seg_is_link(new_hdr)) + new_hdr->headroom = 0; + else + new_hdr->headroom = seg_headroom(new_hdr, 0); + + new_hdr->frame_len = pkt_hdr->frame_len - free_len; + new_hdr->shared_len = pkt_hdr->shared_len; pull_head(new_hdr, pull_len); pkt_hdr = new_hdr; + if (odp_unlikely(links)) + packet_free_multi(link_hdr, links); + packet_free_multi(buf_hdr, num); } else { /* Free last 'num' bufs. @@ -644,10 +720,20 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, seg_entry_next(&hdr, &idx); for (i = 0; i < num; i++) { + tmp_hdr = hdr; seg = seg_entry_next(&hdr, &idx); buf_hdr[i] = seg->hdr; + + /* Free link headers, if those become empty */ + if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { + link_hdr[links] = &tmp_hdr->buf_hdr; + links++; + } } + if (odp_unlikely(links)) + packet_free_multi(link_hdr, links); + packet_free_multi(buf_hdr, num); /* Head segment remains, no need to copy or update majority @@ -767,42 +853,56 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, void odp_packet_free(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - odp_buffer_t hdl = buffer_handle(pkt_hdr); - int num_seg = pkt_hdr->buf_hdr.segcount; - if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) - packet_free_multi((odp_buffer_hdr_t **)&hdl, 1); - else + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) { + odp_buffer_hdr_t *buf_hdr[2]; + int num = 1; + + buf_hdr[0] = &pkt_hdr->buf_hdr; + + if (odp_unlikely(seg_is_link(pkt_hdr))) { + num = 2; + buf_hdr[1] = pkt_hdr->buf_hdr.seg[0].hdr; + } + + packet_free_multi(buf_hdr, num); + } else { free_all_segments(pkt_hdr, num_seg); + } } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - if (CONFIG_PACKET_MAX_SEGS == 1) { - packet_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num); - } else { - odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS]; - int i; - int bufs = 0; - - for (i = 0; i < num; i++) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); - int num_seg = pkt_hdr->buf_hdr.segcount; - odp_buffer_hdr_t *hdr = &pkt_hdr->buf_hdr; + odp_buffer_hdr_t *buf_hdr[num]; + odp_buffer_hdr_t *buf_hdr2[num]; + int i; + int links = 0; + int num_freed = 0; - buf_hdr[bufs] = hdr; - bufs++; + for (i = 0; i < num; i++) { + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); + int num_seg = pkt_hdr->buf_hdr.segcount; - if (odp_likely(num_seg == 1)) - continue; + if (odp_unlikely(num_seg > 1)) { + free_all_segments(pkt_hdr, num_seg); + num_freed++; + continue; + } - copy_buf_hdr(pkt_hdr, 1, num_seg - 1, &buf_hdr[bufs]); - bufs += num_seg - 1; + if (odp_unlikely(seg_is_link(pkt_hdr))) { + buf_hdr2[links] = pkt_hdr->buf_hdr.seg[0].hdr; + links++; } - packet_free_multi(buf_hdr, bufs); + buf_hdr[i - num_freed] = &pkt_hdr->buf_hdr; } + + if (odp_unlikely(links)) + packet_free_multi(buf_hdr2, links); + + if (odp_likely(num - num_freed)) + packet_free_multi(buf_hdr, num - num_freed); } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -1627,7 +1727,9 @@ void odp_packet_print(odp_packet_t pkt) while (seg != ODP_PACKET_SEG_INVALID) { odp_buffer_hdr_t *buf_hdr; + odp_packet_hdr_t *tmp_hdr; + tmp_hdr = seg_hdr; seg_entry = seg_entry_next(&seg_hdr, &idx); buf_hdr = seg_entry->hdr; @@ -1635,8 +1737,16 @@ void odp_packet_print(odp_packet_t pkt) " seg_len %-4" PRIu32 " seg_data %p ", odp_packet_seg_data_len(pkt, seg), odp_packet_seg_data(pkt, seg)); - len += snprintf(&str[len], n - len, "ref_cnt %u\n", + len += snprintf(&str[len], n - len, "ref_cnt %u", buffer_ref(buf_hdr)); + if (seg_is_link(tmp_hdr)) { + uint32_t ref; + + ref = buffer_ref(&tmp_hdr->buf_hdr); + len += snprintf(&str[len], n - len, "L(%u)\n", ref); + } else { + len += snprintf(&str[len], n - len, "\n"); + } seg = odp_packet_next_seg(pkt, seg); } @@ -2034,59 +2144,122 @@ odp_packet_t odp_packet_ref_static(odp_packet_t pkt) odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) { - odp_packet_t new; - int ret; + odp_packet_t ref; + odp_packet_hdr_t *link_hdr; + odp_packet_hdr_t *next_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *hdr = pkt_hdr; + seg_entry_t *seg; + uint32_t seg_idx = 0; + uint8_t idx = 0; + uint32_t seg_offset = 0; + int i, num_copy, segcount; + uint32_t len; - new = odp_packet_copy(pkt, odp_packet_pool(pkt)); + if (offset >= pkt_hdr->frame_len) { + ODP_DBG("offset too large\n"); + return ODP_PACKET_INVALID; + } - if (new == ODP_PACKET_INVALID) { - ODP_ERR("copy failed\n"); + /* Allocate link segment */ + if (packet_alloc(pkt_hdr->buf_hdr.pool_ptr, 0, 1, 1, &ref) != 1) { + ODP_DBG("segment alloc failed\n"); return ODP_PACKET_INVALID; } - ret = odp_packet_trunc_head(&new, offset, NULL, NULL); + link_hdr = packet_hdr(ref); - if (ret < 0) { - ODP_ERR("trunk_head failed\n"); - odp_packet_free(new); - return ODP_PACKET_INVALID; + seg_entry_find_offset(&hdr, &idx, &seg_offset, &seg_idx, offset); + num_copy = hdr->buf_hdr.num_seg - idx; + segcount = pkt_hdr->buf_hdr.segcount; + + /* In addition to segments, update reference count of + * an existing link header. */ + if (seg_is_link(hdr)) + buffer_ref_inc((odp_buffer_hdr_t *)hdr); + + seg = seg_entry_next(&hdr, &idx); + link_hdr->buf_hdr.num_seg = 1; + link_hdr->buf_hdr.seg[0].hdr = seg->hdr; + link_hdr->buf_hdr.seg[0].data = seg->data + seg_offset; + link_hdr->buf_hdr.seg[0].len = seg->len - seg_offset; + buffer_ref_inc(seg->hdr); + + for (i = 1; i < num_copy; i++) { + /* Update link header reference count */ + if (idx == 0 && seg_is_link(hdr)) + buffer_ref_inc((odp_buffer_hdr_t *)hdr); + + seg = seg_entry_next(&hdr, &idx); + + link_hdr->buf_hdr.num_seg++; + link_hdr->buf_hdr.seg[i].hdr = seg->hdr; + link_hdr->buf_hdr.seg[i].data = seg->data; + link_hdr->buf_hdr.seg[i].len = seg->len; + buffer_ref_inc(seg->hdr); + } + + next_hdr = hdr; + + /* Increment ref count for remaining segments */ + for (i = seg_idx + num_copy; i < segcount; i++) { + /* Update link header reference count */ + if (idx == 0 && seg_is_link(hdr)) + buffer_ref_inc((odp_buffer_hdr_t *)hdr); + + seg = seg_entry_next(&hdr, &idx); + buffer_ref_inc(seg->hdr); } - return new; + len = pkt_hdr->frame_len - offset; + link_hdr->buf_hdr.next_seg = next_hdr; + link_hdr->buf_hdr.last_seg = pkt_hdr->buf_hdr.last_seg; + link_hdr->buf_hdr.segcount = segcount - seg_idx; + link_hdr->frame_len = len; + link_hdr->tailroom = pkt_hdr->tailroom; + link_hdr->shared_len = len; + + /* Link header does not have headroom, it just points to other + * buffers. Zero length headroom ensures that head of the other buffer + * is not pushed through a reference. */ + link_hdr->headroom = 0; + + if (pkt_hdr->shared_len < len) + pkt_hdr->shared_len = len; + + return ref; + } odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, odp_packet_t hdr) { - odp_packet_t new; + odp_packet_t ref; int ret; + odp_packet_hdr_t *new_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t len = pkt_hdr->frame_len; - new = odp_packet_copy(pkt, odp_packet_pool(pkt)); + ref = odp_packet_ref(pkt, offset); - if (new == ODP_PACKET_INVALID) { - ODP_ERR("copy failed\n"); + if (ref == ODP_PACKET_INVALID) { + ODP_DBG("reference create failed\n"); return ODP_PACKET_INVALID; } - if (offset) { - ret = odp_packet_trunc_head(&new, offset, NULL, NULL); - - if (ret < 0) { - ODP_ERR("trunk_head failed\n"); - odp_packet_free(new); - return ODP_PACKET_INVALID; - } - } - - ret = odp_packet_concat(&hdr, new); + ret = odp_packet_concat(&hdr, ref); if (ret < 0) { - ODP_ERR("concat failed\n"); - odp_packet_free(new); + ODP_DBG("concat failed\n"); + odp_packet_free(ref); return ODP_PACKET_INVALID; } + new_hdr = packet_hdr(hdr); + new_hdr->shared_len = len - offset; + return hdr; + } int odp_packet_has_ref(odp_packet_t pkt) @@ -2115,8 +2288,12 @@ int odp_packet_has_ref(odp_packet_t pkt) uint32_t odp_packet_unshared_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t len = pkt_hdr->frame_len; + + if (odp_packet_has_ref(pkt)) + return len - pkt_hdr->shared_len; - return packet_len(pkt_hdr) - pkt_hdr->shared_len; + return len; } /* Include non-inlined versions of API functions */ commit 69a1179a05344b59961aaf997a1b406698b6840b Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 25 14:03:59 2017 +0300 linux-gen: packet: compress packet offset table Use uint16_t (instead of size_t) for offsets and thus decrease size of the offset table into one fourth. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 41468bec..96379ebf 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -74,33 +74,33 @@ typedef enum { /** @internal Packet header field offsets for inline functions */ typedef struct _odp_packet_inline_offset_t { /** @internal field offset */ - size_t data; + uint16_t data; /** @internal field offset */ - size_t seg_len; + uint16_t seg_len; /** @internal field offset */ - size_t frame_len; + uint16_t frame_len; /** @internal field offset */ - size_t headroom; + uint16_t headroom; /** @internal field offset */ - size_t tailroom; + uint16_t tailroom; /** @internal field offset */ - size_t pool; + uint16_t pool; /** @internal field offset */ - size_t input; + uint16_t input; /** @internal field offset */ - size_t segcount; + uint16_t segcount; /** @internal field offset */ - size_t user_ptr; + uint16_t user_ptr; /** @internal field offset */ - size_t user_area; + uint16_t user_area; /** @internal field offset */ - size_t user_area_size; + uint16_t user_area_size; /** @internal field offset */ - size_t flow_hash; + uint16_t flow_hash; /** @internal field offset */ - size_t timestamp; + uint16_t timestamp; /** @internal field offset */ - size_t input_flags; + uint16_t input_flags; } _odp_packet_inline_offset_t; commit 6efe7d971a3a4b8f7eaa42660b48eee31493924c Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Aug 24 16:21:51 2017 +0300 linux-gen: packet: enhance segment prints Print segment addresses and reference counts in addition to segment lengths. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 1a21ab0c..5990f878 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1579,11 +1579,13 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, * ******************************************************** * */ - void odp_packet_print(odp_packet_t pkt) { odp_packet_seg_t seg; - int max_len = 512; + seg_entry_t *seg_entry; + odp_packet_hdr_t *seg_hdr; + uint8_t idx; + int max_len = 1024; char str[max_len]; int len = 0; int n = max_len - 1; @@ -1619,12 +1621,22 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " num_segs %i\n", odp_packet_num_segs(pkt)); + seg_hdr = hdr; + idx = 0; seg = odp_packet_first_seg(pkt); while (seg != ODP_PACKET_SEG_INVALID) { + odp_buffer_hdr_t *buf_hdr; + + seg_entry = seg_entry_next(&seg_hdr, &idx); + buf_hdr = seg_entry->hdr; + len += snprintf(&str[len], n - len, - " seg_len %" PRIu32 "\n", - odp_packet_seg_data_len(pkt, seg)); + " seg_len %-4" PRIu32 " seg_data %p ", + odp_packet_seg_data_len(pkt, seg), + odp_packet_seg_data(pkt, seg)); + len += snprintf(&str[len], n - len, "ref_cnt %u\n", + buffer_ref(buf_hdr)); seg = odp_packet_next_seg(pkt, seg); } commit ef4e4d4c4ef1f9f569c236494cb178feb7c90343 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Aug 21 13:51:49 2017 +0300 linux-gen: packet: implement static references Implemented static references with a reference counter. Counter is zero when not used (reference API not used). Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 0c873d2d..cd067a08 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -62,11 +62,16 @@ struct odp_buffer_hdr_t { /* Last header of the segment list */ void *last_seg; - /* Initial buffer data pointer and length */ + /* Initial buffer data pointer */ uint8_t *base_data; - uint8_t *buf_end; - /* --- 40 bytes --- */ + /* Reference count */ + odp_atomic_u32_t ref_cnt; + + /* Event type. Maybe different than pool type (crypto compl event) */ + int8_t event_type; + + /* --- 37 bytes --- */ /* Segments */ seg_entry_t seg[CONFIG_PACKET_MAX_SEGS]; @@ -93,14 +98,17 @@ struct odp_buffer_hdr_t { /* Pool pointer */ void *pool_ptr; + /* Initial buffer tail pointer */ + uint8_t *buf_end; + /* User area pointer */ void *uarea_addr; /* User area size */ uint32_t uarea_size; - /* Event type. Maybe different than pool type (crypto compl event) */ - int8_t event_type; + /* Max data size */ + uint32_t size; /* ipc mapped process can not walk over pointers, * offset has to be used */ @@ -110,9 +118,6 @@ struct odp_buffer_hdr_t { * inlining */ odp_pool_t pool_hdl; - /* Max data size */ - uint32_t size; - /* Data or next header */ uint8_t data[0]; } ODP_ALIGNED_CACHE; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index d8e5766c..d8d4584a 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -116,11 +116,13 @@ typedef struct { packet_parser_t p; + uint32_t frame_len; + odp_pktio_t input; - uint32_t frame_len; uint32_t headroom; uint32_t tailroom; + uint32_t shared_len; /* * Members below are not initialized by packet_init() @@ -217,6 +219,7 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) * segment occupied by the allocated length. */ pkt_hdr->frame_len = len; + pkt_hdr->shared_len = 0; pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; pkt_hdr->tailroom = CONFIG_PACKET_MAX_SEG_LEN - seg_len + CONFIG_PACKET_TAILROOM; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index a4330737..1a21ab0c 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -494,6 +494,76 @@ static inline void copy_buf_hdr(odp_packet_hdr_t *pkt_hdr, int first, int num, } } +static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) +{ + uint32_t ref_cnt = odp_atomic_load_u32(&buf_hdr->ref_cnt); + + /* First count increment after alloc */ + if (odp_likely(ref_cnt) == 0) + odp_atomic_store_u32(&buf_hdr->ref_cnt, 2); + else + odp_atomic_inc_u32(&buf_hdr->ref_cnt); +} + +static inline uint32_t buffer_ref_dec(odp_buffer_hdr_t *buf_hdr) +{ + return odp_atomic_fetch_dec_u32(&buf_hdr->ref_cnt); +} + +static inline uint32_t buffer_ref(odp_buffer_hdr_t *buf_hdr) +{ + return odp_atomic_load_u32(&buf_hdr->ref_cnt); +} + +static inline int is_multi_ref(uint32_t ref_cnt) +{ + return (ref_cnt > 1); +} + +static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) +{ + seg_entry_t *seg; + int i; + int seg_count = pkt_hdr->buf_hdr.segcount; + odp_packet_hdr_t *hdr = pkt_hdr; + uint8_t idx = 0; + + for (i = 0; i < seg_count; i++) { + seg = seg_entry_next(&hdr, &idx); + buffer_ref_inc(seg->hdr); + } +} + +static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) +{ + int i; + uint32_t ref_cnt; + int num_ref = 0; + + for (i = 0; i < num; i++) { + /* Zero when reference API has not been used */ + ref_cnt = buffer_ref(hdr[i]); + + if (odp_unlikely(ref_cnt)) { + ref_cnt = buffer_ref_dec(hdr[i]); + + if (is_multi_ref(ref_cnt)) { + num_ref++; + continue; + } + } + + /* Skip references and pack to be freed headers to array head */ + if (odp_unlikely(num_ref)) + hdr[i - num_ref] = hdr[i]; + } + + num -= num_ref; + + if (odp_likely(num)) + buffer_free_multi(hdr, num); +} + static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { seg_entry_t *seg; @@ -506,7 +576,7 @@ static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) buf_hdr[i] = seg->hdr; } - buffer_free_multi(buf_hdr, num); + packet_free_multi(buf_hdr, num); } static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, @@ -563,7 +633,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, pkt_hdr = new_hdr; - buffer_free_multi(buf_hdr, num); + packet_free_multi(buf_hdr, num); } else { /* Free last 'num' bufs. * First, find the last remaining header. */ @@ -578,7 +648,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, buf_hdr[i] = seg->hdr; } - buffer_free_multi(buf_hdr, num); + packet_free_multi(buf_hdr, num); /* Head segment remains, no need to copy or update majority * of the metadata. */ @@ -702,7 +772,7 @@ void odp_packet_free(odp_packet_t pkt) int num_seg = pkt_hdr->buf_hdr.segcount; if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) - buffer_free_multi((odp_buffer_hdr_t **)&hdl, 1); + packet_free_multi((odp_buffer_hdr_t **)&hdl, 1); else free_all_segments(pkt_hdr, num_seg); } @@ -710,7 +780,7 @@ void odp_packet_free(odp_packet_t pkt) void odp_packet_free_multi(const odp_packet_t pkt[], int num) { if (CONFIG_PACKET_MAX_SEGS == 1) { - buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num); + packet_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num); } else { odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS]; int i; @@ -731,7 +801,7 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) bufs += num_seg - 1; } - buffer_free_multi(buf_hdr, bufs); + packet_free_multi(buf_hdr, bufs); } } @@ -1942,7 +2012,12 @@ uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl) odp_packet_t odp_packet_ref_static(odp_packet_t pkt) { - return odp_packet_copy(pkt, odp_packet_pool(pkt)); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + packet_ref_inc(pkt_hdr); + pkt_hdr->shared_len = pkt_hdr->frame_len; + + return pkt; } odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) @@ -2004,14 +2079,32 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, int odp_packet_has_ref(odp_packet_t pkt) { - (void)pkt; + odp_buffer_hdr_t *buf_hdr; + seg_entry_t *seg; + int i; + uint32_t ref_cnt; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + int seg_count = pkt_hdr->buf_hdr.segcount; + odp_packet_hdr_t *hdr = pkt_hdr; + uint8_t idx = 0; + + for (i = 0; i < seg_count; i++) { + seg = seg_entry_next(&hdr, &idx); + buf_hdr = seg->hdr; + ref_cnt = buffer_ref(buf_hdr); + + if (is_multi_ref(ref_cnt)) + return 1; + } return 0; } uint32_t odp_packet_unshared_len(odp_packet_t pkt) { - return odp_packet_len(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + return packet_len(pkt_hdr) - pkt_hdr->shared_len; } /* Include non-inlined versions of API functions */ diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 47a39f5b..2f65cb20 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -277,6 +277,8 @@ static void init_buffers(pool_t *pool) buf_hdr->seg[0].data = &data[offset]; buf_hdr->seg[0].len = pool->data_size; + odp_atomic_init_u32(&buf_hdr->ref_cnt, 0); + /* Store base values for fast init */ buf_hdr->base_data = buf_hdr->seg[0].data; buf_hdr->buf_end = &data[offset + pool->data_size + commit e4289ab4f73221a9d20ecfb8eac6b79a26df06a3 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 18 10:54:25 2017 +0300 linux-gen: buffer: optimize header layout Pack most often used buffer header fields (including seg[0]) into the first cache line. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 30909465..0c873d2d 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -43,41 +43,46 @@ typedef struct seg_entry_t { /* Common buffer header */ struct odp_buffer_hdr_t { - /* Buffer index in the pool */ - uint32_t index; - /* Initial buffer data pointer and length */ - uint8_t *base_data; - uint8_t *buf_end; + /* Buffer index in the pool */ + uint32_t index; - /* Max data size */ - uint32_t size; + /* Total segment count */ + uint16_t segcount; /* Pool type */ int8_t type; - /* Burst counts */ - uint8_t burst_num; - uint8_t burst_first; - /* Number of seg[] entries used */ uint8_t num_seg; - /* Total segment count */ - uint32_t segcount; - /* Next header which continues the segment list */ void *next_seg; /* Last header of the segment list */ void *last_seg; + /* Initial buffer data pointer and length */ + uint8_t *base_data; + uint8_t *buf_end; + + /* --- 40 bytes --- */ + /* Segments */ seg_entry_t seg[CONFIG_PACKET_MAX_SEGS]; + /* Burst counts */ + uint8_t burst_num; + uint8_t burst_first; + /* Next buf in a list */ struct odp_buffer_hdr_t *next; + /* Burst table */ + struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE]; + + /* --- Mostly read only data --- */ + /* User context pointer or u64 */ union { uint64_t buf_u64; @@ -85,6 +90,9 @@ struct odp_buffer_hdr_t { const void *buf_cctx; /* const alias for ctx */ }; + /* Pool pointer */ + void *pool_ptr; + /* User area pointer */ void *uarea_addr; @@ -94,9 +102,6 @@ struct odp_buffer_hdr_t { /* Event type. Maybe different than pool type (crypto compl event) */ int8_t event_type; - /* Burst table */ - struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE]; - /* ipc mapped process can not walk over pointers, * offset has to be used */ uint64_t ipc_data_offset; @@ -105,8 +110,8 @@ struct odp_buffer_hdr_t { * inlining */ odp_pool_t pool_hdl; - /* Pool pointer */ - void *pool_ptr; + /* Max data size */ + uint32_t size; /* Data or next header */ uint8_t data[0]; commit a787abf96b17d9714a6b892091d19c1be2bc5e6c Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Aug 17 14:09:47 2017 +0300 linux-gen: packet: re-implement packet segmentation Removed limitation to fixed number of segments (CONFIG_PACKET_MAX_SEGS) per packet. Packet headers carrying segment tables are now linked together with 'next_seg' pointer. Last header containing a segment table is pointed by 'last_seg' pointer. Other than the first segment are not referred directly but with seg_entry_ functions. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index e20d1c8d..30909465 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -35,6 +35,12 @@ extern "C" { #define BUFFER_BURST_SIZE CONFIG_BURST_SIZE +typedef struct seg_entry_t { + void *hdr; + uint8_t *data; + uint32_t len; +} seg_entry_t; + /* Common buffer header */ struct odp_buffer_hdr_t { /* Buffer index in the pool */ @@ -54,15 +60,20 @@ struct odp_buffer_hdr_t { uint8_t burst_num; uint8_t burst_first; - /* Segment count */ - uint8_t segcount; + /* Number of seg[] entries used */ + uint8_t num_seg; + + /* Total segment count */ + uint32_t segcount; + + /* Next header which continues the segment list */ + void *next_seg; + + /* Last header of the segment list */ + void *last_seg; /* Segments */ - struct { - void *hdr; - uint8_t *data; - uint32_t len; - } seg[CONFIG_PACKET_MAX_SEGS]; + seg_entry_t seg[CONFIG_PACKET_MAX_SEGS]; /* Next buf in a list */ struct odp_buffer_hdr_t *next; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index f77987c8..d8e5766c 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -172,6 +172,16 @@ static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; } +static inline seg_entry_t *seg_entry_last(odp_packet_hdr_t *hdr) +{ + odp_packet_hdr_t *last; + uint8_t last_seg; + + last = hdr->buf_hdr.last_seg; + last_seg = last->buf_hdr.num_seg - 1; + return &last->buf_hdr.seg[last_seg]; +} + /** * Initialize packet */ @@ -184,10 +194,13 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) seg_len = len; pkt_hdr->buf_hdr.seg[0].len = len; } else { + seg_entry_t *last; + seg_len = len - ((num - 1) * CONFIG_PACKET_MAX_SEG_LEN); /* Last segment data length */ - pkt_hdr->buf_hdr.seg[num - 1].len = seg_len; + last = seg_entry_last(pkt_hdr); + last->len = seg_len; } pkt_hdr->p.input_flags.all = 0; @@ -229,11 +242,11 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr, static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - int last = pkt_hdr->buf_hdr.segcount - 1; + seg_entry_t *last = seg_entry_last(pkt_hdr); pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->buf_hdr.seg[last].len -= len; + last->len -= len; } static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 1e5b136a..a4330737 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -74,15 +74,75 @@ static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) return (odp_buffer_t)pkt; } +static inline seg_entry_t *seg_entry(odp_packet_hdr_t *hdr, + uint32_t seg_idx) +{ + uint32_t idx = 0; + uint8_t num_seg = hdr->buf_hdr.num_seg; + + while (odp_unlikely(idx + num_seg - 1 < seg_idx)) { + idx += num_seg; + hdr = hdr->buf_hdr.next_seg; + num_seg = hdr->buf_hdr.num_seg; + } + + idx = seg_idx - idx; + + return &hdr->buf_hdr.seg[idx]; +} + +static inline void seg_entry_find_idx(odp_packet_hdr_t **p_hdr, + uint8_t *p_idx, + uint32_t find_idx) +{ + odp_packet_hdr_t *hdr = *p_hdr; + uint32_t idx = 0; + uint8_t num_seg = hdr->buf_hdr.num_seg; + + while (odp_unlikely(idx + num_seg - 1 < find_idx)) { + idx += num_seg; + hdr = hdr->buf_hdr.next_seg; + num_seg = hdr->buf_hdr.num_seg; + } + + idx = find_idx - idx; + *p_hdr = hdr; + *p_idx = idx; +} + +/* Return pointer to the current segment entry and step cur_hdr / cur_idx + * forward. + */ +static inline seg_entry_t *seg_entry_next(odp_packet_hdr_t **cur_hdr, + uint8_t *cur_idx) +{ + odp_packet_hdr_t *hdr = *cur_hdr; + uint8_t idx = *cur_idx; + uint8_t num_seg = hdr->buf_hdr.num_seg; + + if (idx == num_seg - 1) { + *cur_hdr = hdr->buf_hdr.next_seg; + *cur_idx = 0; + } else { + *cur_idx = idx + 1; + } + + return &hdr->buf_hdr.seg[idx]; +} + static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { - return pkt_hdr->buf_hdr.seg[seg_idx].len; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + + return seg->len; } static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { - return pkt_hdr->buf_hdr.seg[seg_idx].data; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + + return seg->data; } static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) @@ -95,14 +155,7 @@ static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) { - return packet_seg_len(pkt_hdr, 0); -} - -static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr) -{ - int last = packet_last_seg(pkt_hdr); - - return packet_seg_len(pkt_hdr, last); + return pkt_hdr->buf_hdr.seg[0].len; } static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) @@ -112,26 +165,27 @@ static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) { - int last = packet_last_seg(pkt_hdr); - uint32_t seg_len = pkt_hdr->buf_hdr.seg[last].len; + seg_entry_t *last_seg = seg_entry_last(pkt_hdr); - return pkt_hdr->buf_hdr.seg[last].data + seg_len; + return last_seg->data + last_seg->len; } -static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg) +static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + odp_buffer_hdr_t *hdr = seg->hdr; uint8_t *base = hdr->base_data; - uint8_t *head = pkt_hdr->buf_hdr.seg[seg].data; + uint8_t *head = seg->data; return CONFIG_PACKET_HEADROOM + (head - base); } -static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg) +static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) { - uint32_t seg_len = pkt_hdr->buf_hdr.seg[seg].len; - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint8_t *tail = pkt_hdr->buf_hdr.seg[seg].data + seg_len; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + + odp_buffer_hdr_t *hdr = seg->hdr; + uint8_t *tail = seg->data + seg->len; return hdr->buf_end - tail; } @@ -154,11 +208,11 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - int last = packet_last_seg(pkt_hdr); + seg_entry_t *last_seg = seg_entry_last(pkt_hdr); pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->buf_hdr.seg[last].len += len; + last_seg->len += len; } /* Copy all metadata for segmentation modification. Segment data and lengths @@ -188,6 +242,9 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, /* segmentation data is not copied: * buf_hdr.seg[] * buf_hdr.segcount + * buf_hdr.num_seg + * buf_hdr.next_seg + * buf_hdr.last_seg */ } @@ -196,7 +253,7 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, { void *addr; uint32_t len; - int seg = 0; + int seg_id = 0; int seg_count = pkt_hdr->buf_hdr.segcount; if (odp_unlikely(offset >= pkt_hdr->frame_len)) @@ -207,10 +264,14 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, len = pkt_hdr->buf_hdr.seg[0].len - offset; } else { int i; + seg_entry_t *seg = NULL; uint32_t seg_start = 0, seg_end = 0; + odp_packet_hdr_t *hdr = pkt_hdr; + uint8_t idx = 0; for (i = 0; i < seg_count; i++) { - seg_end += pkt_hdr->buf_hdr.seg[i].len; + seg = seg_entry_next(&hdr, &idx); + seg_end += seg->len; if (odp_likely(offset < seg_end)) break; @@ -218,16 +279,16 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, seg_start = seg_end; } - addr = pkt_hdr->buf_hdr.seg[i].data + (offset - seg_start); - len = pkt_hdr->buf_hdr.seg[i].len - (offset - seg_start); - seg = i; + addr = seg->data + (offset - seg_start); + len = seg->len - (offset - seg_start); + seg_id = i; } if (seg_len) *seg_len = len; if (seg_idx) - *seg_idx = seg; + *seg_idx = seg_id; return addr; } @@ -243,10 +304,43 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; } +static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) +{ + int cur, i; + odp_packet_hdr_t *hdr; + odp_packet_hdr_t *head = pkt_hdr[0]; + + cur = 0; + + while (1) { + hdr = pkt_hdr[cur]; + + for (i = 0; i < CONFIG_PACKET_MAX_SEGS; i++) { + odp_buffer_hdr_t *buf_hdr; + + buf_hdr = &pkt_hdr[cur]->buf_hdr; + hdr->buf_hdr.seg[i].hdr = buf_hdr; + hdr->buf_hdr.seg[i].data = buf_hdr->base_data; + hdr->buf_hdr.seg[i].len = BASE_LEN; + cur++; + + if (cur == num) { + /* Last segment */ + hdr->buf_hdr.num_seg = i + 1; + hdr->buf_hdr.next_seg = NULL; + head->buf_hdr.last_seg = &hdr->buf_hdr; + return; + } + } + + hdr->buf_hdr.num_seg = CONFIG_PACKET_MAX_SEGS; + hdr->buf_hdr.next_seg = pkt_hdr[cur]; + } +} + static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) { odp_packet_hdr_t *hdr; - int i; /* First segment is the packet descriptor */ hdr = pkt_hdr[0]; @@ -258,30 +352,33 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) if (CONFIG_PACKET_MAX_SEGS != 1) { hdr->buf_hdr.segcount = num; + /* Defaults for single segment packet */ + hdr->buf_hdr.num_seg = 1; + hdr->buf_hdr.next_seg = NULL; + hdr->buf_hdr.last_seg = &hdr->buf_hdr; + if (odp_unlikely(num > 1)) { - for (i = 1; i < num; i++) { - odp_buffer_hdr_t *buf_hdr; + link_segments(pkt_hdr, num); - buf_hdr = &pkt_hdr[i]->buf_hdr; - hdr->buf_hdr.seg[i].hdr = buf_hdr; - hdr->buf_hdr.seg[i].data = buf_hdr->base_data; - hdr->buf_hdr.seg[i].len = BASE_LEN; - } } } } static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int first, int num) { - odp_buffer_hdr_t *hdr; + odp_packet_hdr_t *hdr = pkt_hdr; void *base; int i; + seg_entry_t *seg; + uint8_t idx; + + seg_entry_find_idx(&hdr, &idx, first); - for (i = first; i < first + num; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - base = hdr->base_data; - pkt_hdr->buf_hdr.seg[i].len = BASE_LEN; - pkt_hdr->buf_hdr.seg[i].data = base; + for (i = 0; i < num; i++) { + base = hdr->buf_hdr.base_data; + seg = seg_entry_next(&hdr, &idx); + seg->len = BASE_LEN; + seg->data = base; } } @@ -309,31 +406,11 @@ static inline int num_segments(uint32_t len) static inline void add_all_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from) { - int i; - int n = to->buf_hdr.segcount; - int num = from->buf_hdr.segcount; + odp_packet_hdr_t *last = to->buf_hdr.last_seg; - for (i = 0; i < num; i++) { - to->buf_hdr.seg[n + i].hdr = from->buf_hdr.seg[i].hdr; - to->buf_hdr.seg[n + i].data = from->buf_hdr.seg[i].data; - to->buf_hdr.seg[n + i].len = from->buf_hdr.seg[i].len; - } - - to->buf_hdr.segcount = n + num; -} - -static inline void copy_num_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from, - int first, int num) -{ - int i; - - for (i = 0; i < num; i++) { - to->buf_hdr.seg[i].hdr = from->buf_hdr.seg[first + i].hdr; - to->buf_hdr.seg[i].data = from->buf_hdr.seg[first + i].data; - to->buf_hdr.seg[i].len = from->buf_hdr.seg[first + i].len; - } - - to->buf_hdr.segcount = num; + last->buf_hdr.next_seg = from; + to->buf_hdr.last_seg = from->buf_hdr.last_seg; + to->buf_hdr.segcount += from->buf_hdr.segcount; } static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) @@ -385,14 +462,14 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, pkt_hdr = new_hdr; } else { - int last; + seg_entry_t *last_seg; /* add into the tail */ add_all_segs(pkt_hdr, new_hdr); /* adjust last segment length */ - last = packet_last_seg(pkt_hdr); - pkt_hdr->buf_hdr.seg[last].len = seg_len; + last_seg = seg_entry_last(pkt_hdr); + last_seg->len = seg_len; pkt_hdr->frame_len += len; pkt_hdr->tailroom = pool->tailroom + offset; @@ -401,13 +478,33 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, return pkt_hdr; } -static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) +static inline void copy_buf_hdr(odp_packet_hdr_t *pkt_hdr, int first, int num, + odp_buffer_hdr_t *buf_hdr[]) +{ + seg_entry_t *seg; + int i; + uint8_t idx; + odp_packet_hdr_t *hdr = pkt_hdr; + + seg_entry_find_idx(&hdr, &idx, first); + + for (i = 0; i < num; i++) { + seg = seg_entry_next(&hdr, &idx); + buf_hdr[i] = seg->hdr; + } +} + +static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { + seg_entry_t *seg; int i; odp_buffer_hdr_t *buf_hdr[num]; + uint8_t idx = 0; - for (i = 0; i < num; i++) - buf_hdr[i] = pkt_hdr->buf_hdr.seg[first + i].hdr; + for (i = 0; i < num; i++) { + seg = seg_entry_next(&pkt_hdr, &idx); + buf_hdr[i] = seg->hdr; + } buffer_free_multi(buf_hdr, num); } @@ -416,20 +513,45 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, int num, uint32_t free_len, uint32_t pull_len, int head) { + seg_entry_t *seg; + int i; int num_remain = pkt_hdr->buf_hdr.segcount - num; + odp_packet_hdr_t *hdr = pkt_hdr; + odp_packet_hdr_t *last_hdr = pkt_hdr->buf_hdr.last_seg; + uint8_t idx; + uint8_t num_seg; + odp_buffer_hdr_t *buf_hdr[num]; if (head) { odp_packet_hdr_t *new_hdr; - int i; - odp_buffer_hdr_t *buf_hdr[num]; - for (i = 0; i < num; i++) - buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr; + idx = 0; + for (i = 0; i < num; i++) { + seg = seg_entry_next(&hdr, &idx); + buf_hdr[i] = seg->hdr; + } + + /* The first remaining header is the new packet descriptor. + * Copy remaining segments from the last to-be-removed header + * to the new header. */ + new_hdr = hdr->buf_hdr.seg[idx].hdr; + num_seg = hdr->buf_hdr.num_seg - idx; + + new_hdr->buf_hdr.next_seg = hdr->buf_hdr.next_seg; + + if (hdr == last_hdr) + new_hdr->buf_hdr.last_seg = new_hdr; + else + new_hdr->buf_hdr.last_seg = last_hdr; + + new_hdr->buf_hdr.num_seg = num_seg; + new_hdr->buf_hdr.segcount = num_remain; - /* First remaining segment is the new packet descriptor */ - new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; + for (i = 0; i < num_seg; i++) { + seg = seg_entry_next(&hdr, &idx); + new_hdr->buf_hdr.seg[i] = *seg; + } - copy_num_segs(new_hdr, pkt_hdr, num, num_remain); packet_seg_copy_md(new_hdr, pkt_hdr); /* Tailroom not changed */ @@ -443,11 +565,27 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, buffer_free_multi(buf_hdr, num); } else { - /* Free last 'num' bufs */ - free_bufs(pkt_hdr, num_remain, num); + /* Free last 'num' bufs. + * First, find the last remaining header. */ + seg_entry_find_idx(&hdr, &idx, num_remain - 1); + last_hdr = hdr; + num_seg = idx + 1; + + seg_entry_next(&hdr, &idx); + + for (i = 0; i < num; i++) { + seg = seg_entry_next(&hdr, &idx); + buf_hdr[i] = seg->hdr; + } + + buffer_free_multi(buf_hdr, num); /* Head segment remains, no need to copy or update majority * of the metadata. */ + last_hdr->buf_hdr.num_seg = num_seg; + last_hdr->buf_hdr.next_seg = NULL; + + pkt_hdr->buf_hdr.last_seg = last_hdr; pkt_hdr->buf_hdr.segcount = num_remain; pkt_hdr->frame_len -= free_len; pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); @@ -566,7 +704,7 @@ void odp_packet_free(odp_packet_t pkt) if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) buffer_free_multi((odp_buffer_hdr_t **)&hdl, 1); else - free_bufs(pkt_hdr, 0, num_seg); + free_all_segments(pkt_hdr, num_seg); } void odp_packet_free_multi(const odp_packet_t pkt[], int num) @@ -576,7 +714,6 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) } else { odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS]; int i; - int j; int bufs = 0; for (i = 0; i < num; i++) { @@ -590,10 +727,8 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) if (odp_likely(num_seg == 1)) continue; - for (j = 1; j < num_seg; j++) { - buf_hdr[bufs] = hdr->seg[j].hdr; - bufs++; - } + copy_buf_hdr(pkt_hdr, 1, num_seg - 1, &buf_hdr[bufs]); + bufs += num_seg - 1; } buffer_free_multi(buf_hdr, bufs); @@ -664,166 +799,6 @@ void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) return packet_data(pkt_hdr); } -static inline uint32_t pack_seg_head(odp_packet_hdr_t *pkt_hdr, int seg) -{ - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint32_t len = pkt_hdr->buf_hdr.seg[seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[seg].data; - uint8_t *dst = hdr->base_data; - - if (dst != src) { - memmove(dst, src, len); - pkt_hdr->buf_hdr.seg[seg].data = dst; - } - - return len; -} - -static inline uint32_t pack_seg_tail(odp_packet_hdr_t *pkt_hdr, int seg) -{ - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint32_t len = pkt_hdr->buf_hdr.seg[seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[seg].data; - uint8_t *dst = hdr->base_data + BASE_LEN - len; - - if (dst != src) { - memmove(dst, src, len); - pkt_hdr->buf_hdr.seg[seg].data = dst; - } - - return len; -} - -static inline uint32_t fill_seg_head(odp_packet_hdr_t *pkt_hdr, int dst_seg, - int src_seg, uint32_t max_len) -{ - uint32_t len = pkt_hdr->buf_hdr.seg[src_seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[src_seg].data; - uint32_t offset = pkt_hdr->buf_hdr.seg[dst_seg].len; - uint8_t *dst = pkt_hdr->buf_hdr.seg[dst_seg].data + offset; - - if (len > max_len) - len = max_len; - - memmove(dst, src, len); - - pkt_hdr->buf_hdr.seg[dst_seg].len += len; - pkt_hdr->buf_hdr.seg[src_seg].len -= len; - pkt_hdr->buf_hdr.seg[src_seg].data += len; - - if (pkt_hdr->buf_hdr.seg[src_seg].len == 0) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[src_seg].hdr; - - pkt_hdr->buf_hdr.seg[src_seg].data = hdr->base_data; - } - - return len; -} - -static inline uint32_t fill_seg_tail(odp_packet_hdr_t *pkt_hdr, int dst_seg, - int src_seg, uint32_t max_len) -{ - uint32_t src_len = pkt_hdr->buf_hdr.seg[src_seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[src_seg].data; - uint8_t *dst = pkt_hdr->buf_hdr.seg[dst_seg].data; - uint32_t len = src_len; - - if (len > max_len) - len = max_len; - - src += src_len - len; - dst -= len; - - memmove(dst, src, len); - - pkt_hdr->buf_hdr.seg[dst_seg].data -= len; - pkt_hdr->buf_hdr.seg[dst_seg].len += len; - pkt_hdr->buf_hdr.seg[src_seg].len -= len; - - if (pkt_hdr->buf_hdr.seg[src_seg].len == 0) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[src_seg].hdr; - - pkt_hdr->buf_hdr.seg[src_seg].data = hdr->base_data; - } - - return len; -} - -static inline int move_data_to_head(odp_packet_hdr_t *pkt_hdr, int segs) -{ - int dst_seg, src_seg; - uint32_t len, free_len; - uint32_t moved = 0; - - for (dst_seg = 0; dst_seg < segs; dst_seg++) { - len = pack_seg_head(pkt_hdr, dst_seg); - moved += len; - - if (len == BASE_LEN) - continue; - - free_len = BASE_LEN - len; - - for (src_seg = dst_seg + 1; CONFIG_PACKET_MAX_SEGS > 1 && - src_seg < segs; src_seg++) { - len = fill_seg_head(pkt_hdr, dst_seg, src_seg, - free_len); - moved += len; - - if (len == free_len) { - /* dst seg is full */ - break; - } - - /* src seg is empty */ - free_len -= len; - } - - if (moved == pkt_hdr->frame_len) - break; - } - - /* last segment which have data */ - return dst_seg; -} - -static inline int move_data_to_tail(odp_packet_hdr_t *pkt_hdr, int segs) -{ - int dst_seg, src_seg; - uint32_t len, free_len; - uint32_t moved = 0; - - for (dst_seg = segs - 1; dst_seg >= 0; dst_seg--) { - len = pack_seg_tail(pkt_hdr, dst_seg); - moved += len; - - if (len == BASE_LEN) - continue; - - free_len = BASE_LEN - len; - - for (src_seg = dst_seg - 1; src_seg >= 0; src_seg--) { - len = fill_seg_tail(pkt_hdr, dst_seg, src_seg, - free_len); - moved += len; - - if (len == free_len) { - /* dst seg is full */ - break; - } - - /* src seg is empty */ - free_len -= len; - } - - if (moved == pkt_hdr->frame_len) - break; - } - - /* first segment which have data */ - return dst_seg; -} - int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len) { @@ -835,87 +810,23 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, if (len > headroom) { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; - int segs; + void *ptr; if (odp_unlikely((frame_len + len) > pool->max_len)) return -1; - num = num_segments(len - headroom); - segs = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Cannot directly add new segments */ - odp_packet_hdr_t *new_hdr; - int new_segs = 0; - int free_segs = 0; - uint32_t offset; - - num = num_segments(frame_len + len); - - if (num > segs) { - /* Allocate additional segments */ - new_segs = num - segs; - new_hdr = alloc_segments(pool, new_segs); - - if (new_hdr == NULL) - return -1; - - } else if (num < segs) { - free_segs = segs - num; - } - - /* Pack all data to packet tail */ - move_data_to_tail(pkt_hdr, segs); - reset_seg(pkt_hdr, 0, segs); - - if (new_segs) { - add_all_segs(new_hdr, pkt_hdr); - packet_seg_copy_md(new_hdr, pkt_hdr); - segs += new_segs; - - pkt_hdr = new_hdr; - *pkt = packet_handle(pkt_hdr); - } else if (CONFIG_PACKET_MAX_SEGS > 1 && free_segs) { - new_hdr = pkt_hdr->buf_hdr.seg[free_segs].hdr; - packet_seg_copy_md(new_hdr, pkt_hdr); - - /* Free extra segs */ - free_bufs(pkt_hdr, 0, free_segs); - - segs -= free_segs; - pkt_hdr = new_hdr; - *pkt = packet_handle(pkt_hdr); - } - - frame_len += len; - offset = (segs * BASE_LEN) - frame_len; - - pkt_hdr->buf_hdr.seg[0].data += offset; - pkt_hdr->buf_hdr.seg[0].len -= offset; - - pkt_hdr->buf_hdr.segcount = segs; - pkt_hdr->frame_len = frame_len; - pkt_hdr->headroom = offset + pool->headroom; - pkt_hdr->tailroom = pool->tailroom; - - /* Data was moved */ - ret = 1; - } else { - void *ptr; + num = num_segments(len - headroom); + push_head(pkt_hdr, headroom); + ptr = add_segments(pkt_hdr, pool, len - headroom, num, 1); - push_head(pkt_hdr, headroom); - ptr = add_segments(pkt_hdr, pool, len - headroom, - num, 1); - - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_head(pkt_hdr, headroom); - return -1; - } - - *pkt = packet_handle(ptr); - pkt_hdr = ptr; + if (ptr == NULL) { + /* segment alloc failed, rollback changes */ + pull_head(pkt_hdr, headroom); + return -1; } + + *pkt = packet_handle(ptr); + pkt_hdr = ptr; } else { push_head(pkt_hdr, len); } @@ -1001,75 +912,19 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, if (len > tailroom) { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; - int segs; + void *ptr; if (odp_unlikely((frame_len + len) > pool->max_len)) return -1; - num = num_segments(len - tailroom); - segs = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Cannot directly add new segments */ - odp_packet_hdr_t *new_hdr; - int new_segs = 0; - int free_segs = 0; - uint32_t offset; - - num = num_segments(frame_len + len); - - if (num > segs) { - /* Allocate additional segments */ - new_segs = num - segs; - new_hdr = alloc_segments(pool, new_segs); - - if (new_hdr == NULL) - return -1; - - } else if (num < segs) { - free_segs = segs - num; - } - - /* Pack all data to packet head */ - move_data_to_head(pkt_hdr, segs); - reset_seg(pkt_hdr, 0, segs); + num = num_segments(len - tailroom); + push_tail(pkt_hdr, tailroom); + ptr = add_segments(pkt_hdr, pool, len - tailroom, num, 0); - if (new_segs) { - /* Add new segs */ - add_all_segs(pkt_hdr, new_hdr); - segs += new_segs; - } else if (free_segs) { - /* Free extra segs */ - free_bufs(pkt_hdr, segs - free_segs, free_segs); - - segs -= free_segs; - } - - frame_len += len; - offset = (segs * BASE_LEN) - frame_len; - - pkt_hdr->buf_hdr.seg[segs - 1].len -= offset; - - pkt_hdr->buf_hdr.segcount = segs; - pkt_hdr->frame_len = frame_len; - pkt_hdr->headroom = pool->headroom; - pkt_hdr->tailroom = offset + pool->tailroom; - - /* Data was moved */ - ret = 1; - } else { - void *ptr; - - push_tail(pkt_hdr, tailroom); - - ptr = add_segments(pkt_hdr, pool, len - tailroom, - num, 0); - - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_tail(pkt_hdr, tailroom); - return -1; - } + if (ptr == NULL) { + /* segment alloc failed, rollback changes */ + pull_tail(pkt_hdr, tailroom); + return -1; } } else { push_tail(pkt_hdr, len); @@ -1084,8 +939,9 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + seg_entry_t *last_seg = seg_entry_last(pkt_hdr); - if (len > packet_last_seg_len(pkt_hdr)) + if (len > last_seg->len) return NULL; pull_tail(pkt_hdr, len); @@ -1098,13 +954,15 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; + seg_entry_t *last_seg; odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); if (len > pkt_hdr->frame_len) return -1; - last = packet_last_seg(pkt_hdr); - seg_len = packet_seg_len(pkt_hdr, last); + last = packet_last_seg(pkt_hdr); + last_seg = seg_entry_last(pkt_hdr); + seg_len = last_seg->len; if (len < seg_len) { pull_tail(pkt_hdr, len); @@ -1397,17 +1255,13 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) { odp_packet_hdr_t *dst_hdr = packet_hdr(*dst); odp_packet_hdr_t *src_hdr = packet_hdr(src); - int dst_segs = dst_hdr->buf_hdr.segcount; - int src_segs = src_hdr->buf_hdr.segcount; pool_t *dst_pool = dst_hdr->buf_hdr.pool_ptr; pool_t *src_pool = src_hdr->buf_hdr.pool_ptr; uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; - /* Do a copy if resulting packet would be out of segments or packets - * are from different pools. */ - if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool)) { + /* Do a copy if packets are from different pools. */ + if (odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 01edff8b..47a39f5b 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -268,6 +268,9 @@ static void init_buffers(pool_t *pool) /* Show user requested size through API */ buf_hdr->uarea_size = pool->params.pkt.uarea_size; buf_hdr->segcount = 1; + buf_hdr->num_seg = 1; + buf_hdr->next_seg = NULL; + buf_hdr->last_seg = buf_hdr; /* Pointer to data start (of the first segment) */ buf_hdr->seg[0].hdr = buf_hdr; commit 20c75b764af6fd15e5e1d4df969ac33d62525405 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Wed Aug 23 13:30:19 2017 +0300 validation: packet: check reference packet content Check that reference and base packets contain the same data. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/common_plat/validation/api/packet/packet.c b/test/common_plat/validation/api/packet/packet.c index f291b6ee..10e011df 100644 --- a/test/common_plat/validation/api/packet/packet.c +++ b/test/common_plat/validation/api/packet/packet.c @@ -2072,14 +2072,107 @@ void packet_test_ref(void) { odp_packet_t base_pkt, segmented_base_pkt, hdr_pkt[4], ref_pkt[4], refhdr_pkt[4], hdr_cpy; + odp_packet_t pkt, pkt2, pkt3, ref, ref2; uint32_t pkt_len, segmented_pkt_len, hdr_len[4], offset[4], hr[4], base_hr, ref_len[4]; - int i; + int i, ret; + odp_pool_t pool; + + /* Create references and compare data */ + pool = odp_packet_pool(test_packet); + + pkt = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref_static(pkt); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + pkt = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref(pkt, 0); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + pkt = odp_packet_copy(test_packet, pool); + pkt3 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt3 != ODP_PACKET_INVALID) + ret = odp_packet_concat(&pkt3, pkt); + CU_ASSERT_FATAL(ret >= 0); + + pkt = odp_packet_copy(test_packet, pool); + pkt2 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + ref = odp_packet_ref_pkt(pkt, 0, pkt2); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt3, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + odp_packet_free(pkt3); + + /* Do the same for segmented packets */ + pool = odp_packet_pool(segmented_test_packet); + + pkt = odp_packet_copy(segmented_test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref_static(pkt); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + pkt = odp_packet_copy(segmented_test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref(pkt, 0); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + /* Avoid to create too large packets with concat */ + pool = odp_packet_pool(test_packet); + + pkt = odp_packet_copy(test_packet, pool); + pkt2 = odp_packet_copy(test_packet, pool); + pkt3 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt3 != ODP_PACKET_INVALID) + ret = odp_packet_concat(&pkt3, pkt2); + CU_ASSERT_FATAL(ret >= 0); + ret = odp_packet_concat(&pkt3, pkt); + CU_ASSERT_FATAL(ret >= 0); + + pkt = odp_packet_copy(test_packet, pool); + pkt2 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + ref = odp_packet_ref_pkt(pkt, 0, pkt2); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + pkt2 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + ref2 = odp_packet_ref_pkt(ref, 0, pkt2); + CU_ASSERT_FATAL(ref2 != ODP_PACKET_INVALID) + packet_compare_data(pkt3, ref2); + + /* Try print function on a reference */ + odp_packet_print(ref2); + + odp_packet_free(ref); + odp_packet_free(ref2); + odp_packet_free(pkt); + odp_packet_free(pkt3); + /* Test has_ref, unshared_len, lengths, etc */ base_pkt = odp_packet_copy(test_packet, odp_packet_pool(test_packet)); - base_hr = odp_packet_headroom(base_pkt); - pkt_len = odp_packet_len(test_packet); CU_ASSERT_FATAL(base_pkt != ODP_PACKET_INVALID); + base_hr = odp_packet_headroom(base_pkt); + pkt_len = odp_packet_len(test_packet); segmented_base_pkt = odp_packet_copy(segmented_test_packet, commit 04029613fbdc85221d1a20354a49ada912302fc0 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Mon Aug 21 13:54:23 2017 +0300 validation: packet: unshared len with static reference When static references are used all data is shared, and thus unshared length is zero. When there's no references, all data is unique to the packet (unshared len == len). Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/common_plat/validation/api/packet/packet.c b/test/common_plat/validation/api/packet/packet.c index 898dac87..f291b6ee 100644 --- a/test/common_plat/validation/api/packet/packet.c +++ b/test/common_plat/validation/api/packet/packet.c @@ -2224,8 +2224,14 @@ void packet_test_ref(void) CU_ASSERT(ref_pkt[0] != ODP_PACKET_INVALID); if (odp_packet_has_ref(base_pkt) == 1) { - /* CU_ASSERT needs braces */ CU_ASSERT(odp_packet_has_ref(ref_pkt[0]) == 1); + CU_ASSERT(odp_packet_unshared_len(base_pkt) == 0); + CU_ASSERT(odp_packet_unshared_len(ref_pkt[0]) == 0); + } else { + CU_ASSERT(odp_packet_unshared_len(base_pkt) == + odp_packet_len(base_pkt)); + CU_ASSERT(odp_packet_unshared_len(ref_pkt[0]) == + odp_packet_len(ref_pkt[0])); } CU_ASSERT(odp_packet_len(ref_pkt[0]) == odp_packet_len(base_pkt)); commit 88d36e6811de494708c520cb12e9e5f97628e9e4 Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Fri Aug 18 11:04:49 2017 +0300 validation: packet: relax small concat test Limit number of consecutive small concat to fraction of the number of packets in the pool. An implementation linking packets together (no copy) cannot be expected to support one byte concats up to max packet size. Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/common_plat/validation/api/packet/packet.c b/test/common_plat/validation/api/packet/packet.c index 28bc12b0..898dac87 100644 --- a/test/common_plat/validation/api/packet/packet.c +++ b/test/common_plat/validation/api/packet/packet.c @@ -1376,7 +1376,7 @@ void packet_test_concat_small(void) int ret; uint8_t *data; uint32_t i; - uint32_t len = 32000; + uint32_t len = PACKET_POOL_NUM / 4; uint8_t buf[len]; CU_ASSERT_FATAL(odp_pool_capability(&capa) == 0); commit a820d8a84e132652b4cef295756ccf135e3bd54b Author: Petri Savolainen <petri.savolainen(a)linaro.org> Date: Thu Aug 10 13:43:52 2017 +0300 linux-gen: packet: roll back to copy based references In order to re-implement segmentation for the new reference implementation roll back to the copy based reference implementation, which have been proved to work OK. This zero copy reference implementation introduced multiple issues: * segment pointers and length were not set correctly, but were left to NULL * reference counting has a race condition which causes random assert failures also with non-reference packets * there's larger than expected performance drop with non-reference packets * the first segment of a reference was zero bytes long, which is possible by the API but is unusual for users Signed-off-by: Petri Savolainen <petri.savolainen(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 06b049fc..d0cf1390 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -50,19 +50,7 @@ static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_len(odp_packet_t pkt) { - uint32_t pkt_len = _odp_pkt_get(pkt, uint32_t, frame_len); - void *ref_nxt = _odp_pkt_get(pkt, void *, ref_hdr); - void *ref_pkt = (void *)pkt; - - while (ref_nxt) { - pkt_len += _odp_pkt_get(ref_pkt, uint32_t, ref_len) - - _odp_pkt_get(ref_pkt, uint32_t, ref_offset); - - ref_pkt = ref_nxt; - ref_nxt = _odp_pkt_get(ref_nxt, void *, ref_hdr); - } - - return pkt_len; + return _odp_pkt_get(pkt, uint32_t, frame_len); } /** @internal Inline function @param pkt @return */ @@ -90,6 +78,12 @@ static inline odp_pktio_t _odp_packet_input(odp_packet_t pkt) } /** @internal Inline function @param pkt @return */ +static inline int _odp_packet_num_segs(odp_packet_t pkt) +{ + return _odp_pkt_get(pkt, uint8_t, segcount); +} + +/** @internal Inline function @param pkt @return */ static inline void *_odp_packet_user_ptr(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, user_ptr); @@ -128,8 +122,7 @@ static inline void *_odp_packet_head(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline int _odp_packet_is_segmented(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint8_t, segcount) > 1 || - _odp_pkt_get(pkt, void *, ref_hdr) != NULL; + return _odp_pkt_get(pkt, uint8_t, segcount) > 1; } /** @internal Inline function @param pkt @return */ @@ -140,6 +133,23 @@ static inline odp_packet_seg_t _odp_packet_first_seg(odp_packet_t pkt) return _odp_packet_seg_from_ndx(0); } +/** @internal Inline function @param pkt @return */ +static inline odp_packet_seg_t _odp_packet_last_seg(odp_packet_t pkt) +{ + return _odp_packet_seg_from_ndx(_odp_packet_num_segs(pkt) - 1); +} + +/** @internal Inline function @param pkt @param seg @return */ +static inline odp_packet_seg_t _odp_packet_next_seg(odp_packet_t pkt, + odp_packet_seg_t seg) +{ + if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= + _odp_packet_seg_to_ndx(_odp_packet_last_seg(pkt)))) + return ODP_PACKET_SEG_INVALID; + + return seg + 1; +} + /** @internal Inline function @param pkt @param offset @param len */ static inline void _odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h index f818f820..233bc876 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h @@ -48,6 +48,11 @@ _ODP_INLINE odp_pktio_t odp_packet_input(odp_packet_t pkt) return _odp_packet_input(pkt); } +_ODP_INLINE int odp_packet_num_segs(odp_packet_t pkt) +{ + return _odp_packet_num_segs(pkt); +} + _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) { return _odp_packet_user_ptr(pkt); @@ -88,6 +93,17 @@ _ODP_INLINE odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) return _odp_packet_first_seg(pkt); } +_ODP_INLINE odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) +{ + return _odp_packet_last_seg(pkt); +} + +_ODP_INLINE odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, + odp_packet_seg_t seg) +{ + return _odp_packet_next_seg(pkt, seg); +} + _ODP_INLINE void odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) { diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 1554c3af..41468bec 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -84,14 +84,6 @@ typedef struct _odp_packet_inline_offset_t { /** @internal field offset */ size_t tailroom; /** @internal field offset */ - size_t unshared_len; - /** @internal field offset */ - size_t ref_hdr; - /** @internal field offset */ - size_t ref_offset; - /** *internal field offset */ - size_t ref_len; - /** @internal field offset */ size_t pool; /** @internal field offset */ size_t input; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index d923ee23..f77987c8 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -19,7 +19,6 @@ extern "C" { #include <odp/api/align.h> #include <odp/api/debug.h> -#include <odp_debug_internal.h> #include <odp_buffer_internal.h> #include <odp_pool_internal.h> #include <odp_buffer_inlines.h> @@ -107,7 +106,7 @@ typedef struct { * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct odp_packet_hdr_t { +typedef struct { /* common buffer header */ odp_buffer_hdr_t buf_hdr; @@ -123,19 +122,6 @@ typedef struct odp_packet_hdr_t { uint32_t headroom; uint32_t tailroom; - /* Fields used to support packet references */ - uint32_t unshared_len; - /* Next pkt_hdr in reference chain */ - struct odp_packet_hdr_t *ref_hdr; - /* Offset into next pkt_hdr that ref was created at */ - uint32_t ref_offset; - /* frame_len in next pkt_hdr at time ref was created. This - * allows original offset to be maintained when base pkt len - * is changed */ - uint32_t ref_len; - /* Incremented on refs, decremented on frees. */ - odp_atomic_u32_t ref_count; - /* * Members below are not initialized by packet_init() */ @@ -171,50 +157,6 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) return (odp_packet_hdr_t *)(uintptr_t)pkt; } -static inline odp_packet_hdr_t *packet_last_hdr(odp_packet_t pkt, - uint32_t *offset) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - odp_packet_hdr_t *prev_hdr = pkt_hdr; - uint32_t ref_offset = 0; - - while (pkt_hdr->ref_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - -static inline odp_packet_hdr_t *packet_prev_hdr(odp_packet_hdr_t *pkt_hdr, - odp_packet_hdr_t *cur_hdr, - uint32_t *offset) -{ - uint32_t ref_offset = 0; - odp_packet_hdr_t *prev_hdr = pkt_hdr; - - while (pkt_hdr->ref_hdr != cur_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) { return (odp_packet_t)pkt_hdr; @@ -267,10 +209,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) CONFIG_PACKET_TAILROOM; pkt_hdr->input = ODP_PKTIO_INVALID; - - /* By default packet has no references */ - pkt_hdr->unshared_len = len; - pkt_hdr->ref_hdr = NULL; } static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, @@ -295,49 +233,17 @@ static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[last].len -= len; } static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { - uint32_t pkt_len = pkt_hdr->frame_len; - odp_packet_hdr_t *ref_hdr = pkt_hdr->ref_hdr; - - while (ref_hdr) { - pkt_len += (pkt_hdr->ref_len - pkt_hdr->ref_offset); - pkt_hdr = ref_hdr; - ref_hdr = ref_hdr->ref_hdr; - } - - return pkt_len; -} - -static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) -{ - /* Breach the atomic type to do a peek at the ref count. This - * is used to bypass atomic operations if ref_count == 1 for - * performance reasons. - */ - return pkt_hdr->ref_count.v; -} - -static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n) -{ - /* Only used during init when there are no other possible - * references to this pkt, so avoid the "atomic" overhead by - * a controlled breach of the atomic type here. This saves - * over 10% of the pathlength in routines like packet_alloc(). - */ - pkt_hdr->ref_count.v = n; + return pkt_hdr->frame_len; } static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - pkt_hdr->frame_len = len; - pkt_hdr->unshared_len = len; } /* Forward declarations */ diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 1fc715ca..1e5b136a 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -35,10 +35,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), - .unshared_len = offsetof(odp_packet_hdr_t, unshared_len), - .ref_hdr = offsetof(odp_packet_hdr_t, ref_hdr), - .ref_offset = offsetof(odp_packet_hdr_t, ref_offset), - .ref_len = offsetof(odp_packet_hdr_t, ref_len), .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_hdl), .input = offsetof(odp_packet_hdr_t, input), .segcount = offsetof(odp_packet_hdr_t, buf_hdr.segcount), @@ -63,16 +59,6 @@ static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) return (odp_buffer_t)pkt_hdr; } -static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) -{ - odp_atomic_inc_u32(&pkt_hdr->ref_count); -} - -static inline uint32_t packet_ref_dec(odp_packet_hdr_t *pkt_hdr) -{ - return odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count); -} - static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) { return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); @@ -94,13 +80,12 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, return pkt_hdr->buf_hdr.seg[seg_idx].len; } -static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, - uint32_t seg_idx) +static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { return pkt_hdr->buf_hdr.seg[seg_idx].data; } -static inline uint32_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) +static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) { if (CONFIG_PACKET_MAX_SEGS == 1) return 0; @@ -155,7 +140,6 @@ static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[0].data -= len; pkt_hdr->buf_hdr.seg[0].len += len; } @@ -164,7 +148,6 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[0].data += len; pkt_hdr->buf_hdr.seg[0].len -= len; } @@ -175,7 +158,6 @@ static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[last].len += len; } @@ -203,10 +185,6 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; - /* reference related metadata */ - dst->ref_len = src->ref_len; - dst->unshared_len = src->unshared_len; - /* segmentation data is not copied: * buf_hdr.seg[] * buf_hdr.segcount @@ -221,15 +199,7 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, int seg = 0; int seg_count = pkt_hdr->buf_hdr.segcount; - /* Special processing for references */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - seg_count = pkt_hdr->buf_hdr.segcount; - } - - if (odp_unlikely(offset > pkt_hdr->frame_len)) + if (odp_unlikely(offset >= pkt_hdr->frame_len)) return NULL; if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { @@ -271,9 +241,6 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - - /* Ensure dummy pkt hdrs used in I/O recv classification are valid */ - pkt_hdr->ref_hdr = NULL; } static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) @@ -286,7 +253,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; hdr->buf_hdr.seg[0].len = BASE_LEN; - packet_ref_count_set(hdr, 1); /* Link segments */ if (CONFIG_PACKET_MAX_SEGS != 1) { @@ -296,7 +262,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) for (i = 1; i < num; i++) { odp_buffer_hdr_t *buf_hdr; - packet_ref_count_set(pkt_hdr[i], 1); buf_hdr = &pkt_hdr[i]->buf_hdr; hdr->buf_hdr.seg[i].hdr = buf_hdr; hdr->buf_hdr.seg[i].data = buf_hdr->base_data; @@ -414,10 +379,9 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, new_hdr->buf_hdr.seg[0].len = seg_len; packet_seg_copy_md(new_hdr, pkt_hdr); - new_hdr->frame_len = pkt_hdr->frame_len + len; - new_hdr->unshared_len = pkt_hdr->unshared_len + len; - new_hdr->headroom = pool->headroom + offset; - new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->frame_len = pkt_hdr->frame_len + len; + new_hdr->headroom = pool->headroom + offset; + new_hdr->tailroom = pkt_hdr->tailroom; pkt_hdr = new_hdr; } else { @@ -430,9 +394,8 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, last = packet_last_seg(pkt_hdr); pkt_hdr->buf_hdr.seg[last].len = seg_len; - pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; - pkt_hdr->tailroom = pool->tailroom + offset; + pkt_hdr->frame_len += len; + pkt_hdr->tailroom = pool->tailroom + offset; } return pkt_hdr; @@ -440,20 +403,13 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) { - int i, nfree; + int i; odp_buffer_hdr_t *buf_hdr[num]; - for (i = 0, nfree = 0; i < num; i++) { - odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr; - - if (packet_ref_count(hdr) == 1 || packet_ref_dec(hdr) == 1) { - ODP_ASSERT((packet_ref_count_set(hdr, 0), 1)); - buf_hdr[nfree++] = &hdr->buf_hdr; - } - } + for (i = 0; i < num; i++) + buf_hdr[i] = pkt_hdr->buf_hdr.seg[first + i].hdr; - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + buffer_free_multi(buf_hdr, num); } static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, @@ -464,19 +420,11 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, if (head) { odp_packet_hdr_t *new_hdr; - int i, nfree; + int i; odp_buffer_hdr_t *buf_hdr[num]; - for (i = 0, nfree = 0; i < num; i++) { - new_hdr = pkt_hdr->buf_hdr.seg[i].hdr; - - if (packet_ref_count(new_hdr) == 1 || - packet_ref_dec(new_hdr) == 1) { - ODP_ASSERT((packet_ref_count_set(new_hdr, 0), - 1)); - buf_hdr[nfree++] = &new_hdr->buf_hdr; - } - } + for (i = 0; i < num; i++) + buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr; /* First remaining segment is the new packet descriptor */ new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; @@ -485,17 +433,15 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, packet_seg_copy_md(new_hdr, pkt_hdr); /* Tailroom not changed */ - new_hdr->tailroom = pkt_hdr->tailroom; - new_hdr->headroom = seg_headroom(new_hdr, 0); - new_hdr->frame_len = pkt_hdr->frame_len - free_len; - new_hdr->unshared_len = pkt_hdr->unshared_len - free_len; + new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->headroom = seg_headroom(new_hdr, 0); + new_hdr->frame_len = pkt_hdr->frame_len - free_len; pull_head(new_hdr, pull_len); pkt_hdr = new_hdr; - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + buffer_free_multi(buf_hdr, num); } else { /* Free last 'num' bufs */ free_bufs(pkt_hdr, num_remain, num); @@ -504,7 +450,6 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, * of the metadata. */ pkt_hdr->buf_hdr.segcount = num_remain; pkt_hdr->frame_len -= free_len; - pkt_hdr->unshared_len -= free_len; pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); pull_tail(pkt_hdr, pull_len); @@ -611,80 +556,48 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, return num; } -static inline void packet_free(odp_packet_hdr_t *pkt_hdr) +void odp_packet_free(odp_packet_t pkt) { - odp_packet_hdr_t *ref_hdr; - odp_buffer_hdr_t *buf_hdr; - uint32_t ref_count; - int num_seg; - - do { - buf_hdr = &pkt_hdr->buf_hdr; - ref_count = packet_ref_count(pkt_hdr); - num_seg = pkt_hdr->buf_hdr.segcount; - ref_hdr = pkt_hdr->ref_hdr; - ODP_ASSERT(ref_count >= 1); - - if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) && - ref_count == 1)) { - ODP_ASSERT((packet_ref_count_set(pkt_hdr, 0), 1)); - buffer_free_multi(&buf_hdr, 1); - } else { - free_bufs(pkt_hdr, 0, num_seg); - } + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_buffer_t hdl = buffer_handle(pkt_hdr); - pkt_hdr = ref_hdr; - } while (pkt_hdr); -} + int num_seg = pkt_hdr->buf_hdr.segcount; -void odp_packet_free(odp_packet_t pkt) -{ - packet_free(packet_hdr(pkt)); + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) + buffer_free_multi((odp_buffer_hdr_t **)&hdl, 1); + else + free_bufs(pkt_hdr, 0, num_seg); } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - odp_packet_hdr_t *pkt_hdr, *ref_hdr, *hdr; - int nbufs = num * CONFIG_PACKET_MAX_SEGS * 2; - odp_buffer_hdr_t *buf_hdr[nbufs]; - int num_seg; - int i, j; - uint32_t ref_count; - int nfree = 0; + if (CONFIG_PACKET_MAX_SEGS == 1) { + buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num); + } else { + odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS]; + int i; + int j; + int bufs = 0; - for (i = 0; i < num; i++) { - pkt_hdr = packet_hdr(pkt[i]); + for (i = 0; i < num; i++) { + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); + int num_seg = pkt_hdr->buf_hdr.segcount; + odp_buffer_hdr_t *hdr = &pkt_hdr->buf_hdr; - do { - num_seg = pkt_hdr->buf_hdr.segcount; - ref_hdr = pkt_hdr->ref_hdr; + buf_hdr[bufs] = hdr; + bufs++; - /* Make sure we have enough space for this pkt's segs */ - if (nfree + num_seg > nbufs) { - buffer_free_multi(buf_hdr, nfree); - nfree = 0; - } + if (odp_likely(num_seg == 1)) + continue; - for (j = 0; j < num_seg; j++) { - hdr = pkt_hdr->buf_hdr.seg[j].hdr; - ref_count = packet_ref_count(hdr); - ODP_ASSERT(ref_count >= 1); - - if (ref_count == 1 || - packet_ref_dec(hdr) == 1) { - ODP_ASSERT - ((packet_ref_count_set(hdr, 0), - 1)); - buf_hdr[nfree++] = &hdr->buf_hdr; - } + for (j = 1; j < num_seg; j++) { + buf_hdr[bufs] = hdr->seg[j].hdr; + bufs++; } + } - pkt_hdr = ref_hdr; - } while (pkt_hdr); + buffer_free_multi(buf_hdr, bufs); } - - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -696,9 +609,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (odp_unlikely(len > (pool->max_seg_len * num))) return -1; - if (pkt_hdr->ref_hdr) - packet_free(pkt_hdr->ref_hdr); - reset_seg(pkt_hdr, 0, num); packet_init(pkt_hdr, len); @@ -732,47 +642,13 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt) uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t buf_len = 0; - - do { - buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - return buf_len; -} - -uint32_t odp_packet_unshared_len(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pkt_len = 0, offset = 0; - - if (packet_ref_count(pkt_hdr) == 1) - pkt_hdr->unshared_len = pkt_hdr->frame_len; - - do { - if (packet_ref_count(pkt_hdr) > 1) { - if (offset == 0) - pkt_len += pkt_hdr->unshared_len; - break; - } - - pkt_len += pkt_hdr->frame_len - offset; - offset = pkt_hdr->ref_offset; - - if (pkt_hdr->ref_hdr) - offset += (pkt_hdr->ref_hdr->frame_len - - pkt_hdr->ref_len); - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return pkt_len; + return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; } void *odp_packet_tail(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); return packet_tail(pkt_hdr); } @@ -968,13 +844,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, segs = pkt_hdr->buf_hdr.segcount; if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Corner case: fail request if packet has - * references since we cannot shuffle segments - * since another thread may be accessing them - * concurrently */ - if (packet_ref_count(pkt_hdr) > 1) - return -1; - /* Cannot directly add new segments */ odp_packet_hdr_t *new_hdr; int new_segs = 0; @@ -1026,7 +895,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, pkt_hdr->buf_hdr.segcount = segs; pkt_hdr->frame_len = frame_len; - pkt_hdr->unshared_len = frame_len; pkt_hdr->headroom = offset + pool->headroom; pkt_hdr->tailroom = pool->tailroom; @@ -1052,16 +920,11 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, push_head(pkt_hdr, len); } - if (data_ptr || seg_len) { - uint32_t seg_ln = 0; - void *data = packet_map(pkt_hdr, 0, &seg_ln, NULL); - - if (data_ptr) - *data_ptr = data; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (seg_len) - *seg_len = seg_ln; - } + if (seg_len) + *seg_len = packet_first_seg_len(pkt_hdr); return ret; } @@ -1073,8 +936,6 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->frame_len) return NULL; - ODP_ASSERT(len <= pkt_hdr->unshared_len); - pull_head(pkt_hdr, len); return packet_data(pkt_hdr); } @@ -1082,35 +943,15 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt), *nxt_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); uint32_t seg_len = packet_first_seg_len(pkt_hdr); - int ret = 0; - if (len > packet_len(pkt_hdr)) + if (len > pkt_hdr->frame_len) return -1; - ODP_ASSERT(len <= odp_packet_unshared_len(*pkt)); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - nxt_hdr = pkt_hdr->ref_hdr; - len -= pkt_hdr->frame_len; - len += pkt_hdr->ref_offset + - (nxt_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = nxt_hdr; - seg_len = packet_first_seg_len(pkt_hdr); - *pkt = packet_handle(pkt_hdr); - ret = 1; - } - - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_head(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0; @@ -1125,29 +966,23 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, *pkt = packet_handle(pkt_hdr); } - if (data_ptr || seg_len_out) { - void *data_head = packet_map(pkt_hdr, 0, &seg_len, NULL); + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (data_ptr) - *data_ptr = data_head; + if (seg_len_out) + *seg_len_out = packet_first_seg_len(pkt_hdr); - if (seg_len_out) - *seg_len_out = seg_len; - } - - return ret; + return 0; } void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); void *old_tail; if (len > pkt_hdr->tailroom) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - old_tail = packet_tail(pkt_hdr); push_tail(pkt_hdr, len); @@ -1157,14 +992,12 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(*pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); uint32_t frame_len = pkt_hdr->frame_len; uint32_t tailroom = pkt_hdr->tailroom; uint32_t tail_off = frame_len; int ret = 0; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - if (len > tailroom) { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; @@ -1255,7 +1088,6 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) if (len > packet_last_seg_len(pkt_hdr)) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); pull_tail(pkt_hdr, len); return packet_tail(pkt_hdr); @@ -1266,34 +1098,17 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; - uint32_t offset; - odp_packet_hdr_t *first_hdr = packet_hdr(*pkt); - odp_packet_hdr_t *pkt_hdr, *prev_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - if (len > packet_len(first_hdr)) + if (len > pkt_hdr->frame_len) return -1; - pkt_hdr = packet_last_hdr(*pkt, &offset); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len - offset && first_hdr->ref_hdr) { - len -= (pkt_hdr->frame_len - offset); - prev_hdr = packet_prev_hdr(first_hdr, pkt_hdr, &offset); - ODP_ASSERT(packet_ref_count(prev_hdr) == 1); - prev_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = prev_hdr; - } - - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); last = packet_last_seg(pkt_hdr); seg_len = packet_seg_len(pkt_hdr, last); - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_tail(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0; @@ -1440,46 +1255,6 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) pkt_hdr->p.input_flags.timestamp = 1; } -int odp_packet_num_segs(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t segcount = 0, i; - uint32_t seg_offset = 0, offset; - - do { - segcount += pkt_hdr->buf_hdr.segcount - seg_offset; - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - if (pkt_hdr) { - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - } while (pkt_hdr); - - return segcount; -} - -odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) -{ - return _odp_packet_seg_from_ndx(odp_packet_num_segs(pkt) - 1); -} - -odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= - packet_last_seg(pkt_hdr))) - return ODP_PACKET_SEG_INVALID; - - return seg + 1; -} - /* * * Segment level @@ -1490,53 +1265,23 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg); - - while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= + pkt_hdr->buf_hdr.segcount)) return NULL; - return packet_seg_data(pkt_hdr, seg_ndx + seg_offset) + offset; + return packet_seg_data(pkt_hdr, _odp_packet_seg_to_ndx(seg)); } uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg); - - while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= + pkt_hdr->buf_hdr.segcount)) return 0; - return packet_seg_len(pkt_hdr, seg_ndx + seg_offset) - offset; + return packet_seg_len(pkt_hdr, _odp_packet_seg_to_ndx(seg)); } /* @@ -1557,8 +1302,6 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) if (offset > pktlen) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset); - newpkt = odp_packet_alloc(pool->pool_hdl, pktlen + len); if (newpkt == ODP_PACKET_INVALID) @@ -1582,15 +1325,13 @@ int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pktlen = packet_len(pkt_hdr); + uint32_t pktlen = pkt_hdr->frame_len; pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; odp_packet_t newpkt; - if (offset + len > pktlen) + if (offset > pktlen || offset + len > pktlen) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset + len); - newpkt = odp_packet_alloc(pool->pool_hdl, pktlen - len); if (newpkt == ODP_PACKET_INVALID) @@ -1624,8 +1365,6 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - if (seglen >= len) { misalign = align <= 1 ? 0 : ROUNDUP_ALIGN(uaddr, align) - uaddr; @@ -1665,13 +1404,10 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; - ODP_ASSERT(packet_ref_count(dst_hdr) == 1); - /* Do a copy if resulting packet would be out of segments or packets - * are from different pools or src is a reference. */ + * are from different pools. */ if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool) || - odp_unlikely(packet_ref_count(src_hdr) > 1)) { + odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); @@ -1686,9 +1422,8 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) add_all_segs(dst_hdr, src_hdr); - dst_hdr->frame_len = dst_len + src_len; - dst_hdr->unshared_len = dst_len + src_len; - dst_hdr->tailroom = src_hdr->tailroom; + dst_hdr->frame_len = dst_len + src_len; + dst_hdr->tailroom = src_hdr->tailroom; /* Data was not moved in memory */ return 0; @@ -1701,7 +1436,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) if (len >= pktlen || tail == NULL) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt) >= len); *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt)); @@ -1712,109 +1446,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) } /* - * References - */ - -static inline void packet_ref(odp_packet_hdr_t *pkt_hdr) -{ - uint32_t i; - odp_packet_hdr_t *hdr; - - do { - for (i = 0; i < pkt_hdr->buf_hdr.segcount; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - packet_ref_inc(hdr); - } - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); -} - -static inline odp_packet_t packet_splice(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - odp_packet_hdr_t *ref_hdr) -{ - /* Catch attempted references to stale handles in debug builds */ - ODP_ASSERT(packet_ref_count(pkt_hdr) > 0); - - /* Splicing is from the last section of src pkt */ - while (ref_hdr->ref_hdr) - ref_hdr = ref_hdr->ref_hdr; - - /* Find section where splice begins */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - } - - ref_hdr->ref_hdr = pkt_hdr; - ref_hdr->ref_offset = offset; - ref_hdr->ref_len = pkt_hdr->frame_len; - - if (packet_ref_count(pkt_hdr) == 1 || offset < pkt_hdr->unshared_len) - pkt_hdr->unshared_len = offset; - - packet_ref(pkt_hdr); - return packet_handle(ref_hdr); -} - -odp_packet_t odp_packet_ref_static(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - packet_ref(pkt_hdr); - pkt_hdr->unshared_len = 0; - return pkt; -} - -odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_t hdr; - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - pkt_hdr = packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - hdr = odp_packet_alloc(odp_packet_pool(pkt), 0); - - if (hdr == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); -} - -odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, - odp_packet_t hdr) -{ - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID || - hdr == ODP_PACKET_INVALID || - pkt == hdr) - return ODP_PACKET_INVALID; - - ODP_ASSERT(odp_packet_has_ref(hdr) == 0); - - pkt_hdr = packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); -} - -int odp_packet_has_ref(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->ref_hdr != NULL || packet_ref_count(pkt_hdr) > 1; -} - -/* * * Copy * ******************************************************** @@ -1823,7 +1454,8 @@ int odp_packet_has_ref(odp_packet_t pkt) odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { - uint32_t pktlen = odp_packet_len(pkt); + odp_packet_hdr_t *srchdr = packet_hdr(pkt); + uint32_t pktlen = srchdr->frame_len; odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); if (newpkt != ODP_PACKET_INVALID) { @@ -1862,7 +1494,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, uint8_t *dstaddr = (uint8_t *)dst; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1; while (len > 0) { @@ -1886,11 +1518,9 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, const uint8_t *srcaddr = (const uint8_t *)src; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); - while (len > 0) { mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); cpylen = len > seglen ? seglen : len; @@ -1916,12 +1546,10 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, uint32_t src_seglen = 0; /* GCC */ int overlap; - if (dst_offset + len > packet_len(dst_hdr) || - src_offset + len > packet_len(src_hdr)) + if (dst_offset + len > dst_hdr->frame_len || + src_offset + len > src_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); - overlap = (dst_hdr == src_hdr && ((dst_offset <= src_offset && dst_offset + len >= src_offset) || @@ -2054,7 +1682,7 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", packet_len(hdr)); + " frame_len %" PRIu32 "\n", hdr->frame_len); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); @@ -2458,6 +2086,80 @@ uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl) return _odp_pri(hdl); } +odp_packet_t odp_packet_ref_static(odp_packet_t pkt) +{ + return odp_packet_copy(pkt, odp_packet_pool(pkt)); +} + +odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_t new; + int ret; + + new = odp_packet_copy(pkt, odp_packet_pool(pkt)); + + if (new == ODP_PACKET_INVALID) { + ODP_ERR("copy failed\n"); + return ODP_PACKET_INVALID; + } + + ret = odp_packet_trunc_head(&new, offset, NULL, NULL); + + if (ret < 0) { + ODP_ERR("trunk_head failed\n"); + odp_packet_free(new); + return ODP_PACKET_INVALID; + } + + return new; +} + +odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, + odp_packet_t hdr) +{ + odp_packet_t new; + int ret; + + new = odp_packet_copy(pkt, odp_packet_pool(pkt)); + + if (new == ODP_PACKET_INVALID) { + ODP_ERR("copy failed\n"); + return ODP_PACKET_INVALID; + } + + if (offset) { + ret = odp_packet_trunc_head(&new, offset, NULL, NULL); + + if (ret < 0) { + ODP_ERR("trunk_head failed\n"); + odp_packet_free(new); + return ODP_PACKET_INVALID; + } + } + + ret = odp_packet_concat(&hdr, new); + + if (ret < 0) { + ODP_ERR("concat failed\n"); + odp_packet_free(new); + return ODP_PACKET_INVALID; + } + + return hdr; +} + +int odp_packet_has_ref(odp_packet_t pkt) +{ + (void)pkt; + + return 0; +} + +uint32_t odp_packet_unshared_len(odp_packet_t pkt) +{ + return odp_packet_len(pkt); +} + /* Include non-inlined versions of API functions */ #if ODP_ABI_COMPAT == 1 #include <odp/api/plat/packet_inlines_api.h> ----------------------------------------------------------------------- Summary of changes: include/odp/arch/default/api/abi/packet.h | 2 +- .../include/odp/api/plat/packet_inlines.h | 44 +- .../include/odp/api/plat/packet_inlines_api.h | 16 + .../include/odp/api/plat/packet_types.h | 38 +- .../linux-generic/include/odp_buffer_internal.h | 71 +- .../linux-generic/include/odp_packet_internal.h | 135 +- platform/linux-generic/odp_packet.c | 1518 +++++++++----------- platform/linux-generic/odp_pool.c | 8 + test/common_plat/validation/api/packet/packet.c | 109 +- 9 files changed, 926 insertions(+), 1015 deletions(-) hooks/post-receive --

7 years, 9 months

[COMMIT] branch master updated. v1.15.0.0-192-gf7c5f3e9

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via f7c5f3e973814e0fc9211e008a37080b1c7d4a76 (commit) via d445395c66edb38abedb918afdba37ec9f7f95af (commit) via b38117eeed5d7bd5a334ef2387bf83d5cb8b9188 (commit) via d0821c9c43fb89e4c0021b174bbc12bef543ffa0 (commit) via 17e829a3b78d6513d22496734c6edbec955cbfd4 (commit) via eccc78540c05dc71095179b613f014648385f3df (commit) via f8c6360667380f6b3fb917ecbe5b941b23dbadf1 (commit) from fb3f36cec108ce9c55241d9f0e66d4832a552b8a (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit f7c5f3e973814e0fc9211e008a37080b1c7d4a76 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Wed Sep 13 18:39:20 2017 +0300 travis: move cross-compilation into build matrix As both native and cross-compilation now use the same script it is possible to move cross-compilation setup back to build matrix. Proper cross-compiler is selected based on actual compiler (gcc/clang) and cross-target. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index ac8a4c3a..43141f9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,10 @@ env: - CONF="--enable-schedule-sp" - CONF="--enable-schedule-iquery" - CONF="--enable-dpdk-zero-copy" + - CROSS_ARCH="arm64" + - CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" + - CROSS_ARCH="powerpc" + - CROSS_ARCH="i386" before_install: @@ -94,10 +98,19 @@ before_install: fi - if [ "${CC#clang}" != "${CC}" ] ; then + if [ -n "$CROSS_ARCH" ] ; + then + export CC="${CC} --target=$CROSS_GNU_TYPE" ; + fi ; export CXX="${CC/clang/clang++}"; elif [ "$CROSS_ARCH" = "i386" ] ; then + export CC="gcc -m32" ; export CXX="g++ -m32" ; + elif [ -n "$CROSS_ARCH" ] ; + then + export CC="$CROSS_GNU_TYPE"-gcc ; + export CXX="$CROSS_GNU_TYPE"-g++ ; fi - if test ! -L /usr/lib/ccache/${CC%% *} ; then sudo ln -s -t /usr/lib/ccache/ `which ${CC%% *}` ; fi - ccache -s @@ -128,15 +141,17 @@ install: - sudo mkdir -p /mnt/huge - sudo mount -t hugetlbfs nodev /mnt/huge - - sudo apt-get -qq update - - sudo apt-get install linux-headers-`uname -r` - - sudo pip install coverage + - if [ -z "$CROSS_ARCH" ] ; + then + sudo apt-get -qq update ; + sudo apt-get install linux-headers-`uname -r` ; + fi - gem install asciidoctor # DPDK pktio. Note that cache must be purged if dpdk version changes. - TARGET=${TARGET:-"x86_64-native-linuxapp-gcc"} - | - if [ ! -f "dpdk/${TARGET}/lib/libdpdk.a" ]; then + if [ -z "$CROSS_ARCH" -a ! -f "dpdk/${TARGET}/lib/libdpdk.a" ]; then git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v17.02 http://dpdk.org/git/dpdk dpdk pushd dpdk git log --oneline --decorate @@ -151,7 +166,7 @@ install: # Netmap pktio - | - if [ ! -f "netmap/LINUX/netmap.ko" ]; then + if [ -z "$CROSS_ARCH" -a ! -f "netmap/LINUX/netmap.ko" ]; then git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v11.2 https://github.com/luigirizzo/netmap.git pushd netmap/LINUX ./configure @@ -193,41 +208,10 @@ script: jobs: include: - stage: test - compiler: aarch64-linux-gnu-gcc - env: TEST="aarch64-linux-gnu" CROSS_ARCH="arm64" - install: gem install asciidoctor - - stage: test - compiler: "\"clang-3.8 --target=aarch64-linux-gnu\"" - env: TEST="clang-3.8 aarch64-linux-gnu" CROSS_ARCH="arm64" - install: gem install asciidoctor - - stage: test - compiler: arm-linux-gnueabihf-gcc - env: TEST="arm-linux-gnueabihf" CROSS_ARCH="armhf" - install: gem install asciidoctor - - stage: test - compiler: "\"clang-3.8 --target=arm-linux-gnueabihf\"" - env: TEST="clang-3.8 arm-linux-gnueabihf" CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" - install: gem install asciidoctor - - stage: test - compiler: powerpc-linux-gnu-gcc - env: TEST="powerpc-linux-gnueabi" CROSS_ARCH="powerpc" - install: gem install asciidoctor - - stage: test - compiler: "\"clang-3.8 --target=powerpc-linux-gnu\"" - env: TEST="clang-3.8 powerpc-linux-gnu" CROSS_ARCH="powerpc" - install: gem install asciidoctor - - stage: test - compiler: "\"gcc -m32\"" - env: TEST="i686-linux-gnueabi" CROSS_ARCH="i386" - install: gem install asciidoctor - - stage: test - compiler: "\"clang-3.8 -m32\"" - env: TEST="clang-3.8 i686-linux-gnu" CROSS_ARCH="i386" - install: gem install asciidoctor - - stage: test env: TEST=coverage compiler: gcc script: + - sudo pip install coverage - ./bootstrap - ./configure --prefix=$HOME/odp-install --enable-user-guides commit d445395c66edb38abedb918afdba37ec9f7f95af Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Wed Sep 13 16:37:20 2017 +0300 travis: merge native and cross-compilation scripts Merge native and cross-compilation scripts. This allows us to simplify Travis CI configuration and to enable e.g. testing of out-of-tree compilation. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 06983f8e..ac8a4c3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -161,27 +161,33 @@ install: fi script: + - if [ -z "$CROSS_ARCH" ] ; then + EXTRA_CONF="--with-dpdk-path=`pwd`/dpdk/${TARGET} --with-netmap-path=`pwd`/netmap" ; + fi - ./bootstrap - ./configure --prefix=$HOME/odp-install --enable-user-guides - --with-dpdk-path=`pwd`/dpdk/${TARGET} - --with-netmap-path=`pwd`/netmap - $CONF --enable-debug=full --enable-helper-linux + $CROSS $EXTRA_CONF $CONF - make -j $(nproc) - mkdir /dev/shm/odp - - sudo LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" ODP_SHM_DIR=/dev/shm/odp make check + - if [ -z "$CROSS_ARCH" ] ; then + sudo LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" ODP_SHM_DIR=/dev/shm/odp make check ; + fi - make install - echo "Checking linking and run from install..." - pushd $HOME - echo "Dynamic link.." - - ${CC} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux` - - LD_LIBRARY_PATH="${HOME}/odp-install/lib:$LD_LIBRARY_PATH" ./odp_hello_inst + - ${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_dynamic `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux` - echo "Static link.." - - ${CC} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux --static` -static - - ./odp_hello_inst + - ${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_static `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux --static` -static + - if [ -z "$CROSS_ARCH" ] ; then + LD_LIBRARY_PATH="${HOME}/odp-install/lib:$LD_LIBRARY_PATH" ./odp_hello_inst ; + ./odp_hello_inst_static ; + fi + - popd - ccache -s jobs: @@ -189,83 +195,35 @@ jobs: - stage: test compiler: aarch64-linux-gnu-gcc env: TEST="aarch64-linux-gnu" CROSS_ARCH="arm64" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --enable-helper-linux - - make -j $(nproc) + install: gem install asciidoctor - stage: test compiler: "\"clang-3.8 --target=aarch64-linux-gnu\"" env: TEST="clang-3.8 aarch64-linux-gnu" CROSS_ARCH="arm64" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp/odp-install $CROSS - --enable-debug=full - --enable-helper-linux - - make -j $(nproc) + install: gem install asciidoctor - stage: test compiler: arm-linux-gnueabihf-gcc env: TEST="arm-linux-gnueabihf" CROSS_ARCH="armhf" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --enable-helper-linux - - make -j $(nproc) + install: gem install asciidoctor - stage: test compiler: "\"clang-3.8 --target=arm-linux-gnueabihf\"" env: TEST="clang-3.8 arm-linux-gnueabihf" CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --enable-helper-linux - - make -j $(nproc) + install: gem install asciidoctor - stage: test compiler: powerpc-linux-gnu-gcc env: TEST="powerpc-linux-gnueabi" CROSS_ARCH="powerpc" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --enable-helper-linux - - make -j $(nproc) + install: gem install asciidoctor - stage: test compiler: "\"clang-3.8 --target=powerpc-linux-gnu\"" env: TEST="clang-3.8 powerpc-linux-gnu" CROSS_ARCH="powerpc" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --enable-helper-linux - - make -j $(nproc) + install: gem install asciidoctor - stage: test compiler: "\"gcc -m32\"" env: TEST="i686-linux-gnueabi" CROSS_ARCH="i386" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --enable-helper-linux - - make -j $(nproc) + install: gem install asciidoctor - stage: test compiler: "\"clang-3.8 -m32\"" env: TEST="clang-3.8 i686-linux-gnu" CROSS_ARCH="i386" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --enable-helper-linux - - make -j $(nproc) + install: gem install asciidoctor - stage: test env: TEST=coverage compiler: gcc commit b38117eeed5d7bd5a334ef2387bf83d5cb8b9188 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Wed Sep 13 16:31:52 2017 +0300 travis: enable cpp test during cross-compilation Install proper packages to enable cross-testing of linking C++ apps with ODP. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 3854378d..06983f8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,13 +85,20 @@ before_install: sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install build-essential ; if [ "$CROSS_ARCH" = "i386" ] ; then - sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install gcc-multilib ; + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install g++-multilib ; else - sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install gcc-"$CROSS_GNU_TYPE" ; + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install g++-"$CROSS_GNU_TYPE" ; fi ; - sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install libc6-dev:"$CROSS_ARCH" libssl-dev:"$CROSS_ARCH" zlib1g-dev:"$CROSS_ARCH" libconfig-dev:"$CROSS_ARCH" ; + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install libc6-dev:"$CROSS_ARCH" libssl-dev:"$CROSS_ARCH" zlib1g-dev:"$CROSS_ARCH" libconfig-dev:"$CROSS_ARCH" libstdc++-4.8-dev:"$CROSS_ARCH"; export PKG_CONFIG_PATH=/usr/lib/${CROSS_MULTIARCH}/pkgconfig:/usr/${CROSS_MULTIARCH}/lib/pkgconfig ; fi + - if [ "${CC#clang}" != "${CC}" ] ; + then + export CXX="${CC/clang/clang++}"; + elif [ "$CROSS_ARCH" = "i386" ] ; + then + export CXX="g++ -m32" ; + fi - if test ! -L /usr/lib/ccache/${CC%% *} ; then sudo ln -s -t /usr/lib/ccache/ `which ${CC%% *}` ; fi - ccache -s # Install cunit for the validation tests because distro version is too old and fails C99 compile @@ -187,7 +194,6 @@ jobs: - ./bootstrap - ./configure --prefix=$HOME/odp-install $CROSS --enable-debug=full - --disable-test-cpp --enable-helper-linux - make -j $(nproc) - stage: test @@ -198,7 +204,6 @@ jobs: - ./bootstrap - ./configure --prefix=$HOME/odp/odp-install $CROSS --enable-debug=full - --disable-test-cpp --enable-helper-linux - make -j $(nproc) - stage: test @@ -209,7 +214,6 @@ jobs: - ./bootstrap - ./configure --prefix=$HOME/odp-install $CROSS --enable-debug=full - --disable-test-cpp --enable-helper-linux - make -j $(nproc) - stage: test @@ -220,7 +224,6 @@ jobs: - ./bootstrap - ./configure --prefix=$HOME/odp-install $CROSS --enable-debug=full - --disable-test-cpp --enable-helper-linux - make -j $(nproc) - stage: test @@ -231,7 +234,6 @@ jobs: - ./bootstrap - ./configure --prefix=$HOME/odp-install $CROSS --enable-debug=full - --disable-test-cpp --enable-helper-linux - make -j $(nproc) - stage: test @@ -242,7 +244,6 @@ jobs: - ./bootstrap - ./configure --prefix=$HOME/odp-install $CROSS --enable-debug=full - --disable-test-cpp --enable-helper-linux - make -j $(nproc) - stage: test @@ -253,7 +254,6 @@ jobs: - ./bootstrap - ./configure --prefix=$HOME/odp-install $CROSS --enable-debug=full - --disable-test-cpp --enable-helper-linux - make -j $(nproc) - stage: test @@ -264,7 +264,6 @@ jobs: - ./bootstrap - ./configure --prefix=$HOME/odp-install $CROSS --enable-debug=full - --disable-test-cpp --enable-helper-linux - make -j $(nproc) - stage: test commit d0821c9c43fb89e4c0021b174bbc12bef543ffa0 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 12 11:43:23 2017 +0300 travis: add cross-testing for 32-bit x86 Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index b6e98184..3854378d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -246,6 +246,28 @@ jobs: --enable-helper-linux - make -j $(nproc) - stage: test + compiler: "\"gcc -m32\"" + env: TEST="i686-linux-gnueabi" CROSS_ARCH="i386" + install: true + script: + - ./bootstrap + - ./configure --prefix=$HOME/odp-install $CROSS + --enable-debug=full + --disable-test-cpp + --enable-helper-linux + - make -j $(nproc) + - stage: test + compiler: "\"clang-3.8 -m32\"" + env: TEST="clang-3.8 i686-linux-gnu" CROSS_ARCH="i386" + install: true + script: + - ./bootstrap + - ./configure --prefix=$HOME/odp-install $CROSS + --enable-debug=full + --disable-test-cpp + --enable-helper-linux + - make -j $(nproc) + - stage: test env: TEST=coverage compiler: gcc script: commit 17e829a3b78d6513d22496734c6edbec955cbfd4 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 12 17:49:30 2017 +0300 travis: don't use cross-pkg-config Cross-pkg-config rewrites PKG_CONFIG_PATH to static value. Simulate it, but allow further script to expand PKG_CONFIG_PATH value. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 1ca4b5af..b6e98184 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,8 +82,15 @@ before_install: sudo -E apt-add-repository -y "deb http://ports.ubuntu.com trusty main" ; sudo -E apt-add-repository -y "deb http://ports.ubuntu.com trusty-updates main" ; sudo -E apt-get -y update || true ; - sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install build-essential gcc-"$CROSS_GNU_TYPE" pkg-config-"$CROSS_GNU_TYPE" ; + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install build-essential ; + if [ "$CROSS_ARCH" = "i386" ] ; + then + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install gcc-multilib ; + else + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install gcc-"$CROSS_GNU_TYPE" ; + fi ; sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install libc6-dev:"$CROSS_ARCH" libssl-dev:"$CROSS_ARCH" zlib1g-dev:"$CROSS_ARCH" libconfig-dev:"$CROSS_ARCH" ; + export PKG_CONFIG_PATH=/usr/lib/${CROSS_MULTIARCH}/pkgconfig:/usr/${CROSS_MULTIARCH}/lib/pkgconfig ; fi - if test ! -L /usr/lib/ccache/${CC%% *} ; then sudo ln -s -t /usr/lib/ccache/ `which ${CC%% *}` ; fi - ccache -s commit eccc78540c05dc71095179b613f014648385f3df Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 12 14:24:52 2017 +0300 travis: don't override prefix when cross-compiling Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 329b9195..1ca4b5af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,7 +77,7 @@ before_install: BUILD_GNU_TYPE=`dpkg-architecture -a"$CROSS_ARCH" -qDEB_BUILD_GNU_TYPE` ; CROSS_GNU_TYPE=`dpkg-architecture -a"$CROSS_ARCH" -qDEB_HOST_GNU_TYPE` ; CROSS_MULTIARCH=`dpkg-architecture -a"$CROSS_ARCH" -qDEB_HOST_MULTIARCH` ; - CROSS="--host="$CROSS_GNU_TYPE" --build="$BUILD_GNU_TYPE" --prefix=/usr --includedir=/usr/include/"$CROSS_MULTIARCH" --libdir=/usr/lib/"$CROSS_MULTIARCH" --libexecdir=/usr/lib/"$CROSS_MULTIARCH"" ; + CROSS="--host="$CROSS_GNU_TYPE" --build="$BUILD_GNU_TYPE"" ; sudo dpkg --add-architecture "$CROSS_ARCH" ; sudo -E apt-add-repository -y "deb http://ports.ubuntu.com trusty main" ; sudo -E apt-add-repository -y "deb http://ports.ubuntu.com trusty-updates main" ; commit f8c6360667380f6b3fb917ecbe5b941b23dbadf1 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 12 11:43:49 2017 +0300 travis: fix powerpc test name Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 38586242..329b9195 100644 --- a/.travis.yml +++ b/.travis.yml @@ -218,7 +218,7 @@ jobs: - make -j $(nproc) - stage: test compiler: powerpc-linux-gnu-gcc - env: TEST="powerpc-linux-gnueabihf" CROSS_ARCH="powerpc" + env: TEST="powerpc-linux-gnueabi" CROSS_ARCH="powerpc" install: true script: - ./bootstrap ----------------------------------------------------------------------- Summary of changes: .travis.yml | 134 +++++++++++++++++++++++------------------------------------- 1 file changed, 52 insertions(+), 82 deletions(-) hooks/post-receive --

7 years, 9 months

[COMMIT] branch cloud-dev updated. v1.15.0.0-601-ga1f50ad7

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, cloud-dev has been updated via a1f50ad720e11a54b13c4786cad4687cb5c4ec2a (commit) via 4a81959d49847df03aa11acb7bddefe78e238116 (commit) via 183376f96550b24b6601c7e07afe3eed4fe188d6 (commit) from 344fe21a06ef4273f34c4441edd0caef876517d9 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit a1f50ad720e11a54b13c4786cad4687cb5c4ec2a Author: Brian Brooks <brian.brooks(a)arm.com> Date: Fri Sep 1 12:53:32 2017 -0500 linux-gen: cleanup internal schedule interface Signed-off-by: Brian Brooks <brian.brooks(a)arm.com> Reviewed-by: Ola Liljedahl <ola.liljedahl(a)arm.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Yi He <yi.he(a)linaro.org> diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 8e158dc9..14b6a8d3 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -207,6 +207,9 @@ extern const pktio_if_ops_t loopback_pktio_ops; extern const pktio_if_ops_t dpdk_pktio_ops; extern const pktio_if_ops_t * const pktio_if_ops[]; +int pktin_poll(int pktio_index, int num_queue, int index[]); +void pktio_stop_finalize(int pktio_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index 1a4e345f..dc049f46 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -271,6 +271,9 @@ int sock_stats_fd(pktio_entry_t *pktio_entry, int fd); int sock_stats_reset_fd(pktio_entry_t *pktio_entry, int fd); +int pktin_poll(int pktio_index, int num_queue, int index[]); +void pktio_stop_finalize(int pktio_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_internal.h index dd846d59..a1b2a73d 100644 --- a/platform/linux-generic/include/odp_queue_internal.h +++ b/platform/linux-generic/include/odp_queue_internal.h @@ -76,6 +76,11 @@ static inline queue_t qentry_to_int(queue_entry_t *qentry) return (queue_t)(qentry); } +odp_queue_t queue_handle(uint32_t queue_index); +void queue_destroy_finalize(uint32_t queue_index); +int queue_idx_deq_multi(uint32_t queue_index, odp_event_t ev[], int num); +int queue_empty(uint32_t queue_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 2bcf23f2..8f39eec1 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -7,64 +7,31 @@ #ifndef ODP_SCHEDULE_IF_H_ #define ODP_SCHEDULE_IF_H_ -#ifdef __cplusplus -extern "C" { -#endif - #include <odp/api/queue.h> #include <odp_queue_if.h> #include <odp/api/schedule.h> -typedef void (*schedule_pktio_start_fn_t)(int pktio_index, int num_in_queue, - int in_queue_idx[]); -typedef int (*schedule_thr_add_fn_t)(odp_schedule_group_t group, int thr); -typedef int (*schedule_thr_rem_fn_t)(odp_schedule_group_t group, int thr); -typedef int (*schedule_num_grps_fn_t)(void); -typedef int (*schedule_init_queue_fn_t)( - uint32_t queue_index, const odp_schedule_param_t *sched_param); -typedef void (*schedule_destroy_queue_fn_t)(uint32_t queue_index); -typedef int (*schedule_sched_queue_fn_t)(uint32_t queue_index); -typedef int (*schedule_unsched_queue_fn_t)(uint32_t queue_index); -typedef int (*schedule_ord_enq_multi_fn_t)(queue_t q_int, - void *buf_hdr[], int num, int *ret); -typedef void (*schedule_order_lock_fn_t)(void); -typedef void (*schedule_order_unlock_fn_t)(void); -typedef unsigned (*schedule_max_ordered_locks_fn_t)(void); -typedef void (*schedule_save_context_fn_t)(uint32_t queue_index); - typedef struct schedule_fn_t { - int status_sync; - schedule_pktio_start_fn_t pktio_start; - schedule_thr_add_fn_t thr_add; - schedule_thr_rem_fn_t thr_rem; - schedule_num_grps_fn_t num_grps; - schedule_init_queue_fn_t init_queue; - schedule_destroy_queue_fn_t destroy_queue; - schedule_sched_queue_fn_t sched_queue; - schedule_ord_enq_multi_fn_t ord_enq_multi; - schedule_order_lock_fn_t order_lock; - schedule_order_unlock_fn_t order_unlock; - schedule_max_ordered_locks_fn_t max_ordered_locks; + int status_sync; + void (*pktio_start)(int pktio_index, int num_in_queue, + int in_queue_idx[]); + int (*thr_add)(odp_schedule_group_t group, int thr); + int (*thr_rem)(odp_schedule_group_t group, int thr); + int (*num_grps)(void); + int (*init_queue)(uint32_t queue_index, + const odp_schedule_param_t *sched_param); + void (*destroy_queue)(uint32_t queue_index); + int (*sched_queue)(uint32_t queue_index); + int (*ord_enq_multi)(queue_t q_int, void *buf_hdr[], int num, int *ret); + void (*order_lock)(void); + void (*order_unlock)(void); + unsigned (*max_ordered_locks)(void); /* Called only when status_sync is set */ - schedule_unsched_queue_fn_t unsched_queue; - schedule_save_context_fn_t save_context; - + int (*unsched_queue)(uint32_t queue_index); + void (*save_context)(uint32_t queue_index); } schedule_fn_t; -/* Interface towards the scheduler */ extern const schedule_fn_t *sched_fn; -/* Interface for the scheduler */ -int sched_cb_pktin_poll(int pktio_index, int num_queue, int index[]); -void sched_cb_pktio_stop_finalize(int pktio_index); -odp_queue_t sched_cb_queue_handle(uint32_t queue_index); -void sched_cb_queue_destroy_finalize(uint32_t queue_index); -int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num); -int sched_cb_queue_empty(uint32_t queue_index); - -#ifdef __cplusplus -} -#endif - #endif diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 066f89dc..e78c712e 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -665,7 +665,7 @@ static int pktin_deq_multi(queue_t q_int, odp_buffer_hdr_t *buf_hdr[], int num) return nbr; } -int sched_cb_pktin_poll(int pktio_index, int num_queue, int index[]) +int pktin_poll(int pktio_index, int num_queue, int index[]) { odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; int num, idx; @@ -703,7 +703,7 @@ int sched_cb_pktin_poll(int pktio_index, int num_queue, int index[]) return 0; } -void sched_cb_pktio_stop_finalize(int pktio_index) +void pktio_stop_finalize(int pktio_index) { int state; pktio_entry_t *entry = pktio_entry_by_index(pktio_index); diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index 7ad346ad..46197e00 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -234,7 +234,7 @@ static odp_queue_t queue_create(const char *name, return handle; } -void sched_cb_queue_destroy_finalize(uint32_t queue_index) +void queue_destroy_finalize(uint32_t queue_index) { queue_entry_t *queue = get_qentry(queue_index); @@ -643,19 +643,19 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) return 0; } -odp_queue_t sched_cb_queue_handle(uint32_t queue_index) +odp_queue_t queue_handle(uint32_t queue_index) { return queue_from_id(queue_index); } -int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num) +int queue_idx_deq_multi(uint32_t queue_index, odp_event_t ev[], int num) { queue_entry_t *qe = get_qentry(queue_index); return deq_multi(qe, (odp_buffer_hdr_t **)ev, num); } -int sched_cb_queue_empty(uint32_t queue_index) +int queue_empty(uint32_t queue_index) { queue_entry_t *queue = get_qentry(queue_index); int ret = 0; diff --git a/platform/linux-generic/schedule/generic.c b/platform/linux-generic/schedule/generic.c index 24ab8e06..36ed857b 100644 --- a/platform/linux-generic/schedule/generic.c +++ b/platform/linux-generic/schedule/generic.c @@ -24,6 +24,7 @@ #include <odp_ring_internal.h> #include <odp_timer_internal.h> #include <odp_schedule_subsystem.h> +#include <odp_packet_io_internal.h> /* Should remove this dependency */ #include <odp_queue_internal.h> @@ -368,11 +369,6 @@ static int schedule_init_global(void) return 0; } -static inline void queue_destroy_finalize(uint32_t qi) -{ - sched_cb_queue_destroy_finalize(qi); -} - static int schedule_term_global(void) { int ret = 0; @@ -858,13 +854,13 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], if (ordered && max_num < MAX_DEQ) max_deq = max_num; - num = sched_cb_queue_deq_multi(qi, sched_local.ev_stash, - max_deq); + num = queue_idx_deq_multi(qi, sched_local.ev_stash, + max_deq); if (num < 0) { /* Destroyed queue. Continue scheduling the same * priority queue. */ - sched_cb_queue_destroy_finalize(qi); + queue_destroy_finalize(qi); continue; } @@ -874,7 +870,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], continue; } - handle = sched_cb_queue_handle(qi); + handle = queue_handle(qi); sched_local.num = num; sched_local.index = 0; sched_local.queue = handle; @@ -1003,15 +999,15 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], cmd = &sched->pktio_cmd[cmd_index]; /* Poll packet input */ - if (odp_unlikely(sched_cb_pktin_poll(cmd->pktio_index, - cmd->num_pktin, - cmd->pktin))){ + if (odp_unlikely(pktin_poll(cmd->pktio_index, + cmd->num_pktin, + cmd->pktin))){ /* Pktio stopped or closed. Remove poll command and call * stop_finalize when all commands of the pktio has * been removed. */ if (schedule_pktio_stop(cmd->pktio_index, cmd->pktin[0]) == 0) - sched_cb_pktio_stop_finalize(cmd->pktio_index); + pktio_stop_finalize(cmd->pktio_index); free_pktio_cmd(cmd); } else { diff --git a/platform/linux-generic/schedule/iquery.c b/platform/linux-generic/schedule/iquery.c index f03014b8..ac0d0981 100644 --- a/platform/linux-generic/schedule/iquery.c +++ b/platform/linux-generic/schedule/iquery.c @@ -24,6 +24,7 @@ #include <odp_config_internal.h> #include <odp_timer_internal.h> #include <odp_schedule_subsystem.h> +#include <odp_packet_io_internal.h> /* Should remove this dependency */ #include <odp_queue_internal.h> @@ -293,10 +294,10 @@ static int schedule_term_global(void) odp_event_t events[1]; if (sched->availables[i]) - count = sched_cb_queue_deq_multi(i, events, 1); + count = queue_idx_deq_multi(i, events, 1); if (count < 0) - sched_cb_queue_destroy_finalize(i); + queue_destroy_finalize(i); else if (count > 0) ODP_ERR("Queue (%d) not empty\n", i); } @@ -673,16 +674,16 @@ static inline void pktio_poll_input(void) cmd = &sched->pktio_poll.commands[index]; /* Poll packet input */ - if (odp_unlikely(sched_cb_pktin_poll(cmd->pktio, - cmd->count, - cmd->pktin))) { + if (odp_unlikely(pktin_poll(cmd->pktio, + cmd->count, + cmd->pktin))) { /* Pktio stopped or closed. Remove poll * command and call stop_finalize when all * commands of the pktio has been removed. */ if (schedule_pktio_stop(cmd->pktio, cmd->pktin[0]) == 0) - sched_cb_pktio_stop_finalize(cmd->pktio); + pktio_stop_finalize(cmd->pktio); free_pktio_cmd(cmd); } else { @@ -1514,12 +1515,11 @@ static inline int consume_queue(int prio, unsigned int queue_index) if (is_ordered_queue(queue_index)) max = 1; - count = sched_cb_queue_deq_multi( - queue_index, cache->stash, max); + count = queue_idx_deq_multi(queue_index, cache->stash, max); if (count < 0) { DO_SCHED_UNLOCK(); - sched_cb_queue_destroy_finalize(queue_index); + queue_destroy_finalize(queue_index); DO_SCHED_LOCK(); return 0; } @@ -1529,7 +1529,7 @@ static inline int consume_queue(int prio, unsigned int queue_index) cache->top = &cache->stash[0]; cache->count = count; - cache->queue = sched_cb_queue_handle(queue_index); + cache->queue = queue_handle(queue_index); return count; } diff --git a/platform/linux-generic/schedule/scalable.c b/platform/linux-generic/schedule/scalable.c index 787553ca..0fba38a7 100644 --- a/platform/linux-generic/schedule/scalable.c +++ b/platform/linux-generic/schedule/scalable.c @@ -693,7 +693,7 @@ static inline void _schedule_release_ordered(sched_scalable_thread_state_t *ts) ts->rctx = NULL; } -static void pktin_poll(sched_scalable_thread_state_t *ts) +static void poll_pktin(sched_scalable_thread_state_t *ts) { uint32_t i, tag, hi, npolls = 0; int pktio_index, queue_index; @@ -715,8 +715,8 @@ static void pktin_poll(sched_scalable_thread_state_t *ts) /* Tag grabbed */ pktio_index = TAG_2_PKTIO(tag); queue_index = TAG_2_QUEUE(tag); - if (odp_unlikely(sched_cb_pktin_poll(pktio_index, - 1, &queue_index))) { + if (odp_unlikely(pktin_poll(pktio_index, + 1, &queue_index))) { /* Pktio stopped or closed * Remove tag from pktin_tags */ @@ -729,7 +729,7 @@ static void pktin_poll(sched_scalable_thread_state_t *ts) */ if (__atomic_sub_fetch(&pktin_count[pktio_index], 1, __ATOMIC_RELAXED) == 0) - sched_cb_pktio_stop_finalize(pktio_index); + pktio_stop_finalize(pktio_index); } else { /* We don't know whether any packets were found and enqueued * Write back original tag value to release pktin queue @@ -984,7 +984,7 @@ restart_same: } } - pktin_poll(ts); + poll_pktin(ts); return 0; } diff --git a/platform/linux-generic/schedule/sp.c b/platform/linux-generic/schedule/sp.c index 19700f90..a40d42d8 100644 --- a/platform/linux-generic/schedule/sp.c +++ b/platform/linux-generic/schedule/sp.c @@ -17,6 +17,8 @@ #include <odp_ring_internal.h> #include <odp_timer_internal.h> #include <odp_schedule_subsystem.h> +#include <odp_packet_io_internal.h> +#include <odp_queue_internal.h> #define NUM_THREAD ODP_THREAD_COUNT_MAX #define NUM_QUEUE ODP_CONFIG_QUEUES @@ -224,7 +226,7 @@ static int term_global(void) for (qi = 0; qi < NUM_QUEUE; qi++) { if (sched_global->queue_cmd[qi].s.init) { /* todo: dequeue until empty ? */ - sched_cb_queue_destroy_finalize(qi); + queue_destroy_finalize(qi); } } @@ -500,7 +502,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, if (sched_local.cmd) { /* Continue scheduling if queue is not empty */ - if (sched_cb_queue_empty(sched_local.cmd->s.index) == 0) + if (queue_empty(sched_local.cmd->s.index) == 0) add_tail(sched_local.cmd); sched_local.cmd = NULL; @@ -519,10 +521,10 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, cmd = sched_cmd(); if (cmd && cmd->s.type == CMD_PKTIO) { - if (sched_cb_pktin_poll(cmd->s.index, cmd->s.num_pktin, - cmd->s.pktin_idx)) { + if (pktin_poll(cmd->s.index, cmd->s.num_pktin, + cmd->s.pktin_idx)) { /* Pktio stopped or closed. */ - sched_cb_pktio_stop_finalize(cmd->s.index); + pktio_stop_finalize(cmd->s.index); } else { /* Continue polling pktio. */ add_tail(cmd); @@ -554,20 +556,20 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, } qi = cmd->s.index; - num = sched_cb_queue_deq_multi(qi, events, 1); + num = queue_idx_deq_multi(qi, events, 1); if (num > 0) { sched_local.cmd = cmd; if (from) - *from = sched_cb_queue_handle(qi); + *from = queue_handle(qi); return num; } if (num < 0) { /* Destroyed queue */ - sched_cb_queue_destroy_finalize(qi); + queue_destroy_finalize(qi); continue; } commit 4a81959d49847df03aa11acb7bddefe78e238116 Author: Brian Brooks <brian.brooks(a)arm.com> Date: Wed Aug 23 16:17:55 2017 -0500 linux-gen: schedule: move schedulers into modules Signed-off-by: Brian Brooks <brian.brooks(a)arm.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli(a)arm.com> Reviewed-by: Kevin Wang <kevin.wang(a)arm.com> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Yi He <yi.he(a)linaro.org> diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 8b94b7d2..587d4836 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -252,9 +252,12 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_rwlock.c \ ../linux-generic/odp_rwlock_recursive.c \ ../linux-generic/pool/subsystem.c \ - ../linux-generic/odp_schedule.c \ ../linux-generic/odp_schedule_if.c \ - ../linux-generic/odp_schedule_iquery.c \ + ../linux-generic/schedule/generic.c \ + ../linux-generic/schedule/iquery.c \ + ../linux-generic/schedule/scalable.c \ + ../linux-generic/schedule/scalable_ordered.c \ + ../linux-generic/schedule/sp.c \ ../linux-generic/schedule/subsystem.c \ ../linux-generic/odp_shared_memory.c \ ../linux-generic/odp_sorted_list.c \ @@ -297,6 +300,19 @@ __LIB__libodp_dpdk_la_SOURCES += arch/x86/cpu_flags.c \ endif pool/dpdk.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +if ODP_SCHEDULE_SCALABLE +../linux-generic/schedule/scalable.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +else +if ODP_SCHEDULE_SP +../linux-generic/schedule/sp.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +else +if ODP_SCHEDULE_IQUERY +../linux-generic/schedule/iquery.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +else +../linux-generic/schedule/generic.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +endif +endif +endif # Build modular framework into odp-linux library modularframeworkdir = $(top_srcdir)/frameworks/modular diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 96c535ba..cd7abce6 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -358,7 +358,7 @@ int odp_init_global(odp_instance_t *instance, } stage = QUEUE_INIT; - if (sched_fn->init_global()) { + if (odp_schedule_init_global()) { ODP_ERR("ODP schedule init failed.\n"); goto init_failed; } @@ -488,7 +488,7 @@ int _odp_term_global(enum init_stage stage) /* Fall through */ case SCHED_INIT: - if (sched_fn->term_global()) { + if (odp_schedule_term_global()) { ODP_ERR("ODP schedule term failed.\n"); rc = -1; } @@ -590,7 +590,7 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) } stage = POOL_INIT; - if (sched_fn->init_local()) { + if (odp_schedule_init_local()) { ODP_ERR("ODP schedule local init failed.\n"); goto init_fail; } @@ -617,7 +617,7 @@ int _odp_term_local(enum init_stage stage) case ALL_INIT: case SCHED_INIT: - if (sched_fn->term_local()) { + if (odp_schedule_term_local()) { ODP_ERR("ODP schedule local term failed.\n"); rc = -1; } diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 78e95760..cbbefa4e 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -281,12 +281,12 @@ __LIB__libodp_linux_la_SOURCES = \ odp_queue_scalable.c \ odp_rwlock.c \ odp_rwlock_recursive.c \ - odp_schedule.c \ odp_schedule_if.c \ - odp_schedule_sp.c \ - odp_schedule_iquery.c \ - odp_schedule_scalable.c \ - odp_schedule_scalable_ordered.c \ + schedule/generic.c \ + schedule/iquery.c \ + schedule/scalable.c \ + schedule/scalable_ordered.c \ + schedule/sp.c \ schedule/subsystem.c \ odp_shared_memory.c \ odp_sorted_list.c \ @@ -334,6 +334,17 @@ endif pool/generic.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE buffer/generic.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +if ODP_SCHEDULE_SCALABLE +schedule/scalable.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +else +schedule/generic.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +endif +if ODP_SCHEDULE_SP +schedule/sp.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +endif +if ODP_SCHEDULE_IQUERY +schedule/iquery.lo: AM_CFLAGS += -DIM_ACTIVE_MODULE +endif # Build modular framework into odp-linux library modularframeworkdir = $(top_srcdir)/frameworks/modular diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index bb70bd72..2c935a9f 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -113,6 +113,11 @@ int odp_queue_term_global(void); int odp_crypto_init_global(void); int odp_crypto_term_global(void); +int odp_schedule_init_global(void); +int odp_schedule_term_global(void); +int odp_schedule_init_local(void); +int odp_schedule_term_local(void); + int odp_timer_init_global(const odp_init_t *params); int odp_timer_term_global(void); int odp_timer_disarm_all(void); diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 657993b1..2bcf23f2 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -20,18 +20,13 @@ typedef void (*schedule_pktio_start_fn_t)(int pktio_index, int num_in_queue, typedef int (*schedule_thr_add_fn_t)(odp_schedule_group_t group, int thr); typedef int (*schedule_thr_rem_fn_t)(odp_schedule_group_t group, int thr); typedef int (*schedule_num_grps_fn_t)(void); -typedef int (*schedule_init_queue_fn_t)(uint32_t queue_index, - const odp_schedule_param_t *sched_param - ); +typedef int (*schedule_init_queue_fn_t)( + uint32_t queue_index, const odp_schedule_param_t *sched_param); typedef void (*schedule_destroy_queue_fn_t)(uint32_t queue_index); typedef int (*schedule_sched_queue_fn_t)(uint32_t queue_index); typedef int (*schedule_unsched_queue_fn_t)(uint32_t queue_index); typedef int (*schedule_ord_enq_multi_fn_t)(queue_t q_int, void *buf_hdr[], int num, int *ret); -typedef int (*schedule_init_global_fn_t)(void); -typedef int (*schedule_term_global_fn_t)(void); -typedef int (*schedule_init_local_fn_t)(void); -typedef int (*schedule_term_local_fn_t)(void); typedef void (*schedule_order_lock_fn_t)(void); typedef void (*schedule_order_unlock_fn_t)(void); typedef unsigned (*schedule_max_ordered_locks_fn_t)(void); @@ -47,10 +42,6 @@ typedef struct schedule_fn_t { schedule_destroy_queue_fn_t destroy_queue; schedule_sched_queue_fn_t sched_queue; schedule_ord_enq_multi_fn_t ord_enq_multi; - schedule_init_global_fn_t init_global; - schedule_term_global_fn_t term_global; - schedule_init_local_fn_t init_local; - schedule_term_local_fn_t term_local; schedule_order_lock_fn_t order_lock; schedule_order_unlock_fn_t order_unlock; schedule_max_ordered_locks_fn_t max_ordered_locks; @@ -72,32 +63,6 @@ void sched_cb_queue_destroy_finalize(uint32_t queue_index); int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num); int sched_cb_queue_empty(uint32_t queue_index); -/* API functions */ -typedef struct { - uint64_t (*schedule_wait_time)(uint64_t); - odp_event_t (*schedule)(odp_queue_t *, uint64_t); - int (*schedule_multi)(odp_queue_t *, uint64_t, odp_event_t [], int); - void (*schedule_pause)(void); - void (*schedule_resume)(void); - void (*schedule_release_atomic)(void); - void (*schedule_release_ordered)(void); - void (*schedule_prefetch)(int); - int (*schedule_num_prio)(void); - odp_schedule_group_t (*schedule_group_create)(const char *, - const odp_thrmask_t *); - int (*schedule_group_destroy)(odp_schedule_group_t); - odp_schedule_group_t (*schedule_group_lookup)(const char *); - int (*schedule_group_join)(odp_schedule_group_t, const odp_thrmask_t *); - int (*schedule_group_leave)(odp_schedule_group_t, - const odp_thrmask_t *); - int (*schedule_group_thrmask)(odp_schedule_group_t, odp_thrmask_t *); - int (*schedule_group_info)(odp_schedule_group_t, - odp_schedule_group_info_t *); - void (*schedule_order_lock)(unsigned); - void (*schedule_order_unlock)(unsigned); - -} schedule_api_t; - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 0d5ee710..340171a5 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -178,7 +178,7 @@ int odp_init_global(odp_instance_t *instance, } stage = QUEUE_INIT; - if (sched_fn->init_global()) { + if (odp_schedule_init_global()) { ODP_ERR("ODP schedule init failed.\n"); goto init_failed; } @@ -306,7 +306,7 @@ int _odp_term_global(enum init_stage stage) /* Fall through */ case SCHED_INIT: - if (sched_fn->term_global()) { + if (odp_schedule_term_global()) { ODP_ERR("ODP schedule term failed.\n"); rc = -1; } @@ -414,7 +414,7 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) } stage = QUEUE_INIT; - if (sched_fn->init_local()) { + if (odp_schedule_init_local()) { ODP_ERR("ODP schedule local init failed.\n"); goto init_fail; } @@ -447,7 +447,7 @@ int _odp_term_local(enum init_stage stage) case ALL_INIT: case SCHED_INIT: - if (sched_fn->term_local()) { + if (odp_schedule_term_local()) { ODP_ERR("ODP schedule local term failed.\n"); rc = -1; } diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index 2f07aafe..b06ab130 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -7,123 +7,16 @@ #include <odp_schedule_if.h> extern const schedule_fn_t schedule_sp_fn; -extern const schedule_api_t schedule_sp_api; - -extern const schedule_fn_t schedule_default_fn; -extern const schedule_api_t schedule_default_api; - extern const schedule_fn_t schedule_iquery_fn; -extern const schedule_api_t schedule_iquery_api; - -extern const schedule_fn_t schedule_scalable_fn; -extern const schedule_api_t schedule_scalable_api; +extern const schedule_fn_t schedule_scalable_fn; +extern const schedule_fn_t schedule_default_fn; -#ifdef ODP_SCHEDULE_SP -const schedule_fn_t *sched_fn = &schedule_sp_fn; -const schedule_api_t *sched_api = &schedule_sp_api; +#if defined(ODP_SCHEDULE_SP) +const schedule_fn_t *sched_fn = &schedule_sp_fn; #elif defined(ODP_SCHEDULE_IQUERY) -const schedule_fn_t *sched_fn = &schedule_iquery_fn; -const schedule_api_t *sched_api = &schedule_iquery_api; +const schedule_fn_t *sched_fn = &schedule_iquery_fn; #elif defined(ODP_SCHEDULE_SCALABLE) -const schedule_fn_t *sched_fn = &schedule_scalable_fn; -const schedule_api_t *sched_api = &schedule_scalable_api; +const schedule_fn_t *sched_fn = &schedule_scalable_fn; #else -const schedule_fn_t *sched_fn = &schedule_default_fn; -const schedule_api_t *sched_api = &schedule_default_api; +const schedule_fn_t *sched_fn = &schedule_default_fn; #endif - -uint64_t odp_schedule_wait_time(uint64_t ns) -{ - return sched_api->schedule_wait_time(ns); -} - -odp_event_t odp_schedule(odp_queue_t *from, uint64_t wait) -{ - return sched_api->schedule(from, wait); -} - -int odp_schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t events[], - int num) -{ - return sched_api->schedule_multi(from, wait, events, num); -} - -void odp_schedule_pause(void) -{ - return sched_api->schedule_pause(); -} - -void odp_schedule_resume(void) -{ - return sched_api->schedule_resume(); -} - -void odp_schedule_release_atomic(void) -{ - return sched_api->schedule_release_atomic(); -} - -void odp_schedule_release_ordered(void) -{ - return sched_api->schedule_release_ordered(); -} - -void odp_schedule_prefetch(int num) -{ - return sched_api->schedule_prefetch(num); -} - -int odp_schedule_num_prio(void) -{ - return sched_api->schedule_num_prio(); -} - -odp_schedule_group_t odp_schedule_group_create(const char *name, - const odp_thrmask_t *mask) -{ - return sched_api->schedule_group_create(name, mask); -} - -int odp_schedule_group_destroy(odp_schedule_group_t group) -{ - return sched_api->schedule_group_destroy(group); -} - -odp_schedule_group_t odp_schedule_group_lookup(const char *name) -{ - return sched_api->schedule_group_lookup(name); -} - -int odp_schedule_group_join(odp_schedule_group_t group, - const odp_thrmask_t *mask) -{ - return sched_api->schedule_group_join(group, mask); -} - -int odp_schedule_group_leave(odp_schedule_group_t group, - const odp_thrmask_t *mask) -{ - return sched_api->schedule_group_leave(group, mask); -} - -int odp_schedule_group_thrmask(odp_schedule_group_t group, - odp_thrmask_t *thrmask) -{ - return sched_api->schedule_group_thrmask(group, thrmask); -} - -int odp_schedule_group_info(odp_schedule_group_t group, - odp_schedule_group_info_t *info) -{ - return sched_api->schedule_group_info(group, info); -} - -void odp_schedule_order_lock(unsigned lock_index) -{ - return sched_api->schedule_order_lock(lock_index); -} - -void odp_schedule_order_unlock(unsigned lock_index) -{ - return sched_api->schedule_order_unlock(lock_index); -} diff --git a/platform/linux-generic/odp_schedule.c b/platform/linux-generic/schedule/generic.c similarity index 98% rename from platform/linux-generic/odp_schedule.c rename to platform/linux-generic/schedule/generic.c index a696d251..24ab8e06 100644 --- a/platform/linux-generic/odp_schedule.c +++ b/platform/linux-generic/schedule/generic.c @@ -23,6 +23,7 @@ #include <odp/api/packet_io.h> #include <odp_ring_internal.h> #include <odp_timer_internal.h> +#include <odp_schedule_subsystem.h> /* Should remove this dependency */ #include <odp_queue_internal.h> @@ -389,9 +390,9 @@ static int schedule_term_global(void) odp_event_t events[1]; int num; - num = sched_cb_queue_deq_multi(qi, - events, - 1); + num = queue_idx_deq_multi(qi, + events, + 1); if (num < 0) queue_destroy_finalize(qi); @@ -1397,10 +1398,6 @@ const schedule_fn_t schedule_default_fn = { .destroy_queue = schedule_destroy_queue, .sched_queue = schedule_sched_queue, .ord_enq_multi = schedule_ord_enq_multi, - .init_global = schedule_init_global, - .term_global = schedule_term_global, - .init_local = schedule_init_local, - .term_local = schedule_term_local, .order_lock = order_lock, .order_unlock = order_unlock, .max_ordered_locks = schedule_max_ordered_locks, @@ -1409,8 +1406,15 @@ const schedule_fn_t schedule_default_fn = { }; /* Fill in scheduler API calls */ -const schedule_api_t schedule_default_api = { - .schedule_wait_time = schedule_wait_time, +odp_schedule_module_t schedule_generic = { + .base = { + .name = "schedule_generic", + .init_global = schedule_init_global, + .term_global = schedule_term_global, + .init_local = schedule_init_local, + .term_local = schedule_term_local, + }, + .wait_time = schedule_wait_time, .schedule = schedule, .schedule_multi = schedule_multi, .schedule_pause = schedule_pause, @@ -1429,3 +1433,9 @@ const schedule_api_t schedule_default_api = { .schedule_order_lock = schedule_order_lock, .schedule_order_unlock = schedule_order_unlock }; + +ODP_MODULE_CONSTRUCTOR(schedule_generic) +{ + odp_module_constructor(&schedule_generic); + odp_subsystem_register_module(schedule, &schedule_generic); +} diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/schedule/iquery.c similarity index 98% rename from platform/linux-generic/odp_schedule_iquery.c rename to platform/linux-generic/schedule/iquery.c index 86f5d53e..f03014b8 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/schedule/iquery.c @@ -23,6 +23,7 @@ #include <odp/api/packet_io.h> #include <odp_config_internal.h> #include <odp_timer_internal.h> +#include <odp_schedule_subsystem.h> /* Should remove this dependency */ #include <odp_queue_internal.h> @@ -1336,10 +1337,6 @@ const schedule_fn_t schedule_iquery_fn = { .destroy_queue = destroy_sched_queue, .sched_queue = schedule_sched_queue, .ord_enq_multi = schedule_ord_enq_multi, - .init_global = schedule_init_global, - .term_global = schedule_term_global, - .init_local = schedule_init_local, - .term_local = schedule_term_local, .order_lock = order_lock, .order_unlock = order_unlock, .max_ordered_locks = schedule_max_ordered_locks, @@ -1348,8 +1345,15 @@ const schedule_fn_t schedule_iquery_fn = { }; /* Fill in scheduler API calls */ -const schedule_api_t schedule_iquery_api = { - .schedule_wait_time = schedule_wait_time, +odp_schedule_module_t schedule_iquery = { + .base = { + .name = "schedule_iquery", + .init_global = schedule_init_global, + .term_global = schedule_term_global, + .init_local = schedule_init_local, + .term_local = schedule_term_local, + }, + .wait_time = schedule_wait_time, .schedule = schedule, .schedule_multi = schedule_multi, .schedule_pause = schedule_pause, @@ -1369,6 +1373,12 @@ const schedule_api_t schedule_iquery_api = { .schedule_order_unlock = schedule_order_unlock }; +ODP_MODULE_CONSTRUCTOR(schedule_iquery) +{ + odp_module_constructor(&schedule_iquery); + odp_subsystem_register_module(schedule, &schedule_iquery); +} + static void thread_set_interest(sched_thread_local_t *thread, unsigned int queue_index, int prio) { diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/schedule/scalable.c similarity index 99% rename from platform/linux-generic/odp_schedule_scalable.c rename to platform/linux-generic/schedule/scalable.c index 78159b53..787553ca 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/schedule/scalable.c @@ -28,6 +28,7 @@ #include <odp_llqueue.h> #include <odp_queue_scalable_internal.h> #include <odp_schedule_if.h> +#include <odp_schedule_subsystem.h> #include <odp_bitset.h> #include <odp_packet_io_internal.h> @@ -1949,17 +1950,20 @@ const schedule_fn_t schedule_scalable_fn = { .destroy_queue = destroy_queue, .sched_queue = sched_queue, .ord_enq_multi = ord_enq_multi, - .init_global = schedule_init_global, - .term_global = schedule_term_global, - .init_local = schedule_init_local, - .term_local = schedule_term_local, .order_lock = order_lock, .order_unlock = order_unlock, .max_ordered_locks = schedule_max_ordered_locks, }; -const schedule_api_t schedule_scalable_api = { - .schedule_wait_time = schedule_wait_time, +odp_schedule_module_t schedule_scalable = { + .base = { + .name = "schedule_scalable", + .init_global = schedule_init_global, + .term_global = schedule_term_global, + .init_local = schedule_init_local, + .term_local = schedule_term_local, + }, + .wait_time = schedule_wait_time, .schedule = schedule, .schedule_multi = schedule_multi, .schedule_pause = schedule_pause, @@ -1978,3 +1982,9 @@ const schedule_api_t schedule_scalable_api = { .schedule_order_lock = schedule_order_lock, .schedule_order_unlock = schedule_order_unlock, }; + +ODP_MODULE_CONSTRUCTOR(schedule_scalable) +{ + odp_module_constructor(&schedule_scalable); + odp_subsystem_register_module(schedule, &schedule_scalable); +} diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/schedule/scalable_ordered.c similarity index 100% rename from platform/linux-generic/odp_schedule_scalable_ordered.c rename to platform/linux-generic/schedule/scalable_ordered.c diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/schedule/sp.c similarity index 97% rename from platform/linux-generic/odp_schedule_sp.c rename to platform/linux-generic/schedule/sp.c index 9829acc5..19700f90 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/schedule/sp.c @@ -16,6 +16,7 @@ #include <odp_config_internal.h> #include <odp_ring_internal.h> #include <odp_timer_internal.h> +#include <odp_schedule_subsystem.h> #define NUM_THREAD ODP_THREAD_COUNT_MAX #define NUM_QUEUE ODP_CONFIG_QUEUES @@ -836,10 +837,6 @@ const schedule_fn_t schedule_sp_fn = { .destroy_queue = destroy_queue, .sched_queue = sched_queue, .ord_enq_multi = ord_enq_multi, - .init_global = init_global, - .term_global = term_global, - .init_local = init_local, - .term_local = term_local, .order_lock = order_lock, .order_unlock = order_unlock, .max_ordered_locks = max_ordered_locks, @@ -848,8 +845,15 @@ const schedule_fn_t schedule_sp_fn = { }; /* Fill in scheduler API calls */ -const schedule_api_t schedule_sp_api = { - .schedule_wait_time = schedule_wait_time, +odp_schedule_module_t schedule_sp = { + .base = { + .name = "schedule_sp", + .init_global = init_global, + .term_global = term_global, + .init_local = init_local, + .term_local = term_local, + }, + .wait_time = schedule_wait_time, .schedule = schedule, .schedule_multi = schedule_multi, .schedule_pause = schedule_pause, @@ -868,3 +872,9 @@ const schedule_api_t schedule_sp_api = { .schedule_order_lock = schedule_order_lock, .schedule_order_unlock = schedule_order_unlock }; + +ODP_MODULE_CONSTRUCTOR(schedule_sp) +{ + odp_module_constructor(&schedule_sp); + odp_subsystem_register_module(schedule, &schedule_sp); +} diff --git a/platform/linux-generic/schedule/subsystem.c b/platform/linux-generic/schedule/subsystem.c index ae2bf3fd..6ca6459e 100644 --- a/platform/linux-generic/schedule/subsystem.c +++ b/platform/linux-generic/schedule/subsystem.c @@ -6,7 +6,12 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/* API header files */ +#include <odp.h> + /* Internal header files */ +#include <odp_debug_internal.h> +#include <odp_internal.h> #include <odp_module.h> #include <odp_schedule_subsystem.h> @@ -17,3 +22,251 @@ ODP_SUBSYSTEM_CONSTRUCTOR(schedule) { odp_subsystem_constructor(schedule); } + +int odp_schedule_init_global(void) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->base.init_global); + + return module->base.init_global(); +} + +int odp_schedule_term_global(void) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->base.term_global); + + return module->base.term_global(); +} + +int odp_schedule_init_local(void) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->base.init_local); + + return module->base.init_local(); +} + +int odp_schedule_term_local(void) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->base.term_local); + + return module->base.term_local(); +} + +uint64_t odp_schedule_wait_time(uint64_t ns) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->wait_time); + + return module->wait_time(ns); +} + +odp_event_t odp_schedule(odp_queue_t *from, uint64_t wait) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule); + + return module->schedule(from, wait); +} + +int odp_schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t events[], + int num) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_multi); + + return module->schedule_multi(from, wait, events, num); +} + +void odp_schedule_pause(void) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_pause); + + return module->schedule_pause(); +} + +void odp_schedule_resume(void) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_resume); + + return module->schedule_resume(); +} + +void odp_schedule_release_atomic(void) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_release_atomic); + + return module->schedule_release_atomic(); +} + +void odp_schedule_release_ordered(void) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_release_ordered); + + return module->schedule_release_ordered(); +} + +void odp_schedule_prefetch(int num) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_prefetch); + + return module->schedule_prefetch(num); +} + +int odp_schedule_num_prio(void) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_num_prio); + + return module->schedule_num_prio(); +} + +odp_schedule_group_t odp_schedule_group_create(const char *name, + const odp_thrmask_t *mask) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_group_create); + + return module->schedule_group_create(name, mask); +} + +int odp_schedule_group_destroy(odp_schedule_group_t group) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_group_destroy); + + return module->schedule_group_destroy(group); +} + +odp_schedule_group_t odp_schedule_group_lookup(const char *name) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_group_lookup); + + return module->schedule_group_lookup(name); +} + +int odp_schedule_group_join(odp_schedule_group_t group, + const odp_thrmask_t *mask) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_group_join); + + return module->schedule_group_join(group, mask); +} + +int odp_schedule_group_leave(odp_schedule_group_t group, + const odp_thrmask_t *mask) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_group_leave); + + return module->schedule_group_leave(group, mask); +} + +int odp_schedule_group_thrmask(odp_schedule_group_t group, + odp_thrmask_t *thrmask) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_group_thrmask); + + return module->schedule_group_thrmask(group, thrmask); +} + +int odp_schedule_group_info(odp_schedule_group_t group, + odp_schedule_group_info_t *info) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_group_info); + + return module->schedule_group_info(group, info); +} + +void odp_schedule_order_lock(unsigned lock_index) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_order_lock); + + return module->schedule_order_lock(lock_index); +} + +void odp_schedule_order_unlock(unsigned lock_index) +{ + odp_schedule_module_t *module = + odp_subsystem_active_module(schedule, module); + + ODP_ASSERT(module); + ODP_ASSERT(module->schedule_order_unlock); + + return module->schedule_order_unlock(lock_index); +} commit 183376f96550b24b6601c7e07afe3eed4fe188d6 Author: Brian Brooks <brian.brooks(a)arm.com> Date: Wed Aug 23 14:54:49 2017 -0500 linux-gen: schedule: add subsystem Signed-off-by: Brian Brooks <brian.brooks(a)arm.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli(a)arm.com> Reviewed-by: Yi He <yi.he(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 6e7f0b1d..8b94b7d2 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -203,6 +203,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_queue_if.h \ ${top_srcdir}/platform/linux-generic/include/odp_ring_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_schedule_if.h \ + ${top_srcdir}/platform/linux-generic/include/odp_schedule_subsystem.h \ ${top_srcdir}/platform/linux-generic/include/odp_sorted_list_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_shm_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_time_internal.h \ @@ -254,6 +255,7 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_schedule.c \ ../linux-generic/odp_schedule_if.c \ ../linux-generic/odp_schedule_iquery.c \ + ../linux-generic/schedule/subsystem.c \ ../linux-generic/odp_shared_memory.c \ ../linux-generic/odp_sorted_list.c \ ../linux-generic/odp_spinlock.c \ diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 3e3e0c22..78e95760 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -203,6 +203,7 @@ noinst_HEADERS = \ ${srcdir}/include/odp_schedule_scalable.h \ ${srcdir}/include/odp_schedule_scalable_config.h \ ${srcdir}/include/odp_schedule_scalable_ordered.h \ + ${srcdir}/include/odp_schedule_subsystem.h \ ${srcdir}/include/odp_sorted_list_internal.h \ ${srcdir}/include/odp_shm_internal.h \ ${srcdir}/include/odp_time_internal.h \ @@ -286,6 +287,7 @@ __LIB__libodp_linux_la_SOURCES = \ odp_schedule_iquery.c \ odp_schedule_scalable.c \ odp_schedule_scalable_ordered.c \ + schedule/subsystem.c \ odp_shared_memory.c \ odp_sorted_list.c \ odp_spinlock.c \ diff --git a/platform/linux-generic/include/odp_schedule_subsystem.h b/platform/linux-generic/include/odp_schedule_subsystem.h new file mode 100644 index 00000000..c3edef63 --- /dev/null +++ b/platform/linux-generic/include/odp_schedule_subsystem.h @@ -0,0 +1,77 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_SCHEDULE_SUBSYSTEM_H_ +#define ODP_SCHEDULE_SUBSYSTEM_H_ + +/* API header files */ +#include <odp/api/align.h> +#include <odp/api/schedule.h> + +/* Internal header files */ +#include <odp_module.h> + +#define SCHEDULE_SUBSYSTEM_VERSION 0x00010000UL + +ODP_SUBSYSTEM_DECLARE(schedule); + +ODP_SUBSYSTEM_API(schedule, uint64_t, wait_time, uint64_t ns); +ODP_SUBSYSTEM_API(schedule, odp_event_t, schedule, odp_queue_t *from, + uint64_t wait); +ODP_SUBSYSTEM_API(schedule, int, schedule_multi, odp_queue_t *from, + uint64_t wait, odp_event_t events[], int num); +ODP_SUBSYSTEM_API(schedule, void, schedule_pause, void); +ODP_SUBSYSTEM_API(schedule, void, schedule_resume, void); +ODP_SUBSYSTEM_API(schedule, void, schedule_release_atomic, void); +ODP_SUBSYSTEM_API(schedule, void, schedule_release_ordered, void); +ODP_SUBSYSTEM_API(schedule, void, schedule_prefetch, int num); +ODP_SUBSYSTEM_API(schedule, int, schedule_num_prio, void); +ODP_SUBSYSTEM_API(schedule, odp_schedule_group_t, schedule_group_create, + const char *name, const odp_thrmask_t *mask); +ODP_SUBSYSTEM_API(schedule, int, schedule_group_destroy, + odp_schedule_group_t group); +ODP_SUBSYSTEM_API(schedule, odp_schedule_group_t, schedule_group_lookup, + const char *name); +ODP_SUBSYSTEM_API(schedule, int, schedule_group_join, + odp_schedule_group_t group, const odp_thrmask_t *mask); +ODP_SUBSYSTEM_API(schedule, int, schedule_group_leave, + odp_schedule_group_t group, const odp_thrmask_t *mask); +ODP_SUBSYSTEM_API(schedule, int, schedule_group_thrmask, + odp_schedule_group_t group, odp_thrmask_t *thrmask); +ODP_SUBSYSTEM_API(schedule, int, schedule_group_info, + odp_schedule_group_t group, odp_schedule_group_info_t *info); +ODP_SUBSYSTEM_API(schedule, void, schedule_order_lock, unsigned lock_index); +ODP_SUBSYSTEM_API(schedule, void, schedule_order_unlock, unsigned lock_index); + +typedef ODP_MODULE_CLASS(schedule) { + odp_module_base_t base; + /* Called from CP threads */ + odp_api_proto(schedule, schedule_group_create) schedule_group_create; + odp_api_proto(schedule, schedule_group_destroy) schedule_group_destroy; + odp_api_proto(schedule, schedule_group_lookup) schedule_group_lookup; + odp_api_proto(schedule, schedule_group_join) schedule_group_join; + odp_api_proto(schedule, schedule_group_leave) schedule_group_leave; + odp_api_proto(schedule, schedule_group_thrmask) schedule_group_thrmask; + odp_api_proto(schedule, schedule_group_info) schedule_group_info; + odp_api_proto(schedule, schedule_num_prio) schedule_num_prio; + /* Called from DP threads */ + odp_api_proto(schedule, schedule) schedule ODP_ALIGNED_CACHE; + odp_api_proto(schedule, schedule_multi) schedule_multi; + odp_api_proto(schedule, schedule_prefetch) schedule_prefetch; + odp_api_proto(schedule, schedule_order_lock) schedule_order_lock; + odp_api_proto(schedule, schedule_order_unlock) schedule_order_unlock; + odp_api_proto(schedule, schedule_release_atomic) + schedule_release_atomic; + odp_api_proto(schedule, schedule_release_ordered) + schedule_release_ordered; + odp_api_proto(schedule, wait_time) wait_time; + odp_api_proto(schedule, schedule_pause) schedule_pause; + odp_api_proto(schedule, schedule_resume) schedule_resume; +} odp_schedule_module_t; + +#endif /* ODP_SCHEDULE_SUBSYSTEM_H_ */ diff --git a/platform/linux-generic/schedule/subsystem.c b/platform/linux-generic/schedule/subsystem.c new file mode 100644 index 00000000..ae2bf3fd --- /dev/null +++ b/platform/linux-generic/schedule/subsystem.c @@ -0,0 +1,19 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* Internal header files */ +#include <odp_module.h> +#include <odp_schedule_subsystem.h> + +ODP_SUBSYSTEM_DEFINE(schedule, "schedule public APIs", + SCHEDULE_SUBSYSTEM_VERSION); + +ODP_SUBSYSTEM_CONSTRUCTOR(schedule) +{ + odp_subsystem_constructor(schedule); +} ----------------------------------------------------------------------- Summary of changes: platform/linux-dpdk/Makefile.am | 22 +- .../linux-dpdk/include/odp_packet_io_internal.h | 3 + platform/linux-dpdk/odp_init.c | 8 +- platform/linux-generic/Makefile.am | 23 +- platform/linux-generic/include/odp_internal.h | 5 + .../linux-generic/include/odp_packet_io_internal.h | 3 + .../linux-generic/include/odp_queue_internal.h | 5 + platform/linux-generic/include/odp_schedule_if.h | 100 ++------ .../linux-generic/include/odp_schedule_subsystem.h | 77 ++++++ platform/linux-generic/odp_init.c | 8 +- platform/linux-generic/odp_packet_io.c | 4 +- platform/linux-generic/odp_queue.c | 8 +- platform/linux-generic/odp_schedule_if.c | 121 +-------- .../{odp_schedule.c => schedule/generic.c} | 50 ++-- .../{odp_schedule_iquery.c => schedule/iquery.c} | 42 ++-- .../scalable.c} | 32 ++- .../scalable_ordered.c} | 0 .../{odp_schedule_sp.c => schedule/sp.c} | 40 +-- platform/linux-generic/schedule/subsystem.c | 272 +++++++++++++++++++++ 19 files changed, 541 insertions(+), 282 deletions(-) create mode 100644 platform/linux-generic/include/odp_schedule_subsystem.h rename platform/linux-generic/{odp_schedule.c => schedule/generic.c} (97%) rename platform/linux-generic/{odp_schedule_iquery.c => schedule/iquery.c} (97%) rename platform/linux-generic/{odp_schedule_scalable.c => schedule/scalable.c} (98%) rename platform/linux-generic/{odp_schedule_scalable_ordered.c => schedule/scalable_ordered.c} (100%) rename platform/linux-generic/{odp_schedule_sp.c => schedule/sp.c} (96%) create mode 100644 platform/linux-generic/schedule/subsystem.c hooks/post-receive --

7 years, 9 months

[COMMIT] branch api-next updated. v1.15.0.0-654-g6b6253c3

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 6b6253c30f88c80bf632436ff06c1b000860a2f1 (commit) via c0c20e6ed7d0f86dca02a276a7f991586fa7f33d (commit) via fb3f36cec108ce9c55241d9f0e66d4832a552b8a (commit) via 3b31169bee4f036bacbe823c27c9b199fc35fe75 (commit) via 2cb0edd5820fc7fc14d6f4018a605873fdf47033 (commit) via d6e2cd7830bd474e78980414ad7046443a4a3720 (commit) via e114fba150e07e7f25b86306c30003416324955e (commit) via 316d020fc2e3fee86b955eec4946290d90fb2eb1 (commit) via 4c8e7df6337e79cef937ad3246d61e75b7d2164d (commit) via 01f75d6582aadc1aeb6d41745c3b0a2fdbe7b142 (commit) via 29cc24dc2effa5cc76af7b365bcdafe671f03545 (commit) via b7181a3f13d58d87a561cd06e00fc37a9fc237b3 (commit) via 5581eaa55c9d32429be88b068b149ebc8b235f2c (commit) via 71e078e380f81d972cb82908a8d13dcc155f5cad (commit) via 3be7ab995e5f2c4472b20008f63299d93a3a806c (commit) via 1882b608abc37314f90bdd2de8ef7f0501a8d5d8 (commit) via 863f2ca462a7dd0a17b0828d037d2594767de092 (commit) via de3ad51a88daa12e9b822e6df339c0e10448d6dd (commit) via e7cf0a6c24811d768d4df91d2c03b71676f9f783 (commit) via 360c1d34b52a2356619b9290811862b9de41de00 (commit) from 7fa8e2c97ed18f8dd6e95cbc78b7e668ccb98869 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 6b6253c30f88c80bf632436ff06c1b000860a2f1 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Mon Sep 11 19:25:11 2017 +0300 test: drv: remove -Wno-deprecated-declarations set cflags are set from top level directory, drv should not modify them internaly. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> diff --git a/test/common_plat/validation/drv/Makefile.am b/test/common_plat/validation/drv/Makefile.am index bcdb92e2..e445adf5 100644 --- a/test/common_plat/validation/drv/Makefile.am +++ b/test/common_plat/validation/drv/Makefile.am @@ -2,6 +2,3 @@ ODPDRV_MODULES = drvatomic \ drvshmem SUBDIRS = $(ODPDRV_MODULES) - -#The tests will need to retain the deprecated test implementation -AM_CFLAGS += -Wno-deprecated-declarations commit c0c20e6ed7d0f86dca02a276a7f991586fa7f33d Merge: 7fa8e2c9 fb3f36ce Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Mon Sep 11 19:05:34 2017 +0300 Merge branch 'master' into api-next Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --cc platform/linux-generic/Makefile.am index 7556b9ad,f5f65384..3e26aab4 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@@ -2,15 -2,13 +2,15 @@@ #export CUSTOM_STR=https://git.linaro.org/lng/odp.git include $(top_srcdir)/platform/Makefile.inc - include $(top_srcdir)/platform/@with_platform@/Makefile.inc - AM_CFLAGS += -I$(srcdir)/include - AM_CFLAGS += -I$(top_srcdir)/include - AM_CFLAGS += -I$(top_srcdir)/include/odp/arch/@ARCH_ABI@ - AM_CFLAGS += -I$(top_builddir)/include - AM_CFLAGS += -I$(top_srcdir)/arch/@ARCH_DIR@ - AM_CFLAGS += -Iinclude - AM_CFLAGS += -DSYSCONFDIR=\"@sysconfdir@\" + AM_CPPFLAGS = -I$(srcdir)/include + AM_CPPFLAGS += -I$(top_srcdir)/include + AM_CPPFLAGS += -I$(top_srcdir)/include/odp/arch/@ARCH_ABI@ + AM_CPPFLAGS += -I$(top_builddir)/include + AM_CPPFLAGS += -Iinclude + AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/$(ARCH_DIR) ++AM_CPPFLAGS += -Iinclude ++AM_CPPFLAGS += -DSYSCONFDIR=\"@sysconfdir@\" AM_CPPFLAGS += $(OPENSSL_CPPFLAGS) AM_CPPFLAGS += $(DPDK_CPPFLAGS) @@@ -205,26 -170,8 +205,25 @@@ noinst_HEADERS = ${srcdir}/include/protocols/ip.h \ ${srcdir}/include/protocols/ipsec.h \ ${srcdir}/include/protocols/tcp.h \ + ${srcdir}/include/protocols/thash.h \ - ${srcdir}/include/protocols/udp.h \ - ${srcdir}/Makefile.inc + ${srcdir}/include/protocols/udp.h +if ARCH_IS_ARM +noinst_HEADERS += ${srcdir}/arch/arm/odp_atomic.h \ + ${srcdir}/arch/arm/odp_cpu.h \ + ${srcdir}/arch/arm/odp_cpu_idling.h \ + ${srcdir}/arch/arm/odp_llsc.h +endif +if ARCH_IS_MIPS64 +noinst_HEADERS += ${srcdir}/arch/mips64/odp_cpu.h +endif +if ARCH_IS_POWERPC +noinst_HEADERS += ${srcdir}/arch/powerpc/odp_cpu.h +endif +if ARCH_IS_X86 +noinst_HEADERS += ${srcdir}/arch/x86/odp_cpu.h +endif + __LIB__libodp_linux_la_SOURCES = \ _fdserver.c \ _ishm.c \ diff --cc test/common_plat/validation/Makefile.am index ef58b25c,32854378..17e0444b --- a/test/common_plat/validation/Makefile.am +++ b/test/common_plat/validation/Makefile.am @@@ -1,4 -1,3 +1,4 @@@ - if cunit_support + if test_vald - SUBDIRS = api + SUBDIRS = api \ + drv endif ----------------------------------------------------------------------- Summary of changes: .travis.yml | 9 ++--- DEPENDENCIES | 31 ++++++++++------- configure.ac | 25 ++++---------- example/Makefile.inc | 6 ++-- helper/Makefile.am | 19 +++++----- helper/test/Makefile.am | 23 ++++++------- platform/Makefile.inc | 7 ++-- platform/linux-generic/Makefile.am | 19 +++++----- platform/linux-generic/Makefile.inc | 2 -- platform/linux-generic/odp_crypto.c | 3 -- platform/linux-generic/pktio/socket.c | 2 +- test/Makefile.inc | 9 +++-- test/common_plat/common/Makefile.am | 8 ++--- test/common_plat/m4/validation.m4 | 8 ++++- test/common_plat/miscellaneous/Makefile.am | 6 +--- test/common_plat/performance/Makefile.am | 34 +++++------------- test/common_plat/validation/Makefile.am | 2 +- test/common_plat/validation/api/Makefile.am | 3 -- test/common_plat/validation/api/Makefile.inc | 1 - test/common_plat/validation/api/atomic/Makefile.am | 9 ++--- .../common_plat/validation/api/barrier/Makefile.am | 9 ++--- test/common_plat/validation/api/buffer/Makefile.am | 9 ++--- .../validation/api/classification/Makefile.am | 20 +++++------ .../common_plat/validation/api/cpumask/Makefile.am | 10 ++---- test/common_plat/validation/api/crypto/Makefile.am | 16 ++++----- test/common_plat/validation/api/errno/Makefile.am | 9 ++--- test/common_plat/validation/api/hash/Makefile.am | 9 ++--- test/common_plat/validation/api/init/Makefile.am | 16 ++++----- test/common_plat/validation/api/lock/Makefile.am | 9 ++--- test/common_plat/validation/api/packet/Makefile.am | 9 ++--- test/common_plat/validation/api/pktio/Makefile.am | 9 ++--- test/common_plat/validation/api/pool/Makefile.am | 9 ++--- test/common_plat/validation/api/queue/Makefile.am | 9 ++--- test/common_plat/validation/api/random/Makefile.am | 9 ++--- .../validation/api/scheduler/Makefile.am | 9 ++--- test/common_plat/validation/api/shmem/Makefile.am | 9 ++--- .../validation/api/std_clib/Makefile.am | 9 ++--- test/common_plat/validation/api/system/Makefile.am | 9 ++--- test/common_plat/validation/api/thread/Makefile.am | 12 ++----- test/common_plat/validation/api/time/Makefile.am | 8 ++--- test/common_plat/validation/api/timer/Makefile.am | 9 ++--- .../validation/api/traffic_mngr/Makefile.am | 8 ++--- test/common_plat/validation/drv/Makefile.am | 3 -- test/linux-generic/Makefile.inc | 4 +-- test/linux-generic/mmap_vlan_ins/Makefile.am | 9 ++--- test/linux-generic/performance/Makefile.am | 2 +- test/linux-generic/pktio_ipc/Makefile.am | 13 ++----- test/linux-generic/pktio_ipc/ipc_common.c | 12 +++---- test/linux-generic/pktio_ipc/ipc_common.h | 2 +- test/linux-generic/pktio_ipc/pktio_ipc1.c | 40 +++++++++++----------- test/linux-generic/pktio_ipc/pktio_ipc2.c | 36 +++++++++---------- test/linux-generic/ring/Makefile.am | 20 +++++------ .../linux-generic/validation/api/shmem/Makefile.am | 15 +++----- test/linux-generic/validation/api/shmem/shmem.h | 21 ------------ 54 files changed, 219 insertions(+), 409 deletions(-) delete mode 100644 platform/linux-generic/Makefile.inc delete mode 100644 test/linux-generic/validation/api/shmem/shmem.h hooks/post-receive --

7 years, 9 months

[COMMIT] branch master updated. v1.15.0.0-185-gfb3f36ce

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via fb3f36cec108ce9c55241d9f0e66d4832a552b8a (commit) via 3b31169bee4f036bacbe823c27c9b199fc35fe75 (commit) via 2cb0edd5820fc7fc14d6f4018a605873fdf47033 (commit) from d6e2cd7830bd474e78980414ad7046443a4a3720 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit fb3f36cec108ce9c55241d9f0e66d4832a552b8a Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Fri Sep 8 13:08:36 2017 +0300 configure: call PKG_PROG_PKG_CONFIG explicitly Since it well might be that first PKG_CHECK_* call ends up under if, it is required (by pkg-config docs) to call PKG_PROG_PKG_CONFIG explicitly. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/configure.ac b/configure.ac index d4c07a2c..4af2c7c7 100644 --- a/configure.ac +++ b/configure.ac @@ -62,6 +62,8 @@ LT_INIT([]) AC_SUBST([LIBTOOL_DEPS]) AM_PROG_LIBTOOL +PKG_PROG_PKG_CONFIG + # Checks for library functions. dnl breaks cross-compilation and malloc(0) behaviour is not that important dnl AC_FUNC_MALLOC commit 3b31169bee4f036bacbe823c27c9b199fc35fe75 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Fri Sep 8 13:05:52 2017 +0300 test: try detecting CUnit location, if it is not found by pkg-config CUnit 2.1-2 supplied incorrect cunit.pc file, resulting in pkg-config failing to detect library. Try checking CUnit header/library in case of pkg-config failure. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/common_plat/m4/validation.m4 b/test/common_plat/m4/validation.m4 index e303f8f2..78c4ec03 100644 --- a/test/common_plat/m4/validation.m4 +++ b/test/common_plat/m4/validation.m4 @@ -12,7 +12,13 @@ AM_CONDITIONAL([test_vald], [test x$test_vald = xyes ]) ########################################################################## cunit_support=$test_vald AS_IF([test "x$cunit_support" = "xyes"], - [PKG_CHECK_MODULES([CUNIT], [cunit])]) + [PKG_CHECK_MODULES([CUNIT], [cunit], [], + [AC_MSG_WARN([pkg-config could not find CUnit, guessing]) + AC_CHECK_HEADERS([CUnit/Basic.h], [], + [AC_MSG_ERROR(["can't find CUnit headers"])]) + AC_CHECK_LIB([cunit],[CU_get_error], [CUNIT_LIBS="-lcunit"], + [AC_MSG_ERROR([CUnit libraries required])]) +])]) AC_SUBST([CUNIT_CFLAGS]) AC_SUBST([CUNIT_LIBS]) commit 2cb0edd5820fc7fc14d6f4018a605873fdf47033 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Tue Sep 5 14:57:47 2017 +0300 DEPENDENCIES: document changes in CUnit localization Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/DEPENDENCIES b/DEPENDENCIES index a194cad1..e309b339 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -9,12 +9,13 @@ Prerequisites for building the OpenDataPlane (ODP) API automake autoconf libtool + pkg-config On Debian/Ubuntu systems: - $ sudo apt-get install automake autoconf libtool + $ sudo apt-get install automake autoconf libtool pkg-config On CentOS/RedHat/Fedora systems: - $ sudo yum install automake autoconf libtool + $ sudo yum install automake autoconf libtool pkgconfig 3. Required libraries @@ -199,11 +200,11 @@ Prerequisites for building the OpenDataPlane (ODP) API 4.0 Packages needed to build API tests - Cunit test framework version 2.1-3 is required - Cunit prvodes a framework to run the API test suite that proves conformance to the + CUnit test framework version 2.1-3 is required + CUnit provides a framework to run the API test suite that proves conformance to the ODP API. The home page http://cunit.sourceforge.net/doc/introduction.html -4.1 Native Cunit install +4.1 Native CUnit install # Debian/Ubuntu $ apt-get install libcunit1-dev @@ -223,7 +224,7 @@ Prerequisites for building the OpenDataPlane (ODP) API sudo make install # ... OR ... Install CUnit into user defined location. The same path is - # used in step 4.4 --with-cunit-path=/home/<my_cunit_path> + # used in step 4.4: PKG_CONFIG_PATH=/home/<my_cunit_path>/lib/pkgconfig ./configure --prefix=/home/<my_cunit_path> make make install @@ -232,20 +233,24 @@ Prerequisites for building the OpenDataPlane (ODP) API # reboot, before trying to run e.g. 'make distcheck'. sudo ldconfig -4.3 Cross compile of Cunit +4.3 Cross compile of CUnit $ git svn clone http://svn.code.sf.net/p/cunit/code/trunk cunit-code $ cd cunit-code $ ./bootstrap $ ./configure --host=arm-linux-gnueabihf --prefix=/home/${USER}/src/install-cunit -4.4 Using Cunit with ODP - $ Add the configuration option to the regular configuration options - # Use the default path ... - ./configure --enable-cunit +4.4 Using CUnit with ODP - # ... OR the user defined path. - ./configure --with-cunit-path=/home/<my_cunit_path> + Configure will automatically look for CUnit if validation testsuite is + enabled. By default it uses pkg-config to locate CUnit. Usually no + additional configuration will be required. Few corner cases: + + # User directory installation + ./configure PKG_CONFIG_PATH=/home/<my_cunit_path>/lib/pkgconfig + + # ... OR directly specifying flags + ./configure CUNIT_CFLAGS="-I/home/<my_cunit_path>/include" CUNIT_LIBS="/home/<my_cunit_path>/lib -lcunit" 5.0 Documentation Images & Doxygen ----------------------------------------------------------------------- Summary of changes: DEPENDENCIES | 31 ++++++++++++++++++------------- configure.ac | 2 ++ test/common_plat/m4/validation.m4 | 8 +++++++- 3 files changed, 27 insertions(+), 14 deletions(-) hooks/post-receive --

7 years, 9 months

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

git-lng-odp