Add per-component sysfs attribute groups (enable_sink, enable_source, label) and a connections/ group with nr_links and bidirectional out:X / in:Y symlinks. Drivers can attach extra groups via pdata->groups; they are registered and cleaned up inside gtrace_sysfs_add/remove.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- drivers/hwtracing/gtrace/Makefile | 2 +- drivers/hwtracing/gtrace/gtrace-core.c | 47 ++- drivers/hwtracing/gtrace/gtrace-sysfs.c | 440 ++++++++++++++++++++++++ include/linux/gtrace.h | 104 ++++++ 4 files changed, 583 insertions(+), 10 deletions(-) create mode 100644 drivers/hwtracing/gtrace/gtrace-sysfs.c

diff --git a/drivers/hwtracing/gtrace/Makefile b/drivers/hwtracing/gtrace/Makefile index 33274a1ed0b8..ff79a49bdb5e 100644 --- a/drivers/hwtracing/gtrace/Makefile +++ b/drivers/hwtracing/gtrace/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0

obj-$(CONFIG_GTRACE) += gtrace.o -gtrace-y := gtrace-core.o gtrace-perf.o +gtrace-y := gtrace-core.o gtrace-perf.o gtrace-sysfs.o gtrace-$(CONFIG_OF) += gtrace-of.o

obj-$(CONFIG_RVTRACE) += rvtrace-platform.o diff --git a/drivers/hwtracing/gtrace/gtrace-core.c b/drivers/hwtracing/gtrace/gtrace-core.c index 5704e426abe8..cae0a7fa820f 100644 --- a/drivers/hwtracing/gtrace/gtrace-core.c +++ b/drivers/hwtracing/gtrace/gtrace-core.c @@ -17,7 +17,7 @@ #define GTRACE_POLL_TIMEOUT_US_DEFAULT 10

/* Mutex to serialize component registration/unregistration */ -static DEFINE_MUTEX(gtrace_mutex); +DEFINE_MUTEX(gtrace_mutex);

/* Per-CPU source instances */ static DEFINE_PER_CPU(struct gtrace_component *, gtrace_cpu_source_comp); @@ -36,11 +36,20 @@ struct gtrace_comp_priv { pid_t owner; bool ready; bool visited; + enum gtrace_component_mode mode; };

#define to_gtrace_comp_priv(__comp) \ container_of_const(__comp, struct gtrace_comp_priv, comp)

+enum gtrace_component_mode gtrace_get_mode(struct gtrace_component *comp) +{ + struct gtrace_comp_priv *cpriv = to_gtrace_comp_priv(comp); + + return cpriv->mode; +} +EXPORT_SYMBOL_GPL(gtrace_get_mode); + /* Component type based id generator */ struct gtrace_type_idx { /* Lock to protect the type ID generator */ @@ -463,6 +472,12 @@ struct gtrace_component *gtrace_register_component(struct gtrace_component_id *i goto err_out_unlock; }

+ ret = gtrace_sysfs_add(comp); + if (ret < 0) { + device_unregister(&comp->dev); + goto err_out_unlock; + } + if (comp->pdata->bound_cpu >= 0) { gtrace_get_component(comp); per_cpu(gtrace_cpu_source_comp, comp->pdata->bound_cpu) = comp; @@ -507,6 +522,7 @@ void gtrace_unregister_component(struct gtrace_component *comp) }

gtrace_cleanup_inconns_from_outconns(comp); + gtrace_sysfs_remove(comp); device_unregister(&comp->dev);

mutex_unlock(&gtrace_mutex); @@ -636,7 +652,8 @@ static void gtrace_release_path_nodes(struct gtrace_path *path) } }

-static int __gtrace_comp_start(struct gtrace_component *comp, pid_t owner) +static int __gtrace_comp_start(struct gtrace_component *comp, + struct gtrace_path *path) { struct gtrace_comp_priv *cpriv = to_gtrace_comp_priv(comp); const struct gtrace_driver *gtdrv = to_gtrace_driver(comp->dev.driver); @@ -648,17 +665,21 @@ static int __gtrace_comp_start(struct gtrace_component *comp, pid_t owner)

raw_spin_lock_irqsave(&cpriv->lock, flags); if (cpriv->start_count) { - if (cpriv->owner != owner) { + if (cpriv->owner != path->owner || cpriv->mode != path->mode) { ret = -EBUSY; goto out; } } else { if (gtdrv->start) { + comp->path = path; ret = gtdrv->start(comp); - if (ret) + if (ret) { + comp->path = NULL; goto out; + } } - cpriv->owner = owner; + cpriv->owner = path->owner; + cpriv->mode = path->mode; }

cpriv->start_count++; @@ -667,7 +688,8 @@ static int __gtrace_comp_start(struct gtrace_component *comp, pid_t owner) return ret; }

-static int __gtrace_comp_stop(struct gtrace_component *comp) +static int __gtrace_comp_stop(struct gtrace_component *comp, + struct gtrace_path *path) { struct gtrace_comp_priv *cpriv = to_gtrace_comp_priv(comp); const struct gtrace_driver *gtdrv = to_gtrace_driver(comp->dev.driver); @@ -680,6 +702,11 @@ static int __gtrace_comp_stop(struct gtrace_component *comp) goto out; }

+ if (cpriv->mode != path->mode) { + ret = -EBUSY; + goto out; + } + cpriv->start_count--; if (!cpriv->start_count) { if (!gtdrv) { @@ -689,6 +716,8 @@ static int __gtrace_comp_stop(struct gtrace_component *comp)

if (gtdrv->stop) ret = gtdrv->stop(comp); + comp->path = NULL; + cpriv->mode = GTRACE_COMPONENT_MODE_DISABLED; } out: raw_spin_unlock_irqrestore(&cpriv->lock, flags); @@ -701,7 +730,7 @@ int gtrace_path_start(struct gtrace_path *path) int ret = 0;

list_for_each_entry_reverse(node, &path->comp_list, head) { - ret = __gtrace_comp_start(node->comp, path->owner); + ret = __gtrace_comp_start(node->comp, path); if (ret) break; } @@ -709,7 +738,7 @@ int gtrace_path_start(struct gtrace_path *path) /* If a component failed to start, stop all the components that were started before it */ if (ret) list_for_each_entry_continue(node, &path->comp_list, head) - __gtrace_comp_stop(node->comp); + __gtrace_comp_stop(node->comp, path);

return ret; } @@ -721,7 +750,7 @@ int gtrace_path_stop(struct gtrace_path *path) int ret = 0, err;

list_for_each_entry(node, &path->comp_list, head) { - err = __gtrace_comp_stop(node->comp); + err = __gtrace_comp_stop(node->comp, path); if (err && !ret) ret = err; } diff --git a/drivers/hwtracing/gtrace/gtrace-sysfs.c b/drivers/hwtracing/gtrace/gtrace-sysfs.c new file mode 100644 index 000000000000..5f4f3fcb56a4 --- /dev/null +++ b/drivers/hwtracing/gtrace/gtrace-sysfs.c @@ -0,0 +1,440 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Spacemit Technologies, Inc. + * + * Based on drivers/hwtracing/coresight/coresight-sysfs.c + */ + +#include <linux/device.h> +#include <linux/gtrace.h> +#include <linux/property.h> + +ssize_t gtrace_simple_show_pair(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct gtrace_pair_attribute *gtrace_attr = + container_of(attr, struct gtrace_pair_attribute, attr); + u64 val; + + val = gtrace_relaxed_read_pair(comp->pdata, gtrace_attr->lo_off, gtrace_attr->hi_off); + return sysfs_emit(buf, "0x%llx\n", val); +} +EXPORT_SYMBOL_GPL(gtrace_simple_show_pair); + +ssize_t gtrace_simple_show32(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct gtrace_off_attribute *gtrace_attr = + container_of(attr, struct gtrace_off_attribute, attr); + u32 val; + + val = gtrace_relaxed_read32(comp->pdata, gtrace_attr->off); + return sysfs_emit(buf, "0x%x\n", val); +} +EXPORT_SYMBOL_GPL(gtrace_simple_show32); + +/* + * gtrace_find_activated_sysfs_sink - find the first sink activated via + * walks outconns forward from @comp. + */ +static struct gtrace_component * +gtrace_find_activated_sysfs_sink(struct gtrace_component *comp) +{ + int i; + struct gtrace_component *sink = NULL; + struct gtrace_platform_data *pdata = comp->pdata; + + if (comp->sysfs_sink_activated) + return comp; + + for (i = 0; i < pdata->nr_outconns; i++) { + struct gtrace_component *dest = pdata->outconns[i]->dest_comp; + + if (dest) + sink = gtrace_find_activated_sysfs_sink(dest); + if (sink) + return sink; + } + + return NULL; +} + +/** gtrace_validate_source_sysfs - make sure a source has the right credentials to + * be used via sysfs. + * @comp: the component structure for a source. + * @function: the function this was called from. + * + * Assumes the gtrace_mutex is held. + */ +static int gtrace_validate_source_sysfs(struct gtrace_component *comp, + const char *function) +{ + if (!gtrace_is_source(comp->pdata)) { + dev_err(&comp->dev, "component is not a trace source in %s\n", function); + return -EINVAL; + } + + if (!cpu_online(comp->pdata->bound_cpu)) + return -ENODEV; + + return 0; +} + +int gtrace_enable_sysfs(struct gtrace_component *comp) +{ + struct gtrace_component *sink; + struct gtrace_path *path; + int ret; + + mutex_lock(&gtrace_mutex); + ret = gtrace_validate_source_sysfs(comp, __func__); + if (ret) + goto out; + + /* mode == SYSFS implies that it's already enabled. */ + if (gtrace_get_mode(comp) == GTRACE_COMPONENT_MODE_SYSFS) + goto out; + + sink = gtrace_find_activated_sysfs_sink(comp); + + /* + * gtrace_create_path() acquires gtrace_mutex internally via + * gtrace_walk_output_components(). Release it here first to + * avoid a deadlock. + */ + mutex_unlock(&gtrace_mutex); + + path = gtrace_create_path(comp, sink, GTRACE_COMPONENT_MODE_SYSFS); + if (IS_ERR(path)) { + pr_err("building path(s) failed\n"); + return PTR_ERR(path); + } + + mutex_lock(&gtrace_mutex); + + ret = gtrace_path_start(path); + if (ret) + gtrace_destroy_path(path); + +out: + mutex_unlock(&gtrace_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(gtrace_enable_sysfs); + +int gtrace_disable_sysfs(struct gtrace_component *comp) +{ + struct gtrace_path *path; + int ret; + + mutex_lock(&gtrace_mutex); + + ret = gtrace_validate_source_sysfs(comp, __func__); + if (ret) + goto out; + + path = comp->path; + + if (gtrace_get_mode(comp) != GTRACE_COMPONENT_MODE_SYSFS) + goto out; + + ret = gtrace_path_stop(path); + if (ret) + pr_err("Error. Tracing not stopped\n"); + + gtrace_destroy_path(path); +out: + mutex_unlock(&gtrace_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(gtrace_disable_sysfs); + +static ssize_t enable_sink_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + + guard(mutex)(&gtrace_mutex); + + return sysfs_emit(buf, "%u\n", comp->sysfs_sink_activated); +} + +static ssize_t enable_sink_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct gtrace_component *comp = to_gtrace_component(dev); + + ret = kstrtoul(buf, 10, &val); + if (ret) + return ret; + + comp->sysfs_sink_activated = !!val; + + return size; +} +static DEVICE_ATTR_RW(enable_sink); + +static ssize_t enable_source_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + + guard(mutex)(&gtrace_mutex); + return sysfs_emit(buf, "%u\n", + gtrace_get_mode(comp) == GTRACE_COMPONENT_MODE_SYSFS); +} + +static ssize_t enable_source_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret = 0; + unsigned long val; + struct gtrace_component *comp = to_gtrace_component(dev); + + ret = kstrtoul(buf, 10, &val); + if (ret) + return ret; + + + /* + * gtrace hotplug callbacks in core layer control a activated path + * from its source to sink. Taking hotplug lock here protects a race + * condition with hotplug callbacks. + */ + guard(cpus_read_lock)(); + + if (val) { + ret = gtrace_enable_sysfs(comp); + if (ret) + return ret; + } else { + ret = gtrace_disable_sysfs(comp); + if (ret) + return ret; + } + + return size; +} +static DEVICE_ATTR_RW(enable_source); + +static ssize_t label_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + const char *str; + int ret; + + ret = fwnode_property_read_string(dev_fwnode(dev), "label", &str); + if (ret) + return ret; + + return sysfs_emit(buf, "%s\n", str); +} +static DEVICE_ATTR_RO(label); + +static umode_t label_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = kobj_to_dev(kobj); + + if (attr == &dev_attr_label.attr) { + if (fwnode_property_present(dev_fwnode(dev), "label")) + return attr->mode; + else + return 0; + } + + return attr->mode; +} + +static struct attribute *gtrace_sink_attrs[] = { + &dev_attr_enable_sink.attr, + &dev_attr_label.attr, + NULL, +}; + +static const struct attribute_group gtrace_sink_group = { + .attrs = gtrace_sink_attrs, + .is_visible = label_is_visible, +}; + + +static struct attribute *gtrace_source_attrs[] = { + &dev_attr_enable_source.attr, + &dev_attr_label.attr, + NULL, +}; + +static const struct attribute_group gtrace_source_group = { + .attrs = gtrace_source_attrs, + .is_visible = label_is_visible, +}; + +static struct attribute *gtrace_link_attrs[] = { + &dev_attr_label.attr, + NULL, +}; + +static const struct attribute_group gtrace_link_group = { + .attrs = gtrace_link_attrs, + .is_visible = label_is_visible, +}; + +static ssize_t nr_links_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + + return sysfs_emit(buf, "%u\n", comp->pdata->nr_inconns + comp->pdata->nr_outconns); +} +static DEVICE_ATTR_RO(nr_links); + +static struct attribute *gtrace_conns_attrs[] = { + &dev_attr_nr_links.attr, + NULL, +}; + +static struct attribute_group gtrace_conns_group = { + .attrs = gtrace_conns_attrs, + .name = "connections", +}; + +static int gtrace_make_links(struct gtrace_component *comp) +{ + struct gtrace_platform_data *pdata = comp->pdata; + struct gtrace_connection *conn; + char *outs, *ins; + int i, ret; + + for (i = 0; i < pdata->nr_outconns; i++) { + conn = pdata->outconns[i]; + if (!conn->dest_comp) + continue; + + outs = devm_kasprintf(pdata->dev, GFP_KERNEL, + "out:%d", conn->src_port); + ins = devm_kasprintf(&conn->dest_comp->dev, GFP_KERNEL, + "in:%d", conn->dest_port); + if (!outs || !ins) + return -ENOMEM; + + ret = sysfs_add_link_to_group(&comp->dev.kobj, + "connections", + &conn->dest_comp->dev.kobj, outs); + if (ret) + return ret; + + ret = sysfs_add_link_to_group(&conn->dest_comp->dev.kobj, + "connections", + &comp->dev.kobj, ins); + if (ret) { + sysfs_remove_link_from_group(&comp->dev.kobj, + "connections", outs); + return ret; + } + } + return 0; +} + +static void gtrace_remove_links(struct gtrace_component *comp) +{ + struct gtrace_platform_data *pdata = comp->pdata; + struct gtrace_connection *conn; + char buf[32]; + int i; + + for (i = 0; i < pdata->nr_outconns; i++) { + conn = pdata->outconns[i]; + if (!conn->dest_comp) + continue; + + snprintf(buf, sizeof(buf), "out:%d", conn->src_port); + sysfs_remove_link_from_group(&comp->dev.kobj, + "connections", buf); + + snprintf(buf, sizeof(buf), "in:%d", conn->dest_port); + sysfs_remove_link_from_group(&conn->dest_comp->dev.kobj, + "connections", buf); + } +} + +int gtrace_sysfs_add(struct gtrace_component *comp) +{ + const struct attribute_group *grp; + int ret; + + if (gtrace_is_sink(comp->pdata) || comp->pdata->is_linksink) + grp = &gtrace_sink_group; + else if (gtrace_is_source(comp->pdata)) + grp = &gtrace_source_group; + else + grp = &gtrace_link_group; + + ret = sysfs_create_group(&comp->dev.kobj, grp); + if (ret) { + dev_err(&comp->dev, "failed to create sysfs group: %d\n", ret); + goto out; + } + + ret = sysfs_create_group(&comp->dev.kobj, &gtrace_conns_group); + if (ret) { + dev_err(&comp->dev, "failed to create connections sysfs group: %d\n", ret); + goto out_remove_group; + } + + ret = gtrace_make_links(comp); + if (ret) { + dev_err(&comp->dev, "failed to make sysfs links: %d\n", ret); + goto out_remove_conns; + } + + if (comp->pdata->groups) { + ret = device_add_groups(&comp->dev, comp->pdata->groups); + if (ret) { + dev_err(&comp->dev, "failed to create sysfs component groups: %d\n", ret); + goto out_remove_links; + } + } + + return ret; + +out_remove_links: + gtrace_remove_links(comp); +out_remove_conns: + sysfs_remove_group(&comp->dev.kobj, &gtrace_conns_group); +out_remove_group: + sysfs_remove_group(&comp->dev.kobj, grp); +out: + return ret; +} +EXPORT_SYMBOL_GPL(gtrace_sysfs_add); + +void gtrace_sysfs_remove(struct gtrace_component *comp) +{ + mutex_lock(&gtrace_mutex); + if (comp->path) { + gtrace_path_stop(comp->path); + gtrace_destroy_path(comp->path); + comp->path = NULL; + } + mutex_unlock(&gtrace_mutex); + + if (comp->pdata->groups) + device_remove_groups(&comp->dev, comp->pdata->groups); + gtrace_remove_links(comp); + sysfs_remove_group(&comp->dev.kobj, &gtrace_conns_group); + + if (gtrace_is_sink(comp->pdata) || comp->pdata->is_linksink) + sysfs_remove_group(&comp->dev.kobj, &gtrace_sink_group); + else if (gtrace_is_source(comp->pdata)) + sysfs_remove_group(&comp->dev.kobj, &gtrace_source_group); + else + sysfs_remove_group(&comp->dev.kobj, &gtrace_link_group); +} +EXPORT_SYMBOL_GPL(gtrace_sysfs_remove); diff --git a/include/linux/gtrace.h b/include/linux/gtrace.h index da8ba3779193..9945303d2713 100644 --- a/include/linux/gtrace.h +++ b/include/linux/gtrace.h @@ -14,6 +14,8 @@ #include <linux/property.h> #include <linux/types.h>

+extern struct mutex gtrace_mutex; + /* * Global list of trace component types across all architectures. Each backend * uses the entries relevant to it. @@ -29,6 +31,8 @@ enum gtrace_component_type {

/* Supported usage modes for trace components */ enum gtrace_component_mode { + GTRACE_COMPONENT_MODE_DISABLED, + GTRACE_COMPONENT_MODE_SYSFS, GTRACE_COMPONENT_MODE_PERF, GTRACE_COMPONENT_MODE_MAX }; @@ -97,6 +101,9 @@ struct gtrace_hw_ops { * @inconns: Array of pointers to input connections. * @nr_outconns: Number of output connections. * @outconns: Array of pointers to output connections. + * @is_linksink: A component that can serve as both a link and a sink. + * @groups: operations specific to this component. These will end up in + * the component's sysfs sub-directory. */ struct gtrace_platform_data { struct device *dev; @@ -138,6 +145,13 @@ struct gtrace_platform_data { */ unsigned int nr_outconns; struct gtrace_connection **outconns; + + /* + * True for components like ETF that can operate as both a link and + * a sink. + */ + bool is_linksink; + const struct attribute_group **groups; };

static inline u32 gtrace_read32(struct gtrace_platform_data *pdata, u32 offset) @@ -156,6 +170,28 @@ static inline u32 gtrace_relaxed_read32(struct gtrace_platform_data *pdata, u32 return pdata->read(pdata, offset, true); }

+static inline u64 gtrace_read_pair(struct gtrace_platform_data *pdata, + u32 lo_offset, u32 hi_offset) +{ + if (likely(pdata->io_mem)) + return readl(pdata->base + lo_offset) | + ((u64)readl(pdata->base + hi_offset) << 32); + + return pdata->read(pdata, lo_offset, false) | + ((u64)pdata->read(pdata, hi_offset, false) << 32); +} + +static inline u64 gtrace_relaxed_read_pair(struct gtrace_platform_data *pdata, + u32 lo_offset, u32 hi_offset) +{ + if (likely(pdata->io_mem)) + return readl_relaxed(pdata->base + lo_offset) | + ((u64)readl_relaxed(pdata->base + hi_offset) << 32); + + return pdata->read(pdata, lo_offset, true) | + ((u64)pdata->read(pdata, hi_offset, true) << 32); +} + static inline void gtrace_write32(struct gtrace_platform_data *pdata, u32 val, u32 offset) { if (likely(pdata->io_mem)) @@ -173,6 +209,30 @@ static inline void gtrace_relaxed_write32(struct gtrace_platform_data *pdata, pdata->write(pdata, val, offset, true); }

+static inline void gtrace_relaxed_write_pair(struct gtrace_platform_data *pdata, + u64 val, u32 lo_offset, u32 hi_offset) +{ + if (likely(pdata->io_mem)) { + writel_relaxed((u32)val, pdata->base + lo_offset); + writel_relaxed((u32)(val >> 32), pdata->base + hi_offset); + } else { + pdata->write(pdata, (u32)val, lo_offset, true); + pdata->write(pdata, (u32)(val >> 32), hi_offset, true); + } +} + +static inline void gtrace_write_pair(struct gtrace_platform_data *pdata, + u64 val, u32 lo_offset, u32 hi_offset) +{ + if (likely(pdata->io_mem)) { + writel((u32)val, pdata->base + lo_offset); + writel((u32)(val >> 32), pdata->base + hi_offset); + } else { + pdata->write(pdata, (u32)val, lo_offset, false); + pdata->write(pdata, (u32)(val >> 32), hi_offset, false); + } +} + static inline bool gtrace_is_source(struct gtrace_platform_data *pdata) { return !pdata->nr_inconns ? true : false; @@ -200,11 +260,18 @@ struct gtrace_component_id { * @pdata: Pointer to underlying platform data. * @id: Details to match the component. * @dev: Device instance. + * @path: Used for sysfs sources to get path, and for ETF link/sink detection. + * @sysfs_sink_activated: When set, this sink is preferred over other + * reachable sinks when a source walks forward + * looking for a sysfs trace target (e.g. stop + * at ETF instead of ETR). */ struct gtrace_component { struct gtrace_platform_data *pdata; struct gtrace_component_id id; struct device dev; + struct gtrace_path *path; + bool sysfs_sink_activated; };

#define to_gtrace_component(__dev) container_of_const(__dev, struct gtrace_component, dev) @@ -329,4 +396,41 @@ static inline void gtrace_unregister_driver(struct gtrace_driver *gtdrv) int gtrace_perf_init(void); void gtrace_perf_exit(void);

+enum gtrace_component_mode gtrace_get_mode(struct gtrace_component *comp); + +struct gtrace_pair_attribute { + struct device_attribute attr; + u32 lo_off; + u32 hi_off; +}; + +struct gtrace_off_attribute { + struct device_attribute attr; + u32 off; +}; + +ssize_t gtrace_simple_show32(struct device *dev, struct device_attribute *attr, char *buf); +ssize_t gtrace_simple_show_pair(struct device *dev, struct device_attribute *attr, char *buf); + +#define gtrace_simple_reg32(name, offset) \ + (&((struct gtrace_off_attribute[]) { \ + { \ + __ATTR(name, 0444, gtrace_simple_show32, NULL), \ + offset \ + } \ + })[0].attr.attr) + +#define gtrace_simple_reg64(name, lo_off, hi_off) \ + (&((struct gtrace_pair_attribute[]) { \ + { \ + __ATTR(name, 0444, gtrace_simple_show_pair, NULL), \ + lo_off, hi_off \ + } \ + })[0].attr.attr) + +int gtrace_enable_sysfs(struct gtrace_component *comp); +int gtrace_disable_sysfs(struct gtrace_component *comp); +int gtrace_sysfs_add(struct gtrace_component *comp); +void gtrace_sysfs_remove(struct gtrace_component *comp); + #endif /* __LINUX_GTRACE_H__ */

Add sysfs attribute support for RISC-V trace encoder components in the platform driver. The encoder attribute groups are now assigned to pdata->groups based on component type during registration, enabling encoder configuration through sysfs interfaces.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- drivers/hwtracing/gtrace/Makefile | 1 + ...trace-encoder.c => rvtrace-encoder-core.c} | 117 +++++-- .../hwtracing/gtrace/rvtrace-encoder-sysfs.c | 290 ++++++++++++++++++ drivers/hwtracing/gtrace/rvtrace-encoder.h | 53 ++++ drivers/hwtracing/gtrace/rvtrace-platform.c | 8 + drivers/hwtracing/gtrace/rvtrace.h | 3 + 6 files changed, 438 insertions(+), 34 deletions(-) rename drivers/hwtracing/gtrace/{rvtrace-encoder.c => rvtrace-encoder-core.c} (60%) create mode 100644 drivers/hwtracing/gtrace/rvtrace-encoder-sysfs.c create mode 100644 drivers/hwtracing/gtrace/rvtrace-encoder.h

diff --git a/drivers/hwtracing/gtrace/Makefile b/drivers/hwtracing/gtrace/Makefile index ff79a49bdb5e..87cb5add78d4 100644 --- a/drivers/hwtracing/gtrace/Makefile +++ b/drivers/hwtracing/gtrace/Makefile @@ -6,4 +6,5 @@ gtrace-$(CONFIG_OF) += gtrace-of.o

obj-$(CONFIG_RVTRACE) += rvtrace-platform.o obj-$(CONFIG_RVTRACE_ENCODER) += rvtrace-encoder.o +rvtrace-encoder-y := rvtrace-encoder-core.o rvtrace-encoder-sysfs.o obj-$(CONFIG_RVTRACE_RAMSINK) += rvtrace-ramsink.o diff --git a/drivers/hwtracing/gtrace/rvtrace-encoder.c b/drivers/hwtracing/gtrace/rvtrace-encoder-core.c similarity index 60% rename from drivers/hwtracing/gtrace/rvtrace-encoder.c rename to drivers/hwtracing/gtrace/rvtrace-encoder-core.c index f24780b1ca1e..01925ebdfbde 100644 --- a/drivers/hwtracing/gtrace/rvtrace-encoder.c +++ b/drivers/hwtracing/gtrace/rvtrace-encoder-core.c @@ -10,28 +10,13 @@ #include <linux/minmax.h> #include <linux/slab.h> #include <linux/types.h> +#include <linux/bitfield.h> #include "rvtrace.h" +#include "rvtrace-encoder.h"

-#define RVTRACE_COMPONENT_CTRL_ITRACE_SHIFT 2 -#define RVTRACE_COMPONENT_CTRL_INSTMODE_MASK 0x7 -#define RVTRACE_COMPONENT_CTRL_INSTMODE_SHIFT 4 -#define RVTRACE_COMPONENT_CTRL_INSTMODE_OPIT 0x6 -#define RVTRACE_COMPONENT_CTRL_INHIBITSRC_SHIFT 15 -#define RVTRACE_COMPONENT_CTRL_FORMAT_MASK 0x7 -#define RVTRACE_COMPONENT_CTRL_FORMAT_SHIFT 24 -#define RVTRACE_COMPONENT_CTRL_FORMAT_ETRACE 0x0 -#define RVTRACE_COMPONENT_CTRL_FORMAT_NTRACE 0x1 - -#define RVTRACE_ENCODER_INSTFEAT_OFFSET 0x8 -#define RVTRACE_ENCODER_INSTFEAT_SRCID_MASK 0xfff -#define RVTRACE_ENCODER_INSTFEAT_SRCID_SHIFT 16 -#define RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK 0xf -#define RVTRACE_ENCODER_INSTFEAT_SRCBITS_SHIFT 28 -#define RVTRACE_ENCODER_INSTFEAT_SRCBITS_MAX 12 - -struct rvtrace_encoder_priv { - u32 srcbits; -}; +static int boot_enable; +module_param(boot_enable, int, 0444); +MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");

/* * Set the srcid and clear trTeInhibitSrc bit so that the emitted trace messages carry @@ -42,10 +27,10 @@ static void rvtrace_encoder_set_srcid(struct gtrace_platform_data *pdata, u32 sr u32 val;

val = gtrace_read32(pdata, RVTRACE_ENCODER_INSTFEAT_OFFSET); - val &= ~(RVTRACE_ENCODER_INSTFEAT_SRCID_MASK << RVTRACE_ENCODER_INSTFEAT_SRCID_SHIFT); - val &= ~(RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK << RVTRACE_ENCODER_INSTFEAT_SRCBITS_SHIFT); - val |= pdata->bound_cpu << RVTRACE_ENCODER_INSTFEAT_SRCID_SHIFT; - val |= srcbits << RVTRACE_ENCODER_INSTFEAT_SRCBITS_SHIFT; + val &= ~RVTRACE_ENCODER_INSTFEAT_SRCID_MASK; + val &= ~RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK; + val |= FIELD_PREP(RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK, srcbits); + val |= FIELD_PREP(RVTRACE_ENCODER_INSTFEAT_SRCID_MASK, pdata->bound_cpu); gtrace_write32(pdata, val, RVTRACE_ENCODER_INSTFEAT_OFFSET);

val = gtrace_read32(pdata, RVTRACE_COMPONENT_CTRL_OFFSET); @@ -53,11 +38,32 @@ static void rvtrace_encoder_set_srcid(struct gtrace_platform_data *pdata, u32 sr gtrace_write32(pdata, val, RVTRACE_COMPONENT_CTRL_OFFSET); }

+static void rvtrace_encoder_get_syncmode_hw(struct gtrace_platform_data *pdata, + bool *syncmode_hw) +{ + u32 ctrl, val; + enum rvtrace_encoder_syncmode mode; + + ctrl = gtrace_read32(pdata, RVTRACE_COMPONENT_CTRL_OFFSET); + + for (mode = RVTRACE_ENCODER_SYNCMODE_OFF; mode < RVTRACE_ENCODER_SYNCMODE_MAX; mode++) { + val = ctrl & ~RVTRACE_COMPONENT_CTRL_SYNCMODE_MASK; + val |= FIELD_PREP(RVTRACE_COMPONENT_CTRL_SYNCMODE_MASK, mode); + gtrace_write32(pdata, val, RVTRACE_COMPONENT_CTRL_OFFSET); + val = gtrace_read32(pdata, RVTRACE_COMPONENT_CTRL_OFFSET); + val = FIELD_GET(RVTRACE_COMPONENT_CTRL_SYNCMODE_MASK, val); + syncmode_hw[mode] = (mode == val); + } + + gtrace_write32(pdata, ctrl, RVTRACE_COMPONENT_CTRL_OFFSET); +} + static int rvtrace_encoder_probe(struct gtrace_component *comp) { struct gtrace_platform_data *pdata = comp->pdata; struct rvtrace_encoder_priv *priv; u32 val, hw_srcbits, srcbits; + int ret;

if (pdata->bound_cpu < 0) { dev_err(&comp->dev, "cpu property missing. Please fix firmware.\n"); @@ -68,6 +74,28 @@ static int rvtrace_encoder_probe(struct gtrace_component *comp) if (!priv) return -ENOMEM;

+ /* Get trTeFormat */ + val = gtrace_read32(pdata, RVTRACE_COMPONENT_CTRL_OFFSET); + priv->format = FIELD_GET(RVTRACE_COMPONENT_CTRL_FORMAT_MASK, val); + if (priv->format > GTRACE_FORMAT_NTRACE) { + dev_err(&comp->dev, "unsupported trace format %d\n", priv->format); + return -EINVAL; + } + + /* Get trTeInstSyncMode */ + priv->syncmode = FIELD_GET(RVTRACE_COMPONENT_CTRL_SYNCMODE_MASK, val); + rvtrace_encoder_get_syncmode_hw(pdata, priv->syncmode_hw); + + /* Check if trTeContext is hardwired to 0 */ + val |= BIT(RVTRACE_COMPONENT_CTRL_CONTEXT_SHIFT); + gtrace_write32(pdata, val, RVTRACE_COMPONENT_CTRL_OFFSET); + val = gtrace_read32(pdata, RVTRACE_COMPONENT_CTRL_OFFSET); + if (val & BIT(RVTRACE_COMPONENT_CTRL_CONTEXT_SHIFT)) { + priv->has_context = true; + val &= ~BIT(RVTRACE_COMPONENT_CTRL_CONTEXT_SHIFT); + gtrace_write32(pdata, val, RVTRACE_COMPONENT_CTRL_OFFSET); + } + /* Check if trTeInhibitSrc is hardwired to 1 */ val = gtrace_read32(pdata, RVTRACE_COMPONENT_CTRL_OFFSET); val &= ~BIT(RVTRACE_COMPONENT_CTRL_INHIBITSRC_SHIFT); @@ -76,14 +104,16 @@ static int rvtrace_encoder_probe(struct gtrace_component *comp) if (!(val & BIT(RVTRACE_COMPONENT_CTRL_INHIBITSRC_SHIFT))) { /* Probe how many bits can trTeSrcID span in the emitted Trace messages */ val = gtrace_read32(pdata, RVTRACE_ENCODER_INSTFEAT_OFFSET); - val |= (RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK << - RVTRACE_ENCODER_INSTFEAT_SRCBITS_SHIFT); - gtrace_write32(pdata, val, RVTRACE_ENCODER_INSTFEAT_OFFSET); - val = gtrace_read32(pdata, RVTRACE_ENCODER_INSTFEAT_OFFSET); - hw_srcbits = (val >> RVTRACE_ENCODER_INSTFEAT_SRCBITS_SHIFT) & - RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK; + hw_srcbits = RVTRACE_ENCODER_INSTFEAT_SRCBITS_MAX; + do { + val &= ~RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK; + val |= FIELD_PREP(RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK, hw_srcbits); + gtrace_write32(pdata, val, RVTRACE_ENCODER_INSTFEAT_OFFSET); + val = gtrace_read32(pdata, RVTRACE_ENCODER_INSTFEAT_OFFSET); + if (FIELD_GET(RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK, val) == hw_srcbits) + break; + } while (--hw_srcbits); /* Determine if there are sufficient bits to hold max cpus */ - hw_srcbits = min_t(u32, hw_srcbits, RVTRACE_ENCODER_INSTFEAT_SRCBITS_MAX); srcbits = min_t(u32, hw_srcbits, order_base_2(nr_cpu_ids)); if (srcbits && pdata->bound_cpu < BIT(srcbits)) { rvtrace_encoder_set_srcid(pdata, srcbits); @@ -98,6 +128,15 @@ static int rvtrace_encoder_probe(struct gtrace_component *comp) gtrace_write32(pdata, val, RVTRACE_COMPONENT_CTRL_OFFSET); } dev_set_drvdata(&comp->dev, priv); + + if (boot_enable) { + ret = gtrace_enable_sysfs(comp); + if (ret) { + dev_err(&comp->dev, "failed to enable tracing on boot.\n"); + return -EINVAL; + } + priv->boot_enable = true; + } return 0; }

@@ -117,10 +156,20 @@ static int rvtrace_encoder_start(struct gtrace_component *comp) return ret; }

- /* set mode */ val = gtrace_read32(pdata, RVTRACE_COMPONENT_CTRL_OFFSET); - val &= ~(RVTRACE_COMPONENT_CTRL_INSTMODE_MASK << RVTRACE_COMPONENT_CTRL_INSTMODE_SHIFT); - val |= (RVTRACE_COMPONENT_CTRL_INSTMODE_OPIT << RVTRACE_COMPONENT_CTRL_INSTMODE_SHIFT); + /* set mode */ + val &= ~RVTRACE_COMPONENT_CTRL_INSTMODE_MASK; + val |= FIELD_PREP(RVTRACE_COMPONENT_CTRL_INSTMODE_MASK, + RVTRACE_COMPONENT_CTRL_INSTMODE_OPIT); + /* set context */ + val &= ~BIT(RVTRACE_COMPONENT_CTRL_CONTEXT_SHIFT); + val |= FIELD_PREP(BIT(RVTRACE_COMPONENT_CTRL_CONTEXT_SHIFT), !!priv->context); + /* set syncmode */ + val &= ~RVTRACE_COMPONENT_CTRL_SYNCMODE_MASK; + val |= FIELD_PREP(RVTRACE_COMPONENT_CTRL_SYNCMODE_MASK, priv->syncmode); + /* set syncmax */ + val &= ~RVTRACE_COMPONENT_CTRL_SYNCMAX_MASK; + val |= FIELD_PREP(RVTRACE_COMPONENT_CTRL_SYNCMAX_MASK, priv->syncmax); gtrace_write32(pdata, val, RVTRACE_COMPONENT_CTRL_OFFSET);

val = gtrace_read32(pdata, RVTRACE_COMPONENT_CTRL_OFFSET); diff --git a/drivers/hwtracing/gtrace/rvtrace-encoder-sysfs.c b/drivers/hwtracing/gtrace/rvtrace-encoder-sysfs.c new file mode 100644 index 000000000000..e973d0e43036 --- /dev/null +++ b/drivers/hwtracing/gtrace/rvtrace-encoder-sysfs.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#include <linux/device.h> +#include <linux/gtrace.h> +#include <linux/types.h> +#include "rvtrace.h" +#include "rvtrace-encoder.h" + +struct rvtrace_component_reg { + struct gtrace_component *comp; + u32 offset; + u32 data; +}; + +static void do_smp_cross_read(void *data) +{ + struct rvtrace_component_reg *reg = data; + + reg->data = gtrace_relaxed_read32(reg->comp->pdata, reg->offset); +} + +static u32 rvtrace_cross_read(struct gtrace_component *comp, u32 offset) +{ + struct gtrace_platform_data *pdata = comp->pdata; + struct rvtrace_component_reg reg; + + reg.comp = comp; + reg.offset = offset; + /* + * smp cross call ensures the CPU will be powered up before + * accessing the RISC-V trace core registers + */ + smp_call_function_single(pdata->bound_cpu, do_smp_cross_read, &reg, 1); + return reg.data; +} + +static u32 rvtrace_attr_to_offset(struct device_attribute *attr) +{ + struct dev_ext_attribute *eattr; + + eattr = container_of(attr, struct dev_ext_attribute, attr); + return (u32)(unsigned long)eattr->var; +} + +static ssize_t rvtrace_reg_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + u32 val, offset; + struct gtrace_component *comp = to_gtrace_component(dev); + + offset = rvtrace_attr_to_offset(attr); + + val = rvtrace_cross_read(comp, offset); + + return scnprintf(buf, PAGE_SIZE, "0x%x\n", val); +} + +/* + * Macro to set an RO ext attribute with offset and show function. + */ +#define rvtrace_reg_showfn(name, offset, showfn) ( \ + &((struct dev_ext_attribute[]) { \ + { \ + __ATTR(name, 0444, showfn, NULL), \ + (void *)(unsigned long)offset \ + } \ + })[0].attr.attr \ + ) + +/* macro using the default rvtrace_reg_show function */ +#define rvtrace_reg(name, offset) \ + rvtrace_reg_showfn(name, offset, rvtrace_reg_show) + +static ssize_t cpu_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + + return sysfs_emit(buf, "%d\n", comp->pdata->bound_cpu); +} +static DEVICE_ATTR_RO(cpu); + +static struct attribute *rvtrace_encoder_attrs[] = { + &dev_attr_cpu.attr, + NULL, +}; + +static struct attribute *rvtrace_encoder_mgmt_attrs[] = { + rvtrace_reg(control, RVTRACE_COMPONENT_CTRL_OFFSET), + rvtrace_reg(features, RVTRACE_ENCODER_INSTFEAT_OFFSET), + NULL, +}; + +static ssize_t format_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + + return sysfs_emit(buf, "%u\n", priv->format); +} +static DEVICE_ATTR_RO(format); + +static ssize_t has_context_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + + return sysfs_emit(buf, "%u\n", priv->has_context); +} +static DEVICE_ATTR_RO(has_context); + +static ssize_t context_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + + return sysfs_emit(buf, "%u\n", priv->context); +} + +static ssize_t context_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + + if (!priv->has_context) + return -EINVAL; + + ret = kstrtoul(buf, 10, &val); + if (ret) + return ret; + + priv->context = !!val; + + return size; +} +static DEVICE_ATTR_RW(context); + +static const char *const syncmodes_str[] = { + [RVTRACE_ENCODER_SYNCMODE_OFF] = "off", + [RVTRACE_ENCODER_SYNCMODE_MESSAGES] = "messages", + [RVTRACE_ENCODER_SYNCMODE_CLOCK] = "clock", + [RVTRACE_ENCODER_SYNCMODE_INSTRUCTION] = "instruction", +}; + +static ssize_t syncmodes_available_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + ssize_t size = 0; + enum rvtrace_encoder_syncmode mode; + + for (mode = RVTRACE_ENCODER_SYNCMODE_OFF; mode < RVTRACE_ENCODER_SYNCMODE_MAX; mode++) { + if (priv->syncmode_hw[mode]) + size += sysfs_emit_at(buf, size, "%s ", + syncmodes_str[mode]); + } + + size += sysfs_emit_at(buf, size, "\n"); + return size; +} +static DEVICE_ATTR_RO(syncmodes_available); + +static ssize_t syncmode_preferred_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + + return sysfs_emit(buf, "%s\n", syncmodes_str[priv->syncmode]); +} + +static ssize_t syncmode_preferred_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + + if (sysfs_streq(buf, syncmodes_str[RVTRACE_ENCODER_SYNCMODE_OFF]) && + priv->syncmode_hw[RVTRACE_ENCODER_SYNCMODE_OFF]) + priv->syncmode = RVTRACE_ENCODER_SYNCMODE_OFF; + else if (sysfs_streq(buf, syncmodes_str[RVTRACE_ENCODER_SYNCMODE_MESSAGES]) && + priv->syncmode_hw[RVTRACE_ENCODER_SYNCMODE_MESSAGES]) + priv->syncmode = RVTRACE_ENCODER_SYNCMODE_MESSAGES; + else if (sysfs_streq(buf, syncmodes_str[RVTRACE_ENCODER_SYNCMODE_CLOCK]) && + priv->syncmode_hw[RVTRACE_ENCODER_SYNCMODE_CLOCK]) + priv->syncmode = RVTRACE_ENCODER_SYNCMODE_CLOCK; + else if (sysfs_streq(buf, syncmodes_str[RVTRACE_ENCODER_SYNCMODE_INSTRUCTION]) && + priv->syncmode_hw[RVTRACE_ENCODER_SYNCMODE_INSTRUCTION]) + priv->syncmode = RVTRACE_ENCODER_SYNCMODE_INSTRUCTION; + else + return -EINVAL; + return size; +} +static DEVICE_ATTR_RW(syncmode_preferred); + +static ssize_t syncmax_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + + return sysfs_emit(buf, "%u\n", priv->syncmax); +} + +static ssize_t syncmax_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + unsigned long val; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret) + return ret; + + if (val > RVTRACE_COMPONENT_CTRL_SYNCMAX_MAX) + return -EINVAL; + + priv->syncmax = val; + return size; +} +static DEVICE_ATTR_RW(syncmax); + +static struct attribute *rvtrace_encoder_control_attrs[] = { + &dev_attr_format.attr, + &dev_attr_has_context.attr, + &dev_attr_context.attr, + &dev_attr_syncmodes_available.attr, + &dev_attr_syncmode_preferred.attr, + &dev_attr_syncmax.attr, + NULL, +}; + +static ssize_t srcbits_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_encoder_priv *priv = dev_get_drvdata(&comp->dev); + + return sysfs_emit(buf, "%u\n", priv->srcbits); +} +static DEVICE_ATTR_RO(srcbits); + +static struct attribute *rvtrace_encoder_features_attrs[] = { + &dev_attr_srcbits.attr, + NULL, +}; + +static const struct attribute_group rvtrace_encoder_group = { + .attrs = rvtrace_encoder_attrs, +}; + +static const struct attribute_group rvtrace_encoder_mgmt_group = { + .attrs = rvtrace_encoder_mgmt_attrs, + .name = "mgmt", +}; + +static const struct attribute_group rvtrace_encoder_control_group = { + .attrs = rvtrace_encoder_control_attrs, + .name = "control", +}; + +static const struct attribute_group rvtrace_encoder_features_group = { + .attrs = rvtrace_encoder_features_attrs, + .name = "features", +}; + +const struct attribute_group *rvtrace_encoder_groups[] = { + &rvtrace_encoder_group, + &rvtrace_encoder_mgmt_group, + &rvtrace_encoder_control_group, + &rvtrace_encoder_features_group, + NULL, +}; +EXPORT_SYMBOL_GPL(rvtrace_encoder_groups); diff --git a/drivers/hwtracing/gtrace/rvtrace-encoder.h b/drivers/hwtracing/gtrace/rvtrace-encoder.h new file mode 100644 index 000000000000..733d44c0e641 --- /dev/null +++ b/drivers/hwtracing/gtrace/rvtrace-encoder.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __RVTRACE_ENCODER_H__ +#define __RVTRACE_ENCODER_H__ + +#define RVTRACE_COMPONENT_CTRL_ITRACE_SHIFT 2 +#define RVTRACE_COMPONENT_CTRL_INSTMODE_MASK GENMASK(6, 4) +#define RVTRACE_COMPONENT_CTRL_INSTMODE_OPIT 0x6 +#define RVTRACE_COMPONENT_CTRL_CONTEXT_SHIFT 9 +#define RVTRACE_COMPONENT_CTRL_INHIBITSRC_SHIFT 15 +#define RVTRACE_COMPONENT_CTRL_SYNCMODE_MASK GENMASK(17, 16) +#define RVTRACE_COMPONENT_CTRL_SYNCMAX_MASK GENMASK(23, 20) +#define RVTRACE_COMPONENT_CTRL_SYNCMAX_MAX 15 +#define RVTRACE_COMPONENT_CTRL_FORMAT_MASK GENMASK(26, 24) +#define RVTRACE_COMPONENT_CTRL_FORMAT_ETRACE 0x0 +#define RVTRACE_COMPONENT_CTRL_FORMAT_NTRACE 0x1 + +#define RVTRACE_ENCODER_INSTFEAT_OFFSET 0x8 +#define RVTRACE_ENCODER_INSTFEAT_SRCID_MASK GENMASK(27, 16) +#define RVTRACE_ENCODER_INSTFEAT_SRCBITS_MASK GENMASK(31, 28) +#define RVTRACE_ENCODER_INSTFEAT_SRCBITS_MAX 12 + +enum rvtrace_encoder_syncmode { + RVTRACE_ENCODER_SYNCMODE_OFF, + RVTRACE_ENCODER_SYNCMODE_MESSAGES, + RVTRACE_ENCODER_SYNCMODE_CLOCK, + RVTRACE_ENCODER_SYNCMODE_INSTRUCTION, + RVTRACE_ENCODER_SYNCMODE_MAX, +}; + +/** + * struct rvtrace_encoder_priv - Private driver data for the rvtrace encoder. + * @syncmode_hw: Hardware capability for synchronization modes. + * @syncmode: Current operational synchronization mode configured in hardware. + * @syncmax: Maximum interval value allowed for synchronization. + * @srcbits: Width of source bits allocated for trace generation. + * @format: Trace format type used by this encoder compinent. + * @has_context: Hardware flag for scontext/mcontext and privilege tracking. + * @context: Current configuration for scontext/mcontext and privilege tracking. + * @boot_enable: Flag to automatically enable tracing on boot. + */ +struct rvtrace_encoder_priv { + bool syncmode_hw[RVTRACE_ENCODER_SYNCMODE_MAX]; + u32 syncmode; + u32 syncmax; + u32 srcbits; + enum gtrace_format format; + bool has_context; + bool context; + bool boot_enable; +}; + +#endif/* __RVTRACE_ENCODER_H__ */ diff --git a/drivers/hwtracing/gtrace/rvtrace-platform.c b/drivers/hwtracing/gtrace/rvtrace-platform.c index e5c1f47388a2..64496a638359 100644 --- a/drivers/hwtracing/gtrace/rvtrace-platform.c +++ b/drivers/hwtracing/gtrace/rvtrace-platform.c @@ -129,6 +129,14 @@ static int rvtrace_platform_probe(struct platform_device *pdev) id.type = gtype; id.version = rvtrace_component_mkversion(major, minor);

+ switch (gtype) { + case GTRACE_RVTRACE_ENCODER: + pdata->groups = rvtrace_encoder_groups; + break; + default: + pdata->groups = NULL; + } + comp = gtrace_register_component(&id, rvtrace_type_name(gtype), pdata); if (IS_ERR(comp)) return PTR_ERR(comp); diff --git a/drivers/hwtracing/gtrace/rvtrace.h b/drivers/hwtracing/gtrace/rvtrace.h index e6be1ce303cc..fee8e2e4f42a 100644 --- a/drivers/hwtracing/gtrace/rvtrace.h +++ b/drivers/hwtracing/gtrace/rvtrace.h @@ -118,4 +118,7 @@ static inline int rvtrace_comp_poll_empty(struct gtrace_component *comp)

extern const struct gtrace_hw_ops rvtrace_hw_ops;

+/* Sysfs attribute groups for RISC-V trace components */ +extern const struct attribute_group *rvtrace_encoder_groups[]; + #endif /* __LINUX_RVTRACE_H__ */

Replace 'pid_t owner' with 'struct perf_event *event' in struct gtrace_path. This allows component drivers to directly access the perf_event for PMU configuration.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- drivers/hwtracing/gtrace/gtrace-core.c | 12 ++++++++++-- drivers/hwtracing/gtrace/gtrace-perf.c | 2 +- include/linux/gtrace.h | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/drivers/hwtracing/gtrace/gtrace-core.c b/drivers/hwtracing/gtrace/gtrace-core.c index cae0a7fa820f..3f55d8b7302d 100644 --- a/drivers/hwtracing/gtrace/gtrace-core.c +++ b/drivers/hwtracing/gtrace/gtrace-core.c @@ -13,6 +13,7 @@ #include <linux/spinlock.h> #include <linux/xarray.h> #include <linux/gtrace.h> +#include <linux/perf_event.h>

#define GTRACE_POLL_TIMEOUT_US_DEFAULT 10

@@ -665,10 +666,16 @@ static int __gtrace_comp_start(struct gtrace_component *comp,

raw_spin_lock_irqsave(&cpriv->lock, flags); if (cpriv->start_count) { - if (cpriv->owner != path->owner || cpriv->mode != path->mode) { + if (cpriv->mode != path->mode) { ret = -EBUSY; goto out; } + if (cpriv->mode == GTRACE_COMPONENT_MODE_PERF) { + if (cpriv->owner != task_pid_nr(path->event->owner)) { + ret = -EBUSY; + goto out; + } + } } else { if (gtdrv->start) { comp->path = path; @@ -678,8 +685,9 @@ static int __gtrace_comp_start(struct gtrace_component *comp, goto out; } } - cpriv->owner = path->owner; cpriv->mode = path->mode; + if (cpriv->mode == GTRACE_COMPONENT_MODE_PERF) + cpriv->owner = task_pid_nr(path->event->owner); }

cpriv->start_count++; diff --git a/drivers/hwtracing/gtrace/gtrace-perf.c b/drivers/hwtracing/gtrace/gtrace-perf.c index 703c12b95665..2fcfbae9acf6 100644 --- a/drivers/hwtracing/gtrace/gtrace-perf.c +++ b/drivers/hwtracing/gtrace/gtrace-perf.c @@ -137,7 +137,7 @@ static void *gtrace_setup_aux(struct perf_event *event, void **pages, continue; }

- path->owner = task_pid_nr(event->owner); + path->event = event; *gtrace_event_cpu_path_ptr(event_data, cpu) = path; }

diff --git a/include/linux/gtrace.h b/include/linux/gtrace.h index 76b1aa9cfe79..3d769b405030 100644 --- a/include/linux/gtrace.h +++ b/include/linux/gtrace.h @@ -325,7 +325,7 @@ struct gtrace_path { enum gtrace_component_mode mode; u32 trace_id; #define GTRACE_INVALID_TRACE_ID 0 - pid_t owner; + struct perf_event *event; };

struct gtrace_component *gtrace_path_source(struct gtrace_path *path);

Implement gtrace_perf_symlink() to let RISC-V Trace encoder link themselves to the gtrace PMU directory as "cpuX" based on their bound CPU. --- drivers/hwtracing/gtrace/gtrace-perf.c | 29 +++++++++++++++++-- .../hwtracing/gtrace/rvtrace-encoder-core.c | 10 +++++++ include/linux/gtrace.h | 2 ++ 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/drivers/hwtracing/gtrace/gtrace-perf.c b/drivers/hwtracing/gtrace/gtrace-perf.c index 2fcfbae9acf6..604b673a29d5 100644 --- a/drivers/hwtracing/gtrace/gtrace-perf.c +++ b/drivers/hwtracing/gtrace/gtrace-perf.c @@ -23,6 +23,7 @@ #define GTRACE_PMU_NAME "gtrace" static struct pmu gtrace_pmu; static struct workqueue_struct *gtrace_wq; +static bool gtrace_perf_up;

/** * struct gtrace_event_data - generic hardware trace perf event data @@ -351,6 +352,27 @@ static const struct attribute_group *gtrace_pmu_attr_groups[] = { NULL, };

+int gtrace_perf_symlink(struct gtrace_component *comp, bool link) +{ + char entry[sizeof("cpu9999999")]; + int ret = 0, cpu = comp->pdata->bound_cpu; + struct device *pmu_dev = gtrace_pmu.dev; + struct device *comp_dev = &comp->dev; + + sprintf(entry, "cpu%d", cpu); + + if (!gtrace_perf_up) + return -EPROBE_DEFER; + + if (link) + ret = sysfs_create_link(&pmu_dev->kobj, &comp_dev->kobj, entry); + else + sysfs_remove_link(&pmu_dev->kobj, entry); + + return ret; +} +EXPORT_SYMBOL_GPL(gtrace_perf_symlink); + int __init gtrace_perf_init(void) { int ret; @@ -373,10 +395,13 @@ int __init gtrace_perf_init(void) return -ENOMEM;

ret = perf_pmu_register(&gtrace_pmu, GTRACE_PMU_NAME, -1); - if (ret) + if (ret) { destroy_workqueue(gtrace_wq); + return ret; + }

- return ret; + gtrace_perf_up = true; + return 0; }

void __exit gtrace_perf_exit(void) diff --git a/drivers/hwtracing/gtrace/rvtrace-encoder-core.c b/drivers/hwtracing/gtrace/rvtrace-encoder-core.c index 817535ff9558..17a3dde1314c 100644 --- a/drivers/hwtracing/gtrace/rvtrace-encoder-core.c +++ b/drivers/hwtracing/gtrace/rvtrace-encoder-core.c @@ -129,6 +129,10 @@ static int rvtrace_encoder_probe(struct gtrace_component *comp) } dev_set_drvdata(&comp->dev, priv);

+ ret = gtrace_perf_symlink(comp, true); + if (ret) + return ret; + if (boot_enable) { ret = gtrace_enable_sysfs(comp); if (ret) { @@ -140,6 +144,11 @@ static int rvtrace_encoder_probe(struct gtrace_component *comp) return 0; }

+static void rvtrace_encoder_remove(struct gtrace_component *comp) +{ + gtrace_perf_symlink(comp, false); +} + static void rvtrace_encoder_parse_event_config(struct rvtrace_encoder_priv *priv, struct perf_event *event) { @@ -240,6 +249,7 @@ static const struct gtrace_component_id rvtrace_encoder_ids[] = { static struct gtrace_driver rvtrace_encoder_driver = { .id_table = rvtrace_encoder_ids, .probe = rvtrace_encoder_probe, + .remove = rvtrace_encoder_remove, .start = rvtrace_encoder_start, .stop = rvtrace_encoder_stop, .driver = { diff --git a/include/linux/gtrace.h b/include/linux/gtrace.h index 3d769b405030..4bbe5ac835ec 100644 --- a/include/linux/gtrace.h +++ b/include/linux/gtrace.h @@ -437,4 +437,6 @@ int gtrace_disable_sysfs(struct gtrace_component *comp); int gtrace_sysfs_add(struct gtrace_component *comp); void gtrace_sysfs_remove(struct gtrace_component *comp);

+int gtrace_perf_symlink(struct gtrace_component *comp, bool link); + #endif /* __LINUX_GTRACE_H__ */

Add initial implementation of RISC-V Trace funnel driver. The funnel is defined in the RISC-V Trace Control Interface specification.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- drivers/hwtracing/gtrace/Kconfig | 14 ++ drivers/hwtracing/gtrace/Makefile | 1 + drivers/hwtracing/gtrace/rvtrace-funnel.c | 165 ++++++++++++++++++++ drivers/hwtracing/gtrace/rvtrace-funnel.h | 21 +++ drivers/hwtracing/gtrace/rvtrace-platform.c | 3 + drivers/hwtracing/gtrace/rvtrace.h | 1 + 6 files changed, 205 insertions(+) create mode 100644 drivers/hwtracing/gtrace/rvtrace-funnel.c create mode 100644 drivers/hwtracing/gtrace/rvtrace-funnel.h

diff --git a/drivers/hwtracing/gtrace/Kconfig b/drivers/hwtracing/gtrace/Kconfig index e5b62d65e2fd..f6026965c54d 100644 --- a/drivers/hwtracing/gtrace/Kconfig +++ b/drivers/hwtracing/gtrace/Kconfig @@ -57,3 +57,17 @@ config RVTRACE_RAMSINK

To compile this driver as a module, choose M here: the module will be called rvtrace-ramsink. + +config RVTRACE_FUNNEL + tristate "RISC-V Trace Funnel driver" + depends on RVTRACE + default y + help + This driver provides support for the RISC-V trace funnel component + which is defined by the RISC-V Trace Control Interface specification. + The funnel merges trace data from multiple input ports into a single + output stream. It is a link component in the trace path between + sources and sinks. + + To compile this driver as a module, choose M here: the module + will be called rvtrace-funnel. diff --git a/drivers/hwtracing/gtrace/Makefile b/drivers/hwtracing/gtrace/Makefile index 87cb5add78d4..9c99d917b9f4 100644 --- a/drivers/hwtracing/gtrace/Makefile +++ b/drivers/hwtracing/gtrace/Makefile @@ -8,3 +8,4 @@ obj-$(CONFIG_RVTRACE) += rvtrace-platform.o obj-$(CONFIG_RVTRACE_ENCODER) += rvtrace-encoder.o rvtrace-encoder-y := rvtrace-encoder-core.o rvtrace-encoder-sysfs.o obj-$(CONFIG_RVTRACE_RAMSINK) += rvtrace-ramsink.o +obj-$(CONFIG_RVTRACE_FUNNEL) += rvtrace-funnel.o diff --git a/drivers/hwtracing/gtrace/rvtrace-funnel.c b/drivers/hwtracing/gtrace/rvtrace-funnel.c new file mode 100644 index 000000000000..941cd3360395 --- /dev/null +++ b/drivers/hwtracing/gtrace/rvtrace-funnel.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Spacemit Technologies, Inc. + */ + +#include <linux/device.h> +#include <linux/gtrace.h> +#include <linux/types.h> +#include "rvtrace.h" +#include "rvtrace-funnel.h" + +static int rvtrace_funnel_start(struct gtrace_component *comp) +{ + int ret; + + ret = gtrace_enable_component(comp); + if (ret) + dev_err(&comp->dev, "failed to start rvtrace funnel.\n"); + + return ret; +} + +static int rvtrace_funnel_stop(struct gtrace_component *comp) +{ + struct rvtrace_funnel_priv *priv = dev_get_drvdata(&comp->dev); + int ret; + + ret = gtrace_disable_component(comp); + if (ret) { + dev_err(&comp->dev, "failed to stop rvtrace funnel.\n"); + return ret; + } + + if (priv->disinput_mask) + gtrace_write32(comp->pdata, RVTRACE_FUNNEL_DISINPUT_MASK, + RVTRACE_FUNNEL_DISINPUT_OFFSET); + + return 0; +} + +static int _rvtrace_funnel_start_port(struct gtrace_component *comp, int port) +{ + struct gtrace_platform_data *pdata = comp->pdata; + struct rvtrace_funnel_priv *priv = dev_get_drvdata(&comp->dev); + u32 disinput; + int ret; + + if (!(priv->disinput_mask & BIT(port))) + return 0; + + disinput = gtrace_read32(pdata, RVTRACE_FUNNEL_DISINPUT_OFFSET); + disinput &= ~BIT(port); + gtrace_write32(pdata, disinput, RVTRACE_FUNNEL_DISINPUT_OFFSET); + ret = gtrace_poll_bit(pdata, RVTRACE_FUNNEL_DISINPUT_OFFSET, + port, 0, comp->pdata->control_poll_timeout_usecs); + if (ret) + dev_err(&comp->dev, "failed to enable rvtrace funnel inport %d.\n", port); + + return ret; +} + +static int rvtrace_funnel_start_port(struct gtrace_component *comp, + struct gtrace_connection *in, + struct gtrace_connection *out) +{ + int ret = 0; + + if (in->dest_refcnt == 0) + ret = _rvtrace_funnel_start_port(comp, in->dest_port); + if (!ret) + in->dest_refcnt++; + + return ret; +} + +static void rvtrace_funnel_stop_port(struct gtrace_component *comp, + struct gtrace_connection *in, + struct gtrace_connection *out) +{ + /* + * Setting trFunnelEnable to 0 flushes any queued trace data to output. + * Due to this flush behavior of the RISC-V trace funnel during component + * disablement, we do not disable individual ports here. Instead, we + * defer it and disable all ports at once when the entire component is + * stopped. For now, we only need to decrement the reference count. + */ + --in->dest_refcnt; +} + +static ssize_t disinput_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u32 val; + struct gtrace_component *comp = to_gtrace_component(dev); + + val = gtrace_read32(comp->pdata, RVTRACE_FUNNEL_DISINPUT_OFFSET); + + return scnprintf(buf, PAGE_SIZE, "0x%x\n", val); +} +static DEVICE_ATTR_RO(disinput); + +static struct attribute *rvtrace_funnel_attrs[] = { + &dev_attr_disinput.attr, + NULL, +}; + +static struct attribute_group rvtrace_funnel_group = { + .attrs = rvtrace_funnel_attrs, +}; + +const struct attribute_group *rvtrace_funnel_groups[] = { + &rvtrace_funnel_group, + NULL, +}; +EXPORT_SYMBOL_GPL(rvtrace_funnel_groups); + +static int rvtrace_funnel_probe(struct gtrace_component *comp) +{ + struct gtrace_platform_data *pdata = comp->pdata; + struct rvtrace_funnel_priv *priv; + + priv = devm_kzalloc(&comp->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + gtrace_write32(pdata, RVTRACE_FUNNEL_DISINPUT_MASK, RVTRACE_FUNNEL_DISINPUT_OFFSET); + priv->disinput_mask = gtrace_read32(pdata, RVTRACE_FUNNEL_DISINPUT_OFFSET); + dev_set_drvdata(&comp->dev, priv); + return 0; +} + +static const struct gtrace_component_id rvtrace_funnel_ids[] = { + { .type = GTRACE_RVTRACE_FUNNEL, + .version = rvtrace_component_mkversion(1, 0), }, + {}, +}; + +static struct gtrace_driver rvtrace_funnel_driver = { + .id_table = rvtrace_funnel_ids, + .start = rvtrace_funnel_start, + .stop = rvtrace_funnel_stop, + .start_port = rvtrace_funnel_start_port, + .stop_port = rvtrace_funnel_stop_port, + .probe = rvtrace_funnel_probe, + .driver = { + .name = "rvtrace-funnel", + }, +}; + +static int __init rvtrace_funnel_init(void) +{ + return gtrace_register_driver(&rvtrace_funnel_driver); +} + +static void __exit rvtrace_funnel_exit(void) +{ + gtrace_unregister_driver(&rvtrace_funnel_driver); +} + +module_init(rvtrace_funnel_init); +module_exit(rvtrace_funnel_exit); + +MODULE_AUTHOR("Zane Leung"); +MODULE_DESCRIPTION("RISC-V Trace Funnel Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwtracing/gtrace/rvtrace-funnel.h b/drivers/hwtracing/gtrace/rvtrace-funnel.h new file mode 100644 index 000000000000..41547c8971ba --- /dev/null +++ b/drivers/hwtracing/gtrace/rvtrace-funnel.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Spacemit Technologies, Inc. + */ + +#ifndef __RVTRACE_HENNEL_H__ +#define __RVTRACE_HENNEL_H__ + +#define RVTRACE_FUNNEL_DISINPUT_OFFSET 0x008 +#define RVTRACE_FUNNEL_DISINPUT_MASK 0xffff + +/** + * struct rvtrace_encoder_priv - Private driver data for the rvtrace funnel + * @disinput_mask: Bitmask of inputs supporting disable. + */ + +struct rvtrace_funnel_priv { + u32 disinput_mask; +}; + +#endif /* __RVTRACE_ENCODER_H__ */ diff --git a/drivers/hwtracing/gtrace/rvtrace-platform.c b/drivers/hwtracing/gtrace/rvtrace-platform.c index 64496a638359..ad8daa06e13e 100644 --- a/drivers/hwtracing/gtrace/rvtrace-platform.c +++ b/drivers/hwtracing/gtrace/rvtrace-platform.c @@ -133,6 +133,9 @@ static int rvtrace_platform_probe(struct platform_device *pdev) case GTRACE_RVTRACE_ENCODER: pdata->groups = rvtrace_encoder_groups; break; + case GTRACE_RVTRACE_FUNNEL: + pdata->groups = rvtrace_funnel_groups; + break; default: pdata->groups = NULL; } diff --git a/drivers/hwtracing/gtrace/rvtrace.h b/drivers/hwtracing/gtrace/rvtrace.h index fee8e2e4f42a..a345ba110267 100644 --- a/drivers/hwtracing/gtrace/rvtrace.h +++ b/drivers/hwtracing/gtrace/rvtrace.h @@ -120,5 +120,6 @@ extern const struct gtrace_hw_ops rvtrace_hw_ops;

/* Sysfs attribute groups for RISC-V trace components */ extern const struct attribute_group *rvtrace_encoder_groups[]; +extern const struct attribute_group *rvtrace_funnel_groups[];

#endif /* __LINUX_RVTRACE_H__ */

Add initial implementation of RISC-V Trace atbbridge driver. The atbbridge is defined in the RISC-V Trace Control Interface specification.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- drivers/hwtracing/gtrace/Kconfig | 15 ++ drivers/hwtracing/gtrace/Makefile | 1 + drivers/hwtracing/gtrace/rvtrace-atbbridge.c | 153 +++++++++++++++++++ drivers/hwtracing/gtrace/rvtrace-platform.c | 3 + drivers/hwtracing/gtrace/rvtrace.h | 1 + 5 files changed, 173 insertions(+) create mode 100644 drivers/hwtracing/gtrace/rvtrace-atbbridge.c

diff --git a/drivers/hwtracing/gtrace/Kconfig b/drivers/hwtracing/gtrace/Kconfig index f6026965c54d..3af47d7c3ce3 100644 --- a/drivers/hwtracing/gtrace/Kconfig +++ b/drivers/hwtracing/gtrace/Kconfig @@ -71,3 +71,18 @@ config RVTRACE_FUNNEL

To compile this driver as a module, choose M here: the module will be called rvtrace-funnel. + +config RVTRACE_ATBBRIDGE + tristate "RISC-V Trace ATB Bridge driver" + depends on RVTRACE + default y + help + This driver provides support for the RISC-V trace ATB bridge + component which is defined by the RISC-V Trace Control Interface + specification. The ATB bridge converts the RISC-V trace stream + to the ARM Advanced Trace Bus (ATB) protocol, allowing RISC-V + trace sources to feed into ARM CoreSight trace components. It is + a link component in the trace path. + + To compile this driver as a module, choose M here: the module + will be called rvtrace-atbbridge. diff --git a/drivers/hwtracing/gtrace/Makefile b/drivers/hwtracing/gtrace/Makefile index b60d3cfe5248..3f1dcb5664a2 100644 --- a/drivers/hwtracing/gtrace/Makefile +++ b/drivers/hwtracing/gtrace/Makefile @@ -10,3 +10,4 @@ obj-$(CONFIG_RVTRACE_ENCODER) += rvtrace-encoder.o rvtrace-encoder-y := rvtrace-encoder-core.o rvtrace-encoder-sysfs.o obj-$(CONFIG_RVTRACE_RAMSINK) += rvtrace-ramsink.o obj-$(CONFIG_RVTRACE_FUNNEL) += rvtrace-funnel.o +obj-$(CONFIG_RVTRACE_ATBBRIDGE) += rvtrace-atbbridge.o diff --git a/drivers/hwtracing/gtrace/rvtrace-atbbridge.c b/drivers/hwtracing/gtrace/rvtrace-atbbridge.c new file mode 100644 index 000000000000..92a59ed80a11 --- /dev/null +++ b/drivers/hwtracing/gtrace/rvtrace-atbbridge.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Spacemit Technologies, Inc. + */ + +#include <linux/device.h> +#include <linux/gtrace.h> +#include <linux/idr.h> +#include <linux/types.h> +#include <linux/bitfield.h> +#include "rvtrace.h" + +#define RVTRACE_ATBBRIDGE_CTRL_ATID_MASK GENMASK(14, 8) +#define RVTRACE_ATBBRIDGE_ATID_MIN 0x1 +#define RVTRACE_ATBBRIDGE_ATID_MAX 0x6f + +static DEFINE_IDA(rvtrace_atbbridge_atid_ida); + +struct rvtrace_atbbridge_priv { + int atid; +}; + +static int rvtrace_atbbridge_start(struct gtrace_component *comp) +{ + int ret; + + ret = gtrace_enable_component(comp); + if (ret) + dev_err(&comp->dev, "failed to start atbbridge.\n"); + + return ret; +} + +static int rvtrace_atbbridge_stop(struct gtrace_component *comp) +{ + int ret; + + ret = gtrace_disable_component(comp); + if (ret) + dev_err(&comp->dev, "failed to stop atbbridge.\n"); + + return ret; +} + +static ssize_t cpu_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + + return sysfs_emit(buf, "%d\n", comp->pdata->bound_cpu); +} +static DEVICE_ATTR_RO(cpu); + +static ssize_t atid_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gtrace_component *comp = to_gtrace_component(dev); + struct rvtrace_atbbridge_priv *priv = dev_get_drvdata(&comp->dev); + + return sysfs_emit(buf, "%u\n", priv->atid); +} +static DEVICE_ATTR_RO(atid); + +static struct attribute *rvtrace_atbbridge_attrs[] = { + &dev_attr_cpu.attr, + &dev_attr_atid.attr, + NULL, +}; + +static const struct attribute_group rvtrace_atbbridge_group = { + .attrs = rvtrace_atbbridge_attrs, +}; + +const struct attribute_group *rvtrace_atbbridge_groups[] = { + &rvtrace_atbbridge_group, + NULL, +}; +EXPORT_SYMBOL_GPL(rvtrace_atbbridge_groups); + +static void rvtrace_atbbridge_cleanup(struct gtrace_component *comp) +{ + struct rvtrace_atbbridge_priv *priv = dev_get_drvdata(&comp->dev); + + ida_free(&rvtrace_atbbridge_atid_ida, priv->atid); +} + +static int rvtrace_atbbridge_probe(struct gtrace_component *comp) +{ + struct rvtrace_atbbridge_priv *priv; + u32 val; + + priv = devm_kzalloc(&comp->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->atid = ida_alloc_range(&rvtrace_atbbridge_atid_ida, + RVTRACE_ATBBRIDGE_ATID_MIN, + RVTRACE_ATBBRIDGE_ATID_MAX, + GFP_KERNEL); + if (priv->atid < 0) { + dev_err(&comp->dev, "failed to allocate ATID.\n"); + return priv->atid; + } + + val = gtrace_read32(comp->pdata, RVTRACE_COMPONENT_CTRL_OFFSET); + val &= ~RVTRACE_ATBBRIDGE_CTRL_ATID_MASK; + val |= FIELD_PREP(RVTRACE_ATBBRIDGE_CTRL_ATID_MASK, priv->atid); + gtrace_write32(comp->pdata, val, RVTRACE_COMPONENT_CTRL_OFFSET); + + dev_set_drvdata(&comp->dev, priv); + + return 0; +} + +static void rvtrace_atbbridge_remove(struct gtrace_component *comp) +{ + rvtrace_atbbridge_cleanup(comp); +} + +static const struct gtrace_component_id rvtrace_atbbridge_ids[] = { + { .type = GTRACE_RVTRACE_ATBBRIDGE, + .version = rvtrace_component_mkversion(1, 0), }, + {}, +}; + +static struct gtrace_driver rvtrace_atbbridge_driver = { + .id_table = rvtrace_atbbridge_ids, + .probe = rvtrace_atbbridge_probe, + .remove = rvtrace_atbbridge_remove, + .start = rvtrace_atbbridge_start, + .stop = rvtrace_atbbridge_stop, + .driver = { + .name = "rvtrace-atbbridge", + }, +}; + +static int __init rvtrace_atbbridge_init(void) +{ + return gtrace_register_driver(&rvtrace_atbbridge_driver); +} + +static void __exit rvtrace_atbbridge_exit(void) +{ + gtrace_unregister_driver(&rvtrace_atbbridge_driver); + ida_destroy(&rvtrace_atbbridge_atid_ida); +} + +module_init(rvtrace_atbbridge_init); +module_exit(rvtrace_atbbridge_exit); + +MODULE_AUTHOR("Zane Leung"); +MODULE_DESCRIPTION("RISC-V Trace ATB Bridge Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwtracing/gtrace/rvtrace-platform.c b/drivers/hwtracing/gtrace/rvtrace-platform.c index ad8daa06e13e..9bd315867082 100644 --- a/drivers/hwtracing/gtrace/rvtrace-platform.c +++ b/drivers/hwtracing/gtrace/rvtrace-platform.c @@ -136,6 +136,9 @@ static int rvtrace_platform_probe(struct platform_device *pdev) case GTRACE_RVTRACE_FUNNEL: pdata->groups = rvtrace_funnel_groups; break; + case GTRACE_RVTRACE_ATBBRIDGE: + pdata->groups = rvtrace_atbbridge_groups; + break; default: pdata->groups = NULL; } diff --git a/drivers/hwtracing/gtrace/rvtrace.h b/drivers/hwtracing/gtrace/rvtrace.h index a345ba110267..5cbe3405e274 100644 --- a/drivers/hwtracing/gtrace/rvtrace.h +++ b/drivers/hwtracing/gtrace/rvtrace.h @@ -121,5 +121,6 @@ extern const struct gtrace_hw_ops rvtrace_hw_ops; /* Sysfs attribute groups for RISC-V trace components */ extern const struct attribute_group *rvtrace_encoder_groups[]; extern const struct attribute_group *rvtrace_funnel_groups[]; +extern const struct attribute_group *rvtrace_atbbridge_groups[];

#endif /* __LINUX_RVTRACE_H__ */

Add a CoreSight funnel driver for the gtrace framework, derived from drivers/hwtracing/coresight/coresight-funnel.c.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- drivers/hwtracing/gtrace/Kconfig | 13 + drivers/hwtracing/gtrace/Makefile | 1 + drivers/hwtracing/gtrace/coresight-funnel.c | 430 ++++++++++++++++++++ 3 files changed, 444 insertions(+) create mode 100644 drivers/hwtracing/gtrace/coresight-funnel.c

diff --git a/drivers/hwtracing/gtrace/Kconfig b/drivers/hwtracing/gtrace/Kconfig index 5dce4474db7c..081af9c0c670 100644 --- a/drivers/hwtracing/gtrace/Kconfig +++ b/drivers/hwtracing/gtrace/Kconfig @@ -101,3 +101,16 @@ config GTRACE_CORESIGHT

To compile this driver as a module, choose M here: the module will be called coresight. + +config CORESIGHT_FUNNEL + tristate "CoreSight Trace Funnel driver" + depends on GTRACE_CORESIGHT + default y + help + This driver provides support for the CoreSight funnel component. + The funnel merges trace data from multiple input ports into a single + output stream. It supports both static funnels (platform bus, + arm,coresight-static-funnel) and dynamic funnels (AMBA bus). + + To compile this driver as a module, choose M here: the module + will be called coresight-funnel. diff --git a/drivers/hwtracing/gtrace/Makefile b/drivers/hwtracing/gtrace/Makefile index 0833a40c397c..d2745c961a75 100644 --- a/drivers/hwtracing/gtrace/Makefile +++ b/drivers/hwtracing/gtrace/Makefile @@ -12,3 +12,4 @@ obj-$(CONFIG_RVTRACE_RAMSINK) += rvtrace-ramsink.o obj-$(CONFIG_RVTRACE_FUNNEL) += rvtrace-funnel.o obj-$(CONFIG_RVTRACE_ATBBRIDGE) += rvtrace-atbbridge.o obj-$(CONFIG_GTRACE_CORESIGHT) += coresight-platform.o +obj-$(CONFIG_CORESIGHT_FUNNEL) += coresight-funnel.o diff --git a/drivers/hwtracing/gtrace/coresight-funnel.c b/drivers/hwtracing/gtrace/coresight-funnel.c new file mode 100644 index 000000000000..e7a3f185da56 --- /dev/null +++ b/drivers/hwtracing/gtrace/coresight-funnel.c @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. + * + * Description: CoreSight Funnel driver for the gtrace framework. + * Derived from drivers/hwtracing/coresight/coresight-funnel.c. + */ + +#include <linux/acpi.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/amba/bus.h> +#include <linux/clk.h> + +#include "coresight.h" + +#define FUNNEL_FUNCTL 0x000 +#define FUNNEL_PRICTL 0x004 + +#define FUNNEL_HOLDTIME_MASK 0xf00 +#define FUNNEL_HOLDTIME_SHFT 0x8 +#define FUNNEL_HOLDTIME (0x7 << FUNNEL_HOLDTIME_SHFT) +#define FUNNEL_ENSx_MASK 0xff + +/** + * struct funnel_drvdata - specifics associated to a funnel component + * @base: memory mapped base address for this component. + * @atclk: optional clock for the core parts of the funnel. + * @pclk: APB clock if present, otherwise NULL + * @comp: component vitals needed by the framework. + * @priority: port selection order. + */ +struct funnel_drvdata { + void __iomem *base; + struct clk *atclk; + struct clk *pclk; + struct gtrace_component *comp; + unsigned long priority; +}; + +static int dynamic_funnel_enable_hw(struct funnel_drvdata *drvdata, int port) +{ + u32 functl; + int rc = 0; + struct gtrace_component *comp = drvdata->comp; + + CS_UNLOCK(drvdata->base); + + functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); + /* Claim the device only when we enable the first slave */ + if (!(functl & FUNNEL_ENSx_MASK)) { + rc = coresight_claim_device_unlocked(comp); + if (rc) + goto done; + } + + functl &= ~FUNNEL_HOLDTIME_MASK; + functl |= FUNNEL_HOLDTIME; + functl |= (1 << port); + writel_relaxed(functl, drvdata->base + FUNNEL_FUNCTL); + writel_relaxed(drvdata->priority, drvdata->base + FUNNEL_PRICTL); +done: + CS_LOCK(drvdata->base); + return rc; +} + +static int funnel_enable(struct gtrace_component *comp, + struct gtrace_connection *in, + struct gtrace_connection *out) +{ + int rc = 0; + struct funnel_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + bool first_enable = false; + + if (in->dest_refcnt == 0) { + if (drvdata->base) + rc = dynamic_funnel_enable_hw(drvdata, in->dest_port); + if (!rc) + first_enable = true; + } + if (!rc) + in->dest_refcnt++; + + if (first_enable) + dev_dbg(&comp->dev, "FUNNEL inport %d enabled\n", + in->dest_port); + return rc; +} + +static void dynamic_funnel_disable_hw(struct funnel_drvdata *drvdata, + int inport) +{ + u32 functl; + struct gtrace_component *comp = drvdata->comp; + + CS_UNLOCK(drvdata->base); + + functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); + functl &= ~(1 << inport); + writel_relaxed(functl, drvdata->base + FUNNEL_FUNCTL); + + /* Disclaim the device if none of the slaves are now active */ + if (!(functl & FUNNEL_ENSx_MASK)) + coresight_disclaim_device_unlocked(comp); + + CS_LOCK(drvdata->base); +} + +static void funnel_disable(struct gtrace_component *comp, + struct gtrace_connection *in, + struct gtrace_connection *out) +{ + struct funnel_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + bool last_disable = false; + + if (--in->dest_refcnt == 0) { + if (drvdata->base) + dynamic_funnel_disable_hw(drvdata, in->dest_port); + last_disable = true; + } + + if (last_disable) + dev_dbg(&comp->dev, "FUNNEL inport %d disabled\n", + in->dest_port); +} + +static ssize_t priority_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct funnel_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val = drvdata->priority; + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t priority_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct funnel_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->priority = val; + return size; +} +static DEVICE_ATTR_RW(priority); + +static u32 get_funnel_ctrl_hw(struct funnel_drvdata *drvdata) +{ + u32 functl; + + CS_UNLOCK(drvdata->base); + functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); + CS_LOCK(drvdata->base); + + return functl; +} + +static ssize_t funnel_ctrl_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u32 val; + struct funnel_drvdata *drvdata = dev_get_drvdata(dev->parent); + + pm_runtime_get_sync(dev->parent); + + val = get_funnel_ctrl_hw(drvdata); + + pm_runtime_put(dev->parent); + + return sprintf(buf, "%#x\n", val); +} +static DEVICE_ATTR_RO(funnel_ctrl); + +static struct attribute *coresight_funnel_attrs[] = { + &dev_attr_funnel_ctrl.attr, + &dev_attr_priority.attr, + NULL, +}; +ATTRIBUTE_GROUPS(coresight_funnel); + +static int funnel_probe(struct device *dev, struct resource *res) +{ + void __iomem *base; + struct gtrace_platform_data *pdata = NULL; + struct funnel_drvdata *drvdata; + struct gtrace_component_id id = { + .type = GTRACE_CORESIGHT_FUNNEL, + .version = CORESIGHT_COMPONENT_ID_VERSION, + }; + int ret; + + if (is_of_node(dev_fwnode(dev)) && + of_device_is_compatible(dev->of_node, "arm,coresight-funnel")) + dev_warn_once(dev, "Uses OBSOLETE CoreSight funnel binding\n"); + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + ret = coresight_get_enable_clocks(dev, &drvdata->pclk, &drvdata->atclk); + if (ret) + return ret; + + pdata = coresight_get_platform_data(dev); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + + /* + * Map the device base for dynamic-funnel, which has been + * validated by AMBA core. + */ + if (res) { + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + drvdata->base = base; + pdata->groups = coresight_funnel_groups; + pdata->io_mem = true; + pdata->base = base; + coresight_clear_self_claim_tag(pdata); + } + + dev_set_drvdata(dev, drvdata); + + drvdata->comp = gtrace_register_component(&id, "cs_funnel", pdata); + if (IS_ERR(drvdata->comp)) + return PTR_ERR(drvdata->comp); + + return 0; +} + +static void funnel_remove(struct device *dev) +{ + struct funnel_drvdata *drvdata = dev_get_drvdata(dev); + + gtrace_unregister_component(drvdata->comp); +} + +#ifdef CONFIG_PM +static int funnel_runtime_suspend(struct device *dev) +{ + struct funnel_drvdata *drvdata = dev_get_drvdata(dev); + + clk_disable_unprepare(drvdata->atclk); + clk_disable_unprepare(drvdata->pclk); + + return 0; +} + +static int funnel_runtime_resume(struct device *dev) +{ + struct funnel_drvdata *drvdata = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(drvdata->pclk); + if (ret) + return ret; + + ret = clk_prepare_enable(drvdata->atclk); + if (ret) + clk_disable_unprepare(drvdata->pclk); + + return ret; +} +#endif + +static const struct dev_pm_ops funnel_dev_pm_ops = { + SET_RUNTIME_PM_OPS(funnel_runtime_suspend, funnel_runtime_resume, NULL) +}; + +static int funnel_platform_probe(struct platform_device *pdev) +{ + struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + int ret; + + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + ret = funnel_probe(&pdev->dev, res); + pm_runtime_put(&pdev->dev); + if (ret) + pm_runtime_disable(&pdev->dev); + + return ret; +} + +static void funnel_platform_remove(struct platform_device *pdev) +{ + struct funnel_drvdata *drvdata = dev_get_drvdata(&pdev->dev); + + if (WARN_ON(!drvdata)) + return; + + funnel_remove(&pdev->dev); + pm_runtime_disable(&pdev->dev); +} + +static const struct of_device_id funnel_match[] = { + {.compatible = "arm,coresight-static-funnel"}, + {} +}; + +MODULE_DEVICE_TABLE(of, funnel_match); + +#ifdef CONFIG_ACPI +static const struct acpi_device_id funnel_acpi_ids[] = { + {"ARMHC9FE", 0, 0, 0}, /* ARM Coresight Static Funnel */ + {"ARMHC9FF", 0, 0, 0}, /* ARM CoreSight Dynamic Funnel */ + {}, +}; +MODULE_DEVICE_TABLE(acpi, funnel_acpi_ids); +#endif + +static struct platform_driver funnel_driver = { + .probe = funnel_platform_probe, + .remove = funnel_platform_remove, + .driver = { + .name = "coresight-funnel", + /* THIS_MODULE is taken care of by platform_driver_register() */ + .of_match_table = funnel_match, + .acpi_match_table = ACPI_PTR(funnel_acpi_ids), + .pm = &funnel_dev_pm_ops, + .suppress_bind_attrs = true, + }, +}; + +static int dynamic_funnel_probe(struct amba_device *adev, + const struct amba_id *id) +{ + int ret; + + ret = funnel_probe(&adev->dev, &adev->res); + if (!ret) + pm_runtime_put(&adev->dev); + + return ret; +} + +static void dynamic_funnel_remove(struct amba_device *adev) +{ + funnel_remove(&adev->dev); +} + +static const struct amba_id dynamic_funnel_ids[] = { + { + .id = 0x000bb908, + .mask = 0x000fffff, + }, + { + /* Coresight SoC-600 */ + .id = 0x000bb9eb, + .mask = 0x000fffff, + }, + { 0, 0, NULL }, +}; + +MODULE_DEVICE_TABLE(amba, dynamic_funnel_ids); + +static struct amba_driver dynamic_funnel_driver = { + .drv = { + .name = "coresight-dynamic-funnel", + .pm = &funnel_dev_pm_ops, + .suppress_bind_attrs = true, + }, + .probe = dynamic_funnel_probe, + .remove = dynamic_funnel_remove, + .id_table = dynamic_funnel_ids, +}; + +static const struct gtrace_component_id coresight_funnel_ids[] = { + { .type = GTRACE_CORESIGHT_FUNNEL, + .version = CORESIGHT_COMPONENT_ID_VERSION, }, + {}, +}; + +static struct gtrace_driver coresight_funnel_driver = { + .id_table = coresight_funnel_ids, + .start_port = funnel_enable, + .stop_port = funnel_disable, + .driver = { + .name = "coresight-funnel", + }, +}; + +static int __init funnel_init(void) +{ + int ret; + + ret = gtrace_register_driver(&coresight_funnel_driver); + if (ret) + return ret; + + ret = coresight_init_driver("funnel", &dynamic_funnel_driver, &funnel_driver); + if (ret) { + gtrace_unregister_driver(&coresight_funnel_driver); + return ret; + } + + return 0; +} + +static void __exit funnel_exit(void) +{ + coresight_remove_driver(&dynamic_funnel_driver, &funnel_driver); + gtrace_unregister_driver(&coresight_funnel_driver); +} + +module_init(funnel_init); +module_exit(funnel_exit); + +MODULE_AUTHOR("Pratik Patel pratikp@codeaurora.org"); +MODULE_AUTHOR("Mathieu Poirier mathieu.poirier@linaro.org"); +MODULE_AUTHOR("Zane Leung liangzhen@linux.spacemit.com"); +MODULE_DESCRIPTION("Arm CoreSight Funnel Driver"); +MODULE_LICENSE("GPL v2");

Define PERF_AUX_FLAG_GTRACE_FORMAT_CORESIGHT for the gtrace format which reports whether the data has been wrapped in CoreSight frame formatting by a CoreSight sink.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- include/uapi/linux/perf_event.h | 1 + tools/include/uapi/linux/perf_event.h | 1 + 2 files changed, 2 insertions(+)

diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index 2ac00a409a04..9d9ed2adc1d4 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -1315,6 +1315,7 @@ enum perf_callchain_context { #define PERF_AUX_FLAG_GTRACE_FORMAT_UNKNOWN 0x0000 #define PERF_AUX_FLAG_GTRACE_FORMAT_ETRACE 0x0100 /* RISC-V E-trace format */ #define PERF_AUX_FLAG_GTRACE_FORMAT_NTRACE 0x0200 /* RISC-V N-Trace format */ +#define PERF_AUX_FLAG_GTRACE_FORMAT_CORESIGHT 0x0300 /* Trace with CoreSight formatted frames */

#define PERF_FLAG_FD_NO_GROUP (1UL << 0) #define PERF_FLAG_FD_OUTPUT (1UL << 1) diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h index 2ac00a409a04..9d9ed2adc1d4 100644 --- a/tools/include/uapi/linux/perf_event.h +++ b/tools/include/uapi/linux/perf_event.h @@ -1315,6 +1315,7 @@ enum perf_callchain_context { #define PERF_AUX_FLAG_GTRACE_FORMAT_UNKNOWN 0x0000 #define PERF_AUX_FLAG_GTRACE_FORMAT_ETRACE 0x0100 /* RISC-V E-trace format */ #define PERF_AUX_FLAG_GTRACE_FORMAT_NTRACE 0x0200 /* RISC-V N-Trace format */ +#define PERF_AUX_FLAG_GTRACE_FORMAT_CORESIGHT 0x0300 /* Trace with CoreSight formatted frames */

#define PERF_FLAG_FD_NO_GROUP (1UL << 0) #define PERF_FLAG_FD_OUTPUT (1UL << 1)

Extend gtrace_driver with alloc_buf() and free_buf() so a sink can allocate its own buffer (e.g. an ETR SG table) during perf setup and release it when the event data is freed. Store the resulting buffer in gtrace_perf_auxbuf along with the event pid and snapshot mode.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- drivers/hwtracing/gtrace/gtrace-perf.c | 34 ++++++++++++++++++++++++++ include/linux/gtrace.h | 12 +++++++++ 2 files changed, 46 insertions(+)

diff --git a/drivers/hwtracing/gtrace/gtrace-perf.c b/drivers/hwtracing/gtrace/gtrace-perf.c index 9b874daa807f..e05a833aa6a4 100644 --- a/drivers/hwtracing/gtrace/gtrace-perf.c +++ b/drivers/hwtracing/gtrace/gtrace-perf.c @@ -86,6 +86,27 @@ static struct gtrace_path **gtrace_event_cpu_path_ptr(struct gtrace_event_data * return per_cpu_ptr(data->path, cpu); }

+static void free_sink_buffer(struct gtrace_event_data *event_data) +{ + int cpu; + cpumask_t *mask = &event_data->mask; + struct gtrace_path *path; + struct gtrace_component *sink; + struct gtrace_driver *sinkdrv; + + if (!event_data->buf.buf) + return; + + if (WARN_ON(cpumask_empty(mask))) + return; + + cpu = cpumask_first(mask); + path = *gtrace_event_cpu_path_ptr(event_data, cpu); + sink = gtrace_path_sink(path); + sinkdrv = to_gtrace_driver(sink->dev.driver); + sinkdrv->free_buf(event_data->buf.buf); +} + static void free_event_data(struct work_struct *work) { struct gtrace_event_data *event_data; @@ -95,6 +116,7 @@ static void free_event_data(struct work_struct *work)

event_data = container_of(work, struct gtrace_event_data, work); mask = &event_data->mask; + free_sink_buffer(event_data); for_each_cpu(cpu, mask) { path = *gtrace_event_cpu_path_ptr(event_data, cpu); gtrace_destroy_path(path); @@ -110,6 +132,7 @@ static void *gtrace_setup_aux(struct perf_event *event, void **pages, { struct gtrace_event_data *event_data = NULL; struct gtrace_component *sink = NULL; + struct gtrace_driver *sinkdrv = NULL; struct page **pagelist; int cpu = event->cpu, i; cpumask_t *mask; @@ -175,6 +198,17 @@ static void *gtrace_setup_aux(struct perf_event *event, void **pages, event_data->buf.nr_pages = nr_pages; event_data->buf.length = nr_pages * PAGE_SIZE; event_data->buf.pos = 0; + event_data->buf.snapshot = overwrite; + event_data->buf.pages = pages; + + sinkdrv = to_gtrace_driver(sink->dev.driver); + if (sinkdrv && sinkdrv->alloc_buf) { + event_data->buf.buf = sinkdrv->alloc_buf(sink, event, pages, + nr_pages, overwrite); + if (!event_data->buf.buf) + goto err; + } + return event_data; err: gtrace_free_aux(event_data); diff --git a/include/linux/gtrace.h b/include/linux/gtrace.h index 39ae13701615..a3d125882ec9 100644 --- a/include/linux/gtrace.h +++ b/include/linux/gtrace.h @@ -355,12 +355,18 @@ int gtrace_path_stop(struct gtrace_path *path); * @nr_pages: Number of pages of the AUX buffer. * @base: Start address of AUX buffer. * @pos: Position in the AUX buffer to commit traced data. + * @snapshot: Is this run in snapshot mode. + * @pages: a handle to the ring buffer. + * @buf: Optional sink-specific buffer. */ struct gtrace_perf_auxbuf { size_t length; int nr_pages; void *base; long pos; + bool snapshot; + void **pages; + void *buf; };

int gtrace_path_copyto_auxbuf(struct gtrace_path *path, @@ -371,6 +377,8 @@ int gtrace_path_copyto_auxbuf(struct gtrace_path *path, /** * struct gtrace_driver - Representation of a trace driver. * @id_table: Table to match components handled by the driver. + * @alloc_buf: Callback to alloc sink-specific buffer. + * @free_buf: Callback to free sink-specific buffer. * @copyto_auxbuf: Callback to copy data into perf AUX buffer. The driver * reports the PMU specific trace format of the copied data * via @handle (see PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK). @@ -386,6 +394,10 @@ int gtrace_path_copyto_auxbuf(struct gtrace_path *path, */ struct gtrace_driver { const struct gtrace_component_id *id_table; + void *(*alloc_buf)(struct gtrace_component *comp, + struct perf_event *event, void **pages, + int nr_pages, bool overwrite); + void (*free_buf)(void *buf); size_t (*copyto_auxbuf)(struct gtrace_component *comp, struct perf_output_handle *handle, struct gtrace_perf_auxbuf *buf);

Add CoreSight Trace Memory Controller (TMC) driver as part of the gtrace framework. ETB and ETF share one driver (ETF additionally works as a link), while ETR uses system memory and provides perf buffer alloc/free callbacks.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- .../hwtracing/coresight/coresight-tmc-etf.c | 1 - drivers/hwtracing/gtrace/Kconfig | 13 + drivers/hwtracing/gtrace/Makefile | 2 + drivers/hwtracing/gtrace/coresight-tmc-core.c | 1111 ++++++++++ drivers/hwtracing/gtrace/coresight-tmc-etf.c | 573 +++++ drivers/hwtracing/gtrace/coresight-tmc-etr.c | 1888 +++++++++++++++++ drivers/hwtracing/gtrace/coresight-tmc.h | 469 ++++ 7 files changed, 4056 insertions(+), 1 deletion(-) create mode 100644 drivers/hwtracing/gtrace/coresight-tmc-core.c create mode 100644 drivers/hwtracing/gtrace/coresight-tmc-etf.c create mode 100644 drivers/hwtracing/gtrace/coresight-tmc-etr.c create mode 100644 drivers/hwtracing/gtrace/coresight-tmc.h

diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 8882b1c4cdc0..11082fd249eb 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -80,7 +80,6 @@ static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)

if (lost) coresight_insert_barrier_packet(drvdata->buf); - return; }

static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) diff --git a/drivers/hwtracing/gtrace/Kconfig b/drivers/hwtracing/gtrace/Kconfig index 081af9c0c670..83ee442ad860 100644 --- a/drivers/hwtracing/gtrace/Kconfig +++ b/drivers/hwtracing/gtrace/Kconfig @@ -114,3 +114,16 @@ config CORESIGHT_FUNNEL

To compile this driver as a module, choose M here: the module will be called coresight-funnel. + +config CORESIGHT_TMC + tristate "CoreSight generic TMC driver" + depends on GTRACE_CORESIGHT + default y + help + This enables support for the CoreSight Trace Memory Controller + (TMC) driver. Depending on its configuration the device can act + as an embedded trace buffer (ETB), an embedded trace FIFO (ETF) + or a trace router (ETR) that stores trace data in system memory. + + To compile this driver as a module, choose M here: the module + will be called coresight-tmc. diff --git a/drivers/hwtracing/gtrace/Makefile b/drivers/hwtracing/gtrace/Makefile index d2745c961a75..e995ba8e7fc1 100644 --- a/drivers/hwtracing/gtrace/Makefile +++ b/drivers/hwtracing/gtrace/Makefile @@ -13,3 +13,5 @@ obj-$(CONFIG_RVTRACE_FUNNEL) += rvtrace-funnel.o obj-$(CONFIG_RVTRACE_ATBBRIDGE) += rvtrace-atbbridge.o obj-$(CONFIG_GTRACE_CORESIGHT) += coresight-platform.o obj-$(CONFIG_CORESIGHT_FUNNEL) += coresight-funnel.o +obj-$(CONFIG_CORESIGHT_TMC) += coresight-tmc.o +coresight-tmc-y := coresight-tmc-core.o coresight-tmc-etf.o coresight-tmc-etr.o diff --git a/drivers/hwtracing/gtrace/coresight-tmc-core.c b/drivers/hwtracing/gtrace/coresight-tmc-core.c new file mode 100644 index 000000000000..26572f1d3b6f --- /dev/null +++ b/drivers/hwtracing/gtrace/coresight-tmc-core.c @@ -0,0 +1,1111 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2012, The Linux Foundation. All rights reserved. + * + * Description: CoreSight Trace Memory Controller driver + * Derived from drivers/hwtracing/coresight/coresight-tmc-core.c + */ + +#include <linux/acpi.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/device.h> +#include <linux/idr.h> +#include <linux/io.h> +#include <linux/iommu.h> +#include <linux/err.h> +#include <linux/fs.h> +#include <linux/miscdevice.h> +#include <linux/mutex.h> +#include <linux/property.h> +#include <linux/uaccess.h> +#include <linux/slab.h> +#include <linux/dma-mapping.h> +#include <linux/spinlock.h> +#include <linux/pm_runtime.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_reserved_mem.h> +#include <linux/gtrace.h> +#include <linux/amba/bus.h> +#include <linux/platform_device.h> + +#include "coresight.h" +#include "coresight-tmc.h" + +int tmc_wait_for_tmcready(struct tmc_drvdata *drvdata) +{ + struct gtrace_component *comp = drvdata->comp; + struct gtrace_platform_data *pdata = comp->pdata; + + /* Ensure formatter, unformatter and hardware fifo are empty */ + if (gtrace_poll_bit(pdata, TMC_STS, TMC_STS_TMCREADY_BIT, 1, + pdata->control_poll_timeout_usecs)) { + dev_err(&comp->dev, + "timeout while waiting for TMC to be Ready\n"); + return -EBUSY; + } + return 0; +} + +void tmc_flush_and_stop(struct tmc_drvdata *drvdata) +{ + struct gtrace_component *comp = drvdata->comp; + struct gtrace_platform_data *pdata = comp->pdata; + u32 ffcr; + + ffcr = readl_relaxed(drvdata->base + TMC_FFCR); + ffcr |= TMC_FFCR_STOP_ON_FLUSH; + writel_relaxed(ffcr, drvdata->base + TMC_FFCR); + ffcr |= BIT(TMC_FFCR_FLUSHMAN_BIT); + writel_relaxed(ffcr, drvdata->base + TMC_FFCR); + /* Ensure flush completes */ + if (gtrace_poll_bit(pdata, TMC_FFCR, TMC_FFCR_FLUSHMAN_BIT, 0, + pdata->control_poll_timeout_usecs)) { + dev_err(&comp->dev, + "timeout while waiting for completion of Manual Flush\n"); + } + + tmc_wait_for_tmcready(drvdata); +} + +void tmc_enable_hw(struct tmc_drvdata *drvdata) +{ + writel_relaxed(TMC_CTL_CAPT_EN, drvdata->base + TMC_CTL); +} + +void tmc_disable_hw(struct tmc_drvdata *drvdata) +{ + writel_relaxed(0x0, drvdata->base + TMC_CTL); +} + +u32 tmc_get_memwidth_mask(struct tmc_drvdata *drvdata) +{ + u32 mask = 0; + + /* + * When moving RRP or an offset address forward, the new values must + * be byte-address aligned to the width of the trace memory databus + * _and_ to a frame boundary (16 byte), whichever is the biggest. For + * example, for 32-bit, 64-bit and 128-bit wide trace memory, the four + * LSBs must be 0s. For 256-bit wide trace memory, the five LSBs must + * be 0s. + */ + switch (drvdata->memwidth) { + case TMC_MEM_INTF_WIDTH_32BITS: + case TMC_MEM_INTF_WIDTH_64BITS: + case TMC_MEM_INTF_WIDTH_128BITS: + mask = GENMASK(31, 4); + break; + case TMC_MEM_INTF_WIDTH_256BITS: + mask = GENMASK(31, 5); + break; + } + + return mask; +} + +static bool is_tmc_crashdata_valid(struct tmc_drvdata *drvdata) +{ + struct tmc_crash_metadata *mdata; + + if (!tmc_has_reserved_buffer(drvdata) || + !tmc_has_crash_mdata_buffer(drvdata)) + return false; + + mdata = drvdata->crash_mdata.vaddr; + + /* Check version match */ + if (mdata->version != CS_CRASHDATA_VERSION) + return false; + + /* Check for valid metadata */ + if (!mdata->valid) { + dev_dbg(&drvdata->comp->dev, + "Data invalid in tmc crash metadata\n"); + return false; + } + + /* + * Buffer address given by metadata for retrieval of trace data + * from previous boot is expected to be same as the reserved + * trace buffer memory region provided through DTS + */ + if (drvdata->resrv_buf.paddr != mdata->trace_paddr) { + dev_dbg(&drvdata->comp->dev, + "Trace buffer address of previous boot invalid\n"); + return false; + } + + /* Check data integrity of metadata */ + if (mdata->crc32_mdata != find_crash_metadata_crc(mdata)) { + dev_err(&drvdata->comp->dev, + "CRC mismatch in tmc crash metadata\n"); + return false; + } + /* Check data integrity of tracedata */ + if (mdata->crc32_tdata != find_crash_tracedata_crc(drvdata, mdata)) { + dev_err(&drvdata->comp->dev, + "CRC mismatch in tmc crash tracedata\n"); + return false; + } + + return true; +} + +static inline ssize_t tmc_get_resvbuf_trace(struct tmc_drvdata *drvdata, + loff_t pos, size_t len, char **bufpp) +{ + s64 offset; + ssize_t actual = len; + struct tmc_resrv_buf *rbuf = &drvdata->resrv_buf; + + if (pos + actual > rbuf->len) + actual = rbuf->len - pos; + if (actual <= 0) + return 0; + + /* Compute the offset from which we read the data */ + offset = rbuf->offset + pos; + if (offset >= rbuf->size) + offset -= rbuf->size; + + /* Adjust the length to limit this transaction to end of buffer */ + actual = (actual < (rbuf->size - offset)) ? + actual : rbuf->size - offset; + + *bufpp = (char *)rbuf->vaddr + offset; + + return actual; +} + +static int tmc_prepare_crashdata(struct tmc_drvdata *drvdata) +{ + char *bufp; + ssize_t len; + u32 status, size; + u64 rrp, rwp, dba; + struct tmc_resrv_buf *rbuf; + struct tmc_crash_metadata *mdata; + + mdata = drvdata->crash_mdata.vaddr; + rbuf = &drvdata->resrv_buf; + + rrp = mdata->tmc_rrp; + rwp = mdata->tmc_rwp; + dba = mdata->tmc_dba; + status = mdata->tmc_sts; + size = mdata->tmc_ram_size << 2; + + /* Sync the buffer pointers */ + rbuf->offset = rrp - dba; + if (status & TMC_STS_FULL) + rbuf->len = size; + else + rbuf->len = rwp - rrp; + + /* Additional sanity checks for validating metadata */ + if ((rbuf->offset > size) || + (rbuf->len > size)) { + dev_dbg(&drvdata->comp->dev, + "Offset and length invalid in tmc crash metadata\n"); + return -EINVAL; + } + + if (status & TMC_STS_FULL) { + len = tmc_get_resvbuf_trace(drvdata, 0x0, + CORESIGHT_BARRIER_PKT_SIZE, &bufp); + if (len >= CORESIGHT_BARRIER_PKT_SIZE) { + coresight_insert_barrier_packet(bufp); + /* Recalculate crc */ + mdata->crc32_tdata = find_crash_tracedata_crc(drvdata, + mdata); + mdata->crc32_mdata = find_crash_metadata_crc(mdata); + } + } + + return 0; +} + +static int tmc_read_prepare(struct tmc_drvdata *drvdata) +{ + int ret = 0; + + switch (drvdata->config_type) { + case TMC_CONFIG_TYPE_ETB: + case TMC_CONFIG_TYPE_ETF: + ret = tmc_read_prepare_etb(drvdata); + break; + case TMC_CONFIG_TYPE_ETR: + ret = tmc_read_prepare_etr(drvdata); + break; + default: + ret = -EINVAL; + } + + if (!ret) + dev_dbg(&drvdata->comp->dev, "TMC read start\n"); + + return ret; +} + +static int tmc_read_unprepare(struct tmc_drvdata *drvdata) +{ + int ret = 0; + + switch (drvdata->config_type) { + case TMC_CONFIG_TYPE_ETB: + case TMC_CONFIG_TYPE_ETF: + ret = tmc_read_unprepare_etb(drvdata); + break; + case TMC_CONFIG_TYPE_ETR: + ret = tmc_read_unprepare_etr(drvdata); + break; + default: + ret = -EINVAL; + } + + if (!ret) + dev_dbg(&drvdata->comp->dev, "TMC read end\n"); + + return ret; +} + +static int tmc_open(struct inode *inode, struct file *file) +{ + int ret; + struct tmc_drvdata *drvdata = container_of(file->private_data, + struct tmc_drvdata, miscdev); + + ret = tmc_read_prepare(drvdata); + if (ret) + return ret; + + nonseekable_open(inode, file); + + dev_dbg(&drvdata->comp->dev, "%s: successfully opened\n", __func__); + return 0; +} + +static ssize_t tmc_get_sysfs_trace(struct tmc_drvdata *drvdata, loff_t pos, size_t len, + char **bufpp) +{ + switch (drvdata->config_type) { + case TMC_CONFIG_TYPE_ETB: + case TMC_CONFIG_TYPE_ETF: + return tmc_etb_get_sysfs_trace(drvdata, pos, len, bufpp); + case TMC_CONFIG_TYPE_ETR: + return tmc_etr_get_sysfs_trace(drvdata, pos, len, bufpp); + } + + return -EINVAL; +} + +static ssize_t tmc_read(struct file *file, char __user *data, size_t len, + loff_t *ppos) +{ + char *bufp; + ssize_t actual; + struct tmc_drvdata *drvdata = container_of(file->private_data, + struct tmc_drvdata, miscdev); + actual = tmc_get_sysfs_trace(drvdata, *ppos, len, &bufp); + if (actual <= 0) + return 0; + + if (copy_to_user(data, bufp, actual)) { + dev_dbg(&drvdata->comp->dev, + "%s: copy_to_user failed\n", __func__); + return -EFAULT; + } + + *ppos += actual; + dev_dbg(&drvdata->comp->dev, "%zu bytes copied\n", actual); + + return actual; +} + +static int tmc_release(struct inode *inode, struct file *file) +{ + int ret; + struct tmc_drvdata *drvdata = container_of(file->private_data, + struct tmc_drvdata, miscdev); + + ret = tmc_read_unprepare(drvdata); + if (ret) + return ret; + + dev_dbg(&drvdata->comp->dev, "%s: released\n", __func__); + return 0; +} + +static const struct file_operations tmc_fops = { + .owner = THIS_MODULE, + .open = tmc_open, + .read = tmc_read, + .release = tmc_release, +}; + +static int tmc_crashdata_open(struct inode *inode, struct file *file) +{ + int err = 0; + unsigned long flags; + struct tmc_resrv_buf *rbuf; + struct tmc_crash_metadata *mdata; + struct tmc_drvdata *drvdata = container_of(file->private_data, + struct tmc_drvdata, + crashdev); + + mdata = drvdata->crash_mdata.vaddr; + rbuf = &drvdata->resrv_buf; + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + if (mdata->valid) + rbuf->reading = true; + else + err = -ENOENT; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + if (err) + goto exit; + + nonseekable_open(inode, file); + dev_dbg(&drvdata->comp->dev, "%s: successfully opened\n", __func__); +exit: + return err; +} + +static ssize_t tmc_crashdata_read(struct file *file, char __user *data, + size_t len, loff_t *ppos) +{ + char *bufp; + ssize_t actual; + struct tmc_drvdata *drvdata = container_of(file->private_data, + struct tmc_drvdata, + crashdev); + + actual = tmc_get_resvbuf_trace(drvdata, *ppos, len, &bufp); + if (actual <= 0) + return 0; + + if (copy_to_user(data, bufp, actual)) { + dev_dbg(&drvdata->comp->dev, + "%s: copy_to_user failed\n", __func__); + return -EFAULT; + } + + *ppos += actual; + dev_dbg(&drvdata->comp->dev, "%zu bytes copied\n", actual); + + return actual; +} + +static int tmc_crashdata_release(struct inode *inode, struct file *file) +{ + unsigned long flags; + struct tmc_resrv_buf *rbuf; + struct tmc_drvdata *drvdata = container_of(file->private_data, + struct tmc_drvdata, + crashdev); + + rbuf = &drvdata->resrv_buf; + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + rbuf->reading = false; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + dev_dbg(&drvdata->comp->dev, "%s: released\n", __func__); + return 0; +} + +static const struct file_operations tmc_crashdata_fops = { + .owner = THIS_MODULE, + .open = tmc_crashdata_open, + .read = tmc_crashdata_read, + .release = tmc_crashdata_release, +}; + +static enum tmc_mem_intf_width tmc_get_memwidth(u32 devid) +{ + enum tmc_mem_intf_width memwidth; + + /* + * Excerpt from the TRM: + * + * DEVID::MEMWIDTH[10:8] + * 0x2 Memory interface databus is 32 bits wide. + * 0x3 Memory interface databus is 64 bits wide. + * 0x4 Memory interface databus is 128 bits wide. + * 0x5 Memory interface databus is 256 bits wide. + */ + switch (BMVAL(devid, 8, 10)) { + case 0x2: + memwidth = TMC_MEM_INTF_WIDTH_32BITS; + break; + case 0x3: + memwidth = TMC_MEM_INTF_WIDTH_64BITS; + break; + case 0x4: + memwidth = TMC_MEM_INTF_WIDTH_128BITS; + break; + case 0x5: + memwidth = TMC_MEM_INTF_WIDTH_256BITS; + break; + default: + memwidth = 0; + } + + return memwidth; +} + +static struct attribute *coresight_tmc_mgmt_attrs[] = { + gtrace_simple_reg32(rsz, TMC_RSZ), + gtrace_simple_reg32(sts, TMC_STS), + gtrace_simple_reg64(rrp, TMC_RRP, TMC_RRPHI), + gtrace_simple_reg64(rwp, TMC_RWP, TMC_RWPHI), + gtrace_simple_reg32(trg, TMC_TRG), + gtrace_simple_reg32(ctl, TMC_CTL), + gtrace_simple_reg32(ffsr, TMC_FFSR), + gtrace_simple_reg32(ffcr, TMC_FFCR), + gtrace_simple_reg32(mode, TMC_MODE), + gtrace_simple_reg32(pscr, TMC_PSCR), + gtrace_simple_reg32(devid, CORESIGHT_DEVID), + gtrace_simple_reg64(dba, TMC_DBALO, TMC_DBAHI), + gtrace_simple_reg32(axictl, TMC_AXICTL), + gtrace_simple_reg32(authstatus, TMC_AUTHSTATUS), + NULL, +}; + +static ssize_t trigger_cntr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + unsigned long val = drvdata->trigger_cntr; + + return sprintf(buf, "%#lx\n", val); +} + +static ssize_t trigger_cntr_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtoul(buf, 16, &val); + if (ret) + return ret; + + drvdata->trigger_cntr = val; + return size; +} +static DEVICE_ATTR_RW(trigger_cntr); + +static ssize_t buffer_size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + + return sprintf(buf, "%#x\n", drvdata->size); +} + +static ssize_t buffer_size_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + unsigned long val; + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + + /* Only permitted for TMC-ETRs */ + if (drvdata->config_type != TMC_CONFIG_TYPE_ETR) + return -EPERM; + + ret = kstrtoul(buf, 0, &val); + if (ret) + return ret; + /* The buffer size should be page aligned */ + if (val & (PAGE_SIZE - 1)) + return -EINVAL; + drvdata->size = val; + return size; +} + +static DEVICE_ATTR_RW(buffer_size); + +static ssize_t stop_on_flush_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + + return sprintf(buf, "%#x\n", drvdata->stop_on_flush); +} + +static ssize_t stop_on_flush_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + int ret; + u8 val; + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + + ret = kstrtou8(buf, 0, &val); + if (ret) + return ret; + if (val) + drvdata->stop_on_flush = true; + else + drvdata->stop_on_flush = false; + + return size; +} + +static DEVICE_ATTR_RW(stop_on_flush); + + +static struct attribute *coresight_tmc_attrs[] = { + &dev_attr_trigger_cntr.attr, + &dev_attr_buffer_size.attr, + &dev_attr_stop_on_flush.attr, + NULL, +}; + +static const struct attribute_group coresight_tmc_group = { + .attrs = coresight_tmc_attrs, +}; + +static const struct attribute_group coresight_tmc_mgmt_group = { + .attrs = coresight_tmc_mgmt_attrs, + .name = "mgmt", +}; + +static const struct attribute_group *coresight_etf_groups[] = { + &coresight_tmc_group, + &coresight_tmc_mgmt_group, + NULL, +}; + +static const struct attribute_group *coresight_etr_groups[] = { + &coresight_etr_group, + &coresight_tmc_group, + &coresight_tmc_mgmt_group, + NULL, +}; + +static bool tmc_etr_can_use_sg(struct device *dev) +{ + int ret; + u8 val_u8; + + /* + * Presence of the property 'arm,scatter-gather' is checked + * on the platform for the feature support, rather than its + * value. + */ + if (is_of_node(dev->fwnode)) { + return fwnode_property_present(dev->fwnode, "arm,scatter-gather"); + } else if (is_acpi_device_node(dev->fwnode)) { + /* + * TMC_DEVID_NOSCAT test in tmc_etr_setup_caps(), has already ensured + * this property is only checked for Coresight SoC 400 TMC configured + * as ETR. + */ + ret = fwnode_property_read_u8(dev->fwnode, "arm-armhc97c-sg-enable", &val_u8); + if (!ret) + return !!val_u8; + + if (fwnode_property_present(dev->fwnode, "arm,scatter-gather")) { + pr_warn_once("Deprecated ACPI property - arm,scatter-gather\n"); + return true; + } + } + return false; +} + +static bool tmc_etr_has_non_secure_access(struct tmc_drvdata *drvdata) +{ + u32 auth = readl_relaxed(drvdata->base + TMC_AUTHSTATUS); + + return (auth & TMC_AUTH_NSID_MASK) == 0x3; +} + +static const struct amba_id tmc_ids[]; + +static int of_tmc_get_reserved_resource_by_name(struct device *dev, + const char *name, + struct resource *res) +{ + int rc = -ENODEV; + + rc = of_reserved_mem_region_to_resource_byname(dev->of_node, name, res); + if (rc < 0) + return rc; + + if (res->start == 0 || resource_size(res) == 0) + rc = -ENODEV; + + return rc; +} + +static void tmc_get_reserved_region(struct device *parent) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(parent); + struct resource res; + + if (of_tmc_get_reserved_resource_by_name(parent, "tracedata", &res)) + return; + + drvdata->resrv_buf.vaddr = memremap(res.start, + resource_size(&res), + MEMREMAP_WC); + if (IS_ERR_OR_NULL(drvdata->resrv_buf.vaddr)) { + dev_err(parent, "Reserved trace buffer mapping failed\n"); + return; + } + + drvdata->resrv_buf.paddr = res.start; + drvdata->resrv_buf.size = resource_size(&res); + + if (of_tmc_get_reserved_resource_by_name(parent, "metadata", &res)) + return; + + drvdata->crash_mdata.vaddr = memremap(res.start, + resource_size(&res), + MEMREMAP_WC); + if (IS_ERR_OR_NULL(drvdata->crash_mdata.vaddr)) { + dev_err(parent, "Metadata memory mapping failed\n"); + return; + } + + drvdata->crash_mdata.paddr = res.start; + drvdata->crash_mdata.size = resource_size(&res); +} + +/* Detect and initialise the capabilities of a TMC ETR */ +static int tmc_etr_setup_caps(struct device *parent, u32 devid, + struct gtrace_platform_data *pdata) +{ + int rc; + u32 tmc_pid, dma_mask = 0; + struct tmc_drvdata *drvdata = dev_get_drvdata(parent); + void *dev_caps; + + if (!tmc_etr_has_non_secure_access(drvdata)) + return -EACCES; + + tmc_pid = coresight_get_pid(pdata); + dev_caps = coresight_get_uci_data_from_amba(tmc_ids, tmc_pid); + + /* Set the unadvertised capabilities */ + tmc_etr_init_caps(drvdata, (u32)(unsigned long)dev_caps); + + if (!(devid & TMC_DEVID_NOSCAT) && tmc_etr_can_use_sg(parent)) + tmc_etr_set_cap(drvdata, TMC_ETR_SG); + + /* Check if the AXI address width is available */ + if (devid & TMC_DEVID_AXIAW_VALID) + dma_mask = ((devid >> TMC_DEVID_AXIAW_SHIFT) & + TMC_DEVID_AXIAW_MASK); + + /* + * Unless specified in the device configuration, ETR uses a 40-bit + * AXI master in place of the embedded SRAM of ETB/ETF. + */ + switch (dma_mask) { + case 32: + case 40: + case 44: + case 48: + case 52: + dev_info(parent, "Detected dma mask %dbits\n", dma_mask); + break; + default: + dma_mask = 40; + } + + rc = dma_set_mask_and_coherent(parent, DMA_BIT_MASK(dma_mask)); + if (rc) + dev_err(parent, "Failed to setup DMA mask: %d\n", rc); + return rc; +} + +static u32 tmc_etr_get_default_buffer_size(struct device *dev) +{ + u32 size; + + if (fwnode_property_read_u32(dev->fwnode, "arm,buffer-size", &size)) + size = SZ_1M; + return size; +} + +static u32 tmc_etr_get_max_burst_size(struct device *dev) +{ + u32 burst_size; + + if (fwnode_property_read_u32(dev->fwnode, "arm,max-burst-size", + &burst_size)) + return TMC_AXICTL_WR_BURST_16; + + /* Only permissible values are 0 to 15 */ + if (burst_size > 0xF) + burst_size = TMC_AXICTL_WR_BURST_16; + + return burst_size; +} + +static void register_crash_dev_interface(struct tmc_drvdata *drvdata, + const char *name) +{ + drvdata->crashdev.name = + devm_kasprintf(&drvdata->comp->dev, GFP_KERNEL, "%s_%s", "crash", name); + drvdata->crashdev.minor = MISC_DYNAMIC_MINOR; + drvdata->crashdev.fops = &tmc_crashdata_fops; + if (misc_register(&drvdata->crashdev)) { + dev_dbg(&drvdata->comp->dev, + "Failed to setup user interface for crashdata\n"); + drvdata->crashdev.fops = NULL; + } else + dev_info(&drvdata->comp->dev, + "Valid crash tracedata found\n"); +} + +static int __tmc_probe(struct device *dev, struct resource *res) +{ + int ret = 0; + u32 devid; + void __iomem *base; + struct gtrace_platform_data *pdata = NULL; + struct tmc_drvdata *drvdata; + struct gtrace_component_id id = { + .version = CORESIGHT_COMPONENT_ID_VERSION, + }; + const char *comp_name = NULL; + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + dev_set_drvdata(dev, drvdata); + + ret = coresight_get_enable_clocks(dev, &drvdata->pclk, &drvdata->atclk); + if (ret) + return ret; + + ret = -ENOMEM; + + /* Validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) { + ret = PTR_ERR(base); + goto out; + } + + drvdata->base = base; + + raw_spin_lock_init(&drvdata->spinlock); + + devid = readl_relaxed(drvdata->base + CORESIGHT_DEVID); + drvdata->config_type = BMVAL(devid, 6, 7); + drvdata->memwidth = tmc_get_memwidth(devid); + drvdata->etr_mode = ETR_MODE_AUTO; + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { + drvdata->size = tmc_etr_get_default_buffer_size(dev); + drvdata->max_burst_size = tmc_etr_get_max_burst_size(dev); + } else { + drvdata->size = readl_relaxed(drvdata->base + TMC_RSZ) * 4; + } + + tmc_get_reserved_region(dev); + + pdata = coresight_get_platform_data(dev); + if (IS_ERR(pdata)) { + ret = PTR_ERR(pdata); + goto out; + } + pdata->io_mem = true; + pdata->base = base; + + switch (drvdata->config_type) { + case TMC_CONFIG_TYPE_ETB: + pdata->groups = coresight_etf_groups; + id.type = GTRACE_CORESIGHT_TMC_ETB; + comp_name = "tmc_etb"; + break; + case TMC_CONFIG_TYPE_ETR: + pdata->groups = coresight_etr_groups; + id.type = GTRACE_CORESIGHT_TMC_ETR; + ret = tmc_etr_setup_caps(dev, devid, pdata); + if (ret) + goto out; + idr_init(&drvdata->idr); + mutex_init(&drvdata->idr_mutex); + comp_name = "tmc_etr"; + break; + case TMC_CONFIG_TYPE_ETF: + pdata->is_linksink = true; + pdata->groups = coresight_etf_groups; + id.type = GTRACE_CORESIGHT_TMC_ETF; + comp_name = "tmc_etf"; + break; + default: + pr_err("%u: Unsupported TMC config\n", drvdata->config_type); + ret = -EINVAL; + goto out; + } + + coresight_clear_self_claim_tag(pdata); + + drvdata->comp = gtrace_register_component(&id, comp_name, pdata); + if (IS_ERR(drvdata->comp)) { + ret = PTR_ERR(drvdata->comp); + goto out; + } + + drvdata->miscdev.name = dev_name(&drvdata->comp->dev); + drvdata->miscdev.minor = MISC_DYNAMIC_MINOR; + drvdata->miscdev.fops = &tmc_fops; + ret = misc_register(&drvdata->miscdev); + if (ret) { + gtrace_unregister_component(drvdata->comp); + goto out; + } + +out: + if (is_tmc_crashdata_valid(drvdata) && + !tmc_prepare_crashdata(drvdata)) + register_crash_dev_interface(drvdata, dev_name(&drvdata->comp->dev)); + return ret; +} + +static int tmc_probe(struct amba_device *adev, const struct amba_id *id) +{ + int ret; + + ret = __tmc_probe(&adev->dev, &adev->res); + if (!ret) + pm_runtime_put(&adev->dev); + + return ret; +} + +static void tmc_shutdown(struct amba_device *adev) +{ + unsigned long flags; + struct tmc_drvdata *drvdata = amba_get_drvdata(adev); + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + + if (gtrace_get_mode(drvdata->comp) == GTRACE_COMPONENT_MODE_DISABLED) + goto out; + + if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) + tmc_etr_disable_hw(drvdata); + + /* + * We do not care about coresight unregister here unlike remove + * callback which is required for making coresight modular since + * the system is going down after this. + */ +out: + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); +} + +static void __tmc_remove(struct device *dev) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev); + + /* + * Since misc_open() holds a refcount on the f_ops, which is + * etb fops in this case, device is there until last file + * handler to this device is closed. + */ + misc_deregister(&drvdata->miscdev); + if (drvdata->crashdev.fops) + misc_deregister(&drvdata->crashdev); + gtrace_unregister_component(drvdata->comp); +} + +static void tmc_remove(struct amba_device *adev) +{ + __tmc_remove(&adev->dev); +} + +static const struct amba_id tmc_ids[] = { + CS_AMBA_ID(0x000bb961), + /* Coresight SoC 600 TMC-ETR/ETS */ + CS_AMBA_ID_DATA(0x000bb9e8, (unsigned long)CORESIGHT_SOC_600_ETR_CAPS), + /* Coresight SoC 600 TMC-ETB */ + CS_AMBA_ID(0x000bb9e9), + /* Coresight SoC 600 TMC-ETF */ + CS_AMBA_ID(0x000bb9ea), + { 0, 0, NULL }, +}; + +MODULE_DEVICE_TABLE(amba, tmc_ids); + +static struct amba_driver tmc_driver = { + .drv = { + .name = "coresight-tmc", + .suppress_bind_attrs = true, + }, + .probe = tmc_probe, + .shutdown = tmc_shutdown, + .remove = tmc_remove, + .id_table = tmc_ids, +}; + +static int tmc_platform_probe(struct platform_device *pdev) +{ + struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + int ret = 0; + + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + ret = __tmc_probe(&pdev->dev, res); + pm_runtime_put(&pdev->dev); + if (ret) + pm_runtime_disable(&pdev->dev); + + return ret; +} + +static void tmc_platform_remove(struct platform_device *pdev) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(&pdev->dev); + + if (WARN_ON(!drvdata)) + return; + + __tmc_remove(&pdev->dev); + pm_runtime_disable(&pdev->dev); +} + +#ifdef CONFIG_PM +static int tmc_runtime_suspend(struct device *dev) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev); + + clk_disable_unprepare(drvdata->atclk); + clk_disable_unprepare(drvdata->pclk); + + return 0; +} + +static int tmc_runtime_resume(struct device *dev) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(drvdata->pclk); + if (ret) + return ret; + + ret = clk_prepare_enable(drvdata->atclk); + if (ret) + clk_disable_unprepare(drvdata->pclk); + + return ret; +} +#endif + +static const struct dev_pm_ops tmc_dev_pm_ops = { + SET_RUNTIME_PM_OPS(tmc_runtime_suspend, tmc_runtime_resume, NULL) +}; + +#ifdef CONFIG_ACPI +static const struct acpi_device_id tmc_acpi_ids[] = { + {"ARMHC501", 0, 0, 0}, /* ARM CoreSight ETR */ + {"ARMHC97C", 0, 0, 0}, /* ARM CoreSight SoC-400 TMC, SoC-600 ETF/ETB */ + {}, +}; +MODULE_DEVICE_TABLE(acpi, tmc_acpi_ids); +#endif + +static struct platform_driver tmc_platform_driver = { + .probe = tmc_platform_probe, + .remove = tmc_platform_remove, + .driver = { + .name = "coresight-tmc-platform", + .acpi_match_table = ACPI_PTR(tmc_acpi_ids), + .suppress_bind_attrs = true, + .pm = &tmc_dev_pm_ops, + }, +}; + +static const struct gtrace_component_id tmc_etr_ids[] = { + { .type = GTRACE_CORESIGHT_TMC_ETR, + .version = CORESIGHT_COMPONENT_ID_VERSION, }, + {}, +}; + +static struct gtrace_driver tmc_etr_driver = { + .id_table = tmc_etr_ids, + .setup_sysfs_buf = tmc_etr_setup_sysfs_buf, + .alloc_buf = tmc_alloc_etr_buffer, + .free_buf = tmc_free_etr_buffer, + .copyto_auxbuf = tmc_update_etr_buffer, + .start = tmc_enable_etr_sink, + .stop = tmc_disable_etr_sink, + .driver = { + .name = "coresight-tmc-etr" + }, +}; + +static const struct gtrace_component_id tmc_etf_ids[] = { + { .type = GTRACE_CORESIGHT_TMC_ETB, + .version = CORESIGHT_COMPONENT_ID_VERSION, }, + { .type = GTRACE_CORESIGHT_TMC_ETF, + .version = CORESIGHT_COMPONENT_ID_VERSION, }, + {}, +}; + +static struct gtrace_driver tmc_etf_driver = { + .id_table = tmc_etf_ids, + .setup_sysfs_buf = tmc_etf_setup_sysfs_buf, + .copyto_auxbuf = tmc_update_etf_buffer, + .start = tmc_enable_etf, + .stop = tmc_disable_etf, + .driver = { + .name = "coresight-tmc-etf" + }, +}; + +static int __init tmc_init(void) +{ + int ret; + + ret = gtrace_register_driver(&tmc_etr_driver); + if (ret) + return ret; + + ret = gtrace_register_driver(&tmc_etf_driver); + if (ret) + goto err_etr; + + ret = coresight_init_driver("tmc", &tmc_driver, &tmc_platform_driver); + if (ret) + goto err_etf; + + return 0; + +err_etf: + gtrace_unregister_driver(&tmc_etf_driver); +err_etr: + gtrace_unregister_driver(&tmc_etr_driver); + return ret; +} + +static void __exit tmc_exit(void) +{ + coresight_remove_driver(&tmc_driver, &tmc_platform_driver); + gtrace_unregister_driver(&tmc_etf_driver); + gtrace_unregister_driver(&tmc_etr_driver); +} +module_init(tmc_init); +module_exit(tmc_exit); + +MODULE_AUTHOR("Pratik Patel pratikp@codeaurora.org"); +MODULE_AUTHOR("Zane Leung liangzhen@linux.spacemit.com"); +MODULE_DESCRIPTION("Arm CoreSight Trace Memory Controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/hwtracing/gtrace/coresight-tmc-etf.c b/drivers/hwtracing/gtrace/coresight-tmc-etf.c new file mode 100644 index 000000000000..cb35a398bf28 --- /dev/null +++ b/drivers/hwtracing/gtrace/coresight-tmc-etf.c @@ -0,0 +1,573 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright(C) 2016 Linaro Limited. All rights reserved. + * Author: Mathieu Poirier mathieu.poirier@linaro.org + * Derived from drivers/hwtracing/coresight/coresight-tmc-etf.c + */ + +#include <linux/atomic.h> +#include <linux/circ_buf.h> +#include <linux/gtrace.h> +#include <linux/perf_event.h> +#include <linux/slab.h> +#include "coresight.h" +#include "coresight-tmc.h" + +static int __tmc_etb_enable_hw(struct tmc_drvdata *drvdata) +{ + int rc = 0; + u32 ffcr; + + CS_UNLOCK(drvdata->base); + + /* Wait for TMCSReady bit to be set */ + rc = tmc_wait_for_tmcready(drvdata); + if (rc) { + dev_err(&drvdata->comp->dev, + "Failed to enable: TMC not ready\n"); + CS_LOCK(drvdata->base); + return rc; + } + + writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE); + + ffcr = TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | TMC_FFCR_FON_FLIN | + TMC_FFCR_FON_TRIG_EVT | TMC_FFCR_TRIGON_TRIGIN; + if (drvdata->stop_on_flush) + ffcr |= TMC_FFCR_STOP_ON_FLUSH; + writel_relaxed(ffcr, drvdata->base + TMC_FFCR); + + writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG); + tmc_enable_hw(drvdata); + + CS_LOCK(drvdata->base); + return rc; +} + +static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) +{ + int rc = coresight_claim_device(drvdata->comp); + + if (rc) + return rc; + + rc = __tmc_etb_enable_hw(drvdata); + if (rc) + coresight_disclaim_device(drvdata->comp); + return rc; +} + +static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata) +{ + char *bufp; + u32 read_data, lost; + + /* Check if the buffer wrapped around. */ + lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL; + bufp = drvdata->buf; + drvdata->len = 0; + while (1) { + read_data = readl_relaxed(drvdata->base + TMC_RRD); + if (read_data == 0xFFFFFFFF) + break; + memcpy(bufp, &read_data, 4); + bufp += 4; + drvdata->len += 4; + } + + if (lost) + coresight_insert_barrier_packet(drvdata->buf); + return; +} + +static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) +{ + CS_UNLOCK(drvdata->base); + + tmc_flush_and_stop(drvdata); + /* + * When operating in sysFS mode the content of the buffer needs to be + * read before the TMC is disabled. + */ + if (gtrace_get_mode(drvdata->comp) == GTRACE_COMPONENT_MODE_SYSFS) + tmc_etb_dump_hw(drvdata); + tmc_disable_hw(drvdata); + + CS_LOCK(drvdata->base); +} + +static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) +{ + __tmc_etb_disable_hw(drvdata); + coresight_disclaim_device(drvdata->comp); +} + +static int __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +{ + int rc = 0; + + CS_UNLOCK(drvdata->base); + + /* Wait for TMCSReady bit to be set */ + rc = tmc_wait_for_tmcready(drvdata); + if (rc) { + dev_err(&drvdata->comp->dev, + "Failed to enable : TMC is not ready\n"); + CS_LOCK(drvdata->base); + return rc; + } + + writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE); + writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI, + drvdata->base + TMC_FFCR); + writel_relaxed(0x0, drvdata->base + TMC_BUFWM); + tmc_enable_hw(drvdata); + + CS_LOCK(drvdata->base); + return rc; +} + +static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) +{ + int rc = coresight_claim_device(drvdata->comp); + + if (rc) + return rc; + + rc = __tmc_etf_enable_hw(drvdata); + if (rc) + coresight_disclaim_device(drvdata->comp); + return rc; +} + +static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) +{ + struct gtrace_component *comp = drvdata->comp; + + CS_UNLOCK(drvdata->base); + + tmc_flush_and_stop(drvdata); + tmc_disable_hw(drvdata); + coresight_disclaim_device_unlocked(comp); + CS_LOCK(drvdata->base); +} + +/* + * Return the available trace data in the buffer from @pos, with + * a maximum limit of @len, updating the @bufpp on where to + * find it. + */ +ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata, + loff_t pos, size_t len, char **bufpp) +{ + ssize_t actual = len; + + /* Adjust the len to available size @pos */ + if (pos + actual > drvdata->len) + actual = drvdata->len - pos; + if (actual > 0) + *bufpp = drvdata->buf + pos; + return actual; +} + +int tmc_etf_setup_sysfs_buf(struct gtrace_component *comp) +{ + int ret = 0; + bool used = false; + char *buf = NULL; + unsigned long flags; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + + /* + * If we don't have a buffer release the lock and allocate memory. + * Otherwise keep the lock and move along. + */ + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + if (!drvdata->buf) { + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* Allocating the memory here while outside of the spinlock */ + buf = kzalloc(drvdata->size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + /* Let's try again */ + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + } + + if (drvdata->reading) { + ret = -EBUSY; + goto out; + } + + /* + * If drvdata::buf isn't NULL, memory was allocated for a previous + * trace run but wasn't read. If so simply zero-out the memory. + * Otherwise use the memory allocated above. + * + * The memory is freed when users read the buffer using the + * /dev/xyz.{etf|etb} interface. See tmc_read_unprepare_etf() for + * details. + */ + if (drvdata->buf) { + memset(drvdata->buf, 0, drvdata->size); + } else { + used = true; + drvdata->buf = buf; + } +out: + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* Free memory outside the spinlock if need be */ + if (!used) + kfree(buf); + + return ret; +} + +int tmc_enable_etf(struct gtrace_component *comp) +{ + int ret = 0; + struct gtrace_path *path = comp->path; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + + if (WARN_ON(!path)) + return -EINVAL; + + if (drvdata->reading) + return -EBUSY; + + drvdata->is_sink = gtrace_path_sink(path); + if (drvdata->is_sink) + ret = tmc_etb_enable_hw(drvdata); + else + ret = tmc_etf_enable_hw(drvdata); + + if (!ret) + dev_dbg(&comp->dev, "TMC-ETB/ETF enabled\n"); + + return ret; +} + +int tmc_disable_etf(struct gtrace_component *comp) +{ + struct gtrace_path *path = comp->path; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + + if (WARN_ON(!path)) + return -EINVAL; + + if (drvdata->reading) + return -EBUSY; + + if (drvdata->is_sink) + tmc_etb_disable_hw(drvdata); + else + tmc_etf_disable_hw(drvdata); + + dev_dbg(&comp->dev, "TMC-ETB/ETF disabled\n"); + return 0; +} + +unsigned long tmc_update_etf_buffer(struct gtrace_component *comp, + struct perf_output_handle *handle, + struct gtrace_perf_auxbuf *buf) +{ + bool lost = false; + int i, cur; + const u32 *barrier; + u32 *buf_ptr; + u64 read_ptr, write_ptr; + u32 status; + unsigned long offset, to_read = 0, flags; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + struct perf_event *event = handle->event; + + /* This shouldn't happen */ + if (WARN_ON_ONCE(gtrace_get_mode(comp) != GTRACE_COMPONENT_MODE_PERF)) + return 0; + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + + CS_UNLOCK(drvdata->base); + + tmc_flush_and_stop(drvdata); + + read_ptr = tmc_read_rrp(drvdata); + write_ptr = tmc_read_rwp(drvdata); + + /* + * Get a hold of the status register and see if a wrap around + * has occurred. If so adjust things accordingly. + */ + status = readl_relaxed(drvdata->base + TMC_STS); + if (status & TMC_STS_FULL) { + lost = true; + to_read = drvdata->size; + } else { + to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size); + } + + /* + * The TMC RAM buffer may be bigger than the space available in the + * perf ring buffer (handle->size). If so advance the RRP so that we + * get the latest trace data. In snapshot mode none of that matters + * since we are expected to clobber stale data in favour of the latest + * traces. + */ + if (!buf->snapshot && to_read > handle->size) { + u32 mask = tmc_get_memwidth_mask(drvdata); + + /* + * Make sure the new size is aligned in accordance with the + * requirement explained in function tmc_get_memwidth_mask(). + */ + to_read = handle->size & mask; + /* Move the RAM read pointer up */ + read_ptr = (write_ptr + drvdata->size) - to_read; + /* Make sure we are still within our limits */ + if (read_ptr > (drvdata->size - 1)) + read_ptr -= drvdata->size; + /* Tell the HW */ + tmc_write_rrp(drvdata, read_ptr); + lost = true; + } + + /* + * Don't set the TRUNCATED flag in snapshot mode because 1) the + * captured buffer is expected to be truncated and 2) a full buffer + * prevents the event from being re-enabled by the perf core, + * resulting in stale data being send to user space. + */ + if (!buf->snapshot && lost) + perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + + cur = buf->pos / PAGE_SIZE; + offset = buf->pos % PAGE_SIZE; + barrier = coresight_barrier_pkt; + + /* for every byte to read */ + for (i = 0; i < to_read; i += 4) { + buf_ptr = buf->pages[cur] + offset; + *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD); + + if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) { + *buf_ptr = *barrier; + barrier++; + } + + offset += 4; + if (offset >= PAGE_SIZE) { + offset = 0; + cur++; + /* wrap around at the end of the buffer */ + cur &= buf->nr_pages - 1; + } + } + + /* + * In snapshot mode we simply increment the head by the number of byte + * that were written. User space will figure out how many bytes to get + * from the AUX buffer based on the position of the head. + */ + if (buf->snapshot) + handle->head += to_read; + + /* + * CS_LOCK() contains mb() so it can ensure visibility of the AUX trace + * data before the aux_head is updated via perf_aux_output_end(), which + * is expected by the perf ring buffer. + */ + CS_LOCK(drvdata->base); + + /* + * If the event is active, it is triggered during an AUX pause. + * Re-enable the sink so that it is ready when AUX resume is invoked. + */ + if (!event->hw.state) + __tmc_etb_enable_hw(drvdata); + + /* set the GTRACE_FORMAT_CORESIGHT flag */ + perf_aux_output_flag(handle, PERF_AUX_FLAG_GTRACE_FORMAT_CORESIGHT); + + return to_read; +} + +int tmc_panic_sync_etf(struct gtrace_component *comp) +{ + u32 val; + struct tmc_crash_metadata *mdata; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + + mdata = (struct tmc_crash_metadata *)drvdata->crash_mdata.vaddr; + + /* Make sure we have valid reserved memory */ + if (!tmc_has_reserved_buffer(drvdata) || + !tmc_has_crash_mdata_buffer(drvdata)) + return 0; + + tmc_crashdata_set_invalid(drvdata); + + CS_UNLOCK(drvdata->base); + + /* Proceed only if ETF is enabled or configured as sink */ + val = readl(drvdata->base + TMC_CTL); + if (!(val & TMC_CTL_CAPT_EN)) + goto out; + val = readl(drvdata->base + TMC_MODE); + if (val != TMC_MODE_CIRCULAR_BUFFER) + goto out; + + val = readl(drvdata->base + TMC_FFSR); + /* Do manual flush and stop only if its not auto-stopped */ + if (!(val & TMC_FFSR_FT_STOPPED)) { + dev_dbg(&comp->dev, + "%s: Triggering manual flush\n", __func__); + tmc_flush_and_stop(drvdata); + } else + tmc_wait_for_tmcready(drvdata); + + /* Sync registers from hardware to metadata region */ + mdata->tmc_sts = readl(drvdata->base + TMC_STS); + mdata->tmc_mode = readl(drvdata->base + TMC_MODE); + mdata->tmc_ffcr = readl(drvdata->base + TMC_FFCR); + mdata->tmc_ffsr = readl(drvdata->base + TMC_FFSR); + + /* Sync Internal SRAM to reserved trace buffer region */ + drvdata->buf = drvdata->resrv_buf.vaddr; + tmc_etb_dump_hw(drvdata); + /* Store as per RSZ register convention */ + mdata->tmc_ram_size = drvdata->len >> 2; + + /* Other fields for processing trace buffer reads */ + mdata->tmc_rrp = 0; + mdata->tmc_dba = 0; + mdata->tmc_rwp = drvdata->len; + mdata->trace_paddr = drvdata->resrv_buf.paddr; + + mdata->version = CS_CRASHDATA_VERSION; + + /* + * Make sure all previous writes are ordered, + * before we mark valid + */ + smp_wmb(); + mdata->valid = true; + /* + * Below order need to maintained, since crc of metadata + * is dependent on first + */ + mdata->crc32_tdata = find_crash_tracedata_crc(drvdata, mdata); + mdata->crc32_mdata = find_crash_metadata_crc(mdata); + + tmc_disable_hw(drvdata); + + dev_dbg(&comp->dev, "%s: success\n", __func__); +out: + CS_UNLOCK(drvdata->base); + return 0; +} + +int tmc_read_prepare_etb(struct tmc_drvdata *drvdata) +{ + enum tmc_mode mode; + int ret = 0; + unsigned long flags; + + /* config types are set a boot time and never change */ + if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB && + drvdata->config_type != TMC_CONFIG_TYPE_ETF)) + return -EINVAL; + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + + if (drvdata->reading) { + ret = -EBUSY; + goto out; + } + + /* Don't interfere if operated from Perf */ + if (gtrace_get_mode(drvdata->comp) == GTRACE_COMPONENT_MODE_PERF) { + ret = -EINVAL; + goto out; + } + + /* If drvdata::buf is NULL the trace data has been read already */ + if (drvdata->buf == NULL) { + ret = -EINVAL; + goto out; + } + + /* Disable the TMC if need be */ + if (gtrace_get_mode(drvdata->comp) == GTRACE_COMPONENT_MODE_SYSFS) { + /* There is no point in reading a TMC in HW FIFO mode */ + mode = readl_relaxed(drvdata->base + TMC_MODE); + if (mode != TMC_MODE_CIRCULAR_BUFFER) { + ret = -EINVAL; + goto out; + } + __tmc_etb_disable_hw(drvdata); + } + + drvdata->reading = true; +out: + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + return ret; +} + +int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata) +{ + char *buf = NULL; + enum tmc_mode mode; + unsigned long flags; + + /* config types are set a boot time and never change */ + if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB && + drvdata->config_type != TMC_CONFIG_TYPE_ETF)) + return -EINVAL; + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + + /* Re-enable the TMC if need be */ + if (gtrace_get_mode(drvdata->comp) == GTRACE_COMPONENT_MODE_SYSFS) { + /* There is no point in reading a TMC in HW FIFO mode */ + mode = readl_relaxed(drvdata->base + TMC_MODE); + if (mode != TMC_MODE_CIRCULAR_BUFFER) { + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return -EINVAL; + } + /* + * The trace run will continue with the same allocated trace + * buffer. As such zero-out the buffer so that we don't end + * up with stale data. + * + * Since the tracer is still enabled drvdata::buf + * can't be NULL. + */ + memset(drvdata->buf, 0, drvdata->size); + /* + * Ignore failures to enable the TMC to make sure, we don't + * leave the TMC in a "reading" state. + */ + __tmc_etb_enable_hw(drvdata); + } else { + /* + * The ETB/ETF is not tracing and the buffer was just read. + * As such prepare to free the trace buffer. + */ + buf = drvdata->buf; + drvdata->buf = NULL; + } + + drvdata->reading = false; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* + * Free allocated memory outside of the spinlock. There is no need + * to assert the validity of 'buf' since calling kfree(NULL) is safe. + */ + kfree(buf); + + return 0; +} diff --git a/drivers/hwtracing/gtrace/coresight-tmc-etr.c b/drivers/hwtracing/gtrace/coresight-tmc-etr.c new file mode 100644 index 000000000000..11b7df8ae1b2 --- /dev/null +++ b/drivers/hwtracing/gtrace/coresight-tmc-etr.c @@ -0,0 +1,1888 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright(C) 2016 Linaro Limited. All rights reserved. + * Author: Mathieu Poirier mathieu.poirier@linaro.org + * Derived from drivers/hwtracing/coresight/coresight-tmc-etr.c + */ + +#include <linux/atomic.h> +#include <linux/gtrace.h> +#include <linux/dma-mapping.h> +#include <linux/iommu.h> +#include <linux/idr.h> +#include <linux/mutex.h> +#include <linux/refcount.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <linux/vmalloc.h> +#include "coresight.h" +#include "coresight-tmc.h" + +struct etr_flat_buf { + struct device *dev; + dma_addr_t daddr; + void *vaddr; + size_t size; +}; + +struct etr_buf_hw { + bool has_iommu; + bool has_etr_sg; + bool has_catu; + bool has_resrv; +}; + +/* Convert the perf index to an offset within the ETR buffer */ +#define PERF_IDX2OFF(idx, buf) \ + ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT)) + +/* Lower limit for ETR hardware buffer */ +#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M + +/* + * The TMC ETR SG has a page size of 4K. The SG table contains pointers + * to 4KB buffers. However, the OS may use a PAGE_SIZE different from + * 4K (i.e, 16KB or 64KB). This implies that a single OS page could + * contain more than one SG buffer and tables. + * + * A table entry has the following format: + * + * ---Bit31------------Bit4-------Bit1-----Bit0-- + * | Address[39:12] | SBZ | Entry Type | + * ---------------------------------------------- + * + * Address: Bits [39:12] of a physical page address. Bits [11:0] are + * always zero. + * + * Entry type: + * b00 - Reserved. + * b01 - Last entry in the tables, points to 4K page buffer. + * b10 - Normal entry, points to 4K page buffer. + * b11 - Link. The address points to the base of next table. + */ + +typedef u32 sgte_t; + +#define ETR_SG_PAGE_SHIFT 12 +#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT) +#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE) +#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t)) +#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t)) + +#define ETR_SG_ET_MASK 0x3 +#define ETR_SG_ET_LAST 0x1 +#define ETR_SG_ET_NORMAL 0x2 +#define ETR_SG_ET_LINK 0x3 + +#define ETR_SG_ADDR_SHIFT 4 + +#define ETR_SG_ENTRY(addr, type) \ + ((sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \ + ((type) & ETR_SG_ET_MASK))) + +#define ETR_SG_ADDR(entry) \ + (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT) +#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK) + +/* + * struct etr_sg_table : ETR SG Table + * @sg_table: Generic SG Table holding the data/table pages. + * @hwaddr: hwaddress used by the TMC, which is the base + * address of the table. + */ +struct etr_sg_table { + struct tmc_sg_table *sg_table; + dma_addr_t hwaddr; +}; + +/* + * tmc_etr_sg_table_entries: Total number of table entries required to map + * @nr_pages system pages. + * + * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages. + * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers, + * with the last entry pointing to another page of table entries. + * If we spill over to a new page for mapping 1 entry, we could as + * well replace the link entry of the previous page with the last entry. + */ +static unsigned long __attribute_const__ +tmc_etr_sg_table_entries(int nr_pages) +{ + unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE; + unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1); + /* + * If we spill over to a new page for 1 entry, we could as well + * make it the LAST entry in the previous page, skipping the Link + * address. + */ + if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2)) + nr_sglinks--; + return nr_sgpages + nr_sglinks; +} + +/* + * tmc_pages_get_offset: Go through all the pages in the tmc_pages + * and map the device address @addr to an offset within the virtual + * contiguous buffer. + */ +static long +tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr) +{ + int i; + dma_addr_t page_start; + + for (i = 0; i < tmc_pages->nr_pages; i++) { + page_start = tmc_pages->daddrs[i]; + if (addr >= page_start && addr < (page_start + PAGE_SIZE)) + return (long)i * PAGE_SIZE + (addr - page_start); + } + + return -EINVAL; +} + +/* + * tmc_pages_free : Unmap and free the pages used by tmc_pages. + * If the pages were not allocated in tmc_pages_alloc(), we would + * simply drop the refcount. + */ +static void tmc_pages_free(struct tmc_pages *tmc_pages, + struct device *dev, enum dma_data_direction dir) +{ + int i; + struct device *real_dev = dev->parent; + + for (i = 0; i < tmc_pages->nr_pages; i++) { + if (tmc_pages->daddrs && tmc_pages->daddrs[i]) + dma_unmap_page(real_dev, tmc_pages->daddrs[i], + PAGE_SIZE, dir); + if (tmc_pages->pages && tmc_pages->pages[i]) + __free_page(tmc_pages->pages[i]); + } + + kfree(tmc_pages->pages); + kfree(tmc_pages->daddrs); + tmc_pages->pages = NULL; + tmc_pages->daddrs = NULL; + tmc_pages->nr_pages = 0; +} + +/* + * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages. + * If @pages is not NULL, the list of page virtual addresses are + * used as the data pages. The pages are then dma_map'ed for @dev + * with dma_direction @dir. + * + * Returns 0 upon success, else the error number. + */ +static int tmc_pages_alloc(struct tmc_pages *tmc_pages, + struct device *dev, int node, + enum dma_data_direction dir, void **pages) +{ + int i, nr_pages; + dma_addr_t paddr; + struct page *page; + struct device *real_dev = dev->parent; + + nr_pages = tmc_pages->nr_pages; + tmc_pages->daddrs = kzalloc_objs(*tmc_pages->daddrs, nr_pages); + if (!tmc_pages->daddrs) + return -ENOMEM; + tmc_pages->pages = kzalloc_objs(*tmc_pages->pages, nr_pages); + if (!tmc_pages->pages) { + kfree(tmc_pages->daddrs); + tmc_pages->daddrs = NULL; + return -ENOMEM; + } + + for (i = 0; i < nr_pages; i++) { + if (pages && pages[i]) { + page = virt_to_page(pages[i]); + /* Hold a refcount on the page */ + get_page(page); + } else { + page = alloc_pages_node(node, + GFP_KERNEL | __GFP_ZERO, 0); + if (!page) + goto err; + } + paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir); + if (dma_mapping_error(real_dev, paddr)) + goto err; + tmc_pages->daddrs[i] = paddr; + tmc_pages->pages[i] = page; + } + return 0; +err: + tmc_pages_free(tmc_pages, dev, dir); + return -ENOMEM; +} + +static long +tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr) +{ + return tmc_pages_get_offset(&sg_table->data_pages, addr); +} + +static void tmc_free_table_pages(struct tmc_sg_table *sg_table) +{ + if (sg_table->table_vaddr) + vunmap(sg_table->table_vaddr); + tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE); +} + +static void tmc_free_data_pages(struct tmc_sg_table *sg_table) +{ + if (sg_table->data_vaddr) + vunmap(sg_table->data_vaddr); + tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE); +} + +void tmc_free_sg_table(struct tmc_sg_table *sg_table) +{ + tmc_free_table_pages(sg_table); + tmc_free_data_pages(sg_table); + kfree(sg_table); +} +EXPORT_SYMBOL_GPL(tmc_free_sg_table); + +/* + * Alloc pages for the table. Since this will be used by the device, + * allocate the pages closer to the device (i.e, dev_to_node(dev) + * rather than the CPU node). + */ +static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table) +{ + int rc; + struct tmc_pages *table_pages = &sg_table->table_pages; + + rc = tmc_pages_alloc(table_pages, sg_table->dev, + dev_to_node(sg_table->dev), + DMA_TO_DEVICE, NULL); + if (rc) + return rc; + sg_table->table_vaddr = vmap(table_pages->pages, + table_pages->nr_pages, + VM_MAP, + PAGE_KERNEL); + if (!sg_table->table_vaddr) + rc = -ENOMEM; + else + sg_table->table_daddr = table_pages->daddrs[0]; + return rc; +} + +static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages) +{ + int rc; + + /* Allocate data pages on the node requested by the caller */ + rc = tmc_pages_alloc(&sg_table->data_pages, + sg_table->dev, sg_table->node, + DMA_FROM_DEVICE, pages); + if (!rc) { + sg_table->data_vaddr = vmap(sg_table->data_pages.pages, + sg_table->data_pages.nr_pages, + VM_MAP, + PAGE_KERNEL); + if (!sg_table->data_vaddr) + rc = -ENOMEM; + } + return rc; +} + +/* + * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table + * and data buffers. TMC writes to the data buffers and reads from the SG + * Table pages. + * + * @dev - Coresight device to which page should be DMA mapped. + * @node - Numa node for mem allocations + * @nr_tpages - Number of pages for the table entries. + * @nr_dpages - Number of pages for Data buffer. + * @pages - Optional list of virtual address of pages. + */ +struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev, + int node, + int nr_tpages, + int nr_dpages, + void **pages) +{ + long rc; + struct tmc_sg_table *sg_table; + + sg_table = kzalloc_obj(*sg_table); + if (!sg_table) + return ERR_PTR(-ENOMEM); + sg_table->data_pages.nr_pages = nr_dpages; + sg_table->table_pages.nr_pages = nr_tpages; + sg_table->node = node; + sg_table->dev = dev; + + rc = tmc_alloc_data_pages(sg_table, pages); + if (!rc) + rc = tmc_alloc_table_pages(sg_table); + if (rc) { + tmc_free_sg_table(sg_table); + return ERR_PTR(rc); + } + + return sg_table; +} +EXPORT_SYMBOL_GPL(tmc_alloc_sg_table); + +/* + * tmc_sg_table_sync_data_range: Sync the data buffer written + * by the device from @offset up to a @size bytes. + */ +void tmc_sg_table_sync_data_range(struct tmc_sg_table *table, + u64 offset, u64 size) +{ + int i, index, start; + int npages = DIV_ROUND_UP(size, PAGE_SIZE); + struct device *real_dev = table->dev->parent; + struct tmc_pages *data = &table->data_pages; + + start = offset >> PAGE_SHIFT; + for (i = start; i < (start + npages); i++) { + index = i % data->nr_pages; + dma_sync_single_for_cpu(real_dev, data->daddrs[index], + PAGE_SIZE, DMA_FROM_DEVICE); + } +} +EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range); + +/* tmc_sg_sync_table: Sync the page table */ +void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table) +{ + int i; + struct device *real_dev = sg_table->dev->parent; + struct tmc_pages *table_pages = &sg_table->table_pages; + + for (i = 0; i < table_pages->nr_pages; i++) + dma_sync_single_for_device(real_dev, table_pages->daddrs[i], + PAGE_SIZE, DMA_TO_DEVICE); +} +EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table); + +/* + * tmc_sg_table_get_data: Get the buffer pointer for data @offset + * in the SG buffer. The @bufpp is updated to point to the buffer. + * Returns : + * the length of linear data available at @offset. + * or + * <= 0 if no data is available. + */ +ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table, + u64 offset, size_t len, char **bufpp) +{ + size_t size; + int pg_idx = offset >> PAGE_SHIFT; + int pg_offset = offset & (PAGE_SIZE - 1); + struct tmc_pages *data_pages = &sg_table->data_pages; + + size = tmc_sg_table_buf_size(sg_table); + if (offset >= size) + return -EINVAL; + + /* Make sure we don't go beyond the end */ + len = (len < (size - offset)) ? len : size - offset; + /* Respect the page boundaries */ + len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset); + if (len > 0) + *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset; + return len; +} +EXPORT_SYMBOL_GPL(tmc_sg_table_get_data); + +#ifdef ETR_SG_DEBUG +/* Map a dma address to virtual address */ +static unsigned long +tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table, + dma_addr_t addr, bool table) +{ + long offset; + unsigned long base; + struct tmc_pages *tmc_pages; + + if (table) { + tmc_pages = &sg_table->table_pages; + base = (unsigned long)sg_table->table_vaddr; + } else { + tmc_pages = &sg_table->data_pages; + base = (unsigned long)sg_table->data_vaddr; + } + + offset = tmc_pages_get_offset(tmc_pages, addr); + if (offset < 0) + return 0; + return base + offset; +} + +/* Dump the given sg_table */ +static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) +{ + sgte_t *ptr; + int i = 0; + dma_addr_t addr; + struct tmc_sg_table *sg_table = etr_table->sg_table; + + ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table, + etr_table->hwaddr, true); + while (ptr) { + addr = ETR_SG_ADDR(*ptr); + switch (ETR_SG_ET(*ptr)) { + case ETR_SG_ET_NORMAL: + dev_dbg(sg_table->dev, + "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr); + ptr++; + break; + case ETR_SG_ET_LINK: + dev_dbg(sg_table->dev, + "%05d: *** %p\t:{L} 0x%llx ***\n", + i, ptr, addr); + ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table, + addr, true); + break; + case ETR_SG_ET_LAST: + dev_dbg(sg_table->dev, + "%05d: ### %p\t:[L] 0x%llx ###\n", + i, ptr, addr); + return; + default: + dev_dbg(sg_table->dev, + "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n", + i, ptr, addr); + return; + } + i++; + } + dev_dbg(sg_table->dev, "******* End of Table *****\n"); +} +#else +static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {} +#endif + +/* + * Populate the SG Table page table entries from table/data + * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages. + * So does a Table page. So we keep track of indices of the tables + * in each system page and move the pointers accordingly. + */ +#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size)) +static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table) +{ + dma_addr_t paddr; + int i, type, nr_entries; + int tpidx = 0; /* index to the current system table_page */ + int sgtidx = 0; /* index to the sg_table within the current syspage */ + int sgtentry = 0; /* the entry within the sg_table */ + int dpidx = 0; /* index to the current system data_page */ + int spidx = 0; /* index to the SG page within the current data page */ + sgte_t *ptr; /* pointer to the table entry to fill */ + struct tmc_sg_table *sg_table = etr_table->sg_table; + dma_addr_t *table_daddrs = sg_table->table_pages.daddrs; + dma_addr_t *data_daddrs = sg_table->data_pages.daddrs; + + nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages); + /* + * Use the contiguous virtual address of the table to update entries. + */ + ptr = sg_table->table_vaddr; + /* + * Fill all the entries, except the last entry to avoid special + * checks within the loop. + */ + for (i = 0; i < nr_entries - 1; i++) { + if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) { + /* + * Last entry in a sg_table page is a link address to + * the next table page. If this sg_table is the last + * one in the system page, it links to the first + * sg_table in the next system page. Otherwise, it + * links to the next sg_table page within the system + * page. + */ + if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) { + paddr = table_daddrs[tpidx + 1]; + } else { + paddr = table_daddrs[tpidx] + + (ETR_SG_PAGE_SIZE * (sgtidx + 1)); + } + type = ETR_SG_ET_LINK; + } else { + /* + * Update the indices to the data_pages to point to the + * next sg_page in the data buffer. + */ + type = ETR_SG_ET_NORMAL; + paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE; + if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE)) + dpidx++; + } + *ptr++ = ETR_SG_ENTRY(paddr, type); + /* + * Move to the next table pointer, moving the table page index + * if necessary + */ + if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) { + if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE)) + tpidx++; + } + } + + /* Set up the last entry, which is always a data pointer */ + paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE; + *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST); +} + +/* + * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and + * populate the table. + * + * @dev - Device pointer for the TMC + * @node - NUMA node where the memory should be allocated + * @size - Total size of the data buffer + * @pages - Optional list of page virtual address + */ +static struct etr_sg_table * +tmc_init_etr_sg_table(struct device *dev, int node, + unsigned long size, void **pages) +{ + int nr_entries, nr_tpages; + int nr_dpages = size >> PAGE_SHIFT; + struct tmc_sg_table *sg_table; + struct etr_sg_table *etr_table; + + etr_table = kzalloc_obj(*etr_table); + if (!etr_table) + return ERR_PTR(-ENOMEM); + nr_entries = tmc_etr_sg_table_entries(nr_dpages); + nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE); + + sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages); + if (IS_ERR(sg_table)) { + kfree(etr_table); + return ERR_CAST(sg_table); + } + + etr_table->sg_table = sg_table; + /* TMC should use table base address for DBA */ + etr_table->hwaddr = sg_table->table_daddr; + tmc_etr_sg_table_populate(etr_table); + /* Sync the table pages for the HW */ + tmc_sg_table_sync_table(sg_table); + tmc_etr_sg_table_dump(etr_table); + + return etr_table; +} + +/* + * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer. + */ +static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata, + struct etr_buf *etr_buf, int node, + void **pages) +{ + struct etr_flat_buf *flat_buf; + struct device *real_dev = drvdata->comp->dev.parent; + + /* We cannot reuse existing pages for flat buf */ + if (pages) + return -EINVAL; + + flat_buf = kzalloc_obj(*flat_buf); + if (!flat_buf) + return -ENOMEM; + + flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size, + &flat_buf->daddr, + DMA_FROM_DEVICE, + GFP_KERNEL | __GFP_NOWARN); + if (!flat_buf->vaddr) { + kfree(flat_buf); + return -ENOMEM; + } + + flat_buf->size = etr_buf->size; + flat_buf->dev = &drvdata->comp->dev; + etr_buf->hwaddr = flat_buf->daddr; + etr_buf->mode = ETR_MODE_FLAT; + etr_buf->private = flat_buf; + return 0; +} + +static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf) +{ + struct etr_flat_buf *flat_buf = etr_buf->private; + + if (flat_buf && flat_buf->daddr) { + struct device *real_dev = flat_buf->dev->parent; + + dma_free_noncoherent(real_dev, etr_buf->size, + flat_buf->vaddr, flat_buf->daddr, + DMA_FROM_DEVICE); + } + kfree(flat_buf); +} + +static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp) +{ + struct etr_flat_buf *flat_buf = etr_buf->private; + struct device *real_dev = flat_buf->dev->parent; + + /* + * Adjust the buffer to point to the beginning of the trace data + * and update the available trace data. + */ + etr_buf->offset = rrp - etr_buf->hwaddr; + if (etr_buf->full) + etr_buf->len = etr_buf->size; + else + etr_buf->len = rwp - rrp; + + /* + * The driver always starts tracing at the beginning of the buffer, + * the only reason why we would get a wrap around is when the buffer + * is full. Sync the entire buffer in one go for this case. + */ + if (etr_buf->offset + etr_buf->len > etr_buf->size) + dma_sync_single_for_cpu(real_dev, flat_buf->daddr, + etr_buf->size, DMA_FROM_DEVICE); + else + dma_sync_single_for_cpu(real_dev, + flat_buf->daddr + etr_buf->offset, + etr_buf->len, DMA_FROM_DEVICE); +} + +static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf, + u64 offset, size_t len, char **bufpp) +{ + struct etr_flat_buf *flat_buf = etr_buf->private; + + *bufpp = (char *)flat_buf->vaddr + offset; + /* + * tmc_etr_buf_get_data already adjusts the length to handle + * buffer wrapping around. + */ + return len; +} + +static const struct etr_buf_operations etr_flat_buf_ops = { + .alloc = tmc_etr_alloc_flat_buf, + .free = tmc_etr_free_flat_buf, + .sync = tmc_etr_sync_flat_buf, + .get_data = tmc_etr_get_data_flat_buf, +}; + +/* + * tmc_etr_alloc_resrv_buf: Allocate a contiguous DMA buffer from reserved region. + */ +static int tmc_etr_alloc_resrv_buf(struct tmc_drvdata *drvdata, + struct etr_buf *etr_buf, int node, + void **pages) +{ + struct etr_flat_buf *resrv_buf; + struct device *real_dev = drvdata->comp->dev.parent; + + /* We cannot reuse existing pages for resrv buf */ + if (pages) + return -EINVAL; + + resrv_buf = kzalloc_obj(*resrv_buf); + if (!resrv_buf) + return -ENOMEM; + + resrv_buf->daddr = dma_map_resource(real_dev, drvdata->resrv_buf.paddr, + drvdata->resrv_buf.size, + DMA_FROM_DEVICE, 0); + if (dma_mapping_error(real_dev, resrv_buf->daddr)) { + dev_err(real_dev, "failed to map source buffer address\n"); + kfree(resrv_buf); + return -ENOMEM; + } + + resrv_buf->vaddr = drvdata->resrv_buf.vaddr; + resrv_buf->size = etr_buf->size = drvdata->resrv_buf.size; + resrv_buf->dev = &drvdata->comp->dev; + etr_buf->hwaddr = resrv_buf->daddr; + etr_buf->mode = ETR_MODE_RESRV; + etr_buf->private = resrv_buf; + return 0; +} + +static void tmc_etr_free_resrv_buf(struct etr_buf *etr_buf) +{ + struct etr_flat_buf *resrv_buf = etr_buf->private; + + if (resrv_buf && resrv_buf->daddr) { + struct device *real_dev = resrv_buf->dev->parent; + + dma_unmap_resource(real_dev, resrv_buf->daddr, + resrv_buf->size, DMA_FROM_DEVICE, 0); + } + kfree(resrv_buf); +} + +static void tmc_etr_sync_resrv_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp) +{ + /* + * Adjust the buffer to point to the beginning of the trace data + * and update the available trace data. + */ + etr_buf->offset = rrp - etr_buf->hwaddr; + if (etr_buf->full) + etr_buf->len = etr_buf->size; + else + etr_buf->len = rwp - rrp; +} + +static const struct etr_buf_operations etr_resrv_buf_ops = { + .alloc = tmc_etr_alloc_resrv_buf, + .free = tmc_etr_free_resrv_buf, + .sync = tmc_etr_sync_resrv_buf, + .get_data = tmc_etr_get_data_flat_buf, +}; + +/* + * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters + * appropriately. + */ +static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata, + struct etr_buf *etr_buf, int node, + void **pages) +{ + struct etr_sg_table *etr_table; + struct device *dev = &drvdata->comp->dev; + + etr_table = tmc_init_etr_sg_table(dev, node, + etr_buf->size, pages); + if (IS_ERR(etr_table)) + return -ENOMEM; + etr_buf->hwaddr = etr_table->hwaddr; + etr_buf->mode = ETR_MODE_ETR_SG; + etr_buf->private = etr_table; + return 0; +} + +static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf) +{ + struct etr_sg_table *etr_table = etr_buf->private; + + if (etr_table) { + tmc_free_sg_table(etr_table->sg_table); + kfree(etr_table); + } +} + +static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset, + size_t len, char **bufpp) +{ + struct etr_sg_table *etr_table = etr_buf->private; + + return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp); +} + +static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp) +{ + long r_offset, w_offset; + struct etr_sg_table *etr_table = etr_buf->private; + struct tmc_sg_table *table = etr_table->sg_table; + + /* Convert hw address to offset in the buffer */ + r_offset = tmc_sg_get_data_page_offset(table, rrp); + if (r_offset < 0) { + dev_warn(table->dev, + "Unable to map RRP %llx to offset\n", rrp); + etr_buf->len = 0; + return; + } + + w_offset = tmc_sg_get_data_page_offset(table, rwp); + if (w_offset < 0) { + dev_warn(table->dev, + "Unable to map RWP %llx to offset\n", rwp); + etr_buf->len = 0; + return; + } + + etr_buf->offset = r_offset; + if (etr_buf->full) + etr_buf->len = etr_buf->size; + else + etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) + + w_offset - r_offset; + tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len); +} + +static const struct etr_buf_operations etr_sg_buf_ops = { + .alloc = tmc_etr_alloc_sg_buf, + .free = tmc_etr_free_sg_buf, + .sync = tmc_etr_sync_sg_buf, + .get_data = tmc_etr_get_data_sg_buf, +}; + +/* + * TMC ETR could be connected to a CATU device, which can provide address + * translation service. This is represented by the Output port of the TMC + * (ETR) connected to the input port of the CATU. + * + * Returns : gtrace_component ptr for the CATU device if a CATU is found. + * : NULL otherwise. + */ +struct gtrace_component * +tmc_etr_get_catu_device(struct tmc_drvdata *drvdata) +{ + return NULL; +} +EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device); + +static const struct etr_buf_operations *etr_buf_ops[] = { + [ETR_MODE_FLAT] = &etr_flat_buf_ops, + [ETR_MODE_ETR_SG] = &etr_sg_buf_ops, + [ETR_MODE_CATU] = NULL, + [ETR_MODE_RESRV] = &etr_resrv_buf_ops +}; + +void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu) +{ + etr_buf_ops[ETR_MODE_CATU] = catu; +} +EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops); + +void tmc_etr_remove_catu_ops(void) +{ + etr_buf_ops[ETR_MODE_CATU] = NULL; +} +EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops); + +static int tmc_etr_mode_alloc_buf(int mode, struct tmc_drvdata *drvdata, struct etr_buf *etr_buf, + int node, void **pages) +{ + int rc = -EINVAL; + + switch (mode) { + case ETR_MODE_FLAT: + case ETR_MODE_ETR_SG: + case ETR_MODE_CATU: + case ETR_MODE_RESRV: + if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc) + rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf, + node, pages); + if (!rc) + etr_buf->ops = etr_buf_ops[mode]; + return rc; + default: + return -EINVAL; + } +} + +static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + + buf_hw->has_iommu = iommu_get_domain_for_dev(dev->parent); + buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG); + buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata); + buf_hw->has_resrv = tmc_has_reserved_buffer(drvdata); +} + +static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size) +{ + bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg; + + return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M; +} + +/* + * tmc_alloc_etr_buf: Allocate a buffer use by ETR. + * @drvdata : ETR device details. + * @size : size of the requested buffer. + * @flags : Required properties for the buffer. + * @node : Node for memory allocations. + * @pages : An optional list of pages. + */ +static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata, + ssize_t size, int flags, + int node, void **pages) +{ + int rc = -ENOMEM; + struct etr_buf *etr_buf; + struct etr_buf_hw buf_hw; + struct device *dev = &drvdata->comp->dev; + + get_etr_buf_hw(dev, &buf_hw); + etr_buf = kzalloc_obj(*etr_buf); + if (!etr_buf) + return ERR_PTR(-ENOMEM); + + etr_buf->size = size; + + /* If there is user directive for buffer mode, try that first */ + if (drvdata->etr_mode != ETR_MODE_AUTO) + rc = tmc_etr_mode_alloc_buf(drvdata->etr_mode, drvdata, + etr_buf, node, pages); + + /* + * If we have to use an existing list of pages, we cannot reliably + * use a contiguous DMA memory (even if we have an IOMMU). Otherwise, + * we use the contiguous DMA memory if at least one of the following + * conditions is true: + * a) The ETR cannot use Scatter-Gather. + * b) we have a backing IOMMU + * c) The requested memory size is smaller (< 1M). + * + * Fallback to available mechanisms. + * + */ + if (rc && !pages && etr_can_use_flat_mode(&buf_hw, size)) + rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata, + etr_buf, node, pages); + if (rc && buf_hw.has_etr_sg) + rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata, + etr_buf, node, pages); + if (rc && buf_hw.has_catu) + rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata, + etr_buf, node, pages); + if (rc) { + kfree(etr_buf); + return ERR_PTR(rc); + } + + /* + * Keep a reference to the ETR this buffer has been allocated for + * in order to have access to the IDR in tmc_free_etr_buffer(). + */ + etr_buf->drvdata = drvdata; + /* per-thread buffers are not shared and never enter the idr */ + etr_buf->pid = -1; + + refcount_set(&etr_buf->refcount, 1); + dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n", + (unsigned long)size >> 10, etr_buf->mode); + return etr_buf; +} + +static void tmc_free_etr_buf(struct etr_buf *etr_buf) +{ + WARN_ON(!etr_buf->ops || !etr_buf->ops->free); + etr_buf->ops->free(etr_buf); + kfree(etr_buf); +} + +/* + * tmc_etr_buf_get_data: Get the pointer the trace data at @offset + * with a maximum of @len bytes. + * Returns: The size of the linear data available @pos, with *bufpp + * updated to point to the buffer. + */ +static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf, + u64 offset, size_t len, char **bufpp) +{ + /* Adjust the length to limit this transaction to end of buffer */ + len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset; + + return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp); +} + +static s64 +tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset) +{ + ssize_t len; + char *bufp; + + len = tmc_etr_buf_get_data(etr_buf, offset, + CORESIGHT_BARRIER_PKT_SIZE, &bufp); + if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE)) + return -EINVAL; + coresight_insert_barrier_packet(bufp); + return offset + CORESIGHT_BARRIER_PKT_SIZE; +} + +/* + * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata. + * Makes sure the trace data is synced to the memory for consumption. + * @etr_buf->offset will hold the offset to the beginning of the trace data + * within the buffer, with @etr_buf->len bytes to consume. + */ +static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata) +{ + struct etr_buf *etr_buf = drvdata->etr_buf; + u64 rrp, rwp; + u32 status; + + rrp = tmc_read_rrp(drvdata); + rwp = tmc_read_rwp(drvdata); + status = readl_relaxed(drvdata->base + TMC_STS); + + /* + * If there were memory errors in the session, truncate the + * buffer. + */ + if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) { + dev_dbg(&drvdata->comp->dev, + "tmc memory error detected, truncating buffer\n"); + etr_buf->len = 0; + etr_buf->full = false; + return; + } + + etr_buf->full = !!(status & TMC_STS_FULL); + + WARN_ON(!etr_buf->ops || !etr_buf->ops->sync); + + etr_buf->ops->sync(etr_buf, rrp, rwp); +} + +static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata) +{ + u32 axictl, sts, ffcr; + struct etr_buf *etr_buf = drvdata->etr_buf; + int rc = 0; + + CS_UNLOCK(drvdata->base); + + /* Wait for TMCSReady bit to be set */ + rc = tmc_wait_for_tmcready(drvdata); + if (rc) { + dev_err(&drvdata->comp->dev, + "Failed to enable : TMC not ready\n"); + CS_LOCK(drvdata->base); + return rc; + } + + writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ); + writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE); + + axictl = readl_relaxed(drvdata->base + TMC_AXICTL); + axictl &= ~TMC_AXICTL_CLEAR_MASK; + axictl |= TMC_AXICTL_PROT_CTL_B1; + axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size); + axictl |= TMC_AXICTL_AXCACHE_OS; + + if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) { + axictl &= ~TMC_AXICTL_ARCACHE_MASK; + axictl |= TMC_AXICTL_ARCACHE_OS; + } + + if (etr_buf->mode == ETR_MODE_ETR_SG) + axictl |= TMC_AXICTL_SCT_GAT_MODE; + + writel_relaxed(axictl, drvdata->base + TMC_AXICTL); + tmc_write_dba(drvdata, etr_buf->hwaddr); + /* + * If the TMC pointers must be programmed before the session, + * we have to set it properly (i.e, RRP/RWP to base address and + * STS to "not full"). + */ + if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) { + tmc_write_rrp(drvdata, etr_buf->hwaddr); + tmc_write_rwp(drvdata, etr_buf->hwaddr); + sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL; + writel_relaxed(sts, drvdata->base + TMC_STS); + } + + ffcr = TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | TMC_FFCR_FON_FLIN | + TMC_FFCR_FON_TRIG_EVT | TMC_FFCR_TRIGON_TRIGIN; + if (drvdata->stop_on_flush) + ffcr |= TMC_FFCR_STOP_ON_FLUSH; + writel_relaxed(ffcr, drvdata->base + TMC_FFCR); + + writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG); + tmc_enable_hw(drvdata); + + CS_LOCK(drvdata->base); + return rc; +} + +static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata, + struct etr_buf *etr_buf) +{ + int rc; + + /* Callers should provide an appropriate buffer for use */ + if (WARN_ON(!etr_buf)) + return -EINVAL; + + if ((etr_buf->mode == ETR_MODE_ETR_SG) && + WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG))) + return -EINVAL; + + if (WARN_ON(drvdata->etr_buf)) + return -EBUSY; + + rc = coresight_claim_device(drvdata->comp); + if (!rc) { + drvdata->etr_buf = etr_buf; + rc = __tmc_etr_enable_hw(drvdata); + if (rc) { + drvdata->etr_buf = NULL; + coresight_disclaim_device(drvdata->comp); + } + } + + return rc; +} + +/* + * Return the available trace data in the buffer (starts at etr_buf->offset, + * limited by etr_buf->len) from @pos, with a maximum limit of @len, + * also updating the @bufpp on where to find it. Since the trace data + * starts at anywhere in the buffer, depending on the RRP, we adjust the + * @len returned to handle buffer wrapping around. + * + * We are protected here by drvdata->reading != 0, which ensures the + * sysfs_buf stays alive. + */ +ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata, + loff_t pos, size_t len, char **bufpp) +{ + s64 offset; + ssize_t actual = len; + struct etr_buf *etr_buf = drvdata->sysfs_buf; + + if (pos + actual > etr_buf->len) + actual = etr_buf->len - pos; + if (actual <= 0) + return actual; + + /* Compute the offset from which we read the data */ + offset = etr_buf->offset + pos; + if (offset >= etr_buf->size) + offset -= etr_buf->size; + return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp); +} + +static struct etr_buf * +tmc_etr_alloc_sysfs_buf(struct tmc_drvdata *drvdata) +{ + return tmc_alloc_etr_buf(drvdata, drvdata->size, + 0, cpu_to_node(0), NULL); +} + +static void +tmc_etr_free_sysfs_buf(struct etr_buf *buf) +{ + if (buf) + tmc_free_etr_buf(buf); +} + +static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata) +{ + struct etr_buf *etr_buf = drvdata->etr_buf; + + if (WARN_ON(drvdata->sysfs_buf != etr_buf)) { + tmc_etr_free_sysfs_buf(drvdata->sysfs_buf); + drvdata->sysfs_buf = NULL; + } else { + tmc_sync_etr_buf(drvdata); + /* + * Insert barrier packets at the beginning, if there was + * an overflow. + */ + if (etr_buf->full) + tmc_etr_buf_insert_barrier_packet(etr_buf, + etr_buf->offset); + } + /* Reset the ETR buf used by hardware */ + drvdata->etr_buf = NULL; +} + +static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata) +{ + CS_UNLOCK(drvdata->base); + + tmc_flush_and_stop(drvdata); + /* + * When operating in sysFS mode the content of the buffer needs to be + * read before the TMC is disabled. + */ + if (gtrace_get_mode(drvdata->comp) == GTRACE_COMPONENT_MODE_SYSFS) + tmc_etr_sync_sysfs_buf(drvdata); + + tmc_disable_hw(drvdata); + + CS_LOCK(drvdata->base); + +} + +void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) +{ + __tmc_etr_disable_hw(drvdata); + coresight_disclaim_device(drvdata->comp); +} + +int tmc_etr_setup_sysfs_buf(struct gtrace_component *comp) +{ + int ret = 0; + unsigned long flags; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL; + + /* + * If we are enabling the ETR from disabled state, we need to make + * sure we have a buffer with the right size. The etr_buf is not reset + * immediately after we stop the tracing in SYSFS mode as we wait for + * the user to collect the data. We may be able to reuse the existing + * buffer, provided the size matches. Any allocation has to be done + * with the lock released. + */ + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + + sysfs_buf = READ_ONCE(drvdata->sysfs_buf); + if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) { + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* Allocate memory with the locks released */ + free_buf = new_buf = tmc_etr_alloc_sysfs_buf(drvdata); + if (IS_ERR(new_buf)) + return PTR_ERR(new_buf); + + /* Let's try again */ + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + } + + if (drvdata->reading) { + ret = -EBUSY; + goto out; + } + + /* + * If we don't have a buffer or it doesn't match the requested size, + * use the buffer allocated above. Otherwise reuse the existing buffer. + */ + sysfs_buf = READ_ONCE(drvdata->sysfs_buf); + if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) { + free_buf = sysfs_buf; + drvdata->sysfs_buf = new_buf; + } + +out: + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* Free memory outside the spinlock if need be */ + if (free_buf) + tmc_etr_free_sysfs_buf(free_buf); + return ret; +} + +struct etr_buf *tmc_etr_get_buffer(struct gtrace_component *comp) +{ + struct gtrace_path *path = comp->path; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + struct etr_buf *etr_buf; + + switch (path->mode) { + case GTRACE_COMPONENT_MODE_SYSFS: + etr_buf = drvdata->sysfs_buf; + break; + case GTRACE_COMPONENT_MODE_PERF: + etr_buf = drvdata->perf_buf; + break; + default: + return ERR_PTR(-EINVAL); + } + + WARN_ON(!etr_buf); + + return etr_buf; +} +EXPORT_SYMBOL_GPL(tmc_etr_get_buffer); + +/* + * alloc_etr_buf: Allocate ETR buffer for use by perf. + * Allocate the largest possible size, scaling down the size by half until it + * reaches a minimum limit (1M), beyond which we give up. + */ +static struct etr_buf * +alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event, + int nr_pages, void **pages, bool snapshot) +{ + int node; + struct etr_buf *etr_buf; + ssize_t size; + + node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu); + + /* Use the minimum limit if the required size is smaller */ + size = (ssize_t)nr_pages << PAGE_SHIFT; + size = max_t(ssize_t, size, TMC_ETR_PERF_MIN_BUF_SIZE); + + /* + * Try to allocate the required size for this ETR, if failed scale + * down until we hit the minimum limit. + */ + do { + etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL); + if (!IS_ERR(etr_buf)) + return etr_buf; + size /= 2; + } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE); + + return ERR_PTR(-ENOMEM); +} + +static struct etr_buf * +get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata, + struct perf_event *event, int nr_pages, + void **pages, bool snapshot) +{ + int ret; + pid_t pid = task_pid_nr(event->owner); + struct etr_buf *etr_buf; + +retry: + /* + * An gtrace_perf_auxbuf is associated with an event and holds a reference + * to the AUX ring buffer that was created for that event. In CPU-wide + * N:1 mode multiple events (one per CPU), each with its own AUX ring + * buffer, share a sink. As such an gtrace_perf_auxbuf is created for each + * event but a single etr_buf associated with the ETR is shared between + * them. The last event in a trace session will copy the content of the + * etr_buf to its AUX ring buffer. Ring buffer associated to other + * events are simply not used an freed as events are destoyed. We still + * need to allocate a ring buffer for each event since we don't know + * which event will be last. + */ + + /* + * The first thing to do here is check if an etr_buf has already been + * allocated for this session. If so it is shared with this event, + * otherwise it is created. + */ + mutex_lock(&drvdata->idr_mutex); + etr_buf = idr_find(&drvdata->idr, pid); + if (etr_buf) { + refcount_inc(&etr_buf->refcount); + mutex_unlock(&drvdata->idr_mutex); + return etr_buf; + } + + /* If we made it here no buffer has been allocated, do so now. */ + mutex_unlock(&drvdata->idr_mutex); + + etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot); + if (IS_ERR(etr_buf)) + return etr_buf; + + /* Now that we have a buffer, add it to the IDR. */ + mutex_lock(&drvdata->idr_mutex); + ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL); + mutex_unlock(&drvdata->idr_mutex); + + /* Another event with this session ID has allocated this buffer. */ + if (ret == -ENOSPC) { + tmc_free_etr_buf(etr_buf); + goto retry; + } + + /* The IDR can't allocate room for a new session, abandon ship. */ + if (ret == -ENOMEM) { + tmc_free_etr_buf(etr_buf); + return ERR_PTR(ret); + } + + /* + * Keep a reference to the ETR this buffer has been allocated for + * in order to have access to the IDR in tmc_free_etr_buffer(). + */ + etr_buf->drvdata = drvdata; + /* cpu-wide buffers are shared and tracked in the idr by session pid */ + etr_buf->pid = pid; + + return etr_buf; +} + +static struct etr_buf * +get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata, + struct perf_event *event, int nr_pages, + void **pages, bool snapshot) +{ + /* + * In per-thread mode the etr_buf isn't shared, so just go ahead + * with memory allocation. + */ + return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot); +} + +static struct etr_buf * +get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event, + int nr_pages, void **pages, bool snapshot) +{ + if (event->cpu == -1) + return get_perf_etr_buf_per_thread(drvdata, event, nr_pages, + pages, snapshot); + + return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages, + pages, snapshot); +} + +void *tmc_alloc_etr_buffer(struct gtrace_component *comp, + struct perf_event *event, void **pages, + int nr_pages, bool snapshot) +{ + int node; + struct etr_buf *etr_buf; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + + node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu); + + etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot); + if (IS_ERR(etr_buf)) + return NULL; + + drvdata->perf_buf = etr_buf; + + return etr_buf; +} + +void tmc_free_etr_buffer(void *config) +{ + struct etr_buf *buf = NULL, *etr_buf = config; + struct tmc_drvdata *drvdata; + + drvdata = etr_buf->drvdata; + + mutex_lock(&drvdata->idr_mutex); + /* If we are not the last one to use the buffer, don't touch it. */ + if (!refcount_dec_and_test(&etr_buf->refcount)) { + mutex_unlock(&drvdata->idr_mutex); + return; + } + + /* only shared buffers are present in the idr */ + if (etr_buf->pid != -1) { + /* We are the last one, remove from the IDR and free the buffer. */ + buf = idr_remove(&drvdata->idr, etr_buf->pid); + + } + mutex_unlock(&drvdata->idr_mutex); + + /* + * Something went very wrong if the buffer associated with this ID + * is not the same in the IDR. Leak to avoid use after free. + */ + if (buf && WARN_ON(buf != etr_buf)) + return; + + tmc_free_etr_buf(etr_buf); + /* Reset perf specific data */ + drvdata->perf_buf = NULL; +} + +/* + * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware + * buffer to the perf ring buffer. + */ +static void tmc_etr_sync_perf_buffer(struct gtrace_perf_auxbuf *etr_perf, + unsigned long head, + unsigned long src_offset, + unsigned long to_copy) +{ + long bytes; + long pg_idx, pg_offset; + char **dst_pages, *src_buf; + struct etr_buf *etr_buf = etr_perf->buf; + + head = PERF_IDX2OFF(head, etr_perf); + pg_idx = head >> PAGE_SHIFT; + pg_offset = head & (PAGE_SIZE - 1); + dst_pages = (char **)etr_perf->pages; + + while (to_copy > 0) { + /* + * In one iteration, we can copy minimum of : + * 1) what is available in the source buffer, + * 2) what is available in the source buffer, before it + * wraps around. + * 3) what is available in the destination page. + * in one iteration. + */ + if (src_offset >= etr_buf->size) + src_offset -= etr_buf->size; + bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy, + &src_buf); + if (WARN_ON_ONCE(bytes <= 0)) + break; + bytes = min(bytes, (long)(PAGE_SIZE - pg_offset)); + + memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes); + + to_copy -= bytes; + + /* Move destination pointers */ + pg_offset += bytes; + if (pg_offset == PAGE_SIZE) { + pg_offset = 0; + if (++pg_idx == etr_perf->nr_pages) + pg_idx = 0; + } + + /* Move source pointers */ + src_offset += bytes; + } +} + +/* + * tmc_update_etr_buffer : Update the perf ring buffer with the + * available trace data. We use software double buffering at the moment. + * + * TODO: Add support for reusing the perf ring buffer. + */ +unsigned long +tmc_update_etr_buffer(struct gtrace_component *comp, + struct perf_output_handle *handle, + struct gtrace_perf_auxbuf *etr_perf) +{ + bool lost = false; + unsigned long offset, size = 0; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + struct etr_buf *etr_buf = etr_perf->buf; + struct perf_event *event = handle->event; + + if (WARN_ON(drvdata->perf_buf != etr_buf)) { + lost = true; + goto out; + } + + CS_UNLOCK(drvdata->base); + + tmc_flush_and_stop(drvdata); + tmc_sync_etr_buf(drvdata); + + CS_LOCK(drvdata->base); + + lost = etr_buf->full; + offset = etr_buf->offset; + size = etr_buf->len; + + /* + * The ETR buffer may be bigger than the space available in the + * perf ring buffer (handle->size). If so advance the offset so that we + * get the latest trace data. In snapshot mode none of that matters + * since we are expected to clobber stale data in favour of the latest + * traces. + */ + if (!etr_perf->snapshot && size > handle->size) { + u32 mask = tmc_get_memwidth_mask(drvdata); + + /* + * Make sure the new size is aligned in accordance with the + * requirement explained in function tmc_get_memwidth_mask(). + */ + size = handle->size & mask; + offset = etr_buf->offset + etr_buf->len - size; + + if (offset >= etr_buf->size) + offset -= etr_buf->size; + lost = true; + } + + /* Insert barrier packets at the beginning, if there was an overflow */ + if (lost) + tmc_etr_buf_insert_barrier_packet(etr_buf, offset); + tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size); + + /* + * In snapshot mode we simply increment the head by the number of byte + * that were written. User space will figure out how many bytes to get + * from the AUX buffer based on the position of the head. + */ + if (etr_perf->snapshot) + handle->head += size; + + /* + * Ensure that the AUX trace data is visible before the aux_head + * is updated via perf_aux_output_end(), as expected by the + * perf ring buffer. + */ + smp_wmb(); + + /* + * If the event is active, it is triggered during an AUX pause. + * Re-enable the sink so that it is ready when AUX resume is invoked. + */ + if (!event->hw.state) + __tmc_etr_enable_hw(drvdata); + + /* Reset the ETR buf used by hardware */ + if (event->hw.state == PERF_HES_STOPPED) + drvdata->etr_buf = NULL; + + /* set the GTRACE_FORMAT_CORESIGHT flag */ + perf_aux_output_flag(handle, PERF_AUX_FLAG_GTRACE_FORMAT_CORESIGHT); +out: + /* + * Don't set the TRUNCATED flag in snapshot mode because 1) the + * captured buffer is expected to be truncated and 2) a full buffer + * prevents the event from being re-enabled by the perf core, + * resulting in stale data being send to user space. + */ + if (!etr_perf->snapshot && lost) + perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + return size; +} + +int tmc_enable_etr_sink(struct gtrace_component *comp) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + struct etr_buf *etr_buf; + int ret; + + etr_buf = tmc_etr_get_buffer(comp); + if (IS_ERR(etr_buf)) + return PTR_ERR(etr_buf); + + ret = tmc_etr_enable_hw(drvdata, etr_buf); + if (!ret) + dev_dbg(&comp->dev, "TMC-ETR enabled\n"); + + return ret; +} + +int tmc_disable_etr_sink(struct gtrace_component *comp) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + + if (drvdata->reading) + return -EBUSY; + + tmc_etr_disable_hw(drvdata); + + dev_dbg(&comp->dev, "TMC-ETR disabled\n"); + return 0; +} + +int tmc_panic_sync_etr(struct gtrace_component *comp) +{ + u32 val; + struct tmc_crash_metadata *mdata; + struct tmc_drvdata *drvdata = dev_get_drvdata(comp->dev.parent); + + mdata = (struct tmc_crash_metadata *)drvdata->crash_mdata.vaddr; + + if (!drvdata->etr_buf) + return 0; + + /* Being in RESRV mode implies valid reserved memory as well */ + if (drvdata->etr_buf->mode != ETR_MODE_RESRV) + return 0; + + if (!tmc_has_crash_mdata_buffer(drvdata)) + return 0; + + CS_UNLOCK(drvdata->base); + + /* Proceed only if ETR is enabled */ + val = readl(drvdata->base + TMC_CTL); + if (!(val & TMC_CTL_CAPT_EN)) + goto out; + + val = readl(drvdata->base + TMC_FFSR); + /* Do manual flush and stop only if its not auto-stopped */ + if (!(val & TMC_FFSR_FT_STOPPED)) { + dev_dbg(&comp->dev, + "%s: Triggering manual flush\n", __func__); + tmc_flush_and_stop(drvdata); + } else + tmc_wait_for_tmcready(drvdata); + + /* Sync registers from hardware to metadata region */ + mdata->tmc_ram_size = readl(drvdata->base + TMC_RSZ); + mdata->tmc_sts = readl(drvdata->base + TMC_STS); + mdata->tmc_mode = readl(drvdata->base + TMC_MODE); + mdata->tmc_ffcr = readl(drvdata->base + TMC_FFCR); + mdata->tmc_ffsr = readl(drvdata->base + TMC_FFSR); + mdata->tmc_rrp = tmc_read_rrp(drvdata); + mdata->tmc_rwp = tmc_read_rwp(drvdata); + mdata->tmc_dba = tmc_read_dba(drvdata); + mdata->trace_paddr = drvdata->resrv_buf.paddr; + mdata->version = CS_CRASHDATA_VERSION; + + /* + * Make sure all previous writes are ordered, + * before we mark valid + */ + smp_wmb(); + mdata->valid = true; + /* + * Below order need to maintained, since crc of metadata + * is dependent on first + */ + mdata->crc32_tdata = find_crash_tracedata_crc(drvdata, mdata); + mdata->crc32_mdata = find_crash_metadata_crc(mdata); + + tmc_disable_hw(drvdata); + + dev_dbg(&comp->dev, "%s: success\n", __func__); +out: + CS_UNLOCK(drvdata->base); + + return 0; +} + +int tmc_read_prepare_etr(struct tmc_drvdata *drvdata) +{ + int ret = 0; + unsigned long flags; + + /* config types are set a boot time and never change */ + if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR)) + return -EINVAL; + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + if (drvdata->reading) { + ret = -EBUSY; + goto out; + } + + /* + * We can safely allow reads even if the ETR is operating in PERF mode, + * since the sysfs session is captured in mode specific data. + * If drvdata::sysfs_data is NULL the trace data has been read already. + */ + if (!drvdata->sysfs_buf) { + ret = -EINVAL; + goto out; + } + + /* Disable the TMC if we are trying to read from a running session. */ + if (gtrace_get_mode(drvdata->comp) == GTRACE_COMPONENT_MODE_SYSFS) + __tmc_etr_disable_hw(drvdata); + + drvdata->reading = true; +out: + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + return ret; +} + +int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata) +{ + unsigned long flags; + struct etr_buf *sysfs_buf = NULL; + + /* config types are set a boot time and never change */ + if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR)) + return -EINVAL; + + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + + /* RE-enable the TMC if need be */ + if (gtrace_get_mode(drvdata->comp) == GTRACE_COMPONENT_MODE_SYSFS) { + /* + * The trace run will continue with the same allocated trace + * buffer. Since the tracer is still enabled drvdata::buf can't + * be NULL. + */ + __tmc_etr_enable_hw(drvdata); + } else { + /* + * The ETR is not tracing and the buffer was just read. + * As such prepare to free the trace buffer. + */ + sysfs_buf = drvdata->sysfs_buf; + drvdata->sysfs_buf = NULL; + } + + drvdata->reading = false; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* Free allocated memory out side of the spinlock */ + if (sysfs_buf) + tmc_etr_free_sysfs_buf(sysfs_buf); + + return 0; +} + +static const char *const buf_modes_str[] = { + [ETR_MODE_FLAT] = "flat", + [ETR_MODE_ETR_SG] = "tmc-sg", + [ETR_MODE_CATU] = "catu", + [ETR_MODE_RESRV] = "resrv", + [ETR_MODE_AUTO] = "auto", +}; + +static ssize_t buf_modes_available_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct etr_buf_hw buf_hw; + ssize_t size = 0; + + get_etr_buf_hw(dev, &buf_hw); + size += sysfs_emit(buf, "%s ", buf_modes_str[ETR_MODE_AUTO]); + size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_FLAT]); + if (buf_hw.has_etr_sg) + size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_ETR_SG]); + + if (buf_hw.has_catu) + size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_CATU]); + + if (buf_hw.has_resrv) + size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_RESRV]); + + size += sysfs_emit_at(buf, size, "\n"); + return size; +} +static DEVICE_ATTR_RO(buf_modes_available); + +static ssize_t buf_mode_preferred_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + + return sysfs_emit(buf, "%s\n", buf_modes_str[drvdata->etr_mode]); +} + +static int buf_mode_set_resrv(struct tmc_drvdata *drvdata) +{ + int err = -EBUSY; + unsigned long flags; + struct tmc_resrv_buf *rbuf; + + rbuf = &drvdata->resrv_buf; + + /* Ensure there are no active crashdata read sessions */ + raw_spin_lock_irqsave(&drvdata->spinlock, flags); + if (!rbuf->reading) { + tmc_crashdata_set_invalid(drvdata); + rbuf->len = 0; + drvdata->etr_mode = ETR_MODE_RESRV; + err = 0; + } + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + return err; +} + +static ssize_t buf_mode_preferred_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct etr_buf_hw buf_hw; + + get_etr_buf_hw(dev, &buf_hw); + if (sysfs_streq(buf, buf_modes_str[ETR_MODE_FLAT])) + drvdata->etr_mode = ETR_MODE_FLAT; + else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg) + drvdata->etr_mode = ETR_MODE_ETR_SG; + else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu) + drvdata->etr_mode = ETR_MODE_CATU; + else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_RESRV]) && buf_hw.has_resrv) + return buf_mode_set_resrv(drvdata) ? : size; + else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_AUTO])) + drvdata->etr_mode = ETR_MODE_AUTO; + else + return -EINVAL; + return size; +} +static DEVICE_ATTR_RW(buf_mode_preferred); + +static struct attribute *coresight_etr_attrs[] = { + &dev_attr_buf_modes_available.attr, + &dev_attr_buf_mode_preferred.attr, + NULL, +}; + +const struct attribute_group coresight_etr_group = { + .attrs = coresight_etr_attrs, +}; diff --git a/drivers/hwtracing/gtrace/coresight-tmc.h b/drivers/hwtracing/gtrace/coresight-tmc.h new file mode 100644 index 000000000000..78281e04f243 --- /dev/null +++ b/drivers/hwtracing/gtrace/coresight-tmc.h @@ -0,0 +1,469 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(C) 2015 Linaro Limited. All rights reserved. + * Author: Mathieu Poirier mathieu.poirier@linaro.org + * Derived from drivers/hwtracing/coresight/coresight-tmc.h + */ + +#ifndef _CORESIGHT_TMC_H +#define _CORESIGHT_TMC_H + +#include <linux/dma-mapping.h> +#include <linux/idr.h> +#include <linux/miscdevice.h> +#include <linux/mutex.h> +#include <linux/refcount.h> +#include <linux/crc32.h> + +#define TMC_RSZ 0x004 +#define TMC_STS 0x00c +#define TMC_RRD 0x010 +#define TMC_RRP 0x014 +#define TMC_RWP 0x018 +#define TMC_TRG 0x01c +#define TMC_CTL 0x020 +#define TMC_RWD 0x024 +#define TMC_MODE 0x028 +#define TMC_LBUFLEVEL 0x02c +#define TMC_CBUFLEVEL 0x030 +#define TMC_BUFWM 0x034 +#define TMC_RRPHI 0x038 +#define TMC_RWPHI 0x03c +#define TMC_AXICTL 0x110 +#define TMC_DBALO 0x118 +#define TMC_DBAHI 0x11c +#define TMC_FFSR 0x300 +#define TMC_FFCR 0x304 +#define TMC_PSCR 0x308 +#define TMC_ITMISCOP0 0xee0 +#define TMC_ITTRFLIN 0xee8 +#define TMC_ITATBDATA0 0xeec +#define TMC_ITATBCTR2 0xef0 +#define TMC_ITATBCTR1 0xef4 +#define TMC_ITATBCTR0 0xef8 +#define TMC_AUTHSTATUS 0xfb8 + +/* register description */ +/* TMC_CTL - 0x020 */ +#define TMC_CTL_CAPT_EN BIT(0) +/* TMC_STS - 0x00C */ +#define TMC_STS_TMCREADY_BIT 2 +#define TMC_STS_FULL BIT(0) +#define TMC_STS_TRIGGERED BIT(1) +#define TMC_STS_MEMERR BIT(5) +/* + * TMC_AXICTL - 0x110 + * + * TMC AXICTL format for SoC-400 + * Bits [0-1] : ProtCtrlBit0-1 + * Bits [2-5] : CacheCtrlBits 0-3 (AXCACHE) + * Bit 6 : Reserved + * Bit 7 : ScatterGatherMode + * Bits [8-11] : WrBurstLen + * Bits [12-31] : Reserved. + * TMC AXICTL format for SoC-600, as above except: + * Bits [2-5] : AXI WCACHE + * Bits [16-19] : AXI RCACHE + * Bits [20-31] : Reserved + */ +#define TMC_AXICTL_CLEAR_MASK 0xfbf +#define TMC_AXICTL_ARCACHE_MASK (0xf << 16) + +#define TMC_AXICTL_PROT_CTL_B0 BIT(0) +#define TMC_AXICTL_PROT_CTL_B1 BIT(1) +#define TMC_AXICTL_SCT_GAT_MODE BIT(7) +#define TMC_AXICTL_WR_BURST(v) (((v) & 0xf) << 8) +#define TMC_AXICTL_WR_BURST_16 0xf +/* Write-back Read and Write-allocate */ +#define TMC_AXICTL_AXCACHE_OS (0xf << 2) +#define TMC_AXICTL_ARCACHE_OS (0xf << 16) + +/* TMC_FFSR - 0x300 */ +#define TMC_FFSR_FT_STOPPED BIT(1) + +/* TMC_FFCR - 0x304 */ +#define TMC_FFCR_FLUSHMAN_BIT 6 +#define TMC_FFCR_EN_FMT BIT(0) +#define TMC_FFCR_EN_TI BIT(1) +#define TMC_FFCR_FON_FLIN BIT(4) +#define TMC_FFCR_FON_TRIG_EVT BIT(5) +#define TMC_FFCR_TRIGON_TRIGIN BIT(8) +#define TMC_FFCR_STOP_ON_FLUSH BIT(12) + + +#define TMC_DEVID_NOSCAT BIT(24) + +#define TMC_DEVID_AXIAW_VALID BIT(16) +#define TMC_DEVID_AXIAW_SHIFT 17 +#define TMC_DEVID_AXIAW_MASK 0x7f + +#define TMC_AUTH_NSID_MASK GENMASK(1, 0) + +/* Major version 1 Minor version 0 */ +#define CS_CRASHDATA_VERSION (1 << 16) + +enum tmc_config_type { + TMC_CONFIG_TYPE_ETB, + TMC_CONFIG_TYPE_ETR, + TMC_CONFIG_TYPE_ETF, +}; + +enum tmc_mode { + TMC_MODE_CIRCULAR_BUFFER, + TMC_MODE_SOFTWARE_FIFO, + TMC_MODE_HARDWARE_FIFO, +}; + +enum tmc_mem_intf_width { + TMC_MEM_INTF_WIDTH_32BITS = 1, + TMC_MEM_INTF_WIDTH_64BITS = 2, + TMC_MEM_INTF_WIDTH_128BITS = 4, + TMC_MEM_INTF_WIDTH_256BITS = 8, +}; + +/* TMC ETR Capability bit definitions */ +#define TMC_ETR_SG (0x1U << 0) +/* ETR has separate read/write cache encodings */ +#define TMC_ETR_AXI_ARCACHE (0x1U << 1) +/* + * TMC_ETR_SAVE_RESTORE - Values of RRP/RWP/STS.Full are + * retained when TMC leaves Disabled state, allowing us to continue + * the tracing from a point where we stopped. This also implies that + * the RRP/RWP/STS.Full should always be programmed to the correct + * value. Unfortunately this is not advertised by the hardware, + * so we have to rely on PID of the IP to detect the functionality. + */ +#define TMC_ETR_SAVE_RESTORE (0x1U << 2) + +/* Coresight SoC-600 TMC-ETR unadvertised capabilities */ +#define CORESIGHT_SOC_600_ETR_CAPS \ + (TMC_ETR_SAVE_RESTORE | TMC_ETR_AXI_ARCACHE) + +/* TMC metadata region for ETR and ETF configurations */ +struct tmc_crash_metadata { + uint32_t crc32_mdata; /* crc of metadata */ + uint32_t crc32_tdata; /* crc of tracedata */ + uint32_t version; /* 31:16 Major version, 15:0 Minor version */ + uint32_t valid; /* Indicate if this ETF/ETR was enabled */ + uint32_t tmc_ram_size; /* Ram Size register */ + uint32_t tmc_sts; /* Status register */ + uint32_t tmc_mode; /* Mode register */ + uint32_t tmc_ffcr; /* Formatter and flush control register */ + uint32_t tmc_ffsr; /* Formatter and flush status register */ + uint32_t reserved32; + uint64_t tmc_rrp; /* Ram Read pointer register */ + uint64_t tmc_rwp; /* Ram Write pointer register */ + uint64_t tmc_dba; /* Data buffer address register */ + uint64_t trace_paddr; /* Phys address of trace buffer */ + uint64_t reserved64[3]; +}; + +enum etr_mode { + ETR_MODE_FLAT, /* Uses contiguous flat buffer */ + ETR_MODE_ETR_SG, /* Uses in-built TMC ETR SG mechanism */ + ETR_MODE_CATU, /* Use SG mechanism in CATU */ + ETR_MODE_RESRV, /* Use reserved region contiguous buffer */ + ETR_MODE_AUTO, /* Use the default mechanism */ +}; + +struct tmc_drvdata; +struct etr_buf_operations; + +/** + * struct etr_buf - Details of the buffer used by ETR + * drvdata ; The ETR drvdaga this buffer has been allocated for. + * @pid ; The PID of the session owner that etr_buf belongs to. + * refcount ; Number of sources currently using this etr_buf. + * @mode : Mode of the ETR buffer, contiguous, Scatter Gather etc. + * @full : Trace data overflow + * @size : Size of the buffer. + * @hwaddr : Address to be programmed in the TMC:DBA{LO,HI} + * @offset : Offset of the trace data in the buffer for consumption. + * @len : Available trace data @buf (may round up to the beginning). + * @ops : ETR buffer operations for the mode. + * @private : Backend specific information for the buf + */ +struct etr_buf { + struct tmc_drvdata *drvdata; + pid_t pid; + refcount_t refcount; + enum etr_mode mode; + bool full; + ssize_t size; + dma_addr_t hwaddr; + unsigned long offset; + s64 len; + const struct etr_buf_operations *ops; + void *private; +}; + +/** + * @paddr : Start address of reserved memory region. + * @vaddr : Corresponding CPU virtual address. + * @size : Size of reserved memory region. + * @offset : Offset of the trace data in the buffer for consumption. + * @reading : Flag to indicate if reading is active + * @len : Available trace data @buf (may round up to the beginning). + */ +struct tmc_resrv_buf { + phys_addr_t paddr; + void *vaddr; + size_t size; + unsigned long offset; + bool reading; + s64 len; +}; + +/** + * struct tmc_drvdata - specifics associated to an TMC component + * @atclk: optional clock for the core parts of the TMC. + * @pclk: APB clock if present, otherwise NULL + * @base: memory mapped base address for this component. + * @comp: component vitals needed by the framework. + * @miscdev: specifics to handle "/dev/xyz.tmc" entry. + * @crashdev: specifics to handle "/dev/crash_tmc_xyz" entry for reading + * crash tracedata. + * @spinlock: only one at a time pls. + * @reading: buffer's in the reading through "/dev/xyz.tmc" entry + * @stop_on_flush: Stop on flush trigger user configuration. + * @is_sink: True if the ETF is configured as a sink. + * @buf: Snapshot of the trace data for ETF/ETB. + * @etr_buf: details of buffer used in TMC-ETR + * @len: size of the available trace for ETF/ETB. + * @size: trace buffer size for this TMC (common for all modes). + * @max_burst_size: The maximum burst size that can be initiated by + * TMC-ETR on AXI bus. + * @config_type: TMC variant, must be of type @tmc_config_type. + * @memwidth: width of the memory interface databus, in bytes. + * @trigger_cntr: amount of words to store after a trigger. + * @etr_caps: Bitmask of capabilities of the TMC ETR, inferred from the + * device configuration register (DEVID) + * @etr_mode: User preferred mode of the ETR device, default auto mode. + * @idr: Holds etr_bufs allocated for this ETR. + * @idr_mutex: Access serialisation for idr. + * @sysfs_buf: SYSFS buffer for ETR. + * @perf_buf: PERF buffer for ETR. + * @resrv_buf: Used by ETR as hardware trace buffer and for trace data + * retention (after crash) only when ETR_MODE_RESRV buffer + * mode is enabled. Used by ETF for trace data retention + * (after crash) by default. + * @crash_mdata: Reserved memory for storing tmc crash metadata. + * Used by ETR/ETF. + */ +struct tmc_drvdata { + struct clk *atclk; + struct clk *pclk; + void __iomem *base; + struct gtrace_component *comp; + struct miscdevice miscdev; + struct miscdevice crashdev; + raw_spinlock_t spinlock; + bool reading; + bool stop_on_flush; + bool is_sink; + union { + char *buf; /* TMC ETB */ + struct etr_buf *etr_buf; /* TMC ETR */ + }; + u32 len; + u32 size; + u32 max_burst_size; + enum tmc_config_type config_type; + enum tmc_mem_intf_width memwidth; + u32 trigger_cntr; + u32 etr_caps; + enum etr_mode etr_mode; + struct idr idr; + struct mutex idr_mutex; + struct etr_buf *sysfs_buf; + struct etr_buf *perf_buf; + struct tmc_resrv_buf resrv_buf; + struct tmc_resrv_buf crash_mdata; +}; + +struct etr_buf_operations { + int (*alloc)(struct tmc_drvdata *drvdata, struct etr_buf *etr_buf, + int node, void **pages); + void (*sync)(struct etr_buf *etr_buf, u64 rrp, u64 rwp); + ssize_t (*get_data)(struct etr_buf *etr_buf, u64 offset, size_t len, + char **bufpp); + void (*free)(struct etr_buf *etr_buf); +}; + +/** + * struct tmc_pages - Collection of pages used for SG. + * @nr_pages: Number of pages in the list. + * @daddrs: Array of DMA'able page address. + * @pages: Array pages for the buffer. + */ +struct tmc_pages { + int nr_pages; + dma_addr_t *daddrs; + struct page **pages; +}; + +/* + * struct tmc_sg_table - Generic SG table for TMC + * @dev: Device for DMA allocations + * @table_vaddr: Contiguous Virtual address for PageTable + * @data_vaddr: Contiguous Virtual address for Data Buffer + * @table_daddr: DMA address of the PageTable base + * @node: Node for Page allocations + * @table_pages: List of pages & dma address for Table + * @data_pages: List of pages & dma address for Data + */ +struct tmc_sg_table { + struct device *dev; + void *table_vaddr; + void *data_vaddr; + dma_addr_t table_daddr; + int node; + struct tmc_pages table_pages; + struct tmc_pages data_pages; +}; + +/* Generic functions */ +int tmc_wait_for_tmcready(struct tmc_drvdata *drvdata); +void tmc_flush_and_stop(struct tmc_drvdata *drvdata); +void tmc_enable_hw(struct tmc_drvdata *drvdata); +void tmc_disable_hw(struct tmc_drvdata *drvdata); +u32 tmc_get_memwidth_mask(struct tmc_drvdata *drvdata); +int tmc_read_prepare_crashdata(struct tmc_drvdata *drvdata); + +/* ETB/ETF functions */ +int tmc_read_prepare_etb(struct tmc_drvdata *drvdata); +int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata); +int tmc_etf_setup_sysfs_buf(struct gtrace_component *comp); +unsigned long tmc_update_etf_buffer(struct gtrace_component *comp, + struct perf_output_handle *handle, + struct gtrace_perf_auxbuf *buf); +int tmc_enable_etf(struct gtrace_component *comp); +int tmc_disable_etf(struct gtrace_component *comp); +int tmc_panic_sync_etf(struct gtrace_component *comp); + +ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata, + loff_t pos, size_t len, char **bufpp); +/* ETR functions */ +int tmc_read_prepare_etr(struct tmc_drvdata *drvdata); +int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata); +void tmc_etr_disable_hw(struct tmc_drvdata *drvdata); +int tmc_etr_setup_sysfs_buf(struct gtrace_component *comp); +void *tmc_alloc_etr_buffer(struct gtrace_component *comp, + struct perf_event *event, void **pages, + int nr_pages, bool snapshot); +void tmc_free_etr_buffer(void *config); +unsigned long +tmc_update_etr_buffer(struct gtrace_component *comp, + struct perf_output_handle *handle, + struct gtrace_perf_auxbuf *buf); +int tmc_enable_etr_sink(struct gtrace_component *comp); +int tmc_disable_etr_sink(struct gtrace_component *comp); +int tmc_panic_sync_etr(struct gtrace_component *comp); +ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata, + loff_t pos, size_t len, char **bufpp); + + +#define TMC_REG_PAIR(name, lo_off, hi_off) \ +static inline u64 \ +tmc_read_##name(struct tmc_drvdata *drvdata) \ +{ \ + return gtrace_relaxed_read_pair(drvdata->comp->pdata, lo_off, hi_off); \ +} \ +static inline void \ +tmc_write_##name(struct tmc_drvdata *drvdata, u64 val) \ +{ \ + gtrace_relaxed_write_pair(drvdata->comp->pdata, val, lo_off, hi_off); \ +} + +TMC_REG_PAIR(rrp, TMC_RRP, TMC_RRPHI) +TMC_REG_PAIR(rwp, TMC_RWP, TMC_RWPHI) +TMC_REG_PAIR(dba, TMC_DBALO, TMC_DBAHI) + +/* Initialise the caps from unadvertised static capabilities of the device */ +static inline void tmc_etr_init_caps(struct tmc_drvdata *drvdata, u32 dev_caps) +{ + WARN_ON(drvdata->etr_caps); + drvdata->etr_caps = dev_caps; +} + +static inline void tmc_etr_set_cap(struct tmc_drvdata *drvdata, u32 cap) +{ + drvdata->etr_caps |= cap; +} + +static inline bool tmc_etr_has_cap(struct tmc_drvdata *drvdata, u32 cap) +{ + return !!(drvdata->etr_caps & cap); +} + +struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev, + int node, + int nr_tpages, + int nr_dpages, + void **pages); +void tmc_free_sg_table(struct tmc_sg_table *sg_table); +void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table); +void tmc_sg_table_sync_data_range(struct tmc_sg_table *table, + u64 offset, u64 size); +ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table, + u64 offset, size_t len, char **bufpp); + +static inline unsigned long +tmc_sg_table_buf_size(struct tmc_sg_table *sg_table) +{ + return (unsigned long)sg_table->data_pages.nr_pages << PAGE_SHIFT; +} + +static inline bool tmc_has_reserved_buffer(struct tmc_drvdata *drvdata) +{ + if (drvdata->resrv_buf.vaddr && + drvdata->resrv_buf.size) + return true; + return false; +} + +static inline bool tmc_has_crash_mdata_buffer(struct tmc_drvdata *drvdata) +{ + if (drvdata->crash_mdata.vaddr && + drvdata->crash_mdata.size) + return true; + return false; +} + +static inline void tmc_crashdata_set_invalid(struct tmc_drvdata *drvdata) +{ + struct tmc_crash_metadata *mdata; + + mdata = (struct tmc_crash_metadata *)drvdata->crash_mdata.vaddr; + + if (tmc_has_crash_mdata_buffer(drvdata)) + mdata->valid = false; +} + +static inline uint32_t find_crash_metadata_crc(struct tmc_crash_metadata *md) +{ + unsigned long crc_size; + + crc_size = sizeof(struct tmc_crash_metadata) - + offsetof(struct tmc_crash_metadata, crc32_tdata); + return crc32_le(0, (void *)&md->crc32_tdata, crc_size); +} + +static inline uint32_t find_crash_tracedata_crc(struct tmc_drvdata *drvdata, + struct tmc_crash_metadata *md) +{ + unsigned long crc_size; + + /* Take CRC of configured buffer size to keep it simple */ + crc_size = md->tmc_ram_size << 2; + return crc32_le(0, (void *)drvdata->resrv_buf.vaddr, crc_size); +} + +struct gtrace_component *tmc_etr_get_catu_device(struct tmc_drvdata *drvdata); + +void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu); +void tmc_etr_remove_catu_ops(void); +struct etr_buf *tmc_etr_get_buffer(struct gtrace_component *comp); +extern const struct attribute_group coresight_etr_group; + +#endif

Add support for RISC-V Nexus Trace decoder based on the reference implementation from the RISC-V Nexus Trace specification. This includes a CoreSight frame deformatter to remove the trace formatter overhead.

Signed-off-by: Zane Leung liangzhen@linux.spacemit.com --- tools/arch/riscv/include/asm/insn.h | 645 +++++++++++++ tools/perf/util/Build | 1 + tools/perf/util/rvtrace-decoder/Build | 2 + .../util/rvtrace-decoder/rvtrace-decoder.c | 423 ++++++++ .../util/rvtrace-decoder/rvtrace-decoder.h | 122 +++ .../rvtrace-decoder/rvtrace-ntrace-decoder.c | 910 ++++++++++++++++++ .../rvtrace-decoder/rvtrace-ntrace-decoder.h | 124 +++ 7 files changed, 2227 insertions(+) create mode 100644 tools/arch/riscv/include/asm/insn.h create mode 100644 tools/perf/util/rvtrace-decoder/Build create mode 100644 tools/perf/util/rvtrace-decoder/rvtrace-decoder.c create mode 100644 tools/perf/util/rvtrace-decoder/rvtrace-decoder.h create mode 100644 tools/perf/util/rvtrace-decoder/rvtrace-ntrace-decoder.c create mode 100644 tools/perf/util/rvtrace-decoder/rvtrace-ntrace-decoder.h

diff --git a/tools/arch/riscv/include/asm/insn.h b/tools/arch/riscv/include/asm/insn.h new file mode 100644 index 000000000000..6c020afe926b --- /dev/null +++ b/tools/arch/riscv/include/asm/insn.h @@ -0,0 +1,645 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 SiFive + */ + +#ifndef _ASM_RISCV_INSN_H +#define _ASM_RISCV_INSN_H + +#include <linux/bits.h> + +#define RV_INSN_FUNCT3_MASK GENMASK(14, 12) +#define RV_INSN_FUNCT3_OPOFF 12 +#define RV_INSN_OPCODE_MASK GENMASK(6, 0) +#define RV_INSN_OPCODE_OPOFF 0 +#define RV_INSN_FUNCT12_OPOFF 20 + +#define RV_ENCODE_FUNCT3(f_) (RVG_FUNCT3_##f_ << RV_INSN_FUNCT3_OPOFF) +#define RV_ENCODE_FUNCT12(f_) (RVG_FUNCT12_##f_ << RV_INSN_FUNCT12_OPOFF) + +/* The bit field of immediate value in I-type instruction */ +#define RV_I_IMM_SIGN_OPOFF 31 +#define RV_I_IMM_11_0_OPOFF 20 +#define RV_I_IMM_SIGN_OFF 12 +#define RV_I_IMM_11_0_OFF 0 +#define RV_I_IMM_11_0_MASK GENMASK(11, 0) + +/* The bit field of immediate value in J-type instruction */ +#define RV_J_IMM_SIGN_OPOFF 31 +#define RV_J_IMM_10_1_OPOFF 21 +#define RV_J_IMM_11_OPOFF 20 +#define RV_J_IMM_19_12_OPOFF 12 +#define RV_J_IMM_SIGN_OFF 20 +#define RV_J_IMM_10_1_OFF 1 +#define RV_J_IMM_11_OFF 11 +#define RV_J_IMM_19_12_OFF 12 +#define RV_J_IMM_10_1_MASK GENMASK(9, 0) +#define RV_J_IMM_11_MASK GENMASK(0, 0) +#define RV_J_IMM_19_12_MASK GENMASK(7, 0) + +/* + * U-type IMMs contain the upper 20bits [31:20] of an immediate with + * the rest filled in by zeros, so no shifting required. Similarly, + * bit31 contains the signed state, so no sign extension necessary. + */ +#define RV_U_IMM_SIGN_OPOFF 31 +#define RV_U_IMM_31_12_OPOFF 0 +#define RV_U_IMM_31_12_MASK GENMASK(31, 12) + +/* The bit field of immediate value in B-type instruction */ +#define RV_B_IMM_SIGN_OPOFF 31 +#define RV_B_IMM_10_5_OPOFF 25 +#define RV_B_IMM_4_1_OPOFF 8 +#define RV_B_IMM_11_OPOFF 7 +#define RV_B_IMM_SIGN_OFF 12 +#define RV_B_IMM_10_5_OFF 5 +#define RV_B_IMM_4_1_OFF 1 +#define RV_B_IMM_11_OFF 11 +#define RV_B_IMM_10_5_MASK GENMASK(5, 0) +#define RV_B_IMM_4_1_MASK GENMASK(3, 0) +#define RV_B_IMM_11_MASK GENMASK(0, 0) + +/* The register offset in RVG instruction */ +#define RVG_RS1_OPOFF 15 +#define RVG_RS2_OPOFF 20 +#define RVG_RD_OPOFF 7 +#define RVG_RS1_MASK GENMASK(4, 0) +#define RVG_RS2_MASK GENMASK(4, 0) +#define RVG_RD_MASK GENMASK(4, 0) + +/* The bit field of immediate value in RVC J instruction */ +#define RVC_J_IMM_SIGN_OPOFF 12 +#define RVC_J_IMM_4_OPOFF 11 +#define RVC_J_IMM_9_8_OPOFF 9 +#define RVC_J_IMM_10_OPOFF 8 +#define RVC_J_IMM_6_OPOFF 7 +#define RVC_J_IMM_7_OPOFF 6 +#define RVC_J_IMM_3_1_OPOFF 3 +#define RVC_J_IMM_5_OPOFF 2 +#define RVC_J_IMM_SIGN_OFF 11 +#define RVC_J_IMM_4_OFF 4 +#define RVC_J_IMM_9_8_OFF 8 +#define RVC_J_IMM_10_OFF 10 +#define RVC_J_IMM_6_OFF 6 +#define RVC_J_IMM_7_OFF 7 +#define RVC_J_IMM_3_1_OFF 1 +#define RVC_J_IMM_5_OFF 5 +#define RVC_J_IMM_4_MASK GENMASK(0, 0) +#define RVC_J_IMM_9_8_MASK GENMASK(1, 0) +#define RVC_J_IMM_10_MASK GENMASK(0, 0) +#define RVC_J_IMM_6_MASK GENMASK(0, 0) +#define RVC_J_IMM_7_MASK GENMASK(0, 0) +#define RVC_J_IMM_3_1_MASK GENMASK(2, 0) +#define RVC_J_IMM_5_MASK GENMASK(0, 0) + +/* The bit field of immediate value in RVC B instruction */ +#define RVC_B_IMM_SIGN_OPOFF 12 +#define RVC_B_IMM_4_3_OPOFF 10 +#define RVC_B_IMM_7_6_OPOFF 5 +#define RVC_B_IMM_2_1_OPOFF 3 +#define RVC_B_IMM_5_OPOFF 2 +#define RVC_B_IMM_SIGN_OFF 8 +#define RVC_B_IMM_4_3_OFF 3 +#define RVC_B_IMM_7_6_OFF 6 +#define RVC_B_IMM_2_1_OFF 1 +#define RVC_B_IMM_5_OFF 5 +#define RVC_B_IMM_4_3_MASK GENMASK(1, 0) +#define RVC_B_IMM_7_6_MASK GENMASK(1, 0) +#define RVC_B_IMM_2_1_MASK GENMASK(1, 0) +#define RVC_B_IMM_5_MASK GENMASK(0, 0) + +#define RVC_INSN_FUNCT4_MASK GENMASK(15, 12) +#define RVC_INSN_FUNCT4_OPOFF 12 +#define RVC_INSN_FUNCT3_MASK GENMASK(15, 13) +#define RVC_INSN_FUNCT3_OPOFF 13 +#define RVC_INSN_J_RS1_MASK GENMASK(11, 7) +#define RVC_INSN_J_RS2_MASK GENMASK(6, 2) +#define RVC_INSN_OPCODE_MASK GENMASK(1, 0) +#define RVC_ENCODE_FUNCT3(f_) (RVC_FUNCT3_##f_ << RVC_INSN_FUNCT3_OPOFF) +#define RVC_ENCODE_FUNCT4(f_) (RVC_FUNCT4_##f_ << RVC_INSN_FUNCT4_OPOFF) + +/* The register offset in RVC op=C0 instruction */ +#define RVC_C0_RS1_OPOFF 7 +#define RVC_C0_RS2_OPOFF 2 +#define RVC_C0_RD_OPOFF 2 + +/* The register offset in RVC op=C1 instruction */ +#define RVC_C1_RS1_OPOFF 7 +#define RVC_C1_RS2_OPOFF 2 +#define RVC_C1_RD_OPOFF 7 + +/* The register offset in RVC op=C2 instruction */ +#define RVC_C2_RS1_OPOFF 7 +#define RVC_C2_RS2_OPOFF 2 +#define RVC_C2_RD_OPOFF 7 +#define RVC_C2_RS1_MASK GENMASK(4, 0) + +/* parts of opcode for RVG*/ +#define RVG_OPCODE_FENCE 0x0f +#define RVG_OPCODE_AUIPC 0x17 +#define RVG_OPCODE_BRANCH 0x63 +#define RVG_OPCODE_JALR 0x67 +#define RVG_OPCODE_JAL 0x6f +#define RVG_OPCODE_SYSTEM 0x73 +#define RVG_SYSTEM_CSR_OFF 20 +#define RVG_SYSTEM_CSR_MASK GENMASK(12, 0) + +/* parts of opcode for RVF, RVD and RVQ */ +#define RVFDQ_FL_FS_WIDTH_OFF 12 +#define RVFDQ_FL_FS_WIDTH_MASK GENMASK(2, 0) +#define RVFDQ_FL_FS_WIDTH_W 2 +#define RVFDQ_FL_FS_WIDTH_D 3 +#define RVFDQ_LS_FS_WIDTH_Q 4 +#define RVFDQ_OPCODE_FL 0x07 +#define RVFDQ_OPCODE_FS 0x27 + +/* parts of opcode for RVV */ +#define RVV_OPCODE_VECTOR 0x57 +#define RVV_VL_VS_WIDTH_8 0 +#define RVV_VL_VS_WIDTH_16 5 +#define RVV_VL_VS_WIDTH_32 6 +#define RVV_VL_VS_WIDTH_64 7 +#define RVV_OPCODE_VL RVFDQ_OPCODE_FL +#define RVV_OPCODE_VS RVFDQ_OPCODE_FS + +/* parts of opcode for RVC*/ +#define RVC_OPCODE_C0 0x0 +#define RVC_OPCODE_C1 0x1 +#define RVC_OPCODE_C2 0x2 + +/* parts of funct3 code for I, M, A extension*/ +#define RVG_FUNCT3_JALR 0x0 +#define RVG_FUNCT3_BEQ 0x0 +#define RVG_FUNCT3_BNE 0x1 +#define RVG_FUNCT3_BLT 0x4 +#define RVG_FUNCT3_BGE 0x5 +#define RVG_FUNCT3_BLTU 0x6 +#define RVG_FUNCT3_BGEU 0x7 + +/* parts of funct3 code for C extension*/ +#define RVC_FUNCT3_C_BEQZ 0x6 +#define RVC_FUNCT3_C_BNEZ 0x7 +#define RVC_FUNCT3_C_J 0x5 +#define RVC_FUNCT3_C_JAL 0x1 +#define RVC_FUNCT4_C_JR 0x8 +#define RVC_FUNCT4_C_JALR 0x9 +#define RVC_FUNCT4_C_EBREAK 0x9 + +#define RVG_FUNCT12_ECALL 0x0 +#define RVG_FUNCT12_EBREAK 0x1 +#define RVG_FUNCT12_SRET 0x102 +#define RVG_FUNCT12_MRET 0x303 + +#define RVG_MATCH_AUIPC (RVG_OPCODE_AUIPC) +#define RVG_MATCH_JALR (RV_ENCODE_FUNCT3(JALR) | RVG_OPCODE_JALR) +#define RVG_MATCH_JAL (RVG_OPCODE_JAL) +#define RVG_MATCH_FENCE (RVG_OPCODE_FENCE) +#define RVG_MATCH_BEQ (RV_ENCODE_FUNCT3(BEQ) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BNE (RV_ENCODE_FUNCT3(BNE) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BLT (RV_ENCODE_FUNCT3(BLT) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BGE (RV_ENCODE_FUNCT3(BGE) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BLTU (RV_ENCODE_FUNCT3(BLTU) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_BGEU (RV_ENCODE_FUNCT3(BGEU) | RVG_OPCODE_BRANCH) +#define RVG_MATCH_ECALL (RV_ENCODE_FUNCT12(EBREAK) | RVG_OPCODE_SYSTEM) +#define RVG_MATCH_EBREAK (RV_ENCODE_FUNCT12(EBREAK) | RVG_OPCODE_SYSTEM) +#define RVG_MATCH_SRET (RV_ENCODE_FUNCT12(SRET) | RVG_OPCODE_SYSTEM) +#define RVG_MATCH_MRET (RV_ENCODE_FUNCT12(MRET) | RVG_OPCODE_SYSTEM) +#define RVC_MATCH_C_BEQZ (RVC_ENCODE_FUNCT3(C_BEQZ) | RVC_OPCODE_C1) +#define RVC_MATCH_C_BNEZ (RVC_ENCODE_FUNCT3(C_BNEZ) | RVC_OPCODE_C1) +#define RVC_MATCH_C_J (RVC_ENCODE_FUNCT3(C_J) | RVC_OPCODE_C1) +#define RVC_MATCH_C_JAL (RVC_ENCODE_FUNCT3(C_JAL) | RVC_OPCODE_C1) +#define RVC_MATCH_C_JR (RVC_ENCODE_FUNCT4(C_JR) | RVC_OPCODE_C2) +#define RVC_MATCH_C_JALR (RVC_ENCODE_FUNCT4(C_JALR) | RVC_OPCODE_C2) +#define RVC_MATCH_C_EBREAK (RVC_ENCODE_FUNCT4(C_EBREAK) | RVC_OPCODE_C2) + +#define RVG_MASK_AUIPC (RV_INSN_OPCODE_MASK) +#define RVG_MASK_JALR (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_JAL (RV_INSN_OPCODE_MASK) +#define RVG_MASK_FENCE (RV_INSN_OPCODE_MASK) +#define RVC_MASK_C_JALR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_JR (RVC_INSN_FUNCT4_MASK | RVC_INSN_J_RS2_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_JAL (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_J (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) +#define RVG_MASK_BEQ (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BNE (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BLT (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BGE (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BLTU (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVG_MASK_BGEU (RV_INSN_FUNCT3_MASK | RV_INSN_OPCODE_MASK) +#define RVC_MASK_C_BEQZ (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_BNEZ (RVC_INSN_FUNCT3_MASK | RVC_INSN_OPCODE_MASK) +#define RVC_MASK_C_EBREAK 0xffff +#define RVG_MASK_EBREAK 0xffffffff +#define RVG_MASK_ECALL 0xffffffff +#define RVG_MASK_SRET 0xffffffff +#define RVG_MASK_MRET 0xffffffff + +#define __INSN_LENGTH_MASK _UL(0x3) +#define __INSN_LENGTH_GE_32 _UL(0x3) +#define __INSN_OPCODE_MASK _UL(0x7F) +#define __INSN_BRANCH_OPCODE _UL(RVG_OPCODE_BRANCH) + +#define __RISCV_INSN_FUNCS(name, mask, val) \ +static __always_inline bool riscv_insn_is_##name(u32 code) \ +{ \ + BUILD_BUG_ON(~(mask) & (val)); \ + return (code & (mask)) == (val); \ +} \ + +#if __riscv_xlen == 32 +/* C.JAL is an RV32C-only instruction */ +__RISCV_INSN_FUNCS(c_jal, RVC_MASK_C_JAL, RVC_MATCH_C_JAL) +#else +#define riscv_insn_is_c_jal(opcode) 0 +#endif +__RISCV_INSN_FUNCS(auipc, RVG_MASK_AUIPC, RVG_MATCH_AUIPC) +__RISCV_INSN_FUNCS(jalr, RVG_MASK_JALR, RVG_MATCH_JALR) +__RISCV_INSN_FUNCS(jal, RVG_MASK_JAL, RVG_MATCH_JAL) +__RISCV_INSN_FUNCS(c_j, RVC_MASK_C_J, RVC_MATCH_C_J) +__RISCV_INSN_FUNCS(beq, RVG_MASK_BEQ, RVG_MATCH_BEQ) +__RISCV_INSN_FUNCS(bne, RVG_MASK_BNE, RVG_MATCH_BNE) +__RISCV_INSN_FUNCS(blt, RVG_MASK_BLT, RVG_MATCH_BLT) +__RISCV_INSN_FUNCS(bge, RVG_MASK_BGE, RVG_MATCH_BGE) +__RISCV_INSN_FUNCS(bltu, RVG_MASK_BLTU, RVG_MATCH_BLTU) +__RISCV_INSN_FUNCS(bgeu, RVG_MASK_BGEU, RVG_MATCH_BGEU) +__RISCV_INSN_FUNCS(c_beqz, RVC_MASK_C_BEQZ, RVC_MATCH_C_BEQZ) +__RISCV_INSN_FUNCS(c_bnez, RVC_MASK_C_BNEZ, RVC_MATCH_C_BNEZ) +__RISCV_INSN_FUNCS(c_ebreak, RVC_MASK_C_EBREAK, RVC_MATCH_C_EBREAK) +__RISCV_INSN_FUNCS(ebreak, RVG_MASK_EBREAK, RVG_MATCH_EBREAK) +__RISCV_INSN_FUNCS(ecall, RVG_MASK_EBREAK, RVG_MATCH_EBREAK) +__RISCV_INSN_FUNCS(sret, RVG_MASK_SRET, RVG_MATCH_SRET) +__RISCV_INSN_FUNCS(mret, RVG_MASK_MRET, RVG_MATCH_MRET) +__RISCV_INSN_FUNCS(fence, RVG_MASK_FENCE, RVG_MATCH_FENCE); + +/* special case to catch _any_ system instruction */ +static __always_inline bool riscv_insn_is_system(u32 code) +{ + return (code & RV_INSN_OPCODE_MASK) == RVG_OPCODE_SYSTEM; +} + +/* special case to catch _any_ branch instruction */ +static __always_inline bool riscv_insn_is_branch(u32 code) +{ + return (code & RV_INSN_OPCODE_MASK) == RVG_OPCODE_BRANCH; +} + +static __always_inline bool riscv_insn_is_c_jr(u32 code) +{ + return (code & RVC_MASK_C_JR) == RVC_MATCH_C_JR && + (code & RVC_INSN_J_RS1_MASK) != 0; +} + +static __always_inline bool riscv_insn_is_c_jalr(u32 code) +{ + return (code & RVC_MASK_C_JALR) == RVC_MATCH_C_JALR && + (code & RVC_INSN_J_RS1_MASK) != 0; +} + +#define INSN_MATCH_LB 0x3 +#define INSN_MASK_LB 0x707f +#define INSN_MATCH_LH 0x1003 +#define INSN_MASK_LH 0x707f +#define INSN_MATCH_LW 0x2003 +#define INSN_MASK_LW 0x707f +#define INSN_MATCH_LD 0x3003 +#define INSN_MASK_LD 0x707f +#define INSN_MATCH_LBU 0x4003 +#define INSN_MASK_LBU 0x707f +#define INSN_MATCH_LHU 0x5003 +#define INSN_MASK_LHU 0x707f +#define INSN_MATCH_LWU 0x6003 +#define INSN_MASK_LWU 0x707f +#define INSN_MATCH_SB 0x23 +#define INSN_MASK_SB 0x707f +#define INSN_MATCH_SH 0x1023 +#define INSN_MASK_SH 0x707f +#define INSN_MATCH_SW 0x2023 +#define INSN_MASK_SW 0x707f +#define INSN_MATCH_SD 0x3023 +#define INSN_MASK_SD 0x707f + +#define INSN_MATCH_C_LD 0x6000 +#define INSN_MASK_C_LD 0xe003 +#define INSN_MATCH_C_SD 0xe000 +#define INSN_MASK_C_SD 0xe003 +#define INSN_MATCH_C_LW 0x4000 +#define INSN_MASK_C_LW 0xe003 +#define INSN_MATCH_C_SW 0xc000 +#define INSN_MASK_C_SW 0xe003 +#define INSN_MATCH_C_LDSP 0x6002 +#define INSN_MASK_C_LDSP 0xe003 +#define INSN_MATCH_C_SDSP 0xe002 +#define INSN_MASK_C_SDSP 0xe003 +#define INSN_MATCH_C_LWSP 0x4002 +#define INSN_MASK_C_LWSP 0xe003 +#define INSN_MATCH_C_SWSP 0xc002 +#define INSN_MASK_C_SWSP 0xe003 + +#define INSN_OPCODE_MASK 0x007c +#define INSN_OPCODE_SHIFT 2 +#define INSN_OPCODE_SYSTEM 28 + +#define INSN_MASK_WFI 0xffffffff +#define INSN_MATCH_WFI 0x10500073 + +#define INSN_MASK_WRS 0xffffffff +#define INSN_MATCH_WRS 0x00d00073 + +#define INSN_MATCH_CSRRW 0x1073 +#define INSN_MASK_CSRRW 0x707f +#define INSN_MATCH_CSRRS 0x2073 +#define INSN_MASK_CSRRS 0x707f +#define INSN_MATCH_CSRRC 0x3073 +#define INSN_MASK_CSRRC 0x707f +#define INSN_MATCH_CSRRWI 0x5073 +#define INSN_MASK_CSRRWI 0x707f +#define INSN_MATCH_CSRRSI 0x6073 +#define INSN_MASK_CSRRSI 0x707f +#define INSN_MATCH_CSRRCI 0x7073 +#define INSN_MASK_CSRRCI 0x707f + +#define INSN_MATCH_FLW 0x2007 +#define INSN_MASK_FLW 0x707f +#define INSN_MATCH_FLD 0x3007 +#define INSN_MASK_FLD 0x707f +#define INSN_MATCH_FLQ 0x4007 +#define INSN_MASK_FLQ 0x707f +#define INSN_MATCH_FSW 0x2027 +#define INSN_MASK_FSW 0x707f +#define INSN_MATCH_FSD 0x3027 +#define INSN_MASK_FSD 0x707f +#define INSN_MATCH_FSQ 0x4027 +#define INSN_MASK_FSQ 0x707f + +#define INSN_MATCH_C_FLD 0x2000 +#define INSN_MASK_C_FLD 0xe003 +#define INSN_MATCH_C_FLW 0x6000 +#define INSN_MASK_C_FLW 0xe003 +#define INSN_MATCH_C_FSD 0xa000 +#define INSN_MASK_C_FSD 0xe003 +#define INSN_MATCH_C_FSW 0xe000 +#define INSN_MASK_C_FSW 0xe003 +#define INSN_MATCH_C_FLDSP 0x2002 +#define INSN_MASK_C_FLDSP 0xe003 +#define INSN_MATCH_C_FSDSP 0xa002 +#define INSN_MASK_C_FSDSP 0xe003 +#define INSN_MATCH_C_FLWSP 0x6002 +#define INSN_MASK_C_FLWSP 0xe003 +#define INSN_MATCH_C_FSWSP 0xe002 +#define INSN_MASK_C_FSWSP 0xe003 + +#define INSN_MATCH_C_LHU 0x8400 +#define INSN_MASK_C_LHU 0xfc43 +#define INSN_MATCH_C_LH 0x8440 +#define INSN_MASK_C_LH 0xfc43 +#define INSN_MATCH_C_SH 0x8c00 +#define INSN_MASK_C_SH 0xfc43 + +#define INSN_16BIT_MASK 0x3 +#define INSN_IS_16BIT(insn) (((insn) & INSN_16BIT_MASK) != INSN_16BIT_MASK) +#define INSN_LEN(insn) (INSN_IS_16BIT(insn) ? 2 : 4) + +#define SHIFT_RIGHT(x, y) \ + ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) + +#define REG_MASK \ + ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) + +#define REG_OFFSET(insn, pos) \ + (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) + +#define REG_PTR(insn, pos, regs) \ + ((ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))) + +#define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) +#define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) +#define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) +#define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) +#define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) +#define GET_SP(regs) (*REG_PTR(2, 0, regs)) +#define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) +#define IMM_I(insn) ((s32)(insn) >> 20) +#define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ + (s32)(((insn) >> 7) & 0x1f)) + +#define SH_RD 7 +#define SH_RS1 15 +#define SH_RS2 20 +#define SH_RS2C 2 +#define MASK_RX 0x1f + +#if defined(CONFIG_64BIT) +#define LOG_REGBYTES 3 +#else +#define LOG_REGBYTES 2 +#endif + +#define MASK_FUNCT3 0x7000 + +#define GET_FUNCT3(insn) (((insn) >> 12) & 7) + +#define RV_IMM_SIGN(x) (-(((x) >> 31) & 1)) +#define RVC_IMM_SIGN(x) (-(((x) >> 12) & 1)) +#define RV_X_MASK(X, s, mask) (((X) >> (s)) & (mask)) +#define RV_X(X, s, n) RV_X_MASK(X, s, ((1 << (n)) - 1)) +#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ + (RV_X(x, 10, 3) << 3) | \ + (RV_X(x, 5, 1) << 6)) +#define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ + (RV_X(x, 5, 2) << 6)) +#define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ + (RV_X(x, 12, 1) << 5) | \ + (RV_X(x, 2, 2) << 6)) +#define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ + (RV_X(x, 12, 1) << 5) | \ + (RV_X(x, 2, 3) << 6)) +#define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ + (RV_X(x, 7, 2) << 6)) +#define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ + (RV_X(x, 7, 3) << 6)) +#define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) +#define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) +#define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) +#define RVC_X(X, s, mask) RV_X_MASK(X, s, mask) + +#define RV_EXTRACT_FUNCT3(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RV_INSN_FUNCT3_OPOFF, \ + RV_INSN_FUNCT3_MASK >> RV_INSN_FUNCT3_OPOFF)); }) + +#define RV_EXTRACT_RS1_REG(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RVG_RS1_OPOFF, RVG_RS1_MASK)); }) + +#define RV_EXTRACT_RS2_REG(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RVG_RS2_OPOFF, RVG_RS2_MASK)); }) + +#define RV_EXTRACT_RD_REG(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RVG_RD_OPOFF, RVG_RD_MASK)); }) + +#define RV_EXTRACT_UTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RV_U_IMM_31_12_OPOFF, RV_U_IMM_31_12_MASK)); }) + +#define RV_EXTRACT_JTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RV_J_IMM_10_1_OPOFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OFF) | \ + (RV_X_MASK(x_, RV_J_IMM_11_OPOFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OFF) | \ + (RV_X_MASK(x_, RV_J_IMM_19_12_OPOFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OFF) | \ + (RV_IMM_SIGN(x_) << RV_J_IMM_SIGN_OFF); }) + +#define RV_EXTRACT_ITYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RV_I_IMM_11_0_OPOFF, RV_I_IMM_11_0_MASK)) | \ + (RV_IMM_SIGN(x_) << RV_I_IMM_SIGN_OFF); }) + +#define RV_EXTRACT_BTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RV_B_IMM_4_1_OPOFF, RV_B_IMM_4_1_MASK) << RV_B_IMM_4_1_OFF) | \ + (RV_X_MASK(x_, RV_B_IMM_10_5_OPOFF, RV_B_IMM_10_5_MASK) << RV_B_IMM_10_5_OFF) | \ + (RV_X_MASK(x_, RV_B_IMM_11_OPOFF, RV_B_IMM_11_MASK) << RV_B_IMM_11_OFF) | \ + (RV_IMM_SIGN(x_) << RV_B_IMM_SIGN_OFF); }) + +#define RVC_EXTRACT_C2_RS1_REG(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RVC_C2_RS1_OPOFF, RVC_C2_RS1_MASK)); }) + +#define RVC_EXTRACT_JTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RVC_X(x_, RVC_J_IMM_3_1_OPOFF, RVC_J_IMM_3_1_MASK) << RVC_J_IMM_3_1_OFF) | \ + (RVC_X(x_, RVC_J_IMM_4_OPOFF, RVC_J_IMM_4_MASK) << RVC_J_IMM_4_OFF) | \ + (RVC_X(x_, RVC_J_IMM_5_OPOFF, RVC_J_IMM_5_MASK) << RVC_J_IMM_5_OFF) | \ + (RVC_X(x_, RVC_J_IMM_6_OPOFF, RVC_J_IMM_6_MASK) << RVC_J_IMM_6_OFF) | \ + (RVC_X(x_, RVC_J_IMM_7_OPOFF, RVC_J_IMM_7_MASK) << RVC_J_IMM_7_OFF) | \ + (RVC_X(x_, RVC_J_IMM_9_8_OPOFF, RVC_J_IMM_9_8_MASK) << RVC_J_IMM_9_8_OFF) | \ + (RVC_X(x_, RVC_J_IMM_10_OPOFF, RVC_J_IMM_10_MASK) << RVC_J_IMM_10_OFF) | \ + (RVC_IMM_SIGN(x_) << RVC_J_IMM_SIGN_OFF); }) + +#define RVC_EXTRACT_BTYPE_IMM(x) \ + ({typeof(x) x_ = (x); \ + (RVC_X(x_, RVC_B_IMM_2_1_OPOFF, RVC_B_IMM_2_1_MASK) << RVC_B_IMM_2_1_OFF) | \ + (RVC_X(x_, RVC_B_IMM_4_3_OPOFF, RVC_B_IMM_4_3_MASK) << RVC_B_IMM_4_3_OFF) | \ + (RVC_X(x_, RVC_B_IMM_5_OPOFF, RVC_B_IMM_5_MASK) << RVC_B_IMM_5_OFF) | \ + (RVC_X(x_, RVC_B_IMM_7_6_OPOFF, RVC_B_IMM_7_6_MASK) << RVC_B_IMM_7_6_OFF) | \ + (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) + +#define RVG_EXTRACT_SYSTEM_CSR(x) \ + ({typeof(x) x_ = (x); RV_X_MASK(x_, RVG_SYSTEM_CSR_OFF, RVG_SYSTEM_CSR_MASK); }) + +#define RVFDQ_EXTRACT_FL_FS_WIDTH(x) \ + ({typeof(x) x_ = (x); RV_X_MASK(x_, RVFDQ_FL_FS_WIDTH_OFF, \ + RVFDQ_FL_FS_WIDTH_MASK); }) + +#define RVV_EXTRACT_VL_VS_WIDTH(x) RVFDQ_EXTRACT_FL_FS_WIDTH(x) + +/* + * Get the immediate from a J-type instruction. + * + * @insn: instruction to process + * Return: immediate + */ +static inline s32 riscv_insn_extract_jtype_imm(u32 insn) +{ + return RV_EXTRACT_JTYPE_IMM(insn); +} + +/* + * Update a J-type instruction with an immediate value. + * + * @insn: pointer to the jtype instruction + * @imm: the immediate to insert into the instruction + */ +static inline void riscv_insn_insert_jtype_imm(u32 *insn, s32 imm) +{ + /* drop the old IMMs, all jal IMM bits sit at 31:12 */ + *insn &= ~GENMASK(31, 12); + *insn |= (RV_X_MASK(imm, RV_J_IMM_10_1_OFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OPOFF) | + (RV_X_MASK(imm, RV_J_IMM_11_OFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OPOFF) | + (RV_X_MASK(imm, RV_J_IMM_19_12_OFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OPOFF) | + (RV_X_MASK(imm, RV_J_IMM_SIGN_OFF, 1) << RV_J_IMM_SIGN_OPOFF); +} + +/* + * Put together one immediate from a U-type and I-type instruction pair. + * + * The U-type contains an upper immediate, meaning bits[31:12] with [11:0] + * being zero, while the I-type contains a 12bit immediate. + * Combined these can encode larger 32bit values and are used for example + * in auipc + jalr pairs to allow larger jumps. + * + * @utype_insn: instruction containing the upper immediate + * @itype_insn: instruction + * Return: combined immediate + */ +static inline s32 riscv_insn_extract_utype_itype_imm(u32 utype_insn, u32 itype_insn) +{ + s32 imm; + + imm = RV_EXTRACT_UTYPE_IMM(utype_insn); + imm += RV_EXTRACT_ITYPE_IMM(itype_insn); + + return imm; +} + +/* + * Update a set of two instructions (U-type + I-type) with an immediate value. + * + * Used for example in auipc+jalrs pairs the U-type instructions contains + * a 20bit upper immediate representing bits[31:12], while the I-type + * instruction contains a 12bit immediate representing bits[11:0]. + * + * This also takes into account that both separate immediates are + * considered as signed values, so if the I-type immediate becomes + * negative (BIT(11) set) the U-type part gets adjusted. + * + * @utype_insn: pointer to the utype instruction of the pair + * @itype_insn: pointer to the itype instruction of the pair + * @imm: the immediate to insert into the two instructions + */ +static inline void riscv_insn_insert_utype_itype_imm(u32 *utype_insn, u32 *itype_insn, s32 imm) +{ + /* drop possible old IMM values */ + *utype_insn &= ~(RV_U_IMM_31_12_MASK); + *itype_insn &= ~(RV_I_IMM_11_0_MASK << RV_I_IMM_11_0_OPOFF); + + /* add the adapted IMMs */ + *utype_insn |= (imm & RV_U_IMM_31_12_MASK) + ((imm & BIT(11)) << 1); + *itype_insn |= ((imm & RV_I_IMM_11_0_MASK) << RV_I_IMM_11_0_OPOFF); +} + +/* + * Get the immediate from a B-type instruction. + * + * @insn: instruction to process + * Return: immediate + */ +static inline s32 riscv_insn_extract_btype_imm(u32 insn) +{ + return RV_EXTRACT_BTYPE_IMM(insn); +} + +/* + * Get the immediate from a RVC B-type instruction. + * + * @insn: instruction to process + * Return: immediate + */ +static inline s32 riscv_insn_extract_cbtype_imm(u32 insn) +{ + return RVC_EXTRACT_BTYPE_IMM(insn); +} + +/* + * Get the immediate from a RVC J-type instruction. + * + * @insn: instruction to process + * Return: immediate + */ +static inline s32 riscv_insn_extract_cjtype_imm(u16 insn) +{ + return RVC_EXTRACT_JTYPE_IMM(insn); +} + +#endif /* _ASM_RISCV_INSN_H */ diff --git a/tools/perf/util/Build b/tools/perf/util/Build index d25140a0d64f..342b4f75457b 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -150,6 +150,7 @@ perf-util-y += cs-etm-decoder/ endif perf-util-y += cs-etm-base.o perf-util-y += gtrace-decoder.o +perf-util-y += rvtrace-decoder/

perf-util-y += parse-branch-options.o perf-util-y += parse-regs-options.o diff --git a/tools/perf/util/rvtrace-decoder/Build b/tools/perf/util/rvtrace-decoder/Build new file mode 100644 index 000000000000..dc447eec372a --- /dev/null +++ b/tools/perf/util/rvtrace-decoder/Build @@ -0,0 +1,2 @@ +perf-util-y += rvtrace-decoder.o +perf-util-y += rvtrace-ntrace-decoder.o diff --git a/tools/perf/util/rvtrace-decoder/rvtrace-decoder.c b/tools/perf/util/rvtrace-decoder/rvtrace-decoder.c new file mode 100644 index 000000000000..e4b54fe8f51a --- /dev/null +++ b/tools/perf/util/rvtrace-decoder/rvtrace-decoder.c @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 IAR Systems AB. + * Copyright (c) 2026 Spacemit Technologies, Inc. + */ + +#include <linux/err.h> +#include <linux/zalloc.h> +#include <linux/types.h> +#include <errno.h> +#include <api/fs/fs.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "../debug.h" +#include "../intlist.h" +#include "rvtrace-decoder.h" + +#define BUFFER_SIZE 1024 + +/* + * Shared by the per-source staging buffer used during CoreSight frame + * de-formatting and by the contiguous de-formatted output. + */ +static int rvtrace_buf_append(struct rvtrace_defmt_buf *b, const void *src, size_t len) +{ + if (b->size + len > b->capacity) { + size_t new_cap = b->capacity ? b->capacity : BUFFER_SIZE; + unsigned char *p; + + while (new_cap < b->size + len) + new_cap *= 2; + + p = realloc(b->buf, new_cap); + if (!p) + return -ENOMEM; + b->buf = p; + b->capacity = new_cap; + } + memcpy(b->buf + b->size, src, len); + b->size += len; + return 0; +} + +static struct rvtrace_defmt_buf *rvtrace_get_buf(struct rvtrace_defmt_buf defmt_bufs[], + unsigned char id) +{ + if (defmt_bufs[id].buf != NULL) + return &defmt_bufs[id]; + + defmt_bufs[id].buf = (unsigned char *)malloc(BUFFER_SIZE); + if (!defmt_bufs[id].buf) + return NULL; + + defmt_bufs[id].size = 0; + defmt_bufs[id].capacity = BUFFER_SIZE; + + return &defmt_bufs[id]; +} + +static int rvtrace_defmt_buf_append(struct rvtrace_defmt_buf *defmt_buf, + struct rvtrace_defmt_buf *out, unsigned char data) +{ + int err; + + err = rvtrace_buf_append(defmt_buf, &data, 1); + if (err) + return err; + + /* A Nexus message ends when the MSEO bits of the last byte are '11'. */ + if ((data & 0x3) == 0x3) { + err = rvtrace_buf_append(out, defmt_buf->buf, defmt_buf->size); + if (err) { + pr_err("Encoder: failed to write nexus data\n"); + return err; + } + defmt_buf->size = 0; + } + + return 0; +} + +static size_t skip_frame_syncs(const unsigned char *buf, size_t len, size_t *fsync_count) +{ + const uint32_t FSYNC_PATTERN = 0x7FFFFFFF; /* LE host pattern for Frame SYNC */ + size_t skipped_bytes = 0; + + while (skipped_bytes + 4 <= len) { + if (*((uint32_t *)(buf + skipped_bytes)) == FSYNC_PATTERN) { + skipped_bytes += 4; + (*fsync_count)++; + } else { + break; + } + } + + return skipped_bytes; +} + +/* remove coresight formatter frame */ +static int rvtrace_data_defmt(struct rvtrace_defmt_buf defmt_bufs[], + struct rvtrace_defmt_buf *out, + const unsigned char *buf, size_t len) +{ + int err; + uint8_t flag_bit; + unsigned char data = 0; + int cur_id = -1, old_id = -1, new_id = -1; + + /* 16 bytes is output by coresight trace formatter */ + while (len >= 16) { + /* some linux drivers (e.g. for perf) will insert FSYNCS to pad or differentiate + * between blocks of aligned data, always in frame aligned complete 16 byte frames. + * we need to skip past these frames, resetting as we go. + */ + size_t fsync_count = 0; + size_t fsync_bytes = skip_frame_syncs(buf, len, &fsync_count); + + if (fsync_bytes > 0) { + if (fsync_count % 4 == 0) { + cur_id = -1; + old_id = -1; + new_id = -1; + } else { + pr_err("Incorrect FSYNC reset pattern\n"); + return -EINVAL; + } + buf += fsync_bytes; + len -= fsync_bytes; + continue; + } + + flag_bit = 0x1; + for (int i = 0; i < 15; i += 2) { + if (buf[i] & 0x1) { + /* it's id */ + old_id = new_id; + new_id = buf[i] >> 1; /* get new_id */ + cur_id = (flag_bit & buf[15]) ? old_id : new_id; + } else { + /* it's data */ + data = buf[i] | ((flag_bit & buf[15]) ? 0x1 : 0x0); + } + + if (IS_VALID_ID(cur_id)) { + struct rvtrace_defmt_buf *defmt_buf = + rvtrace_get_buf(defmt_bufs, cur_id); + + if (!defmt_buf) + return -ENOMEM; + + /* it's data */ + if ((buf[i] & 0x1) == 0) { + err = rvtrace_defmt_buf_append(defmt_buf, out, data); + if (err) + return err; + } + + /* buf[i+1] is alway data when i != 14 */ + if (i != 14) { + err = rvtrace_defmt_buf_append(defmt_buf, out, + buf[i + 1]); + if (err) + return err; + } + } + + if (cur_id != new_id) + cur_id = new_id; + + flag_bit <<= 1; + } + buf += 16; + len -= 16; + } + return 0; +} + +/* + * Common processing for both dump and decode: de-format the raw AUX buffer + * into a contiguous Nexus byte stream (in memory), then dispatch to the + * format specific parser based on 'type'. + */ +static int rvtrace_decode(struct rvtrace_decoder *decoder, + enum rvtrace_decode_type type, + const unsigned char *buf, size_t len) +{ + int err; + const unsigned char *data = buf; + size_t size = len; + struct rvtrace_defmt_buf out = { 0 }; + + if (decoder->coresight_formatted) { + err = rvtrace_data_defmt(decoder->defmt_bufs, &out, buf, len); + if (err) { + pr_err("Encoder: failed to remove coresight formatter frame\n"); + free(out.buf); + return err; + } + data = out.buf; + size = out.size; + } + + switch (decoder->format) { + case GTRACE_FORMAT_NTRACE: + err = rvtrace_ntrace_decode(decoder, type, data, size); + break; + case GTRACE_FORMAT_ETRACE: + default: + err = -EINVAL; + pr_err("unsupported trace format %d\n", decoder->format); + break; + } + + free(out.buf); + return err; +} + +static int rvtrace_init_packet_buffer(struct rvtrace_packet_buffer *packet_buffer) +{ + if (!packet_buffer->packets) { + packet_buffer->packets = calloc(BUFFER_SIZE, sizeof(struct rvtrace_packet)); + if (!packet_buffer->packets) + return -ENOMEM; + } + + packet_buffer->size = 0; + packet_buffer->capacity = BUFFER_SIZE; + + return 0; +} + +static void rvtrace_free_packet_buffer(struct rvtrace_packet_buffer *packet_buffer) +{ + free(packet_buffer->packets); +} + +int rvtrace_packet_buffer_append(struct rvtrace_packet_buffer *packet_buffer, + struct rvtrace_packet packet) +{ + if (packet_buffer->size == packet_buffer->capacity) { + packet_buffer->capacity *= 2; + packet_buffer->packets = realloc(packet_buffer->packets, + sizeof(struct rvtrace_packet) * packet_buffer->capacity); + if (!packet_buffer->packets) + return -ENOMEM; + } + packet_buffer->packets[packet_buffer->size++] = packet; + + return 0; +} + +int rvtrace_init_stack(struct rvtrace_stack *stack) +{ + if (!stack->data) { + stack->data = (u64 *)malloc(sizeof(u64) * BUFFER_SIZE); + if (!stack->data) + return -ENOMEM; + } + + stack->top = -1; + stack->capacity = BUFFER_SIZE; + + return 0; +} + +int rvtrace_stack_push(struct rvtrace_stack *stack, u64 value) +{ + if (stack->top == stack->capacity - 1) { + stack->capacity *= 2; + stack->data = (u64 *)realloc(stack->data, sizeof(u64) * stack->capacity); + if (!stack->data) + return -ENOMEM; + } + stack->data[++stack->top] = value; + + return 0; +} + +int rvtrace_stack_pop(struct rvtrace_stack *stack) +{ + u64 value; + + if (stack->top == -1) + return 1; /* Empty */ + + value = stack->data[stack->top--]; + return value; +} + +static void rvtrace_free_stack(struct rvtrace_stack *stack) +{ + free(stack->data); +} + +struct rvtrace_src_context *rvtrace_get_src_context(struct rvtrace_decoder *decoder, int srcid) +{ + struct int_node *node; + struct rvtrace_src_context *ctx; + + node = intlist__find(decoder->src_contexts, srcid); + if (node) + return (struct rvtrace_src_context *)node->priv; + + ctx = zalloc(sizeof(struct rvtrace_src_context)); + if (!ctx) + return NULL; + + ctx->srcid = srcid; + ctx->nexdeco_pc = 1; + ctx->nexdeco_lastaddr = 1; + ctx->prv = 0; + ctx->v = 0; + ctx->timestamp = 0; + ctx->context = -1; + ctx->resourcefull_icnt = 0; + + if (rvtrace_init_stack(&ctx->stack)) { + free(ctx); + return NULL; + } + + /* Add to RB Tree */ + node = intlist__findnew(decoder->src_contexts, srcid); + if (!node) { + rvtrace_free_stack(&ctx->stack); + free(ctx); + return NULL; + } + + node->priv = ctx; + + return ctx; +} + +static void rvtrace_free_src_contexts(struct intlist *src_contexts) +{ + struct int_node *node; + + intlist__for_each_entry(node, src_contexts) { + struct rvtrace_src_context *ctx = (struct rvtrace_src_context *)node->priv; + + rvtrace_free_stack(&ctx->stack); + free(ctx); + } + intlist__delete(src_contexts); +} + +struct rvtrace_decoder *rvtrace_decoder_new(struct rvtrace_decoder_params *params) +{ + int err; + struct rvtrace_decoder *decoder; + + if (!params) + return NULL; + + decoder = zalloc(sizeof(struct rvtrace_decoder)); + if (!decoder) + return NULL; + + decoder->data = params->data; + decoder->coresight_formatted = params->coresight_formatted; + decoder->format = params->format; + decoder->srcbits = params->srcbits; + decoder->mem_access = params->mem_access; + decoder->decode = rvtrace_decode; + decoder->current_src = -1; + decoder->src_contexts = intlist__new(NULL); + if (!decoder->src_contexts) + goto err_out_free_decoder; + + err = rvtrace_init_packet_buffer(&decoder->packet_buffer); + if (err) + goto err_out_free_context; + + return decoder; + +err_out_free_context: + rvtrace_free_src_contexts(decoder->src_contexts); +err_out_free_decoder: + zfree(&decoder); + return NULL; +} + +void rvtrace_decoder_free(struct rvtrace_decoder *decoder) +{ + for (int i = 0; i < MAX_ID; ++i) + free(decoder->defmt_bufs[i].buf); + + rvtrace_free_packet_buffer(&decoder->packet_buffer); + rvtrace_free_src_contexts(decoder->src_contexts); + zfree(&decoder); +} + +int rvtrace_decoder_reset(struct rvtrace_decoder *decoder) +{ + int err; + struct int_node *node; + struct rvtrace_src_context *ctx; + + if (decoder->src_contexts) { + intlist__for_each_entry(node, decoder->src_contexts) { + ctx = (struct rvtrace_src_context *)node->priv; + ctx->nexdeco_pc = 1; + ctx->nexdeco_lastaddr = 1; + ctx->prv = 0; + ctx->v = 0; + ctx->timestamp = 0; + ctx->context = -1; + ctx->resourcefull_icnt = 0; + rvtrace_init_stack(&ctx->stack); + } + } + decoder->current_src_ctx = NULL; + decoder->current_src = -1; + + err = rvtrace_init_packet_buffer(&decoder->packet_buffer); + if (err) + return err; + + return 0; +} diff --git a/tools/perf/util/rvtrace-decoder/rvtrace-decoder.h b/tools/perf/util/rvtrace-decoder/rvtrace-decoder.h new file mode 100644 index 000000000000..27c27d5fc3b8 --- /dev/null +++ b/tools/perf/util/rvtrace-decoder/rvtrace-decoder.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Spacemit Technologies, Inc. + */ + +#ifndef INCLUDE__RVTRACE_DECODER_H__ +#define INCLUDE__RVTRACE_DECODER_H__ + +#include "../gtrace.h" +#include "rvtrace-ntrace-decoder.h" + +#define MAX_ID 112 + +/* check an ID is in the valid range */ +#define IS_VALID_ID(id) \ + ((id > 0) && (id < MAX_ID)) + +struct rvtrace_defmt_buf { + unsigned char *buf; + size_t size; + size_t capacity; +}; + +struct rvtrace_stack { + u64 *data; + int top; + int capacity; +}; + +enum rvtrace_sample_type { + RVTRACE_EMPTY, + RVTRACE_RANGE, + RVTRACE_LOSS, + RVTRACE_ERROR, +}; + +enum riscv_privilege_mode { + RISCV_PRIV_USER_MODE, + RISCV_PRIV_SUPERVISOR_MODE, + RISCV_PRIV_MACHINE_MODE = 3, +}; + +typedef u32 (*rvtrace_mem_access_type)(void *, u64, enum riscv_privilege_mode, + int, size_t, u8 *); + +struct rvtrace_packet { + enum rvtrace_sample_type sample_type; + u64 start_addr; + u64 end_addr; + u32 insn_cnt; + int cpu; + enum riscv_privilege_mode prv; + bool v; + int context; + u64 timestamp; +}; + +struct rvtrace_packet_buffer { + struct rvtrace_packet *packets; + int size; + int capacity; +}; + +struct rvtrace_src_context { + int srcid; + u64 nexdeco_pc; + u64 nexdeco_lastaddr; + u64 current_pc; + u64 timestamp; + enum riscv_privilege_mode prv; + bool v; + int context; + int resourcefull_icnt; + struct rvtrace_stack stack; +}; + +struct rvtrace_decoder_params { + void *data; + bool coresight_formatted; + enum gtrace_format format; + u32 srcbits; + rvtrace_mem_access_type mem_access; +}; + +/* What the decoder should do with the de-formatted trace stream. */ +enum rvtrace_decode_type { + RVTRACE_DECODE_MSG, /* print the Nexus messages */ + RVTRACE_DECODE_INSN, /* reconstruct instruction trace packets */ +}; + +struct rvtrace_decoder { + void *data; + bool coresight_formatted; + enum gtrace_format format; + u32 srcbits; + rvtrace_mem_access_type mem_access; + int (*decode)(struct rvtrace_decoder *decoder, enum rvtrace_decode_type type, + const unsigned char *buf, size_t size); + struct rvtrace_defmt_buf defmt_bufs[MAX_ID]; + int current_src; + struct intlist *src_contexts; + struct rvtrace_src_context *current_src_ctx; + struct rvtrace_packet_buffer packet_buffer; + struct rvtrace_ntrace_msg msg; /* most recently parsed message */ + struct rvtrace_ntrace_msg saved_msg; /* previous message, for RepeatBranch replay */ +}; + +int rvtrace_packet_buffer_append(struct rvtrace_packet_buffer *packet_buffer, + struct rvtrace_packet packet); +int rvtrace_init_stack(struct rvtrace_stack *stack); +int rvtrace_stack_push(struct rvtrace_stack *stack, u64 value); +int rvtrace_stack_pop(struct rvtrace_stack *stack); +struct rvtrace_src_context * +rvtrace_get_src_context(struct rvtrace_decoder *decoder, int srcid); +struct rvtrace_decoder *rvtrace_decoder_new(struct rvtrace_decoder_params *params); +void rvtrace_decoder_free(struct rvtrace_decoder *decoder); +int rvtrace_decoder_reset(struct rvtrace_decoder *decoder); + +int rvtrace_ntrace_decode(struct rvtrace_decoder *decoder, enum rvtrace_decode_type type, + const unsigned char *buf, size_t size); + +#endif diff --git a/tools/perf/util/rvtrace-decoder/rvtrace-ntrace-decoder.c b/tools/perf/util/rvtrace-decoder/rvtrace-ntrace-decoder.c new file mode 100644 index 000000000000..f72e5baa323e --- /dev/null +++ b/tools/perf/util/rvtrace-decoder/rvtrace-ntrace-decoder.c @@ -0,0 +1,910 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 IAR Systems AB. + * Copyright (c) 2026 Spacemit Technologies, Inc. + */ + +#include <linux/err.h> +#include <linux/zalloc.h> +#include <linux/types.h> +#include <errno.h> +#include <api/fs/fs.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "../debug.h" +#include "../color.h" +#include "../intlist.h" +#include "rvtrace-decoder.h" +#include "rvtrace-ntrace-decoder.h" +#include "../../../arch/riscv/include/asm/insn.h" + +#define RV_REGNO_ZERO 0 +#define RV_REGNO_RA 1 + +/* Definition of Nexus Messages (subset applicable to RISC-V PC trace). */ +const struct rvtrace_ntrace_msg_field_def rvtrace_ntrace_msg_defs[] = { + NEXM_BEG(Ownership), + NEXM_FLD_PAR(SRC), + NEXM_FLD(FORMAT), + NEXM_FLD(PRV), + NEXM_FLD(V), + NEXM_VAR(CONTEXT), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(DirectBranch), + NEXM_FLD_PAR(SRC), + NEXM_VAR(ICNT), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(IndirectBranch), + NEXM_FLD_PAR(SRC), + NEXM_FLD(BTYPE), + NEXM_VAR(ICNT), + NEXM_ADR(UADDR), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(Error), + NEXM_FLD_PAR(SRC), + NEXM_FLD(ETYPE), + NEXM_VAR(PAD), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(ProgTraceSync), + NEXM_FLD_PAR(SRC), + NEXM_FLD(SYNC), + NEXM_VAR(ICNT), + NEXM_ADR(FADDR), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(DirectBranchSync), + NEXM_FLD_PAR(SRC), + NEXM_FLD(SYNC), + NEXM_VAR(ICNT), + NEXM_ADR(FADDR), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(IndirectBranchSync), + NEXM_FLD_PAR(SRC), + NEXM_FLD(SYNC), + NEXM_FLD(BTYPE), + NEXM_VAR(ICNT), + NEXM_ADR(FADDR), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(ResourceFull), + NEXM_FLD_PAR(SRC), + NEXM_FLD(RCODE), + NEXM_VAR(RDATA), + NEXM_VAR(HREPEAT), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(IndirectBranchHist), + NEXM_FLD_PAR(SRC), + NEXM_FLD(BTYPE), + NEXM_VAR(ICNT), + NEXM_ADR(UADDR), + NEXM_VAR(HIST), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(IndirectBranchHistSync), + NEXM_FLD_PAR(SRC), + NEXM_FLD(SYNC), + NEXM_FLD(BTYPE), + NEXM_VAR(ICNT), + NEXM_ADR(FADDR), + NEXM_VAR(HIST), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(RepeatBranch), + NEXM_FLD_PAR(SRC), + NEXM_VAR(BCNT), + NEXM_VAR(TSTAMP), + NEXM_END(), + + NEXM_BEG(ProgTraceCorrelation), + NEXM_FLD_PAR(SRC), + NEXM_FLD(EVCODE), + NEXM_FLD(CDF), + NEXM_VAR(ICNT), + NEXM_VAR(HIST), + NEXM_VAR(TSTAMP), + NEXM_END(), + + { NULL, 0 } /* end marker */ +}; + +/* + * Look up the rvtrace_ntrace_msg_defs entry index of a message header whose low 8 bits + * match 'tcode'. Returns the index, or -1 if the TCODE is not defined. + */ +static int rvtrace_ntrace_find_tcode(unsigned int tcode) +{ + for (int d = 0; rvtrace_ntrace_msg_defs[d].def != 0; d++) { + if ((rvtrace_ntrace_msg_defs[d].def & NEXM_DEF_MSG) == 0) + continue; + if ((rvtrace_ntrace_msg_defs[d].def & 0xff) == tcode) + return d; + } + return -1; +} + +/* + * Return the bit width of the fixed-size field at rvtrace_ntrace_msg_defs[def]. Fields + * marked NEXM_DEF_PARAM take their width from the decoder's srcbits. + */ +static int rvtrace_ntrace_field_size(struct rvtrace_decoder *decoder, int def) +{ + int size = rvtrace_ntrace_msg_defs[def].def & 0xff; + + if (rvtrace_ntrace_msg_defs[def].def & NEXM_DEF_PARAM) + size = decoder->srcbits; + return size; +} + +/* + * Feed the byte stream one byte at a time. The parser accumulates + * MDO/MSEO bits into 'msg' and returns: + * + * 1 a complete message has been parsed into *msg + * 0 more bytes are needed (or an idle/partial byte was consumed) + */ +static int rvtrace_ntrace_msg_parse(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_msg_parser *p, + struct rvtrace_ntrace_msg *msg, + unsigned char byte, unsigned char prev_byte) +{ + unsigned int mdo = byte >> 2; + unsigned int mseo = byte & 0x3; + + pr_debug3("0x%02x ", byte); + for (int b = 0x80; b != 0; b >>= 1) { + if (b == 0x2) + pr_debug3("_"); + pr_debug3("%d", !!(byte & b)); + } + pr_debug3("\n"); + + if (p->resync) { + if ((prev_byte & 0x3) == 0x3) { /* MSEO==3: message end */ + p->resync = false; + p->field_def = -1; + p->field_bits = 0; + p->field_val = 0; + } + return 0; + } + + /* Skip long runs of idle bytes (0xFF). */ + if (byte == 0xFF && prev_byte == 0xFF) + return 0; + + if (mseo == 0x2) { + pr_err("ERROR: MSEO='10' is not allowed\n"); + p->resync = true; + return 0; + } + + /* Message header: the first byte's MDO is the TCODE. */ + if (p->field_def < 0) { + if (mseo == 0x3) + return 0; /* skip idle */ + + if (mseo != 0x0) { + pr_err("ERROR: Message must start from MSEO='00'\n"); + p->resync = true; + return 0; + } + + p->field_def = rvtrace_ntrace_find_tcode(mdo); + if (p->field_def < 0) { + pr_err("ERROR: Message with TCODE=%d is not defined for RISC-V\n", mdo); + p->resync = true; + return 0; + } + + msg->field_pos = p->field_def; + msg->tcode = mdo; + msg->nfields = 0; + msg->fields[msg->nfields++] = mdo; + + p->field_def++; + p->field_bits = 0; + p->field_val = 0; + return 0; + } + + /* Accumulate this byte's MDO into the current field value. */ + p->field_val |= (u64)mdo << p->field_bits; + p->field_bits += 6; + + /* Emit as many complete fixed-size fields as we now have bits for. */ + while ((rvtrace_ntrace_msg_defs[p->field_def].def & NEXM_DEF_FLD) && + p->field_bits >= rvtrace_ntrace_field_size(decoder, p->field_def)) { + int width = rvtrace_ntrace_field_size(decoder, p->field_def); + + msg->fields[msg->nfields++] = p->field_val & ((1ULL << width) - 1); + p->field_def++; + p->field_val >>= width; + p->field_bits -= width; + } + + if (mseo == 0x0) + return 0; + + if (rvtrace_ntrace_msg_defs[p->field_def].def & NEXM_DEF_VAR) { + /* Variable-size field: the whole remaining value. */ + msg->fields[msg->nfields++] = p->field_val; + + if (rvtrace_ntrace_msg_defs[p->field_def].def & NEXM_DEF_ADR) + msg->addr_bits = p->field_bits; + + if (mseo == 3) { + p->field_def = -1; + p->field_bits = 0; + p->field_val = 0; + return 1; /* complete message */ + } + p->field_def++; + p->field_bits = 0; + p->field_val = 0; + return 0; + } + + if (p->field_bits > 0) { + pr_err("Decode: Not enough bits for non-variable field\n"); + p->resync = true; + } + return 0; +} + +static int rvtrace_ntrace_msg_sprint(char *buf, size_t size, + struct rvtrace_ntrace_msg *msg, int msgs, int bytes) +{ + int len; + + len = snprintf(buf, size, "MSG #%d +%d - %s TCODE[6]=%d", + msgs, bytes, + rvtrace_ntrace_msg_defs[msg->field_pos].name, msg->tcode); + + for (int i = 1; i < (int)msg->nfields && len < (int)size; i++) + len += snprintf(buf + len, size - len, " %s=0x%llx", + rvtrace_ntrace_msg_defs[msg->field_pos + i].name, + (unsigned long long)msg->fields[i]); + + return len; +} + +static int handle_error_msg(struct rvtrace_src_context *ctx, const char *err) +{ + pr_err("\nERROR: %s\n", err); + ctx->nexdeco_pc = 1; /* 1 means, that last address is unknown */ + rvtrace_init_stack(&ctx->stack); + return -EINVAL; +} + +static int rvtrace_ntrace_insn_info_get(struct rvtrace_decoder *decoder, u8 *info, u64 *dest) +{ + u32 insn; + struct rvtrace_src_context *ctx = decoder->current_src_ctx; + u64 addr = ctx->nexdeco_pc; + + if (!decoder->mem_access(decoder->data, addr, ctx->prv, ctx->context, + sizeof(insn), (u8 *)&insn)) + return -EINVAL; + + *info = INFO_LINEAR; + if ((insn & (__INSN_LENGTH_MASK)) != (__INSN_LENGTH_GE_32)) { + if (riscv_insn_is_c_beqz(insn) || riscv_insn_is_c_bnez(insn)) { + *info = INFO_BRANCH; + *dest = addr + riscv_insn_extract_cbtype_imm(insn); + } else if (riscv_insn_is_c_j(insn)) { + /* c.j offset */ + *info = INFO_JUMP; + *dest = addr + riscv_insn_extract_cjtype_imm(insn); + } else if (riscv_insn_is_c_jr(insn)) { + *info = INFO_JUMP | INFO_INDIRECT; + /* ret => c.jr x1 */ + if (RVC_EXTRACT_C2_RS1_REG(insn) == RV_REGNO_RA) + *info |= INFO_RET; + } else if (riscv_insn_is_c_jalr(insn)) { + *info = INFO_JUMP | INFO_CALL | INFO_INDIRECT; + } else { + *info = INFO_LINEAR; + } + } else { + if (riscv_insn_is_branch(insn)) { + *info = INFO_BRANCH; + *dest = addr + riscv_insn_extract_btype_imm(insn); + } else if (riscv_insn_is_jalr(insn)) { + *info = INFO_JUMP | INFO_INDIRECT; + if (RV_EXTRACT_RD_REG(insn) == RV_REGNO_ZERO + && riscv_insn_extract_jtype_imm(insn) == 0) { + /* ret => jalr x0,x1,0 */ + if (RV_EXTRACT_RS1_REG(insn) == RV_REGNO_RA) + *info |= INFO_RET; + else + /* jr rs1 => jalr x0,rs1,0 */ + *info |= INFO_CALL; + } + } else if (riscv_insn_is_jal(insn)) { + *info = INFO_JUMP; + /* j offset => jal x0,offset */ + if (RV_EXTRACT_RD_REG(insn) != RV_REGNO_ZERO) + *info |= INFO_CALL; + *dest = addr + riscv_insn_extract_jtype_imm(insn); + } else if (riscv_insn_is_sret(insn) || riscv_insn_is_mret(insn)) { + *info = INFO_JUMP | INFO_INDIRECT | INFO_RET; + } else if (riscv_insn_is_ecall(insn)) { + *info = INFO_JUMP | INFO_INDIRECT | INFO_CALL; + } else { + *info = INFO_LINEAR; + } + *info |= INFO_4; + } + + pr_debug2(".nexdeco_pc=0x%lx .insn=0x%0*x\n", addr, + (*info & INFO_4) ? 8 : 4, + (*info & INFO_4) ? insn : (u16)insn); + return 0; +} + +static int rvtrace_ntrace_emit_icnt(struct rvtrace_decoder *decoder, int n, u32 hist) +{ + u64 a = 0; + u32 hist_mask; + u8 info; + int done_icnt = 0; /* Number of done instruction count */ + struct rvtrace_src_context *ctx = decoder->current_src_ctx; + + if (ctx->nexdeco_pc & 1) + return 0; /* Not synchronized ... */ + + pr_debug2(".PC=0x%lX, EmitICNT(n=%d,hist=0x%x)\n", ctx->nexdeco_pc, n, hist); + + /* + * Adjust ICNT by what was handled by ResourceFull message[s] + * before this message (message with normal ICNT field). + * NOTE: ctx->resourcefull_icnt maybe positive or negative! + */ + if (n >= 0 && ctx->resourcefull_icnt != 0) { + n += ctx->resourcefull_icnt; + if (n < 0) + return handle_error_msg(ctx, "ICNT adjustment ERROR"); + + ctx->resourcefull_icnt = 0; /* Make adjustment 'consumed' */ + } + + hist_mask = 0; /* MSB is first in history, so we need sliding mask */ + if (hist != 0) { + if (hist & (1 << (sizeof(u32) * 8 - 1))) { + hist_mask = (1 << (sizeof(u32) * 8 - 2)); + } else { + hist_mask = 0x1; + while (hist_mask <= hist) + hist_mask <<= 1; + hist_mask >>= 2; + } + } + + while (n != 0) { + ctx->current_pc = ctx->nexdeco_pc; + if (rvtrace_ntrace_insn_info_get(decoder, &info, &a)) + return handle_error_msg(ctx, "failed to get insn info"); + + if (n > 0) { + n -= (info & INFO_4) ? 2 : 1; + if (n < 0) + return handle_error_msg(ctx, "ICNT too small"); + done_icnt += 1; + } + + if (info & INFO_CALL) { + u64 ret = ctx->nexdeco_pc + ((info & INFO_4) ? 4 : 2); + + if (rvtrace_stack_push(&ctx->stack, ret)) + return handle_error_msg(ctx, "failed to push value to stack"); + } + + if (info & INFO_INDIRECT) { /* Cannot continue over indirect... */ + if (info & INFO_RET) { + u64 ret = rvtrace_stack_pop(&ctx->stack); + + ctx->nexdeco_pc = ret; + if (n != 0) { + if (ret == 1) + return handle_error_msg(ctx, + "Not enough entries on callstack"); + continue; + } + } + + if (n > 0) + return handle_error_msg(ctx, + "indirect address encountered in ICNT"); + + break; + } + + if (info & INFO_BRANCH) { + if (hist == 0) { + /* This is calling as DirectBranch */ + if (n == 0) + info |= INFO_JUMP; /* Force PC change below */ + } else { + if (hist_mask & hist) + info |= INFO_JUMP; /* Force PC change below */ + hist_mask >>= 1; + if (hist_mask == 0 && n < 0) + n = 0; + } + } + + if (info & INFO_JUMP) + ctx->nexdeco_pc = a; /* Direct jump/call/branch */ + else if (info & INFO_4) + ctx->nexdeco_pc += 4; + else + ctx->nexdeco_pc += 2; + + } + + return done_icnt; +} + +static u64 rvtrace_ntrace_field_get(struct rvtrace_decoder *decoder, const char *name) +{ + struct rvtrace_ntrace_msg *msg = &decoder->msg; + + for (int d = msg->field_pos; rvtrace_ntrace_msg_defs[d].def != 1; d++) { + if (strcmp(rvtrace_ntrace_msg_defs[d].name, name) == 0) { + int fi = d - msg->field_pos; + + if (fi < (int)msg->nfields) + return msg->fields[fi]; + return 0; + } + } + return 0; +} + +#define NEX_FLDGET(n) rvtrace_ntrace_field_get(decoder, #n) + +/* + * Switch the active source context to 'new_src', allocating it on first use. + * Returns 0 on success, negative on error. + */ +static int rvtrace_ntrace_switch_src(struct rvtrace_decoder *decoder, int new_src) +{ + if (new_src == decoder->current_src) + return 0; + + decoder->current_src_ctx = rvtrace_get_src_context(decoder, new_src); + if (!decoder->current_src_ctx) + return -ENOMEM; + + decoder->current_src = new_src; + return 0; +} + +/* + * Fill 'packet' from the active source context and append it to the decoder's + * packet buffer. 'ret' is the instruction count produced by emit_icnt. + */ +static int rvtrace_ntrace_emit_packet(struct rvtrace_decoder *decoder, int tcode, + u64 start_addr, int ret, int src) +{ + struct rvtrace_packet packet; + + if (ret >= 0) { + packet.sample_type = (tcode == NEXUS_TCODE_Error) ? + RVTRACE_LOSS : + (ret ? RVTRACE_RANGE : RVTRACE_EMPTY); + packet.start_addr = start_addr; + packet.end_addr = decoder->current_src_ctx->current_pc; + packet.insn_cnt = ret; + packet.cpu = src; + packet.prv = decoder->current_src_ctx->prv; + packet.v = decoder->current_src_ctx->v; + packet.context = decoder->current_src_ctx->context; + packet.timestamp = decoder->current_src_ctx->timestamp; + } else { + packet.sample_type = RVTRACE_ERROR; + } + + return rvtrace_packet_buffer_append(&decoder->packet_buffer, packet); +} + +/* Reconstruct an address from the transmitted F-ADDR/U-ADDR field */ +static u64 rvtrace_ntrace_calculate_addr(struct rvtrace_decoder *decoder, + u64 fu_addr, int full, u64 prev_addr) +{ + u32 bits = decoder->msg.addr_bits; + + fu_addr <<= 1; /* LSB bit is never sent */ + + /* + * After shifting the LSB back in, the most significant transmitted bit + * sits at position 'bits'. Extend it (0 or 1) all the way to bit 63. + * This is sign-extension in effect, but the spec notes it is NOT a sign + * bit -- it is simply the value of the skipped identical bits. + */ + if (bits && bits < 64 && (fu_addr & (1ULL << bits))) + fu_addr |= ~0ULL << bits; + + /* Update (NEW or XOR) */ + if (!full) { + if (prev_addr & 1) /* Not Sync */ + fu_addr = prev_addr; + else + fu_addr ^= prev_addr; + } + return fu_addr; +} + +/* + * Per-message handler result. Each handler fills these and the dispatcher + * emits a single packet from them. + */ +struct rvtrace_ntrace_result { + int icnt; /* instruction count from emit_icnt() */ + u64 start_addr; + int src; +}; + +/* Common prologue: switch to the message's source context. */ +static int rvtrace_ntrace_handler_prologue(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + res->src = NEX_FLDGET(SRC); + return rvtrace_ntrace_switch_src(decoder, res->src); +} + +/* + * Common F-ADDR/U-ADDR handling: reconstruct the address and make it the + * current PC. 'full' selects full (sync) vs XOR (compressed) update. + */ +static void rvtrace_ntrace_update_addr(struct rvtrace_decoder *decoder, u64 addr, int full) +{ + struct rvtrace_src_context *ctx = decoder->current_src_ctx; + + ctx->nexdeco_lastaddr = rvtrace_ntrace_calculate_addr(decoder, addr, full, + ctx->nexdeco_lastaddr); + pr_debug2(".nexdeco_lastaddr=0x%lx\n", ctx->nexdeco_lastaddr); + ctx->nexdeco_pc = ctx->nexdeco_lastaddr; +} + +static int rvtrace_ntrace_handle_ownership(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + + if (ret) + return ret; + + decoder->current_src_ctx->prv = NEX_FLDGET(PRV); + decoder->current_src_ctx->v = NEX_FLDGET(V); + if (NEX_FLDGET(FORMAT) && NEX_FLDGET(CONTEXT)) + decoder->current_src_ctx->context = NEX_FLDGET(CONTEXT); + decoder->current_src_ctx->timestamp += NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_handle_direct_branch(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + + if (ret) + return ret; + + res->start_addr = decoder->current_src_ctx->nexdeco_pc; + res->icnt = rvtrace_ntrace_emit_icnt(decoder, NEX_FLDGET(ICNT), 0x0); + decoder->current_src_ctx->timestamp += NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_handle_indirect_branch(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + + if (ret) + return ret; + + res->start_addr = decoder->current_src_ctx->nexdeco_pc; + res->icnt = rvtrace_ntrace_emit_icnt(decoder, NEX_FLDGET(ICNT), 0x0); + + rvtrace_ntrace_update_addr(decoder, NEX_FLDGET(UADDR), 0); + decoder->current_src_ctx->timestamp += NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_handle_prog_trace_sync(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + + if (ret) + return ret; + + res->start_addr = decoder->current_src_ctx->nexdeco_pc; + res->icnt = rvtrace_ntrace_emit_icnt(decoder, NEX_FLDGET(ICNT), 0x0); + + rvtrace_ntrace_update_addr(decoder, NEX_FLDGET(FADDR), 1); + decoder->current_src_ctx->timestamp = NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_handle_direct_branch_sync(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + + if (ret) + return ret; + + res->start_addr = decoder->current_src_ctx->nexdeco_pc; + res->icnt = rvtrace_ntrace_emit_icnt(decoder, NEX_FLDGET(ICNT), 0x0); + + rvtrace_ntrace_update_addr(decoder, NEX_FLDGET(FADDR), 1); + decoder->current_src_ctx->timestamp = NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_handle_indirect_branch_sync(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + + if (ret) + return ret; + + res->start_addr = decoder->current_src_ctx->nexdeco_pc; + res->icnt = rvtrace_ntrace_emit_icnt(decoder, NEX_FLDGET(ICNT), 0x0); + + rvtrace_ntrace_update_addr(decoder, NEX_FLDGET(FADDR), 1); + decoder->current_src_ctx->timestamp = NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_handle_indirect_branch_hist(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + + if (ret) + return ret; + + res->start_addr = decoder->current_src_ctx->nexdeco_pc; + res->icnt = rvtrace_ntrace_emit_icnt(decoder, NEX_FLDGET(ICNT), NEX_FLDGET(HIST)); + + rvtrace_ntrace_update_addr(decoder, NEX_FLDGET(UADDR), 0); + decoder->current_src_ctx->timestamp += NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_handle_indirect_branch_hist_sync(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + + if (ret) + return ret; + + res->start_addr = decoder->current_src_ctx->nexdeco_pc; + res->icnt = rvtrace_ntrace_emit_icnt(decoder, NEX_FLDGET(ICNT), NEX_FLDGET(HIST)); + + rvtrace_ntrace_update_addr(decoder, NEX_FLDGET(FADDR), 1); + decoder->current_src_ctx->timestamp = NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_handle_resource_full(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + int rcode = NEX_FLDGET(RCODE); + int hrepeat = 0; + + if (ret) + return ret; + + if (rcode == 2) + hrepeat = NEX_FLDGET(HREPEAT); + + res->start_addr = decoder->current_src_ctx->nexdeco_pc; + decoder->current_src_ctx->timestamp += NEX_FLDGET(TSTAMP); + if (rcode == 1 || rcode == 2) { + int rdata = NEX_FLDGET(RDATA); + + if (rdata > 1) { + do { + /* ICNT is unknown (-1), what will process only HIST bits */ + res->icnt = rvtrace_ntrace_emit_icnt(decoder, -1, rdata); + if (res->icnt < 0) + break; + + /* Consume, so next time ICNT will be adjusted */ + decoder->current_src_ctx->resourcefull_icnt -= res->icnt; + hrepeat--; + } while (hrepeat > 0); + } + } else if (rcode == 0) { + decoder->current_src_ctx->resourcefull_icnt += NEX_FLDGET(RDATA); + } + return 0; +} + +static int rvtrace_ntrace_handle_prog_trace_correlation(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + int hist = 0; + + if (ret) + return ret; + + if (NEX_FLDGET(CDF) == 1) + hist = NEX_FLDGET(HIST); + + res->start_addr = decoder->current_src_ctx->nexdeco_pc; + res->icnt = rvtrace_ntrace_emit_icnt(decoder, NEX_FLDGET(ICNT), hist); + decoder->current_src_ctx->timestamp += NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_handle_error(struct rvtrace_decoder *decoder, + struct rvtrace_ntrace_result *res) +{ + int ret = rvtrace_ntrace_handler_prologue(decoder, res); + + if (ret) + return ret; + + decoder->current_src_ctx->timestamp += NEX_FLDGET(TSTAMP); + return 0; +} + +static int rvtrace_ntrace_msg_handle(struct rvtrace_decoder *decoder) +{ + struct rvtrace_ntrace_result res = { 0 }; + int tcode = decoder->msg.tcode; + int err; + + switch (tcode) { + case NEXUS_TCODE_Ownership: + err = rvtrace_ntrace_handle_ownership(decoder, &res); + break; + case NEXUS_TCODE_DirectBranch: + err = rvtrace_ntrace_handle_direct_branch(decoder, &res); + break; + case NEXUS_TCODE_IndirectBranch: + err = rvtrace_ntrace_handle_indirect_branch(decoder, &res); + break; + case NEXUS_TCODE_ProgTraceSync: + err = rvtrace_ntrace_handle_prog_trace_sync(decoder, &res); + break; + case NEXUS_TCODE_DirectBranchSync: + err = rvtrace_ntrace_handle_direct_branch_sync(decoder, &res); + break; + case NEXUS_TCODE_IndirectBranchSync: + err = rvtrace_ntrace_handle_indirect_branch_sync(decoder, &res); + break; + case NEXUS_TCODE_IndirectBranchHist: + err = rvtrace_ntrace_handle_indirect_branch_hist(decoder, &res); + break; + case NEXUS_TCODE_IndirectBranchHistSync: + err = rvtrace_ntrace_handle_indirect_branch_hist_sync(decoder, &res); + break; + case NEXUS_TCODE_ResourceFull: + err = rvtrace_ntrace_handle_resource_full(decoder, &res); + break; + case NEXUS_TCODE_ProgTraceCorrelation: + err = rvtrace_ntrace_handle_prog_trace_correlation(decoder, &res); + break; + case NEXUS_TCODE_Error: + err = rvtrace_ntrace_handle_error(decoder, &res); + break; + case NEXUS_TCODE_RepeatBranch: /* Handled differently! */ + default: + return -EINVAL; + } + + if (err) + return err; + + return rvtrace_ntrace_emit_packet(decoder, tcode, res.start_addr, res.icnt, res.src); +} + +/* + * Decode the N-Trace byte stream. + * + * RVTRACE_DECODE_MSG - print the messages + * RVTRACE_DECODE_INSN - reconstruct instruction flow + */ +int rvtrace_ntrace_decode(struct rvtrace_decoder *decoder, enum rvtrace_decode_type type, + const unsigned char *buf, size_t size) +{ + const char *color = PERF_COLOR_BLUE; + struct rvtrace_ntrace_msg_parser p = { .field_def = -1 }; + struct rvtrace_ntrace_msg *msg = &decoder->msg; + struct { + int msgs; + int bytes; + int idles; + int errors; + int invalid; + } stat = { 0 }; + unsigned char prev_byte = 0; + bool was_resync = false; + size_t pos; + int err; + + msg->nfields = 0; + + for (pos = 0; pos < size; pos++) { + unsigned char byte = buf[pos]; + char line[256]; + int done = rvtrace_ntrace_msg_parse(decoder, &p, msg, byte, prev_byte); + int cnt = 1; + + prev_byte = byte; + stat.bytes++; + + if (p.resync && !was_resync) + stat.invalid++; + was_resync = p.resync; + + if (done <= 0) + continue; + + stat.msgs++; + if (msg->tcode == NEXUS_TCODE_Error) + stat.errors++; + + rvtrace_ntrace_msg_sprint(line, sizeof(line), msg, stat.msgs, stat.bytes); + if (type == RVTRACE_DECODE_MSG) { + color_fprintf(stdout, color, "%s\n", line); + continue; + } + pr_debug2("%s\n", line); + + if (msg->tcode == NEXUS_TCODE_RepeatBranch) { + cnt = msg->fields[1]; + *msg = decoder->saved_msg; + } else { + /* Save previous message fields */ + decoder->saved_msg = *msg; + } + + while (cnt-- > 0) { + err = rvtrace_ntrace_msg_handle(decoder); + if (err) + return err; + } + } + + if (type == RVTRACE_DECODE_MSG) { + color_fprintf(stdout, color, + "\nN-Trace: %d bytes, %d messages, %d error messages, %d invalid messages", + stat.bytes, stat.msgs, stat.errors, stat.invalid); + } else { + pr_debug2("\nN-Trace: %d bytes, %d messages, %d errors, %d invalid\n", + stat.bytes, stat.msgs, stat.errors, stat.invalid); + } + + return 0; +} + diff --git a/tools/perf/util/rvtrace-decoder/rvtrace-ntrace-decoder.h b/tools/perf/util/rvtrace-decoder/rvtrace-ntrace-decoder.h new file mode 100644 index 000000000000..4575e631ba5e --- /dev/null +++ b/tools/perf/util/rvtrace-decoder/rvtrace-ntrace-decoder.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2020 IAR Systems AB. + * Copyright (c) 2026 Spacemit Technologies, Inc. + */ + +#ifndef INCLUDE__RVTRACE_NTRACE_DECODER_H__ +#define INCLUDE__RVTRACE_NTRACE_DECODER_H__ + +#include <stdint.h> // For uint32_t and uint64_t +#include <stdbool.h> +#include <linux/types.h> + +/****************************************************************************/ +/* Instruction information */ +#define INFO_LINEAR 0x1 // Linear (plain instruction or not taken BRANCH) +#define INFO_4 0x2 // If not 4, it must be 2 on RISC-V +// 0x4 // Reserved (for exception or so ...) +#define INFO_INDIRECT 0x8 // Possible for most types above +#define INFO_BRANCH 0x10 // Always direct on RISC-V (may have LINEAR too) +#define INFO_JUMP 0x20 // Direct or indirect +#define INFO_CALL 0x40 // Direct or indirect (and always a jump) +#define INFO_RET 0x80 // Return (always indirect and always a jump) + +/****************************************************************************/ + +/****************************************************************************/ +/* Nexus specific values (based on Nexus Standard PDF) */ + +#define NEXUS_FLDSIZE_TCODE 6 // This is standard + +/* Nexus TCODE values applicable to RISC-V */ +#define NEXUS_TCODE_Ownership 2 +#define NEXUS_TCODE_DirectBranch 3 +#define NEXUS_TCODE_IndirectBranch 4 +#define NEXUS_TCODE_Error 8 +#define NEXUS_TCODE_ProgTraceSync 9 +#define NEXUS_TCODE_DirectBranchSync 11 +#define NEXUS_TCODE_IndirectBranchSync 12 +#define NEXUS_TCODE_ResourceFull 27 +#define NEXUS_TCODE_IndirectBranchHist 28 +#define NEXUS_TCODE_IndirectBranchHistSync 29 +#define NEXUS_TCODE_RepeatBranch 30 +#define NEXUS_TCODE_ProgTraceCorrelation 33 + +/* End of standard values */ +/****************************************************************************/ + +/****************************************************************************/ +/* RISC-V Nexus Trace related values (most 'recommended' by Nexus) */ + +// Sizes of fields: +#define NEXUS_FLDSIZE_BTYPE 2 // Branch type +#define NEXUS_FLDSIZE_SYNC 4 // Synchronization code +#define NEXUS_FLDSIZE_ETYPE 4 // Error type +#define NEXUS_FLDSIZE_EVCODE 4 // Event code (correlation) +#define NEXUS_FLDSIZE_CDF 2 +#define NEXUS_FLDSIZE_RCODE 4 // Resource full code size +// from PROCESS fields: +#define NEXUS_FLDSIZE_FORMAT 2 +#define NEXUS_FLDSIZE_PRV 2 +#define NEXUS_FLDSIZE_V 1 + +#define NEXUS_PAR_SIZE_SRC 0 // SRC field size is defined by parameter #0 + +#define NEXUS_HIST_BITS 31 // Number of valid HIST bits + +// Address skipping (on RISC-V LSB of PC is always 0, so it is not encoded) +#define NEXUS_PARAM_AddrSkip 1 +#define NEXUS_PARAM_AddrUnit 1 + +// End of RISC-V related values +/****************************************************************************/ + +/* + * Field definition flags packed into rvtrace_ntrace_msg_field_def.def. The + * low 8 bits of a fixed field carry its bit width; bit 7 marks a + * runtime-sized (parameter) field such as SRC. + */ +#define NEXM_DEF_MSG 0x100 /* message header: low bits are the TCODE */ +#define NEXM_DEF_FLD 0x200 /* fixed-size field */ +#define NEXM_DEF_VAR 0x400 /* variable-size field */ +#define NEXM_DEF_ADR 0x800 /* address field (variable + addr semantics) */ +#define NEXM_DEF_PARAM 0x80 /* field size is a runtime parameter */ + +// Macros to define Nexus Messages (NEXM_...) +// NOTE: These macros refer to 'NEXUS_TCODE_...' and 'NEXUS_FLDSIZE_...' +// It is also possible to NOT do it, but it provides less flexibility. +// +// name def (marker | value) +#define NEXM_BEG(n) {#n, NEXM_DEF_MSG | (NEXUS_TCODE_##n) } +#define NEXM_FLD(n) {#n, NEXM_DEF_FLD | (NEXUS_FLDSIZE_##n) } +#define NEXM_FLD_PAR(n) {#n, NEXM_DEF_FLD | NEXM_DEF_PARAM } +#define NEXM_VAR(n) {#n, NEXM_DEF_VAR } +#define NEXM_ADR(n) {#n, NEXM_DEF_VAR | NEXM_DEF_ADR } +#define NEXM_END() {NULL, 1 } + +#define MSGFIELDS_MAX 10 + +struct rvtrace_ntrace_msg_field_def { + const char *name; /* Name of message/field */ + int def; /* Definition of field (see NEXM_... above) */ +}; + +extern const struct rvtrace_ntrace_msg_field_def rvtrace_ntrace_msg_defs[]; + +/* A single decoded N-Trace message */ +struct rvtrace_ntrace_msg { + u32 tcode; + u32 field_pos; + u32 nfields; + u32 addr_bits; /* transmitted bit width of the last F-ADDR/U-ADDR field */ + u64 fields[MSGFIELDS_MAX]; +}; + +/* Bit-level N-Trace message parser state */ +struct rvtrace_ntrace_msg_parser { + int field_def; /* index into rvtrace_ntrace_msg_defs of the current field */ + int field_bits; /* bits accumulated for the current field */ + u64 field_val; /* accumulated value of the current field */ + bool resync; /* skip bytes until the next MSEO==3 terminator */ +}; + +#endif

Implement the gtrace auxtrace decoder: queue AUX buffers, run timeless or timestamped decoding, and synthesize branch samples from decoded trace packets. Support machine-mode instruction access via new '-m/--firmware' and '--firmware-base' options. --- tools/perf/builtin-script.c | 4 + tools/perf/util/gtrace-decoder.c | 1206 +++++++++++++++++++++++++++++- tools/perf/util/symbol_conf.h | 2 + 3 files changed, 1197 insertions(+), 15 deletions(-)

diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c index ad8ca08ceb5f..d1173c80b6fa 100644 --- a/tools/perf/builtin-script.c +++ b/tools/perf/builtin-script.c @@ -4088,6 +4088,10 @@ int cmd_script(int argc, const char **argv) "file", "vmlinux pathname"), OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name, "file", "kallsyms pathname"), + OPT_STRING('m', "firmware", &symbol_conf.firmware_name, + "file", "machine mode firmware binary pathname"), + OPT_STRING(0, "firmware-base", &symbol_conf.firmware_base, + "address", "base address for machine mode firmware (e.g., 0x100000000)"), OPT_BOOLEAN('G', "hide-call-graph", &no_callchain, "When printing symbols do not display call chain"), OPT_CALLBACK(0, "symfs", NULL, "directory[,layout]", SYMFS_HELP, diff --git a/tools/perf/util/gtrace-decoder.c b/tools/perf/util/gtrace-decoder.c index d70cb7cd399f..b2973f7ad3fe 100644 --- a/tools/perf/util/gtrace-decoder.c +++ b/tools/perf/util/gtrace-decoder.c @@ -13,6 +13,24 @@ #include "session.h" #include "gtrace.h" #include "debug.h" +#include "color.h" +#include <linux/bitfield.h> +#include "intlist.h" +#include "machine.h" +#include "map.h" +#include "thread.h" +#include "thread_map.h" +#include "thread-stack.h" +#include "util.h" +#include "dso.h" +#include "maps.h" +#include "symbol.h" +#include "addr_location.h" +#include "util/synthetic-events.h" +#include <fcntl.h> +#include <unistd.h> +#include "rvtrace-decoder/rvtrace-decoder.h" +#include "../../arch/riscv/include/asm/insn.h"

struct gtrace_decoder { struct auxtrace auxtrace; @@ -22,31 +40,242 @@ struct gtrace_decoder { u32 pmu_type; int num_cpu; u64 **metadata; + struct auxtrace_queues queues; + struct auxtrace_heap heap; + struct itrace_synth_opts synth_opts; + struct dso *firmware_dso; + struct maps *firmware_maps; + bool timeless_decoding; + bool per_thread_decoding; + bool data_queued; + u64 latest_kernel_timestamp; + u64 branches_sample_type; + u64 branches_id; };

-static int gtrace_process_event(struct perf_session *session __maybe_unused, - union perf_event *event __maybe_unused, - struct perf_sample *sample __maybe_unused, - const struct perf_tool *tool __maybe_unused) +enum coresight_format { + UNSET, + FORMATTED, + UNFORMATTED +}; + +struct gtrace_queue { + struct gtrace_decoder *ptr; + struct thread *thread; + struct rvtrace_decoder *decoder; + struct auxtrace_buffer *buffer; + union perf_event *event_buf; + unsigned int queue_nr; + u64 offset; + enum coresight_format cs_format; +}; + +static int gtrace_process_timeless_queues(struct gtrace_decoder *ptr, + pid_t tid); +static int gtrace_process_timestamped_queues(struct gtrace_decoder *ptr); + +static int gtrace_process_itrace_start(struct gtrace_decoder *ptr, + union perf_event *event) { + struct thread *th; + + if (ptr->timeless_decoding) + return 0; + + /* + * Add the tid/pid to the log so that we can get a match when + * we get a contextID from the decoder. + */ + th = machine__findnew_thread(ptr->machine, + event->itrace_start.pid, + event->itrace_start.tid); + if (!th) + return -ENOMEM; + + thread__put(th); + return 0; }

-static int gtrace_process_auxtrace_event(struct perf_session *session __maybe_unused, - union perf_event *event __maybe_unused, - const struct perf_tool *tool __maybe_unused) +static int gtrace_process_switch_cpu_wide(struct gtrace_decoder *ptr, + union perf_event *event) { + struct thread *th; + bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; + + if (ptr->timeless_decoding) + return 0; + + if (!out) + return 0; + + th = machine__findnew_thread(&ptr->session->machines.host, + event->context_switch.next_prev_pid, + event->context_switch.next_prev_tid); + + if (!th) + return -ENOMEM; + + thread__put(th); + return 0; }

-static int gtrace_flush(struct perf_session *session __maybe_unused, - const struct perf_tool *tool __maybe_unused) +static int gtrace_process_event(struct perf_session *session, + union perf_event *event, + struct perf_sample *sample, + const struct perf_tool *tool) { + struct gtrace_decoder *ptr = container_of(session->auxtrace, struct gtrace_decoder, + auxtrace); + if (dump_trace) + return 0; + + if (!tool->ordered_events) { + pr_err("GTRACE requires ordered events\n"); + return -EINVAL; + } + + switch (event->header.type) { + case PERF_RECORD_EXIT: + if (ptr->per_thread_decoding && ptr->timeless_decoding) + return gtrace_process_timeless_queues(ptr, event->fork.tid); + break; + + case PERF_RECORD_ITRACE_START: + return gtrace_process_itrace_start(ptr, event); + + case PERF_RECORD_SWITCH_CPU_WIDE: + return gtrace_process_switch_cpu_wide(ptr, event); + + case PERF_RECORD_AUX: + if (sample->time && (sample->time != (u64)-1)) + ptr->latest_kernel_timestamp = sample->time; + break; + + default: + break; + } + + return 0; +} + +static void gtrace_dump_event(struct gtrace_queue *gtraceq, + struct auxtrace_buffer *buffer) +{ + int ret; + const char *color = PERF_COLOR_BLUE; + struct rvtrace_decoder *decoder = gtraceq->decoder; + + color_fprintf(stdout, color, + ". ... Trace Encoder Trace data: size %#zx bytes\n", + buffer->size); + + ret = decoder->decode(gtraceq->decoder, RVTRACE_DECODE_MSG, + buffer->data, buffer->size); + if (ret) + color_fprintf(stdout, color, " Bad packet!\n"); +} + +static void dump_queued_data(struct gtrace_decoder *ptr, + struct perf_record_auxtrace *event) +{ + struct auxtrace_buffer *buf; + unsigned int i; + /* + * Find all buffers with same reference in the queues and dump them. + * This is because the queues can contain multiple entries of the same + * buffer that were split on aux records. + */ + for (i = 0; i < ptr->queues.nr_queues; ++i) + list_for_each_entry(buf, &ptr->queues.queue_array[i].head, list) + if (buf->reference == event->reference) + gtrace_dump_event(ptr->queues.queue_array[i].priv, buf); +} + +static int gtrace_process_auxtrace_event(struct perf_session *session, + union perf_event *event, + const struct perf_tool *tool __maybe_unused) +{ + struct gtrace_decoder *ptr = container_of(session->auxtrace, struct gtrace_decoder, + auxtrace); + + if (!ptr->data_queued) { + struct auxtrace_buffer *buffer; + off_t data_offset; + int fd = perf_data__fd(session->data); + bool is_pipe = perf_data__is_pipe(session->data); + int err; + int idx = event->auxtrace.idx; + + if (is_pipe) + data_offset = 0; + else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) + return -errno; + } + + err = auxtrace_queues__add_event(&ptr->queues, session, + event, data_offset, &buffer); + if (err) + return err; + + if (dump_trace) + if (auxtrace_buffer__get_data(buffer, fd)) { + gtrace_dump_event(ptr->queues.queue_array[idx].priv, buffer); + auxtrace_buffer__put_data(buffer); + } + } else if (dump_trace) + dump_queued_data(ptr, &event->auxtrace); + return 0; }

-static void gtrace_free_events(struct perf_session *session __maybe_unused) +static int gtrace_flush(struct perf_session *session, + const struct perf_tool *tool) { + struct gtrace_decoder *ptr = container_of(session->auxtrace, struct gtrace_decoder, + auxtrace); + if (dump_trace) + return 0; + + if (!tool->ordered_events) + return -EINVAL; + + if (ptr->timeless_decoding) + return gtrace_process_timeless_queues(ptr, -1); + + return gtrace_process_timestamped_queues(ptr); +} + +static void gtrace_free_queue(void *priv) +{ + struct gtrace_queue *gtraceq = priv; + + if (!gtraceq) + return; + + if (gtraceq->decoder) + rvtrace_decoder_free(gtraceq->decoder); + thread__zput(gtraceq->thread); + zfree(&gtraceq->event_buf); + free(gtraceq); +} + +static void gtrace_free_events(struct perf_session *session) +{ + unsigned int i; + struct gtrace_decoder *ptr = container_of(session->auxtrace, struct gtrace_decoder, + auxtrace); + struct auxtrace_queues *queues = &ptr->queues; + + for (i = 0; i < queues->nr_queues; i++) { + gtrace_free_queue(queues->queue_array[i].priv); + queues->queue_array[i].priv = NULL; + } + + auxtrace_queues__free(queues); }

static void gtrace_free(struct perf_session *session) @@ -54,6 +283,7 @@ static void gtrace_free(struct perf_session *session) struct gtrace_decoder *ptr = container_of(session->auxtrace, struct gtrace_decoder, auxtrace);

+ gtrace_free_events(session); session->auxtrace = NULL; for (int i = 0; i < ptr->num_cpu; i++) zfree(&ptr->metadata[i]); @@ -70,6 +300,643 @@ static bool gtrace_evsel_is_auxtrace(struct perf_session *session, return evsel->core.attr.type == ptr->pmu_type; }

+static void gtrace_set_thread(struct gtrace_queue *gtraceq, + pid_t tid) +{ + struct gtrace_decoder *ptr = gtraceq->ptr; + + if (tid != -1) { + thread__zput(gtraceq->thread); + gtraceq->thread = machine__find_thread(ptr->machine, -1, tid); + } + + /* Couldn't find a known thread */ + if (!gtraceq->thread) + gtraceq->thread = machine__idle_thread(ptr->machine); +} + +/* + * Register firmware ELF PT_LOAD segments as maps in the machine's kernel maps, + * so that perf script can resolve symbols for M-mode (machine mode) samples. + * + * base is the absolute physical load address of the firmware (from --firmware-base). + * p_paddr in the ELF is the relative address within the firmware image. + * + * map->start = base + p_paddr + * map->end = base + p_paddr + p_memsz + * map->pgoff = p_offset + * + * map__map_ip(paddr) = paddr - start + pgoff = p_offset + (paddr - base - p_paddr) + */ +static int gtrace_register_firmware_maps(struct gtrace_decoder *ptr, + struct dso *dso, u64 base) +{ + Elf64_Ehdr ehdr; + Elf64_Phdr phdr; + int fd, i, ret = 0; + const char *filename = dso__long_name(dso); + + fd = open(filename, O_RDONLY); + if (fd < 0) + return -1; + + if (pread(fd, &ehdr, sizeof(ehdr), 0) != sizeof(ehdr)) { + ret = -1; + goto out; + } + + for (i = 0; i < ehdr.e_phnum; i++) { + struct map *map; + + if (pread(fd, &phdr, sizeof(phdr), + ehdr.e_phoff + (u64)i * ehdr.e_phentsize) != sizeof(phdr)) + continue; + + if (phdr.p_type != PT_LOAD || phdr.p_memsz == 0) + continue; + + map = map__new2(base + phdr.p_paddr, dso); + if (!map) { + ret = -ENOMEM; + goto out; + } + + map__set_end(map, base + phdr.p_paddr + phdr.p_memsz); + map__set_pgoff(map, phdr.p_offset); + + pr_debug("gtrace: machine code map [%#"PRIx64"-%#"PRIx64"] pgoff=%#"PRIx64" %s\n", + base + phdr.p_paddr, base + phdr.p_paddr + phdr.p_memsz, + (u64)phdr.p_offset, dso__long_name(dso)); + + ret = maps__insert(ptr->firmware_maps, map); + map__put(map); + if (ret) + goto out; + } +out: + close(fd); + return ret; +} + +static int gtrace_insert_map_cb(struct map *map, void *data) +{ + struct maps *kmaps = data; + + maps__insert(kmaps, map); + return 0; +} + +static u32 gtrace_machine_mode_access(struct gtrace_queue *gtraceq, u64 address, + size_t size, u8 *buffer) +{ + struct machine *machine = gtraceq->ptr->machine; + struct map *map; + struct dso *dso; + u64 offset; + int len; + + if (!gtraceq->ptr->firmware_dso) { + ui__warning_once("RISC-V Trace: Machine mode code binary not provided.\n" + " Failed to read address 0x%lx.\n" + " Use option '-m /path/to/firmware --firmware-base 0xBASE'\n" + " to specify the binary and its base address.\n", + address); + return 0; + } + + map = maps__find(gtraceq->ptr->firmware_maps, address); + if (!map) + return 0; + + /* + * Insert firmware maps into kernel maps for symbol resolution. + * Re-insert if kcore loading has removed them (kcore calls + * maps__remove_maps which clears all non-kernel maps). + */ + if (!maps__find(machine__kernel_maps(machine), address)) { + maps__for_each_map(gtraceq->ptr->firmware_maps, + gtrace_insert_map_cb, + machine__kernel_maps(machine)); + map__load(map); + } + + dso = map__dso(map); + offset = map__map_ip(map, address); + len = dso__data_read_offset(dso, machine, offset, buffer, size); + map__put(map); + return len > 0 ? len : 0; +} + +static u32 gtrace_mem_access(void *data, u64 address, enum riscv_privilege_mode prv, + int context, size_t size, u8 *buffer) +{ + u8 cpumode; + u64 offset; + int len; + struct machine *machine; + struct addr_location al; + struct dso *dso; + int ret = 0; + struct gtrace_queue *gtraceq = data; + + if (!gtraceq) + goto out; + + addr_location__init(&al); + + /* If the riscv_privilege_mode is machine mode, access physical address by /dev/mem */ + if (prv == RISCV_PRIV_MACHINE_MODE) + return gtrace_machine_mode_access(gtraceq, address, size, buffer); + + machine = gtraceq->ptr->machine; + if (address >= machine__kernel_start(machine)) + cpumode = PERF_RECORD_MISC_KERNEL; + else + cpumode = PERF_RECORD_MISC_USER; + + if (thread__tid(gtraceq->thread) != context) + gtrace_set_thread(gtraceq, context); + + if (!thread__find_map(gtraceq->thread, cpumode, address, &al)) + goto out; + + dso = map__dso(al.map); + if (!dso) + goto out; + + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR && + dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE)) + goto out; + + offset = map__map_ip(al.map, address); + + map__load(al.map); + + len = dso__data_read_offset(dso, machine, offset, buffer, size); + if (len <= 0) { + ui__warning_once("GTRACE: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n" + " Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n"); + if (!dso__auxtrace_warned(dso)) { + pr_err("RISC-V Trace: Debug data not found for address %#"PRIx64" in %s\n", + address, + dso__long_name(dso) ? dso__long_name(dso) : "Unknown"); + dso__set_auxtrace_warned(dso); + } + goto out; + } + ret = len; +out: + addr_location__exit(&al); + return ret; +} + +static u8 gtrace_cpu_mode(enum riscv_privilege_mode prv) +{ + u8 cpumode; + + switch (prv) { + case RISCV_PRIV_USER_MODE: + cpumode = PERF_RECORD_MISC_USER; + break; + case RISCV_PRIV_SUPERVISOR_MODE: + case RISCV_PRIV_MACHINE_MODE: + default: + cpumode = PERF_RECORD_MISC_KERNEL; + break; + } + return cpumode; +} + +static void gtrace_synth_copy_insn(struct gtrace_queue *gtraceq, + struct rvtrace_packet *packet, + struct perf_sample *sample) +{ + int ret; + u32 insn; + + ret = gtrace_mem_access(gtraceq, sample->ip, packet->prv, packet->context, + sizeof(insn), (u8 *)&insn); + if (!ret) + return; + + sample->insn_len = ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_GE_32) ? 4 : 2; + memcpy(sample->insn, &insn, sample->insn_len); +} + +static inline u64 gtrace_resolve_sample_time(struct gtrace_queue *gtraceq, + struct rvtrace_packet *packet) +{ + struct gtrace_decoder *ptr = gtraceq->ptr; + + if (!ptr->timeless_decoding) + return packet->timestamp; + else + return ptr->latest_kernel_timestamp; +} + +static int gtrace_synth_branch_sample(struct gtrace_queue *gtraceq, + struct rvtrace_packet *packet) +{ + int ret = 0; + struct gtrace_decoder *ptr = gtraceq->ptr; + struct perf_sample sample; + union perf_event *event = gtraceq->event_buf; + + perf_sample__init(&sample, /*all=*/true); + event->sample.header.type = PERF_RECORD_SAMPLE; + event->sample.header.misc = gtrace_cpu_mode(packet->prv); + event->sample.header.size = sizeof(struct perf_event_header); + + if (thread__tid(gtraceq->thread) != packet->context) + gtrace_set_thread(gtraceq, packet->context); + + /* Set time field based on rvtrace auxtrace config. */ + sample.time = gtrace_resolve_sample_time(gtraceq, packet); + + sample.ip = packet->start_addr; + sample.pid = thread__pid(gtraceq->thread); + sample.tid = thread__tid(gtraceq->thread); + sample.addr = packet->end_addr; + sample.insn_cnt = packet->insn_cnt; + sample.id = gtraceq->ptr->branches_id; + sample.stream_id = gtraceq->ptr->branches_id; + sample.period = 1; + sample.cpu = packet->cpu; + sample.flags = 0; + sample.cpumode = event->sample.header.misc; + + gtrace_synth_copy_insn(gtraceq, packet, &sample); + + ret = perf_session__deliver_synth_event(ptr->session, event, &sample); + if (ret) + pr_err( + "RISC-V Trace: failed to deliver instruction event, error %d\n", + ret); + perf_sample__exit(&sample); + + return ret; +} + +static int gtrace_synth_events(struct gtrace_decoder *ptr, + struct perf_session *session) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + struct perf_event_attr attr; + bool found = false; + u64 id; + int err; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type == ptr->pmu_type) { + found = true; + break; + } + } + + if (!found) { + pr_debug("No selected events with RISC-V Trace data\n"); + return 0; + } + + memset(&attr, 0, sizeof(struct perf_event_attr)); + attr.size = sizeof(struct perf_event_attr); + attr.type = PERF_TYPE_HARDWARE; + attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK; + attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | + PERF_SAMPLE_PERIOD; + if (ptr->timeless_decoding) + attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; + else + attr.sample_type |= PERF_SAMPLE_TIME; + + attr.exclude_user = evsel->core.attr.exclude_user; + attr.exclude_kernel = evsel->core.attr.exclude_kernel; + attr.exclude_hv = evsel->core.attr.exclude_hv; + attr.exclude_host = evsel->core.attr.exclude_host; + attr.exclude_guest = evsel->core.attr.exclude_guest; + attr.sample_id_all = evsel->core.attr.sample_id_all; + attr.read_format = evsel->core.attr.read_format; + + /* create new id val to be a fixed offset from evsel id */ + id = evsel->core.id[0] + 1000000000; + if (!id) + id = 1; + + if (ptr->synth_opts.branches) { + attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS; + attr.sample_period = 1; + attr.sample_type |= PERF_SAMPLE_ADDR; + + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + ptr->branches_sample_type = attr.sample_type; + ptr->branches_id = id; + id += 1; + attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR; + } + + return 0; +} + +static int gtrace_process_queue(struct gtrace_queue *gtraceq) +{ + struct rvtrace_packet_buffer *packet_buffer = &gtraceq->decoder->packet_buffer; + + for (int i = 0; i < packet_buffer->size; i++) { + struct rvtrace_packet packet = packet_buffer->packets[i]; + + if (packet.sample_type == RVTRACE_RANGE) + gtrace_synth_branch_sample(gtraceq, &packet); + else if (packet.sample_type == RVTRACE_LOSS) + fprintf(stdout, "RISC-V Trace: FIFO overrun, messages lost\n"); + else if (packet.sample_type == RVTRACE_ERROR) + fprintf(stdout, "RISC-V Trace: Packet data parsing error\n"); + } + + return 0; +} + +static int gtrace_get_trace(struct gtrace_queue *gtraceq) +{ + struct auxtrace_buffer *aux_buffer = gtraceq->buffer; + struct auxtrace_buffer *old_buffer = aux_buffer; + struct auxtrace_queue *queue; + + queue = &gtraceq->ptr->queues.queue_array[gtraceq->queue_nr]; + + aux_buffer = auxtrace_buffer__next(queue, aux_buffer); + + /* If no more data, drop the previous auxtrace_buffer and return */ + if (!aux_buffer) { + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + return 0; + } + + gtraceq->buffer = aux_buffer; + + /* If the aux_buffer doesn't have data associated, try to load it */ + if (!aux_buffer->data) { + /* get the file desc associated with the perf data file */ + int fd = perf_data__fd(gtraceq->ptr->session->data); + + aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd); + if (!aux_buffer->data) + return -ENOMEM; + } + + /* If valid, drop the previous buffer */ + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + + return aux_buffer->size; +} + +/* + * gtrace_get_data_block: Fetch a block from the auxtrace_buffer queue + * if need be. + * Returns: < 0 if error + * = 0 if no more auxtrace_buffer to read + * > 0 if the current buffer isn't empty yet + */ +static int gtrace_get_data_block(struct gtrace_queue *gtraceq) +{ + int ret; + + ret = gtrace_get_trace(gtraceq); + if (ret <= 0) + return ret; + + /* + * We cannot assume consecutive blocks in the data file + * are contiguous, reset the decoder to force re-sync. + */ + ret = rvtrace_decoder_reset(gtraceq->decoder); + if (ret) + return ret; + + return gtraceq->buffer->size; +} + +static int gtrace_run_timeless_decoder(struct gtrace_queue *gtraceq) +{ + struct rvtrace_decoder *decoder = gtraceq->decoder; + int ret; + + while (1) { + ret = gtrace_get_data_block(gtraceq); + if (ret < 0) + return ret; + + if (ret == 0) + break; + + ret = decoder->decode(decoder, RVTRACE_DECODE_INSN, + gtraceq->buffer->data, + gtraceq->buffer->size); + if (ret) + return ret; + + ret = gtrace_process_queue(gtraceq); + if (ret) + return ret; + } + + return ret; +} + +static int gtrace_process_timeless_queues(struct gtrace_decoder *ptr, + pid_t tid) +{ + unsigned int i; + struct auxtrace_queues *queues = &ptr->queues; + + for (i = 0; i < queues->nr_queues; i++) { + struct auxtrace_queue *queue = &ptr->queues.queue_array[i]; + struct gtrace_queue *gtraceq = queue->priv; + + if (gtraceq && ((tid == -1) || (queue->tid == tid))) { + gtrace_set_thread(gtraceq, queue->tid); + gtrace_run_timeless_decoder(gtraceq); + } + } + + return 0; +} + +static u64 gtrace_queue_get_timestamp(struct gtrace_queue *gtraceq) +{ + struct rvtrace_packet_buffer *packet_buffer = &gtraceq->decoder->packet_buffer; + + for (int i = 0; i < packet_buffer->size; i++) { + struct rvtrace_packet packet = packet_buffer->packets[i]; + + if (packet.sample_type == RVTRACE_RANGE) + return packet.timestamp; + } + + return 0; +} + +static int gtrace_queue_first_timestamp(struct gtrace_decoder *ptr, + struct gtrace_queue *gtraceq, + unsigned int queue_nr) +{ + struct rvtrace_decoder *decoder = gtraceq->decoder; + u64 timestamp = 0; + int ret; + + /* Decode the first segment of data until a timestamp is found, + * then add it to the heap for sorting by time across multiple + * queues. + */ + while (1) { + /* + * Fetch an aux_buffer from this gtraceq. Bail if no more + * blocks or an error has been encountered. + */ + ret = gtrace_get_data_block(gtraceq); + if (ret <= 0) + goto out; + + ret = decoder->decode(decoder, RVTRACE_DECODE_INSN, + gtraceq->buffer->data, + gtraceq->buffer->size); + if (ret) + goto out; + + timestamp = gtrace_queue_get_timestamp(gtraceq); + + /* We found a timestamp, no need to continue. */ + if (timestamp) + break; + } + + /* We have a timestamp and add it to the min heap */ + ret = auxtrace_heap__add(&ptr->heap, queue_nr, timestamp); +out: + return ret; +} + +static int gtrace_process_timestamped_queues(struct gtrace_decoder *ptr) +{ + int ret = 0; + unsigned int queue_nr, i; + struct auxtrace_queue *queue; + struct gtrace_queue *gtraceq; + + /* First, find the first timestamp for each queue and add it to the heap. */ + for (i = 0; i < ptr->queues.nr_queues; i++) { + queue = &ptr->queues.queue_array[i]; + gtraceq = queue->priv; + if (!gtraceq) + continue; + + gtrace_set_thread(gtraceq, queue->tid); + + ret = gtrace_queue_first_timestamp(ptr, gtraceq, i); + if (ret) + return ret; + } + + /* Process queues in the heap in timestamp order */ + while (1) { + if (!ptr->heap.heap_cnt) + break; + + /* Take the entry at the top of the min heap */ + queue_nr = ptr->heap.heap_array[0].queue_nr; + queue = &ptr->queues.queue_array[queue_nr]; + gtraceq = queue->priv; + + /* + * Remove the top entry from the heap since we are about + * to process it. + */ + auxtrace_heap__pop(&ptr->heap); + + /* + * Packets associated with this timestamp are already in + * the gtraceq->packet_buffer, so process them. + */ + ret = gtrace_process_queue(gtraceq); + if (ret) + return ret; + + /* + * Packets for this timestamp have been processed, time to + * move on to the next timestamp, find the next timestamp + * for this gtraceq. + */ + ret = gtrace_queue_first_timestamp(ptr, gtraceq, queue_nr); + if (ret) + return ret; + } + + return 0; +} + +static struct gtrace_queue *gtrace_alloc_queue(struct gtrace_decoder *ptr, + unsigned int queue_nr) +{ + struct gtrace_queue *gtraceq; + + gtraceq = zalloc(sizeof(*gtraceq)); + if (!gtraceq) + return NULL; + + gtraceq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); + if (!gtraceq->event_buf) + goto out_free; + + gtraceq->ptr = ptr; + gtraceq->queue_nr = queue_nr; + gtraceq->offset = 0; + + return gtraceq; + +out_free: + zfree(&gtraceq->event_buf); + free(gtraceq); + + return NULL; + +} + +static int gtrace_setup_queue(struct gtrace_decoder *ptr, + struct auxtrace_queue *queue, + unsigned int queue_nr) +{ + struct gtrace_queue *gtraceq = queue->priv; + + if (gtraceq) + return 0; + + gtraceq = gtrace_alloc_queue(ptr, queue_nr); + if (!gtraceq) + return -ENOMEM; + + queue->priv = gtraceq; + queue->cpu = queue_nr; + + return 0; +} + +static void gtrace_setup_timeless_decoding(struct gtrace_decoder *ptr) +{ + u64 *metadata = ptr->metadata[0]; + + /* Override timeless mode with user input from --itrace=Z */ + if (ptr->synth_opts.timeless_decoding) { + ptr->timeless_decoding = true; + return; + } + + ptr->timeless_decoding = metadata[RVTRACE_ENCODER_TIMESTAMP]; +} + static const char * const gtrace_global_header_fmts[] = { [GTRACE_PMU_TYPE_CPUS] = " PMU type/num cpus %llx\n", }; @@ -99,6 +966,251 @@ static void gtrace_print_auxtrace_info(u64 *val, int num_cpu) } }

+static struct gtrace_queue *gtrace_get_queue(struct gtrace_decoder *ptr, int cpu) +{ + if (ptr->per_thread_decoding) + return ptr->queues.queue_array[0].priv; + + if (cpu < 0 || cpu >= (int)ptr->queues.nr_queues) + return NULL; + + return ptr->queues.queue_array[cpu].priv; +} + +/** + * Puts a fragment of an auxtrace buffer into the auxtrace queues based + * on the bounds of aux_event, if it matches with the buffer that's at + * file_offset. + * + * Normally, whole auxtrace buffers would be added to the queue. But we + * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder + * is reset across each buffer, so splitting the buffers up in advance has + * the same effect. + */ +static int gtrace_queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz, + struct perf_record_aux *aux_event, struct perf_sample *sample) +{ + int err; + char buf[PERF_SAMPLE_MAX_SIZE]; + union perf_event *auxtrace_event_union; + struct perf_record_auxtrace *auxtrace_event; + union perf_event auxtrace_fragment; + __u64 aux_offset, aux_size; + enum coresight_format cs_format; + + struct gtrace_decoder *ptr = container_of(session->auxtrace, struct gtrace_decoder, + auxtrace); + + /* + * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got + * from looping through the auxtrace index. + */ + err = perf_session__peek_event(session, file_offset, buf, + PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL); + if (err) + return err; + auxtrace_event = &auxtrace_event_union->auxtrace; + if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE) + return -EINVAL; + + if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) || + auxtrace_event->header.size != sz) { + return -EINVAL; + } + + /* + * Compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1. + * Return 'not found' if mismatch. + */ + if (auxtrace_event->cpu == (__u32) -1) { + ptr->per_thread_decoding = true; + if (auxtrace_event->tid != sample->tid) + return 1; + } else if (auxtrace_event->cpu != sample->cpu) { + if (ptr->per_thread_decoding) { + /* + * Found a per-cpu buffer after a per-thread one was + * already found + */ + pr_err("GTRACE: Inconsistent per-thread/per-cpu mode.\n"); + return -EINVAL; + } + return 1; + } + + if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) { + /* + * Clamp size in snapshot mode. The buffer size is clamped in + * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect + * the buffer size. + */ + aux_size = min(aux_event->aux_size, auxtrace_event->size); + + /* + * In this mode, the head also points to the end of the buffer so aux_offset + * needs to have the size subtracted so it points to the beginning as in normal mode + */ + aux_offset = aux_event->aux_offset - aux_size; + } else { + aux_size = aux_event->aux_size; + aux_offset = aux_event->aux_offset; + } + + if (aux_offset >= auxtrace_event->offset && + aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { + struct gtrace_queue *gtraceq = gtrace_get_queue(ptr, auxtrace_event->cpu); + + if (!gtraceq) + return -EINVAL; + + /* + * If this AUX event was inside this buffer somewhere, create a new auxtrace event + * based on the sizes of the aux event, and queue that fragment. + */ + auxtrace_fragment.auxtrace = *auxtrace_event; + auxtrace_fragment.auxtrace.size = aux_size; + auxtrace_fragment.auxtrace.offset = aux_offset; + auxtrace_fragment.auxtrace.idx = gtraceq->queue_nr; + file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size; + + pr_debug3("GTRACE: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64 + " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu); + err = auxtrace_queues__add_event(&ptr->queues, session, &auxtrace_fragment, + file_offset, NULL); + if (err) + return err; + + cs_format = (aux_event->flags & PERF_AUX_FLAG_GTRACE_FORMAT_CORESIGHT) ? + FORMATTED : UNFORMATTED; + if (gtraceq->cs_format != UNSET && cs_format != gtraceq->cs_format) { + pr_err("GTRACE: mixed coresight formatted and unformatted trace not supported\n"); + return -EINVAL; + } + gtraceq->cs_format = cs_format; + return 0; + } + + /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */ + return 1; +} + +static int gtrace_queue_aux_records_cb(struct perf_session *session, union perf_event *event, + u64 offset __maybe_unused, void *data __maybe_unused) +{ + struct perf_sample sample; + int ret; + struct auxtrace_index_entry *ent; + struct auxtrace_index *auxtrace_index; + struct evsel *evsel; + size_t i; + + /* Don't care about any other events, we're only queuing buffers for AUX events */ + if (event->header.type != PERF_RECORD_AUX) + return 0; + + if (event->header.size < sizeof(struct perf_record_aux)) + return -EINVAL; + + /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */ + if (!event->aux.aux_size) + return 0; + + /* + * Parse the sample, we need the sample_id_all data that comes after the event so that the + * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID. + */ + evsel = evlist__event2evsel(session->evlist, event); + if (!evsel) + return -EINVAL; + perf_sample__init(&sample, /*all=*/false); + ret = evsel__parse_sample(evsel, event, &sample); + if (ret) + goto out; + + /* + * Loop through the auxtrace index to find the buffer that matches up with this aux event. + */ + list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) { + for (i = 0; i < auxtrace_index->nr; i++) { + ent = &auxtrace_index->entries[i]; + ret = gtrace_queue_aux_fragment(session, ent->file_offset, + ent->sz, &event->aux, &sample); + /* + * Stop search on error or successful values. Continue search on + * 1 ('not found') + */ + if (ret != 1) + goto out; + } + } + + /* + * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but + * don't exit with an error because it will still be possible to decode other aux records. + */ + pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64 + " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu); + ret = 0; +out: + perf_sample__exit(&sample); + return ret; + +} + +static int gtrace_queue_aux_records(struct perf_session *session) +{ + struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index, + struct auxtrace_index, list); + + if (index && index->nr > 0) + return perf_session__peek_events(session, session->header.data_offset, + session->header.data_size, + gtrace_queue_aux_records_cb, NULL); + + return 0; +} + +static int gtrace_create_queue_decoder(struct gtrace_queue *gtraceq) +{ + struct gtrace_decoder *ptr = gtraceq->ptr; + struct rvtrace_decoder_params params; + + params.data = gtraceq; + params.mem_access = gtrace_mem_access; + params.coresight_formatted = gtraceq->cs_format == FORMATTED; + params.format = ptr->metadata[gtraceq->queue_nr][RVTRACE_ENCODER_FORMAT]; + params.srcbits = ptr->metadata[gtraceq->queue_nr][RVTRACE_ENCODER_SRCBITS]; + gtraceq->decoder = rvtrace_decoder_new(&params); + if (!gtraceq->decoder) + return -EINVAL; + + return 0; +} + +static int gtrace_create_decoders(struct gtrace_decoder *ptr) +{ + struct auxtrace_queues *queues = &ptr->queues; + + for (unsigned int i = 0; i < queues->nr_queues; i++) { + bool empty = list_empty(&queues->queue_array[i].head); + struct gtrace_queue *gtraceq = queues->queue_array[i].priv; + int ret; + + /* + * Don't create decoders for empty queues, mainly because + * q->format is unknown for empty queues. + */ + assert(empty || gtraceq->cs_format != UNSET); + if (empty) + continue; + + ret = gtrace_create_queue_decoder(gtraceq); + if (ret) + return ret; + } + return 0; +} + int gtrace__process_auxtrace_info(union perf_event *event, struct perf_session *session) { @@ -106,7 +1218,7 @@ int gtrace__process_auxtrace_info(union perf_event *event, struct gtrace_decoder *ptr; u64 *priv; u64 **metadata; - int num_cpu; + int num_cpu, max_cpu = 0; int priv_size; int err; int i; @@ -115,7 +1227,6 @@ int gtrace__process_auxtrace_info(union perf_event *event, num_cpu = priv[GTRACE_PMU_TYPE_CPUS] & 0xffffffff; if (dump_trace) gtrace_print_auxtrace_info(priv, num_cpu); - priv_size = auxtrace_info->header.size - sizeof(struct perf_event_header) - INFO_HEADER_SIZE - GTRACE_HEADER_SIZE; if (num_cpu <= 0 || priv_size <= 0 || @@ -126,7 +1237,7 @@ int gtrace__process_auxtrace_info(union perf_event *event, if (!metadata) return -ENOMEM;

- i = GTRACE_HEADER_SIZE; + i = GTRACE_HEADER_MAX;

for (int j = 0; j < num_cpu; j++) { metadata[j] = zalloc(sizeof(*metadata[j]) * RVTRACE_ENCODER_PRIV_MAX); @@ -137,6 +1248,9 @@ int gtrace__process_auxtrace_info(union perf_event *event,

for (int k = 0; k < RVTRACE_ENCODER_PRIV_MAX; k++) metadata[j][k] = priv[i + k]; + + if ((int)metadata[j][RVTRACE_ENCODER_CPU] > max_cpu) + max_cpu = metadata[j][RVTRACE_ENCODER_CPU]; i += RVTRACE_ENCODER_PRIV_MAX; }

@@ -146,6 +1260,26 @@ int gtrace__process_auxtrace_info(union perf_event *event, goto err_free_metadata; }

+ err = auxtrace_queues__init_nr(&ptr->queues, max_cpu + 1); + if (err) + goto err_free_metadata; + + for (unsigned int j = 0; j < ptr->queues.nr_queues; ++j) { + err = gtrace_setup_queue(ptr, &ptr->queues.queue_array[j], j); + if (err) + goto err_free_queues; + } + + if (session->itrace_synth_opts->set) { + ptr->synth_opts = *session->itrace_synth_opts; + } else { + itrace_synth_opts__set_default(&ptr->synth_opts, + session->itrace_synth_opts->default_no_sample); + ptr->synth_opts.callchain = false; + ptr->synth_opts.thread_stack = session->itrace_synth_opts->thread_stack; + } + + ptr->session = session; ptr->machine = &session->machines.host; ptr->auxtrace_type = auxtrace_info->type; @@ -161,10 +1295,52 @@ int gtrace__process_auxtrace_info(union perf_event *event, ptr->auxtrace.evsel_is_auxtrace = gtrace_evsel_is_auxtrace; session->auxtrace = &ptr->auxtrace;

- return 0; + gtrace_setup_timeless_decoding(ptr); + + /* Load firmware binary if specified */ + if (symbol_conf.firmware_name) { + u64 base_addr = 0; + + if (symbol_conf.firmware_base) + base_addr = strtoull(symbol_conf.firmware_base, NULL, 0); + + ptr->firmware_dso = dso__new(symbol_conf.firmware_name); + if (ptr->firmware_dso) { + ptr->firmware_maps = maps__new(ptr->machine); + if (!ptr->firmware_maps || + gtrace_register_firmware_maps(ptr, + ptr->firmware_dso, + base_addr) < 0) { + pr_err("Failed to register machine code maps: %s\n", + symbol_conf.firmware_name); + dso__put(ptr->firmware_dso); + ptr->firmware_dso = NULL; + maps__put(ptr->firmware_maps); + ptr->firmware_maps = NULL; + } + } + }

+ err = gtrace_synth_events(ptr, session); + if (err) + goto err_free_queues; + + err = gtrace_queue_aux_records(session); + if (err) + goto err_free_queues; + + err = gtrace_create_decoders(ptr); + if (err) + goto err_free_queues; + + ptr->data_queued = ptr->queues.populated; + return 0; +err_free_queues: + auxtrace_queues__free(&ptr->queues); + session->auxtrace = NULL; err_free_metadata: for (int j = 0; j < num_cpu; j++) - zfree(&metadata); + zfree(&metadata[j]); + zfree(&metadata); return err; } diff --git a/tools/perf/util/symbol_conf.h b/tools/perf/util/symbol_conf.h index 0dee5aa6a534..83fc134ea8ac 100644 --- a/tools/perf/util/symbol_conf.h +++ b/tools/perf/util/symbol_conf.h @@ -72,6 +72,8 @@ struct symbol_conf { prefer_latency; const char *vmlinux_name, *kallsyms_name, + *firmware_name, + *firmware_base, *source_prefix, *field_sep, *graph_function;