/* * hwlat_detector.c - A simple Hardware Latency detector. * * Use this module to detect large system latencies induced by the behavior of * certain underlying system hardware or firmware, independent of Linux itself. * The code was developed originally to detect the presence of SMIs on Intel * and AMD systems, although there is no dependency upon x86 herein. * * The classical example usage of this module is in detecting the presence of * SMIs or System Management Interrupts on Intel and AMD systems. An SMI is a * somewhat special form of hardware interrupt spawned from earlier CPU debug * modes in which the (BIOS/EFI/etc.) firmware arranges for the South Bridge * LPC (or other device) to generate a special interrupt under certain * circumstances, for example, upon expiration of a special SMI timer device, * due to certain external thermal readings, on certain I/O address accesses, * and other situations. An SMI hits a special CPU pin, triggers a special * SMI mode (complete with special memory map), and the OS is unaware. * * Although certain hardware-inducing latencies are necessary (for example, * a modern system often requires an SMI handler for correct thermal control * and remote management) they can wreak havoc upon any OS-level performance * guarantees toward low-latency, especially when the OS is not even made * aware of the presence of these interrupts. For this reason, we need a * somewhat brute force mechanism to detect these interrupts. In this case, * we do it by hogging all of the CPU(s) for configurable timer intervals, * sampling the built-in CPU timer, looking for discontiguous readings. * * WARNING: This implementation necessarily introduces latencies. Therefore, * you should NEVER use this module in a production environment * requiring any kind of low-latency performance guarantee(s). * * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. * * Includes useful feedback from Clark Williams * * This file is licensed under the terms of the GNU General Public * License version 2. This program is licensed "as is" without any * warranty of any kind, whether express or implied. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #define BUF_SIZE_DEFAULT 262144UL /* 8K*(sizeof(entry)) */ #define BUF_FLAGS (RB_FL_OVERWRITE) /* no block on full */ #define U64STR_SIZE 22 /* 20 digits max */ #define VERSION "1.0.0" #define BANNER "hwlat_detector: " #define DRVNAME "hwlat_detector" #define DEFAULT_SAMPLE_WINDOW 1000000 /* 1s */ #define DEFAULT_SAMPLE_WIDTH 500000 /* 0.5s */ #define DEFAULT_LAT_THRESHOLD 10 /* 10us */ /* Module metadata */ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Jon Masters "); MODULE_DESCRIPTION("A simple hardware latency detector"); MODULE_VERSION(VERSION); /* Module parameters */ static int debug; static int enabled; static int threshold; module_param(debug, int, 0); /* enable debug */ module_param(enabled, int, 0); /* enable detector */ module_param(threshold, int, 0); /* latency threshold */ /* Buffering and sampling */ static struct ring_buffer *ring_buffer; /* sample buffer */ static DEFINE_MUTEX(ring_buffer_mutex); /* lock changes */ static unsigned long buf_size = BUF_SIZE_DEFAULT; static struct task_struct *kthread; /* sampling thread */ /* DebugFS filesystem entries */ static struct dentry *debug_dir; /* debugfs directory */ static struct dentry *debug_max; /* maximum TSC delta */ static struct dentry *debug_count; /* total detect count */ static struct dentry *debug_sample_width; /* sample width us */ static struct dentry *debug_sample_window; /* sample window us */ static struct dentry *debug_sample; /* raw samples us */ static struct dentry *debug_threshold; /* threshold us */ static struct dentry *debug_enable; /* enable/disable */ /* Individual samples and global state */ struct sample; /* latency sample */ struct data; /* Global state */ /* Sampling functions */ static int __buffer_add_sample(struct sample *sample); static struct sample *buffer_get_sample(struct sample *sample); /* Threading and state */ static int kthread_fn(void *unused); static int start_kthread(void); static int stop_kthread(void); static void __reset_stats(void); static int init_stats(void); /* Debugfs interface */ static ssize_t simple_data_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, const u64 *entry); static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u64 *entry); static int debug_sample_fopen(struct inode *inode, struct file *filp); static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos); static int debug_sample_release(struct inode *inode, struct file *filp); static int debug_enable_fopen(struct inode *inode, struct file *filp); static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos); static ssize_t debug_enable_fwrite(struct file *file, const char __user *user_buffer, size_t user_size, loff_t *offset); /* Initialization functions */ static int init_debugfs(void); static void free_debugfs(void); static int detector_init(void); static void detector_exit(void); /* Individual latency samples are stored here when detected and packed into * the ring_buffer circular buffer, where they are overwritten when * more than buf_size/sizeof(sample) samples are received. */ struct sample { u64 seqnum; /* unique sequence */ u64 duration; /* ktime delta */ u64 outer_duration; /* ktime delta (outer loop) */ struct timespec timestamp; /* wall time */ unsigned long lost; }; /* keep the global state somewhere. */ static struct data { struct mutex lock; /* protect changes */ u64 count; /* total since reset */ u64 max_sample; /* max hardware latency */ u64 threshold; /* sample threshold level */ u64 sample_window; /* total sampling window (on+off) */ u64 sample_width; /* active sampling portion of window */ atomic_t sample_open; /* whether the sample file is open */ wait_queue_head_t wq; /* waitqeue for new sample values */ } data; /** * __buffer_add_sample - add a new latency sample recording to the ring buffer * @sample: The new latency sample value * * This receives a new latency sample and records it in a global ring buffer. * No additional locking is used in this case. */ static int __buffer_add_sample(struct sample *sample) { return ring_buffer_write(ring_buffer, sizeof(struct sample), sample); } /** * buffer_get_sample - remove a hardware latency sample from the ring buffer * @sample: Pre-allocated storage for the sample * * This retrieves a hardware latency sample from the global circular buffer */ static struct sample *buffer_get_sample(struct sample *sample) { struct ring_buffer_event *e = NULL; struct sample *s = NULL; unsigned int cpu = 0; if (!sample) return NULL; mutex_lock(&ring_buffer_mutex); for_each_online_cpu(cpu) { e = ring_buffer_consume(ring_buffer, cpu, NULL, &sample->lost); if (e) break; } if (e) { s = ring_buffer_event_data(e); memcpy(sample, s, sizeof(struct sample)); } else sample = NULL; mutex_unlock(&ring_buffer_mutex); return sample; } #ifndef CONFIG_TRACING #define time_type ktime_t #define time_get() ktime_get() #define time_to_us(x) ktime_to_us(x) #define time_sub(a, b) ktime_sub(a, b) #define init_time(a, b) (a).tv64 = b #define time_u64(a) ((a).tv64) #else #define time_type u64 #define time_get() trace_clock_local() #define time_to_us(x) div_u64(x, 1000) #define time_sub(a, b) ((a) - (b)) #define init_time(a, b) (a = b) #define time_u64(a) a #endif /** * get_sample - sample the CPU TSC and look for likely hardware latencies * * Used to repeatedly capture the CPU TSC (or similar), looking for potential * hardware-induced latency. Called with interrupts disabled and with * data.lock held. */ static int get_sample(void) { time_type start, t1, t2, last_t2; s64 diff, total = 0; u64 sample = 0; u64 outer_sample = 0; int ret = -1; init_time(last_t2, 0); start = time_get(); /* start timestamp */ do { t1 = time_get(); /* we'll look for a discontinuity */ t2 = time_get(); if (time_u64(last_t2)) { /* Check the delta from outer loop (t2 to next t1) */ diff = time_to_us(time_sub(t1, last_t2)); /* This shouldn't happen */ if (diff < 0) { pr_err(BANNER "time running backwards\n"); goto out; } if (diff > outer_sample) outer_sample = diff; } last_t2 = t2; total = time_to_us(time_sub(t2, start)); /* sample width */ /* This checks the inner loop (t1 to t2) */ diff = time_to_us(time_sub(t2, t1)); /* current diff */ /* This shouldn't happen */ if (diff < 0) { pr_err(BANNER "time running backwards\n"); goto out; } if (diff > sample) sample = diff; /* only want highest value */ } while (total <= data.sample_width); ret = 0; /* If we exceed the threshold value, we have found a hardware latency */ if (sample > data.threshold || outer_sample > data.threshold) { struct sample s; ret = 1; data.count++; s.seqnum = data.count; s.duration = sample; s.outer_duration = outer_sample; s.timestamp = CURRENT_TIME; __buffer_add_sample(&s); /* Keep a running maximum ever recorded hardware latency */ if (sample > data.max_sample) data.max_sample = sample; } out: return ret; } /* * kthread_fn - The CPU time sampling/hardware latency detection kernel thread * @unused: A required part of the kthread API. * * Used to periodically sample the CPU TSC via a call to get_sample. We * disable interrupts, which does (intentionally) introduce latency since we * need to ensure nothing else might be running (and thus pre-empting). * Obviously this should never be used in production environments. * * Currently this runs on which ever CPU it was scheduled on, but most * real-worald hardware latency situations occur across several CPUs, * but we might later generalize this if we find there are any actualy * systems with alternate SMI delivery or other hardware latencies. */ static int kthread_fn(void *unused) { int ret; u64 interval; while (!kthread_should_stop()) { mutex_lock(&data.lock); local_irq_disable(); ret = get_sample(); local_irq_enable(); if (ret > 0) wake_up(&data.wq); /* wake up reader(s) */ interval = data.sample_window - data.sample_width; do_div(interval, USEC_PER_MSEC); /* modifies interval value */ mutex_unlock(&data.lock); if (msleep_interruptible(interval)) break; } return 0; } /** * start_kthread - Kick off the hardware latency sampling/detector kthread * * This starts a kernel thread that will sit and sample the CPU timestamp * counter (TSC or similar) and look for potential hardware latencies. */ static int start_kthread(void) { kthread = kthread_run(kthread_fn, NULL, DRVNAME); if (IS_ERR(kthread)) { pr_err(BANNER "could not start sampling thread\n"); enabled = 0; return -ENOMEM; } return 0; } /** * stop_kthread - Inform the hardware latency samping/detector kthread to stop * * This kicks the running hardware latency sampling/detector kernel thread and * tells it to stop sampling now. Use this on unload and at system shutdown. */ static int stop_kthread(void) { int ret; ret = kthread_stop(kthread); return ret; } /** * __reset_stats - Reset statistics for the hardware latency detector * * We use data to store various statistics and global state. We call this * function in order to reset those when "enable" is toggled on or off, and * also at initialization. Should be called with data.lock held. */ static void __reset_stats(void) { data.count = 0; data.max_sample = 0; ring_buffer_reset(ring_buffer); /* flush out old sample entries */ } /** * init_stats - Setup global state statistics for the hardware latency detector * * We use data to store various statistics and global state. We also use * a global ring buffer (ring_buffer) to keep raw samples of detected hardware * induced system latencies. This function initializes these structures and * allocates the global ring buffer also. */ static int init_stats(void) { int ret = -ENOMEM; mutex_init(&data.lock); init_waitqueue_head(&data.wq); atomic_set(&data.sample_open, 0); ring_buffer = ring_buffer_alloc(buf_size, BUF_FLAGS); if (WARN(!ring_buffer, KERN_ERR BANNER "failed to allocate ring buffer!\n")) goto out; __reset_stats(); data.threshold = threshold ?: DEFAULT_LAT_THRESHOLD; /* threshold us */ data.sample_window = DEFAULT_SAMPLE_WINDOW; /* window us */ data.sample_width = DEFAULT_SAMPLE_WIDTH; /* width us */ ret = 0; out: return ret; } /* * simple_data_read - Wrapper read function for global state debugfs entries * @filp: The active open file structure for the debugfs "file" * @ubuf: The userspace provided buffer to read value into * @cnt: The maximum number of bytes to read * @ppos: The current "file" position * @entry: The entry to read from * * This function provides a generic read implementation for the global state * "data" structure debugfs filesystem entries. It would be nice to use * simple_attr_read directly, but we need to make sure that the data.lock * is held during the actual read. */ static ssize_t simple_data_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, const u64 *entry) { char buf[U64STR_SIZE]; u64 val = 0; int len = 0; memset(buf, 0, sizeof(buf)); if (!entry) return -EFAULT; mutex_lock(&data.lock); val = *entry; mutex_unlock(&data.lock); len = snprintf(buf, sizeof(buf), "%llu\n", (unsigned long long)val); return simple_read_from_buffer(ubuf, cnt, ppos, buf, len); } /* * simple_data_write - Wrapper write function for global state debugfs entries * @filp: The active open file structure for the debugfs "file" * @ubuf: The userspace provided buffer to write value from * @cnt: The maximum number of bytes to write * @ppos: The current "file" position * @entry: The entry to write to * * This function provides a generic write implementation for the global state * "data" structure debugfs filesystem entries. It would be nice to use * simple_attr_write directly, but we need to make sure that the data.lock * is held during the actual write. */ static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u64 *entry) { char buf[U64STR_SIZE]; int csize = min(cnt, sizeof(buf)); u64 val = 0; int err = 0; memset(buf, '\0', sizeof(buf)); if (copy_from_user(buf, ubuf, csize)) return -EFAULT; buf[U64STR_SIZE-1] = '\0'; /* just in case */ err = kstrtoull(buf, 10, &val); if (err) return -EINVAL; mutex_lock(&data.lock); *entry = val; mutex_unlock(&data.lock); return csize; } /** * debug_count_fopen - Open function for "count" debugfs entry * @inode: The in-kernel inode representation of the debugfs "file" * @filp: The active open file structure for the debugfs "file" * * This function provides an open implementation for the "count" debugfs * interface to the hardware latency detector. */ static int debug_count_fopen(struct inode *inode, struct file *filp) { return 0; } /** * debug_count_fread - Read function for "count" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The userspace provided buffer to read value into * @cnt: The maximum number of bytes to read * @ppos: The current "file" position * * This function provides a read implementation for the "count" debugfs * interface to the hardware latency detector. Can be used to read the * number of latency readings exceeding the configured threshold since * the detector was last reset (e.g. by writing a zero into "count"). */ static ssize_t debug_count_fread(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { return simple_data_read(filp, ubuf, cnt, ppos, &data.count); } /** * debug_count_fwrite - Write function for "count" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The user buffer that contains the value to write * @cnt: The maximum number of bytes to write to "file" * @ppos: The current position in the debugfs "file" * * This function provides a write implementation for the "count" debugfs * interface to the hardware latency detector. Can be used to write a * desired value, especially to zero the total count. */ static ssize_t debug_count_fwrite(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { return simple_data_write(filp, ubuf, cnt, ppos, &data.count); } /** * debug_enable_fopen - Dummy open function for "enable" debugfs interface * @inode: The in-kernel inode representation of the debugfs "file" * @filp: The active open file structure for the debugfs "file" * * This function provides an open implementation for the "enable" debugfs * interface to the hardware latency detector. */ static int debug_enable_fopen(struct inode *inode, struct file *filp) { return 0; } /** * debug_enable_fread - Read function for "enable" debugfs interface * @filp: The active open file structure for the debugfs "file" * @ubuf: The userspace provided buffer to read value into * @cnt: The maximum number of bytes to read * @ppos: The current "file" position * * This function provides a read implementation for the "enable" debugfs * interface to the hardware latency detector. Can be used to determine * whether the detector is currently enabled ("0\n" or "1\n" returned). */ static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { char buf[4]; if ((cnt < sizeof(buf)) || (*ppos)) return 0; buf[0] = enabled ? '1' : '0'; buf[1] = '\n'; buf[2] = '\0'; if (copy_to_user(ubuf, buf, strlen(buf))) return -EFAULT; return *ppos = strlen(buf); } /** * debug_enable_fwrite - Write function for "enable" debugfs interface * @filp: The active open file structure for the debugfs "file" * @ubuf: The user buffer that contains the value to write * @cnt: The maximum number of bytes to write to "file" * @ppos: The current position in the debugfs "file" * * This function provides a write implementation for the "enable" debugfs * interface to the hardware latency detector. Can be used to enable or * disable the detector, which will have the side-effect of possibly * also resetting the global stats and kicking off the measuring * kthread (on an enable) or the converse (upon a disable). */ static ssize_t debug_enable_fwrite(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { char buf[4]; int csize = min(cnt, sizeof(buf)); long val = 0; int err = 0; memset(buf, '\0', sizeof(buf)); if (copy_from_user(buf, ubuf, csize)) return -EFAULT; buf[sizeof(buf)-1] = '\0'; /* just in case */ err = kstrtoul(buf, 10, &val); if (0 != err) return -EINVAL; if (val) { if (enabled) goto unlock; enabled = 1; __reset_stats(); if (start_kthread()) return -EFAULT; } else { if (!enabled) goto unlock; enabled = 0; err = stop_kthread(); if (err) { pr_err(BANNER "cannot stop kthread\n"); return -EFAULT; } wake_up(&data.wq); /* reader(s) should return */ } unlock: return csize; } /** * debug_max_fopen - Open function for "max" debugfs entry * @inode: The in-kernel inode representation of the debugfs "file" * @filp: The active open file structure for the debugfs "file" * * This function provides an open implementation for the "max" debugfs * interface to the hardware latency detector. */ static int debug_max_fopen(struct inode *inode, struct file *filp) { return 0; } /** * debug_max_fread - Read function for "max" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The userspace provided buffer to read value into * @cnt: The maximum number of bytes to read * @ppos: The current "file" position * * This function provides a read implementation for the "max" debugfs * interface to the hardware latency detector. Can be used to determine * the maximum latency value observed since it was last reset. */ static ssize_t debug_max_fread(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { return simple_data_read(filp, ubuf, cnt, ppos, &data.max_sample); } /** * debug_max_fwrite - Write function for "max" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The user buffer that contains the value to write * @cnt: The maximum number of bytes to write to "file" * @ppos: The current position in the debugfs "file" * * This function provides a write implementation for the "max" debugfs * interface to the hardware latency detector. Can be used to reset the * maximum or set it to some other desired value - if, then, subsequent * measurements exceed this value, the maximum will be updated. */ static ssize_t debug_max_fwrite(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { return simple_data_write(filp, ubuf, cnt, ppos, &data.max_sample); } /** * debug_sample_fopen - An open function for "sample" debugfs interface * @inode: The in-kernel inode representation of this debugfs "file" * @filp: The active open file structure for the debugfs "file" * * This function handles opening the "sample" file within the hardware * latency detector debugfs directory interface. This file is used to read * raw samples from the global ring_buffer and allows the user to see a * running latency history. Can be opened blocking or non-blocking, * affecting whether it behaves as a buffer read pipe, or does not. * Implements simple locking to prevent multiple simultaneous use. */ static int debug_sample_fopen(struct inode *inode, struct file *filp) { if (!atomic_add_unless(&data.sample_open, 1, 1)) return -EBUSY; else return 0; } /** * debug_sample_fread - A read function for "sample" debugfs interface * @filp: The active open file structure for the debugfs "file" * @ubuf: The user buffer that will contain the samples read * @cnt: The maximum bytes to read from the debugfs "file" * @ppos: The current position in the debugfs "file" * * This function handles reading from the "sample" file within the hardware * latency detector debugfs directory interface. This file is used to read * raw samples from the global ring_buffer and allows the user to see a * running latency history. By default this will block pending a new * value written into the sample buffer, unless there are already a * number of value(s) waiting in the buffer, or the sample file was * previously opened in a non-blocking mode of operation. */ static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { int len = 0; char buf[64]; struct sample *sample = NULL; if (!enabled) return 0; sample = kzalloc(sizeof(struct sample), GFP_KERNEL); if (!sample) return -ENOMEM; while (!buffer_get_sample(sample)) { DEFINE_WAIT(wait); if (filp->f_flags & O_NONBLOCK) { len = -EAGAIN; goto out; } prepare_to_wait(&data.wq, &wait, TASK_INTERRUPTIBLE); schedule(); finish_wait(&data.wq, &wait); if (signal_pending(current)) { len = -EINTR; goto out; } if (!enabled) { /* enable was toggled */ len = 0; goto out; } } len = snprintf(buf, sizeof(buf), "%010lu.%010lu\t%llu\t%llu\n", sample->timestamp.tv_sec, sample->timestamp.tv_nsec, sample->duration, sample->outer_duration); /* handling partial reads is more trouble than it's worth */ if (len > cnt) goto out; if (copy_to_user(ubuf, buf, len)) len = -EFAULT; out: kfree(sample); return len; } /** * debug_sample_release - Release function for "sample" debugfs interface * @inode: The in-kernel inode represenation of the debugfs "file" * @filp: The active open file structure for the debugfs "file" * * This function completes the close of the debugfs interface "sample" file. * Frees the sample_open "lock" so that other users may open the interface. */ static int debug_sample_release(struct inode *inode, struct file *filp) { atomic_dec(&data.sample_open); return 0; } /** * debug_threshold_fopen - Open function for "threshold" debugfs entry * @inode: The in-kernel inode representation of the debugfs "file" * @filp: The active open file structure for the debugfs "file" * * This function provides an open implementation for the "threshold" debugfs * interface to the hardware latency detector. */ static int debug_threshold_fopen(struct inode *inode, struct file *filp) { return 0; } /** * debug_threshold_fread - Read function for "threshold" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The userspace provided buffer to read value into * @cnt: The maximum number of bytes to read * @ppos: The current "file" position * * This function provides a read implementation for the "threshold" debugfs * interface to the hardware latency detector. It can be used to determine * the current threshold level at which a latency will be recorded in the * global ring buffer, typically on the order of 10us. */ static ssize_t debug_threshold_fread(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { return simple_data_read(filp, ubuf, cnt, ppos, &data.threshold); } /** * debug_threshold_fwrite - Write function for "threshold" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The user buffer that contains the value to write * @cnt: The maximum number of bytes to write to "file" * @ppos: The current position in the debugfs "file" * * This function provides a write implementation for the "threshold" debugfs * interface to the hardware latency detector. It can be used to configure * the threshold level at which any subsequently detected latencies will * be recorded into the global ring buffer. */ static ssize_t debug_threshold_fwrite(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { int ret; ret = simple_data_write(filp, ubuf, cnt, ppos, &data.threshold); if (enabled) wake_up_process(kthread); return ret; } /** * debug_width_fopen - Open function for "width" debugfs entry * @inode: The in-kernel inode representation of the debugfs "file" * @filp: The active open file structure for the debugfs "file" * * This function provides an open implementation for the "width" debugfs * interface to the hardware latency detector. */ static int debug_width_fopen(struct inode *inode, struct file *filp) { return 0; } /** * debug_width_fread - Read function for "width" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The userspace provided buffer to read value into * @cnt: The maximum number of bytes to read * @ppos: The current "file" position * * This function provides a read implementation for the "width" debugfs * interface to the hardware latency detector. It can be used to determine * for how many us of the total window us we will actively sample for any * hardware-induced latecy periods. Obviously, it is not possible to * sample constantly and have the system respond to a sample reader, or, * worse, without having the system appear to have gone out to lunch. */ static ssize_t debug_width_fread(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_width); } /** * debug_width_fwrite - Write function for "width" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The user buffer that contains the value to write * @cnt: The maximum number of bytes to write to "file" * @ppos: The current position in the debugfs "file" * * This function provides a write implementation for the "width" debugfs * interface to the hardware latency detector. It can be used to configure * for how many us of the total window us we will actively sample for any * hardware-induced latency periods. Obviously, it is not possible to * sample constantly and have the system respond to a sample reader, or, * worse, without having the system appear to have gone out to lunch. It * is enforced that width is less that the total window size. */ static ssize_t debug_width_fwrite(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { char buf[U64STR_SIZE]; int csize = min(cnt, sizeof(buf)); u64 val = 0; int err = 0; memset(buf, '\0', sizeof(buf)); if (copy_from_user(buf, ubuf, csize)) return -EFAULT; buf[U64STR_SIZE-1] = '\0'; /* just in case */ err = kstrtoull(buf, 10, &val); if (0 != err) return -EINVAL; mutex_lock(&data.lock); if (val < data.sample_window) data.sample_width = val; else { mutex_unlock(&data.lock); return -EINVAL; } mutex_unlock(&data.lock); if (enabled) wake_up_process(kthread); return csize; } /** * debug_window_fopen - Open function for "window" debugfs entry * @inode: The in-kernel inode representation of the debugfs "file" * @filp: The active open file structure for the debugfs "file" * * This function provides an open implementation for the "window" debugfs * interface to the hardware latency detector. The window is the total time * in us that will be considered one sample period. Conceptually, windows * occur back-to-back and contain a sample width period during which * actual sampling occurs. */ static int debug_window_fopen(struct inode *inode, struct file *filp) { return 0; } /** * debug_window_fread - Read function for "window" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The userspace provided buffer to read value into * @cnt: The maximum number of bytes to read * @ppos: The current "file" position * * This function provides a read implementation for the "window" debugfs * interface to the hardware latency detector. The window is the total time * in us that will be considered one sample period. Conceptually, windows * occur back-to-back and contain a sample width period during which * actual sampling occurs. Can be used to read the total window size. */ static ssize_t debug_window_fread(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) { return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_window); } /** * debug_window_fwrite - Write function for "window" debugfs entry * @filp: The active open file structure for the debugfs "file" * @ubuf: The user buffer that contains the value to write * @cnt: The maximum number of bytes to write to "file" * @ppos: The current position in the debugfs "file" * * This function provides a write implementation for the "window" debufds * interface to the hardware latency detetector. The window is the total time * in us that will be considered one sample period. Conceptually, windows * occur back-to-back and contain a sample width period during which * actual sampling occurs. Can be used to write a new total window size. It * is enfoced that any value written must be greater than the sample width * size, or an error results. */ static ssize_t debug_window_fwrite(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { char buf[U64STR_SIZE]; int csize = min(cnt, sizeof(buf)); u64 val = 0; int err = 0; memset(buf, '\0', sizeof(buf)); if (copy_from_user(buf, ubuf, csize)) return -EFAULT; buf[U64STR_SIZE-1] = '\0'; /* just in case */ err = kstrtoull(buf, 10, &val); if (0 != err) return -EINVAL; mutex_lock(&data.lock); if (data.sample_width < val) data.sample_window = val; else { mutex_unlock(&data.lock); return -EINVAL; } mutex_unlock(&data.lock); return csize; } /* * Function pointers for the "count" debugfs file operations */ static const struct file_operations count_fops = { .open = debug_count_fopen, .read = debug_count_fread, .write = debug_count_fwrite, .owner = THIS_MODULE, }; /* * Function pointers for the "enable" debugfs file operations */ static const struct file_operations enable_fops = { .open = debug_enable_fopen, .read = debug_enable_fread, .write = debug_enable_fwrite, .owner = THIS_MODULE, }; /* * Function pointers for the "max" debugfs file operations */ static const struct file_operations max_fops = { .open = debug_max_fopen, .read = debug_max_fread, .write = debug_max_fwrite, .owner = THIS_MODULE, }; /* * Function pointers for the "sample" debugfs file operations */ static const struct file_operations sample_fops = { .open = debug_sample_fopen, .read = debug_sample_fread, .release = debug_sample_release, .owner = THIS_MODULE, }; /* * Function pointers for the "threshold" debugfs file operations */ static const struct file_operations threshold_fops = { .open = debug_threshold_fopen, .read = debug_threshold_fread, .write = debug_threshold_fwrite, .owner = THIS_MODULE, }; /* * Function pointers for the "width" debugfs file operations */ static const struct file_operations width_fops = { .open = debug_width_fopen, .read = debug_width_fread, .write = debug_width_fwrite, .owner = THIS_MODULE, }; /* * Function pointers for the "window" debugfs file operations */ static const struct file_operations window_fops = { .open = debug_window_fopen, .read = debug_window_fread, .write = debug_window_fwrite, .owner = THIS_MODULE, }; /** * init_debugfs - A function to initialize the debugfs interface files * * This function creates entries in debugfs for "hwlat_detector", including * files to read values from the detector, current samples, and the * maximum sample that has been captured since the hardware latency * dectector was started. */ static int init_debugfs(void) { int ret = -ENOMEM; debug_dir = debugfs_create_dir(DRVNAME, NULL); if (!debug_dir) goto err_debug_dir; debug_sample = debugfs_create_file("sample", 0444, debug_dir, NULL, &sample_fops); if (!debug_sample) goto err_sample; debug_count = debugfs_create_file("count", 0444, debug_dir, NULL, &count_fops); if (!debug_count) goto err_count; debug_max = debugfs_create_file("max", 0444, debug_dir, NULL, &max_fops); if (!debug_max) goto err_max; debug_sample_window = debugfs_create_file("window", 0644, debug_dir, NULL, &window_fops); if (!debug_sample_window) goto err_window; debug_sample_width = debugfs_create_file("width", 0644, debug_dir, NULL, &width_fops); if (!debug_sample_width) goto err_width; debug_threshold = debugfs_create_file("threshold", 0644, debug_dir, NULL, &threshold_fops); if (!debug_threshold) goto err_threshold; debug_enable = debugfs_create_file("enable", 0644, debug_dir, &enabled, &enable_fops); if (!debug_enable) goto err_enable; else { ret = 0; goto out; } err_enable: debugfs_remove(debug_threshold); err_threshold: debugfs_remove(debug_sample_width); err_width: debugfs_remove(debug_sample_window); err_window: debugfs_remove(debug_max); err_max: debugfs_remove(debug_count); err_count: debugfs_remove(debug_sample); err_sample: debugfs_remove(debug_dir); err_debug_dir: out: return ret; } /** * free_debugfs - A function to cleanup the debugfs file interface */ static void free_debugfs(void) { /* could also use a debugfs_remove_recursive */ debugfs_remove(debug_enable); debugfs_remove(debug_threshold); debugfs_remove(debug_sample_width); debugfs_remove(debug_sample_window); debugfs_remove(debug_max); debugfs_remove(debug_count); debugfs_remove(debug_sample); debugfs_remove(debug_dir); } /** * detector_init - Standard module initialization code */ static int detector_init(void) { int ret = -ENOMEM; pr_info(BANNER "version %s\n", VERSION); ret = init_stats(); if (0 != ret) goto out; ret = init_debugfs(); if (0 != ret) goto err_stats; if (enabled) ret = start_kthread(); goto out; err_stats: ring_buffer_free(ring_buffer); out: return ret; } /** * detector_exit - Standard module cleanup code */ static void detector_exit(void) { int err; if (enabled) { enabled = 0; err = stop_kthread(); if (err) pr_err(BANNER "cannot stop kthread\n"); } free_debugfs(); ring_buffer_free(ring_buffer); /* free up the ring buffer */ } module_init(detector_init); module_exit(detector_exit);