libbpf-tools/biotop: Add --output for Line Protocol metrics

Add --output option to biotop to print system call metrics in Line Protocol
format for time-series analysis.
Include biotop_example.txt with usage, examples, and visualization guide for
tools like InfluxDB and Grafana. Add images for I/O and Latency modes.

Author: ekyooo

libbpf-tools/biotop.c

@@ -31,6 +31,8 @@
 #define warn(...) fprintf(stderr, __VA_ARGS__)
 #define OUTPUT_ROWS_LIMIT 10240
 
+#define OPT_OUTPUT			1 /* --output */
+
 enum SORT {
 	ALL,
 	IO,
@@ -50,6 +52,11 @@ struct vector {
 	void **elems;
 };
 
+struct data_t {
+	struct info_t key;
+	struct val_t value;
+};
+
 int grow_vector(struct vector *vector) {
 	if (vector->nr >= vector->capacity) {
 		void **reallocated;
@@ -87,6 +94,8 @@ static int interval = 1;
 static int count = 99999999;
 static pid_t target_pid = 0;
 static bool verbose = false;
+enum output_format output = 0;
+static struct data_t datas[OUTPUT_ROWS_LIMIT];
 
 const char *argp_program_version = "biotop 0.1";
 const char *argp_program_bug_address =
@@ -107,6 +116,7 @@ static const struct argp_option opts[] = {
 	{ "rows", 'r', "ROWS", 0, "Maximum rows to print, default 20", 0 },
 	{ "pid", 'p', "PID", 0, "Process ID to trace", 0 },
 	{ "verbose", 'v', NULL, 0, "Verbose debug output", 0 },
+	{ "output", OPT_OUTPUT, "FORMAT", OPTION_ARG_OPTIONAL, "Output metrics in specified format (currently only 'line' supported)", 0 },
 	{ NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help", 0 },
 	{},
 };
@@ -160,6 +170,16 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state)
 	case 'h':
 		argp_state_help(state, stderr, ARGP_HELP_STD_HELP);
 		break;
+	case OPT_OUTPUT:
+		output = FORMAT_LINE_PROTOCOL;
+		if (arg) {
+			if (strcmp(arg, "line") == 0)
+				output = FORMAT_LINE_PROTOCOL;
+			else
+				argp_error(state, "Invalid output format: %s. "
+					   "Only 'line' is supported.", arg);
+		}
+		break;
 	case ARGP_KEY_ARG:
 		errno = 0;
 		if (pos_args == 0) {
@@ -198,11 +218,6 @@ static void sig_int(int signo)
 	exiting = 1;
 }
 
-struct data_t {
-	struct info_t key;
-	struct val_t value;
-};
-
 static int sort_column(const void *obj1, const void *obj2)
 {
 	struct data_t *d1 = (struct data_t *) obj1;
@@ -299,13 +314,58 @@ static int read_stat(struct biotop_bpf *obj, struct data_t *datas, __u32 *count)
 	return 0;
 }
 
+static int print_metrics(struct biotop_bpf *obj)
+{
+	int i, err = 0, rows = OUTPUT_ROWS_LIMIT;
+	time_t ts = time(NULL);
+	struct metric m = {
+		.name = "biotop",
+		.tags = {{ "pid", "" }},
+		.nr_tags = 1,
+		.fields = {
+			{ "I/O", 0 },
+			{ "Kbytes", 0},
+			{ "AVGms", 0}
+		},
+		.nr_fields = 3,
+		.ts = ts
+	};
+
+	err = read_stat(obj, datas, (__u32*) &rows);
+	if (err) {
+		fprintf(stderr, "read stat failed: %s\n", strerror(errno));
+		return err;
+	}
+
+	for (i = 0; i < rows; i++) {
+		struct info_t *key = &datas[i].key;
+		struct val_t *value = &datas[i].value;
+		float avg_ms = 0;
+
+		/* Tag */
+		snprintf(m.tags[0].value, sizeof(m.tags[0].value), "%u", key->pid);
+
+		/* To avoid floating point exception. */
+		if (value->io)
+			avg_ms = ((float) value->us) / 1000 / value->io;
+
+		/* Fields */
+		m.fields[0].value = value->io;
+		m.fields[1].value = value->bytes;
+		m.fields[2].value = avg_ms;
+
+		print_metric(&m, output);
+	}
+
+	return 0;
+}
+
 static int print_stat(struct biotop_bpf *obj)
 {
 	FILE *f;
 	time_t t;
 	struct tm *tm;
 	char ts[16], buf[256];
-	static struct data_t datas[OUTPUT_ROWS_LIMIT];
 	int n, i, err = 0, rows = OUTPUT_ROWS_LIMIT;
 
 	f = fopen("/proc/loadavg", "r");
@@ -319,6 +379,7 @@ static int print_stat(struct biotop_bpf *obj)
 			printf("%8s loadavg: %s\n", ts, buf);
 		fclose(f);
 	}
+
 	printf("%-7s %-16s %1s %-3s %-3s %-8s %5s %7s %6s\n",
 	       "PID", "COMM", "D", "MAJ", "MIN", "DISK", "I/O", "Kbytes", "AVGms");
 
@@ -448,15 +509,21 @@ int main(int argc, char **argv)
 	while (1) {
 		sleep(interval);
 
-		if (clear_screen) {
-			err = system("clear");
+		if (output) {
+			err = print_metrics(obj);
 			if (err)
 				goto cleanup;
-		}
+		} else {
+			if (clear_screen) {
+				err = system("clear");
+				if (err)
+					goto cleanup;
+			}
 
-		err = print_stat(obj);
-		if (err)
-			goto cleanup;
+			err = print_stat(obj);
+			if (err)
+				goto cleanup;
+		}
 
 		count--;
 		if (exiting || !count)

tools/biotop_example.txt

@@ -163,6 +163,81 @@ This shows another "dd" command reading from xvda1. On this system, various
 creating and updating "status" files).
 
 
+Output Metrics in Line Protocol
+-------------------------------
+
+The `--output` option enables `biotop` to output metrics in InfluxDB Line
+Protocol format, a text-based format for writing data points to time-series
+databases like InfluxDB. This is useful for tracking I/O operations, kilobytes
+transferred, and average latency per process ID (PID) for monitoring systems.
+
+Usage
+-----
+Use `--output` to switch to Line Protocol output.
+
+Example Output
+--------------
+Running `biotop --output` might produce:
+biotop,pid=279 I/O=6,Kbytes=69632,AVGms=0 1745473912
+biotop,pid=0 I/O=9,Kbytes=0,AVGms=94 1745473912
+biotop,pid=1585 I/O=14,Kbytes=94208,AVGms=1 1745473912
+
+This format is easy to save to files or process with scripts and databases to
+monitor I/O activity over time.
+
+Using the Output with InfluxDB and Grafana
+------------------------------------------
+
+The Line Protocol output can be stored in InfluxDB and visualized in Grafana,
+similar to how folded stacks from `profile` are turned into flame graphs.
+Here's how:
+
+1. Storing in InfluxDB
+Pipe the output into InfluxDB using the `influx` CLI. For example:
+
+# biotop --output > biotop_metrics.txt
+influx write -b mybucket -f biotop_metrics.txt -p s
+
+For real-time ingestion:
+# unbuffer biotop --output | influx write -b mybucket -p s
+
+Note: The -p s option specifies second-precision timestamps.
+
+2. Visualizing Metrics
+Once stored in InfluxDB, query the `biotop` measurement in Grafana to create
+time-series graphs. Here are some examples:
+
+2.1) I/O Operations per PID
+Command: `biotop --output`
+Shows I/O operations per process ID as a time-series graph, useful for finding
+processes with high I/O activity.
+See: bcc/tools/images/biotop_io.png
+Query:
+from(bucket: "mybucket")
+  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
+  |> filter(fn: (r) => r._measurement == "biotop" and r._field == "I/O")
+
+2.2) Latency per PID
+Command: `biotop --output`
+Shows average I/O latency per process ID in milliseconds, helping identify
+processes with high I/O delays.
+See: bcc/tools/images/biotop_latency.png
+Query:
+from(bucket: "mybucket")
+  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
+  |> filter(fn: (r) => r._measurement == "biotop" and r._field == "AVGms")
+
+2.3) Combined I/O Metrics Dashboard
+Command: `biotop --output`
+Displays I/O operations, kilobytes transferred, and average latency per process
+ID in a single Grafana dashboard, providing a comprehensive view of I/O activity
+for real-time monitoring and trend analysis.
+See: bcc/tools/images/biotops.png
+
+These graphs can be added to a Grafana dashboard to monitor I/O activity in
+real-time or analyze trends.
+
+
 USAGE message:
 
 # ./biotop.py -h