summaryrefslogtreecommitdiff
path: root/scripts/dtc/of_unittest_expect
blob: 7b25bb6931623f0b04aaf836d21c2fa9d428a81f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
#!/usr/bin/perl
# SPDX-License-Identifier: GPL-2.0
#
# Copyright 2020, 2022 Sony Corporation
#
# Author: Frank Rowand

# This program is meant to be an aid to reading the verbose output of
# on the console log that results from executing the Linux kernel
# devicetree unittest (drivers/of/unitest.c).

$VUFX = "230121a";

use strict 'refs';
use strict subs;

use Getopt::Long;
use Text::Wrap;

# strip off everything before final "/"
(undef, $script_name) = split(/^.*\//, $0);

# following /usr/include/sysexits.h
$EX_OK=0;
$EX_USAGE=64;


#______________________________________________________________________________
sub compare {
	my ($expect, $got) = @_;
	my $expect_next;
	my $expect_next_lit;
	my $got_next;
	my $type;

	while ($expect) {

		($expect_next, $type) = split(/<</, $expect);
		($type) = split(/>>/, $type);
		$expect =~ s/^.*?>>//;	# '?' is non-greedy, minimal match

		# literal, ignore all metacharacters when used in a regex
		$expect_next_lit = quotemeta($expect_next);

		$got_next = $got;
		$got_next =~ s/^($expect_next_lit).*/\1/;
		$got       =~ s/^$expect_next_lit//;

		if ($expect_next ne $got_next) {
			return 0;
		}

		if ($type eq "int") {
			if ($got =~ /^[+-]*[0-9]+/) {
				$got =~ s/^[+-]*[0-9]+//;
			} else {
				return 0;
			}
		} elsif ($type eq "hex") {
			if ($got =~ /^(0x)*[0-9a-f]+/) {
				$got =~ s/^(0x)*[0-9a-f]+//;
			} else {
				return 0;
			}
		} elsif ($type eq "") {
			if ($expect_next ne $got_next) {
				return 0;
			} else {
				return 1;
			}
		} else {
			$internal_err++;
			print "** ERROR: special pattern not recognized: <<$type>>, CONSOLE_LOG line: $.\n";
			return 0;
		}

	}

	# should not get here
	$internal_err++;
	print "** ERROR: $script_name internal error, at end of compare(), CONSOLE_LOG line: $.\n";

	return 0;
}


#______________________________________________________________________________
sub usage {

# ***** when editing, be careful to not put tabs in the string printed:

	print STDERR
"
usage:

  $script_name CONSOLE_LOG

     -h                print program usage
    --help             print program usage
    --hide-expect      suppress output of EXPECTed lines
    --line-num         report line number of CONSOLE_LOG
    --no-expect-stats  do not report EXPECT statistics
    --no-strip-ts      do not strip leading console timestamps
    --verbose          do not suppress EXPECT begin and end lines
    --version          print program version and exit


  Process a console log for EXPECTed test related messages to either
  highlight expected devicetree unittest related messages or suppress
  the messages.  Leading console timestamps will be stripped.

  Various unittests may trigger kernel messages from outside the
  unittest code.  The unittest annotates that it expects the message
  to occur with an 'EXPECT \\ : text' (begin) before triggering the
  message, and an 'EXPECT / : text' (end) after triggering the message.

  If an expected message does not occur, that will be reported.

  For each expected message, the 'EXPECT \\ : text' (begin) and
  'EXPECT / : text' (end), 'text' will contain the message text.

  If 'EXPECT \\' (begin) and 'EXPECT /' (end) lines do not contain
  matching 'text', that will be reported.

  If EXPECT lines are nested, 'EXPECT /' (end) lines must be in the
  reverse order of the corresponding 'EXPECT \\' (begin) lines.

  'EXPECT \\ : text' (begin) and 'EXPECT / : text' (end) lines can
  contain special patterns in 'text':

     <<int>> matches: [+-]*[0-9]+
     <<hex>> matches: (0x)*[0-9a-f]+

  'EXPECT \\' (begin) and 'EXPECT /' (end) lines are suppressed.

  A prefix is added to every line of output:

    'ok ' Line matches an enclosing EXPECT begin/end pair

    '** ' Line reports $script_name warning or error

    '-> ' Line reports start or end of the unittests

    '>> ' Line reports a unittest test FAIL

    '   ' Lines that are not otherwise prefixed

  Issues detected in CONSOLE_LOG are reported to STDOUT, not to STDERR.

  Known Issues:

    --line-num causes the CONSOLE_LOG line number to be printed in 4 columns.
       If CONSOLE_LOG contains more than 9999 lines then more columns will be
       used to report the line number for lines greater than 9999 (eg for
       lines 10000 - 99999, 5 columns will be used).
";

	return {};
}

#______________________________________________________________________________
#______________________________________________________________________________

if (!GetOptions(
	"h"               => \$help,
	"help"            => \$help,
	"hide-expect"     => \$hide_expect,
	"line-num"        => \$print_line_num,
	"no-expect-stats" => \$no_expect_stats,
	"no-strip-ts"     => \$no_strip_ts,
	"verbose"         => \$verbose,
	"version"         => \$version,
	)) {
	print STDERR "\n";
	print STDERR "ERROR processing command line options\n";
	print STDERR "\n";
	print STDERR "For help, type '$script_name --help'\n";
	print STDERR "\n";

	exit $EX_OK;
}


if ($no_strip_ts) {
	$strip_ts = 1;
	$no_strip_ts = 0;
} else {
	$strip_ts = 0;
	$no_strip_ts = 1;
}


# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if ($help){

	&usage;

	exit $EX_OK;
}


# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

if ($version) {
	print STDERR "\n$script_name  $VUFX\n\n";
	print STDERR "\n";

	exit $EX_OK;
}


# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if ($#ARGV != 0) {

	# Limit input files to exactly one.
	#
	# 'while ($line = <ARGV>) {' in the code below supports multiple file
	# names on the command line, but the EXPECT statistics are reported
	# once for all input - it is not an expected use case to generate one
	# set of statistics for multiple input files.

	print STDERR "\n";
	print STDERR "Required arguments: CONSOLE_LOG\n";
	print STDERR "\n";

	exit $EX_USAGE;
}


#______________________________________________________________________________

# Patterns to match 'EXPECT \ : ' (begin) and 'EXPECT / : ' (end)
#
# $exp_* are used as regex match patterns,
# so '\\\\' in $exp_begin matches a single '\'
# quotemeta() does not do the right thing in this case
#
# $pr_fmt is the prefix that unittest prints for every message

$pr_fmt = "### dt-test ### ";
$exp_begin = "${pr_fmt}EXPECT \\\\ : ";
$exp_end   = "${pr_fmt}EXPECT / : ";


$line_num = "";
$timestamp = "";

LINE:
while ($line = <ARGV>) {

	chomp $line;

	$prefix = "  ";  ## 2 characters


	if ($strip_ts) {

		$timestamp = $line;

		if ($timestamp =~ /^\[\s*[0-9]+\.[0-9]*\] /) {
			($timestamp, $null) = split(/]/, $line);
			$timestamp = $timestamp . "] ";

		} else {
			$timestamp = "";
		}
	}

	$line =~ s/^\[\s*[0-9]+\.[0-9]*\] //;


	# -----  find EXPECT begin

	if ($line =~ /^\s*$exp_begin/) {
		$data = $line;
		$data =~ s/^\s*$exp_begin//;
		push @exp_begin_stack, $data;

		if ($verbose) {
			if ($print_line_num) {
				$line_num = sprintf("%4s ", $.);
			}
			printf "%s %s%s%s\n", $prefix, $line_num,  $timestamp, $line;
		}

		next LINE;
	}


	# -----  find EXPECT end

	if ($line =~ /^\s*$exp_end/) {
		$data = $line;
		$data =~ s/^\s*$exp_end//;

		if ($verbose) {
			if ($print_line_num) {
				$line_num = sprintf("%4s ", $.);
			}
			printf "%s %s%s%s\n", $prefix, $line_num,  $timestamp, $line;
		}

		$found = 0;
		$no_begin = 0;
		if (@exp_found_or_begin > 0) {
			$begin = pop @exp_found_or_begin;
			if (compare($data, $begin)) {
				$found = 1;
			}
		} elsif (@begin > 0) {
			$begin = pop @exp_begin_stack;
		} else {
			$no_begin = 1;
		}

		if ($no_begin) {

			$exp_missing_begin++;
			print "** ERROR: EXPECT end without any EXPECT begin:\n";
			print "       end ---> $line\n";

		} elsif (! $found) {

			if ($print_line_num) {
				$line_num = sprintf("%4s ", $.);
			}

			$exp_missing++;
			printf "** %s%s$script_name WARNING - not found ---> %s\n",
					$line_num,  $timestamp, $data;

		} elsif (! compare($data, $begin)) {

			$exp_missing_end++;
			print "** ERROR: EXPECT end does not match EXPECT begin:\n";
			print "       begin -> $begin\n";
			print "       end ---> $line\n";

		} else {

			$exp_found++;

		}

		next LINE;
	}


	# -----  not an EXPECT line

	if (($line =~ /^${pr_fmt}start of unittest - you will see error messages$/) ||
	    ($line =~ /^${pr_fmt}end of unittest - [0-9]+ passed, [0-9]+ failed$/ )   ) {
		$prefix = "->"; # 2 characters
	} elsif ($line =~ /^${pr_fmt}FAIL /) {
		$unittest_fail++;
		$prefix = ">>"; # 2 characters
	}

	$found = 0;
	foreach $begin (@exp_begin_stack) {
		if (compare($begin, $line)) {
			$found = 1;
			last;
		}
	}

	if ($found) {
		$begin = shift @exp_begin_stack;
		while (! compare($begin, $line)) {
			push @exp_found_or_begin, $begin;
			$begin = shift @exp_begin_stack;
		}
		push @exp_found_or_begin, $line;

		if ($hide_expect) {
			$suppress_line = 1;
			next LINE;
		}
		$prefix = "ok"; # 2 characters
	}


	if ($print_line_num) {
		$line_num = sprintf("%4s ", $.);
	}

	printf "%s %s%s%s\n", $prefix, $line_num,  $timestamp, $line;
}

if (! $no_expect_stats) {
	print  "\n";
	print  "** EXPECT statistics:\n";
	print  "**\n";
	printf "**   EXPECT found          : %4i\n", $exp_found;
	printf "**   EXPECT not found      : %4i\n", $exp_missing;
	printf "**   missing EXPECT begin  : %4i\n", $exp_missing_begin;
	printf "**   missing EXPECT end    : %4i\n", $exp_missing_end;
	printf "**   unittest FAIL         : %4i\n", $unittest_fail;
	printf "**   internal error        : %4i\n", $internal_err;
}

if (@exp_begin_stack) {
	print "** ERROR: EXPECT begin without any EXPECT end:\n";
	print "          This list may be misleading.\n";
	foreach $begin (@exp_begin_stack) {
		print "       begin ---> $begin\n";
	}
}