1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
|
# -*- cperl -*-
# Copyright (c) 2008, 2013, Oracle and/or its affiliates. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
package My::CoreDump;
use strict;
use Carp;
use My::Platform;
use File::Temp qw/ tempfile tempdir /;
use mtr_results;
my $hint_mysqld; # Last resort guess for executable path
# If path in core file is 79 chars we assume it's been truncated
# Looks like we can still find the full path using 'strings'
# If that doesn't work, use the hint (mysqld path) as last resort.
sub _verify_binpath {
my ($binary, $core_name)= @_;
my $binpath;
if (length $binary != 79) {
$binpath= $binary;
print "Core generated by '$binpath'\n";
} else {
# Last occurrence of path ending in /mysql*, cut from first /
if (`strings '$core_name' | grep "/mysql[^/. ]*\$" | tail -1` =~ /(\/.*)/) {
$binpath= $1;
print "Guessing that core was generated by '$binpath'\n";
} else {
return unless $hint_mysqld;
$binpath= $hint_mysqld;
print "Wild guess that core was generated by '$binpath'\n";
}
}
return $binpath;
}
sub _gdb {
my ($core_name)= @_;
# Check that gdb exists
`gdb --version`;
if ($?) {
print "gdb not found, cannot get the stack trace\n";
return;
}
if (-f $core_name) {
print "\nTrying 'gdb' to get a backtrace from coredump $core_name\n";
} else {
print "\nCoredump $core_name does not exist, cannot run 'gdb'\n";
return;
}
# Find out name of binary that generated core
`gdb -c '$core_name' --batch 2>&1` =~
/Core was generated by `([^\s\'\`]+)/;
my $binary= $1 or return;
$binary= _verify_binpath ($binary, $core_name) or return;
# Create tempfile containing gdb commands
my ($tmp, $tmp_name) = tempfile();
print $tmp
"bt\n",
"thread apply all bt\n",
"quit\n";
close $tmp or die "Error closing $tmp_name: $!";
# Run gdb
my $gdb_output=
`gdb '$binary' -c '$core_name' -x '$tmp_name' --batch 2>&1`;
unlink $tmp_name or die "Error removing $tmp_name: $!";
return if $? >> 8;
return unless $gdb_output;
resfile_print <<EOF . $gdb_output . "\n";
Output from gdb follows. The first stack trace is from the failing thread.
The following stack traces are from all threads (so the failing one is
duplicated).
--------------------------
EOF
return 1;
}
sub _dbx {
my ($core_name)= @_;
print "\nTrying 'dbx' to get a backtrace\n";
return unless -f $core_name;
# Find out name of binary that generated core
`echo | dbx - '$core_name' 2>&1` =~
/Corefile specified executable: "([^"]+)"/;
my $binary= $1 or return;
$binary= _verify_binpath ($binary, $core_name) or return;
# Find all threads
my @thr_ids = `echo threads | dbx '$binary' '$core_name' 2>&1` =~ /t@\d+/g;
# Create tempfile containing dbx commands
my ($tmp, $tmp_name) = tempfile();
foreach my $thread (@thr_ids) {
print $tmp "where $thread\n";
}
print $tmp "exit\n";
close $tmp or die "Error closing $tmp_name: $!";
# Run dbx
my $dbx_output=
`cat '$tmp_name' | dbx '$binary' '$core_name' 2>&1`;
unlink $tmp_name or die "Error removing $tmp_name: $!";
return if $? >> 8;
return unless $dbx_output;
resfile_print <<EOF . $dbx_output . "\n";
Output from dbx follows. Stack trace is printed for all threads in order,
above this you should see info about which thread was the failing one.
----------------------------
EOF
return 1;
}
# Check that Debugging tools for Windows are installed
sub cdb_check {
`cdb -? 2>&1`;
if ($? >> 8)
{
print "Cannot find cdb. Please Install Debugging tools for Windows\n";
print "from http://www.microsoft.com/whdc/devtools/debugging/";
if($ENV{'ProgramW6432'})
{
print "install64bit.mspx (native x64 version)\n";
}
else
{
print "installx86.mspx\n";
}
}
}
sub _cdb {
my ($core_name)= @_;
print "\nTrying 'cdb' to get a backtrace\n";
return unless -f $core_name;
# Try to set environment for debugging tools for Windows
if ($ENV{'PATH'} !~ /Debugging Tools/)
{
if ($ENV{'ProgramW6432'})
{
# On x64 computer
$ENV{'PATH'}.= ";".$ENV{'ProgramW6432'}."\\Debugging Tools For Windows (x64)";
}
else
{
# On x86 computer. Newest versions of Debugging tools are installed in the
# directory with (x86) suffix, older versions did not have this suffix.
$ENV{'PATH'}.= ";".$ENV{'ProgramFiles'}."\\Debugging Tools For Windows (x86)";
$ENV{'PATH'}.= ";".$ENV{'ProgramFiles'}."\\Debugging Tools For Windows";
}
}
# Read module list, find out the name of executable and
# build symbol path (required by cdb if executable was built on
# different machine)
my $tmp_name= $core_name.".cdb_lmv";
`cdb -z $core_name -c \"lmv;q\" > $tmp_name 2>&1`;
if ($? >> 8)
{
unlink($tmp_name);
# check if cdb is installed and complain if not
cdb_check();
return;
}
open(temp,"< $tmp_name");
my %dirhash=();
while(<temp>)
{
if($_ =~ /Image path\: (.*)/)
{
if (rindex($1,'\\') != -1)
{
my $dir= substr($1, 0, rindex($1,'\\'));
$dirhash{$dir}++;
}
}
}
close(temp);
unlink($tmp_name);
my $image_path= join(";", (keys %dirhash),".");
# For better callstacks, setup _NT_SYMBOL_PATH to include
# OS symbols. Note : Dowloading symbols for the first time
# can take some minutes
if (!$ENV{'_NT_SYMBOL_PATH'})
{
my $windir= $ENV{'windir'};
my $symbol_cache= substr($windir ,0, index($windir,'\\'))."\\cdb_symbols";
print "OS debug symbols will be downloaded and stored in $symbol_cache.\n";
print "You can control the location of symbol cache with _NT_SYMBOL_PATH\n";
print "environment variable. Please refer to Microsoft KB article\n";
print "http://support.microsoft.com/kb/311503 for details about _NT_SYMBOL_PATH\n";
print "-------------------------------------------------------------------------\n";
$ENV{'_NT_SYMBOL_PATH'}.=
"srv*".$symbol_cache."*http://msdl.microsoft.com/download/symbols";
}
my $symbol_path= $image_path.";".$ENV{'_NT_SYMBOL_PATH'};
# Run cdb. Use "analyze" extension to print crashing thread stacktrace
# and "uniqstack" to print other threads
my $cdb_cmd = "!sym prompts off; !analyze -v; .ecxr; !for_each_frame dv /t;!uniqstack -p;q";
my $cdb_output=
`cdb -c "$cdb_cmd" -z $core_name -i "$image_path" -y "$symbol_path" -t 0 -lines 2>&1`;
return if $? >> 8;
return unless $cdb_output;
# Remove comments (lines starting with *), stack pointer and frame
# pointer adresses and offsets to function to make output better readable
$cdb_output=~ s/^\*.*\n//gm;
$cdb_output=~ s/^([\:0-9a-fA-F\`]+ )+//gm;
$cdb_output=~ s/^ChildEBP RetAddr//gm;
$cdb_output=~ s/^Child\-SP RetAddr Call Site//gm;
$cdb_output=~ s/\+0x([0-9a-fA-F]+)//gm;
resfile_print <<EOF . $cdb_output . "\n";
Output from cdb follows. Faulting thread is printed twice,with and without function parameters
Search for STACK_TEXT to see the stack trace of
the faulting thread. Callstacks of other threads are printed after it.
EOF
return 1;
}
sub _lldb
{
my ($core_name)= @_;
print "\nTrying 'lldb' to get a backtrace from coredump $core_name\n";
# Create tempfile containing lldb commands
my ($tmp, $tmp_name)= tempfile();
print $tmp
"bt\n",
"thread backtrace all\n",
"quit\n";
close $tmp or die "Error closing $tmp_name: $!";
my $lldb_output= `lldb -c '$core_name' -s '$tmp_name' 2>&1`;
unlink $tmp_name or die "Error removing $tmp_name: $!";
if ($? == 127)
{
print "lldb not found, cannot get the stack trace\n";
return;
}
return if $?;
return unless $lldb_output;
resfile_print <<EOF . $lldb_output . "\n";
Output from lldb follows. The first stack trace is from the failing thread.
The following stack traces are from all threads (so the failing one is
duplicated).
--------------------------
EOF
return 1;
}
sub show {
my ($class, $core_name, $exe_mysqld, $parallel)= @_;
$hint_mysqld= $exe_mysqld;
# On Windows, rely on cdb to be there...
if (IS_WINDOWS)
{
_cdb($core_name);
return;
}
# We try dbx first; gdb itself may coredump if run on a Sun Studio
# compiled binary on Solaris.
my @debuggers =
(
\&_dbx,
\&_gdb,
\&_lldb,
# TODO...
);
# Try debuggers until one succeeds
foreach my $debugger (@debuggers){
if ($debugger->($core_name)){
return;
}
}
return;
}
1;
|