#!/usr/bin/env bash # This program is part of Percona Toolkit: http://www.percona.com/software/ # See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal # notices and disclaimers. # ########################################################################### # log_warn_die package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/bash/log_warn_die.sh # t/lib/bash/log_warn_die.sh # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### set -u PTFUNCNAME="" PTDEBUG="${PTDEBUG:-""}" EXIT_STATUS=0 ts() { TS=$(date +%F-%T | tr ':-' '_') echo "$TS $*" } info() { [ ${OPT_VERBOSE:-3} -ge 3 ] && ts "$*" } log() { [ ${OPT_VERBOSE:-3} -ge 2 ] && ts "$*" } warn() { [ ${OPT_VERBOSE:-3} -ge 1 ] && ts "$*" >&2 EXIT_STATUS=1 } die() { ts "$*" >&2 EXIT_STATUS=1 exit 1 } _d () { [ "$PTDEBUG" ] && echo "# $PTFUNCNAME: $(ts "$*")" >&2 } # ########################################################################### # End log_warn_die package # ########################################################################### # ########################################################################### # parse_options package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/bash/parse_options.sh # t/lib/bash/parse_options.sh # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### set -u ARGV="" # Non-option args (probably input files) EXT_ARGV="" # Everything after -- (args for an external command) HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV OPT_ERRS=0 # How many command line option errors OPT_VERSION="" # If --version was specified OPT_HELP="" # If --help was specified PO_DIR="" # Directory with program option spec files usage() { local file="$1" local usage="$(grep '^Usage: ' "$file")" echo $usage echo echo "For more information, 'man $TOOL' or 'perldoc $file'." } usage_or_errors() { local file="$1" local version="" if [ "$OPT_VERSION" ]; then version=$(grep '^pt-[^ ]\+ [0-9]' "$file") echo "$version" return 1 fi if [ "$OPT_HELP" ]; then usage "$file" echo echo "Command line options:" echo perl -e ' use strict; use warnings FATAL => qw(all); my $lcol = 20; # Allow this much space for option names. my $rcol = 80 - $lcol; # The terminal is assumed to be 80 chars wide. my $name; while ( <> ) { my $line = $_; chomp $line; if ( $line =~ s/^long:/ --/ ) { $name = $line; } elsif ( $line =~ s/^desc:// ) { $line =~ s/ +$//mg; my @lines = grep { $_ } $line =~ m/(.{0,$rcol})(?:\s+|\Z)/g; if ( length($name) >= $lcol ) { print $name, "\n", (q{ } x $lcol); } else { printf "%-${lcol}s", $name; } print join("\n" . (q{ } x $lcol), @lines); print "\n"; } } ' "$PO_DIR"/* echo echo "Options and values after processing arguments:" echo ( cd "$PO_DIR" for opt in *; do local varname="OPT_$(echo "$opt" | tr a-z- A-Z_)" eval local varvalue=\$$varname if ! grep -q "type:" "$PO_DIR/$opt" >/dev/null; then if [ "$varvalue" -a "$varvalue" = "yes" ]; then varvalue="TRUE" else varvalue="FALSE" fi fi printf -- " --%-30s %s" "$opt" "${varvalue:-(No value)}" echo done ) return 1 fi if [ $OPT_ERRS -gt 0 ]; then echo usage "$file" return 1 fi return 0 } option_error() { local err="$1" OPT_ERRS=$(($OPT_ERRS + 1)) echo "$err" >&2 } parse_options() { local file="$1" shift ARGV="" EXT_ARGV="" HAVE_EXT_ARGV="" OPT_ERRS=0 OPT_VERSION="" OPT_HELP="" PO_DIR="$PT_TMPDIR/po" if [ ! -d "$PO_DIR" ]; then mkdir "$PO_DIR" if [ $? -ne 0 ]; then echo "Cannot mkdir $PO_DIR" >&2 exit 1 fi fi rm -rf "$PO_DIR"/* if [ $? -ne 0 ]; then echo "Cannot rm -rf $PO_DIR/*" >&2 exit 1 fi _parse_pod "$file" # Parse POD into program option (po) spec files _eval_po # Eval po into existence with default values if [ $# -ge 2 ] && [ "$1" = "--config" ]; then shift # --config local user_config_files="$1" shift # that ^ local IFS="," for user_config_file in $user_config_files; do _parse_config_files "$user_config_file" done else _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" if [ "${HOME:-}" ]; then _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" fi fi _parse_command_line "${@:-""}" } _parse_pod() { local file="$1" PO_FILE="$file" PO_DIR="$PO_DIR" perl -e ' $/ = ""; my $file = $ENV{PO_FILE}; open my $fh, "<", $file or die "Cannot open $file: $!"; while ( defined(my $para = <$fh>) ) { next unless $para =~ m/^=head1 OPTIONS/; while ( defined(my $para = <$fh>) ) { last if $para =~ m/^=head1/; chomp; if ( $para =~ m/^=item --(\S+)/ ) { my $opt = $1; my $file = "$ENV{PO_DIR}/$opt"; open my $opt_fh, ">", $file or die "Cannot open $file: $!"; print $opt_fh "long:$opt\n"; $para = <$fh>; chomp; if ( $para =~ m/^[a-z ]+:/ ) { map { chomp; my ($attrib, $val) = split(/: /, $_); print $opt_fh "$attrib:$val\n"; } split(/; /, $para); $para = <$fh>; chomp; } my ($desc) = $para =~ m/^([^?.]+)/; print $opt_fh "desc:$desc.\n"; close $opt_fh; } } last; } ' } _eval_po() { local IFS=":" for opt_spec in "$PO_DIR"/*; do local opt="" local default_val="" local neg=0 local size=0 while read key val; do case "$key" in long) opt=$(echo $val | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]') ;; default) default_val="$val" ;; "short form") ;; type) [ "$val" = "size" ] && size=1 ;; desc) ;; negatable) if [ "$val" = "yes" ]; then neg=1 fi ;; *) echo "Invalid attribute in $opt_spec: $line" >&2 exit 1 esac done < "$opt_spec" if [ -z "$opt" ]; then echo "No long attribute in option spec $opt_spec" >&2 exit 1 fi if [ $neg -eq 1 ]; then if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then echo "Option $opt_spec is negatable but not default: yes" >&2 exit 1 fi fi if [ $size -eq 1 -a -n "$default_val" ]; then default_val=$(size_to_bytes $default_val) fi eval "OPT_${opt}"="$default_val" done } _parse_config_files() { for config_file in "${@:-""}"; do test -f "$config_file" || continue while read config_opt; do echo "$config_opt" | grep '^[ ]*[^#]' >/dev/null 2>&1 || continue config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')" [ "$config_opt" = "" ] && continue echo "$config_opt" | grep -v 'version-check' >/dev/null 2>&1 || continue if ! [ "$HAVE_EXT_ARGV" ]; then config_opt="--$config_opt" fi _parse_command_line "$config_opt" done < "$config_file" HAVE_EXT_ARGV="" # reset for each file done } _parse_command_line() { local opt="" local val="" local next_opt_is_val="" local opt_is_ok="" local opt_is_negated="" local real_opt="" local required_arg="" local spec="" for opt in "${@:-""}"; do if [ "$opt" = "--" -o "$opt" = "----" ]; then HAVE_EXT_ARGV=1 continue fi if [ "$HAVE_EXT_ARGV" ]; then if [ "$EXT_ARGV" ]; then EXT_ARGV="$EXT_ARGV $opt" else EXT_ARGV="$opt" fi continue fi if [ "$next_opt_is_val" ]; then next_opt_is_val="" if [ $# -eq 0 ] || [ $(expr "$opt" : "\-") -eq 1 ]; then option_error "$real_opt requires a $required_arg argument" continue fi val="$opt" opt_is_ok=1 else if [ $(expr "$opt" : "\-") -eq 0 ]; then if [ -z "$ARGV" ]; then ARGV="$opt" else ARGV="$ARGV $opt" fi continue fi real_opt="$opt" if $(echo $opt | grep '^--no[^-]' >/dev/null); then local base_opt=$(echo $opt | sed 's/^--no//') if [ -f "$PT_TMPDIR/po/$base_opt" ]; then opt_is_negated=1 opt="$base_opt" else opt_is_negated="" opt=$(echo $opt | sed 's/^-*//') fi else if $(echo $opt | grep '^--no-' >/dev/null); then opt_is_negated=1 opt=$(echo $opt | sed 's/^--no-//') else opt_is_negated="" opt=$(echo $opt | sed 's/^-*//') fi fi if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then val="$(echo $opt | awk -F= '{print $2}')" opt="$(echo $opt | awk -F= '{print $1}')" fi if [ -f "$PT_TMPDIR/po/$opt" ]; then spec="$PT_TMPDIR/po/$opt" else spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1) if [ -z "$spec" ]; then option_error "Unknown option: $real_opt" continue fi fi required_arg=$(cat "$spec" | awk -F: '/^type:/{print $2}') if [ "$required_arg" ]; then if [ "$val" ]; then opt_is_ok=1 else next_opt_is_val=1 fi else if [ "$val" ]; then option_error "Option $real_opt does not take a value" continue fi if [ "$opt_is_negated" ]; then val="" else val="yes" fi opt_is_ok=1 fi fi if [ "$opt_is_ok" ]; then opt=$(cat "$spec" | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]') if grep "^type:size" "$spec" >/dev/null; then val=$(size_to_bytes $val) fi eval "OPT_$opt"="'$val'" opt="" val="" next_opt_is_val="" opt_is_ok="" opt_is_negated="" real_opt="" required_arg="" spec="" fi done } size_to_bytes() { local size="$1" echo $size | perl -ne '%f=(B=>1, K=>1_024, M=>1_048_576, G=>1_073_741_824, T=>1_099_511_627_776); m/^(\d+)([kMGT])?/i; print $1 * $f{uc($2 || "B")};' } # ########################################################################### # End parse_options package # ########################################################################### # ########################################################################### # tmpdir package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/bash/tmpdir.sh # t/lib/bash/tmpdir.sh # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### set -u PT_TMPDIR="" mk_tmpdir() { local dir="${1:-""}" if [ -n "$dir" ]; then if [ ! -d "$dir" ]; then mkdir "$dir" || die "Cannot make tmpdir $dir" fi PT_TMPDIR="$dir" else local tool="${0##*/}" local pid="$$" PT_TMPDIR=`mktemp -d -t "${tool}.${pid}.XXXXXX"` \ || die "Cannot make secure tmpdir" fi } rm_tmpdir() { if [ -n "$PT_TMPDIR" ] && [ -d "$PT_TMPDIR" ]; then rm -rf "$PT_TMPDIR" fi PT_TMPDIR="" } # ########################################################################### # End tmpdir package # ########################################################################### # ########################################################################### # alt_cmds package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/bash/alt_cmds.sh # t/lib/bash/alt_cmds.sh # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### set -u _seq() { local i="$1" awk "BEGIN { for(i=1; i<=$i; i++) print i; }" } _pidof() { local cmd="$1" if ! pidof "$cmd" 2>/dev/null; then ps -eo pid,ucomm | awk -v comm="$cmd" '$2 == comm { print $1 }' fi } _lsof() { local pid="$1" if ! lsof -p $pid 2>/dev/null; then /bin/ls -l /proc/$pid/fd 2>/dev/null fi } _which() { if [ -x /usr/bin/which ]; then /usr/bin/which "$1" 2>/dev/null | awk '{print $1}' elif which which 1>/dev/null 2>&1; then which "$1" 2>/dev/null | awk '{print $1}' else echo "$1" fi } # ########################################################################### # End alt_cmds package # ########################################################################### # ########################################################################### # Global variables # ########################################################################### TOOL="pt-ioprofile" # ########################################################################### # Subroutines # ########################################################################### # Read the 'lsof' and 'strace' from the file, and convert it into lines: # pid function fd_no size timing filename # The arguments are the files to summarize. tabulate_strace() { cat > $PT_TMPDIR/tabulate_strace.awk < function call if ( \$3 == "<..." ) { funcn = \$4; fd = unfinished[pid "," funcn]; if ( fd > 0 ) { filename = filename_for[fd]; if ( filename != "" ) { if ( funcn ~ /open/ ) { size = 0; } else { size_field = NF - 1; size = \$size_field; } timing = \$NF; gsub(/[<>]/, "", timing); print pid, funcn, fd, size, timing, filename; } } } # The beginning of a function call (not resumed). There are basically # two cases here: the whole call is on one line, and it's unfinished # and ends on a later line. else { funcn = substr(\$3, 1, index(\$3, "(") - 1); if ( funcn ~ wanted_pat ) { # Save the file descriptor and name for lookup later. if ( funcn ~ /open/ ) { filename = substr(\$3, index(\$3, "(") + 2); filename = substr(filename, 1, index(filename, "\\"") - 1); if ( "./" == substr(filename, 1, 2) ) { # Translate relative filenames into absolute ones. filename = cwd substr(filename, 2); } fd_field = NF - 1; fd = \$fd_field; filename_for[fd] = filename; } else { fd = substr(\$3, index(\$3, "(") + 1); gsub(/[^0-9].*/, "", fd); } # Save unfinished calls for later if ( \$NF == "...>" ) { unfinished[pid "," funcn] = fd; } # Function calls that are all on one line, not else { filename = filename_for[fd]; if ( filename != "" ) { if ( funcn ~ /open/ ) { size = 0; } else { size_field = NF - 1; size = \$size_field; } timing = \$NF; gsub(/[<>]/, "", timing); print pid, funcn, fd, size, timing, filename; } } } } } } EOF awk -f $PT_TMPDIR/tabulate_strace.awk "$@" } # Takes as input the output from tabulate_strace. Arguments are just a subset # of the overall command-line options, but no validation is needed. The last # command-line option is the filename of the tabulate_strace output. summarize_strace() { local func="$1" local cell="$2" local group_by="$3" local file="$4" cat > "$PT_TMPDIR/summarize_strace.awk" < 0 ) { result /= count[funcn "," thing]; } else { result = 0; } } if ( "$group_by" != "all" ) { output = output sprintf(col_pat, result); } else { printf(col_pat funcn "\\n", result); } } total_result = total_$cell; if ( "$func" == "avg" ) { if ( total_count > 0 ) { total_result /= total_count; } else { total_result = 0; } } printf(col_pat, total_result); if ( "$group_by" != "all" ) { print(output thing); } else { print "TOTAL"; } } } EOF awk -f $PT_TMPDIR/summarize_strace.awk "$file" > $PT_TMPDIR/summarized_samples if [ "$group_by" != "all" ]; then head -n1 $PT_TMPDIR/summarized_samples tail -n +2 $PT_TMPDIR/summarized_samples | sort -rn -k1 else grep TOTAL $PT_TMPDIR/summarized_samples grep -v TOTAL $PT_TMPDIR/summarized_samples | sort -rn -k1 fi } sigtrap() { warn "Caught signal, forcing exit" rm_tmpdir exit $EXIT_STATUS } main() { trap sigtrap HUP INT TERM if [ $# -gt 0 ]; then # Summarize the files the user passed in. tabulate_strace "$@" > $PT_TMPDIR/tabulated_samples else # There's no file to analyze, so we'll make one. if which strace > /dev/null 2>&1; then local samples=${OPT_SAVE_SAMPLES:-"$PT_TMPDIR/samples"} # Get the PID of the process to profile, unless the user # gave us it explicitly with --profile-pid. local proc_pid="$OPT_PROFILE_PID" if [ -z "$proc_pid" ]; then proc_pid=$(_pidof "$OPT_PROFILE_PROCESS" | awk '{print $1; exit;'}) fi date if [ "$proc_pid" ]; then echo "Tracing process ID $proc_pid" _lsof "$proc_pid" > "$samples" 2>&1 if [ "$?" -ne "0" ]; then echo "Error: could not execute lsof, error code $?" EXIT_STATUS=1 return 1 fi strace -T -s 0 -f -p $proc_pid >> "$samples" 2>&1 & if [ "$?" -ne "0" ]; then echo "Error: could not execute strace, error code $?" EXIT_STATUS=1 return 1 fi strace_pid=$! # sleep one second then check to make sure the strace is # actually running sleep 1 ps -p $strace_pid > /dev/null 2>&1 if [ "$?" -ne "0" ]; then echo "Cannot find strace process" >&2 tail "$samples" >&2 EXIT_STATUS=1 return 1 fi # sleep for interval -1, since we did a one second sleep # before checking for the PID of strace if [ $((${OPT_RUN_TIME}-1)) -gt 0 ]; then sleep $((${OPT_RUN_TIME}-1)) fi kill -s 2 $strace_pid sleep 1 kill -s 15 $strace_pid 2>/dev/null # Sometimes strace leaves threads/processes in T status. kill -s 18 $proc_pid # Summarize the output we just generated. tabulate_strace "$samples" > $PT_TMPDIR/tabulated_samples else echo "Cannot determine PID of $OPT_PROFILE_PROCESS process" >&2 EXIT_STATUS=1 return 1 fi else echo "strace is not in PATH" >&2 EXIT_STATUS=1 return 1 fi fi summarize_strace \ $OPT_AGGREGATE \ $OPT_CELL \ $OPT_GROUP_BY \ "$PT_TMPDIR/tabulated_samples" } # Execute the program if it was not included from another file. # This makes it possible to include without executing, and thus test. if [ "${0##*/}" = "$TOOL" ] \ || [ "${0##*/}" = "bash" -a "${_:-""}" = "$0" ]; then # Parse command line options. We must do this first so we can # see if --daemonize was specified. mk_tmpdir parse_options "$0" "$@" usage_or_errors "$0" po_status=$? rm_tmpdir if [ $po_status -eq 0 ]; then # Make a secure tmpdir. mk_tmpdir # XXX # TODO: This should be quoted but because the way parse_options() # currently works, it flattens files in $@ (i.e. given on the cmd # line) into the string $ARGV. So if we pass "$ARGV" then other # functions will see 1 file named "file1 file2" instead of "file1" # "file2". main $ARGV # Clean up. rm_tmpdir else [ $OPT_ERRS -gt 0 ] && EXIT_STATUS=1 fi exit $EXIT_STATUS fi # ############################################################################ # Documentation # ############################################################################ :<<'DOCUMENTATION' =pod =head1 NAME pt-ioprofile - Watch process IO and print a table of file and I/O activity. =head1 SYNOPSIS Usage: pt-ioprofile [OPTIONS] [FILE] pt-ioprofile does two things: 1) get lsof+strace for -s seconds, 2) aggregate the result. If you specify a FILE, then step 1) is not performed. =head1 RISKS B: pt-ioprofile freezes the server and may crash the process, or make it perform badly after detaching, or leave it in a sleeping state! Before using this tool, please: =over =item * Read the tool's documentation =item * Review the tool's known L<"BUGS"> =item * Test the tool on a non-production server =item * Backup your production server and verify the backups pt-ioprofile should be considered an intrusive tool, and should not be used on production servers unless you understand and accept the risks. =back =head1 DESCRIPTION pt-ioprofile uses C and C to watch a process's IO and print out a table of files and I/O activity. By default, it watches the mysqld process for 30 seconds. The output is like: Tue Dec 27 15:33:57 PST 2011 Tracing process ID 1833 total read write lseek ftruncate filename 0.000150 0.000029 0.000068 0.000038 0.000015 /tmp/ibBE5opS You probably need to run this tool as root. pt-ioprofile works by attaching C to the process using C, which will make it run very slowly until C detaches. In addition to freezing the server, there is some risk of the process crashing or performing badly after C detaches from it, or of C not detaching cleanly and leaving the process in a sleeping state. As a result, this should be considered an intrusive tool, and should not be used on production servers unless you are comfortable with that. =head1 OPTIONS =over =item --aggregate short form: -a; type: string; default: sum The aggregate function, either C or C. If sum, then each cell will contain the sum of the values in it. If avg, then each cell will contain the average of the values in it. =item --cell short form: -c; type: string; default: times The cell contents. Valid values are: VALUE CELLS CONTAIN ===== ======================= count Count of I/O operations sizes Sizes of I/O operations times I/O operation timing =item --group-by short form: -g; type: string; default: filename The group-by item. Valid values are: VALUE GROUPING ===== ====================================== all Summarize into a single line of output filename One line of output per filename pid One line of output per process ID =item --help Print help and exit. =item --profile-pid short form: -p; type: int The PID to profile, overrides L<"--profile-process">. =item --profile-process short form: -b; type: string; default: mysqld The process name to profile. =item --run-time type: int; default: 30 How long to profile. =item --save-samples type: string Filename to save samples in; these can be used for later analysis. =item --version Print the tool's version and exit. =back =head1 ENVIRONMENT This tool does not use any environment variables. =head1 SYSTEM REQUIREMENTS This tool requires the Bourne shell (F). =head1 BUGS For a list of known bugs, see L. Please report bugs at L. Include the following information in your bug report: =over =item * Complete command-line used to run the tool =item * Tool L<"--version"> =item * MySQL version of all servers involved =item * Output from the tool including STDERR =item * Input files (log/dump/config files, etc.) =back If possible, include debugging output by running the tool with C; see L<"ENVIRONMENT">. =head1 DOWNLOADING Visit L to download the latest release of Percona Toolkit. Or, get the latest release from the command line: wget percona.com/get/percona-toolkit.tar.gz wget percona.com/get/percona-toolkit.rpm wget percona.com/get/percona-toolkit.deb You can also get individual tools from the latest release: wget percona.com/get/TOOL Replace C with the name of any tool. =head1 AUTHORS Baron Schwartz =head1 ABOUT PERCONA TOOLKIT This tool is part of Percona Toolkit, a collection of advanced command-line tools for MySQL developed by Percona. Percona Toolkit was forked from two projects in June, 2011: Maatkit and Aspersa. Those projects were created by Baron Schwartz and primarily developed by him and Daniel Nichter. Visit L to learn about other free, open-source software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2; OR the Perl Artistic License. On UNIX and similar systems, you can issue `man perlgpl' or `man perlartistic' to read these licenses. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION pt-ioprofile 2.2.16 =cut DOCUMENTATION