#!/bin/ksh
#
# License:	Please refer to the license file (license.txt) for license and support information
#
# Purpose:	Dump process information from both kernel(kdb) and user space (proctools/dbx).
# Usage:    	See print_usage().
# Last update:  See PDUMPVERSION
# Author:	Tao Chen

export PDUMPVERSION="Version 07.31.2007"
export LANG=C
export LC_ALL=C
export PATH=/sbin:/usr/sbin:/usr/bin

print_usage()
{
        print "\nUsage: ${0##/*/} [ -k | -d ] <PID>"
	print "       -k: skip kdb"
	print "       -p: call proctools"
	print "       -d: call dbx (implies -p)"
	print "       -l: long mode with more output\n"
}

clean_up()
{
        print -u2 "\nScript is stopped. Partial output is saved in $OFILE.\n"
        exit
}

#################################################################################################### general
#
# This section should only use commands that least likely to hang.
#

get_general_config()
{
    print "\nGetting general environment data ..."
    {
	print "Date:     $(date +%d%h%Y-%H.%M.%S)"
	print "Machine:  $(hostname) - $(uname -M)"
	print "pdump.sh: $PDUMPVERSION"
	print "User:     $(whoami)"

	## if not root, BIT is not a must, since we won't run kdb
	if [[ $(whoami) = "root" ]]; then  
		BIT=$((`bootinfo -K`)) 
		print "Kernel:   $BIT-bit"
	fi

	print ""
	lslpp -Lc \
		bos.up \
		bos.mp \
		bos.mp64 \
		bos.rte.libc \
		bos.rte.libpthreads \
		bos.rte.filesystem \
		bos.rte.lvm \
		bos.adt.debug \
		bos.sysmgt.serv_aid 2>&1 | awk '
		/^bos/ {
			gsub(/:/, " ")
			printf("%-8s - %-s\n", $3, $2)
		}'

	print "\nemgr -l:"
	emgr -l 2>&1 | grep -p ID

	## 'C' and 'TIME' give hint on CPU usage (looping or hanging)
	print "\n# ps -fp $PID"
	ps -fp $PID

	if [[ $(whoami) = "root" ]]; then  
		print "\n# svmon -P $PID"
		svmon -P $PID 2>/dev/null | egrep -v "\-\-"
	fi

        print "\n# proctree $PID" 
        proctree $PID 2>&1

    } >> $OFILE

}
## END of get_general_config()

#################################################################################################### kdb

run_kdb()
{
	print "Dumping process information from kdb ...\n"

	print "\n# kdb \n" >> $OFILE
	print "\tdumping process slot $PSLT ..."

	BK="\n\n\n\n\n"

	## 08/08/2006: tpid reports wrong list of threads, use th -p pslot instead
	printf " th -p %d $BK proc %d $BK lle -v -p %d\n" $PSLT $PSLT $PSLT \
	| kdb | sed -n -e '/^(0)/,$p' >> $OFILE 

	> pdump.tslot
	awk '
	/^pvthread|^thread/ {
		sub(/!/, " "); 
		sub(/>/, " "); 
		if ($4 != "ZOMB") 
			print $2 >> "pdump.tslot"
	}' < $OFILE

	if [[ ! -s pdump.tslot ]]; then
		print -u2 "Error getting thread list. Skip other kdb commands."
		return
	fi

	# USER is a proc structure shared by all threads, 
	# in kdb though, 'user' command actually prints u-block of each threads.
	# ( USER = u-block - uthread ) 
	# so only the first thread's full user output is saved here.

	# buld a command list
	print "\tbuilding kdb commands ..."
	COMMS=""

	FIRSTT="true"
	for TSLT in `cat pdump.tslot`
	do 
		print "\tthread slot $TSLT ..."
		if [[ $FIRSTT = "true" ]]; then
			COMMS=$COMMS"$BK th $TSLT $BK user $TSLT $BK f $TSLT $BK f -v $TSLT $BK sw $TSLT $BK mst $BK dr iar $BK sr64 $BK segst64 $BK u -ad"
			FIRSTT="false"
		else
			COMMS=$COMMS"$BK th $TSLT $BK user -ut $TSLT $BK f $TSLT $BK f -v $TSLT $BK sw $TSLT $BK mst"
		fi
	done
	rm pdump.tslot >/dev/null 2>&1 

	############################################## lock in kdb ##############################################
	#
	#
	# for each thread in this section, f and f -v output are collected.
	# f is for easy copy/paste into pmr/email
	# f -v is more comprehensive than "set 10; set 18".

	# Get lock info if any thread is waiting on lock
	# Sample:
	# pvthread+01F300  499 harmad   SLEEP 1F3001 03C   9  0 F10000E33E1D8000 slist_table+000800 
	# pvthread+020900  521 lspv     SLEEP 209061 03C   2  0 082522C0         slist_table+000E20 

	# select threads based on unique name and lock address,
	# because if two threads have the same name and lock address,
	# chances are the are in the same stack.
 	# "uniq -f1" means ignore one field (tslot).

	COMMS=$COMMS"$BK lq"
	COMMS=$COMMS"$BK th -w WLOCK"

	echo "th -w WLOCK" | kdb | grep ^pvthread | sort -k9 -k3 > pdump.lock
	awk '{ print $2, $3, $9 }' < pdump.lock | uniq -f1 | while read tslot tname lkaddr
	do 
		## clk command covers slk just fine.
		COMMS=$COMMS"$BK clk $lkaddr $BK f $tslot $BK f -v $tslot"
	done
	rm pdump.lock >/dev/null 2>&1

	# Get stack for those holding a lock but in SLEEP
	# Sample:
	# pvthread+018500  389 scopeux  SLEEP 185015 03C   8 0 F10000E33E1D9538 
	# pvthread+01F300  499 harmad   SLEEP 1F3001 03C   9 0 F10000E33E1D8000 slist_table+000800
	# pvthread+02EB00  747 sas      SLEEP 2EB04F 064  13 0 vmmswpft+77793160
	#
	# if thread is in SLEEP because of another lock, we should have
	# that lock covered already in WLOCK section above.

	COMMS=$COMMS"$BK th -lk"  

	echo "th -lk" | kdb | awk '/SLEEP/ {if (NF == 9) print $2}' | while read tslot 
	do
		COMMS=$COMMS"$BK f $tslot $BK f -v $tslot"
	done
	
	## 01/25/2006: dla may generate endless "Out of lock descriptors" error
	# COMMS=$COMMS"$BK dla"

	#
	#
	############################################## end of lock ##############################################

	# I/O
	# mounted filesystem
	COMMS=$COMMS"$BK vfs"
	COMMS=$COMMS"$BK pdt *"
	COMMS=$COMMS"$BK th -w WPGIN"

	# runqueue
	COMMS=$COMMS"$BK rq"
	COMMS=$COMMS"$BK rqi"
	COMMS=$COMMS"$BK th -r"

	# kernel extension: 
	COMMS=$COMMS"$BK lke"

	if [[ -n $LONGMODE ]]
	then

		# library 
		COMMS=$COMMS"$BK lle -l32"
		COMMS=$COMMS"$BK lle -l64"

		# IPC :
		COMMS=$COMMS"$BK ipc 1 1"
		COMMS=$COMMS"$BK ipc 2 1"
		COMMS=$COMMS"$BK ipc 3 1"
	fi

	# send commands to kdb
	print "\texecuting kdb commands ..."
	print $COMMS | kdb | sed -n -e '/^(0)/,$p' >> $OFILE



} 
## END of run_kdb()

#################################################################################################### proc

run_proctools()
{
	print "\nDumping process information with proc tools ...\n"
	
	print "\n# proccred $PID" >> $OFILE
	proccred $PID 	>> $OFILE 2>&1

	print "\n# procfiles $PID" >> $OFILE
	procfiles $PID 	>> $OFILE 2>&1

	print "\n# procflags $PID" >> $OFILE
	procflags $PID 	>> $OFILE 2>&1

	print "\n# procldd $PID" >> $OFILE
	procldd $PID 	>> $OFILE 2>&1

	print "\n# procmap $PID" >> $OFILE
	procmap  $PID 	>> $OFILE 2>&1

	print "\n# procsig $PID" >> $OFILE
	procsig $PID 	>> $OFILE 2>&1

	print "\n# procstack $PID" >> $OFILE
	procstack $PID 	>> $OFILE 2>&1

	# now in general
	#print "\n# proctree $PID" >> $OFILE
	#proctree $PID 	>> $OFILE 2>&1

	print "\n# procwdx $PID" >> $OFILE
	procwdx $PID 	>> $OFILE 2>&1
}

#################################################################################################### dbx

run_dbx()
{
	print "\nDumping process information from dbx ...\n"

        DBXCMD="/usr/bin/dbx -a $PID"

        print "\n# dbx -a $PID \n" >> $OFILE

	## Get the current thread's output first
	print "\n p '(dbx) where' \n where \
	      \n p '(dbx) x' \n x     \
	      \n p '(dbx) (\$stkp)/200' \n (\$stkp)/200 \
	      \n p '(dbx) map' \n map   \
	      \n p '(dbx) p __n_pthreads' \n p __n_pthreads \
	      \n p '(dbx) p __multi_threaded' \n p __multi_threaded \
	      \n p '(dbx) mutex' \n mutex \
	      \n p '(dbx) condition' \n condition \
	      \n p '(dbx) rwlock' \n rwlock \
	      \n p '(dbx) dump .' \n dump . \
	      \n p '(dbx) th' \n th \
	      \n p '(dbx) detach' \n detach" | $DBXCMD >> $OFILE 2>&1

	if [[ $? -ne 0 ]]; then
		print -u "Can not dbx attach to the process. Skip dbx."
		return 
	fi

	sed -n -e '/(dbx) th/,$p' $OFILE | grep "k-tid" >/dev/null 2>&1 
	## multi-threaded
	if [[ $? -eq 0 ]] 
	then
		## build a command list
		BRK='p "."'
		SECT=">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
		COMMS="$BRK\n"

		COMMS=$COMMS"p '$SECT thread info'\n$BRK\n thread info\n$BRK\n"

		## Since thread number is not necessarily continuous,
		## we have to get the exact thread number.

		sed -n -e '/(dbx) th/,$p' $OFILE | awk '/^.\$t[0-9]/ {print $1}' | while read TPT
		do
			PT=${TPT#*t}

			print "\tdumping tid $PT ..."
			COMMS=$COMMS"p '$SECT thread current $PT' \n$BRK\n thread current $PT\n"
			COMMS=$COMMS"p '$SECT x $PT' \n$BRK\n x \n$BRK\n"
			COMMS=$COMMS"p '$SECT where $PT' \n$BRK\n where \n$BRK\n"
			COMMS=$COMMS"p '$SECT (\$stkp)/100 $PT' \n$BRK\n (\$stkp)/100 \n$BRK\n"
		done
		COMMS=$COMMS"p '(dbx) detach' \n detach"

		print "\n# dbx -a $PID \n" >> $OFILE
		print $COMMS | $DBXCMD 2>/dev/null >> $OFILE
	fi

	print "\tlisting object files ..."
	print "\n + List of object files: \n" >> $OFILE
	awk '/Object name: / { print $NF }' $OFILE | /usr/bin/sort | /usr/bin/uniq | while read obj
	do
		## the main program may not locate in the current directory
		## so ls -l may return error > /dev/null
		realobj=$obj
		/bin/ls -l $realobj >> $OFILE 2>/dev/null
		/bin/ls -l $realobj 2>/dev/null | /usr/bin/grep ^lrwx >/dev/null 2>&1
		while [[ $? -eq 0 ]]
		do
			realobj=`/bin/ls -l $realobj 2>/dev/null | awk '{print $NF}'`
			/bin/ls -l $realobj >> $OFILE  2>/dev/null
			/bin/ls -l $realobj 2>/dev/null | /usr/bin/grep ^lrwx >/dev/null 2>&1
		done
	done
} 
## END of run_dbx

#################################################################################################### main

while getopts :kdplh flag ; do
        case $flag in
                k)      NOKDB=1;;
                d)      USEDBX=1;;
                p)      PROCTOOL=1;;
		l)	LONGMODE=1;;
                h)      print_usage
                        return 0;;
                \?)     print -u2 "\nInvalid parameter"
                        print_usage
                        return 1;;
        esac
done
shift $(($OPTIND -1))

## check tools
{
	## check permission to run kdb
	if [[ -z $NOKDB && $(whoami) != root ]]; then
		print -u2 "\n'root' authority is required for kdb (use '-k' to skip kdb).\n" 
		return 1
	fi

	## check dbx tool
	if [[ -n $USEDBX && (! -f /usr/bin/dbx) ]]; then
		print -u2 "/usr/bin/dbx doesn't exist. Install bos.adt.debug or use '-d' to skip dbx."
		return 1
	fi
}

## validate PID
{
	## need one parameter
	if [[ $# -ne 1 ]]; then
		print_usage
		return 1
	fi

	## numeric?
	if [[ ${1##+([0-9])} != "" ]] ; then
		print -u2 "\n$1 is not a PID"
		print_usage
		return 1
	fi

	## existing PID?
	/bin/ps -p $1 > /dev/null 2>&1
	if [[ $? -eq 1 ]]; then
		print -u2 "\nPID $1 doesn't exist.\n"
		return 1
	fi

	PID=$1
}


## create output file
{
	OFILE="pdump.$(/bin/ps -p $PID -ocomm=).$PID.`date +%d%h%Y-%H.%M.%S`.out"
	> $OFILE
	if [[ $? -ne 0 ]]; then
		print -u2 "\nCannot create output file in the current directory. Please check permission.\n"
		return 1
	fi
}

trap clean_up TERM INT

## collect output
{
	get_general_config

	if [[ -z $NOKDB ]]; then

		## from PID to PSLT
                osl=$(oslevel | cut -c1-3)
                if [[ $BIT -eq 32 ]]
                then
                        let PSLT=$PID/256
                else
                        if [[ $osl = "5.1" || $osl = "4.3" ]]
                        then
                                let PSLT=$PID/8192
                        else
                                let PSLT=$((`echo $PID/256%16*16384+$PID/4096 | bc`))
                        fi
                fi
                # init is an exception (1/$DIVISOR = 0)
                if [[ $PID -eq 1 ]]; then
                        PSLT=1
                fi

		run_kdb
	fi

	if [[ -f /usr/bin/procstack ]]; then
		if [[ -n $PROCTOOL || -n $USEDBX ]]; then
			run_proctools ## new in 5.2+
		fi
	fi

	if [[ -n $USEDBX ]]; then
		run_dbx
	fi

	# extra data
	{
		print "\n# ps -mp $PID -o THREAD\n"
		ps -mp $PID -o THREAD
		print "\n# ps auxeww $PID\n"
		ps auxeww $PID
		print "\n# ps -efk \n" 
		ps -efk 

		if [[ -n $LONGMODE ]]; then
			print "\n# ipcs -a \n"
			ipcs -a
			if [[ `oslevel` = "5.3.0.0" ]]; then
				print "\n# genld -ld | egrep -p "^Proc_pid:[[:blank:]]+$PID[[:blank:]]"\n"
				genld -ld | egrep -p "^Proc_pid:[[:blank:]]+$PID[[:blank:]]"
				print "\n# genkld -d \n"
				genkld -d
			else
				print "\n# genld -l | egrep -p "^Proc_pid:[[:blank:]]+$PID[[:blank:]]"\n"
				genld -l | egrep -p "^Proc_pid:[[:blank:]]+$PID[[:blank:]]"
			fi
			
			## network stuff
			netstat -Aan
		fi
	} >> $OFILE
}

print "\nDone.\nOutput file is $OFILE\n"
/bin/ls -l $OFILE 
print

## The End
## Change Log:
## 07.31.2007 - move some long output to $LONGMODE only, such as ipc, sys loader, netstat, etc., to reduce default output file size
##            - add dr iar & segment register commands
