Showing posts with label monitoring. Show all posts
Showing posts with label monitoring. Show all posts

Thursday, 4 July 2013

Use wsadmin to monitoring WebSphere App Server SSL certificate expiry

A Jython script to check all certificates that are stored in keystores under Cell management. At my client's site I added IHS, Plugin and CACerts keystores to the Cell so that they too can be checked.
If you have access to an SMTP service this script will send an email when a cert is due to expire in less than X days. I run this from a bourne wrapper (I'll place the code for this at the end of this post) which also sends an email if it can't run the AdminTask methods it needs to for any reason.

Here's the Jython code

# --------------------------------------------------------------------------------
# checkCertificates.py
# Author: Bob Clarke (IBM)
# Date: 19/06/2013
# --------------------------------------------------------------------------------

# --------------------------------------------------------------------------------
# Setup
# --------------------------------------------------------------------------------
import re
import sys
import time
import os
import javaos
from java.text import SimpleDateFormat ;
dateFormat = SimpleDateFormat("dd-MMM-yyyy");
emailRecipients = "bob.clarke@stack1.com"
emailContent = "props/email.content"
smtpUrl = "smtp=smtp://smtphub.stack1.com"

# --------------------------------------------------------------------------------
# Define Subroutines 
# --------------------------------------------------------------------------------
def dateDiff(keystoreName, issuedTo, expString, scopeName):
        todayString =  time.strftime("%d-%b-%Y", time.gmtime())
        todayDate = dateFormat.parse(todayString)
        expiryDate = dateFormat.parse(expString)
        e = expiryDate.getTime()
        t = todayDate.getTime()
        d = e - t
        days = d / (1000 * 60 * 60 * 24)
        print "\tExpires "+expString
        if(days < 31):
                if(re.search("blueworks",  issuedTo)):
                        print "\tIgnoring BlueworksLive cert"
                else:
                        print "\tALERT - this certificate will expire in "+str(days)+" days"
                        file = open('props/email.content','w')
                        file.write('ACTION REQUIRED : The following certificate will expire in '+str(days)+' days\n\n')
                        file.write('- Environment '+env+'\n\n')
                        file.write('- Expiry Date '+expString+'\n\n')
                        file.write('- '+str(issuedTo)+'\n\n')
                        file.write('- Keystore name '+str(keystoreName)+'\n\n')
                        file.write('- Keystore scope '+str(scopeName)+'\n')
                        file.close()
                        sendEmail()

def sendEmail():
        os.system('cat '+emailContent+' | mailx -v -s "Certificate expiry notice for '+env+'" -S '+smtpUrl+' -S from="smtp@stack1.com" '+emailRecipients+' >> logs/smtp.log 2>&1')

# ---------------------------------------------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------------------------------------------

env = sys.argv[0]
print
print 'Obtaining keystore information for '+env

# Iterate through all keystores, print each cert with expiry dates and ask user if they want to replace
for ks in AdminTask.listKeyStores('[-all true -keyStoreUsage SSLKeys ]').splitlines():
        keystoreName =  AdminConfig.showAttribute(ks, 'name')
        ms = AdminConfig.showAttribute(ks, 'managementScope')
        scopeName = AdminConfig.showAttribute(ms, 'scopeName')

        print '\n## START '+keystoreName +' in scope '+scopeName+'##'

        print '\n\t## START personal certificates ##'
        personalCertsFound=0
        for cert in AdminTask.listPersonalCertificates('[-keyStoreName '+keystoreName+' -keyStoreScope '+scopeName+']').splitlines():
                personalCertsFound=1
                issuedTo=""
                for property in re.split("\] \[", cert):
                        if(re.search("\[\[",  property)):
                                tmp = property
                                property = re.split("\[\[",tmp)[1]
                        if(re.search("] ]",  property)):
                                tmp = property
                                property = re.split("] ]",tmp)[0]
                        if(re.search("alias", property)):
                                alias = re.split("\s+", property)[1]
                                print "\n\t"+property
                        if(re.search("issuedTo", property)):
                                issuedTo=property
                                print "\t"+property
                        if(re.search("issuedBy", property)):
                                print "\t"+property
                        if(re.search("validity", property)):
                                expString = property.split()[5].split('.]')[0]
                                #dateDiff(expString)
                                dateDiff(keystoreName, issuedTo, expString, scopeName)


        if(personalCertsFound==0):
                print '\tNo personal certificates found in '+keystoreName+' in scope '+scopeName
        print '\n\t## END personal certificates ##'

        print '\n\t## START signer certificates ##'
        signerCertsFound=0
        for cert in AdminTask.listSignerCertificates('[-keyStoreName '+keystoreName+' -keyStoreScope '+scopeName+']').splitlines():
                signerCertsFound=1
                issuedTo=""
                for property in re.split("\] \[", cert):
                        if(re.search("\[\[",  property)):
                                tmp = property
                                property = re.split("\[\[",tmp)[1]
                        if(re.search("] ]",  property)):
                                tmp = property
                                property = re.split("] ]",tmp)[0]
                        if(re.search("alias", property)):
                                alias = re.split("\s+", property)[1]
                                print "\n\t"+property
                        if(re.search("issuedTo", property)):
                                issuedTo=property
                                print "\t"+property
                        if(re.search("issuedBy", property)):
                                print "\t"+property
                        if(re.search("validity", property)):
                                expString = property.split()[5].split('.]')[0]
                                #dateDiff(expString)
                                dateDiff(keystoreName, issuedTo, expString, scopeName)

        if(signerCertsFound==0):
                print '\tNo signer certificates found in '+keystoreName+' in scope '+scopeName
        print '\n\t## END signer certificates ##'

        print '\n## END '+keystoreName +' in scope '+scopeName+'##'



And here's the wrapper script to run it. NOTE: This uses a cutdown version of wadmin.sh (which I've renamed to wsAdminLite.sh - see this post) which I will describe in an upcoming post. They key advantage of this for me is that I was able to point the wsadmin client at my own keystore in which I'd loaded the cell signers for each environment rather than loading all of these (untidily) into the Cell default trust store of another WAS environment.
You'll notice that this script writes a new soap.client.props for each environment, in this way you can XOR encode each password (better than plain text).


#!/bin/sh
# -----------------------------------------------------------------------------
# run.sh
# Author: Bob Clarke (IBM)
# Date: 01/07/2013
# -----------------------------------------------------------------------------
wsaCmd="./wsAdminLite.sh -lang jython"
envProps="props/env.props"
soapProps="props/soap.client.props"
run_log="logs/run.log"
emailRecipients="bob.clarke@stack1.com"
pwd=`pwd`
timestamp=`date "+%d/%m/%y %H:%M:%S"`
scripthost=`hostname`

cat $envProps | grep -v '^#' | while read line
do
        # Parse props
        env=`echo $line | awk -F: '{print $1}'`
        host=`echo $line | awk -F: '{print $2}'`
        port=`echo $line | awk -F: '{print $3}'`
        user=`echo $line | awk -F: '{print $4}'`
        pw=`echo $line | awk -F: '{print $5}'`

        # Write soap.client.props
        echo "com.ibm.SOAP.loginUserid=admin" > ${soapProps}
        echo "com.ibm.SOAP.loginPassword={xor}"${pw} >> ${soapProps}
        echo "com.ibm.ssl.alias=DefaultSSLSettings" >> ${soapProps}

        echo >> $run_log
        echo ENV is $env >> $run_log
        echo Timestamp is $timestamp >> $run_log
        echo "Running: $wsaCmd -host $host -port $port -f checkCertificates.py $env using $soapProps" >> $run_log
        out=`$wsaCmd -host $host -port $port -f checkCertificates.py $env`
        echo "Command output is: $out" >> $run_log

        # Check if we successfully connected to the deployment manager and obtained cert details
        echo $out | grep "END signer certificates" 2>&1 > /dev/null
        if [ $? -ne 0 ]; then
                echo
                echo "It seems the attempt to run checkCertificates.py on environment (${env}) has failed"
                echo "Invocation of checkCertificates.py on host (${scripthost}) to check certificates on environment (${env}) has failed, please check ${pwd}/${run_log}" | mailx -v -s "ACTION REQUIRED :Certificate check for $env has failed" -S smtp=smtp://smtphub.stack1.com -S from="smtp@stack1.com" $emailRecipients >> logs/smtp.log 2>&1
                echo
        fi
done


.... and here's an example env.props file


#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# NOTE: The first column is a simple flag that is passed
# to wsadmin so that it can print out a meaningful
# string to describe the environment being checked
# It's up to the person who edits this file to ensure it's accurate
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
PNL_PERF002:jupiter.stack1.com:11005:admin:gfe8Shm1sPSjy
PNL_DEV001:pluto.stack1.com:8887:admin:HQ8SbhdsS8y
PNL_PROC_CENTRE:saturn.stack1.com::12005:admin:Lhjhju98zxubW==
PNL_SIT001:mars.stack1.com::8879:admin:HQ8Sjkj654kjk==
PNL_PERF001:venus.stack1.com::11005:admin:t65hhjtPS8y=

Tuesday, 4 June 2013

Lombardi BPM Instrumentation

Great article taken from http://www-01.ibm.com/support/docview.wss?uid=swg21613989

  1. Log into the Process Admin Console as a user in the tw_admins group, such as tw_admin or admin. The error SECJ0305 may appear if this is run without the base internal users.

    For example:
    http://server_name:port_number/ProcessAdmin

  2. Under Monitoring, click Instrumentation.

  3. Click Start Logging to collect data. Note where it stores the file. For example, in IBM Business Process Manager Version 7.5, the file might be stored in the following directory:
    C:\IBM\BPM\v7.5\profiles\ProcCtr01\logs\inst001.dat

  4. Click Stop Logging.

  5. Retrieve the file for processing. 
 
How do I to decode the instrumentation .dat files? 
Use the following command to extract the data: 
java -Xmx1024M -cp svrcoreclnt.jar  com.lombardisoftware.instrumentation.log.tools.NonXMLDump inst001.dat >  inst001.txt 
 
Notes: 
  • The svrcoreclnt.jar file needs to be in a path that Java can execute. You can copy this file to a separate directory to your desktop to run. The location of  the Java archive (JAR) file is: [install_dir]\Lombardi\lib

  • 1024MB heap will be sufficient for almost all cases.

  • inst001.dat is the instrumentation file.

  • inst001.txt is a decoded file in the plain text format. 

Reading the decoded file
The instrumentation log files are important because they tell you exactly how long it took for something to run. The following command finds all threads that took longer than 1000 milliseconds (1 second) to complete, which are the threads that you should examine first.
 
grep -E "(THREAD|period [0-9]{4,}ms)" inst001.txt 

Example Sample output: 
>> THREAD Thread-47 << 
>> THREAD WebContainer : 2 << 
03:17:10.356 period 1156ms 'Resume Workflow Engine' { 
03:17:10.418    period 1031ms 'Do Job'  
Worker=com.lombardisoftware.component.javaconnector.worker.JavaConnectorWorker { 
03:17:10.418       period 1031ms 'Java Execution' { 
>> THREAD WebContainer : 13 << 
>> THREAD WebContainer : 16 << 

The times are nested,  1156ms 'Resume Workflow Engine' is running a coach and took 1.2 seconds. Within here, you see 1.0 seconds for executing Java. After this function, there might be another function that took 400 milliseconds.  

Thursday, 2 May 2013

Solaris CPU info


This script is courtesy of Oracle and was taken from
https://blogs.oracle.com/mandalika/entry/solaris_show_me_the_cpu


#!/bin/bash

/usr/bin/kstat -m cpu_info | egrep "chip_id|core_id|module: cpu_info" > /var/tmp/cpu_info.log

nproc=`(grep chip_id /var/tmp/cpu_info.log | awk '{ print $2 }' | sort -u | wc -l | tr -d ' ')`
ncore=`(grep core_id /var/tmp/cpu_info.log | awk '{ print $2 }' | sort -u | wc -l | tr -d ' ')`
vproc=`(grep 'module: cpu_info' /var/tmp/cpu_info.log | awk '{ print $4 }' | sort -u | wc -l | tr -d ' ')`

nstrandspercore=$(($vproc/$ncore))
ncoresperproc=$(($ncore/$nproc))

speedinmhz=`(/usr/bin/kstat -m cpu_info | grep clock_MHz | awk '{ print $2 }' | sort -u)`
speedinghz=`echo "scale=2; $speedinmhz/1000" | bc`

echo "Total number of physical processors: $nproc"
echo "Number of virtual processors: $vproc"
echo "Total number of cores: $ncore"
echo "Number of cores per physical processor: $ncoresperproc"
echo "Number of hardware threads (strands or vCPUs) per core: $nstrandspercore"
echo "Processor speed: $speedinmhz MHz ($speedinghz GHz)"

# now derive the vcpu-to-core mapping based on above information #

echo -e "\n** Socket-Core-vCPU mapping **"
let linenum=2

for ((i = 1; i <= ${nproc}; ++i ))
do
        chipid=`sed -n ${linenum}p /var/tmp/cpu_info.log | awk '{ print $2 }'`
        echo -e "\nPhysical Processor $i (chip id: $chipid):"

        for ((j = 1; j <= ${ncoresperproc}; ++j ))
        do
                let linenum=($linenum + 1)
                coreid=`sed -n ${linenum}p /var/tmp/cpu_info.log | awk '{ print $2 }'`
                echo -e "\tCore $j (core id: $coreid):"

                let linenum=($linenum - 2)
                vcpustart=`sed -n ${linenum}p /var/tmp/cpu_info.log | awk '{ print $4 }'`

                let linenum=(3 * $nstrandspercore + $linenum - 3)
                vcpuend=`sed -n ${linenum}p /var/tmp/cpu_info.log | awk '{ print $4 }'`

                echo -e "\t\tvCPU ids: $vcpustart - $vcpuend"
                let linenum=($linenum + 4)
        done
done

rm /var/tmp/cpu_info.log
It's also useful to verify that this agrees with what you see in /usr/sbin/psrinfo -pv