Udiff functions.ksh
--- /workspace/du105637/oscposthot/webrev/usr/src/cmd/ha-services/gds-agents/PostgreSQL/functions.ksh-  Tue Apr 22 05:14:55 2008
+++ functions.ksh       Tue Apr 22 04:42:23 2008
@@ -18,15 +18,15 @@
 #
 # CDDL HEADER END
 #
 
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 
-#ident "%Z%%M% %I%     %E% SMI"
+# ident   "%Z%%M% %I%     %E% SMI"
 
 PKG=SUNWscPostgreSQL
 METHOD=`basename $0`
 TASK_COMMAND=""
 
@@ -55,10 +55,18 @@
 ZLOGIN=/usr/sbin/zlogin
 TEST=/usr/bin/test
 PING=/usr/sbin/ping
 SED=/usr/bin/sed
 IFCONFIG=/usr/sbin/ifconfig
+SSH=/usr/bin/ssh
+SSH_AGENT=/usr/bin/ssh-agent
+PKILL=/usr/bin/pkill
+WC=/usr/bin/wc
+SLEEP=/usr/bin/sleep
+TOUCH=/usr/bin/touch
+CHOWN=/usr/bin/chown
+CP=/usr/bin/cp
 
 get_fmri_parameters ()
 {
 
 # extract the smf properties, you need to call your agent commands
@@ -257,10 +265,47 @@
 SCUSER=${SCUSER}
 SCTABLE=${SCTABLE}
 SCPASS=${SCPASS}
 NOCONRET=${NOCONRET}
 
+# The following parameters need to be configured only if logfile shipping is configured to ship
+# the PosgreSQL WAL logs between a designated primary and a designated standby resource.
+# They need to be configured only on the primary.
+
+#        STDBY_RS  The resource name of the PostgreSQL standby resource.
+#        STDBY_RG  The resource group name of the PostgreSQL standby resource group.
+#      STDBY_USER  User which is the owner of the standby postgres database.
+#      STDBY_HOST  Resolvable name of the standby host or the standby zone, 
+#                  this name has to be reachable via ssh.
+#   STDBY_PARFILE  The standbys postgres parameter file to get the rest of the necessary parameters.
+#   STDBY_PING     The number of of packets the primary uses to ping the standby host. If this variable is
+#                  empty, it will be set to 5 packets.
+#   ROLECHG_RS     The rolechangers resource name.
+#  SSH_PASSDIR     A directory where the ssh passphrase is stored in a the file <resourcename>-phrase.
+#                  This parameter is needed only if you configured WAL file shipping and secured your
+#                  ssh key with a passphrase.
+#                  Leave it undefined if the passprase is empty.
+#
+# If you configure the logfile shipping in a shared nothing topology, do not set the LH parameter.
+#
+# Configure the following paramters on the primary host.
+
+STDBY_RS=${STDBY_RS}
+STDBY_RG=${STDBY_RG}
+STDBY_USER=${STDBY_USER}
+STDBY_HOST=${STDBY_HOST}
+STDBY_PARFILE=${STDBY_PARFILE}
+STDBY_PING=${STDBY_PING}
+#
+# Configure the following paramter on the standby host.
+#
+ROLECHG_RS=${ROLECHG_RS}
+#
+# Configure the following parameter on both hosts.
+#
+SSH_PASSDIR=${SSH_PASSDIR}
+
 EOF
        if [ $? -ne 0 ]
        then
                ${ECHO} "ERROR: could not create the temporary parameter file ${pfile_tmp}"
                return 1
@@ -835,10 +880,99 @@
                                        else
                                                debug_message "Function: validate - NOCONRET OK"        
                                        fi
                                fi
                         fi;;
+
+                       STDBY_HOST) 
+
+                       # Test the standby host variable and the neccessary other variables if the standby 
+                       # host variable is not set.
+                             
+                       if [ -n "${STDBY_HOST}" ]; then
+                               if ! ${GETENT} hosts ${STDBY_HOST} >/dev/null 2>&1
+                               then
+                       
+                                       # SCMSGS
+                                       # @explanation
+                                       # The host specified in the STDBY_HOST variable is not
+                                       # resolvable.
+                                       # @user_action
+                                       # Add the host to one of your configured name services, 
+                                       # so it can get listed with getent.
+                                       scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                                               "Function: validate - The standby host %s is not resolvable" \
+                                               "${STDBY_HOST}"
+                                       rc_validate=1
+                                               
+                               else
+                                       debug_message "Function: validate - STDBY_HOST resolvable"      
+                               fi
+
+                               # Test if all the other standby related variables are set.
+                               # There will be no other validation, because it may be that the necessary
+                               # resources are not up right now, and then the validation will fail without 
+                               # a valid reason which is related to the parameter configuration.
+
+                               if [ -z "${STDBY_RS}" ]
+                               then
+                                       scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                                               "Function: validate: The %s variable is not set, but it is required" \
+                                               "STDBY_RS"
+                                       rc_validate=1
+                               else
+                                       debug_message "Function: validate - STDBY_RS OK"
+                               fi
+
+                               if [ -z "${STDBY_RG}" ]
+                               then
+                                       scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                                               "Function: validate: The %s variable is not set, but it is required" \
+                                               "STDBY_RG"
+                                       rc_validate=1
+                               else
+                                       debug_message "Function: validate - STDBY_RG OK"
+                               fi
+
+                               if [ -z "${STDBY_USER}" ]
+                               then
+                                       scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                                               "Function: validate: The %s variable is not set, but it is required" \
+                                               "STDBY_USER"
+                                       rc_validate=1
+                               else
+                                       debug_message "Function: validate - STDBY_USER OK"
+                               fi
+
+                               if [ -z "${STDBY_PARFILE}" ]
+                               then
+                                       scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                                               "Function: validate: The %s variable is not set, but it is required" \
+                                               "STDBY_PARFILE"
+                                       rc_validate=1
+                               else
+                                       debug_message "Function: validate - STDBY_PARFILE OK"
+                               fi
+
+                               if [ -n "${STDBY_PING}" ]
+                               then
+                                       if  ! let x=${STDBY_PING} >/dev/null 2>&1
+                                       then
+                                               scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                                                       "Function: validate: The STDBY_PING value of %s is not numeric" \
+                                                       "${STDBY_PING}"
+                                               rc_validate=1
+                                       else
+                                               debug_message "Function: validate - STDBY_PING OK"
+                                       fi
+                               else
+                                       # set a default of 5 packets
+                                       STDBY_PING=5
+                                       debug_message "Function: validate - STDBY_PING OK, it was set to 5 packets"
+                               fi
+
+                       fi;;
                esac
        done
 
        debug_message "Function: validate - End"
        return ${rc_validate}
@@ -1102,10 +1236,11 @@
        # Start PostgreSQL
        #
 
         debug_message "Function: start_pgs - Begin"
        ${SET_DEBUG}
+       SECONDS=0
 
        # construct the necessary redirection and source variables
 
        set_shell_specifics
 
@@ -1132,20 +1267,124 @@
 
        # remove the logfile, before creating it to survive potential noclobber settings
 
        ${RM} ${LOGFILE} 2>/dev/null
 
-       #
+       do_start=0
+
+       # start an ssh-agent and store the decrypted private key if SSH_PASSDIR is configured
+
+       
+       if [ -n "${SSH_PASSDIR}" ]
+       then
+       
+               ${CP} -p ${SSH_PASSDIR}/${RESOURCE}-phrase /tmp/${RESOURCE}-phrase
+
+               if [ -z "${SMF_FMRI}" ]
+               then
+                       ${CHOWN} ${USER} /tmp/${RESOURCE}-phrase
+                       ${SU} ${USER} -c " ${TASK_COMMAND} ${MYDIR}/${MYNAME} -R ${RESOURCE} -G ${RESOURCEGROUP} -P ${PARFILE} start_ssh_agent "> /dev/null 2>&1
+                       if [ ${?} -ne 0 ]
+                       then
+                               do_start=1
+                       fi
+               else
+                       ${MYDIR}/${MYNAME} -R ${RESOURCE} -G ${RESOURCEGROUP} -P ${PARFILE} start_ssh_agent > /dev/null 2>&1
+                       if [ ${?} -ne 0 ]
+                       then
+                               do_start=1
+                       fi
+               fi
+
+               ${RM} /tmp/${RESOURCE}-phrase
+
+
+               # store the SSH_AUTH_SOCK value in the SOCK variable
+
+               SOCK=
+               if [ ${do_start} -eq 0 ]
+               then
+                       SOCK=`${GREP} SSH_AUTH_SOCK /tmp/${RESOURCE}-ssh`
+                       SOCK="${ENV} ${SOCK}"
+               fi
+       fi
+
+       # If a standby host is configured, check if the standby host is available and configured/acting
+       # as a standby. If everything is ok continue with the startup. If not, do not start the PostgreSQL
+       # resource.
+
+       if [ -n "${STDBY_HOST}" ] && [ ${do_start} -eq 0 ]
+       then
+
+               if ${PING} ${STDBY_HOST} ${STDBY_PING} >/dev/null 2>&1
+               then
+                       debug_message "Function: start_pgs - the host  ${STDBY_HOST} is accessible"
+               
+                       # As the PostgreSQL user perform a check on the standby host, to determine if it
+                       # is configured as a primary, it runs as a primary, or it runs as a standby database.
+       
+                       if [ -z "${SMF_FMRI}" ]
+                       then
+       
+                               ${SU} ${USER} -c " ${TASK_COMMAND} ${SOCK} ${SSH} ${STDBY_USER}@${STDBY_HOST} ${MYDIR}/${MYNAME} -R ${STDBY_RS} -G ${STDBY_RG} -P ${STDBY_PARFILE} check_stdby "> /dev/null 2>&1
+                               if [ ${?} -ne 0 ]
+                               then
+                                       do_start=1
+                               fi
+                       else
+                               ${SOCK} ${SSH} ${STDBY_USER}@${STDBY_HOST} ${MYDIR}/${MYNAME} -R ${STDBY_RS} -G ${STDBY_RG} -P ${STDBY_PARFILE} check_stdby ${OUTPUT} >/dev/null 2>&1
+                               if [ ${?} -ne 0 ]
+                               then
+                                       do_start=1
+                               fi
+       
+                       fi
+                       if [ $do_start -ne 0 ]
+                       then 
+                               # SCMSGS
+                               # @explanation
+                               # The PostgreSQL resource on the standby host is not 
+                               # configured as a standby or acting as a primary.
+                               # @user_action
+                               # Consult the logs on the standby host and insure, that
+                               # the standby host is configured as a standby database, 
+                               # if it is running as a primary, reconfigure and restart it.
+                               scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                                       "Function: start_pgs - resource %s on host %s is not configured/running as a standby database" \
+                                       "${STDBY_RS}" "${STDBY_HOST}"
+                       else
+                               debug_message "Function: start_pgs - the resource ${STDBY_RS} on host ${STDBY_HOST} is configured/running as a standby database"
+                       fi
+       
+               else
+                       # SCMSGS
+                       # @explanation
+                       # The standby host is not up and running.
+                       # @user_action
+                       # Consult the logs on the standby host and insure, that
+                       # the standby host is answering on ping requests of the primary host.
+                       scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                               "Function: start_pgs - the standby host %s is not answering on ping" \
+                               "${STDBY_HOST}"
+                       do_start=1
+               fi
+       fi
+
+       # continue on the start process
+
+       if [ ${do_start} -eq 0 ] 
+       then
+               
        # Start PostgreSql, the postmaster searches for databases in the directory of the PGDATA variable,
        # and it will listen on the port of the PGORT varaible. The logfile will be stored in the file name
        # contained in the PGLOGFILE variable.
        # The postmaster is started via the pg_ctl utility.
        #
 
        if [ -z "${SMF_FMRI}" ]
        then
-               ${SU} - ${USER} -c " ${ENVSC} ${TASK_COMMAND} ${LIBPATH} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/pg_ctl start -l ${PGLOGFILE} ${OUTPUT}" > /dev/null
+                       ${SU} ${USER} -c " ${ENVSC} ${TASK_COMMAND} ${SOCK} ${LIBPATH} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/pg_ctl start -l ${PGLOGFILE} ${OUTPUT}" > /dev/null
                rc_start_command=$?
        else
 
                # start under the right user in an smf manifest you source a ksh script, regardless of the users login shell
 
@@ -1152,14 +1391,54 @@
                if [ -n "${ENVSCRIPT}" ]
                then
                        . ${ENVSCRIPT}
                fi
 
-               ${LIBPATH} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/pg_ctl start -l ${PGLOGFILE} >${LOGFILE} 2>&1
+                       ${SOCK} ${LIBPATH} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/pg_ctl start -l ${PGLOGFILE} >${LOGFILE} 2>&1
                rc_start_command=$?
        fi
 
+               # if an ssh-agent was started, remember the postmaster pid in /tmp/${RESOURCE}-pid
+
+               if [ -n "${SOCK}" ] && [ ${rc_start_command} -eq 0 ]
+               then
+                       START_TIMEOUT=`${SCHA_RESOURCE_GET} -O START_TIMEOUT -R ${RESOURCE} -G ${RESOURCEGROUP}`
+                       MAX_START=`expr ${START_TIMEOUT} \* 70 \/ 100`
+                       
+                       # give PostgreSQL a chance to construct the ${PGDATA}/postmaster pid, it is 
+                       # done asynchronously
+
+                       PID_LINES=0
+                       while [ ${SECONDS} -lt ${MAX_START} ] && [ ${PID_LINES} -eq 0 ]
+                       do
+                               if [ -f ${PGDATA}/postmaster.pid ]
+                               then
+                                       PID_LINES=`${WC} -l ${PGDATA}/postmaster.pid|${AWK} '{print $1}'`
+                               fi
+                               ${SLEEP} 2
+                       done
+
+                       ${RM} /tmp/${RESOURCE}-pid 2>/dev/null
+                       ${HEAD} -1 ${PGDATA}/postmaster.pid >/tmp/${RESOURCE}-pid
+               fi
+       else
+               rc_start_command=1
+
+               # disable the pmf tag and run a sleep in the background, to assure, that there will be a valid pmftag during the start phase
+
+               START_TIMEOUT=`${SCHA_RESOURCE_GET} -O START_TIMEOUT -R ${RESOURCE} -G ${RESOURCEGROUP}`
+               sleep ${START_TIMEOUT} &
+               ${PMFADM} -s ${RESOURCEGROUP},${RESOURCE},0.svc
+
+               # As a workaround until RFE 6629606 is implemented, kill the gds_svc start method
+               # to speed up the transition into START_FAILED.
+               # If gds_svc_start gets killed the resource transition into START_FAILED and does not
+               # wait until the start_timeout expires.
+
+               ${PKILL} -u root -f "gds_svc_start .*-R ${RESOURCE} " >/dev/null
+       fi
+
        debug_message "Function: start_pgs - End"
        return ${rc_start_command}
 }
 
 stop_pgs()
@@ -1213,11 +1492,11 @@
              "stop_pgs: Stop PostgreSQL with the option %s " \
              "fast"
        
        if [ -z "${SMF_FMRI}" ]
        then
-               ${SU} - ${USER} -c " ${ENVSC} ${TASK_COMMAND} ${LIBPATH} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/pg_ctl stop -m fast ${OUTPUT}" > /dev/null
+               ${SU} ${USER} -c " ${ENVSC} ${TASK_COMMAND} ${LIBPATH} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/pg_ctl stop -m fast ${OUTPUT}" > /dev/null
                rc_stop_command=$?
        else
 
                # stop under the right user in a smf manifest
 
@@ -1230,22 +1509,27 @@
                rc_stop_command=$?
        fi
 
        # Determine if postgres is really stopped, if not perform an immediate shutdown.
        # The server is specified by the triple user, PGPORT and the PGDATA variable.
-       # The immediate shutdown stop the server processes without disconnecting the clients.
+       # The immediate shutdown stops the server processes without disconnecting the clients.
+       # The immediate shutdown will be performd only, if the postgres database is not configured as
+       # a standby database, because then an immediate shutdown will not succeed in a standby configuration.
        
+       
+       if [ ! -f ${PGDATA}/recovery.conf ]
+       then
        if ${PS} -u ${USER} -o pid,ppid |${GREP} -w ${pgrspid}>/dev/null
        then
                
                scds_syslog -p daemon.notice -t $(syslog_tag) -m \
                     "stop_pgs: Stop PostgreSQL with the option %s " \
                     "immediate"
                
                if [ -z "${SMF_FMRI}" ]
                then
-                       ${SU} - ${USER} -c "${ENVSC} ${TASK_COMMAND} ${LIBPATH} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/pg_ctl stop -m immediate ${OUTPUT_APP}" > /dev/null
+                               ${SU} ${USER} -c "${ENVSC} ${TASK_COMMAND} ${LIBPATH} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/pg_ctl stop -m immediate ${OUTPUT_APP}" > /dev/null
                        rc_stop_command=$?
                else
        
                        # stop under the right user in a smf manifest
        
@@ -1256,10 +1540,13 @@
 
                        ${LIBPATH} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/pg_ctl stop -m immediate >>${LOGFILE} 2>&1
                        rc_stop_command=$?
                fi
        fi
+       else
+               debug_message "Function: stop_pgs - It is a standby database, do not stop with immediate"
+       fi
 
        # determine if postgres is really stopped, if not kill the parent of the postmaster with -9
        
        if ${PS} -u ${USER} -o pid,ppid |${GREP} -w ${pgrspid}>/dev/null
        then
@@ -1275,10 +1562,55 @@
 
                ${KILL} -9 ${pgrspid}
                rc_stop_command=$?
        fi
 
+       if [ -n "${SSH_PASSDIR}" ]
+       then
+               # stop the ssh-agent
+
+               SSH_PID=
+               if  [ -f /tmp/${RESOURCE}-ssh ]
+               then
+                       debug_message "Function: stop_pgs - stop the ssh agent"
+
+                       # source /tmp/${RESOURCE}-ssh to get the variable SSH_AGENT_PID
+
+                       . /tmp/${RESOURCE}-ssh
+                       
+                       SSH_PID="${ENV} SSH_AGENT_PID=${SSH_AGENT_PID}"
+       
+                       if [ -z "${SMF_FMRI}" ]
+                       then
+                               ${SU} ${USER} -c "${TASK_COMMAND} ${SSH_PID} ${SSH_AGENT} -k ${OUTPUT_APP}"
+                               rc_stop_command=$?
+                       else
+                               ${SSH_PID} ${SSH_AGENT} -k >>${LOGFILE}
+                               rc_stop_command=$?
+                       fi
+
+                       # if the ssh-agent process is still there, kill it with -9
+
+                       if ${PS} -p ${SSH_AGENT_PID} >/dev/null 2>&1
+                       then
+                               debug_message "Function: stop_pgs - The ssh-agent ${SSH_AGENT_PID} survived his stop request, kill it with -9"
+                               ${KILL} -9 ${SSH_AGENT_PID}
+                       fi
+               else
+                       # SCMSGS
+                       # @explanation
+                       # It can not be determined which ssh agent is working for the user/database,
+                       # it is now killed by PMF using the configured stop signal.
+                       # @user_action
+                       # None
+                       scds_syslog -p daemon.notice -t $(syslog_tag) -m \
+                            "stop_pgs: Allow pmf to stop the remaining ssh-agent for the user %s" \
+                            "${USER}"
+               fi
+
+       fi
+
        debug_message "Function: stop_pgs - End"
        return ${rc_stop_command}
 }
 
 check_pgs()
@@ -1300,10 +1632,27 @@
        # The Project will be derived according to the call method,
        # either from the smf service or from the cluster resource / resource group
 
        srm_function ${USER}
 
+       # determine the parent postmaster pid
+
+       pgrspid=""
+       if [ -f ${PGDATA}/postmaster.pid ]
+       then
+               pgrspid=`${HEAD} -1 ${PGDATA}/postmaster.pid`
+       else
+
+               # the pid store is created only if a ssh passphrase is defined
+               # in this case get the pid from /tmp/${RESOURCE}-pid
+
+               if [ -n "${SSH_PASSDIR}" ]
+               then
+                       pgrspid=`${CAT} /tmp/${RESOURCE}-pid`
+               fi
+       fi
+
        # Determine if the gds start process is already completed, or the database start is far enough to allow connections
        # The criteria is the existence of the postmaster.pid file in the PGDATA directory.
        # As long as the GDS start is not complete, probe error messages will be suppressed.
 
        wait_for_online=0
@@ -1311,13 +1660,12 @@
        ${PGREP} -u root -f "gds_svc_start .*-R ${RESOURCE} " >/dev/null
        if [ ${?} -eq 0 ]
        then
                debug_message "Function: check_pgs - wait for online detected" 
                wait_for_online=1
-               if [ -f ${PGDATA}/postmaster.pid ]
+               if [ -n "${pgrspid}" ]
                then
-                       pgrspid=`${HEAD} -1 ${PGDATA}/postmaster.pid`
                        if ! ${PS} -u ${USER} -o pid,ppid |${GREP} -w ${pgrspid}>/dev/null
                        then
                                debug_message "Function: check_pgs - wait for online detected, postmaster is not running" 
                                rc_check_command=100
                        else
@@ -1327,12 +1675,81 @@
                        debug_message "Function: check_pgs - wait for online detected, postmaster pid file ${PGDATA}/postmaster.pid not found" 
                        rc_check_command=100
                fi
        fi
 
-       if [ ${rc_check_command} -ne 0 ]
+       # If a passphrase is defined a ssh agent is running together with the PostgreSQL deamon
+       # each of the daemons satisfies pmf/smf, so we have to check manually for the processes to run.
+       # If either the ssh-agent or the PostgreSQL daemons are not running, exit with 100.
+       # This has to be done, because pmf/smf will not trigger a restart if only one of the process trees are gone.
+
+       if [ -n "${SSH_PASSDIR}" ] && [ -n "${pgrspid}" ]
        then
+               
+               if ! ${PS} -u ${USER} -o pid,ppid |${GREP} -w ${pgrspid}>/dev/null
+               then
+                       # SCMSGS
+                       # @explanation
+                       # The PostgreSQL process is not running.
+                       # @user_action
+                       # None
+                       scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                               "check_pgs: The PostgreSQL process %s is not running" \
+                               "${pgrspid}"
+                       rc_check_command=100
+                       return ${rc_check_command}
+                               
+               fi
+               
+               # Determine if the necessary ssh-agent is running. If PostgreSQL is running 
+               # without the ssh-agent, the replication is not working, so we have to return 100
+
+               if [ -f /tmp/${RESOURCE}-ssh ]
+               then
+
+                       sshpid=`${GREP} SSH_AGENT_PID /tmp/${RESOURCE}-ssh|${AWK} -F= '{print $2}'`
+                       if ! ${PS} -u ${USER} -o pid,ppid |${GREP} -w ${sshpid}>/dev/null
+                       then
+                               # SCMSGS
+                               # @explanation
+                               # The ssh-agent process associated with the PostgreSQL database 
+                               # is not running.
+                               # @user_action
+                               # None
+                               scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                                       "check_pgs: The ssh-agent process %s is not running" \
+                                       "${sshpid}"
+                               rc_check_command=100
+                               return ${rc_check_command}
+                       fi
+
+               else
+
+                       # SCMSGS
+                       # @explanation
+                       # The file /tmp/<resourcename>-ssh containing the necessary information to manage 
+                       # the ssh-agent together with the PostgreSQL database is unavailable.
+                       # @user_action
+                       # None
+                       scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                               "check_pgs: The ssh-agent information file %s is unavailable" \
+                               "/tmp/${RESOURCE}-ssh"
+                       rc_check_command=100
+                       return ${rc_check_command}
+
+               fi
+       
+       fi
+
+       # If there is no postmaster.pid during wait for online, or the database will not
+       # accept updates, because it is configured as a standby database, terminate the probe.
+       # To trigger an early exit of the probe, it is enough, that a recover.conf exists in $PGDATA.
+       # So effectively the agent relies on process monitoring if a database is configured as a standby database.
+
+
+       if [ ${rc_check_command} -ne 0 ] || [ -f ${PGDATA}/recovery.conf ]
+       then
                debug_message "Function: check_pgs - End"
                return ${rc_check_command}
        fi
 
        # construct the necessary environment variables
@@ -1374,11 +1791,11 @@
 
        # check if the catalog of the postgres database is accessible
 
        debug_message "Function: check_pgs - check if the database catalog is accessible with ${SCDB} "
 
-       ${SU} - ${USER} -c "${OPEN_BRACKET} ${TASK_COMMAND} ${LIBPATH} $HOST ${PASSWORD} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/psql -d ${SCDB} -U ${SCUSER} -c \"${DBCAT}\" ${CAT_OUTPUT} ${CLOSE_BRACKET} ${CAT_ERRPUT} " >/dev/null
+       ${SU} ${USER} -c "${OPEN_BRACKET} ${TASK_COMMAND} ${LIBPATH} $HOST ${PASSWORD} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/psql -d ${SCDB} -U ${SCUSER} -c \"${DBCAT}\" ${CAT_OUTPUT} ${CLOSE_BRACKET} ${CAT_ERRPUT} " >/dev/null
        rc_check_command=$?
 
        # Return ${NOCONRET} if the connect fails, otherwise proceed with the check
 
        if [ ${rc_check_command} -ne 0 ]
@@ -1431,11 +1848,11 @@
        if [ ${rc_check_command} -eq 0 ]
        then
 
                debug_message "Function: check_pgs - manipulate the table ${SCTABLE}"
 
-               ${SU} - ${USER} -c "${OPEN_BRACKET} ${TASK_COMMAND} ${LIBPATH} $HOST ${PASSWORD} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/psql -d ${SCDB} -U ${SCUSER} -c \"${DBTRUNC};${DBINS};${DBSEL}\" ${TBL_OUTPUT} ${CLOSE_BRACKET} ${TBL_ERRPUT} " >/dev/null
+               ${SU} ${USER} -c "${OPEN_BRACKET} ${TASK_COMMAND} ${LIBPATH} $HOST ${PASSWORD} ${ENV} ${DATA} ${ENV} ${PORT} ${PGROOT}/bin/psql -d ${SCDB} -U ${SCUSER} -c \"${DBTRUNC};${DBINS};${DBSEL}\" ${TBL_OUTPUT} ${CLOSE_BRACKET} ${TBL_ERRPUT} " >/dev/null
                rc_check_command=$?
 
                # Return ${NOCONRET} if the connect fails, otherwise proceed with the check
 
                if [ ${rc_check_command} -ne 0 ]
@@ -1467,11 +1884,11 @@
                                        # specified table failed.
                                        # @user_action
                                        # None
                                        scds_syslog -p daemon.err -t $(syslog_tag) -m \
                                                "check_pgs: The test database manipulation failed for database %s, user %s and table %s" \
-                                               "${SCDB}" "${SCUSER}" "$SCTABLE}"
+                                               "${SCDB}" "${SCUSER}" "${SCTABLE}"
                                fi
 
                                rc_check_command=100
                                debug_message "Function: check_pgs - manipulation of the table ${SCTABLE} failed"
 
@@ -1483,6 +1900,99 @@
                fi
        fi      
 
        debug_message "Function: check_pgs - End"
        return ${rc_check_command}
+}
+
+check_stdby()
+{
+       # 
+       # Check if the current postgres database is configured or running as a standby database.
+       # This function is called from the start command of the remote host.
+       # The error messages are processed by scds_syslog, and they are transported via
+       # standard out to the remote host.
+       #
+       # This function will generate up to two error messages in the postgres logfile.
+
+        debug_message "Function: check_stdby - Begin"
+       ${SET_DEBUG}
+
+       rc_check_stdby_command=0
+       if [ -f ${PGDATA}/recovery.conf ]
+       then
+
+               # We are running as the PostgreSQL user, so exporting the environment is sufficient.
+
+               export PGPORT
+
+               # The variable PASSWORD is set to the database user login password.
+
+               if [ -n "${SCPASS}" ]
+               then
+                       export PGPASSWORD=${SCPASS}
+               fi
+
+               if [ -n "${PGHOST}" ]
+               then
+                       export PGHOST
+               fi
+               export PGPORT
+               export LD_LIBRARY_PATH
+               
+               # check the database if it accepts inserts
+
+               ${PGROOT}/bin/psql -d ${SCDB} -U ${SCUSER} -c "insert into ${SCTABLE} (sccol) values('standby test');" >/dev/null 2>&1
+               if [ ${?} -eq 0 ]
+               then
+                       # SCMSGS
+                       # @explanation
+                       # The database is running as a primary database, but is configured to be a standby.
+                       # @user_action
+                       # If you want to start the old primary database, reload the primary and restart 
+                       # the standby database.
+                       scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                               "check_stdby: The database in %s is not running as a standby database, reload the primary and restart the standby database." \
+                               "${PGDATA}" 
+                       rc_check_stdby_command=1
+
+               else
+
+                       # Check if the rolchanger is currently doing his work.
+
+                       if [ -f /tmp/${ROLECHG_RS}_rolechg.lck ]
+                       then
+
+                               # SCMSGS
+                               # @explanation
+                               # The database is running as a standby database, but is currently reconfigured to run as a primary.
+                               # @user_action
+                               # If you want to start the old primary database, reload the primary and restart 
+                               # the standby database.
+                               scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                                       "check_stdby: The database in %s is reconfigured to run as a primary database, reload the primary and restart the standby database." \
+                                       "${PGDATA}" 
+                               rc_check_stdby_command=1
+                       else
+                               debug_message "Function: check_stdby - the database in ${PGDATA} is configured or running as a standby database"
+                       fi
+
+               fi
+               
+       else
+
+               # SCMSGS
+               # @explanation
+               # The database is not configured as a standby database.
+               # @user_action
+               # Create an appropriate recovery.conf command, reload the primary 
+               # and restart the standby database.
+               scds_syslog -p daemon.err -t $(syslog_tag) -m \
+                       "check_stdby: The database in %s is not configured as a standby database, create the recovery.conf command" \
+                       "${PGDATA}" 
+               rc_check_stdby_command=1
+
+       fi
+
+       debug_message "Function: check_stdby - End"
+       return ${rc_check_stdby_command}
 }