#!/bin/bash
#
#"@(#) SGeSAP/LX - 12.20.00 - $Rev: 1856 $ - Last modified 07/06/17"
#
# **********************************************************************
# *                                                                    *
# * HPE SGeSAP/LX                                                      *
# *                                                                    *
# * SAP HANA Safesync administration script                            *
# *                                                                    *
# * (c) Copyright 2016-2017 Hewlett Packard Enterprise Co.             *
# *                                                                    *
# * SG_LOG_LEVEL 1..5                                                  *
# *                                                                    *
# **********************************************************************


########################################################################
#
# Source in the utility functions used by the module scripts
#
########################################################################

typeset script_name=`echo $0|awk '{ print substr ($0,match($0,"[^/]*$")) }'`

HPLAT=Linux
LX=1
UX=0
USAGE_STRING='Usage: sr_safesync [-n node] [-s|-b|-u|-p|-x|-h] [hana_primary_package_name|SID]'

SAPSYS=${SGeSAP_SAPSYS:-sapsys}
 
if [[ `whoami` != root ]]; then
  echo Must be super-user to run this command.
  exit 2
fi

if [[ -f /etc/cmcluster.conf ]]; then
  . /etc/cmcluster.conf
  if [[ $? -ne 0 ]]; then
    echo "ERROR: Unable to source cluster configuration: /etc/cmcluster.conf"
    exit 2
  fi
else
    echo "ERROR: Unable to find cluster configuration: /etc/cmcluster.conf"
    exit 2
fi

SG_UTILS=${SG_UTILS:-${SGCONF}/scripts/mscripts/utils.sh}
SG_LOG_LEVEL=${SG_LOG_LEVEL:-1}
op_code=validate # prevent that log level gets overwritten

########################################################################
#
# Globals
#
########################################################################

FUN0_FILE=${FUN0_FILE:-${SGCONF}/scripts/sgesap/sap_functions.sh}
CUS0_FILE=${CUS0_FILE:-${SGCONF}/scripts/sgesap/customer_functions.sh}
CUS1_FILE=${CUS1_FILE:-${SGCONF}/scripts/ext/${SG_PACKAGE_NAME}_customer_functions.sh}
CUS2_FILE=${CUS2_FILE:-${SGCONF}/scripts/${SG_PACKAGE_NAME}/customer_functions.sh}

function sg_log
{
  if [[ $1 -le $SG_LOG_LEVEL ]]; then
    shift
    echo $*
  fi
}

# deactivate disablement block for scale-out
# don't change these values
SGeSAP_disablement_block=skip
SGeSAP_eligibility_block=execute
SGeSAP_persistence_block=execute

########################################################################
#
# Read additional variables from configuration files
# and include function pools.
#
########################################################################

function get_source
{
  if [[ -r $1 ]]; then
    sg_log 4 "Found: $1"

    . $1    # display any errors

    if [[ $? -ne 0 ]]; then
      sg_log 0 "ERROR: Corrupt data in file $1"
      exit 2
    fi

  elif [[ "X$2" = "Xcritical" ]]; then
    sg_log 0 "ERROR: Missing file $1"
    exit 2
  else
    sg_log 4 "Not used: $1"
  fi
}

########################################################################
#
# "main()"
#
########################################################################

SGeSAP_SAP_SYSTEM=$SAPSYSTEMNAME
get_source $FUN0_FILE critical
get_source $CUS0_FILE critical
get_source $CUS1_FILE optional
get_source $CUS2_FILE optional

# redefine, if overwritten
CMEXEC_TIMEOUT=${SGeSAP_CMEXEC_TIMEOUT:-35}
CMEXEC_TIMEOUT_SHORT=${SGeSAP_CMEXEC_TIMEOUT_SHORT:-$(expr 15 - 8 '*' \( $($SGSBIN/cmviewcl -fline -lnode -sconfig|grep name=|wc -l) / 33 \) )}
NFS_TIMEOUT=25

function sg_log
{
  if [[ $1 -le $SG_LOG_LEVEL ]]; then
    shift
    echo $*
  fi
}

#
# parse command line options
# and trigger remote execution
#

if [[ "$1" = "-n" ]]; then
  two=$2
  shift 2
  if [[ -n "$two" ]]; then
    # do not specify a cmexec timeout
    $SGSBIN/cmexec $two SG_LOG_LEVEL=${SG_LOG_LEVEL}';'$SGSBIN/sr_safesync $*
    exit $?
  else
    sg_log 0 "Option requires an argument."
    echo >&2 "$USAGE_STRING"
    exit 2 
  fi
fi

case $1 in
  -s|-b|-u|-dbg_abort|-p|-x)
    OP=$1
    shift
  ;;
  -h)
    sg_log 0 "$USAGE_STRING"
    sg_log 0 "-s: (default op) print status information on safesync blocks of a HANA replication pair"
    sg_log 0 "-b: activate the safesync takeover blocks on a single node"
    sg_log 0 "-u: deactivate the safesync takeover blocks on a single node"
    sg_log 0 "-x: activate the safesync double primary block on a single node"
    sg_log 0 "-p: deactivate the safesync double primary block on a single node"
    exit 0
  ;;
  *)
    if [[ $# -le 1 ]]; then
      OP=-s
    else
      echo >&2 "Unknown operation."
      OP=-h
      shift
    fi
  ;;
esac

#
# identify nodes, sites, package, SID and instance
#
one=${1:-${SGeSAP_SAP_SYSTEM}}

cmviewcl_config=`$SGSBIN/cmviewcl -f line -l package -s config -v`

if [[ "$one" = "-dbg_abort" ]]; then
  PKGNAME=$1
else
  typeset -i foundbefore=0
  typeset -i foundnow=0
  typeset -i found=0
  typeset -i foundsofit=0
  typeset type=unknown
  typeset TYPE=unknown
  typeset pkgnodelist=" "
  typeset HDBINSTANCE
  typeset PKGNODELIST
  typeset SITENAME
  typeset -i block
  typeset -i pblock
  typeset -i unknowns
  typeset -i no_resource=0

  for line in `echo -e "${cmviewcl_config}\npackage:final|name=final"`; do

    if [[ $found -gt 2 ]]; then
      if [[ $foundnow -eq 0 ]]; then
        sg_log 4 $pkg is a candidate
        (( foundbefore = $foundbefore + 1 ))
        foundnow=1
        PKGNAME=$pkg
      fi
      SGeSAP_SAP_SYSTEM=${name:-$SGeSAP_SAP_SYSTEM}
      HDBINSTANCE=$hdbinstance
      PKGNODELIST="$pkgnodelist"
    fi  

    sg_log 5 "Parsing [ $line ] with status $found"

    echo $line|grep '^package:[[:alnum:]_-]*|name=' >/dev/null 
    if [[ $? -eq 0 ]]; then
      if [[ $foundnow -eq 1 ]]; then
        sg_log 4 Package achieves score $found
        TYPE=$type
        if [[ $type != multi_node ]]; then
          if [[ $found -eq 4 ]]; then
            break
          elif [[ -n $one ]]; then
            (( foundbefore = $foundbefore - 1 ))
            sg_log 4 $pkg was rejected
          fi
        elif [[ $type = multi_node ]]; then
          if [[ $found -eq 4 ]]; then
            if [[ -z ${PKGNAME[1]} ]]; then
              PKGNAME[1]=$pkg
              SGeSAP_SAP_SYSTEM[1]=$SGeSAP_SAP_SYSTEM
              PKGNODELIST[1]="$pkgnodelist"
              sg_log 4 Detected a fitting scale-out HANA package
              foundsofit=1
              foundbefore=1
            elif [[ $foundsofit -eq 1 ]]; then
              if [[ ${SGeSAP_SAP_SYSTEM[1]} = $SGeSAP_SAP_SYSTEM ]]; then
                foundbefore=2
               sg_log 4 Detected a fitting scale-out HANA package SADTA config
                break
              else
                sg_log 0 ERROR: Cluster appears to be misconfigured
                exit 2
              fi
            elif [[ ${SGeSAP_SAP_SYSTEM[1]} = $SGeSAP_SAP_SYSTEM ]]; then
              foundbefore=2
              sg_log 4 Detected a fitting scale-out HANA package SADTA config
              break
            else
              sg_log 4 Discarded a previous scale-out HANA package
              SGeSAP_SAP_SYSTEM[1]=$SGeSAP_SAP_SYSTEM
              PKGNODELIST[1]="$pkgnodelist"
              PKGNAME[1]=$pkg
            fi
          elif [[ $found -eq 3 ]]; then
            if [[ -z "$one" ]]; then 
              if [[ -z "${PKGNAME[1]}" ]]; then
                PKGNAME[1]=$pkg
                SGeSAP_SAP_SYSTEM[1]=$SGeSAP_SAP_SYSTEM
                PKGNODELIST[1]="$pkgnodelist"
                sg_log 4 Detected a scale-out HANA package
                if [[ $foundbefore -gt 1 ]]; then
                  if [[ -z $one ]]; then
                    sg_log 0 Please specify a SID or primary package name with the command. 
                    exit 2
                  fi
                  sg_log 4 Rejected an ambiguous scale-up HANA package 
                  foundbefore=1
                fi
              elif [[ $foundsofit -eq 1 ]]; then
                if [[ ${SGeSAP_SYSTEM[1]} = $SGeSAP_SYSTEM ]]; then
                  foundbefore=2
                  sg_log 4 Detected a fitting scale-out HANA package SADTA config
                  break
                else
                  sg_log 4 Discarded a worse fit scale-out HANA package
                  (( foundbefore = $foundbefore - 1 ))
                fi
              else
                if [[ ${SGeSAP_SAP_SYSTEM[1]} = $SGeSAP_SAP_SYSTEM ]]; then
                  sg_log 4 Detected a scale-out HANA package SADTA config
                  foundbefore=2
                else
                  sg_log 0 Rejected an ambiguous scale-out HANA package
                  (( foundbefore = $foundbefore - 1 ))
                fi
              fi
            elif [[ -z "${PKGNAME[1]}" ]]; then
              PKGNAME[1]=$pkg
              SGeSAP_SAP_SYSTEM[1]=$SGeSAP_SAP_SYSTEM
              PKGNODELIST[1]="$pkgnodelist"
              sg_log 4 Detected a scale-out HANA package
            elif [[ $foundsofit -eq 0 && ${SGeSAP_SAP_SYSTEM[1]} != $SGeSAP_SAP_SYSTEM ]]; then
                sg_log 0 Please specify a SID or primary package name with the command. 
                exit 2
            else
              sg_log 4 Detected a scale-out HANA package SADTA configuration
            fi 
          fi
        fi    
      fi
      pkg=`echo $line|cut -f 2 -d =`
      sg_log 4 "Parsing configuration of package $pkg (matches so far: $foundbefore)"
      found=0
      foundnow=0
      type=unknown
      pkgnodelist=" "
    fi

    echo $line|grep '^package:'$pkg'|type=multi_node' >/dev/null
    if [[ $? -eq 0 ]]; then
      type=multi_node
      continue
    fi

    echo $line|grep '^package:'$pkg'|sgesap/hdb_global/hdb_system:[[:alnum:]_-]\{3\}|sgesap/hdbinstance_global/hdb_instance=[[:alnum:]]*$' >/dev/null 
    if [[ $? -eq 0 ]]; then
      hdbinstance=`echo $line|cut -f 2 -d =`
      continue
    fi

    echo $line|grep '^package:'$pkg'|node:[[:alnum:]_-]*|name=[[:alnum:]_-]*$' >/dev/null
    if [[ $? -eq 0 ]]; then
      pkgnodelist="$pkgnodelist`echo $line|cut -f 2 -d =` "
      continue
    fi
    
    echo $line|grep '^package:'$pkg'|module_name:sgesap/hdbprimary|module_name=sgesap/hdbprimary$' >/dev/null && 
      if [[ "$one" = "$pkg" ]]; then
        (( found = $found + 3 )) 
        sg_log 4 Found a fitting primary HANA package name
        continue
      else
        sg_log 4 Found a primary HANA package
        (( found = $found + 2 ))
        continue
      fi

    name=`echo ${line}|grep '^package:'$pkg'|sgesap/hdb_global/hdb_system:[[:alnum:]_-]*|sgesap/hdb_global/hdb_system='|cut -f 2 -d =`
    if [[ ${#name} -eq 3 ]]; then
      (( found = $found + 1 ))
      if [[  "$one" = "$name" ]]; then
        sg_log 4 $pkg fits to the specified HANA system
        (( found = $found + 1 ))
      else
        sg_log 4 $pkg belongs to HANA system $name
      fi
    fi

  done

  if [[ ( $TYPE != multi_node && $foundbefore -ne 1 ) || ( $TYPE = multi_node && $foundbefore -ne 2 ) ]]; then
    if [[ $# -eq 0 ]]; then
      sg_log 0 Please specify a SID or primary package name with the command.
    else
      sg_log 0 No fitting package configuration found for [$one].
    fi
    # echo >&2 "$USAGE_STRING"
    exit 2
  fi
  if [[ -n "$one" ]]; then
    if [[ $one != $SGeSAP_SAP_SYSTEM ]]; then
      if [[ $one != $PKGNAME && $one != ${PKGNAME[1]} ]]; then
        sg_log 0 No fitting package configuration for [$one].
        exit 2
      fi
      if [[ $OP != '-s' && $TYPE = multi_node ]]; then
        for i in 0 1; do
          echo "${PKGNODELIST[$i]}"|grep " $HNAME " >/dev/null
          if [[ $? -eq 0 ]]; then
            if [[ ${PKGNAME[$i]} != $one ]]; then
              sg_log 0 No fitting package configuration found in the local site.
              exit 2
            fi
          fi
        done
      fi
    fi
  fi 
fi

sg_log 4 "SAPSYSTEMNAME=$SGeSAP_SAP_SYSTEM"
sg_log 4 "OP=$OP"

if [[ "$TYPE" = "multi_node" ]]; then
  sg_log 4 "PACKAGENAMEs=${PKGNAME[0]}, ${PKGNAME[1]}"
  sg_log 4 "PKGNODELISTS=${PKGNODELIST[0]}, ${PKGNODELIST[1]}"
  # the following path could be retrieved reliably from the DIR_INSTANCE variable of
  # the HANA administrator environment; however, retrieving the value from there
  # bears risks to block this command's execution, so it is avoided.
  TMPDIR=${SGeSAP_TMPDIR:-/usr/sap/$SGeSAP_SAP_SYSTEM/$HDBINSTANCE/work/.sgesap}
  mkdir -p $TMPDIR/api 2>/dev/null &
  watchdog $! $NFS_TIMEOUT 1
  if [[ $? -le 1 ]]; then
    prefix=""
    prefix_visible=""
  else
    sg_log 0 "HANA NFS share tmpdir cannot be accessed ($TMPDIR)."
    exit 2
  fi
else
  sg_log 4 "PACKAGENAME=$PKGNAME"
  prefix=$PKGNAME.sgesap
  prefix_visible=$prefix'.'
fi

MYSIDADM=`ls -l /usr/sap/$SGeSAP_SAP_SYSTEM/$HDBINSTANCE|cut -f 3 -d " "`
MYGROUP=`ls -l /usr/sap/$SGeSAP_SAP_SYSTEM/$HDBINSTANCE|cut -f 4 -d " "`
sg_log 4 "MYSIDADM=$MYSIDADM"
sg_log 4 "MYGROUP=$MYGROUP"
if [[ -z $MYSIDADM ]]; then
  sg_log 0 "Cannot determine sidadm user. Check instance directory access."
  exit 2
fi

if [[ "$TYPE" = "multi_node" ]]; then
  chown $MYSIDADM:$MYGROUP $TMPDIR
  chown -f $MYSIDADM:$MYGROUP $TMPDIR/*
  chmod 755 $TMPDIR
  chmod 755 $TMPDIR/api
  chown -f $MYSIDADM:$MYGROUP $TMPDIR/api/*
fi
  
HDBMON_STICKY_FILE_PRE=$TMPDIR/api/
HDBMON_STICKY_FILE_POST=$prefix.sticky.tmp
HDBMON_ASYNC_FILE=$TMPDIR/api/$prefix.asynclck.tmp
HDBMON_TAKEOVER_FILE=$TMPDIR/api/$prefix.takeover.tmp
HDBMON_PRODUCTION_START_FILE=$TMPDIR/api/$prefix.prodstart.tmp
HDBMON_SAFESYNC_GENRES=safesync${SGeSAP_SAP_SYSTEM}_$GENRES
HDBMON_HISTORY_FILE=$TMPDIR/${prefix_visible}safesync.history.log
sg_log 4 "HDBMON_ASYNC_FILE=$HDBMON_ASYNC_FILE"
sg_log 4 "HDBMON_SAFESYNC_GENRES=$HDBMON_SAFESYNC_GENRES"

if [ `read_ini_config "$USRSAPDIR/$SGeSAP_SAP_SYSTEM/SYS/global/hdb/custom/config/global.ini"  ha_dr_provider_SGeSAPcl provider`X != "SGeSAPclX" ]; then
  sg_log 0 Safesync configuration error: SGeSAPcl is not defined as HANA ha_dr_provider in global.ini file.
  exit 2
fi

#
# perform main operation
#

case $OP in
  -s)
    #
    # get site information
    #
    cmviewcl_out=`$SGSBIN/cmviewcl -f line`
    nnodes=`echo "$cmviewcl_out"|grep '^node:[[:alnum:]_-]*|name=[[:alnum:]_-]*$'|wc -l`
    if [[ $TYPE = multi_node ]]; then
      for i in 0 1; do
        representative=`echo ${PKGNODELIST[$i]}|cut -f 1 -d " "`
        siteid=`echo "$cmviewcl_out"|grep '^node:'$representative'|siteid=[[:digit:]]*$'|cut -f 2 -d =`
        SITENAME[$i]=`echo "$cmviewcl_out"|grep '^site:[[:alnum:]_-]*|id='$siteid|cut -f 2 -d :|cut -f 1 -d "|"`
      done
      if [[ -z ${SITENAME[0]} || -z ${SITENAME[1]} ]]; then
        sg_log 0 Site not found. 
        sg_log 4 Check cluster configuration for correct site settings.
        exit 2
      fi
      echo -e '\n'PRIMARY_PACKAGES
      echo "${PKGNAME[1]}, ${PKGNAME[0]}"
    else
      echo -e '\n'PACKAGE
      echo ${PKGNAME}
    fi
    echo -e '\n  BLOCK               STATUS'

    #
    # check persistence block
    #
    echo -n '  persistence block   '
    typeset persistence_timeout=$CMEXEC_TIMEOUT
    typeset timeout_flag=0
    cmd='[ -e '$HDBMON_ASYNC_FILE' ]'
    if [[ $TYPE = multi_node ]]; then
      for i in 0 1; do
        safesync_site[$i]=0
        skippy=0
        for node in `echo "$cmviewcl_out"|grep -e '^node:[[:alnum:]_-]*|state=halted' -e '^node:[[:alnum:]_-]*|state=running$'|cut -f 2 -d :|cut -f 1 -d "|"`; do
          echo ${PKGNODELIST[$i]}|grep $node >/dev/null
          if [[ $? -eq 0 ]]; then
            if [[ ${skippy} -eq 0 ]]; then
              out=`$SGSBIN/cmexec $node -t $persistence_timeout "$cmd" 2>&1`
              if [[ $? -eq 0 ]]; then
                CHECK_NODELIST[$i]="${CHECK_NODELIST[$i]} $node "
                safesync_site[$i]=1
              else
                echo $out|grep "Unable to execute " >/dev/null
                if [[ $? -eq 0 ]]; then
                  (( timeout_flag++ ))
                  # speed up further checks
                  [ $timeout_flag -eq 3 ] && persistence_timeout=$CMEXEC_TIMEOUT_SHORT
                else
                  CHECK_NODELIST[$i]="${CHECK_NODELIST[$i]} $node "
                  skippy=1
                fi
              fi
            else
              CHECK_NODELIST[$i]="${CHECK_NODELIST[$i]} $node "
            fi
          fi
        done
      done
      if [[ -z "${CHECK_NODELIST[0]}${CHECK_NODELIST[1]}" ]]; then
        eval $cmd
        if [[ $? -eq 0 ]]; then
          if [[ $TYPE -eq multi_node ]]; then
            echo "${PKGNODELIST[0]}"|grep " $HNAME " >/dev/null
            if [[ $? -eq 0 ]]; then
              echo active in ${SITENAME[0]}
            else
              echo active in ${SITENAME[1]}
            fi
          else
            echo active on $HNAME
          fi
          block=1
        else
          echo inactive
          block=0
        fi
      elif [[ ${safesync_site[0]} -eq 1 ]]; then
        if [[ ${safesync_site[1]} -eq 1 ]]; then
          echo active
          block=2
        else
          echo active in ${SITENAME[0]}
          block=1
        fi
      else
        if [[ ${safesync_site[1]} -eq 1 ]]; then
          echo active in ${SITENAME[1]}
          block=1
        else
          if [[ $timeout_flag -gt 0 ]]; then
            echo unknown
            block=2
          else
            echo inactive 
            block=0
          fi
        fi
      fi
    else # scale-up
      for node in `echo "$cmviewcl_out"|grep -e '^node:[[:alnum:]_-]*|state=halted' -e '^node:[[:alnum:]_-]*|state=running$'|cut -f 2 -d :|cut -f 1 -d "|"`; do
        out=`$SGSBIN/cmexec $node -t $CMEXEC_TIMEOUT "$cmd" 2>&1`
        if [[ $? -eq 0 ]]; then
          CHECK_NODELIST="$CHECK_NODELIST $node "
          safesync_nodelist="$safesync_nodelist $node "
        else
          echo $out|grep "Unable to execute " >/dev/null
          if [[ $? -eq 0 ]]; then
            timeout_flag=1
          else
            CHECK_NODELIST="$CHECK_NODELIST $node "
          fi
        fi
      done
      if [[ -z "$safesync_nodelist" ]]; then
        if [[ $timeout_flag -eq 1 ]]; then
          echo unknown
          block=2
        else
          echo inactive 
          block=0
        fi
      elif [[ $nnodes -eq `echo $safesync_nodelist|wc -w` ]]; then
        echo active
        block=2
      else
        echo active on $safesync_nodelist
        block=1
      fi
    fi
    pblock=$block

    #
    # check takeover token
    #
    token_nodelist=""
    cmd='[ -e '$HDBMON_TAKEOVER_FILE' ] && exit 2'
    lastprod=0
    lastprod_loc=""
    if [[ $TYPE = multi_node ]]; then
      for i in 0 1; do
        for node in ${CHECK_NODELIST[$i]}; do
          echo ${PKGNODELIST[$i]}|grep $node >/dev/null
          if [[ $? -eq 0 ]]; then
            out=`$SGSBIN/cmexec $node -t $CMEXEC_TIMEOUT_SHORT "cat ${HDBMON_PRODUCTION_START_FILE} 2>dev/null"`
            if [[ $out =~ [0-9]+ ]]; then
              if [[ $out -gt $lastprod ]]; then
                lastprod=$out
                lastprod_loc=${SITENAME[$i]}
              fi
            fi
            $SGSBIN/cmexec $node -t $CMEXEC_TIMEOUT_SHORT "$cmd" 2>/dev/null
            if [[ $? -eq 2 ]]; then
              token_nodelist="$token_nodelist ${SITENAME[$i]} "
              break
            fi
          fi
        done
      done
    else # scale-up
      for node in $CHECK_NODELIST; do
        $SGSBIN/cmexec $node -t $CMEXEC_TIMEOUT "$cmd" 2>/dev/null
        if [[ $? -eq 2 ]]; then
          token_nodelist="$token_nodelist $node "
        fi
        out=`$SGSBIN/cmexec $node -t $CMEXEC_TIMEOUT "cat ${HDBMON_PRODUCTION_START_FILE} 2>dev/null"`
        if [[ $out =~ [0-9]+ ]]; then
          if [[ $out -gt $lastprod ]]; then
            lastprod=$out
            lastprod_loc=$node
          fi
        fi
      done
    fi

    #
    # check disablement block
    #
    if [[ $TYPE != multi_node || ${SGeSAP_disablement_block:-execute} != skip ]]; then
      echo -n '  disablement block   '
      typeset _node=""
      typeset grepstring
      if [[ $TYPE = multi_node ]]; then
        grepstring='node:[[:alnum:]_-]*|switching'
        grepstring2='|status=d[oe][wt][na]c\?h\?e\?d\?$'
      else
        grepstring='autorun'
        grepstring2='|status=[ud][pe]t\?a\?c\?h\?e\?d\?$'
        cmd="if [ ! -e $HDBMON_STICKY_FILE_PRE$HDBMON_STICKY_FILE_POST -a -e $HDBMON_ASYNC_FILE ]; then exit 2; fi" 
      fi
      for i in 0 1; do
        incomplete[$i]=0
        if [[ -z "${PKGNAME[$i]}" ]]; then 
          disablement_block[1]=${disablement_block[0]}
          break # execute this only once in case of a scale-up cluster
        fi
        disablement_block[$i]=inactive
        echo "$cmviewcl_out"|grep '^package:'${PKGNAME[$i]}'|'$grepstring'=disabled$' >/dev/null
        if [[ $? -eq 0 ]]; then
          # autorun/switching must be disabled somewhere for the disablement block to be active
          echo "$cmviewcl_out"|grep '^package:'${PKGNAME[$i]}$grepstring2 >/dev/null
          if [[ $? -eq 0 ]]; then 
            for node in ${CHECK_NODELIST[$i]}; do
              echo ${PKGNODELIST[$i]}|grep $node >/dev/null
              if [[ $? -eq 0 ]]; then
                [ $TYPE = multi_node ] && cmd="if [ ! -e $HDBMON_STICKY_FILE_PRE.$node$HDBMON_STICKY_FILE_POST ]; then exit 2; fi" 
                $SGSBIN/cmexec $node -t $CMEXEC_TIMEOUT $cmd >/dev/null 2>&1
                if [[ $? -eq 2 ]]; then
                  _reenable_nodelist[$i]="${_reenable_nodelist[$i]} $node "
                else
                  incomplete[$i]=1
                fi
              fi  
            done
            if [[ -n "${_reenable_nodelist[$i]}" && ${incomplete[$i]} -eq 0 ]]; then
              disablement_block[$i]=active
            fi
          fi
        fi
      done
      if [[ ( ${incomplete[0]} -eq 1 && -n "${_reenable_nodelist[0]}" ) || ( ${incomplete[1]} -eq 1  && -n "${_reenable_nodelist[1]}" ) ]]; then
        echo active on ${_reenable_nodelist[0]}${_reenable_nodelist[1]}
        (( block = $block + 1 ))
      elif [[ ${disablement_block[0]} = ${disablement_block[1]} ]]; then
        echo ${disablement_block[0]}
        [ ${disablement_block[0]} = active ] && (( block = $block + 2 ))
      elif [[ ${disablement_block[0]} = active ]]; then
        echo active in ${SITENAME[0]}
        (( block = $block + 1 ))
      else
        echo active in ${SITENAME[1]}
        (( block = $block + 1 ))
      fi
    fi

    #
    # check eligibility block
    #
    unknowns=0
    echo -n '  eligibility block   '
    cmd=$SGSBIN'/cmgetresource -r '$HDBMON_SAFESYNC_GENRES
    eval $cmd 2>&1|grep "does not exist" >/dev/null 
    if [[ $? -eq 0 ]]; then
      echo not configured
      no_resource=1
    elif [[ -z "${CHECK_NODELIST[0]}${CHECK_NODELiST[1]}" ]]; then
      echo unknown
    else
      for i in 0 1; do
        incomplete[$i]=0
        if [[ -z "${PKGNAME[$i]}" ]]; then 
          eligibility_block[1]=${eligibility_block[0]}
          break
        fi
        eligibility_block[$i]=inactive
        for node in  `echo "$cmviewcl_out"|grep '^node:[[:alnum:]_-]*|status=up$'|cut -f 2 -d :|cut -f 1 -d "|"`; do
          echo ${PKGNODELIST[$i]}|grep $node >/dev/null
          if [[ $? -eq 0 ]]; then
            cmexec_out=`$SGSBIN/cmexec $node -t $CMEXEC_TIMEOUT_SHORT $cmd'|tail -1|grep -e " up" -e " unknown" && exit 2' 2>&1`
            if [[ $? -ne 2 ]]; then
               _eligibility_nodelist[i]="${_eligibility_nodelist[i]} $node "
            else
              if [[ $TYPE = multi_node ]]; then 
                echo $cmexec_out| grep -e " unknown" >/dev/null
                if [[ $? -eq 0 ]]; then
                  (( unknowns = $unknowns + 1 ))
                  _eligibility_nodelist[i]="${_eligibility_nodelist[i]} $node "
                  continue
                fi
              fi
              incomplete[$i]=1
            fi
          fi
        done
        if [[ -n "${_eligibility_nodelist[$i]}" && ${incomplete[$i]} -eq 0 ]]; then
          eligibility_block[$i]=active
        fi
      done
      if [[ ( ${incomplete[0]} -eq 1 && -n "${_eligibility_nodelist[0]}" ) || ( ${incomplete[1]} -eq 1  && -n "${_eligibility_nodelist[1]}" ) ]]; then
        echo active on ${_eligibility_nodelist[0]}${_eligibility_nodelist[1]}
        (( block = $block + 1 ))
      elif [[ ${eligibility_block[0]} = ${eligibility_block[1]} ]]; then
        echo ${eligibility_block[0]}
        [ ${eligibility_block[0]} = active ] && (( block = $block + 2 ))
      elif [[ ${eligibility_block[0]} = active ]]; then
        echo active in ${SITENAME[0]}
        (( block = $block + 1 ))
      else
        echo active in ${SITENAME[1]}
        (( block = $block + 1 ))
      fi
    fi

    #
    # print a summary 
    #
    echo
    echo
    sg_log 4 CHECK_NODELIST[0]=$CHECK_NODELIST
    if [[ $TYPE = multi_node ]]; then
      sg_log 4 CHECK_NODELIST[1]=${CHECK_NODELIST[1]}
      sg_log 4 SITENAMES=${SITENAME[1]}, ${SITENAME[0]}
      preposition=in
      local="local site"
      echo "${PKGNODELIST[0]}"|grep " $HNAME " >/dev/null
      if [[ $? -eq 0 ]]; then
        local=${SITENAME[0]}
      else
        echo "${PKGNODELIST[1]}"|grep " $HNAME " >/dev/null
        if [[ $? -eq 0 ]]; then
          local=${SITENAME[1]}
        fi
      fi
    else
      preposition=on
      local=$HNAME
    fi
    sg_log 4 ""

    if [[ ! -z "$lastprod_loc" ]]; then
      echo Current cluster nodes last ran a primary system $preposition $lastprod_loc.
    fi
    if [[ -z "$token_nodelist" ]]; then
      if [[ -e "$HDBMON_TAKEOVER_FILE" ]]; then  # this condition is not redundant!
        echo Production token $preposition $local.
        exit 1
      fi
    else
      echo Production token $preposition ${token_nodelist:0:${#token_nodelist}-1}.
    fi
    if [[ $no_resource -eq 1 ]]; then
      if [[ X$SGeSAP_eligibility_block = Xexecute ]]; then
        echo WARNING: The primary package generic resource configuration appears to be incomplete.
      fi
    fi
    if [[ $block -eq 0 ]]; then
      echo $HNAME expects the replication pair $SGeSAP_SAP_SYSTEM to be in sync. 
      exit 0
    elif [[ $pblock -eq 0 && $unknowns -ne 0 ]]; then
      echo Safesync is not fully initialized - make sure that sitecontroller package is running.
    else
      echo The replication pair $SGeSAP_SAP_SYSTEM may NOT be in sync.
    fi
    exit 1
  ;;
  -u)
    typeset echotext
    if [[ $TYPE = multi_node ]]; then
      cmviewcl_out=`$SGSBIN/cmviewcl -f line`
      for i in 0 1; do
        representative=`echo ${PKGNODELIST[$i]}|cut -f 1 -d " "`
        siteid=`echo "$cmviewcl_out"|grep '^node:'$representative'|siteid=[[:digit:]]*$'|cut -f 2 -d =`
        SITENAME[$i]=`echo "$cmviewcl_out"|grep '^site:[[:alnum:]_-]*|id='$siteid|cut -f 2 -d :|cut -f 1 -d "|"`
      done
      if [[ -z ${SITENAME[0]} || -z ${SITENAME[1]} ]]; then
        sg_log 0 Site not found.
        sg_log 4 Check cluster configuration for correct site settings
        exit 2
      fi
      stat1=halting
      stat2=down
    else
      stat1=starting
      stat2=up
    fi
    for i in 0 1; do
      echo "${PKGNODELIST[$i]}"|grep " $HNAME " >/dev/null
      if [[ $? -eq 0 ]]; then
        echotext="Unblocking ${SITENAME[$i]:-$HNAME} for package ${PKGNAME[$i]}..."
        echo $echotext
        echo "$(date '+%b %d %H:%M:%S')  $(hostname): sr_safesync exec: "$echotext >>$HDBMON_HISTORY_FILE
        chmod 644 $HDBMON_HISTORY_FILE
        chown $MYSIDADM:$MYGROUP $HDBMON_HISTORY_FILE
        if [[ -e $HDBMON_ASYNC_FILE ]]; then
          rm $HDBMON_ASYNC_FILE
          if [[ $TYPE != multi_node || ${SGeSAP_disablement_block:-execute} != skip ]]; then
            $SGSBIN/cmviewcl -f line -p ${PKGNAME[$i]}|grep -e ^status=$stat1$ -e ^status=$stat2$ >/dev/null
            if [[ $? -eq 0 ]]; then
              for node in `echo ${CHECK_NODELIST[$i]}`; do
                [ $TYPE = multi_node ] && _node=$node
                if [[ -e $HDBMON_STICKY_FILE_PRE${_node:+.$_node}$HDBMON_STICKY_FILE_POST ]]; then
                  rm $HDBMON_STICKY_FILE_PRE${_node:+.$_node}$HDBMON_STICKY_FILE_POST 
                else
                  nn="$nn $node"
                fi
              done
              if [[ $TYPE = multi_node ]]; then
                if [[ -n "$nn" ]]; then
                  $SGSBIN/cmmodpkg $nn -e ${PKGNAME[$i]} >/dev/null
                fi
              else
                $SGSBIN/cmmodpkg -e $PKGNAME >/dev/null
              fi
            fi
          fi
        fi
        if [[ $TYPE = multi_node ]]; then
          for j in `echo ${PKGNODELIST[$i]}`; do
            cmexec -n $j -t $CMEXEC_TIMEOUT $SGSBIN/cmsetresource -r $HDBMON_SAFESYNC_GENRES -s up 2>/dev/null >/dev/null 
          done
        else
          $SGSBIN/cmsetresource -r $HDBMON_SAFESYNC_GENRES -s up 2>/dev/null >/dev/null
        fi
        $SGSBIN/sr_safesync -s ${PKGNAME[$i]}
        exit $?
      fi
    done
    sg_log 0 Not found.
    exit 2
  ;;
  -p)
    typeset echotext
    if [[ $TYPE = multi_node ]]; then
      cmviewcl_out=`$SGSBIN/cmviewcl -f line`
      for i in 0 1; do
        representative=`echo ${PKGNODELIST[$i]}|cut -f 1 -d " "`
        siteid=`echo "$cmviewcl_out"|grep '^node:'$representative'|siteid=[[:digit:]]*$'|cut -f 2 -d =`
        SITENAME[$i]=`echo "$cmviewcl_out"|grep '^site:[[:alnum:]_-]*|id='$siteid|cut -f 2 -d :|cut -f 1 -d "|"`
      done
      if [[ -z ${SITENAME[0]} || -z ${SITENAME[1]} ]]; then
        sg_log 0 "Check cluster configuration for correct site settings"
        exit 2
      fi
    fi
    for i in 0 1; do
      echo "${PKGNODELIST[$i]}"|grep " $HNAME " >/dev/null
      if [[ $? -eq 0 ]]; then
        if [[ $TYPE = multi_node ]]; then
          echotext="Assign double primary production token of [${PKGNAME[0]}, ${PKGNAME[1]}] to ${SITENAME[$i]}..."
        else
          echotext="Assign double primary production token of $PKGNAME to $HNAME..."
        fi
        echo $echotext
        echo "$(date '+%b %d %H:%M:%S')  $(hostname): sr_safesync exec: "$echotext >>$HDBMON_HISTORY_FILE
        chmod 644 $HDBMON_HISTORY_FILE
        chown $MYSIDADM:$MYGROUP $HDBMON_HISTORY_FILE
      fi
    done
    date +%s > ${HDBMON_PRODUCTION_START_FILE}
    touch $HDBMON_TAKEOVER_FILE # a token (re-)creation...
    chmod 644 $HDBMON_TAKEOVER_FILE
    chown $MYSIDADM:$MYGROUP $HDBMON_TAKEOVER_FILE
  ;;
  -b)
    typeset echotext
    if [[ $TYPE = multi_node ]]; then
      cmviewcl_out=`$SGSBIN/cmviewcl -f line`
      grepstring='node:[[:alnum:]_-]*|switching'
      for i in 0 1; do
        representative=`echo ${PKGNODELIST[$i]}|cut -f 1 -d " "`
        siteid=`echo "$cmviewcl_out"|grep '^node:'$representative'|siteid=[[:digit:]]*$'|cut -f 2 -d =`
        SITENAME[$i]=`echo "$cmviewcl_out"|grep '^site:[[:alnum:]_-]*|id='$siteid|cut -f 2 -d :|cut -f 1 -d "|"`
      done
      if [[ -z ${SITENAME[0]} || -z ${SITENAME[1]} ]]; then
        sg_log 0 Check cluster configuration for correct site settings
        exit 2
      fi
    else
      grepstring='autorun'
    fi
    for i in 0 1; do
      echo "${PKGNODELIST[$i]}"|grep " $HNAME " >/dev/null
      if [[ $? -eq 0 ]]; then
        echotext="Blocking ${SITENAME[$i]:-$HNAME} for package ${PKGNAME[$i]}..."
        echo $echotext
        echo "$(date '+%b %d %H:%M:%S')  $(hostname): sr_safesync exec: "$echotext >>$HDBMON_HISTORY_FILE
        chmod 644 $HDBMON_HISTORY_FILE
        chown $MYSIDADM:$MYGROUP $HDBMON_HISTORY_FILE
        rm $HDBMON_TAKEOVER_FILE 2>/dev/null
        touch $HDBMON_ASYNC_FILE 
        chmod 644 $HDBMON_ASYNC_FILE
        chown $MYSIDADM:$MYGROUP $HDBMON_ASYNC_FILE
        if [[ $TYPE != multi_node || ${SGeSAP_disablement_block:-execute} != skip ]]; then
          cmviewcl_out=`$SGSBIN/cmviewcl -f line -p ${PKGNAME[$i]}`
          # echo "$cmviewcl_out"|grep -e '^status=up$' -e '^status=starting$' >/dev/null
          # if [[ $? -eq 0 ]]; then
          for node in `echo "$cmviewcl_out"|grep ^$grepstring=disabled$|cut -f 2 -d :|cut -f 1 -d '|'`; do
            [ $TYPE = multi_node ] && _node=$node
            if [[ ! -e $HDBMON_STICKY_FILE_PRE${_node:+.$_node}$HDBMON_STICKY_FILE_POST ]]; then
              touch $HDBMON_STICKY_FILE_PRE${_node:+.$_node}$HDBMON_STICKY_FILE_POST
              chmod 644 $HDBMON_STICKY_FILE_PRE${_node:+.$_node}$HDBMON_STICKY_FILE_POST
              chown $MYSIDADM:$MYGROUP $HDBMON_STICKY_FILE_PRE${_node:+.$_node}$HDBMON_STICKY_FILE_POST
              sg_log 1 Switching already disabled for node $node.
            fi
          done
          if [[ $TYPE = multi_node ]]; then
            for node in `echo "$cmviewcl_out"|grep ^$grepstring=enabled$|cut -f 2 -d :|cut -f 1 -d '|'`; do
              nn="$nn -n $node"
              echo Attempt to disable $node.
            done
            if [[ -n "$nn" ]]; then
              g/SGSBIN/cmmodpkg $nn -d $PKGNAME >/dev/null
            fi
          else
            $SGSBIN/cmmodpkg -d $PKGNAME >/dev/null
          fi
          # fi
        fi
        if [[ $TYPE = multi_node ]]; then
          for j in `echo ${PKGNODELIST[$i]}`; do
            cmexec -n $j -t $CMEXEC_TIMEOUT $SGSBIN/cmsetresource -r $HDBMON_SAFESYNC_GENRES -s down 2>/dev/null >/dev/null 
          done
        else
          $SGSBIN/cmsetresource -r $HDBMON_SAFESYNC_GENRES -s down 2>/dev/null >/dev/null
        fi
        $SGSBIN/sr_safesync -s ${PKGNAME[$i]}
        exit `[[ $? -eq 1 ]]`
      fi
    done
    sg_log 0 Not found.
    exit 2
  ;;
  -x)
    typeset echotext
    if [[ $TYPE = multi_node ]]; then
      cmviewcl_out=`$SGSBIN/cmviewcl -f line`
      for i in 0 1; do
        representative=`echo ${PKGNODELIST[$i]}|cut -f 1 -d " "`
        siteid=`echo "$cmviewcl_out"|grep '^node:'$representative'|siteid=[[:digit:]]*$'|cut -f 2 -d =`
        SITENAME[$i]=`echo "$cmviewcl_out"|grep '^site:[[:alnum:]_-]*|id='$siteid|cut -f 2 -d :|cut -f 1 -d "|"`
      done
      if [[ -z ${SITENAME[0]} || -z ${SITENAME[1]} ]]; then
        sg_log 0 Check cluster configuration for correct site settings
        exit 2
      fi
    fi
    for i in 0 1; do
      echo "${PKGNODELIST[$i]}"|grep " $HNAME " >/dev/null
      if [[ $? -eq 0 ]]; then
        if [[ $TYPE = multi_node ]]; then
          echotext="Remove double primary production token of [${PKGNAME[0]}, ${PKGNAME[1]}] from ${SITENAME[$i]}..."
        else
          echotext="Remove double primary production token of $PKGNAME from $HNAME..."
        fi
        echo $echotext
        echo "$(date '+%b %d %H:%M:%S')  $(hostname): sr_safesync exec: "$echotext >>$HDBMON_HISTORY_FILE
        chmod 644 $HDBMON_HISTORY_FILE
        chown $MYSIDADM:$MYGROUP $HDBMON_HISTORY_FILE
      fi
    done
    rm $HDBMON_TAKEOVER_FILE 2>/dev/null
    rm $HDBMON_PRODUCTION_START_FILE 2>/dev/null
  ;;
  -dbg_abort) # for debugging purposes
    eval `$SGSBIN/cmgetpkgenv $PKGNAME`
    typeset hdb_nonleaderdir=$HANA_SHARED/$SGeSAP_HDB_SYSTEM/${SGeSAP_HDB_INSTANCE[0]}/.sgesap
    
    mkdir $hdb_nonleaderdir 2>/dev/null
    chmod 775 $hdb_nonleaderdir
    chgrp $SAPSYS $hdb_nonleaderdir
    for node in `$SGSBIN/cmviewcl -f line -p $PKGNAME|grep ^node:|grep state=starting|cut -f 2 -d :|cut -f 1 -d '|'`; do
      nodelist=$nodelist' '$node
      typeset abort_file=$hdb_nonleaderdir/.$node.abort
      touch $abort_file
      chmod 664 $abort_file
      chgrp $SAPSYS $abort_file
    done
    [ -n "$nodelist" ] && echo Aborting syncgate'(s)' of $PKGNAME on $nodelist
  ;;
  *) # includes help operation
    echo >&2 "$USAGE_STRING"
  ;;
esac

