#####################################################################
##    Script Name  : setup_shared_storage.sh                       ##
##    Version #    : 1.0                                           ##
##    Date created : 07-Aug-2012                                   ##
##    Created by   : Suresh Neravati                               ##
##    Usage        : sh create_mp_targets.sh <shared_storage_ip>   ##
#####################################################################

#!/bin/sh
#install_dir=`pwd`
install_dir="/home/oracle/scripts/rac/sharedstorage"
cv_dir="/home/oracle/cv_home"
RAC_CONF_DIR="/home/oracle/scripts/rac/racconfig"
ssip="$1"
remote_machine="$2"
runas=""
if [ `whoami` !=  "root" ]; then
   echo "You can run this script as root user only, Please try again ....!"
   exit 1
fi
#clear

if [ "$ssip" = "" ]; then
   echo ""
   echo "shared storage ip should be the first parameter to run the script"
   echo "try again with share storage IP...."
   echo ""
   exit 1
fi

echo -e "\n**************************************************************"
echo "     Script started at $(date)"
echo -e "**************************************************************\n"
if [ "${remote_machine}" = ""  ]; 
then
   remote_machine="YES"
   echo -n "Enter other database server IP to include in the cluster setup : "
   read OTHER_DBSERVER_IP
   if [ ! ${OTHER_DBSERVER_IP} = "" ]; then
      echo -e "\n******** Setting up shared storage on ${OTHER_DBSERVER_IP} remotely ********\n"
      ssh "root@${OTHER_DBSERVER_IP}" "${install_dir}/setup_shared_storage.sh ${ssip} ${remote_machine}"
      echo -e "\n***** Shared storage set up completed on the remote machine *****\n"
   else
      echo -e "\nYou have not entered the IP, existing the setup. Please try again...!\n"
      exit 1
   fi
fi
# add system parameters
sed -i "/kernel.shmall =[.]*/d" /etc/sysctl.conf
grep -q ^kernel.shmall /etc/sysctl.conf
if [ $? -ne 0 ]; then
   echo "kernel.shmall = 12582912" >> /etc/sysctl.conf
fi

grep -q ^kernel.sem /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "kernel.sem = 250 32000 100 128" >> /etc/sysctl.conf
fi

sed -i "/kernel.shmmax =[.]*/d" /etc/sysctl.conf
grep -q ^kernel.shmmax /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "kernel.shmmax = 51539607552" >> /etc/sysctl.conf
        #echo "kernel.shmmax = 2147483648" >> /etc/sysctl.conf
fi

grep -q ^kernel.shmmni /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "kernel.shmmni = 4096" >> /etc/sysctl.conf
fi

grep -q ^net.ipv4.ip_local_port_range /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.ipv4.ip_local_port_range = 9000 65500" >> /etc/sysctl.conf
fi

grep -q ^net.core.rmem_default /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.core.rmem_default = 262144" >> /etc/sysctl.conf
fi

grep -q ^net.core.rmem_max /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.core.rmem_max = 4194304" >> /etc/sysctl.conf
fi

grep -q ^net.core.wmem_default /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.core.wmem_default = 262144" >> /etc/sysctl.conf
fi

grep -q ^net.core.wmem_max /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.core.wmem_max = 1048586" >> /etc/sysctl.conf
fi

grep -q ^fs.aio-max-nr /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf
fi

sed -i "/fs.file-max =[.]*/d" /etc/sysctl.conf
grep -q ^fs.file-max /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "fs.file-max = 6815744" >> /etc/sysctl.conf
fi

grep -q ^vm.hugetlb_shm_group /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "vm.hugetlb_shm_group = 1000" >> /etc/sysctl.conf
fi

grep -q -G "oracle.*soft.*nproc.*2047" /etc/security/limits.conf
if [ $? -ne 0 ]; then
        echo "oracle    soft    nproc   2047" >> /etc/security/limits.conf
fi

grep -q -G "oracle.*hard.*nproc.*16384" /etc/security/limits.conf
if [ $? -ne 0 ]; then
        echo "oracle    hard    nproc   16384" >> /etc/security/limits.conf
fi

grep -q -G "oracle.*soft.*nofile.*1024" /etc/security/limits.conf
if [ $? -ne 0 ]; then
        echo "oracle    soft    nofile  1024" >> /etc/security/limits.conf
fi

grep -q -G "oracle.*hard.*nofile.*65536" /etc/security/limits.conf
if [ $? -ne 0 ]; then
        echo "oracle    hard    nofile  65536" >> /etc/security/limits.conf
fi

grep -q -G "\*.*soft.*memlock.*94371840" /etc/security/limits.conf
if [ $? -ne 0 ]; then
        echo "*         soft    memlock 94371840" >> /etc/security/limits.conf
fi

grep -q -G "\*.*hard.*memlock.*94371840" /etc/security/limits.conf
if [ $? -ne 0 ]; then
        echo "*         hard    memlock 94371840" >> /etc/security/limits.conf
fi

grep -q -G "session.*required.*pam_limits.so" /etc/pam.d/login
if [ $? -ne 0 ]; then
        echo "session   required        pam_limits.so" >> /etc/pam.d/login
fi

sed -i "/net.ipv4.conf.default.rp_filter =[.]*/d" /etc/sysctl.conf
grep -q ^net.ipv4.conf.default.rp_filter /etc/sysctl.conf
if [ $? -ne 0 ]; then
   echo "#net.ipv4.conf.default.rp_filter = 1" >> /etc/sysctl.conf
fi

grep -q ^net.ipv4.conf.eth0.rp_filter /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.ipv4.conf.eth0.rp_filter = 2" >> /etc/sysctl.conf
fi

grep -q ^net.ipv4.conf.eth1.rp_filter /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.ipv4.conf.eth1.rp_filter = 0" >> /etc/sysctl.conf
fi

grep -q ^net.ipv4.conf.eth2.rp_filter /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.ipv4.conf.eth2.rp_filter = 2" >> /etc/sysctl.conf
fi

grep -q ^net.ipv4.conf.eth3.rp_filter /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.ipv4.conf.eth3.rp_filter = 0" >> /etc/sysctl.conf
fi

grep -q ^net.ipv4.conf.eth4.rp_filter /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.ipv4.conf.eth4.rp_filter = 2" >> /etc/sysctl.conf
fi

grep -q ^net.ipv4.conf.eth5.rp_filter /etc/sysctl.conf
if [ $? -ne 0 ]; then
        echo "net.ipv4.conf.eth5.rp_filter = 2" >> /etc/sysctl.conf
fi

/sbin/sysctl -p > /dev/null

usermod -G dba,asmadmin,asmdba oracle

mkdir -p /opt/app/11.2.0/grid
mkdir -p /opt/app/oracle/product/11.2.0/db_1
mkdir -p /home/oracle/orarac

chown -R oracle:oinstall /opt/app
chown oracle:dba /home/oracle/orarac

# Check ResponseFiles file permissions

cp ${install_dir}/bash_profile /home/oracle/.bash_profile
chown oracle:oinstall /home/oracle/.bash_profile

srv=( iscsid iscsi multipathd )

for serv in ${srv[@]}
do
   if [[ -z `$runas /etc/init.d/${serv} status | grep "running..."` ]]
   then
      if [[ -z `$runas chkconfig ${serv} --list` ]]
      then
        echo " adding service ${serv} ....."
        chkconfig --add ${serv} 
        chkconfig --level 345 ${serv} on
      fi
      # start service 
      if [ ${serv} == "iscsid" ]; then
         cp -f ${install_dir}/iscsid_template.conf /etc/iscsi/iscsid.conf
         service ${serv} force-start
         sleep 10
      else
         service ${serv} start
         sleep 10
      fi
   fi
done

# display run status of the service
echo " "
echo " "
for serv in ${srv[@]}
do
   if [[ -z `/etc/init.d/${serv} status | grep "${serv}"` ]]
   then
      echo "${serv} `/etc/init.d/${serv} status`"
   else
      echo "`/etc/init.d/${serv} status`"
   fi
done

# display machine ip and initiator name.
echo " "
#echo "Machine IP: $(ifconfig eth0 | grep "inet addr"|awk '{print(substr($2,6))}')"
#initiator_name="$(cat /etc/iscsi/initiatorname.iscsi | awk '{print substr($1,15) }')"
#echo "iscsi Initiator Name: $initiator_name"

enets=( eth4 eth5 )
for ethx in ${enets[@]}
do
   echo "Adding Ethernet: ${ethx}........."
   iscsiadm -m iface -I ora${ethx} --op=new
   iscsiadm -m iface -I ora${ethx} --op=update -n iface.net_ifacename -v ${ethx}
   echo " "
done

echo -e "\niSCSI interfaces info: "
iscsiadm -m iface

echo -e "\nLogging in targets: "
tgts=`iscsiadm -m discovery -t st -p ${ssip} -I ora${enets[0]} -I ora${enets[1]} | awk '{print($2)}'`
for tgt in ${tgts[@]}
do
   iscsiadm -m node -T ${tgt} -p ${ssip} -l
   iscsiadm -m node -T ${tgt} -p ${ssip} --op update -n node.startup -v automatic
done
sleep 20

echo -e "\nDevices are : "
devices=(`ls -l /dev/disk/by-path | grep ip | awk '{print($11)}' | sed 's/..\///g'`)
for device in ${devices[@]}
do
   echo "   /dev/${device}"
done
echo " "
echo "Create partition on the devices as follows: "
for device in ${devices[@]}
do
echo -e "\n********************* Creating patition on disk /dev/${device}******************"
echo "d
w
q" | /sbin/fdisk /dev/${device}

echo "n
p
1


w
q
" | /sbin/fdisk /dev/${device}

echo "p
q" | /sbin/fdisk /dev/${device}

done

#MapDevices 
vcrs=`sh ${RAC_CONF_DIR}/get_var_value.sh "DB_CLUSTER_CRS_VOLUME"`
vdata=`sh ${RAC_CONF_DIR}/get_var_value.sh "DB_CLUSTER_DATA_VOLUME"`
vfra=`sh ${RAC_CONF_DIR}/get_var_value.sh "DB_CLUSTER_FRA_VOLUME"`
vclusfiles=`sh ${RAC_CONF_DIR}/get_var_value.sh "DB_CLUSTER_CLUSFILES_VOLUME"`

echo -e "CRS_VOLUME=${vcrs} \nDATA_VOLUMNE=${vdata} \nFRA_VOLUME=${vfra}\nCLUSFILES_VOLUME=${vclusfiles}"
if [ "${vcrs}" == "" ] || [ "${vdata}" == "" ] || [ "${vfra}" == "" ] || [ "${vclusfiles}" == "" ]; then
   echo "Configuration file is missing crs or data or fra or clusfiles volume names. Please define these names and run the script again...!"
   exit 1
fi

/sbin/dmsetup remove_all
devs=(`ls -l /dev/disk/by-path | grep ip | awk '{print($11)}' | sed 's/..\///g'`)
tgtnames=(`ls -l /dev/disk/by-path | grep ip | awk '{print($9)}' | sed 's/:/ /g'| awk '{print($5)}' | sed 's/-lun-0//'`)
devices=(`ls -l /dev/disk/by-path | grep ip | awk '{print($11)}' | sed 's/..\///g'`)
for (( i=1; $i <= ${#tgtnames[*]}; i++ ))
do
        #echo "Device /dev/${devs[i-1]} is mapped to target ${tgtnames[i-1]}"
        if [ "${tgtnames[i-1]}" == "${vdata}" ]; then
           dataid=(`scsi_id --whitelisted /dev/${devs[i-1]}`)
        elif [ "${tgtnames[i-1]}" == "${vcrs}" ]; then
           crsid=(`scsi_id --whitelisted /dev/${devs[i-1]}`)
        elif [ "${tgtnames[i-1]}" == "${vfra}" ]; then
           fraid=(`scsi_id --whitelisted /dev/${devs[i-1]}`)
        elif [ "${tgtnames[i-1]}" == "${vclusfiles}" ]; then
           clusfilesid=(`scsi_id --whitelisted /dev/${devs[i-1]}`)
       fi
done

if [ "${crsid}" == "" ] || [ "${dataid}" == "" ] || [ "${fraid}" == "" ] || [ "${clusfilesid}" == "" ]; then
   echo "Can not add crs, data and fra ids in multipath.conf file."
   exit 1
else
   sed -e "s/crsid/${crsid}/g" -e "s/dataid/${dataid}/g" -e "s/fraid/${fraid}/g" -e "s/clusfilesid/${clusfilesid}/g" < ${install_dir}/multipath_template.conf > /etc/multipath.conf
   cp -f ${install_dir}/12-dm-permissions_template.rules /etc/udev/rules.d/12-dm-permissions.rules  
fi

chkconfig --level 345 multipathd on

if [ -f "/etc/multipath.conf" ]; then
   service multipathd restart     
fi
sleep 15
chown -R oracle:dba /dev/iscsi/*

#rpm -e cvuqdisk-1.0.9-1
if [ -z `rpm -qa | grep cvuqdisk` ]; then
   echo "cvuqdisk rpm not found, now installing"
   if [ ! -d /home/oracle/cv_home ]; then
      /usr/bin/unzip ${install_dir}/cvupack_Linux_x86_64.zip -d "${cv_dir}"
      chown -R oracle:oinstall ${cv_dir}
      chmod -R 775 ${cv_dir}
   fi
   /bin/rpm -Uvh ${cv_dir}/rpm/cvuqdisk-1.0.9-1.rpm
fi

echo -e "\nCheck for user permissions on the disks"
ls -l /dev/dm*
echo -e "\nCheck if oracle ASM symbolic links exists for multipath devices"
ls -l /dev/iscsi/*
echo -e "\nCheck multipath setup"
multipath -ll

#echo -e "\nAs an oracle user, run the script ${install_dir}/check_clusterware_pre_requisits.sh "
#echo -e "  to verify Pre-requisits for the Clusterware installation *******\n"

echo -e "\n**************************************************"
echo "  Script ended at $(date)"
echo -e "**************************************************\n"

exit 0
