Move the OCR and Move the Voting disk

Move the OCR
#
# The new raw storage devices for OCR should be owned by the
# root user, must be in the oinstall group, and must have
# permissions set to 640. Provide at least 280MB of disk
# space for each OCR file and verify the raw storage devices
# can be seen from all nodes in the cluster.
#
[root@racnode1 ~]# ls -l /dev/raw/raw[12]
crw-r----- 1 root oinstall 162, 1 Oct 8 21:55 /dev/raw/raw1
crw-r----- 1 root oinstall 162, 2 Oct 8 21:55 /dev/raw/raw2
[root@racnode2 ~]# ls -l /dev/raw/raw[12]
crw-r----- 1 root oinstall 162, 1 Oct 8 21:54 /dev/raw/raw1
crw-r----- 1 root oinstall 162, 2 Oct 8 21:54 /dev/raw/raw2
#
# Use the dd command to zero out the devices and make sure
# no data is written to the raw devices.
#
[root@racnode1 ~]# dd if=/dev/zero of=/dev/raw/raw1
[root@racnode1 ~]# dd if=/dev/zero of=/dev/raw/raw2
#
# Verify CRS is running on node 1.
#
[root@racnode1 ~]# crsctl check crs
CSS appears healthy
CRS appears healthy
EVM appears healthy
#
# Verify CRS is running on node 2.
#
[root@racnode2 ~]# crsctl check crs
CSS appears healthy
CRS appears healthy
EVM appears healthy
#
# Query the current location and number of OCR files on
# the OCFS2 file system.
#
[root@racnode1 ~]# ocrcheck
Status of Oracle Cluster Registry is as follows :
Version : 2
Total space (kbytes) : 262120
Used space (kbytes) : 4676
Available space (kbytes) : 257444
ID : 1513888898
Device/File Name : /u02/oradata/racdb/OCRFile <-- OCR (primary)
Device/File integrity check succeeded
Device/File Name : /u02/oradata/racdb/OCRFile_mirror <-- OCR (mirror)
Device/File integrity check succeeded
Cluster registry integrity check succeeded
#
# Move OCR and OCR mirror to new storage location.
#
[root@racnode1 ~]# ocrconfig -replace ocr /dev/raw/raw1
[root@racnode1 ~]# ocrconfig -replace ocrmirror /dev/raw/raw2
#
# Verify OCR relocation from node 1.
#
[root@racnode1 ~]# ocrcheck
Status of Oracle Cluster Registry is as follows :
Version : 2
Total space (kbytes) : 262120
Used space (kbytes) : 4676
Available space (kbytes) : 257444
ID : 1513888898
Device/File Name : /dev/raw/raw1
Device/File integrity check succeeded
Device/File Name : /dev/raw/raw2
Device/File integrity check succeeded
Cluster registry integrity check succeeded
#
# Verify OCR relocation from node 2.
#
[root@racnode2 ~]# ocrcheck
Status of Oracle Cluster Registry is as follows :
Version : 2
Total space (kbytes) : 262120
Used space (kbytes) : 4676
Available space (kbytes) : 257444
ID : 1513888898
Device/File Name : /dev/raw/raw1
Device/File integrity check succeeded
Device/File Name : /dev/raw/raw2
Device/File integrity check succeeded
Cluster registry integrity check succeeded
#
# Remove all deleted OCR files from the OCFS2 file system.
#
[root@racnode1 ~]# rm /u02/oradata/racdb/OCRFile
[root@racnode1 ~]# rm /u02/oradata/racdb/OCRFile_mirror
Move the Voting Disk
#
# The new raw storage devices for the voting disks should be
# owned by the oracle user, must be in the oinstall group,
# and and must have permissions set to 644. Provide at least
# 20MB of disk space for each voting disk and verify the raw
# storage devices can be seen from all nodes in the cluster.
#
[root@racnode1 ~]# ls -l /dev/raw/raw[345]
crw-r--r-- 1 oracle oinstall 162, 3 Oct 8 22:44 /dev/raw/raw3
crw-r--r-- 1 oracle oinstall 162, 4 Oct 8 22:45 /dev/raw/raw4
crw-r--r-- 1 oracle oinstall 162, 5 Oct 9 00:22 /dev/raw/raw5
[root@racnode2 ~]# ls -l /dev/raw/raw[345]
crw-r--r-- 1 oracle oinstall 162, 3 Oct 8 22:53 /dev/raw/raw3
crw-r--r-- 1 oracle oinstall 162, 4 Oct 8 22:54 /dev/raw/raw4
crw-r--r-- 1 oracle oinstall 162, 5 Oct 9 00:23 /dev/raw/raw5
#
# Use the dd command to zero out the devices and make sure
# no data is written to the raw devices.
#
[root@racnode1 ~]# dd if=/dev/zero of=/dev/raw/raw3
[root@racnode1 ~]# dd if=/dev/zero of=/dev/raw/raw4
[root@racnode1 ~]# dd if=/dev/zero of=/dev/raw/raw5
#
# Query the current location and number of voting disks on
# the OCFS2 file system. There needs to be at least two
# voting disks configured before attempting to perform the
# move.
#
[root@racnode1 ~]# crsctl query css votedisk
0. 0 /u02/oradata/racdb/CSSFile
1. 0 /u02/oradata/racdb/CSSFile_mirror1
2. 0 /u02/oradata/racdb/CSSFile_mirror2
located 3 votedisk(s).
#
# Stop all application processes.
#
[root@racnode1 ~]# srvctl stop database -d racdb
[root@racnode1 ~]# srvctl stop asm -n racnode1
[root@racnode1 ~]# srvctl stop asm -n racnode2
[root@racnode1 ~]# srvctl stop nodeapps -n racnode1
[root@racnode1 ~]# srvctl stop nodeapps -n racnode2
#
# Verify all application processes are OFFLINE.
#
[root@racnode1 ~]# crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.racdb.db application OFFLINE OFFLINE
ora....b1.inst application OFFLINE OFFLINE
ora....b2.inst application OFFLINE OFFLINE
ora....srvc.cs application OFFLINE OFFLINE
ora....db1.srv application OFFLINE OFFLINE
ora....db2.srv application OFFLINE OFFLINE
ora....SM1.asm application OFFLINE OFFLINE
ora....E1.lsnr application OFFLINE OFFLINE
ora....de1.gsd application OFFLINE OFFLINE
ora....de1.ons application OFFLINE OFFLINE
ora....de1.vip application OFFLINE OFFLINE
ora....SM2.asm application OFFLINE OFFLINE
ora....E2.lsnr application OFFLINE OFFLINE
ora....de2.gsd application OFFLINE OFFLINE
ora....de2.ons application OFFLINE OFFLINE
ora....de2.vip application OFFLINE OFFLINE
#
# Shut down CRS on node 1 and verify the CRS stack is not up.
#
[root@racnode1 ~]# crsctl stop crs
Stopping resources. This could take several minutes.
Successfully stopped CRS resources.
Stopping CSSD.
Shutting down CSS daemon.
Shutdown request successfully issued.
[root@racnode1 ~]# ps -ef grep d.bin grep -v grep
#
# Shut down CRS on node 2 and verify the CRS stack is not up.
#
[root@racnode2 ~]# crsctl stop crs
Stopping resources. This could take several minutes.
Successfully stopped CRS resources.
Stopping CSSD.
Shutting down CSS daemon.
Shutdown request successfully issued.
[root@racnode2 ~]# ps -ef grep d.bin grep -v grep
#
# Move all three voting disks to new storage location.
#
[root@racnode1 ~]# crsctl delete css votedisk /u02/oradata/racdb/CSSFile -force
successful deletion of votedisk /u02/oradata/racdb/CSSFile.
[root@racnode1 ~]# crsctl add css votedisk /dev/raw/raw3 -force
Now formatting voting disk: /dev/raw/raw3
successful addition of votedisk /dev/raw/raw3.
[root@racnode1 ~]# crsctl delete css votedisk /u02/oradata/racdb/CSSFile_mirror1 -force
successful deletion of votedisk /u02/oradata/racdb/CSSFile_mirror1.
[root@racnode1 ~]# crsctl add css votedisk /dev/raw/raw4 -force
Now formatting voting disk: /dev/raw/raw4
successful addition of votedisk /dev/raw/raw4.
[root@racnode1 ~]# crsctl delete css votedisk /u02/oradata/racdb/CSSFile_mirror2 -force
successful deletion of votedisk /u02/oradata/racdb/CSSFile_mirror2.
[root@racnode1 ~]# crsctl add css votedisk /dev/raw/raw5 -force
Now formatting voting disk: /dev/raw/raw5
successful addition of votedisk /dev/raw/raw5.
#
# Verify voting disk(s) relocation from node 1.
#
[root@racnode1 ~]# crsctl query css votedisk
0. 0 /dev/raw/raw3
1. 0 /dev/raw/raw4
2. 0 /dev/raw/raw5
located 3 votedisk(s).
#
# Verify voting disk(s) relocation from node 2.
#
[root@racnode2 ~]# crsctl query css votedisk
0. 0 /dev/raw/raw3
1. 0 /dev/raw/raw4
2. 0 /dev/raw/raw5
located 3 votedisk(s).
#
# Remove all deleted voting disk files from the OCFS2 file system.
#
[root@racnode1 ~]# rm /u02/oradata/racdb/CSSFile
[root@racnode1 ~]# rm /u02/oradata/racdb/CSSFile_mirror1
[root@racnode1 ~]# rm /u02/oradata/racdb/CSSFile_mirror2
#
# With all voting disks now located on raw storage devices,
# restart CRS on all Oracle RAC nodes.
#
[root@racnode1 ~]# crsctl start crs
[root@racnode2 ~]# crsctl start crs