Due to the following problem , which is likely a un-fixed bug, and I didn't use md_backup ASM info, thus I have to re-create the ASM instance.
Fri Jan 25 22:00:40 2013
NOTE: attached to recovery domain 1
NOTE: starting recovery of thread=1 ckpt=32.568 group=1 (DATA)
Errors in file /u01/app/oracle/diag/asm/+asm/+ASM1/trace/+ASM1_ora_4561.trc (incident=55419):
ORA-00600: internal error code, arguments: [kfrValAcd30], [DATA], [1], [32], [568], [33], [569], [], [], [], [], []
Incident details in: /u01/app/oracle/diag/asm/+asm/+ASM1/incident/incdir_55419/+ASM1_ora_4561_i55419.tr
[root@rac1 ~]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
FRA
[root@rac1 ~]# oracleasm querydisk -p DISK1
Disk "DISK1" is a valid ASM disk
/dev/sdb1: LABEL="DISK1" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p DISK2
Disk "DISK2" is a valid ASM disk
/dev/sdc1: LABEL="DISK2" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p DISK3
Disk "DISK3" is a valid ASM disk
/dev/sdd1: LABEL="DISK3" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p DISK4
Disk "DISK4" is a valid ASM disk
/dev/sde1: LABEL="DISK4" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p DISK5
Disk "DISK5" is a valid ASM disk
/dev/sdf1: LABEL="DISK5" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p FRA
Disk "FRA" is a valid ASM disk
/dev/sdg1: LABEL="FRA" TYPE="oracleasm"
2. Recreate ASM disks, which belong to two diskgroups: FRA (one disk) and DATA(5 disks).
SQL> alter diskgroup fra mount;
Diskgroup altered.
SQL> select * from v$asm_diskgroup;
GROUP_NUMBER NAME SECTOR_SIZE BLOCK_SIZE
------------ ------------------------------ ----------- ----------
ALLOCATION_UNIT_SIZE STATE TYPE TOTAL_MB FREE_MB HOT_USED_MB
-------------------- ----------- ------ ---------- ---------- -----------
COLD_USED_MB REQUIRED_MIRROR_FREE_MB USABLE_FILE_MB OFFLINE_DISKS
------------ ----------------------- -------------- -------------
COMPATIBILITY
------------------------------------------------------------
DATABASE_COMPATIBILITY V
------------------------------------------------------------ -
0 DATA 0 4096
0 DISMOUNTED 0 0 0
0 0 0 0
0.0.0.0.0
0.0.0.0.0 N
1 FRA 512 4096
1048576 MOUNTED EXTERN 10236 8533 0
1703 0 8533 0
11.2.0.0.0
10.1.0.0.0 N
SQL> select path,group_number from v$asm_disk;
PATH GROUP_NUMBER
------------------------------ ------------
ORCL:DISK1 0
ORCL:DISK2 0
ORCL:FRA 0
ORCL:DISK4 0
ORCL:DISK5 0
ORCL:DISK3 0
6 rows selected.
SQL> drop diskgroup fra force including contents;
Diskgroup dropped.
SQL> select * from v$asm_diskgroup;
GROUP_NUMBER NAME SECTOR_SIZE BLOCK_SIZE
------------ ------------------------------ ----------- ----------
ALLOCATION_UNIT_SIZE STATE TYPE TOTAL_MB FREE_MB HOT_USED_MB
-------------------- ----------- ------ ---------- ---------- -----------
COLD_USED_MB REQUIRED_MIRROR_FREE_MB USABLE_FILE_MB OFFLINE_DISKS
------------ ----------------------- -------------- -------------
COMPATIBILITY
------------------------------------------------------------
DATABASE_COMPATIBILITY V
------------------------------------------------------------ -
0 DATA 0 4096
0 DISMOUNTED 0 0 0
0 0 0 0
0.0.0.0.0
0.0.0.0.0 N
SQL> drop diskgroup DATA force including contents;
drop diskgroup DATA force including contents
*
ERROR at line 1:
ORA-15039: diskgroup not dropped
ORA-15276: ASM diskgroup DATA has cluster voting files
SQL> exit
Disconnected from Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - Production
With the Real Application Clusters and Automatic Storage Management options
[oracle@rac1 ~]$ oerr ora 15276]
[oracle@rac1 ~]$ oerr ora 15276
15276, 00000, "ASM diskgroup %s has cluster voting files"
// *Cause: An attempt was made to drop a diskgroup that contained cluster
// voting files.
// *Action: Move the cluster voting files out of the diskgroup and retry the
// operation.
[root@rac1 ~]# oracleasm deletedisk DISK1
Clearing disk header: failed
Unable to clear disk "DISK1"
[root@rac1 ~]# oracleasm deletedisk DISK2
Clearing disk header: done
Dropping disk: done
[root@rac1 ~]# oracleasm deletedisk DISK3
Clearing disk header: done
Dropping disk: done
[root@rac1 ~]# oracleasm deletedisk DISK4
Clearing disk header: done
Dropping disk: done
[root@rac1 ~]# oracleasm deletedisk DISK5
Clearing disk header: done
Dropping disk: done
[root@rac1 ~]# oracleasm createdisk DISK2 /dev/sdc1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac1 ~]# oracleasm listdisks
DISK1
DISK2
FRA
[root@rac1 ~]# oracleasm createdisk DISK3 /dev/sdd1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm createdisk DISK4 /dev/sde1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm createdisk DISK5 /dev/sdf1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# dd if=/dev/zero of=/dev/sdb1 bs=1024 count=4
4+0 records in
4+0 records out
4096 bytes (4.1 kB) copied, 4.3731e-05 seconds, 93.7 MB/s
[root@rac1 ~]# oracleasm deletedisk DISK1
Disk "DISK1" defines an unmarked device
Dropping disk: done
[root@rac1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac1 ~]# oracleasm listdisks
DISK2
DISK3
DISK4
DISK5
FRA
[root@rac1 ~]# oracleasm createdisk DISK1 /dev/sdb1
Unable to open device "/dev/sdb1": Device or resource busy
[root@rac1 ~]# oracleasm createdisk DISK1 /dev/sdb1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac1 ~]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
FRA
[root@rac1 install]# ./roothas.pl -deconfig -force
Using configuration parameter file: ./crsconfig_params
CRS-4047: No Oracle Clusterware components configured.
CRS-4000: Command Stop failed, or completed with errors.
CRS-4047: No Oracle Clusterware components configured.
CRS-4000: Command Delete failed, or completed with errors.
CRS-4047: No Oracle Clusterware components configured.
CRS-4000: Command Stop failed, or completed with errors.
You must kill ohasd processes or reboot the system to properly
cleanup the processes started by Oracle clusterware
ACFS-9200: Supported
ACFS-9313: No ADVM/ACFS installation detected.
Either /etc/oracle/olr.loc does not exist or is not readable
Make sure the file exists and it has read and execute access
Failure in execution (rc=-1, 256, No such file or directory) for command 1 /etc/init.d/ohasd deinstall
Successfully deconfigured Oracle Restart stack
[root@rac1 install]# ps -ef |grep has
root 5651 4233 0 00:11 pts/1 00:00:00 grep has
--finally run the root.sh to reconfigure clusterware services
[root@rac1 grid]# ./root.sh
Running Oracle 11g root script...
The following environment variables are set as:
ORACLE_OWNER= oracle
ORACLE_HOME= /u01/app/11.2.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
The contents of "dbhome" have not changed. No need to overwrite.
The contents of "oraenv" have not changed. No need to overwrite.
The contents of "coraenv" have not changed. No need to overwrite.
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/app/11.2.0/grid/crs/install/crsconfig_params
LOCAL ADD MODE
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
OLR initialization - successful
Adding daemon to inittab
ACFS-9200: Supported
ACFS-9300: ADVM/ACFS distribution files found.
ACFS-9307: Installing requested ADVM/ACFS software.
ACFS-9308: Loading installed ADVM/ACFS drivers.
ACFS-9321: Creating udev for ADVM/ACFS.
ACFS-9323: Creating module dependencies - this may take some time.
ACFS-9327: Verifying ADVM/ACFS devices.
ACFS-9309: ADVM/ACFS installation correctness verified.
CRS-2672: Attempting to start 'ora.mdnsd' on 'rac1'
CRS-2676: Start of 'ora.mdnsd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rac1'
CRS-2676: Start of 'ora.diskmon' on 'rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rac1' succeeded
ASM created and started successfully.
Disk Group DATA created successfully.
clscfg: -install mode specified
Successfully accumulated necessary OCR keys.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
Successful addition of voting disk 55c34b4c06194f09bf8018170dab7627.
Successfully replaced voting disk group with +DATA.
CRS-4266: Voting file(s) successfully replaced
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 55c34b4c06194f09bf8018170dab7627 (ORCL:DISK1) [DATA]
Located 1 voting disk(s).
CRS-2672: Attempting to start 'ora.asm' on 'rac1'
CRS-2676: Start of 'ora.asm' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.DATA.dg' on 'rac1'
CRS-2676: Start of 'ora.DATA.dg' on 'rac1' succeeded
ACFS-9200: Supported
ACFS-9200: Supported
CRS-2672: Attempting to start 'ora.registry.acfs' on 'rac1'
CRS-2676: Start of 'ora.registry.acfs' on 'rac1' succeeded
Preparing packages for installation...
cvuqdisk-1.0.9-1
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
[root@rac1 grid]# ps -ef |grep ASM
root 1825 4233 0 00:24 pts/1 00:00:00 grep ASM
oracle 13498 1 0 00:21 ? 00:00:00 asm_pmon_+ASM1
oracle 13504 1 0 00:21 ? 00:00:00 asm_psp0_+ASM1
oracle 13738 1 0 00:21 ? 00:00:00 asm_vktm_+ASM1
oracle 13762 1 0 00:21 ? 00:00:00 asm_gen0_+ASM1
oracle 13771 1 0 00:21 ? 00:00:00 asm_diag_+ASM1
oracle 13776 1 0 00:21 ? 00:00:00 asm_ping_+ASM1
oracle 13783 1 0 00:21 ? 00:00:00 asm_dia0_+ASM1
oracle 13790 1 0 00:21 ? 00:00:00 asm_lmon_+ASM1
oracle 13798 1 0 00:21 ? 00:00:00 asm_lmd0_+ASM1
oracle 13802 1 0 00:21 ? 00:00:00 asm_lms0_+ASM1
oracle 13811 1 0 00:21 ? 00:00:00 asm_lmhb_+ASM1
oracle 13820 1 0 00:21 ? 00:00:00 asm_mman_+ASM1
oracle 13830 1 0 00:21 ? 00:00:00 asm_dbw0_+ASM1
oracle 13838 1 0 00:21 ? 00:00:00 asm_lgwr_+ASM1
oracle 13843 1 0 00:21 ? 00:00:00 asm_ckpt_+ASM1
oracle 13852 1 0 00:21 ? 00:00:00 asm_smon_+ASM1
oracle 13860 1 0 00:21 ? 00:00:00 asm_rbal_+ASM1
oracle 13866 1 0 00:21 ? 00:00:00 asm_gmon_+ASM1
oracle 13872 1 0 00:21 ? 00:00:00 asm_mmon_+ASM1
oracle 13880 1 0 00:21 ? 00:00:00 asm_mmnl_+ASM1
oracle 13938 1 0 00:21 ? 00:00:00 asm_lck0_+ASM1
oracle 15325 1 0 00:21 ? 00:00:00 oracle+ASM1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15727 1 1 00:21 ? 00:00:01 oracle+ASM1_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15733 1 0 00:21 ? 00:00:00 asm_asmb_+ASM1
oracle 15743 1 0 00:21 ? 00:00:00 oracle+ASM1_asmb_+asm1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15754 1 0 00:21 ? 00:00:00 asm_o000_+ASM1
oracle 15762 1 0 00:21 ? 00:00:00 oracle+ASM1_o000_+asm1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 26710 1 0 00:22 ? 00:00:00 oracle+ASM1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
[root@rac1 grid]# ps -ef |grep oracle
root 1827 4233 0 00:24 pts/1 00:00:00 grep oracle
root 4812 4300 0 Mar26 pts/2 00:00:00 su - oracle
oracle 4813 4812 0 Mar26 pts/2 00:00:00 -bash
oracle 13498 1 0 00:21 ? 00:00:00 asm_pmon_+ASM1
oracle 13504 1 0 00:21 ? 00:00:00 asm_psp0_+ASM1
oracle 13738 1 0 00:21 ? 00:00:00 asm_vktm_+ASM1
oracle 13762 1 0 00:21 ? 00:00:00 asm_gen0_+ASM1
oracle 13771 1 0 00:21 ? 00:00:00 asm_diag_+ASM1
oracle 13776 1 0 00:21 ? 00:00:00 asm_ping_+ASM1
oracle 13783 1 0 00:21 ? 00:00:00 asm_dia0_+ASM1
oracle 13790 1 0 00:21 ? 00:00:00 asm_lmon_+ASM1
oracle 13798 1 0 00:21 ? 00:00:00 asm_lmd0_+ASM1
oracle 13802 1 0 00:21 ? 00:00:00 asm_lms0_+ASM1
oracle 13811 1 0 00:21 ? 00:00:00 asm_lmhb_+ASM1
oracle 13820 1 0 00:21 ? 00:00:00 asm_mman_+ASM1
oracle 13830 1 0 00:21 ? 00:00:00 asm_dbw0_+ASM1
oracle 13838 1 0 00:21 ? 00:00:00 asm_lgwr_+ASM1
oracle 13843 1 0 00:21 ? 00:00:00 asm_ckpt_+ASM1
oracle 13852 1 0 00:21 ? 00:00:00 asm_smon_+ASM1
oracle 13860 1 0 00:21 ? 00:00:00 asm_rbal_+ASM1
oracle 13866 1 0 00:21 ? 00:00:00 asm_gmon_+ASM1
oracle 13872 1 0 00:21 ? 00:00:00 asm_mmon_+ASM1
oracle 13880 1 0 00:21 ? 00:00:00 asm_mmnl_+ASM1
oracle 13938 1 0 00:21 ? 00:00:00 asm_lck0_+ASM1
oracle 15325 1 0 00:21 ? 00:00:00 oracle+ASM1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15720 1 0 00:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmd.bin
oracle 15727 1 0 00:21 ? 00:00:01 oracle+ASM1_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15733 1 0 00:21 ? 00:00:00 asm_asmb_+ASM1
oracle 15743 1 0 00:21 ? 00:00:00 oracle+ASM1_asmb_+asm1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15754 1 0 00:21 ? 00:00:00 asm_o000_+ASM1
oracle 15762 1 0 00:21 ? 00:00:00 oracle+ASM1_o000_+asm1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 16222 15720 0 00:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmlogger.bin -o /u01/app/11.2.0/grid/evm/log/evmlogger.info -l /u01/app/11.2.0/grid/evm/log/evmlogger.log
oracle 26509 1 0 00:22 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 26710 1 0 00:22 ? 00:00:00 oracle+ASM1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 30286 1 0 00:23 ? 00:00:00 /u01/app/11.2.0/grid/opmn/bin/ons -d
oracle 30287 30286 0 00:23 ? 00:00:00 /u01/app/11.2.0/grid/opmn/bin/ons -d
oracle 30801 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 30840 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/mdnsd.bin
oracle 31165 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/gpnpd.bin
oracle 31433 1 0 00:23 ? 00:00:00 /u01/app/11.2.0/grid/bin/tnslsnr LISTENER_SCAN1 -inherit
oracle 31558 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/gipcd.bin
oracle 31851 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/diskmon.bin -d -f
oracle 31923 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/ocssd.bin
oracle 32011 1 0 00:23 ? 00:00:00 /u01/app/11.2.0/grid/bin/scriptagent.bin
oracle 32093 1 9 00:23 ? 00:00:07 /u01/app/11.2.0/grid/jdk/jre//bin/java -server -Xcheck:jni -Xms128M -Xmx384M -Djava.awt.headless=true -Ddisable.checkForUpdate=true -Dstdstream.filesize=100 -Dstdstream.filenumber=10 -DTRACING.ENABLED=false -Doracle.wlm.dbwlmlogger.logging.level=INFO -Dport.rmi=23792 -jar /u01/app/11.2.0/grid/oc4j/j2ee/home/oc4j.jar -config /u01/app/11.2.0/grid/oc4j/j2ee/home/OC4J_DBWLM_config/server.xml -out /u01/app/11.2.0/grid/oc4j/j2ee/home/log/oc4j.out -err /u01/app/11.2.0/grid/oc4j/j2ee/home/log/oc4j.err
SQL> set pages 1000
SQL> select * from v$asm_diskgroup;
GROUP_NUMBER NAME SECTOR_SIZE BLOCK_SIZE
------------ ------------------------------ ----------- ----------
ALLOCATION_UNIT_SIZE STATE TYPE TOTAL_MB FREE_MB HOT_USED_MB
-------------------- ----------- ------ ---------- ---------- -----------
COLD_USED_MB REQUIRED_MIRROR_FREE_MB USABLE_FILE_MB OFFLINE_DISKS
------------ ----------------------- -------------- -------------
COMPATIBILITY
------------------------------------------------------------
DATABASE_COMPATIBILITY V
------------------------------------------------------------ -
1 DATA 512 4096
1048576 MOUNTED EXTERN 51180 50776 0
404 0 50776 0
11.2.0.0.0
10.1.0.0.0 Y
6. do the same in node2
[root@rac2 install]# ls -l root*
-rwxr-xr-x 1 oracle oinstall 29259 Mar 20 2011 rootcrs.pl
-rwxr-xr-x 1 oracle oinstall 13522 Mar 20 2011 roothas.pl
-rwxr-xr-x 1 oracle oinstall 915 Mar 20 2011 rootofs.sh
[root@rac2 install]# ./roothas.pl -deconfig
Using configuration parameter file: ./crsconfig_params
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Stop failed, or completed with errors.
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Delete failed, or completed with errors.
CRS-2796: The command may not proceed when Cluster Ready Services is not running
CRS-4687: Shutdown command has completed with errors.
CRS-4000: Command Stop failed, or completed with errors.
You must kill ohasd processes or reboot the system to properly
cleanup the processes started by Oracle clusterware
ACFS-9200: Supported
Successfully deconfigured Oracle Restart stack
[root@rac2 install]# ./rootcrs.pl -deconfig
Using configuration parameter file: ./crsconfig_params
Oracle Clusterware stack is not active on this node
Restart the clusterware stack (use /u01/app/11.2.0/grid/bin/crsctl start crs) and retry
Failed to verify resources
[root@rac2 install]# which oracleasm
/usr/sbin/oracleasm
[root@rac2 install]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac2 install]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
FRA
[root@rac2 install]# cd ../..
[root@rac2 grid]# pwd
/u01/app/11.2.0/grid
[root@rac2 grid]# ./root.sh
Running Oracle 11g root script...
The following environment variables are set as:
ORACLE_OWNER= oracle
ORACLE_HOME= /u01/app/11.2.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
The contents of "dbhome" have not changed. No need to overwrite.
The contents of "oraenv" have not changed. No need to overwrite.
The contents of "coraenv" have not changed. No need to overwrite.
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/app/11.2.0/grid/crs/install/crsconfig_params
LOCAL ADD MODE
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
OLR initialization - successful
Adding daemon to inittab
ACFS-9200: Supported
ACFS-9300: ADVM/ACFS distribution files found.
ACFS-9307: Installing requested ADVM/ACFS software.
ACFS-9308: Loading installed ADVM/ACFS drivers.
ACFS-9321: Creating udev for ADVM/ACFS.
ACFS-9323: Creating module dependencies - this may take some time.
ACFS-9327: Verifying ADVM/ACFS devices.
ACFS-9309: ADVM/ACFS installation correctness verified.
CRS-4402: The CSS daemon was started in exclusive mode but found an active CSS daemon on node rac1, number 1, and is terminating
An active cluster was found during exclusive startup, restarting to join the cluster
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
[root@rac2 grid]# ps -ef |grep pmon
oracle 8105 1 0 00:30 ? 00:00:00 asm_pmon_+ASM2
root 9282 4030 0 00:40 pts/1 00:00:00 grep pmon
[root@rac2 grid]# ps -ef |grep ASM2
oracle 8105 1 0 00:30 ? 00:00:00 asm_pmon_+ASM2
oracle 8109 1 0 00:30 ? 00:00:00 asm_psp0_+ASM2
oracle 8113 1 0 00:30 ? 00:00:00 asm_vktm_+ASM2
oracle 8119 1 0 00:30 ? 00:00:00 asm_gen0_+ASM2
oracle 8123 1 0 00:30 ? 00:00:00 asm_diag_+ASM2
oracle 8127 1 0 00:30 ? 00:00:00 asm_ping_+ASM2
oracle 8131 1 0 00:30 ? 00:00:00 asm_dia0_+ASM2
oracle 8135 1 0 00:30 ? 00:00:00 asm_lmon_+ASM2
oracle 8139 1 0 00:30 ? 00:00:00 asm_lmd0_+ASM2
oracle 8143 1 0 00:30 ? 00:00:00 asm_lms0_+ASM2
oracle 8149 1 0 00:30 ? 00:00:00 asm_lmhb_+ASM2
oracle 8153 1 0 00:30 ? 00:00:00 asm_mman_+ASM2
oracle 8157 1 0 00:30 ? 00:00:00 asm_dbw0_+ASM2
oracle 8161 1 0 00:30 ? 00:00:00 asm_lgwr_+ASM2
oracle 8165 1 0 00:30 ? 00:00:00 asm_ckpt_+ASM2
oracle 8169 1 0 00:30 ? 00:00:00 asm_smon_+ASM2
oracle 8173 1 0 00:30 ? 00:00:00 asm_rbal_+ASM2
oracle 8177 1 0 00:30 ? 00:00:00 asm_gmon_+ASM2
oracle 8181 1 0 00:30 ? 00:00:00 asm_mmon_+ASM2
oracle 8185 1 0 00:30 ? 00:00:00 asm_mmnl_+ASM2
oracle 8189 1 0 00:31 ? 00:00:00 asm_lck0_+ASM2
oracle 8206 1 0 00:31 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8239 1 0 00:31 ? 00:00:00 oracle+ASM2_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8248 1 0 00:31 ? 00:00:00 asm_asmb_+ASM2
oracle 8252 1 0 00:31 ? 00:00:00 oracle+ASM2_asmb_+asm2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8408 1 0 00:31 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 9221 1 0 00:39 ? 00:00:00 asm_gcr0_+ASM2
root 9286 4030 0 00:40 pts/1 00:00:00 grep ASM2
[root@rac2 grid]# ps -ef |grep oracle
oracle 7800 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 7814 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/mdnsd.bin
oracle 7830 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/gpnpd.bin
oracle 7855 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/gipcd.bin
oracle 7907 1 0 00:30 ? 00:00:01 /u01/app/11.2.0/grid/bin/ocssd.bin
oracle 7910 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/diskmon.bin -d -f
oracle 8105 1 0 00:30 ? 00:00:00 asm_pmon_+ASM2
oracle 8109 1 0 00:30 ? 00:00:00 asm_psp0_+ASM2
oracle 8113 1 0 00:30 ? 00:00:00 asm_vktm_+ASM2
oracle 8119 1 0 00:30 ? 00:00:00 asm_gen0_+ASM2
oracle 8123 1 0 00:30 ? 00:00:00 asm_diag_+ASM2
oracle 8127 1 0 00:30 ? 00:00:00 asm_ping_+ASM2
oracle 8131 1 0 00:30 ? 00:00:00 asm_dia0_+ASM2
oracle 8135 1 0 00:30 ? 00:00:00 asm_lmon_+ASM2
oracle 8139 1 0 00:30 ? 00:00:00 asm_lmd0_+ASM2
oracle 8143 1 0 00:30 ? 00:00:00 asm_lms0_+ASM2
oracle 8149 1 0 00:30 ? 00:00:00 asm_lmhb_+ASM2
oracle 8153 1 0 00:30 ? 00:00:00 asm_mman_+ASM2
oracle 8157 1 0 00:30 ? 00:00:00 asm_dbw0_+ASM2
oracle 8161 1 0 00:30 ? 00:00:00 asm_lgwr_+ASM2
oracle 8165 1 0 00:30 ? 00:00:00 asm_ckpt_+ASM2
oracle 8169 1 0 00:30 ? 00:00:00 asm_smon_+ASM2
oracle 8173 1 0 00:30 ? 00:00:00 asm_rbal_+ASM2
oracle 8177 1 0 00:30 ? 00:00:00 asm_gmon_+ASM2
oracle 8181 1 0 00:30 ? 00:00:00 asm_mmon_+ASM2
oracle 8185 1 0 00:30 ? 00:00:00 asm_mmnl_+ASM2
oracle 8189 1 0 00:31 ? 00:00:00 asm_lck0_+ASM2
oracle 8206 1 0 00:31 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8236 1 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmd.bin
oracle 8239 1 0 00:31 ? 00:00:00 oracle+ASM2_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8248 1 0 00:31 ? 00:00:00 asm_asmb_+ASM2
oracle 8252 1 0 00:31 ? 00:00:00 oracle+ASM2_asmb_+asm2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8326 8236 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmlogger.bin -o /u01/app/11.2.0/grid/evm/log/evmlogger.info -l /u01/app/11.2.0/grid/evm/log/evmlogger.log
oracle 8367 1 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 8408 1 0 00:31 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8431 1 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/opmn/bin/ons -d
oracle 8432 8431 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/opmn/bin/ons -d
oracle 9221 1 0 00:39 ? 00:00:00 asm_gcr0_+ASM2
root 9289 4030 0 00:40 pts/1 00:00:00 grep oracle
SQL> l
1* select * from v$asm_diskgroup
SQL> /
GROUP_NUMBER NAME SECTOR_SIZE BLOCK_SIZE
------------ ------------------------------ ----------- ----------
ALLOCATION_UNIT_SIZE STATE TYPE TOTAL_MB FREE_MB HOT_USED_MB
-------------------- ----------- ------ ---------- ---------- -----------
COLD_USED_MB REQUIRED_MIRROR_FREE_MB USABLE_FILE_MB OFFLINE_DISKS
------------ ----------------------- -------------- -------------
COMPATIBILITY
------------------------------------------------------------
DATABASE_COMPATIBILITY V
------------------------------------------------------------ -
1 DATA 512 4096
1048576 MOUNTED EXTERN 51180 50776 0
404 0 50776 0
11.2.0.0.0
10.1.0.0.0
References:
http://hiteshgondalia.wordpress.com/2013/03/10/crs-4046-invalid-oracle-clusterware-configuration-11gr2/
NOTE: attached to recovery domain 1
NOTE: starting recovery of thread=1 ckpt=32.568 group=1 (DATA)
Errors in file /u01/app/oracle/diag/asm/+asm/+ASM1/trace/+ASM1_ora_4561.trc (incident=55419):
ORA-00600: internal error code, arguments: [kfrValAcd30], [DATA], [1], [32], [568], [33], [569], [], [], [], [], []
Incident details in: /u01/app/oracle/diag/asm/+asm/+ASM1/incident/incdir_55419/+ASM1_ora_4561_i55419.tr
1. ASM disk information before I wipe them out.
[root@rac1 ~]# fdisk -l
Disk /dev/sda: 21.4 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sda1 * 1 13 104391 83 Linux
/dev/sda2 14 2610 20860402+ 8e Linux LVM
Disk /dev/sdb: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdb1 1 1305 10482381 83 Linux
Disk /dev/sdc: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdc1 1 1305 10482381 83 Linux
Disk /dev/sdd: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdd1 1 1305 10482381 83 Linux
Disk /dev/sde: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sde1 1 1305 10482381 83 Linux
Disk /dev/sdf: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdf1 1 1305 10482381 83 Linux
Disk /dev/sdg: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdg1 1 1305 10482381 83 Linux
Disk /dev/sdh: 21.4 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdh1 1 2610 20964793+ 83 Linux
Disk /dev/sdi: 8589 MB, 8589934592 bytes
255 heads, 63 sectors/track, 1044 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdi1 1 1044 8385898+ 83 Linux
Disk /dev/dm-0: 17.1 GB, 17112760320 bytes
255 heads, 63 sectors/track, 2080 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/dm-0 doesn't contain a valid partition table
Disk /dev/dm-1: 4227 MB, 4227858432 bytes
255 heads, 63 sectors/track, 514 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/dm-1 doesn't contain a valid partition table
Disk /dev/sda: 21.4 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sda1 * 1 13 104391 83 Linux
/dev/sda2 14 2610 20860402+ 8e Linux LVM
Disk /dev/sdb: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdb1 1 1305 10482381 83 Linux
Disk /dev/sdc: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdc1 1 1305 10482381 83 Linux
Disk /dev/sdd: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdd1 1 1305 10482381 83 Linux
Disk /dev/sde: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sde1 1 1305 10482381 83 Linux
Disk /dev/sdf: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdf1 1 1305 10482381 83 Linux
Disk /dev/sdg: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdg1 1 1305 10482381 83 Linux
Disk /dev/sdh: 21.4 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdh1 1 2610 20964793+ 83 Linux
Disk /dev/sdi: 8589 MB, 8589934592 bytes
255 heads, 63 sectors/track, 1044 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdi1 1 1044 8385898+ 83 Linux
Disk /dev/dm-0: 17.1 GB, 17112760320 bytes
255 heads, 63 sectors/track, 2080 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/dm-0 doesn't contain a valid partition table
Disk /dev/dm-1: 4227 MB, 4227858432 bytes
255 heads, 63 sectors/track, 514 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/dm-1 doesn't contain a valid partition table
[root@rac1 ~]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
FRA
[root@rac1 ~]# oracleasm querydisk -p DISK1
Disk "DISK1" is a valid ASM disk
/dev/sdb1: LABEL="DISK1" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p DISK2
Disk "DISK2" is a valid ASM disk
/dev/sdc1: LABEL="DISK2" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p DISK3
Disk "DISK3" is a valid ASM disk
/dev/sdd1: LABEL="DISK3" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p DISK4
Disk "DISK4" is a valid ASM disk
/dev/sde1: LABEL="DISK4" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p DISK5
Disk "DISK5" is a valid ASM disk
/dev/sdf1: LABEL="DISK5" TYPE="oracleasm"
[root@rac1 ~]# oracleasm querydisk -p FRA
Disk "FRA" is a valid ASM disk
/dev/sdg1: LABEL="FRA" TYPE="oracleasm"
2. Recreate ASM disks, which belong to two diskgroups: FRA (one disk) and DATA(5 disks).
SQL> alter diskgroup fra mount;
Diskgroup altered.
SQL> select * from v$asm_diskgroup;
GROUP_NUMBER NAME SECTOR_SIZE BLOCK_SIZE
------------ ------------------------------ ----------- ----------
ALLOCATION_UNIT_SIZE STATE TYPE TOTAL_MB FREE_MB HOT_USED_MB
-------------------- ----------- ------ ---------- ---------- -----------
COLD_USED_MB REQUIRED_MIRROR_FREE_MB USABLE_FILE_MB OFFLINE_DISKS
------------ ----------------------- -------------- -------------
COMPATIBILITY
------------------------------------------------------------
DATABASE_COMPATIBILITY V
------------------------------------------------------------ -
0 DATA 0 4096
0 DISMOUNTED 0 0 0
0 0 0 0
0.0.0.0.0
0.0.0.0.0 N
1 FRA 512 4096
1048576 MOUNTED EXTERN 10236 8533 0
1703 0 8533 0
11.2.0.0.0
10.1.0.0.0 N
SQL> select path,group_number from v$asm_disk;
PATH GROUP_NUMBER
------------------------------ ------------
ORCL:DISK1 0
ORCL:DISK2 0
ORCL:FRA 0
ORCL:DISK4 0
ORCL:DISK5 0
ORCL:DISK3 0
6 rows selected.
SQL> drop diskgroup fra force including contents;
Diskgroup dropped.
SQL> select * from v$asm_diskgroup;
GROUP_NUMBER NAME SECTOR_SIZE BLOCK_SIZE
------------ ------------------------------ ----------- ----------
ALLOCATION_UNIT_SIZE STATE TYPE TOTAL_MB FREE_MB HOT_USED_MB
-------------------- ----------- ------ ---------- ---------- -----------
COLD_USED_MB REQUIRED_MIRROR_FREE_MB USABLE_FILE_MB OFFLINE_DISKS
------------ ----------------------- -------------- -------------
COMPATIBILITY
------------------------------------------------------------
DATABASE_COMPATIBILITY V
------------------------------------------------------------ -
0 DATA 0 4096
0 DISMOUNTED 0 0 0
0 0 0 0
0.0.0.0.0
0.0.0.0.0 N
SQL> drop diskgroup DATA force including contents;
drop diskgroup DATA force including contents
*
ERROR at line 1:
ORA-15039: diskgroup not dropped
ORA-15276: ASM diskgroup DATA has cluster voting files
SQL> exit
Disconnected from Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - Production
With the Real Application Clusters and Automatic Storage Management options
[oracle@rac1 ~]$ oerr ora 15276]
[oracle@rac1 ~]$ oerr ora 15276
15276, 00000, "ASM diskgroup %s has cluster voting files"
// *Cause: An attempt was made to drop a diskgroup that contained cluster
// voting files.
// *Action: Move the cluster voting files out of the diskgroup and retry the
// operation.
[root@rac1 ~]# oracleasm deletedisk DISK1
Clearing disk header: failed
Unable to clear disk "DISK1"
[root@rac1 ~]# oracleasm deletedisk DISK2
Clearing disk header: done
Dropping disk: done
[root@rac1 ~]# oracleasm deletedisk DISK3
Clearing disk header: done
Dropping disk: done
[root@rac1 ~]# oracleasm deletedisk DISK4
Clearing disk header: done
Dropping disk: done
[root@rac1 ~]# oracleasm deletedisk DISK5
Clearing disk header: done
Dropping disk: done
[root@rac1 ~]# oracleasm createdisk DISK2 /dev/sdc1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac1 ~]# oracleasm listdisks
DISK1
DISK2
FRA
[root@rac1 ~]# oracleasm createdisk DISK3 /dev/sdd1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm createdisk DISK4 /dev/sde1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm createdisk DISK5 /dev/sdf1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# dd if=/dev/zero of=/dev/sdb1 bs=1024 count=4
4+0 records in
4+0 records out
4096 bytes (4.1 kB) copied, 4.3731e-05 seconds, 93.7 MB/s
[root@rac1 ~]# oracleasm deletedisk DISK1
Disk "DISK1" defines an unmarked device
Dropping disk: done
[root@rac1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac1 ~]# oracleasm listdisks
DISK2
DISK3
DISK4
DISK5
FRA
[root@rac1 ~]# oracleasm createdisk DISK1 /dev/sdb1
Unable to open device "/dev/sdb1": Device or resource busy
# and there are still HAS processes running.
[root@rac1 ~]# ps -ef |grep grid
root 3187 1 0 Mar20 ? 00:00:04 /u01/app/11.2.0/grid/bin/ohasd.bin reboot
oracle 3813 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 3827 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/mdnsd.bin
oracle 3838 1 0 Mar20 ? 00:00:06 /u01/app/11.2.0/grid/bin/gpnpd.bin
oracle 3850 1 0 Mar20 ? 00:00:02 /u01/app/11.2.0/grid/bin/gipcd.bin
root 3916 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/cssdmonitor
root 3931 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/cssdagent
oracle 3982 1 0 Mar20 ? 00:00:10 /u01/app/11.2.0/grid/bin/ocssd.bin
root 3995 1 0 Mar20 ? 00:00:14 /u01/app/11.2.0/grid/bin/orarootagent.bin
root 4009 1 0 Mar20 ? 00:00:14 /u01/app/11.2.0/grid/bin/osysmond.bin
oracle 4011 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/diskmon.bin -d -f
root 4215 1 0 Mar20 ? 00:00:13 /u01/app/11.2.0/grid/bin/ologgerd -M -d /u01/app/11.2.0/grid/crf/db/rac1
root 4267 1 0 Mar20 ? 00:00:01 /u01/app/11.2.0/grid/bin/octssd.bin reboot
oracle 4291 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmd.bin
root 13782 8695 0 00:21 pts/1 00:00:00 grep grid
[root@rac1 ~]# . ~oracle/grid_env
[root@rac1 ~]# crsctl stop crs
CRS-2796: The command may not proceed when Cluster Ready Services is not running
CRS-4687: Shutdown command has completed with errors.
CRS-4000: Command Stop failed, or completed with errors.
--after server reboot
[root@rac1 ~]# ps -ef |grep grid
root 3187 1 0 Mar20 ? 00:00:04 /u01/app/11.2.0/grid/bin/ohasd.bin reboot
oracle 3813 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 3827 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/mdnsd.bin
oracle 3838 1 0 Mar20 ? 00:00:06 /u01/app/11.2.0/grid/bin/gpnpd.bin
oracle 3850 1 0 Mar20 ? 00:00:02 /u01/app/11.2.0/grid/bin/gipcd.bin
root 3916 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/cssdmonitor
root 3931 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/cssdagent
oracle 3982 1 0 Mar20 ? 00:00:10 /u01/app/11.2.0/grid/bin/ocssd.bin
root 3995 1 0 Mar20 ? 00:00:14 /u01/app/11.2.0/grid/bin/orarootagent.bin
root 4009 1 0 Mar20 ? 00:00:14 /u01/app/11.2.0/grid/bin/osysmond.bin
oracle 4011 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/diskmon.bin -d -f
root 4215 1 0 Mar20 ? 00:00:13 /u01/app/11.2.0/grid/bin/ologgerd -M -d /u01/app/11.2.0/grid/crf/db/rac1
root 4267 1 0 Mar20 ? 00:00:01 /u01/app/11.2.0/grid/bin/octssd.bin reboot
oracle 4291 1 0 Mar20 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmd.bin
root 13782 8695 0 00:21 pts/1 00:00:00 grep grid
[root@rac1 ~]# . ~oracle/grid_env
[root@rac1 ~]# crsctl stop crs
CRS-2796: The command may not proceed when Cluster Ready Services is not running
CRS-4687: Shutdown command has completed with errors.
CRS-4000: Command Stop failed, or completed with errors.
--after server reboot
[root@rac1 ~]# oracleasm createdisk DISK1 /dev/sdb1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac1 ~]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
FRA
3. below processes are auto-up and running. I have to stop them ...
[root@rac1 client]# ps -ef |grep grid
root 3187 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/ohasd.bin reboot
oracle 3811 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 3825 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/mdnsd.bin
oracle 3836 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/gpnpd.bin
oracle 3925 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/gipcd.bin
root 4091 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/orarootagent.bin
root 4120 1 0 23:11 ? 00:00:01 /u01/app/11.2.0/grid/bin/osysmond.bin
root 4346 1 0 23:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/cssdmonitor
root 4360 1 0 23:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/cssdagent
oracle 4374 1 0 23:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/ocssd.bin
[root@rac1 init.d]# pwd
/etc/init.d
[root@rac1 init.d]# ls -l init*
-rwxr-xr-x 1 root root 8767 Mar 20 2011 init.ohasd
[root@rac1 init.d]# ./init.ohasd stop
[root@rac1 init.d]# crsctl stop crs
CRS-4544: Unable to connect to OHAS
CRS-4000: Command Stop failed, or completed with errors.
[root@rac1 client]# ps -ef |grep oracle
root 4508 4300 0 23:32 pts/2 00:00:00 grep oracle
# now all process are gone. alternatively, can kill them.
root 3187 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/ohasd.bin reboot
oracle 3811 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 3825 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/mdnsd.bin
oracle 3836 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/gpnpd.bin
oracle 3925 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/gipcd.bin
root 4091 1 0 23:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/orarootagent.bin
root 4120 1 0 23:11 ? 00:00:01 /u01/app/11.2.0/grid/bin/osysmond.bin
root 4346 1 0 23:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/cssdmonitor
root 4360 1 0 23:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/cssdagent
oracle 4374 1 0 23:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/ocssd.bin
[root@rac1 init.d]# pwd
/etc/init.d
[root@rac1 init.d]# ls -l init*
-rwxr-xr-x 1 root root 8767 Mar 20 2011 init.ohasd
[root@rac1 init.d]# ./init.ohasd stop
[root@rac1 init.d]# crsctl stop crs
CRS-4544: Unable to connect to OHAS
CRS-4000: Command Stop failed, or completed with errors.
[root@rac1 client]# ps -ef |grep oracle
root 4508 4300 0 23:32 pts/2 00:00:00 grep oracle
# now all process are gone. alternatively, can kill them.
4. deconfig clusterware services
[oracle@rac1 ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.2.0 Production on Tue Mar 26 23:47:27 2013
Copyright (c) 1982, 2010, Oracle. All rights reserved.
Connected to an idle instance.
SQL> startup nomount;
ORA-01078: failure in processing system parameters
ORA-29701: unable to connect to Cluster Synchronization Service
SQL> exit
Disconnected
[root@rac1 install]# ./rootcrs.pl -deconfig
Using configuration parameter file: ./crsconfig_params
Oracle Clusterware stack is not active on this node
Restart the clusterware stack (use /u01/app/11.2.0/grid/bin/crsctl start crs) and retry
Failed to verify resources
[root@rac1 install]# ./rootcrs.pl -deconfig -force
Using configuration parameter file: ./crsconfig_params
PRCR-1119 : Failed to look up CRS resources of ora.cluster_vip_net1.type type
PRCR-1068 : Failed to query resources
Cannot communicate with crsd
PRCR-1070 : Failed to check if resource ora.gsd is registered
Cannot communicate with crsd
PRCR-1070 : Failed to check if resource ora.ons is registered
Cannot communicate with crsd
ACFS-9200: Supported
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Stop failed, or completed with errors.
CRS-4544: Unable to connect to OHAS
CRS-4000: Command Stop failed, or completed with errors.
Successfully deconfigured Oracle clusterware stack on this node
[oracle@rac1 ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.2.0 Production on Tue Mar 26 23:47:27 2013
Copyright (c) 1982, 2010, Oracle. All rights reserved.
Connected to an idle instance.
SQL> startup nomount;
ORA-01078: failure in processing system parameters
ORA-29701: unable to connect to Cluster Synchronization Service
SQL> exit
Disconnected
[root@rac1 install]# ./rootcrs.pl -deconfig
Using configuration parameter file: ./crsconfig_params
Oracle Clusterware stack is not active on this node
Restart the clusterware stack (use /u01/app/11.2.0/grid/bin/crsctl start crs) and retry
Failed to verify resources
[root@rac1 install]# ./rootcrs.pl -deconfig -force
Using configuration parameter file: ./crsconfig_params
PRCR-1119 : Failed to look up CRS resources of ora.cluster_vip_net1.type type
PRCR-1068 : Failed to query resources
Cannot communicate with crsd
PRCR-1070 : Failed to check if resource ora.gsd is registered
Cannot communicate with crsd
PRCR-1070 : Failed to check if resource ora.ons is registered
Cannot communicate with crsd
ACFS-9200: Supported
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Stop failed, or completed with errors.
CRS-4544: Unable to connect to OHAS
CRS-4000: Command Stop failed, or completed with errors.
Successfully deconfigured Oracle clusterware stack on this node
[root@rac1 install]# ./roothas.pl -deconfig -force
Using configuration parameter file: ./crsconfig_params
CRS-4047: No Oracle Clusterware components configured.
CRS-4000: Command Stop failed, or completed with errors.
CRS-4047: No Oracle Clusterware components configured.
CRS-4000: Command Delete failed, or completed with errors.
CRS-4047: No Oracle Clusterware components configured.
CRS-4000: Command Stop failed, or completed with errors.
You must kill ohasd processes or reboot the system to properly
cleanup the processes started by Oracle clusterware
ACFS-9200: Supported
ACFS-9313: No ADVM/ACFS installation detected.
Either /etc/oracle/olr.loc does not exist or is not readable
Make sure the file exists and it has read and execute access
Failure in execution (rc=-1, 256, No such file or directory) for command 1 /etc/init.d/ohasd deinstall
Successfully deconfigured Oracle Restart stack
[root@rac1 install]# ps -ef |grep has
root 5651 4233 0 00:11 pts/1 00:00:00 grep has
--finally run the root.sh to reconfigure clusterware services
[root@rac1 grid]# ./root.sh
Running Oracle 11g root script...
The following environment variables are set as:
ORACLE_OWNER= oracle
ORACLE_HOME= /u01/app/11.2.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
The contents of "dbhome" have not changed. No need to overwrite.
The contents of "oraenv" have not changed. No need to overwrite.
The contents of "coraenv" have not changed. No need to overwrite.
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/app/11.2.0/grid/crs/install/crsconfig_params
LOCAL ADD MODE
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
OLR initialization - successful
Adding daemon to inittab
ACFS-9200: Supported
ACFS-9300: ADVM/ACFS distribution files found.
ACFS-9307: Installing requested ADVM/ACFS software.
ACFS-9308: Loading installed ADVM/ACFS drivers.
ACFS-9321: Creating udev for ADVM/ACFS.
ACFS-9323: Creating module dependencies - this may take some time.
ACFS-9327: Verifying ADVM/ACFS devices.
ACFS-9309: ADVM/ACFS installation correctness verified.
CRS-2672: Attempting to start 'ora.mdnsd' on 'rac1'
CRS-2676: Start of 'ora.mdnsd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rac1'
CRS-2676: Start of 'ora.diskmon' on 'rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rac1' succeeded
ASM created and started successfully.
Disk Group DATA created successfully.
clscfg: -install mode specified
Successfully accumulated necessary OCR keys.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
Successful addition of voting disk 55c34b4c06194f09bf8018170dab7627.
Successfully replaced voting disk group with +DATA.
CRS-4266: Voting file(s) successfully replaced
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 55c34b4c06194f09bf8018170dab7627 (ORCL:DISK1) [DATA]
Located 1 voting disk(s).
CRS-2672: Attempting to start 'ora.asm' on 'rac1'
CRS-2676: Start of 'ora.asm' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.DATA.dg' on 'rac1'
CRS-2676: Start of 'ora.DATA.dg' on 'rac1' succeeded
ACFS-9200: Supported
ACFS-9200: Supported
CRS-2672: Attempting to start 'ora.registry.acfs' on 'rac1'
CRS-2676: Start of 'ora.registry.acfs' on 'rac1' succeeded
Preparing packages for installation...
cvuqdisk-1.0.9-1
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
5. verification
[root@rac1 grid]# ps -ef |grep ASM
root 1825 4233 0 00:24 pts/1 00:00:00 grep ASM
oracle 13498 1 0 00:21 ? 00:00:00 asm_pmon_+ASM1
oracle 13504 1 0 00:21 ? 00:00:00 asm_psp0_+ASM1
oracle 13738 1 0 00:21 ? 00:00:00 asm_vktm_+ASM1
oracle 13762 1 0 00:21 ? 00:00:00 asm_gen0_+ASM1
oracle 13771 1 0 00:21 ? 00:00:00 asm_diag_+ASM1
oracle 13776 1 0 00:21 ? 00:00:00 asm_ping_+ASM1
oracle 13783 1 0 00:21 ? 00:00:00 asm_dia0_+ASM1
oracle 13790 1 0 00:21 ? 00:00:00 asm_lmon_+ASM1
oracle 13798 1 0 00:21 ? 00:00:00 asm_lmd0_+ASM1
oracle 13802 1 0 00:21 ? 00:00:00 asm_lms0_+ASM1
oracle 13811 1 0 00:21 ? 00:00:00 asm_lmhb_+ASM1
oracle 13820 1 0 00:21 ? 00:00:00 asm_mman_+ASM1
oracle 13830 1 0 00:21 ? 00:00:00 asm_dbw0_+ASM1
oracle 13838 1 0 00:21 ? 00:00:00 asm_lgwr_+ASM1
oracle 13843 1 0 00:21 ? 00:00:00 asm_ckpt_+ASM1
oracle 13852 1 0 00:21 ? 00:00:00 asm_smon_+ASM1
oracle 13860 1 0 00:21 ? 00:00:00 asm_rbal_+ASM1
oracle 13866 1 0 00:21 ? 00:00:00 asm_gmon_+ASM1
oracle 13872 1 0 00:21 ? 00:00:00 asm_mmon_+ASM1
oracle 13880 1 0 00:21 ? 00:00:00 asm_mmnl_+ASM1
oracle 13938 1 0 00:21 ? 00:00:00 asm_lck0_+ASM1
oracle 15325 1 0 00:21 ? 00:00:00 oracle+ASM1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15727 1 1 00:21 ? 00:00:01 oracle+ASM1_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15733 1 0 00:21 ? 00:00:00 asm_asmb_+ASM1
oracle 15743 1 0 00:21 ? 00:00:00 oracle+ASM1_asmb_+asm1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15754 1 0 00:21 ? 00:00:00 asm_o000_+ASM1
oracle 15762 1 0 00:21 ? 00:00:00 oracle+ASM1_o000_+asm1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 26710 1 0 00:22 ? 00:00:00 oracle+ASM1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
[root@rac1 grid]# ps -ef |grep oracle
root 1827 4233 0 00:24 pts/1 00:00:00 grep oracle
root 4812 4300 0 Mar26 pts/2 00:00:00 su - oracle
oracle 4813 4812 0 Mar26 pts/2 00:00:00 -bash
oracle 13498 1 0 00:21 ? 00:00:00 asm_pmon_+ASM1
oracle 13504 1 0 00:21 ? 00:00:00 asm_psp0_+ASM1
oracle 13738 1 0 00:21 ? 00:00:00 asm_vktm_+ASM1
oracle 13762 1 0 00:21 ? 00:00:00 asm_gen0_+ASM1
oracle 13771 1 0 00:21 ? 00:00:00 asm_diag_+ASM1
oracle 13776 1 0 00:21 ? 00:00:00 asm_ping_+ASM1
oracle 13783 1 0 00:21 ? 00:00:00 asm_dia0_+ASM1
oracle 13790 1 0 00:21 ? 00:00:00 asm_lmon_+ASM1
oracle 13798 1 0 00:21 ? 00:00:00 asm_lmd0_+ASM1
oracle 13802 1 0 00:21 ? 00:00:00 asm_lms0_+ASM1
oracle 13811 1 0 00:21 ? 00:00:00 asm_lmhb_+ASM1
oracle 13820 1 0 00:21 ? 00:00:00 asm_mman_+ASM1
oracle 13830 1 0 00:21 ? 00:00:00 asm_dbw0_+ASM1
oracle 13838 1 0 00:21 ? 00:00:00 asm_lgwr_+ASM1
oracle 13843 1 0 00:21 ? 00:00:00 asm_ckpt_+ASM1
oracle 13852 1 0 00:21 ? 00:00:00 asm_smon_+ASM1
oracle 13860 1 0 00:21 ? 00:00:00 asm_rbal_+ASM1
oracle 13866 1 0 00:21 ? 00:00:00 asm_gmon_+ASM1
oracle 13872 1 0 00:21 ? 00:00:00 asm_mmon_+ASM1
oracle 13880 1 0 00:21 ? 00:00:00 asm_mmnl_+ASM1
oracle 13938 1 0 00:21 ? 00:00:00 asm_lck0_+ASM1
oracle 15325 1 0 00:21 ? 00:00:00 oracle+ASM1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15720 1 0 00:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmd.bin
oracle 15727 1 0 00:21 ? 00:00:01 oracle+ASM1_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15733 1 0 00:21 ? 00:00:00 asm_asmb_+ASM1
oracle 15743 1 0 00:21 ? 00:00:00 oracle+ASM1_asmb_+asm1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 15754 1 0 00:21 ? 00:00:00 asm_o000_+ASM1
oracle 15762 1 0 00:21 ? 00:00:00 oracle+ASM1_o000_+asm1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 16222 15720 0 00:21 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmlogger.bin -o /u01/app/11.2.0/grid/evm/log/evmlogger.info -l /u01/app/11.2.0/grid/evm/log/evmlogger.log
oracle 26509 1 0 00:22 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 26710 1 0 00:22 ? 00:00:00 oracle+ASM1 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 30286 1 0 00:23 ? 00:00:00 /u01/app/11.2.0/grid/opmn/bin/ons -d
oracle 30287 30286 0 00:23 ? 00:00:00 /u01/app/11.2.0/grid/opmn/bin/ons -d
oracle 30801 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 30840 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/mdnsd.bin
oracle 31165 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/gpnpd.bin
oracle 31433 1 0 00:23 ? 00:00:00 /u01/app/11.2.0/grid/bin/tnslsnr LISTENER_SCAN1 -inherit
oracle 31558 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/gipcd.bin
oracle 31851 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/diskmon.bin -d -f
oracle 31923 1 0 00:20 ? 00:00:00 /u01/app/11.2.0/grid/bin/ocssd.bin
oracle 32011 1 0 00:23 ? 00:00:00 /u01/app/11.2.0/grid/bin/scriptagent.bin
oracle 32093 1 9 00:23 ? 00:00:07 /u01/app/11.2.0/grid/jdk/jre//bin/java -server -Xcheck:jni -Xms128M -Xmx384M -Djava.awt.headless=true -Ddisable.checkForUpdate=true -Dstdstream.filesize=100 -Dstdstream.filenumber=10 -DTRACING.ENABLED=false -Doracle.wlm.dbwlmlogger.logging.level=INFO -Dport.rmi=23792 -jar /u01/app/11.2.0/grid/oc4j/j2ee/home/oc4j.jar -config /u01/app/11.2.0/grid/oc4j/j2ee/home/OC4J_DBWLM_config/server.xml -out /u01/app/11.2.0/grid/oc4j/j2ee/home/log/oc4j.out -err /u01/app/11.2.0/grid/oc4j/j2ee/home/log/oc4j.err
SQL> set pages 1000
SQL> select * from v$asm_diskgroup;
GROUP_NUMBER NAME SECTOR_SIZE BLOCK_SIZE
------------ ------------------------------ ----------- ----------
ALLOCATION_UNIT_SIZE STATE TYPE TOTAL_MB FREE_MB HOT_USED_MB
-------------------- ----------- ------ ---------- ---------- -----------
COLD_USED_MB REQUIRED_MIRROR_FREE_MB USABLE_FILE_MB OFFLINE_DISKS
------------ ----------------------- -------------- -------------
COMPATIBILITY
------------------------------------------------------------
DATABASE_COMPATIBILITY V
------------------------------------------------------------ -
1 DATA 512 4096
1048576 MOUNTED EXTERN 51180 50776 0
404 0 50776 0
11.2.0.0.0
10.1.0.0.0 Y
6. do the same in node2
[root@rac2 install]# ls -l root*
-rwxr-xr-x 1 oracle oinstall 29259 Mar 20 2011 rootcrs.pl
-rwxr-xr-x 1 oracle oinstall 13522 Mar 20 2011 roothas.pl
-rwxr-xr-x 1 oracle oinstall 915 Mar 20 2011 rootofs.sh
[root@rac2 install]# ./roothas.pl -deconfig
Using configuration parameter file: ./crsconfig_params
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Stop failed, or completed with errors.
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Delete failed, or completed with errors.
CRS-2796: The command may not proceed when Cluster Ready Services is not running
CRS-4687: Shutdown command has completed with errors.
CRS-4000: Command Stop failed, or completed with errors.
You must kill ohasd processes or reboot the system to properly
cleanup the processes started by Oracle clusterware
ACFS-9200: Supported
Successfully deconfigured Oracle Restart stack
[root@rac2 install]# ./rootcrs.pl -deconfig
Using configuration parameter file: ./crsconfig_params
Oracle Clusterware stack is not active on this node
Restart the clusterware stack (use /u01/app/11.2.0/grid/bin/crsctl start crs) and retry
Failed to verify resources
[root@rac2 install]# which oracleasm
/usr/sbin/oracleasm
[root@rac2 install]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac2 install]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
FRA
[root@rac2 install]# cd ../..
[root@rac2 grid]# pwd
/u01/app/11.2.0/grid
[root@rac2 grid]# ./root.sh
Running Oracle 11g root script...
The following environment variables are set as:
ORACLE_OWNER= oracle
ORACLE_HOME= /u01/app/11.2.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
The contents of "dbhome" have not changed. No need to overwrite.
The contents of "oraenv" have not changed. No need to overwrite.
The contents of "coraenv" have not changed. No need to overwrite.
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/app/11.2.0/grid/crs/install/crsconfig_params
LOCAL ADD MODE
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
OLR initialization - successful
Adding daemon to inittab
ACFS-9200: Supported
ACFS-9300: ADVM/ACFS distribution files found.
ACFS-9307: Installing requested ADVM/ACFS software.
ACFS-9308: Loading installed ADVM/ACFS drivers.
ACFS-9321: Creating udev for ADVM/ACFS.
ACFS-9323: Creating module dependencies - this may take some time.
ACFS-9327: Verifying ADVM/ACFS devices.
ACFS-9309: ADVM/ACFS installation correctness verified.
CRS-4402: The CSS daemon was started in exclusive mode but found an active CSS daemon on node rac1, number 1, and is terminating
An active cluster was found during exclusive startup, restarting to join the cluster
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
[root@rac2 grid]# ps -ef |grep pmon
oracle 8105 1 0 00:30 ? 00:00:00 asm_pmon_+ASM2
root 9282 4030 0 00:40 pts/1 00:00:00 grep pmon
[root@rac2 grid]# ps -ef |grep ASM2
oracle 8105 1 0 00:30 ? 00:00:00 asm_pmon_+ASM2
oracle 8109 1 0 00:30 ? 00:00:00 asm_psp0_+ASM2
oracle 8113 1 0 00:30 ? 00:00:00 asm_vktm_+ASM2
oracle 8119 1 0 00:30 ? 00:00:00 asm_gen0_+ASM2
oracle 8123 1 0 00:30 ? 00:00:00 asm_diag_+ASM2
oracle 8127 1 0 00:30 ? 00:00:00 asm_ping_+ASM2
oracle 8131 1 0 00:30 ? 00:00:00 asm_dia0_+ASM2
oracle 8135 1 0 00:30 ? 00:00:00 asm_lmon_+ASM2
oracle 8139 1 0 00:30 ? 00:00:00 asm_lmd0_+ASM2
oracle 8143 1 0 00:30 ? 00:00:00 asm_lms0_+ASM2
oracle 8149 1 0 00:30 ? 00:00:00 asm_lmhb_+ASM2
oracle 8153 1 0 00:30 ? 00:00:00 asm_mman_+ASM2
oracle 8157 1 0 00:30 ? 00:00:00 asm_dbw0_+ASM2
oracle 8161 1 0 00:30 ? 00:00:00 asm_lgwr_+ASM2
oracle 8165 1 0 00:30 ? 00:00:00 asm_ckpt_+ASM2
oracle 8169 1 0 00:30 ? 00:00:00 asm_smon_+ASM2
oracle 8173 1 0 00:30 ? 00:00:00 asm_rbal_+ASM2
oracle 8177 1 0 00:30 ? 00:00:00 asm_gmon_+ASM2
oracle 8181 1 0 00:30 ? 00:00:00 asm_mmon_+ASM2
oracle 8185 1 0 00:30 ? 00:00:00 asm_mmnl_+ASM2
oracle 8189 1 0 00:31 ? 00:00:00 asm_lck0_+ASM2
oracle 8206 1 0 00:31 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8239 1 0 00:31 ? 00:00:00 oracle+ASM2_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8248 1 0 00:31 ? 00:00:00 asm_asmb_+ASM2
oracle 8252 1 0 00:31 ? 00:00:00 oracle+ASM2_asmb_+asm2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8408 1 0 00:31 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 9221 1 0 00:39 ? 00:00:00 asm_gcr0_+ASM2
root 9286 4030 0 00:40 pts/1 00:00:00 grep ASM2
[root@rac2 grid]# ps -ef |grep oracle
oracle 7800 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 7814 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/mdnsd.bin
oracle 7830 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/gpnpd.bin
oracle 7855 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/gipcd.bin
oracle 7907 1 0 00:30 ? 00:00:01 /u01/app/11.2.0/grid/bin/ocssd.bin
oracle 7910 1 0 00:30 ? 00:00:00 /u01/app/11.2.0/grid/bin/diskmon.bin -d -f
oracle 8105 1 0 00:30 ? 00:00:00 asm_pmon_+ASM2
oracle 8109 1 0 00:30 ? 00:00:00 asm_psp0_+ASM2
oracle 8113 1 0 00:30 ? 00:00:00 asm_vktm_+ASM2
oracle 8119 1 0 00:30 ? 00:00:00 asm_gen0_+ASM2
oracle 8123 1 0 00:30 ? 00:00:00 asm_diag_+ASM2
oracle 8127 1 0 00:30 ? 00:00:00 asm_ping_+ASM2
oracle 8131 1 0 00:30 ? 00:00:00 asm_dia0_+ASM2
oracle 8135 1 0 00:30 ? 00:00:00 asm_lmon_+ASM2
oracle 8139 1 0 00:30 ? 00:00:00 asm_lmd0_+ASM2
oracle 8143 1 0 00:30 ? 00:00:00 asm_lms0_+ASM2
oracle 8149 1 0 00:30 ? 00:00:00 asm_lmhb_+ASM2
oracle 8153 1 0 00:30 ? 00:00:00 asm_mman_+ASM2
oracle 8157 1 0 00:30 ? 00:00:00 asm_dbw0_+ASM2
oracle 8161 1 0 00:30 ? 00:00:00 asm_lgwr_+ASM2
oracle 8165 1 0 00:30 ? 00:00:00 asm_ckpt_+ASM2
oracle 8169 1 0 00:30 ? 00:00:00 asm_smon_+ASM2
oracle 8173 1 0 00:30 ? 00:00:00 asm_rbal_+ASM2
oracle 8177 1 0 00:30 ? 00:00:00 asm_gmon_+ASM2
oracle 8181 1 0 00:30 ? 00:00:00 asm_mmon_+ASM2
oracle 8185 1 0 00:30 ? 00:00:00 asm_mmnl_+ASM2
oracle 8189 1 0 00:31 ? 00:00:00 asm_lck0_+ASM2
oracle 8206 1 0 00:31 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8236 1 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmd.bin
oracle 8239 1 0 00:31 ? 00:00:00 oracle+ASM2_ocr (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8248 1 0 00:31 ? 00:00:00 asm_asmb_+ASM2
oracle 8252 1 0 00:31 ? 00:00:00 oracle+ASM2_asmb_+asm2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8326 8236 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/bin/evmlogger.bin -o /u01/app/11.2.0/grid/evm/log/evmlogger.info -l /u01/app/11.2.0/grid/evm/log/evmlogger.log
oracle 8367 1 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/bin/oraagent.bin
oracle 8408 1 0 00:31 ? 00:00:00 oracle+ASM2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
oracle 8431 1 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/opmn/bin/ons -d
oracle 8432 8431 0 00:31 ? 00:00:00 /u01/app/11.2.0/grid/opmn/bin/ons -d
oracle 9221 1 0 00:39 ? 00:00:00 asm_gcr0_+ASM2
root 9289 4030 0 00:40 pts/1 00:00:00 grep oracle
SQL> l
1* select * from v$asm_diskgroup
SQL> /
GROUP_NUMBER NAME SECTOR_SIZE BLOCK_SIZE
------------ ------------------------------ ----------- ----------
ALLOCATION_UNIT_SIZE STATE TYPE TOTAL_MB FREE_MB HOT_USED_MB
-------------------- ----------- ------ ---------- ---------- -----------
COLD_USED_MB REQUIRED_MIRROR_FREE_MB USABLE_FILE_MB OFFLINE_DISKS
------------ ----------------------- -------------- -------------
COMPATIBILITY
------------------------------------------------------------
DATABASE_COMPATIBILITY V
------------------------------------------------------------ -
1 DATA 512 4096
1048576 MOUNTED EXTERN 51180 50776 0
404 0 50776 0
11.2.0.0.0
10.1.0.0.0
References:
http://hiteshgondalia.wordpress.com/2013/03/10/crs-4046-invalid-oracle-clusterware-configuration-11gr2/
