orarac2poc:ORCL2:/u01/app/oracle/oms12c/gc_inst/user_projects/domains/GCDomain/servers/EMGC_OMS1/logs> tail -100 EMGC_OMS1.out
...
ORA-01653: unable to extend table SYS.AUD$ by 128 in tablespace SYSTEM
ORA-02002: error while writing to audit trail
ORA-00604: error occurred at recursive SQL level 1
ORA-01653: unable to extend table SYS.AUD$ by 128 in tablespace SYSTEM
Error Code: 0.>
2. Confirm the error at database side
orarac2poc:RCAT:/home/oracle/scripts/performance> bdump
orarac2poc:RCAT:/u01/app/oracle/diag/rdbms/rcat/RCAT/trace> tail -100 alert_RCAT.log
ORA-1653: unable to extend table SYS.AUD$ by 128 in tablespace SYSTEM
ORA-1653: unable to extend table SYS.AUD$ by 128 in tablespace SYSTEM
ORA-1653: unable to extend table SYS.AUD$ by 128 in tablespace SYSTEM
Wed Apr 10 09:47:47 2013
orarac2poc:ORCL2:/u01/app/oracle/oms12c/gc_inst/user_projects/domains/GCDomain/servers/EMGC_OMS1/logs> rcat
Disconnected from Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Data Mining and Real Application Testing options
orarac2poc:RCAT:/u01/app/oracle/oms12c/gc_inst/user_projects/domains/GCDomain/servers/EMGC_OMS1/logs> perf
orarac2poc:RCAT:/home/oracle/scripts/performance> sqlplus / as sysdba
SQL*Plus: Release 11.2.0.2.0 Production on Wed Apr 10 09:22:39 2013
Copyright (c) 1982, 2010, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Data Mining and Real Application Testing options
SYS@RCAT> @space_info
MGMT_AD4J_TS , 198.4375 , 200 , 99.2188
UNDOTBS1 , 536.4375 , 590 , 90.9216
RMAN_TBS , 88.25 , 100 , 88.2500
USERS , 4 , 5 , 80.0000
TTS1 , 11 , 20 , 55.0000
MGMT_TABLESPACE , 147 , 4950 , 2.9697
SYSAUX , 38.6875 , 1480 , 2.6140
SYSTEM , .125 , 870 , .0144
MGMT_ECM_DEPOT_TS , .0625 , 640 , .0098
SYS@RCAT> select * from dba_data_files ;
+DGRCAT/rcat/datafile/mgmt_ecm_depot_ts.273.804790907
6 MGMT_ECM_DEPOT_TS 671088640 81920 AVAILABLE 6 YES 3.4360E+10 4194302 2560 670040064 81792 ONLINE
+DGRCAT/rcat/datafile/mgmt_tablespace.272.804790907
7 MGMT_TABLESPACE 5190451200 633600 AVAILABLE 7 YES 3.4360E+10 4194302 6400 5189402624 633472 ONLINE
+DGRCAT/rcat/datafile/mgmt_ad4j_ts.271.804790909
8 MGMT_AD4J_TS 209715200 25600 AVAILABLE 8 YES 3.4360E+10 4194302 6400 208666624 25472 ONLINE
+DGRCAT/rcat/datafile/system.261.803153779
1 SYSTEM 912261120 111360 AVAILABLE 1 YES 3.4360E+10 4194302 1280 911212544 111232 SYSTEM
+DGRCAT/rcat/datafile/sysaux.262.803153789
2 SYSAUX 1551892480 189440 AVAILABLE 2 YES 3.4360E+10 4194302 1280 1550843904 189312 ONLINE
+DGRCAT/rcat/datafile/undotbs1.263.803153795
3 UNDOTBS1 618659840 75520 AVAILABLE 3 YES 3.4360E+10 4194302 640 617611264 75392 ONLINE
+DGRCAT/rcat/datafile/users.265.803153807
4 USERS 5242880 640 AVAILABLE 4 YES 3.4360E+10 4194302 160 4194304 512 ONLINE
+DGRCAT/rcat/datafile/rman_tbs01.dbf
5 RMAN_TBS 104857600 12800 AVAILABLE 5 NO 0 0 0 103809024 12672 ONLINE
+DGRCAT/rcat/datafile/tts1_01.dbf
9 TTS1 20971520 2560 AVAILABLE 5 NO 0 0 0 19922944 2432 ONLINE
SYS@RCAT> alter database datafile '+DGRCAT/rcat/datafile/mgmt_ecm_depot_ts.273.804790907' resize 700m;
alter database datafile '+DGRCAT/rcat/datafile/mgmt_ecm_depot_ts.273.804790907' resize 700m
*
ERROR at line 1:
ORA-01237: cannot extend datafile 6
ORA-01110: data file 6: '+DGRCAT/rcat/datafile/mgmt_ecm_depot_ts.273.804790907'
ORA-17505: ksfdrsz:1 Failed to resize file to size 89600 blocks
ORA-15041: diskgroup "DGRCAT" space exhausted
SYS@RCAT> alter database datafile '+DGRCAT/rcat/datafile/system.261.803153779' resize 880m;
alter database datafile '+DGRCAT/rcat/datafile/system.261.803153779' resize 880m
*
ERROR at line 1:
ORA-01237: cannot extend datafile 1
ORA-01110: data file 1: '+DGRCAT/rcat/datafile/system.261.803153779'
ORA-17505: ksfdrsz:1 Failed to resize file to size 112640 blocks
ORA-15041: diskgroup "DGRCAT" space exhausted
SYS@RCAT> exit
orarac2poc:DGS:/home/oracle/scripts/performance> su - grid
Password:
[grid@orarac2poc ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.2.0 Production on Wed Apr 10 09:30:42 2013
Copyright (c) 1982, 2010, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
SQL> select name, total_mb, free_mb from v$asm_diskgroup;
NAME TOTAL_MB FREE_MB
------------------------------ ---------- ----------
DGDATA 10239 7049
DGFRA 10239 5314
OCR 2046 1650
DGRCAT 10239 0
SQL> exit
Disconnected from Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
2. Attemp to fix using asmca, but asmca can only create new diskgroup. Can't use it to add new disk to diskgroup, this is out of surprise to me. Hope more functions can be introduced in future version.
[grid@orarac2poc ~]$ which asmca
/u01/app/11.2.0/grid/bin/asmca
3. Create ASM disk first.
[grid@orarac2poc ~]$ su -
Password:
[root@orarac2poc ~]# which oracleasm
/usr/sbin/oracleasm
[root@orarac2poc ~]# oracleasm listdisks
OCRVOL
RACDATA
RCATV01
RFRA01
[root@orarac2poc ~]# fdisk -l
Disk /dev/sda: 21.4 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sda1 * 1 13 104391 83 Linux
/dev/sda2 14 2610 20860402+ 8e Linux LVM
Disk /dev/sdb: 37.5 GB, 37580963840 bytes
255 heads, 63 sectors/track, 4568 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdb1 1 4568 36692428+ 83 Linux
Disk /dev/sdc: 10.7 GB, 10737418240 bytes
64 heads, 32 sectors/track, 10240 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdc1 1 10240 10485744 83 Linux
Disk /dev/sdd: 10.7 GB, 10737418240 bytes
64 heads, 32 sectors/track, 10240 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdd1 1 10240 10485744 83 Linux
Disk /dev/sde: 10.7 GB, 10737418240 bytes
64 heads, 32 sectors/track, 10240 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sde1 1 10240 10485744 83 Linux
Disk /dev/sdf: 10.7 GB, 10737418240 bytes
64 heads, 32 sectors/track, 10240 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdf1 1 10240 10485744 83 Linux
Disk /dev/sdg: 10.7 GB, 10737418240 bytes
64 heads, 32 sectors/track, 10240 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdg1 1 10240 10485744 83 Linux
Disk /dev/sdh: 2147 MB, 2147483648 bytes
67 heads, 62 sectors/track, 1009 cylinders
Units = cylinders of 4154 * 512 = 2126848 bytes
Device Boot Start End Blocks Id System
/dev/sdh1 1 1009 2095662 83 Linux
-- check my document, so I know which disk is still not in use.
[grid@orarac2poc ~]$ su -
Password:
[root@orarac2poc ~]# oracleasm listdisks
OCRVOL
RACDATA
RCATV01
RFRA01
--to double check to ensure my document is update-to-date.
[root@orarac2poc ~]# oracleasm querydisk -d RCATV01
Disk "RCATV01" is a valid ASM disk on device /dev/sde1[8,65]
[root@orarac2poc ~]# oracleasm querydisk -d OCRVOL
Disk "OCRVOL" is a valid ASM disk on device /dev/sdh1[8,113]
[root@orarac2poc ~]# oracleasm querydisk -d RACDATA
Disk "RACDATA" is a valid ASM disk on device /dev/sdc1[8,33]
[root@orarac2poc ~]# oracleasm querydisk -d RFRA01
Disk "RFRA01" is a valid ASM disk on device /dev/sdd1[8,49]
[root@orarac2poc ~]# exit
logout
-- see what happens if we don't create ASM disk first.
[grid@orarac2poc ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.2.0 Production on Wed Apr 10 09:40:27 2013
Copyright (c) 1982, 2010, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
SQL> alter diskgroup DGRCAT add disk '/dev/sdf1';
alter diskgroup DGRCAT add disk '/dev/sdf1'
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15031: disk specification '/dev/sdf1' matches no disks
ORA-15025: could not open disk "/dev/sdf1"
ORA-15056: additional error message
Linux-x86_64 Error: 13: Permission denied
Additional information: 42
Additional information: 1962056608
Additional information: 1962056896
SQL> exit
Disconnected from Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
[grid@orarac2poc ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.2.0 Production on Wed Apr 10 09:42:28 2013
Copyright (c) 1982, 2010, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
SQL> select GROUP_NUMBER,name from v$asm_diskgroup;
GROUP_NUMBER NAME
------------ ------------------------------
1 DGDATA
2 DGFRA
3 OCR
4 DGRCAT
SQL> select name, label, path from v$asm_disk where GROUP_NUMBER=4;
NAME LABEL
------------------------------ -------------------------------
PATH
--------------------------------------------------------------------------------
RCATV01 RCATV01
ORCL:RCATV01
SQL> exit
Disconnected from Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
-- seems we need to use oracleasm to create it first.
[grid@orarac2poc ~]$ su -
Password:
[root@orarac2poc ~]# oracleasm createdisk RCATV02 /dev/sdf1
Writing disk header: done
Instantiating disk: done
[root@orarac2poc ~]# oracleasm listdisks
OCRVOL
RACDATA
RCATV01
RCATV02
RFRA01
[root@orarac2poc ~]# ls -l /dev/oracleasm/
total 0
drwxr-xr-x 1 root root 0 Feb 28 01:03 disks
drwxrwx--- 1 grid asmdba 0 Feb 28 01:03 iid
[root@orarac2poc ~]# ls -l /dev/oracleasm/disks/
total 0
brw-rw---- 1 grid asmdba 8, 113 Feb 28 01:03 OCRVOL
brw-rw---- 1 grid asmdba 8, 33 Feb 28 01:03 RACDATA
brw-rw---- 1 grid asmdba 8, 65 Feb 28 01:03 RCATV01
brw-rw---- 1 grid asmdba 8, 81 Apr 10 09:45 RCATV02
brw-rw---- 1 grid asmdba 8, 49 Feb 28 01:03 RFRA01
[root@orarac2poc ~]# exit
logout
4. Now add the new disk to diskgroup
[grid@orarac2poc ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.2.0 Production on Wed Apr 10 09:47:39 2013
Copyright (c) 1982, 2010, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
SQL> alter diskgroup DGRCAT add disk 'ORCL:RCATV02';
Diskgroup altered.
SQL> select name, label, path from v$asm_disk where GROUP_NUMBER=4;
NAME LABEL
------------------------------ -------------------------------
PATH
--------------------------------------------------------------------------------
RCATV01 RCATV01
ORCL:RCATV01
RCATV02 RCATV02
ORCL:RCATV02
SQL> select name, total_mb, free_mb from v$asm_diskgroup;
NAME TOTAL_MB FREE_MB
------------------------------ ---------- ----------
DGDATA 10239 7049
DGFRA 10239 5314
OCR 2046 1650
DGRCAT 20478 10207
SQL> exit
Disconnected from Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
[grid@orarac2poc ~]$ exit
logout
orarac2poc:DGS:/home/oracle/scripts/performance> rcat
orarac2poc:RCAT:/home/oracle/scripts/performance> sqlplus / as sysdba
SQL*Plus: Release 11.2.0.2.0 Production on Wed Apr 10 09:48:53 2013
Copyright (c) 1982, 2010, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Data Mining and Real Application Testing options
SYS@RCAT> @space_info
MGMT_AD4J_TS , 198.4375 , 200 , 99.2188
RMAN_TBS , 88.25 , 100 , 88.2500
UNDOTBS1 , 513.375 , 590 , 87.0127
USERS , 4 , 5 , 80.0000
TTS1 , 11 , 20 , 55.0000
MGMT_TABLESPACE , 138.75 , 4950 , 2.8030
SYSAUX , 38.6875 , 1480 , 2.6140
MGMT_ECM_DEPOT_TS , 12 , 660 , 1.8182
SYSTEM , 9.125 , 880 , 1.0369
SYS@RCAT> exit
Disconnected from Oracle Database 11g Enterprise Edition Release 11.2.0.2.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Data Mining and Real Application Testing options
Wed Apr 10 09:48:03 2013
SQL> alter diskgroup DGRCAT add disk 'ORCL:RCATV02'
NOTE: Assigning number (4,1) to disk (ORCL:RCATV02)
NOTE: requesting all-instance membership refresh for group=4
NOTE: initializing header on grp 4 disk RCATV02
NOTE: requesting all-instance disk validation for group=4
Wed Apr 10 09:48:05 2013
NOTE: skipping rediscovery for group 4/0xad3178ac (DGRCAT) on local instance.
NOTE: requesting all-instance disk validation for group=4
NOTE: skipping rediscovery for group 4/0xad3178ac (DGRCAT) on local instance.
NOTE: initiating PST update: grp = 4
Wed Apr 10 09:48:05 2013
GMON updating group 4 at 14 for pid 35, osid 26052
NOTE: PST update grp = 4 completed successfully
NOTE: membership refresh pending for group 4/0xad3178ac (DGRCAT)
GMON querying group 4 at 15 for pid 18, osid 16512
NOTE: cache opening disk 1 of grp 4: RCATV02 label:RCATV02
GMON querying group 4 at 16 for pid 18, osid 16512
SUCCESS: refreshed membership for 4/0xad3178ac (DGRCAT)
SUCCESS: alter diskgroup DGRCAT add disk 'ORCL:RCATV02'
5. Rebalancing after a while (5 minutes).
SQL> select group_number , name,path, total_mb,free_mb from v$asm_disk where group_number=4;
GROUP_NUMBER NAME
------------ ------------------------------
PATH
--------------------------------------------------------------------------------
TOTAL_MB FREE_MB
---------- ----------
4 RCATV01
ORCL:RCATV01
10239 4961
4 RCATV02
ORCL:RCATV02
10239 4966
NOTE: starting rebalance of group 4/0xad3178ac (DGRCAT) at power 1
Starting background process ARB0
Wed Apr 10 09:48:08 2013
ARB0 started with pid=38, OS id=26115
NOTE: assigning ARB0 to group 4/0xad3178ac (DGRCAT) with 1 parallel I/O
NOTE: Attempting voting file refresh on diskgroup DGRCAT
Wed Apr 10 09:51:07 2013
NOTE: GroupBlock outside rolling migration privileged region
NOTE: requesting all-instance membership refresh for group=4
Wed Apr 10 09:51:10 2013
NOTE: membership refresh pending for group 4/0xad3178ac (DGRCAT)
Wed Apr 10 09:51:13 2013
GMON querying group 4 at 17 for pid 18, osid 16512
SUCCESS: refreshed membership for 4/0xad3178ac (DGRCAT)
NOTE: Attempting voting file refresh on diskgroup DGRCAT
Wed Apr 10 09:52:58 2013
NOTE: stopping process ARB0
SUCCESS: rebalance completed for group 4/0xad3178ac (DGRCAT)
