在青云上部署oraclerac全过程

1.1服务器基本信息

[root@db02 software]# cat /etc/issue

Oracle Linux Server release 6.6

Kernel \r on an \m

[root@db02 software]# uname -a

Linux db02 3.8.13-55.1.2.el6uek.x86_64 #2 SMP Thu Dec 18 00:15:51 PST 2014 x86_64 x86_64 x86_64 GNU/Linux

1.2软件信息

Db11.2.0.4：p13390677_112040_Linux-x86-64_1of7.zip

p13390677_112040_Linux-x86-64_2of7.zip

grid：      p13390677_112040_Linux-x86-64_3of7.zip 

安装介质存放在/u01/software中

p13390677_112040_Linux-x86-64_1of7.zip、p13390677_112040_Linux-x86-64_2of7.zip解压生成datebase目录；

p13390677_112040_Linux-x86-64_3of7.zip解压生成grid目录

1.3本地盘配置

将500GB的本地盘创建文件系统挂载到/u01目录下面，并修改/etc/fstab，让开机自动挂载/u01文件系统。

[root@db01 ~]# df -h

Filesystem     Size Used Avail Use% Mounted on

/dev/vda1       20G 3.0G  16G 16% /

tmpfs          8.0G 132M 7.9G  2% /dev/shm

/dev/vdc1      493G  23G 445G  5% /u01

二 基础配置

基础配置需要在两个节点都要操作的。

2.1网络配置

2.1.1 Host文件

#Public

192.168.0.2 db01

192.168.0.3 db02

#Virtual

192.168.0.102 db01-vip

192.168.0.103 db02-vip

#Private

192.168.101.2 db01-priv

192.168.101.3 db02-priv

#SCAN

192.168.0.100 db-scan

2.1.2网络配置文件

节点1

[root@db01 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0

HWADDR=52:54:46:A6:94:40

TYPE=Ethernet

BOOTPROTO=none

IPADDR=192.168.101.2

PREFIX=24

DEFROUTE=yes

IPV4_FAILURE_FATAL=yes

IPV6INIT=no

NAME=eth0

UUID=d790a693-882a-4b5e-bc9f-44ac2706c5b7

ONBOOT=yes

[root@db01 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth2

DEVICE=eth2

TYPE=Ethernet

IPADDR=192.168.0.2

NETMASK=255.255.255.0

GATEWAY=192.168.0.1

DEFROUTE=yes

NAME=eth2

ONBOOT=yes

HWADDR=52:54:a1:2a:34:3f

[root@db01 ~]#

节点2

[root@db02 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0

HWADDR=52:54:72:2C:50:63

TYPE=Ethernet

BOOTPROTO=none

IPADDR=192.168.101.3

PREFIX=24

DEFROUTE=yes

IPV4_FAILURE_FATAL=yes

IPV6INIT=no

NAME=eth0

UUID=5c37a16a-e9ac-4cf6-a244-c4d0faa592db

ONBOOT=yes

DEVICE=eth0

[root@db02 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth2

DEVICE=eth2

HWADDR=52:54:b0:c6:97:26

TYPE=Ethernet

IPADDR=192.168.0.3

NETMASK=255.255.255.0

GATEWAY=192.168.0.1

DEFROUTE=yes

NAME=eth2

ONBOOT=yes

两个节点都要操作：

关闭networkmanager

[root@i-zq8ytr3k network-scripts]# chkconfig NetworkManager --list

NetworkManager 0:off  1:off  2:on   3:on   4:on   5:on   6:off

[root@i-zq8ytr3k network-scripts]# chkconfig NetworkManager off

[root@i-zq8ytr3k network-scripts]# chkconfig NetworkManager --list

NetworkManager 0:off  1:off  2:off  3:off  4:off  5:off  6:off

重启网络：

service network restart

关闭防火墙

service iptables stop

chkconfig iptables off

关闭selinux

修改：/etc/selinux/config内容改为：SELINUX=disabled

setenforce 0

2.2创建用户

两个节点都要操作：

groupadd -g 500 dba

 groupadd -g 501 oinstall

 groupadd -g 502 asmadmin

 useradd -g oinstall -u 500 oracle

 useradd -g oinstall -u 501 grid

 usermod -G dba oracle

 usermod -G dba grid

 usermod -aG asmadmin grid

[root@db02 software]# id oracle

uid=500(oracle) gid=501(oinstall)组=501(oinstall),500(dba)

[root@db02 software]# id grid

uid=501(grid) gid=501(oinstall)组=501(oinstall),500(dba),502(asmadmin)

[root@db01 ~]# id oracle

uid=500(oracle) gid=501(oinstall) groups=501(oinstall),500(dba)

[root@db01 ~]# id grid

uid=501(grid) gid=501(oinstall) groups=501(oinstall),500(dba),502(asmadmin)

[root@db01 ~]#

2.2.1设置profile

后续要将oracle的环境变量ORACLE_SID分别改为oadb1和oadb2

节点1：

Oracle用户

[oracle@db01 ~]$ cat .bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

       . ~/.bashrc

fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

TMP=/tmp; export TMP   

TMPDIR=$TMP; export TMPDIR

ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE   

ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1; export ORACLE_HOME   

ORACLE_SID=db1; export ORACLE_SID   

ORACLE_TERM=xterm; export ORACLE_TERM

PATH=/usr/sbin:$PATH; export PATH

PATH=$ORACLE_HOME/bin:$PATH; export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib; export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib;

grid用户：

[grid@db01 ~]$ cat .bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

       . ~/.bashrc

fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

TMP=/tmp; export TMP   

TMPDIR=$TMP; export TMPDIR

ORACLE_SID=+ASM1; export ORACLE_SID   

ORACLE_BASE=/u01/app/grid; export ORACLE_BASE

ORACLE_HOME=/u01/app/11.2.0/grid/product/db_1; export ORACLE_HOME

NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT

THREADS_FLAG=native; export THREADS_FLAG

PATH=$ORACLE_HOME/bin:$PATH; export PATHTHREADS_FLAG=native; export THREADS_FLAG

PATH=$ORACLE_HOME/bin:$PATH; export PATH

节点2

Oracle用户：

[oracle@db02 ~]$ cat .bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

       . ~/.bashrc

fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

TMP=/tmp; export TMP   

TMPDIR=$TMP; export TMPDIR

ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE   

ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1; export ORACLE_HOME   

ORACLE_SID=db2; export ORACLE_SID   

ORACLE_TERM=xterm; export ORACLE_TERM

PATH=/usr/sbin:$PATH; export PATH

PATH=$ORACLE_HOME/bin:$PATH; export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib; export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib;

Grid用户：

[grid@db02 ~]$ cat .bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

       . ~/.bashrc

fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

TMP=/tmp; export TMP   

TMPDIR=$TMP; export TMPDIR

ORACLE_SID=+ASM2; export ORACLE_SID   

ORACLE_BASE=/u01/app/grid; export ORACLE_BASE

ORACLE_HOME=/u01/app/11.2.0/grid/product/db_1; export ORACLE_HOME

NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT

THREADS_FLAG=native; export THREADS_FLAG

PATH=$ORACLE_HOME/bin:$PATH; export PATHTHREADS_FLAG=native; export THREADS_FLAG

PATH=$ORACLE_HOME/bin:$PATH; export PATH

2.3配置路径

两台服务器都要配置：

mkdir -p /u01/app

 chown -R root:oinstall /u01/app

 mkdir -p /u01/app/11.2.0/ 

 mkdir -p /u01/app/grid/ 

 mkdir -p /u01/app/oracle/

 chown -R grid:oinstall /u01/app/11.2.0

 chown -R grid:oinstall /u01/app/grid

 chown -R oracle:oinstall /u01/app/oracle

2.4安装rpm包

配置本地镜像

将iso文件挂载

mount -t iso9660 -o loop /u01/software/V52218-01-OEL-6.6.iso /u01/software/mnt

重命名之前的文件

Mv /etc/yum.repos.d/public-yum-ol6.repo /etc/yum.repos.d/public-yum-ol6.repo_bak

[root@db02 ~]# cat /etc/yum.repos.d/public-yum-ol6.repo

[Server]

name=Oracle Linux $releasever Beta - $basearch - Server

baseurl=file:///u01/software/mnt

enabled=1

gpgcheck=1

gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle

安装rpm

yum install gcc compat-libstdc++-33 elfutils-libelf-devel gcc-c++ libaio-devel libstdc++-devel ksh -y

后续根据实际情况也许需要安装一些新的包，都可以在/u01/software/mnt/Packages中找到

2.5 sysctl.conf文件

修改/etc/sysctl.conf

新加

##add by oracle

kernel.shmmax = 4294967295

kernel.shmall = 2097152

kernel.shmmni = 4096

kernel.sem = 1100 32000 100 128

fs.file-max = 6815744

net.ipv4.ip_local_port_range = 9000 65000

net.core.rmem_default = 1048576

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048576

vm.min_free_kbytes= 1048576

fs.aio-max-nr= 1048576

设置生效：

/sbin/sysctl -p

2.6 limit文件

/etc/security/limits.conf添加：

#add by oracle

oracle hard nofile 65536

oracle  soft nofile 4096

oracle  hard nproc 16384

oracle  soft nproc 2047

grid hard nofile 65536

grid soft nofile 1024

grid hard nproc 16384

grid soft nproc 2047

2.7共享盘安装

在云平台上分配共享存储之后，最好在界面上设置可以访问共享存储的客户端，本次设置的是：iqn.2014-12.com.qingcloud:e594a0ee9b82

和iqn.2014-12.com.qingcloud:5ba958139a1

1安装工具包

yum install -y iscsi-initiator-utils

2根据前缀生成客户端id

iscsi-iname -p iqn.2014-12.com.qingcloud

结果：

[root@db02 ~]# iscsi-iname -p iqn.2014-12.com.qingcloud

iqn.2014-12.com.qingcloud:e594a0ee9b82

[root@db01 ~]# iscsi-iname -p iqn.2014-12.com.qingcloud

iqn.2014-12.com.qingcloud:5ba958139a1

3修改配置文件

/etc/iscsi/initiatorname.iscsi 修改为下面的信息（名称是界面上看到的）

节点1:

[root@db01 ~]# cat /etc/iscsi/initiatorname.iscsi

InitiatorName=iqn.2014-12.com.qingcloud:5ba958139a1

节点2：

[root@db02 ~]# cat /etc/iscsi/initiatorname.iscsi

InitiatorName=iqn.2014-12.com.qingcloud:e594a0ee9b82

4客户端发现（两个服务器都要配置）

[root@db01 iscsi]# iscsiadm -m discovery -t st -p 192.168.0.8

Starting iscsid:                                          [ OK ]

192.168.0.8:3260,1 iqn.2014-12.com.qingcloud.s2:sn.data

[root@i-zq8ytr3k iscsi]# iscsiadm -m node

192.168.0.8:3260,1 iqn.2014-12.com.qingcloud.s2:sn.data

5注册（两个服务器都要配置）

[root@db01 iscsi]# iscsiadm -m node -T iqn.2014-12.com.qingcloud.s2:sn.data -p 192.168.0.8 --login

Logging in to [iface: default, target: iqn.2014-12.com.qingcloud.s2:sn.data, portal: 192.168.0.8,3260] (multiple)

Login to [iface: default, target: iqn.2014-12.com.qingcloud.s2:sn.data, portal: 192.168.0.8,3260] successful.

6查看结果

[root@db02 ~]# lsblk

NAME  MAJ:MIN RM SIZE RO TYPE MOUNTPOINT

sda     8:0   0  10G 0 disk

sdb     8:16  0 500G 0 disk

sdc     8:32  0  10G 0 disk

sdd     8:48  0  10G 0 disk

vda   251:0   0  20G 0 disk

`-vda1 251:1   0  20G 0 part /

vdb   251:16  0  16G 0 disk [SWAP]

vdc   251:32  0 500G 0 disk

`-vdc1 251:33  0 500G 0 part /u01

[root@db01 ~]# lsblk

NAME  MAJ:MIN RM SIZE RO TYPE MOUNTPOINT

sda     8:0   0  10G 0 disk

sdb     8:16  0 500G 0 disk

sdc     8:32  0  10G 0 disk

sdd     8:48  0  10G 0 disk

vda   251:0   0  20G 0 disk

`-vda1 251:1   0  20G 0 part /

vdb   251:16  0  16G 0 disk [SWAP]

vdc   251:32  0 500G 0 disk

`-vdc1 251:33  0 500G 0 part /u01

7配置开启自动启动识别

iscsiadm -m node -T iqn.2014-12.com.qingcloud.s2:sn.data -p 192.168.0.8 --op update -n node.startup -v automatic

8配置asm能识别的盘

echo "options=--whitelisted --replace-whitespace">/etc/scsi_id.config

rm -f /etc/udev/rules.d/99-oracle-asmdevices.rules

i=1

id=''

for x in a b c d

 do

   id=`scsi_id --whitelisted --replace-whitespace --device=/dev/sd$x`

   echo "KERNEL==\"sd*\", SUBSYSTEM==\"block\", PROGRAM==\"/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/\$name\", RESULT==\"$id\", NAME=\"asm-disk$i\", OWNER=\"grid\", GROUP=\"asmadmin\", MODE=\"0660\"">>/etc/udev/rules.d/99-oracle-asmdevices.rules

   let i++

 done

 /sbin/start_udev

最后文件生成的/etc/udev/rules.d/99-oracle-asmdevices.rules文件是：

[root@db01 ~]# cat /etc/udev/rules.d/99-oracle-asmdevices.rules

KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name", RESULT=="36001405745529a22afc4f8fb1a685e36", NAME="asm-disk1", OWNER="grid", GROUP="asmadmin", MODE="0660"

KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name", RESULT=="36001405e371bbbb031b4547b5fe217d1", NAME="asm-disk2", OWNER="grid", GROUP="asmadmin", MODE="0660"

KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name", RESULT=="36001405ff11eeac99ac40f69e34a3ec5", NAME="asm-disk3", OWNER="grid", GROUP="asmadmin", MODE="0660"

KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name", RESULT=="3600140517aafddef37c4305a3e6930ee", NAME="asm-disk4", OWNER="grid", GROUP="asmadmin", MODE="0660"

查看生成的设备符号

[root@db01 ~]# ll /dev/asm*

brw-rw---- 1 grid asmadmin 8, 0 Mar 23 19:12 /dev/asm-disk1

brw-rw---- 1 grid asmadmin 8, 16 Mar 23 19:12 /dev/asm-disk2

brw-rw---- 1 grid asmadmin 8, 32 Mar 23 19:12 /dev/asm-disk3

brw-rw---- 1 grid asmadmin 8, 48 Mar 23 19:12 /dev/asm-disk4

[root@db02 ~]# ll /dev/asm*

brw-rw---- 1 grid asmadmin 8, 0 Mar 23 19:12 /dev/asm-disk1

brw-rw---- 1 grid asmadmin 8, 16 Mar 23 19:12 /dev/asm-disk2

brw-rw---- 1 grid asmadmin 8, 32 Mar 23 19:12 /dev/asm-disk3

brw-rw---- 1 grid asmadmin 8, 48 Mar 23 19:12 /dev/asm-disk4

其中asm-disk2是500GB，用作数据。另外3块10GB的盘用作集群vote disk。

2.8关闭ntp

Service ntpd stop

Chkconfig ntpd off

2.9配置操作系统用户互信

修改ssh配置文件：/etc/ssh/ssh_config，注释下面两行

       #StrictHostKeyChecking no

       #UserKnownHostsFile /dev/null

Oracle和grid用户都要操作

mkdir ~/.ssh

 chmod 700 ~/.ssh

 ssh-keygen -t rsa

 ssh-keygen -t dsa

在节点1：

$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

$ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

$ ssh db02 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

$ ssh db02 cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

$ scp ~/.ssh/authorized_keys db02:~/.ssh/authorized_keys

之后分别使用grid、oracle用户在两个节点都运行：

Ssh db01

ssh db02

ssh db01-priv

ssh db02-priv

按照提示输入yes，不输入密码能正常登陆表示正常

三 预检查

节点1，使用grid用户，切换至目录：

/u01/software/u01，运行命令：

./runcluvfy.sh stage -pre crsinst -n db01,db02 -fixup -verbose

结果如下，按照需求再安装rpm，让failed成功：

[grid@db01 grid]$ ./runcluvfy.sh stage -pre crsinst -n db01,db02 -fixup -verbose

四 图形化安装

为避免网络中途掉线，所有图形化均在web控制台上分别使用grid、oracle用户登陆db01进行操作。

4.1安装grid软件

使用grid用户登陆db01，切换至目录：

/u01/software/grid

执行./runInstaller

选择大小为10G的/dev/asm-disk1, /dev/asm-disk3,/dev/asm-disk4

这里需要点击“change discovery path”，设置路径为/dev/asm*

在进行环境检查时，会报错

这个警告在青云平台也有说明：

注解 grid安装环境检查脚本在Oracle Linux 6上面的一个bug导致错误”PRVF-9802”无法解决，具体原因请见Debug Cluvfy error ERROR: PRVF-9802。

勾选忽略这个警告。

弹出脚本分别使用root在两个服务器执行：

/u01/app/oraInventory/orainstRoot.sh

/u01/app/11.2.0/grid/product/db_1/root.sh

第二个脚本执行会比较好时间，大概20分钟。

4.1.1执行脚本

节点1

[root@db01 ~]# /u01/app/oraInventory/orainstRoot.sh

Changing permissions of /u01/app/oraInventory.

Adding read,write permissions for group.

Removing read,write,execute permissions for world.

Changing groupname of /u01/app/oraInventory to oinstall.

The execution of the script is complete.

[root@db01 ~]# /u01/app/11.2.0/grid/product/db_1/root.sh

[root@db01 ~]# su - grid

[grid@db01 ~]$ crs_stat -t

Name          Type          Target   State    Host       

------------------------------------------------------------

ora.CRS.dg    ora....up.type ONLINE   ONLINE   db01       

ora....N1.lsnr ora....er.type ONLINE   ONLINE   db01       

ora.asm       ora.asm.type  ONLINE   ONLINE   db01       

ora.cvu       ora.cvu.type  ONLINE   ONLINE   db01       

ora....SM1.asm application   ONLINE   ONLINE   db01       

ora.db01.gsd  application   OFFLINE  OFFLINE              

ora.db01.ons  application   ONLINE   ONLINE   db01       

ora.db01.vip  ora....t1.type ONLINE   ONLINE   db01       

ora.gsd       ora.gsd.type  OFFLINE  OFFLINE              

ora....network ora....rk.type ONLINE   ONLINE   db01       

ora.oc4j      ora.oc4j.type ONLINE   ONLINE   db01       

ora.ons       ora.ons.type  ONLINE   ONLINE   db01       

ora.scan1.vip ora....ip.type ONLINE   ONLINE   db01

节点2

[root@db02 ~]# /u01/app/oraInventory/orainstRoot.sh

Changing permissions of /u01/app/oraInventory.

Adding read,write permissions for group.

Removing read,write,execute permissions for world.

Changing groupname of /u01/app/oraInventory to oinstall.

The execution of the script is complete.

[root@db02 ~]# /u01/app/11.2.0/grid/product/db_1/root.sh

[root@db02 ~]# su - grid

[grid@db02 ~]$ crs_stat -t

4.2创建asm磁盘组

4.3安装oracle软件

Oracle用户登陆，切换至目录：

/u01/software/database