May 30, 2021 Article blog
Cloudera's Distribution Includeing Apache Hadoop, or "CDH"), supports most Hadoop components, including HDFS, MapReduce, Hive, Pig, Hbase, Zookeeper, and Sqoop, based on the Web user interface.
serial number | IP address | host name | The system version |
---|---|---|---|
1 | 10.211.55.100 | zero | Centos 7.7 |
2 | 10.211.55.101 | Hadoop1 | Centos 7.7 |
3 | 10.211.55.102 | Hadoop2 | Centos 7.7 |
vi /etc/sysconfig/network-scripts/ifcfg-eth0
BOOTPROTO=static
ONBOOT=yes
IPADDR=10.211.55.100
NETMASK=255.255.255.0
GATEWAY=10.211.55.1
systemctl restart network.service #重启网络服务
systemctl stop firewalld
systemctl disable firewalld
vi /etc/sysconfig/selinux
SELINUX=disabled #将SELINUX的值改为disabled
reboot #重启设备使selinux的配置生效
hostnamectl set-hostname zero
vi /etc/hosts
10.211.55.100 zero
10.211.55.101 hadoop1
10.211.55.102 hadoop2
reboot #修改完后重启生效
yum -y install ntp #安装NPT服务
vi /etc/ntp.conf #修改配置文件
#注释掉原有的server指向
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
#zero节点指向
server 0.cn.pool.ntp.org
server 0.asia.pool.ntp.org
server 3.asia.pool.ntp.org
server 10.211.55.100 iburst #Hadoop1/Hadoop2节点指向zero节点
ntpdate -u cn.pool.ntp.org #同步网络时间
#重启ntp服务
systemctl start ntpd
systemctl enable ntpd
hwclock --localtime -w #将系统时间写入本地硬件时钟时间
ssh-keygen -t rsa #不输入密码,一路回车
ssh-copy-id hadoop1
ssh-copy-id hadoop2
#测试连通性
ssh 'zero'
ssh 'hadoop1'
ssh 'hadoop2'
mkdir /usr/Java/ #创建目录
tar -zxvf jdk1.8.0_211.tar.gz -C /usr/Java/ #解压JDK包到指定目录
vi /etc/profile #配置环境变量
#JAVA
export JAVA_HOME=/usr/Java/jdk1.8.0_211
export JRE_HOME=${JAVA_HOME}/jre export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib export PATH=${JAVA_HOME}/bin:$PATH
source /etc/profile #使配置文件生效
java -version #查看版本信息 检测安装是否成功
#安装成功后的返回信息
java version "1.8.0_211"
Java(TM) SE Runtime Environment (build 1.8.0_211-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.211-b12, mixed mode)
#查看系统中是否已安装MySQL服务 如果存在则删除MySQL及其依赖的包
rpm -qa | grep mysql
yum list installed | grep mysql
wget http://repo.mysql.com/mysql57-community-release-el7-8.noarch.rpm
rpm -ivh mysql57-community-release-el7-8.noarch.rpm
yum install mysql-server #安装MySQL 如出现提示 一路y到底
Once installed, run MySQL and automatically generate a random password in
/var/log/mysqld.log
file.
service mysqld start #运行MySQL
grep "password" /var/log/mysqld.log #取得随机密码 用于登录 MySQL 服务端
#返回信息如下 “6WjiI.UxuoaF”即为MySQL登陆密码
2021-01-05T11:04:28.684402Z 1 [Note] A temporary password is generated for root@localhost: 6WjiI.UxuoaF
Sign in to the MySQL service and change your password.
/*修改密码*/
SET PASSWORD = PASSWORD('new password');
ALTER USER 'root'@'localhost' PASSWORD EXPIRE NEVER;
flush privileges;
/*设置用户 root 可以在任意 IP 下被访问*/
grant all privileges on *.* to root@"%" identified by "new password";
/*设置用户 root 可以在本地被访问*/
grant all privileges on *.* to root@"localhost" identified by "new password";
/*刷新权限使其生效*/
flush privileges;
yum -y install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse portmap fuse-libs redhat-lsb #安装第三方依赖包
mkdir -p /opt/cloudera-manager #创建目录
Unzip the Cloudera Manager package to/opt/cloudera-manager directory.
tar -zvxf cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz -C /opt/cloudera-manager/
Modify the agent profile.
vi /opt/cloudera-manager/cm-5.14.0/etc/cloudera-scm-agent/config.ini
#配置文件修改如下
# Hostname of the CM server.
server_host=zero
# Port that the CM server is listening on.
server_port=7182
Create a dedicated normal user cloudera-scm for CM.
useradd --system #表示创建的是系统用户
--home=/opt/cloudera-manager/cm-5.14.0/run/cloudera-scm-server #指定了用户的主目录
--no-create-home #表示不再创建用户的主目录
--shell=/bin/false #不作为登陆用户
--comment "Cloudera SCM User"
cloudera-scm #用户名
Configure database access and add users.
grant all privileges on *.* to 'temp'@'%' identified by 'password' with grant option;
flush privileges;
Execute cm's initialization script in MySQL on the primary node.
mv mysql-connector-java.jar /usr/share/java/mysql-connector-java.jar
#进入初始化脚本目录下
cd /opt/cloudera-manager/cm-5.14.0/share/cmf/schema
#运行初始化脚步
./scm_prepare_database.sh mysql -h zero -utemp -ppassword --scm-host zero scm scm scm
Configure the CDH source (note that the version correspondence is required for download, and el5/6/7 corresponds to CentOS5/6/7, respectively.) )
#创建目录
mkdir -p /opt/cloudera/parcel-repo #主节点
mkdir -p /opt/cloudera/parcels #所有节点
#将parcel文件移动到parcel-repo目录下
mv CDH-5.14.0-1.cdh5.14.0.p0.24-el7.parcel /opt/cloudera/parcel-repo/
#将json文件移动到parcel-repo目录下
mv manifest.json /opt/cloudera/parcel-repo/
#重命名校验码文件并移动到parcel-repo目录下
mv CDH-5.14.0-1.cdh5.14.0.p0.24-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.14.0-1.cdh5.14.0.p0.24-el7.parcel.sha
Change the directory owner
chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/ #主节点
chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcels/ #所有节点
Start server (primary node)
/opt/cloudera-manager/cm-5.14.0/etc/init.d/cloudera-scm-server start
Starting cloudera-scm-server: [ 确定 ] #成功启动后的返回信息
Startagent (all nodes)
/opt/cloudera-manager/cm-5.14.0/etc/init.d/cloudera-scm-agent start
Starting cloudera-scm-agent: [ 确定 ] #成功启动后的返回信息
Open
http://10.211.55.100:7180/
enter the CM initial interface with an account password of
admin
Sign in to check the consent terms, click Continue and select the free version.
Continue to the next step and check the installation host.
The default check for cluster installation is available, click Continue and wait for the installation to complete.
Click Continue to check the host correctness and make repairs with the information given.
Click Continue to select the service you need.
Click Continue to assign roles.
Configure the database to connect successfully.
Review the changes page directly by default, click Continue.
Wait for the deployment to complete.
Click continue and the installation is complete.
#错误
Your password does not satisfy the current policy requirements
#解决方法
vi /etc/my.cnf #修改配置文件my.cnf
validate_password=off #关闭密码策略设置validate_password
service mysqld restart #重启服务
#错误
为CDH群集安装指定主机无法勾选。
#解决方法
/opt/cloudera-manager/cm-5.14.0/etc/init.d/cloudera-scm-server stop #停止server
/opt/cloudera-manager/cm-5.14.0/etc/init.d/cloudera-scm-agent stop #停止agent
#删除/opt/cloudera-manager/cm-5.14.0/lib/cloudera-scm-server-db/data/目录下内容
rm -rf /opt/cloudera-manager/cm-5.14.0/lib/cloudera-scm-server-db/data/*
/*删除MySQL数据库中scm数据库下的hosts表中的数据*/
use scm;
delete from HOSTS;
#错误
MainThread agent ERROR Error, CM server guid updated, expected 7cda865f-a833-40d1-9fa6-761631c74326, received e2678b78-7c5e-4703-91b6-c52e2875988c
#解决方法
rm -rf /opt/cloudera-manager/cm-5.14.0/lib/cloudera-scm-agent/cm_guid
#错误
ERROR Table 'scm.CM_VERSION' doesn't exist
#解决方法
vi /etc/my.cnf #修改配置文件my.cnf
lower_case_table_names = 1 #[mysqld]节点下添加
service mysqld restart #重启服务
#错误
ERROR WebServerImpl:com.cloudera.server.web.cmf.search.components.SearchRepositoryManager: The server storage directory [/var/lib/cloudera-scm-server] doesn't exist.
#解决方法
mkdir /var/lib/cloudera-scm-server
chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-scm-server #创建目录并加上权限
Recommended lessons:
Develop employment from the 0 foundation to the front end
PHP quick entry to development employment