azkaban集群部署

由于azkaban需要下载源码编译，请先在github上下载最新的源码

https://github.com/azkaban/azkaban

编译完成后打包成 azkaban.tar.gz

eg:三台服务器(实现无ssh认证)

   hadoop-master-01(web-server和mysql 5.7)

   hadoop-slave-01(exec-server)

   hadoop-slave-02(exec-server)

当前目录/home/bigdata

tar  --zxvf azkaban.tar.gz

复制/home/bigdata/azkaban/azkaban-db/build/sql/create-all-sql-0.1.0-SNAPSHOT.sql文件内容保存到hadoop-master-01目录/home/bigdata/create_azkaban_sql.sql

进到mysql(用户名root/123456)

操作

mysql -h hadoop-master-01 -P 3306 -u root -p123456

create databases azkaban;

use azkaban;

source /home/bigdata/create_azkaban_sql.sql

scp -rp /home/bigdata/azkaban/azkaban-exec-server/build/distributions/azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz [email protected]:/home/bigdata/azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz

scp -rp /home/bigdata/azkaban/azkaban-exec-server/build/distributions/azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz [email protected]:/home/bigdata/azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz

cp  /home/bigdata/azkaban/azkaban-web-server/build/distributions/azkaban-web-server-0.1.0-SNAPSHOT.tar.gz  /home/bigdata/

tar  –zxvf  azkaban-web-server-0.1.0-SNAPSHOT.tar.gz

mv  azkaban-web-server-0.1.0-SNAPSHOT azkaban-web

cd   azkaban-web

cp  azkaban-web/lib

rm slf4j-log4j12-1.7.21.jar

mv mysql-connector-java-5.1.28.jar  mysql-connector-java-5.1.28.jar.back

cp ../../hive/lib/derby-10.10.2.0.jar  .

cd  azaban-web

mkdir  extlib

mkdir  logs

mkdir web

cp ../hive/lib/mysql-connector-java-8.0.25.jar  extlib/

修改azkaban.properties文件(设置账号密码为123456)

Azkaban Personalization Settings

azkaban.name=project

azkaban.label=My Local Azkaban

azkaban.color=#FF3601

azkaban.default.servlet.path=/index

web.resource.dir=/home/bigdata/azkaban-web/web/

default.timezone.id=Asia/Shanghai

user.manager.class=azkaban.user.XmlUserManager

user.manager.xml.file=/home/bigdata/azkaban-web/conf/azkaban-users.xml

executor.global.properties=/home/bigdata/azkaban-web/conf/global.properties

azkaban.project.dir=projects

velocity.dev.mode=false

jetty.use.ssl=false

jetty.maxThreads=25

jetty.port=8081

jetty.ssl.port=8443

jetty.keystore=keystore

jetty.password=123456

jetty.keypassword=123456

jetty.truststore=keystore

jetty.trustpassword=123456

executor.port=12321

mail.sender=

mail.host=

job.failure.email=

job.success.email=

lockdown.create.projects=false

cache.directory=cache

jetty.connector.stats=true

executor.connector.stats=true

database.type=mysql

mysql.port=3306

mysql.host=hadoop-master-01

mysql.database=azkaban

mysql.user=root

mysql.password=123456

mysql.numconnections=100

#Multiple Executor

azkaban.use.multiple.executors=true

azkaban.executorselector.filters=StaticRemainingFlowSize,MinimumFreeMemory,CpuStatus

azkaban.executorselector.comparator.NumberOfAssignedFlowComparator=1

azkaban.executorselector.comparator.Memory=1

azkaban.executorselector.comparator.LastDispatched=1

azkaban.executorselector.comparator.CpuUsage=1

启动web服务

./ start-web.sh

hadoop-slave-01和hadoop-slave-02:

tar -zxvf azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz

mv azkaban-exec-server-0.1.0-SNAPSHOT  azkaban-exec

cp  azkaban-exec/lib

rm slf4j-log4j12-1.7.21.jar

mv mysql-connector-java-5.1.28.jar  mysql-connector-java-5.1.28.jar.back

cp ../../hive/lib/derby-10.10.2.0.jar  .

cd  azaban-exec

mkdir  extlib

mkdir  logs

mkdir web

cp ../hive/lib/mysql-connector-java-8.0.25.jar  extlib/

修改azkaban.properties文件

azkaban.webserver.url=http://hadoop-master-01:8081

mail.sender=

mail.host=

job.failure.email=

job.success.email=

lockdown.create.projects=false

cache.directory=cache

jetty.connector.stats=true

executor.connector.stats=true

database.type=mysql

mysql.port=3306

mysql.host=hadoop-master-01

mysql.database=azkaban

mysql.user=root

mysql.password=123456

mysql.numconnections=100

# Azkaban Executor settings

executor.maxThreads=50

executor.flow.threads=30

executor.port=12321

开启服务

./ start-exec.sh

进到MySQL数据库

use azkaban;

select * from executors;

update azkaban.executors set active=1;

curl http://hadoop-slave-01:12321/executor?action=activate

curl http://hadoop-slave-02:12321/executor?action=activate

切记：先启动exec进程再启动web进程