1.开启JobManager日志
（1）编辑NameNode配置文件${hadoop_home}/etc/hadoop/yarn-site.xml和mapred-site.xml
- 编辑yarn-site.xml
<!-- Site specific YARN configuration properties -->
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<!-- 开启日志聚合 -->
<!-- 是否启用日志聚集功能 -->
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<!-- 日志存储时间 -->
<property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>10080</value>
</property>
<!--当应用程序运行结束后，日志被转移到的HDFS目录（启用日志聚集功能时有效），如此便可通过appmaster UI查看作业的运行日志。-->
<property>
<name>yarn.nodemanager.remote-app-log-dir</name>
<value>/flink/log</value>
</property>
<!-- 日志服务器的地址 -->
<property>
<name>yarn.log.server.url</name>
<value>http://localhost:19888/jobhistory/logs</value>
</property>
<!-- 正在运行中的日志在hdfs上的存放路径 -->
<property>
<name>mapreduce.jobhistory.intermediate-done-dir</name>
<value>/history/done_intermediate</value>
</property>
<!-- 运行过的日志存放在hdfs上的存放路径 -->
<property>
<name>mapreduce.jobhistory.done-dir</name>
<value>/history/done</value>
</property>
</configuration>
- 编辑mapred-site.xml
<property>
<!-- 表示提交到hadoop中的任务采用yarn来运行，要是已经有该配置则无需重复配置 -->
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<!--日志监控服务的地址，一般填写为namenode机器地址 -->
<name>mapreduce.jobhistroy.address</name>
<value>hadoop101:10020</value>
</property>
<!--填写为namenode机器地址-->
<property>
<name>mapreduce.jobhistroy.webapp.address</name>
<value>hadoop101:19888</value>
</property>
#复制配置文件到集群的其他机器
scp mapred-site.xml 用户@IP地址:/目标机器文件夹路径
scp yarn-site.xml 用户@IP地址:/目标机器文件夹路径
（3）重启yarn,重启历史服务
./stop-yarn.sh && ./start-yarn.sh
#进入到hadoop的安装目录
cd ${hadoophome}/hadoop/sbin
kill -9 117681 && ./mr-jobhistory-daemon.sh start historyserver
（4）查看服务运行情况
jps
（5）运行flink on yarn
./bin/flink run -m yarn-cluster -c com.lixiang.app.FlinkDemo ./flink-demo-jar-with-dependencies.jar
（6）查看hadoop控制台
2.开启TaskManager日志
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://192.168.139.101:3306/metastore?useSSL=false</value>
</property>
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
</property>
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>root</value>
</property>
<property>
<name>hive.metastore.warehouse.dir</name>
<value>/user/hive/warehouse</value>
</property>
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>123456</value>
</property>
<property>
<name>hive.metastore.schema.verification</name>
<value>false</value>
</property>
<property>
<name>hive.metastore.event.db.notification.api.auth</name>
<value>false</value>
</property>
<property>
<name>hive.cli.print.current.db</name>
<value>true</value>
</property>
<property>
<name>hive.cli.print.header</name>
<value>true</value>
</property>
<property>
<name>hive.server2.thrift.bind.host</name>
<value>ip</value>
</property>
<property>
<name>hive.server2.thrift.port</name>
<value>10000</value>
</property>
</configuration>
CREATE EXTERNAL TABLE tweets
COMMENT "A table backed by Avro data with the Avro schema embedded in the CREATE TABLE statement"
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
STORED AS
INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
LOCATION '/user/hive/warehouse'
TBLPROPERTIES (
'avro.schema.literal'='{
"type": "record",
"name": "Tweet",
"namespace": "com.miguno.avro",
"fields": [
{ "name":"username", "type":"string"},
{ "name":"tweet", "type":"string"},
{ "name":"timestamp", "type":"long"}
]
}'
);
insert into tweets values('zhaoliu','Hello word',13800000000);
select * from tweets;
//建立外部 schema
CREATE EXTERNAL TABLE avro_test1
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
STORED AS
INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
LOCATION '/user/tmp'
TBLPROPERTIES (
'avro.schema.url'='hdfs:///user/hive/warehouse/student.avsc'
);
{
"type":"record",
"name":"student",
"namespace":"com.tiejia.avro",
"fields":[
{
"name":"SID",
"type":"string",
"default":""
},
{
"name":"Name",
"type":"string",
"default":""
},
{
"name":"Dept",
"type":"string",
"default":""
},
{
"name":"Phone",
"type":"string",
"default":""
},
{
"name":"Age",
"type":"string",
"default":""
},
{
"name":"Date",
"type":"string",
"default":""
}
]
}
"type": "record",
"name": "Tweet",
"namespace": "com.miguno.avro",
"fields": [
{
"name": "username",
"type": "string"
},
{
"name": "tweet",
"type": "string"
},
{
"name": "timestamp",
"type": "long"
}
]
}
CREATE EXTERNAL TABLE tweets
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
STORED AS
INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
LOCATION '/user/tmp'
TBLPROPERTIES (
'avro.schema.url'='hdfs:///user/hive/warehouse/tweets.avsc'
);