hive on spark配置

标签：logger log 配置 hive DRFA spark appender

hive-site.xml

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<name>javax.jdo.option.ConnectionURL</name>

<value>jdbc:mysql://192.168.12.65:3306/metastore?useSSL=false&useUnicode=true&characterEncoding=UTF-8&allowPublicKeyRetrieval=true</value>

</property>

<name>javax.jdo.option.ConnectionDriverName</name>

<value>com.mysql.cj.jdbc.Driver</value>

</property>

<name>javax.jdo.option.ConnectionUserName</name>

</property>

<name>javax.jdo.option.ConnectionPassword</name>

<value>mysql</value>

</property>

<name>hive.metastore.warehouse.dir</name>

<value>/user/hive/warehouse</value>

</property>

<name>hive.metastore.schema.verification</name>

<value>false</value>

</property>

<name>hive.server2.enable.doAs</name>

<value>false</value>

</property>

<name>hive.server2.thrift.port</name>

</property>

<name>hive.server2.thrift.bind.host</name>

</property>

<name>hive.metastore.event.db.notification.api.auth</name>

<value>false</value>

</property>

<name>hive.cli.print.header</name>

</property>

<name>hive.cli.print.current.db</name>

</property>

<!--

<name>hive.zookeeper.quorum</name>

<value>datanebula7:2181,datanebula8:2181,datanebula9:2181</value>

</property>

<name>hive.zookeeper.client.port</name>

</property>

-->

<name>spark.yarn.jars</name>

<value>hdfs://zszccluster/spark/spark-jars/*</value>

</property>

<!--

<name>hive.execution.engine</name>

<value>spark</value>

</property>

-->

<name>hive.spark.client.connect.timeout</name>

</property>

<name>hive.heapsize</name>

</property>

<name>hive.exec.spark.default.parallelism</name>

</property>

<name>spark.executor.memory</name>

</property>

</configuration>

spark-default.conf

# 指定提交到 yarn 运行

spark.master yarn

# 开启日志并存储到 HDFS 上

spark.eventLog.enabled true

spark.eventLog.dir hdfs://zszccluster/spark/logs

# 指定每个执行器的内存

spark.executor.memory 4g

# 指定每个调度器的内存

spark.driver.memory 2g

log4j.properties

log4j.rootLogger=WARN, CA

log4j.appender.CA=org.apache.log4j.ConsoleAppender

log4j.appender.CA.layout=org.apache.log4j.PatternLayout

log4j.appender.CA.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

hive-log4j2.properties

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements. See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership. The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License. You may obtain a copy of the License at

# http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

status = WARN

name = HiveLog4j2

packages = org.apache.hadoop.hive.ql.log

# list of properties

property.hive.log.level = WARN

property.hive.root.logger = DRFA

property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}

property.hive.log.file = hive.log

property.hive.perflogger.log.level = INFO

# list of all appenders

appenders = console, DRFA

# console appender

appender.console.type = Console

appender.console.name = console

appender.console.target = SYSTEM_ERR

appender.console.layout.type = PatternLayout

appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n

# daily rolling file appender

appender.DRFA.type = RollingRandomAccessFile

appender.DRFA.name = DRFA

appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}

# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session

appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}

appender.DRFA.layout.type = PatternLayout

appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n

appender.DRFA.policies.type = Policies

appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy

appender.DRFA.policies.time.interval = 1

appender.DRFA.policies.time.modulate = true

appender.DRFA.strategy.type = DefaultRolloverStrategy

appender.DRFA.strategy.max = 30

# list of all loggers

loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp

logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn

logger.NIOServerCnxn.level = WARN

logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO

logger.ClientCnxnSocketNIO.level = WARN

logger.DataNucleus.name = DataNucleus

logger.DataNucleus.level = ERROR

logger.Datastore.name = Datastore

logger.Datastore.level = ERROR

logger.JPOX.name = JPOX

logger.JPOX.level = ERROR

logger.AmazonAws.name=com.amazonaws

logger.AmazonAws.level = INFO

logger.ApacheHttp.name=org.apache.http

logger.ApacheHttp.level = INFO

logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger

logger.PerfLogger.level = ${sys:hive.perflogger.log.level}

# root logger

rootLogger.level = ${sys:hive.log.level}

rootLogger.appenderRefs = root

rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}

标签：logger,log,配置,hive,DRFA,spark,appender
From： https://www.cnblogs.com/Mr--Wang/p/18184702

相关文章

赞助商

阅读排行