最近公司上了一个转储服务器，用于存储所有应用系统的数据文件备份，由于每天的备份文件都放在一个固定的文件夹，不需要的文件也会传过来，日后不方便整理，于是便使用Python脚本，将需要的每个文件每日进行归类整理。
一、配置文件格式
{
"otcdDBBAK": {
"foldername": "otcdDBBAK", --源文件目录最后一级的文件夹名
"backupType": 0, --备份类型（0-每周一至周五备份；1-每天备份；2、每周一至周六备份）
"src_path": "/home/oracle/dmpbak/", --源文件需要整理的文件夹
"target_path": "/home/oracle/backup/", --目标存放的目录
"fileName": "otc_yyyymmdd", --文件名
"fileExtension": [ --文件后缀
".tar.gz"
]
}
}
二、脚本内容
脚本中，将需要的执行信息输出到执行日志中，便于后面监控。
脚本执行目的：
1、对源文件进行整理
2、对部分文件进行MD5码值校验
#!/usr/bin/python
# -*- coding: utf-8 -*-
from datetime import datetime
from datetime import timedelta
import json
import os
import subprocess
from glob import glob
import hashlib
# 获取前一天的日期，获取两种格式数据，yyyymmdd和yyyymm
def getYesterday():
today = datetime.today()
oneday = timedelta(days=1)
yesterday1 = (today - oneday).strftime('%Y%m%d')
yesterday2 = (today - oneday).strftime('%Y%m')
return yesterday1, yesterday2
def check_md5_file_list(files):
# files输入的是一个字典，获取字典中fileExtension 包含.md5后缀的元素，并组成一个新的字典
value = '.*' + 'md5'
# 获取需要校验的文件字典集合
for i in list(files):
import re
for f in files[i]["fileExtension"]:
result = re.findall(value, f)
if len(result) == 0:
del files[i]
return files
def get_md5(file):
# 计算文件的MD5值，返回一个字符串
md5 = hashlib.md5()
with open(file, 'rb') as fobj:
while True:
data = fobj.read(4096)
if not data:
break
md5.update(data)
return md5.hexdigest()
def read_md5(file):
# 读取文件的数据，返回一个字符串
with open(file, 'r') as data:
rd = data.read()
md5 = rd[0:32]
return md5
def get_src_file(cf):
# 根据备份策略
# week 取值为0-6 表示周一到周日
# 当week 取值为0时，获取conf/config,json 中{backupType}为1的数据
# 当week 取值为6时，获取conf/config.json 中{backupType}不为0的数据
week = datetime.weekday(datetime.today())
with open(cf, 'r') as f:
json_data = json.load(f)
for i in list(json_data):
if week == 0 and json_data[i]["backupType"] != 1:
del json_data[i]
continue
elif week == 6 and json_data[i]["backupType"] == 0:
del json_data[i]
continue
else:
continue
return json_data
def mv_file_everyday(files, logfile, yd1, yd2):
# 遍历新的字典，获取源文件和目标目录，执行创建文件夹的命令以及mv文件的命令
for i in files:
target_path = files[i]["target_path"] + \
files[i]["foldername"] + '/' + yd2 + '/'
# 目标文件夹不存在是需要创建
if not os.path.exists(target_path):
os.makedirs(target_path)
warn = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
"daily mv task warnning: Folder created: " + '"'+target_path+'"'
print(warn)
subprocess.call(f"echo {warn} >> {logfile}", shell=True)
# 开始处理源文件
for j in files[i]["fileExtension"]:
src_file = (files[i]["src_path"] + files[i]["foldername"] +
'/' + files[i]["fileName"] + j).replace("yyyymmdd", yd1)
try:
try:
# 1、src_file 中有通配符的话，需要进行转换
if '*' in src_file:
src_file = glob(src_file)[0]
except Exception as e:
error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
" daily mv task error: Execution failed " + str(e) + src_file
subprocess.call(f"echo {error} >> {logfile}", shell=True)
# 2、判断源文件是否存在，如果不存在则将信息写入日志，如需监控则从日志中提取信息
if not os.path.exists(src_file):
error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
"daily mv task error srcfile: no such file: " + '"'+src_file+'"'
subprocess.call(f"echo {error} >> {logfile}", shell=True)
# subprocess.call(['touch', src_file])
# 3、开始整理文件操作，将源文件mv 到目标文件夹中
# 不直接mv，分开使用cp + rm 的方式，通过分析日志，可以便于mv文件失败后，辨别正确的文件
else:
try:
subprocess.call(["cp", src_file, target_path])
success_cp = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
"daily mv task successfully copy file : " + '"'+src_file+'"'
subprocess.call(
f"echo {success_cp} >> {logfile}", shell=True)
subprocess.call(["rm", "-f", src_file])
success_rm = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
"daily mv task successfully delete files : " + '"'+src_file+'"'
subprocess.call(
f"echo {success_rm} >> {logfile}", shell=True)
except Exception as e:
error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
"daily mv task error: Execution failed: " + str(e)
subprocess.call(
f"echo {error} >> {logfile}", shell=True)
except Exception as s:
error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
" daily mv task error: Execution failed: " + str(s) + src_file
subprocess.call(f"echo {error} >> {logfile}", shell=True)
return
def check(Files):
for j in Files:
for fx in Files[j]["fileExtension"]:
target_file = (Files[j]["target_path"] + Files[j]["foldername"] +
'/' + yd2 + '/' + Files[j]["fileName"] + fx).replace("yyyymmdd", yd1)
try:
if '*' in target_file:
target_file = glob(target_file)[0]
except Exception as e:
error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
" check md5 task error: Execution failed " + str(e) + target_file
subprocess.call(f"echo {error} >> {logfile}", shell=True)
# 由于md5Files列表中，既包含源文件，也包含其.md5文件
# 这里需要先删选出源文件，计算源文件的md5值
# 再根据源文件名拼接得出其md5文件名，进行读取操作
if not target_file.endswith('.md5'):
# 首先判断文件是否存在
if os.path.exists(target_file):
check_result = get_md5(target_file)
target_file_md5 = target_file + '.md5'
md5_result = read_md5(target_file_md5)
# 判断md5值是否一致
if check_result == md5_result:
check_md5_yes = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
' check md5 task successfully: md5_consistent in ' + '"'+j+'"'
subprocess.call(
f"echo {check_md5_yes} >> {logfile}", shell=True)
else:
check_md5_no = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
' check md5 task error: md5_inconsistent in ' + '"'+j+'"'
subprocess.call(
f"echo {check_md5_no} >> {logfile}", shell=True)
else:
error_check_md5file = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
' check md5 task error: file missing in ' + '"'+j+'"'
subprocess.call(
f"echo {error_check_md5file} >> {logfile}", shell=True)
else:
# print(target_file)
continue
return
if __name__ == '__main__':
logfile = '/home/oracle/task/PYbackup/conf/log.log'
configfile = '/home/oracle/task/PYbackup/conf/config.json'
yd1, yd2 = getYesterday()
# 从json 文件中获取今天需要整理的文件信息
files = get_src_file(configfile)
# 开始执行每日整理操作
dailyTaskMv = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + "daily mv task begin......"
subprocess.call(f"echo {dailyTaskMv} >> {logfile}", shell=True)
mv_file_everyday(files, logfile, yd1, yd2)
task_finished = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
"mv file everyday task finished!"
subprocess.call(f"echo {task_finished} >> {logfile}", shell=True)
dailyTaskCheckMd5 = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + "daily check md5 task begin......"
subprocess.call(f"echo {dailyTaskCheckMd5} >> {logfile}", shell=True)
# 获取需要校验MD5值的元素信息
md5Files = check_md5_file_list(files)
# 开始校验md5值
check(md5Files)
check_md5_finished = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
"check md5 task finished!"
subprocess.call(f"echo {check_md5_finished} >> {logfile}", shell=True)
三、zabbix对执行进行监控
# 1、监控脚本：
$ cat zabbix_check.sh
#!/bin/sh
datetime=$(date +%F)
cat $1 | grep $datetime | grep "$2" | wc -l
# 2、zabbix配置文件配置
cat /etc/zabbix_agent.d/userparameter.conf
UnsafeUserParameters=1
UserParameter=logcheck[*],sh zabbix_check.sh $1 "$2"
# 3、zabbix前台新增监控项，设置键值
logcheck[/home/oracle/task/PYbackup/conf/log.log,"error"]