python实现文件下载功能(Linux+Windows本地)一、前提介绍
我的日常工作中,有一项固定的工作内容,就是须要从不同的服务器中将各个应用系统的数据库备份拉到本地。因为此项工作太过分机械化,占用时间较多,在经历了一段时间的摧残以后,我就开始尝试使用脚本来完成这项工作。
我的备份介质存在比较严重的日期依赖,比如正常情况,T日备份T-1的备份文件,然而当T日为礼拜一时,有的须要拉上周日至上周六的数据,有的只须要拉上周四的数据等,遇见节假日就更麻烦了。起初考虑使用bat脚本实现,后来剖析了一下linux操作系统好吗,认为可行性太低linux下载文件到本地,决定采用python来实现。
开始的时侯,因为服务器环境比较老,而且是生产的外网环境,我不想去折腾一个新的环境,所以我想直接使用服务器本身的环境,Windows+python2.7+pscp来完成linux驱动下载,做完以后发觉拷贝数据太慢,1M下载速率不到,无奈才使用python的paramiko模块进行优化。
二、代码实现实现目标:
①方便维护
②从远程拷贝备份介质到本地;
②拷贝备份介质的日期为上一工作日至当天;
③自动生成本地路径文件夹,文件夹所含的日期为下个月第一个工作日
以上linux下载文件到本地,经过简单剖析以后,第一,我须要将所有须要配置的数据写进配置文件,第二,解决日期问题,第三,解决路径问题
2.1配置文件2.1.1日常维护
首先为了保证之后的维护问题,将往前须要更改数据的部份配置在单独的文件中,我配置的数据有holiday_list(节假日)、linux_address(远程路径)、windows_address(本地路径)、file_name(模糊文件名)、file_extension(文件后缀)。配置的数据使用列表储存,便捷遍历套接产生远程路径和本地路径。
2.1.2日期问题
(1)须要获取上一工作日至当天的所有日期
# 获取日期列表(上一工作日至系统日期)----------------------------------------------
def get_date_list():
date_list = []
today = datetime.datetime.today().strftime("%Y%m%d")
today_date = datetime.datetime.strptime(today, "%Y%m%d")
last_day_get = get_last_work_day()
last_day_date = datetime.datetime.strptime(last_day_get, "%Y%m%d")
while today_date >= last_day_date:
last_day = last_day_date.strftime("%Y%m%d")
date_list.append(last_day)
last_day_date += datetime.timedelta(days=1)
return date_list
# 获取today最近一个工作日--------------------------------------------------------
def get_last_work_day():
i = 1
# 前一自然日
last_day = datetime.date(datetime.datetime.today().year, datetime.datetime.today().month,
datetime.datetime.today().day) - datetime.timedelta(i)
# 前一自然日的星期数
calendar_march = calendar.weekday(last_day.year, last_day.month, last_day.day)
# 格式化日期
last_day_to_str = last_day.strftime("%Y%m%d")
while last_day_to_str in holiday_list or calendar_march in range(5, 7):
i += 1
last_day = datetime.date(datetime.datetime.today().year, datetime.datetime.today().month,
datetime.datetime.today().day) - datetime.timedelta(i)
# 前一自然日的星期数
calendar_march = calendar.weekday(last_day.year, last_day.month, last_day.day)
# 格式化日期
last_day_to_str = last_day.strftime("%Y%m%d")
return last_day_to_str
(2)须要获取下个月第一个工作日
# 获取下个月第一个工作日-----------------------------------------------------------
def get_first_workday_of_next_month():
date = get_date_list()
d = 0
next_days = []
while d < len(date):
dt = datetime.datetime.strptime(date[d], '%Y%m%d')
y = dt.year
m = dt.month
i = 1
# y = datetime.datetime.today().year
# m = datetime.datetime.today().month
if m in [1, 3, 5, 7, 8, 10, 12]:
day = 31
elif m in [4, 6, 9, 11]:
day = 30
else:
if (y % 4 == 0 and y % 100 != 0) or (y % 100 == 0 and y % 400 == 0):
day = 29
else:
day = 28
# 当月最后一个自然子的下一自然日
next_day = datetime.date(y, m, day) + datetime.timedelta(i)
# 当月最后一个自然子的下一自然日的星期数
calendar_march = calendar.weekday(next_day.year, next_day.month, next_day.day)
# 格式化日期
next_day_to_str = next_day.strftime("%Y%m%d")
# 获取工作日
while next_day_to_str in holiday_list or calendar_march in range(5, 7):
i += 1
next_day = datetime.date(y, m, day) + datetime.timedelta(i)
# 下一自然日的星期数
calendar_march = calendar.weekday(next_day.year, next_day.month, next_day.day)
# 格式化日期
next_day_to_str = next_day.strftime("%Y%m%d")
next_days.append(next_day_to_str)
d += 1
return next_days
2.1.3路径处理
(1)paramiko下载路径须要完整路径,包括文件名
# 获取Linux路径
def get_linux_adr():
i = 0
dates = get_date.get_date_list()
add_linux = []
while i < len(linux_address):
d = 0
while d < len(dates):
fe = 0
while fe < len(file_extension):
adr_linux = linux_address[i] + '/' + file_name[i] + dates[d] + file_extension[fe]
add_linux.append(adr_linux)
fe += 1
d += 1
i += 1
return add_linux
# 获取Windows路径--带文件名
def get_windows_adr():
i = 0
add_windows = []
dates = get_date.get_date_list()
next_days = get_date.get_first_workday_of_next_month()
while i < len(windows_address):
d = 0
while d < len(dates):
fe = 0
while fe < len(file_extension):
adr_windows = windows_address[i] + next_days[d] + '\' + file_name[i] + dates[d] +
file_extension[fe]
add_windows.append(adr_windows)
fe += 1
d += 1
i += 1
return add_windows
(2)创建本地路径文件夹,不须要文件名
# 获取Windows路径--不带文件名
def get_windows_adr_no_filename():
next_days = get_date.get_first_workday_of_next_month()
add_windows = []
i = 0
while i < len(windows_address):
d = 0
while d < len(next_days):
adr_windows = windows_address[i] + next_days[d]
add_windows.append(adr_windows)
d += 1
i += 1
return add_windows_set
2.2主代码2.2.1从Linux下载文件到本地Windows(使用paramiko模块)
(1)创建联接,以及获取文件信息
class Pygetfile(object):
def __init__(self, ip, port, user, passwd):
self.ip = ip
self.port = port
self.user = user
self.passwd = passwd
def connect(self):
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(self.ip, self.port, self.user, self.passwd)
print ('connected:' + self.ip)
def stat(self, linux_path):
sftp = paramiko.SFTPClient.from_transport(self.ssh.get_transport())
return sftp.stat(linux_path)
def getfile(self, linux_path, windows_path):
sftp = paramiko.SFTPClient.from_transport(self.ssh.get_transport())
now = datetime.datetime.now()
print ('start time:' + now.strftime('%Y%m%d %H:%M:%S'))
print ('>>>>>>>>>>download start:' + linux_path)
print ('>>>>>>>the local path is:' + windows_path)
sftp.get(linux_path, windows_path)
now1 = datetime.datetime.now()
used = str(now1 - now)
print ('time used:' + used)
def close(self):
self.ssh.close()
print ('close the connect')
(2)下载文件
def copy_file_from_remote():
global ip, port, user, passwd
obj = Pygetfile(ip, port, user, passwd)
obj.connect()
i = 0
while i < len(adr_linuxs):
linux_path = adr_linuxs[i]
try:
obj.stat(linux_path)
except IOError as e:
i += 1
else:
windows_path = adr_windows[i]
obj.getfile(linux_path, windows_path.decode('utf-8'))
i += 1
obj.close()
(3)创建本地文件夹
def mkdir_for_windows():
adr = config.get_windows_adr_no_filename()
i = 0
while i < len(adr):
windows_path = adr[i]
windows_path_fmt = windows_path.decode('utf-8')
if not os.path.exists(windows_path_fmt):
os.makedirs(windows_path_fmt)
print ('directory>>> ' + windows_path_fmt + ' <<<created')
else:
print ('directory>>> ' + windows_path_fmt + ' <<<a style='color:#0000CC;font-size:16px;' lready exists')
i += 1
(4)main函数
if __name__ == '__main__':
mkdir_for_windows()
copy_file_from_remote()
os.system('pause')
2.2.2从本地拷贝文件到另一个文件夹(使用shutil的copy模块)
主代码如下:
# 1、目标文件夹是否存在,如不存在,创建文件夹
def mkdir_target():
target_path = config.get_to_path()
i = 0
while i<len(target_path):
target_path_fmt = target_path[i].decode('utf-8')
if not os.path.exists(target_path_fmt):
os.makedirs(target_path_fmt)
print ('directory>>> ' + target_path_fmt + ' <<<created')
else:
print ('directory>>> ' + target_path_fmt + ' <<<a style='color:#0000CC;font-size:16px;' lready exists')
i += 1
# 2、复制源文件到目标文件夹
def copy_file():
source_path = config.get_from_path()
target_path = config.get_to_path()
p = 0
while p < len(source_path):
source_path_fmt = source_path[p].decode('utf-8')
target_path_fmt = target_path[p].decode('utf-8')
if not os.path.exists(source_path_fmt):
print ('file>>> ' + source_path_fmt + ' <<<not exists')
else:
copy(source_path_fmt, target_path_fmt)
print ('copy file>>> ' + source_path_fmt + ' <<<done')
p += 1
if __name__ == '__main__':
mkdir_target()
copy_file()
os.system('pause')