首页 >> 网站技术 >> Python下载一个网站的所有图片

Python下载一个网站的所有图片

Catagory:网站技术, Date:2011年12月5日,Author: 工业品营销-贾定强

Python语言真是不错,越用越发现它的强大。今天跟大家分享一段python程序,这个程序可以下载一个网站的所有图片哦。下班前开启程序,第二天就能拿到你心仪的网站的所有图片了。

首先,你需要为目标网站做一份网站地图,存在一个txt文件中,每行一个网址。如果对seo比较熟悉的话,这个事有N种方法可以实现。

然后运行下面程序就可以了。程序下载

# -*- coding: utf-8 -*-
“””
some function by metaphy,2007-04-03,copyleft
version 0.2
“””
import urllib, httplib, urlparse
import re
import random

“””judge url exists or not,by others”””
def httpExists(url):
host, path = urlparse.urlsplit(url)[1:3]
if ‘:’ in host:
# port specified, try to use it
host, port = host.split(‘:’, 1)
try:
port = int(port)
except ValueError:
print ‘invalid port number %r’ % (port,)
return False
else:
# no port specified, use default port
port = None
try:
connection = httplib.HTTPConnection(host, port=port)
connection.request(“HEAD”, path)
resp = connection.getresponse( )
if resp.status == 200:       # normal ‘found’ status
found = True
elif resp.status == 302:     # recurse on temporary redirect
found = httpExists(urlparse.urljoin(url,resp.getheader(‘location’, ”)))
else:                        # everything else -> not found
print “Status %d %s : %s” % (resp.status, resp.reason, url)
found = False
except Exception, e:
print e.__class__, e, url
found = False
return found

“””get html src,return lines[]”””
def gGetHtmlLines(url):
if url==None : return
if not httpExists(url): return
try:
page = urllib.urlopen(url)
html = page.readlines()
page.close()
return html
except:
print “gGetHtmlLines() error!”
return
“””get html src,return string”””
def gGetHtml(url):
if url==None : return
if not httpExists(url): return
try:
page = urllib.urlopen(url)
html = page.read()
page.close()
return html
except:
print “gGetHtml() error!”
return

“””根据url获取文件名”””
def gGetFileName(url):
if url==None: return None
if url==”” : return “”
arr=url.split(“/”)
return arr[len(arr)-1]

“””生成随机文件名”””
def gRandFilename(type):
fname = ”
for i in range(16):
fname = fname + chr(random.randint(65,90))
fname = fname + chr(random.randint(48,57))
return fname + ‘.’ + type
“””根据url和其上的link,得到link的绝对地址”””
def gGetAbslLink(url,link):
if url==None or link == None : return
if url==” or link==” : return url
addr = ”
if link[0] == ‘/’ :
addr = gGetHttpAddr(url) + link
elif len(link)>3 and link[0:4] == ‘http’:
addr =  link
elif len(link)>2 and link[0:2] == ‘..’:
addr = gGetHttpAddrFatherAssign(url,link)
else:
addr = gGetHttpAddrFather(url) + link

return addr

“””根据输入的lines,匹配正则表达式,返回list”””
def gGetRegList(linesList,regx):
if linesList==None : return
rtnList=[]
for line in linesList:
matchs = re.search(regx, line, re.IGNORECASE)
if matchs!=None:
allGroups = matchs.groups()
for foundStr in allGroups:
if foundStr not in rtnList:
rtnList.append(foundStr)
return rtnList
“””根据url下载文件,文件名参数指定”””
def gDownloadWithFilename(url,savePath,file):
#参数检查,现忽略
try:
urlopen=urllib.URLopener()
fp = urlopen.open(url)
data = fp.read()
fp.close()
file=open(savePath + file,’w+b’)
file.write(data)
file.close()
except IOError:
print “download error!”+ url

“””根据url下载文件,文件名自动从url获取”””
def gDownload(url,savePath):
#参数检查,现忽略
fileName = gGetFileName(url)
#fileName =gRandFilename(‘jpg’)
gDownloadWithFilename(url,savePath,fileName)

“””根据某网页的url,下载该网页的jpg”””
def gDownloadHtmlJpg(downloadUrl,savePath):
lines= gGetHtmlLines(downloadUrl)
regx = r”””src\s*=”?(\S+)\.jpg”””
lists =gGetRegList(lines,regx)
if lists==None: return
for jpg in lists:
jpg = gGetAbslLink(downloadUrl,jpg) + ‘.jpg’
gDownload(jpg,savePath)
###     print gGetFileName(jpg)
“””根据url取主站地址”””
def gGetHttpAddr(url):
if url== ” : return ”
arr=url.split(“/”)
return arr[0]+”//”+arr[2]
“””根据url取上级目录”””
def gGetHttpAddrFather(url):
if url==” : return ”
arr=url.split(“/”)
addr = arr[0]+’//’+arr[2]+ ‘/’
if len(arr)-1>3 :
for i in range(3,len(arr)-1):
addr = addr + arr[i] + ‘/’
return addr

“””根据url和上级的link取link的绝对地址”””
def gGetHttpAddrFatherAssign(url,link):
if url==” : return ”
if link==”: return ”
linkArray=link.split(“/”)
urlArray = url.split(“/”)
partLink =”
partUrl = ”
for i in range(len(linkArray)):
if linkArray[i]==’..’:
numOfFather = i + 1    #上级数
else:
partLink = partLink + ‘/’  + linkArray[i]
for i in range(len(urlArray)-1-numOfFather):
partUrl = partUrl + urlArray[i]
if i < len(urlArray)-1-numOfFather -1 :
partUrl = partUrl + ‘/’
return  partUrl + partLink

“””根据url获取其上的相关htm、html链接,返回list”””
def gGetHtmlLink(url):
#参数检查,现忽略
rtnList=[]
lines=gGetHtmlLines(url)
regx = r”””href=”?(\S+)\.htm”””
for link in gGetRegList(lines,regx):
link = gGetAbslLink(url,link) + ‘.htm’
if link not in rtnList:
rtnList.append(link)
print link
return rtnList

“””根据url,抓取其上的jpg和其链接htm上的jpg”””
def gDownloadAllJpg(url,savePath):
#参数检查,现忽略
gDownloadHtmlJpg(url,savePath)
#抓取link上的jpg
links=gGetHtmlLink(url)
for link in links:
gDownloadHtmlJpg(link,savePath)

“””test”””
def test():
read = open(“sitemap.txt”)
line=read.readline()
while line:
if len(line)>10:
u = line
save=’d:/long/’
print ‘download pic from [‘ + u +’]’
print ‘save to [‘ +save+’] …’
gDownloadHtmlJpg(u,save)
print “download finished”
line=read.readline()#如果没有这行会造成死循环
read.close

test()

 

来源:工业品营销,原文地址:http://www.jiadingqiang.com/2323.html,欢迎分享本文,转载请保留出处!

Comments: 3 Responses to “ Python下载一个网站的所有图片 ”
七月 27th, 2012
月水社seo 说 Said:

python是好东西,不用纠结什么写,只要把逻辑流程搞定了就能写出像样的python代码来。

[回复]

十二月 8th, 2011
北京多日游 说 Said:

好久没写过python的代码了

[回复]

十二月 7th, 2011
放放电影 说 Said:

这个语言用在塞班软件开发啊。呵呵

[回复]

你也来说几句:

名字*必填
邮箱*必填,不会被公开
网站
QQ群交流
最近文章
文章归档
  • 2017 (3)
  • 2016 (30)
  • 2015 (63)
  • 2014 (21)
  • 2013 (38)
  • 2012 (25)
  • 2011 (67)
  • 2010 (268)
  • 2009 (19)