注册
登录
| 关键词： nbsp URL match regex MEDIA LOCAL print Exception exceptionLog SITE |
#IMPORTS--------------------------------------------------------------------------------------------------------------import re,urllib,os #GLOBALS--------------------------------------------------------------------------------------------------------------#Global that holds all our spidered URL'SsiteSet = [] #Open our log filessiteLog = open("sitelog","w")mediaLog = open("medialog","w")exceptionLog = open("exceptionlog","w") #Spider recursive paramaters RECURSION == "On" means depth first search, "Off" == BFS RECURSION = "Off"MAXDEPTH = 1SITE_FILE = "siteList2.txt" #REGEXS#Compile to regex one simple the other complex to allow for fairly funky url'sregex_URL_CONTAINS = ""regex_LOCAL_HREF = "(\"\S+(.htm|.html)\")"#regex_HREF_Simple = "(http://\S+(\.com|\.htm|\.html|\.net|\.org))"#regex_HREF_Complex = "(http://\S+(\.com|\.net|\.org|\.htm|\.html)\S*(\.com|\.net|\.org|\.htm|\.html)(\S*)(?=\"))"regex_HREF_CONTAINS = "" regex_HREF_Simple = "((http://\S/?[^<>\"']*))" #IIIIIIdontkonw #This can be anything you want try .rpm .txt .html .php .cgi .gzipregex_LOCAL_MEDIA = "(\"\S+(\.mpg|\.avi|\.mpeg|\.rm)\")" regex_MEDIA = "(http://\S+(\.mpg|\.avi|\.mpeg|\.rm))"#regex_MEDIA = "(http://\S+(\.mpg|\.avi|\.mpeg|\.rm))"regex_MEDIA_NAME_CONTAINS = "" #---------------------------------------------------------------------------------------------------------------------- def findRefs(site,lines=[]): hrefs = [] try: #For each line of the URL for x in lines: #REGEX match for a simple and a complex HREF matchSimple = re.search(regex_HREF_Simple,x) mediaCheck = re.search(regex_MEDIA,x) #matchComplex = re.search(regex_HREF_Complex,x) #Disable Local file naming EX: ./index.html vs http://local/inde.html matchLocal = None #re.search(regex_LOCAL_HREF,x) #Make sure we got a match if (mediaCheck == "" or mediaCheck == None): if (matchSimple != "" and matchSimple != None) or (matchLocal != "" and matchLocal != None): #Order the check to make sure the complex one isnt overridden if matchLocal != None and matchLocal != "": url = matchLocal.string[matchLocal.start():matchLocal.end()] url = site + '/' + url[1:-1] if matchSimple != None and matchSimple != "": url = matchSimple.string[matchSimple.start():matchSimple.end()] url=url.split(" ")[0] #if url.find(" "): # temp=url.split(" ") # url=temp[0] print url #Here we need to check if hrefs is already in our set of to be spidered sites found = "false" for y in siteSet: if y == url: found = "true" #print "Duplicate url!" #If the site is not in our list then add it to the set to be spidered if found == "false": #print "New url appended to spider list = %s" % url siteSet.append(url) hrefs.append(url) siteLog.write(url) siteLog.write("\n") #Flush the contents of the buffer to the file siteLog.flush() except: print "Exception in findRefs :(" exceptionLog.write("Exception in findRefs :(\n") #Return a new list or HREFS to be spidered return hrefs#----------------------------------------------------------------------------------------------------------------------def findMedia(site,lines=[]): try: #The downloaded files we found media = [] for x in lines: #Search for a media match match = re.search(regex_MEDIA,x) local_match = None #re.search(regex_LOCAL_MEDIA,x) if local_match != None and local_match != "": url = local_match.string[local_match.start():local_match.end()] #Search to see if there is htm/html in the sites url temp_match = re.search("\S+(.htm|.html)",x) #if so then remove from the last / to html if temp_match != None and temp_match != "": backPnt = len(site)-1 while site[backPnt] != '/': --backPnt site = site[0:backPnt] url = site + '/' + url[1:-1] print url print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++changed url is now " + url if match != None and match != "": url = match.string[match.start():match.end()] if (local_match != None and local_match != "") or (match != None and match != ""): end = len(url)-1 #Get the file name itself not just the whole URL while end > 0: if url[end] == '/': end = end + 1 break end = end - 1 name = url[end:] #We need to change this to check to see if it exists locally, if not then download it # this saves MUCH time existing = os.listdir("./") found = 0 for x in existing: if x == name: found = 1 #print "*****Duplicate File :| [%s] Source[ %s ]*****" % (name,site) if found == 0: print "*****New File :) [%s] downloading...*****" % url try: blankFile = open(name,"w") urllib.urlretrieve(url,name) #And download it media.append(url) mediaLog.write(url) mediaLog.write("\n") except: print "Error: Downloading; File Exception!" exceptionLog.write("Error downloading %s\n" % url) mediaLog.flush() except: print "Exception in find media :(" exceptionLog.write("Exception in find media :(\n") return media#----------------------------------------------------------------------------------------------------------------------#The controlling recursive function spider using Depth First Recursion Breadth First would be better.def spider(site,depth): #Continue recursively untill we reach the maximum allowed depth if depth <= MAXDEPTH: if site != None and site != "" : try: #Connect to the page, read all the lines print "Current depth = %d site = %s" % (depth,site) usock = urllib.urlopen(site) lines = usock.readlines() usock.close() #Find any media on the page media = findMedia(site,lines) #Find any HREFS on the page hrefs = findRefs(site,lines) #For each HREF run spider on it for x in hrefs: spider(x,depth+1) except: print "Exception in spider :(" exceptionLog.write("Exception in spider :( at %s depth %d\n" % (site,depth)) #else: # print "----maxdepth reached, returning..."#----------------------------------------------------------------------------------------------------------------------#Try to implement spider using BFS instead of DFSdef BFSpider(L = []): #Connect to the page, read all the lines try: while(len(L) > 0): try: usock = urllib.urlopen(L[0]) except: print "Exception connecting to :( %s" % L[0] del(L[0]) continue exceptionLog.write("Exception connecting to :( %s" % L[0]) lines = usock.readlines() usock.close() media = findMedia(L[0],lines) hrefs = findRefs(L[0],lines) print "<From %s> Appended: %s \n{ Media: %s }" % (L[0], hrefs ,media) del(L[0]) except: print "Exception in BFSpider :( %s" % L exceptionLog.write("Exception in BFSpider :( %s\n" % L) #----------------------------------------------------------------------------------------------------------------------# Main Execution point def run(): file = open(SITE_FILE,"r") #Open the site file for reading (should be an argument) sites = file.readlines() #Read in all the lines from the file a = 0 if RECURSION == "On": try: for x in sites: spider(x,a) except: print "Exception in Main :(" else: BFSpider(sites) siteLog.close() mediaLog.close() exceptionLog.close() #----------------------------------------------------------------------------------------------------------------------##NOTES# |
|
声明：文章版权归原作者所有 部分文章转自互联网 如有侵权请联系
[邮箱地址] 删除
|
