Python网络爬虫与信息提取——正则表达式

 import re

    
 match = re.search(r'[1-9]\d{5}','bat 100081')
    
 if match:
    
     print match.group(0)   ##100081
    
    
    
    
    AI写代码

 import re

    
 match = re.match(r'[1-9]\d{5}','bit 100081')
    
 if match:
    
     print match.group(0)
    
  
    
 match = re.match(r'[1-9]\d{5}','100081 bit')
    
 if match:
    
     print match.group(0)  #100081
    
    
    
    
    AI写代码

 ls = re.findall(r'[1-9]\d{5}','bit100081 tsu100084')

    
 print ls   #['100081', '100084']
    
    
    
    
    AI写代码

 print re.split(r'[1-9]\d{5}','bit100081 tsu100084')

    
 #['bit', ' tsu', '']
    
 print re.split(r'[1-9]\d{5}','bit100081 tsu100084', maxsplit=1)
    
 #['bit', ' tsu100084']
    
    
    
    
    AI写代码

 for m in re.finditer(r'[1-9]\d{5}','bit100081 tsu100084'):

    
     if m:
    
     print m.group(0)
    
 #100081
    
 #100084
    
    
    
    
    AI写代码

 print re.sub(r'[1-9]\d{5}',':zipcode','bit100081 tsu100084')

    
 #bit:zipcode tsu:zipcode
    
    
    
    
    AI写代码

 pat = re.compile(r'[1-9]\d{5}')

    
 res = pat.search('bit 100081')
    
  
    
 regex = re.compile(pattern, flags=0)
    
    
    
    
    AI写代码

 m = re.search(r'[1-9]\d{5}','bit100081 tsu100084')

    
 print m.string  #bit100081 tsu100084
    
 print m.re  #<_sre.SRE_Pattern object at 0x10854d7b0>
    
 print m.pos  #0
    
 print m.endpos   #19
    
 print m.group(0)  #100081
    
 print m.start()  #3
    
 print m.end()   #9
    
 print m.span()   #(3,9)
    
    
    
    
    AI写代码

 def gethtmltext(url):

    
     try:
    
     r = requests.get(url, timeout=30)
    
     r.raise_for_status()
    
     r.encoding = r.apparent_encoding
    
     return r.text
    
     except:
    
     return ""
    
  
    
 def parsepage(ilt, html):
    
     try:
    
     plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"', html)
    
     tlt = re.findall(r'\"raw_title\"\:\".*?\"', html)
    
     for i in range(len(plt)):
    
         price = eval(plt[i].split(':')[1])
    
         title = eval(tlt[i].split(':')[1])
    
         ilt.append([price, title])
    
     except:
    
     print ""
    
  
    
 def printgoodslist(ilt):
    
     tplt = "{:4}\t{:8}\t{:16}"
    
     print tplt.format("序号", "价格", "商品名称")
    
     count = 0
    
     for g in ilt:
    
     print tplt.format(count, g[0], g[1])
    
  
    
 def main():
    
     goods = '书包'
    
     depth = 2
    
     start_url = 'https://s.taobao.com/search?q=' + goods
    
     infolist = []
    
     for i in range(depth):
    
     try:
    
         url = start_url + '&s=' + str(44+i)
    
         html = gethtmltext(url)
    
         parsepage(infolist, html)
    
     except:
    
         continue
    
     printgoodslist(infolist)
    
  
    
 main()
    
  
    
  
    
 实例：股票数据定向爬虫
    
  
    
  
    
 def gethtmltext(url):
    
   try:
    
     r = requests.get(url, timeout=30)
    
     r.raise_for_status()
    
     r.encoding = r.apparent_encoding
    
     return r.text
    
   except:
    
     return ""
    
  
    
 def getstocklist(lst, stockurl):
    
   html = gethtmltext(stockurl)
    
   soup = BeautifulSoup(html, 'html.parser')
    
   a = soup.find_all('a')
    
   for i in a:
    
     try:
    
       href = i.attrs['href']
    
       lst.append(re.findall(r"[s][hz]\d{6}", href))
    
     except:
    
       continue
    
  
    
 def getstockinfo(lst, stockurl, fpath):
    
   for stock in lst:
    
     url = stockurl + stock + '.html'
    
     html = gethtmltext(url)
    
     try:
    
       if html == "":
    
     continue
    
       infodict = []
    
       soup = BeautifulSoup(html, 'html.parser')
    
       stockinfo = soup.find('div', attrs={'class':'stock-bets'})
    
  
    
       name = stockinfo.find_all(attrs={'class':'bets-name'})[0]
    
       infodict.update({'股票名称':name.text.split()[0]})
    
  
    
       keylist = stockinfo.find_all('dt')
    
       valuelist = stocklist.find_all('dd')
    
       for i in range(len(keylist)):
    
     key = keylist[i].text
    
     val = valuelist[i].text
    
     infodict[key] = val
    
  
    
       with open(fpath, 'a', encoding='utf-8') as f:
    
     f.write(str(infodict) + '\n')
    
  
    
 def main():
    
   stock_list_url = 'http://quote.eastmoney.com/stocklist.html'
    
   stock_info_url = 'https://gupiao.baidu.com/stock'
    
   output_file = './baiduinfostock.txt'
    
   slist = []
    
   getstocklist(slist, stock_list_url)
    
   getstockinfo(slist, stock_info_url, output_file)
    
  
    
 main()
    
    
    
    
    AI写代码