import scrapy
from bs4 import BeautifulSoup,SoupStrainer
import urllib2
from scrapy.selector import Selector
from scrapy.http import HtmlResponse
import re
import pkgutil
from pkg_resources import resource_string
data = pkgutil.get_data("friday2","resources/urllist.txt")
class FridaySpider (scrapy.Spider):
name = 'fridayspider'
start_urls = [url.strip() for url in data]
def parse(self, response):
soup = BeautifulSoup(response.text,'lxml')
url = response.url
yield{
"title" : soup.title.string,
"url" : response.url,
}
Thanks in advance :) !!