class DmozSpider(BaseSpider):
    name = "dmoz"
    allowed_domains = ["dmoz.org"]
    start_urls = [
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
    ]

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        sites = hxs.select('//ul/li')
        items = []
        for site in sites:
            item = Website()
            item['name'] = site.select('a/text()').extract()
            item['url'] = site.select('a/@href').extract()
            item['description'] = site.select('text()').extract()
            items.append(item)
        return items

from scrapy.item import Item, Field

class DbItem(Item):
    title = Field()
    link = Field()

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from dirbot.items import DbItem

class DailycallerSpider(BaseSpider):
    name = "dailycaller.com"
    allowed_domains = ["dailycaller.com"]
    start_urls = ["http://dailycaller.com"]

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        item = DbItem()
        item["title"] = hxs.select("//h1/text()").extract()[0]
        item["link"] = hxs.select("//link[@rel='canonical']/@href").extract()[0]
        return item

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from dirbot.items import DbItem

from scrapy.contrib.loader import XPathItemLoader
from scrapy.contrib.loader.processor import TakeFirst

class ArstechnicaSpider(BaseSpider):
    name = "arstechnica.com"
    allowed_domains = ["arstechnica.com"]
    start_urls = ["http://arstechnica.com"]

    def parse(self, response):
        loader = XPathItemLoader(item=DbItem(), response=response)
        loader.add_xpath("title", "//meta/[@property='og:title']/@content")
        loader.add_xpath("link", "//link[@rel='canonical']/@href")
        item = loader.load_item()
        item["title"] = loader.get_value(item["title"], TakeFirst(), unicode.title)
        item["link"] = loader.get_value(item["link"], TakeFirst())
        return item

>>> fetch("http://dailycaller.com/2012...-most-rallies/")
[dailycaller.com] INFO: Spider opened
[dailycaller.com] DEBUG: Crawled (200) <GET http://dailycaller.com/...ies/>
[s] Available Scrapy objects:
[s]   hxs        <HtmlXPathSelector xpath...>
[s]   item       Website: name=None url=None
[s]   request    <GET http://dailycaller.com/...es/>
[s]   response   <200 http://dailycaller.com/...es/>
[s]   settings   <CrawlerSettings module=<module 'dirbot.settings'>
[s]   spider     <DailycallerSpider 'dailycaller.com' at 0x2484d90>
[s] Useful shortcuts:
[s]   shelp()           Shell help
[s]   fetch(req_or_url) Fetch request (or URL) and update local objects
[s]   view(response)    View response in a browser
>>> hxs.select("//title/text()")

from distillers import Distill, Distiller
class NewsDistiller(Distiller):
    title = Distill("s:headline", "og:title")
    image_url = Distill("s:associatedMedia.ImageObject/url", "og:image")
    pub_date = Distill("s:datePublished")
    author = Distill("s:author", "s:creator.Person/name")
    section = Distill("s:articleSection")
    description = Distill("s:description", "og:description")
    link = Distill("s:url", "og:url")
    site = Distill("og:site_id")
    id = Distill("s:identifier")

>>> from distillery import NewsDistiller
>>> from schemato import Schemato
>>> lnk = "http://www.cnn.com/2012/10/26/world/europe/italy-berlusconi-convicted/index.html"
>>> cnn = Schemato(lnk)
>>> distiller = NewsDistiller(cnn)
>>> distiller.distill()
{'author': "Ben Wedeman",
'id': None,
'image_url': 'http://i2.cdn.turner.com/cnn/...-video-tease.jpg',
'link': 'http://www.cnn.com/2012/10/26/world/europe/italy-berlusconi-convicted/index.html',
'pub_date': '2012-10-26T14:36:35Z',
'section': 'world',
'title': 'Ex-Italian PM Berlusconi handed 4-year prison term for tax fraud',
'site': 'CNN',
'description': 'Flamboyant former Italian Prime Minister...'}

>>> distiller.sources
{'author': 's:author',
'id': None,
'image_url': 'og:image',
'link': 'og:url',
'pub_date': 's:datePublished',
'section': 's:articleSection',
'title': 's:headline',
'site': 'og:site_id',
'description': 's:description'}