Scrapy 字段加载

from scrapy.loader import ItemLoader 
from scrapy.loader.processors import TakeFirst, MapCompose, Join  
class DemoLoader(ItemLoader):  
   default_output_processor = TakeFirst()  
   title_in = MapCompose(unicode.title) 
   title_out = Join()  
   size_in = MapCompose(unicode.strip)  
   # you can continue scraping here

from scrapy.loader import ItemLoader 
from demoproject.items import Demo  
def parse(self, response): 
   l = ItemLoader(item = Product(), response = response)
   l.add_xpath("title", "//div[@class = 'product_title']")
   l.add_xpath("title", "//div[@class = 'product_name']")
   l.add_xpath("desc", "//div[@class = 'desc']")
   l.add_css("size", "div#size]")
   l.add_value("last_updated", "yesterday")
   return l.load_item()

1. //div[@class = "product_title"] 

2. //div[@class = "product_name"]

l = ItemLoader(Product(), some_selector)
l.add_xpath("title", xpath1) # [1]
l.add_xpath("title", xpath2) # [2]
l.add_css("title", css)      # [3]
l.add_value("title", "demo") # [4]
return l.load_item()         # [5]

import scrapy 
from scrapy.loader.processors import Join, MapCompose, TakeFirst 
from w3lib.html import remove_tags  
def filter_size(value): 
   if value.isdigit(): 
      return value  
class Item(scrapy.Item): 
   name = scrapy.Field( 
      input_processor = MapCompose(remove_tags), 
      output_processor = Join(), 
   )
   size = scrapy.Field( 
      input_processor = MapCompose(remove_tags, filter_price), 
      output_processor = TakeFirst(), 
   ) 
>>> from scrapy.loader import ItemLoader 
>>> il = ItemLoader(item = Product()) 
>>> il.add_value('title', [u'Hello', u'<strong>world</strong>']) 
>>> il.add_value('size', [u'<span>100 kg</span>']) 
>>> il.load_item()

{'title': u'Hello world', 'size': u'100 kg'}

def parse_length(text, loader_context): 
   unit = loader_context.get('unit', 'cm') 
   
   # You can write parsing code of length here  
   return parsed_length

loader = ItemLoader (product)
loader.context ["unit"] = "mm"

loader = ItemLoader(product, unit = "mm")

class ProductLoader(ItemLoader):
   length_out = MapCompose(parse_length, unit = "mm")

class scrapy.loader.ItemLoader([item, selector, response, ]**kwargs)

<header>
   <a class = "social" href = "http://facebook.com/whatever">facebook</a>
   <a class = "social" href = "http://twitter.com/whatever">twitter</a>
   <a class = "email" href = "mailto:someone@example.com">send mail</a>
</header>

loader = ItemLoader(item = Item())
header_loader = loader.nested_xpath('//header')
header_loader.add_xpath('social', 'a[@class = "social"]/@href')
header_loader.add_xpath('email', 'a[@class = "email"]/@href')
loader.load_item()

from scrapy.loader.processors import MapCompose 
from demoproject.ItemLoaders import DemoLoader  
def strip_dashes(x): 
   return x.strip('-')  
class SiteSpecificLoader(DemoLoader): 
   title_in = MapCompose(strip_dashes, DemoLoader.title_in)

>>> from scrapy.loader.processors import Identity
>>> proc = Identity()
>>> proc(['a', 'b', 'c'])
['a', 'b', 'c']

>>> from scrapy.loader.processors import TakeFirst
>>> proc = TakeFirst()
>>> proc(['', 'a', 'b', 'c'])
'a'

>>> from scrapy.loader.processors import Join
>>> proc = Join()
>>> proc(['a', 'b', 'c'])
u'a b c'
>>> proc = Join('<br>')
>>> proc(['a', 'b', 'c'])
u'a<br>b<br>c'

>>> from scrapy.loader.processors import Compose
>>> proc = Compose(lambda v: v[0], str.upper)
>>> proc(['python', 'scrapy'])
'PYTHON'

>>> def filter_scrapy(x): 
   return None if x == 'scrapy' else x  
>>> from scrapy.loader.processors import MapCompose 
>>> proc = MapCompose(filter_scrapy, unicode.upper) 
>>> proc([u'hi', u'everyone', u'im', u'pythonscrapy']) 
[u'HI, u'IM', u'PYTHONSCRAPY'] 

>>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
>>> proc = SelectJmes("hello")
>>> proc({'hello': 'scrapy'})
'scrapy'
>>> proc({'hello': {'scrapy': 'world'}})
{'scrapy': 'world'}

>>> import json
>>> proc_single_json_str = Compose(json.loads, SelectJmes("hello"))
>>> proc_single_json_str('{"hello": "scrapy"}')
u'scrapy'
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('hello')))
>>> proc_json_list('[{"hello":"scrapy"}, {"world":"env"}]')
[u'scrapy']