def get_children(node):
    for child in node['children']:
        yield child 
        for grandchild in get_children(child):
            yield grandchild


for line in f:
    d = json.loads(line)
    child_dic={}
    for child in get_children(d):
        if child not in child_dic.keys():
            child_dic[child["id"]]=1

当我用他的代码来查找json树有的孩子数时,
我收到一个错误
调用Python对象时超出了最大递归深度.
请帮我一样.

>>> def build_nested(depth):
...     d = {'children': []}
...     for i in range(depth - 1):
...         d['children'].append({'children': []})
...         d = d['children'][0]
...     return d
... 
>>> try:
...     len(list(get_children(build_nested(998))))
... except RuntimeError:
...     print 'too deep'
... 
997
>>> try:
...     len(list(get_children(build_nested(999))))
... except RuntimeError:
...     print 'too deep'
... 
too deep

在这种情况下不要使用递归.使用堆栈：

def get_children(node):
    stack = [node]
    while stack:
        node = stack.pop()
        stack.extend(node['children'][::-1])
        yield node

这个简单的方法首先遍历您的树深度,就像递归版本一样,以相同的顺序遍历.

这仅受您可以为Python提供的内存量的限制：

>>> try:
...     len(list(get_children(build_nested(999))))
... except RuntimeError:
...     print 'too deep'
... 
998
>>> try:
...     len(list(get_children(build_nested(10**6))))
... except RuntimeError:
...     print 'too deep'
... 
999999

然而,这无助于加载这样的物体; json库也有限制：

>>> import json
>>> json.loads('{"a":' * 100000 + '1' + '}' * 100000)
Traceback (most recent call last):
   File "<stdin>",line 1,in <module>
   File "/.../lib/python2.7/json/__init__.py",line 338,in loads
     return _default_decoder.decode(s)
   File "/.../lib/python2.7/json/decoder.py",line 366,in decode
     obj,end = self.raw_decode(s,idx=_w(s,0).end())
   File "/.../lib/python2.7/json/decoder.py",line 382,in raw_decode
     obj,end = self.scan_once(s,idx)
RuntimeError: maximum recursion depth exceeded while calling a Python object

您可以尝试使用sys.setrecursionlimit()提高Python递归限制,但要小心.将它设置得太高,你将使用段错误来破坏Python解释器.首先查看sys.getrecursionlimit()并将其作为起点来增加限制,直到您可以加载数据.

我不知道任何其他Python JSON库可以处理您看起来具有如此深度的JSON对象. jsonlib2只是段错误,ujson有1024个对象的硬编码深度限制,而且demjson也给你一个最大的递归深度错误.

码：

#!/usr/bin/env python
from bs4 import BeautifulSoup
from urllib2 import urlopen
import pickle

# open page and return soup list
def get_page_startups(page_url):
  html = urlopen(page_url).read()
  soup = BeautifulSoup(html,"lxml")
  return soup.find_all("div","startup item")

#
# Get certain text from startup soup
#
def get_name(startup):
  return startup.find("a","profile").string
def get_website(startup):
  return startup.find("a","visit")["href"]
def get_status(startup):
  return startup.find("p","status").strong.string[8:]
def get_twitter(startup):
  return startup.find("a","comment").string
def get_high_concept_pitch(startup):
  return startup.find("div","headline").find_all("em")[1].string
def get_elevator_pitch(startup):
  startup_soup = BeautifulSoup(urlopen("http://startupli.st" + startup.find("a","profile")["href"]).read(),"lxml")
  return startup_soup.find("p","desc").string.rstrip().lstrip()
def get_tags(startup):
  return startup.find("p","tags").string
def get_blog(startup):
  try:
    return startup.find("a","visit blog")["href"]
  except TypeError:
    return None
def get_facebook(startup):
  try:
    return startup.find("a","visit facebook")["href"]
  except TypeError:
    return None
def get_angellist(startup):
  try:
    return startup.find("a","visit angellist")["href"]
  except TypeError:
    return None
def get_linkedin(startup):
  try:
    return startup.find("a","visit linkedin")["href"]
  except TypeError:
    return None
def get_crunchbase(startup):
  try:
    return startup.find("a","visit crunchbase")["href"]
  except TypeError:
    return None


# site to scrape
BASE_URL = "http://startupli.st/startups/latest/"

# scrape all pages
for page_no in xrange(1,142):
  startups = get_page_startups(BASE_URL + str(page_no))

  # search soup and pickle data
  for i,startup in enumerate(startups):
    s = {}
    s[''name''] = get_name(startup)
    s[''website''] = get_website(startup)
    s[''status''] = get_status(startup)
    s[''high_concept_pitch''] = get_high_concept_pitch(startup)
    s[''elevator_pitch''] = get_elevator_pitch(startup)
    s[''tags''] = get_tags(startup)
    s[''twitter''] = get_twitter(startup)
    s[''facebook''] = get_facebook(startup)
    s[''blog''] = get_blog(startup)
    s[''angellist''] = get_angellist(startup)
    s[''linkedin''] = get_linkedin(startup)
    s[''crunchbase''] = get_crunchbase(startup)

    f = open(str(i)+".pkl","wb")
    pickle.dump(s,f)
    f.close()

  print "Done " + str(page_no)

这是引发异常后的0.pkl的内容：

http://pastebin.com/DVS1GKzz千行长！

在泡菜中有一些来自BASE_URL的HTML ……但是我没有腌制任何html字符串……

>>> from bs4 import BeautifulSoup
>>> soup = BeautifulSoup(''<div>Foo</div>'')
>>> soup.find(''div'').string
    u''Foo''
>>> type(soup.find(''div'').string)
    bs4.element.NavigableString

尝试使用str(soup.find(‘div’).string),看看它是否有帮助.另外,我不认为Pickle真的是这里最好的格式.在这种情况下,JSON更容易.