CKAN: Ticket Query

#372: Fix system limits on CKAN for DGU

johnbywater — Fri, 23 Jul 2010 11:49:12 GMT

Set limits in /etc/security/limits.conf so that we can always ssh in at least. Requested by DGU.

#436: Investigate exception: resource search JSON

dread — Thu, 19 Aug 2010 08:35:59 GMT

Here's the dump from 22:10 last night:

URL: http://ckan.net/api/search/resource?all_fields=1&offset=0&limit=20&qjson=%3Cspan%20class= Module weberror.errormiddleware:162 in call << traceback_supplement = Supplement, self, environ

sr_checker = ResponseStartChecker?(start_response) app_iter = self.application(environ, sr_checker) return self.make_catching_iter(app_iter, environ, sr_checker)

except:

app_iter = self.application(environ, sr_checker)

Module beaker.middleware:73 in call << self.cache_manager)

environ[self.environ_key] = self.cache_manager return self.app(environ, start_response)

return self.app(environ, start_response)

Module beaker.middleware:152 in call << headers.append(('Set-cookie', cookie))

return start_response(status, headers, exc_info)

return self.wrap_app(environ, session_start_response)

def _get_session(self):

return self.wrap_app(environ, session_start_response)

Module routes.middleware:130 in call << environSCRIPT_NAME? = environSCRIPT_NAME?[:-1]

response = self.app(environ, start_response)

# Wrapped in try as in rare cases the attribute will be gone already

response = self.app(environ, start_response)

Module pylons.wsgiapp:125 in call <<

controller = self.resolve(environ, start_response) response = self.dispatch(controller, environ, start_response)

if 'paste.testing_variables' in environ and hasattr(response,

response = self.dispatch(controller, environ, start_response)

Module pylons.wsgiapp:324 in dispatch << if log_debug:

log.debug("Calling controller class with WSGI interface")

return controller(environ, start_response)

def load_test_env(self, environ):

return controller(environ, start_response)

Module ckan.lib.base:73 in call << # available in environpylons.routes_dict?

try:

return WSGIController.call(self, environ, start_response)

finally:

model.Session.remove()

return WSGIController.call(self, environ, start_response)

Module pylons.controllers.core:221 in call << return response(environ, self.start_response)

response = self._dispatch_call() if not start_response_called:

self.start_response = start_response

response = self._dispatch_call()

Module pylons.controllers.core:172 in _dispatch_call << req.environpylons.action_method? = func

response = self._inspect_call(func)

else:

if log_debug:

response = self._inspect_call(func)

Module pylons.controllers.core:107 in _inspect_call << func.name, args)

try:

result = self._perform_call(func, args)

except HTTPException, httpe:

if log_debug:

result = self._perform_call(func, args)

Module pylons.controllers.core:60 in _perform_call << """Hide the traceback for everything above this method"""

traceback_hide = 'before_and_this' return func(args)

def _inspect_call(self, func):

return func(args)

Module ckan.controllers.rest:400 in search << response.status_int = 400

return gettext('Blank qjson parameter')

params = json.loads(request.paramsqjson?)

elif request.params.values() and request.params.values() != [u] and request.params.values() != [u'1']:

params = request.params

params = json.loads(request.paramsqjson?)

Module simplejson:384 in loads << parse_constant is None and object_pairs_hook is None

and not use_decimal and not kw):

return _default_decoder.decode(s)

if cls is None:

cls = JSONDecoder

return _default_decoder.decode(s)

Module simplejson.decoder:402 in decode << """

obj, end = self.raw_decode(s, idx=_w(s, 0).end()) end = _w(s, end).end() if end != len(s):

obj, end = self.raw_decode(s, idx=_w(s, 0).end())

Module simplejson.decoder:420 in raw_decode << obj, end = self.scan_once(s, idx)

except StopIteration?:

raise JSONDecodeError("No JSON object could be decoded", s, idx)

return obj, end

raise JSONDecodeError("No JSON object could be decoded", s, idx)

JSONDecodeError: No JSON object could be decoded: line 1 column 0 (char 0) CGI Variables DOCUMENT_ROOT '/htdocs' GATEWAY_INTERFACE 'CGI/1.1' HTTP_ACCEPT 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8, application/json' HTTP_ACCEPT_CHARSET 'ISO-8859-1,utf-8;q=0.7,*;q=0.7' HTTP_ACCEPT_ENCODING 'gzip,deflate' HTTP_ACCEPT_LANGUAGE 'en-us,en;q=0.5' HTTP_CONNECTION 'keep-alive' HTTP_COOKIE 'utma=27730403.1245320310.1281386803.1281386803.1282164955.2; utmz=27730403.1282164955.2.2.utmcsr=jira|utmccn=(referral)|utmcmd=referral|utmcct=/browse/PLATFORM-892; utmb=27730403.3.10.1282164955; utmc=27730403' HTTP_HOST 'ckan.net' HTTP_KEEP_ALIVE '300' HTTP_REFERER 'http://jira/browse/PLATFORM-892' HTTP_USER_AGENT 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7' PATH '/usr/local/bin:/usr/bin:/bin' PATH_INFO '/api/search/resource' PATH_TRANSLATED '/home/okfn/var/srvc/ckan.net/pyenv/bin/ckan.net.py/api/search/resource' QUERY_STRING 'all_fields=1&offset=0&limit=20&qjson=%3Cspan%20class=' REMOTE_ADDR '64.235.97.218' REMOTE_PORT '20720' REQUEST_METHOD 'GET' REQUEST_URI '/api/search/resource?all_fields=1&offset=0&limit=20&qjson=%3Cspan%20class=' SCRIPT_FILENAME '/home/okfn/var/srvc/ckan.net/pyenv/bin/ckan.net.py' SCRIPT_URI 'http://ckan.net/api/search/resource' SCRIPT_URL '/api/search/resource' SERVER_ADDR '10.226.226.118' SERVER_ADMIN '[no address given]' SERVER_NAME 'ckan.net' SERVER_PORT '80' SERVER_PROTOCOL 'HTTP/1.1' SERVER_SIGNATURE '<address>Apache/2.2.9 (Debian) mod_wsgi/2.5 Python/2.5.2 Server at ckan.net Port 80</address>\n' SERVER_SOFTWARE 'Apache/2.2.9 (Debian) mod_wsgi/2.5 Python/2.5.2' WSGI Variables application <beaker.middleware.CacheMiddleware? object at 0xa1c13ec> beaker.cache <beaker.cache.CacheManager? object at 0xa1c142c> beaker.get_session <bound method SessionMiddleware?._get_session of <beaker.middleware.SessionMiddleware? object at 0xa1c12ac>> beaker.session {'_accessed_time': 1282165818.0880959, '_creation_time': 1282165818.0880959} mod_wsgi.application_group 'ckan.net|' mod_wsgi.callable_object 'application' mod_wsgi.listener_host mod_wsgi.listener_port '80' mod_wsgi.process_group mod_wsgi.reload_mechanism '0' mod_wsgi.script_reloading '1' mod_wsgi.version (2, 5) paste.cookies (<SimpleCookie: __utma='27730403.1245320310.1281386803.1281386803.1282164955.2' __utmb='27730403.3.10.1282164955' __utmc='27730403' __utmz='27730403.1282164955.2.2.utmcsr=jira|utmccn=(referral)|utmcmd=referral|utmcct=/browse/PLATFORM-892'>, 'utma=27730403.1245320310.1281386803.1281386803.1282164955.2; utmz=27730403.1282164955.2.2.utmcsr=jira|utmccn=(referral)|utmcmd=referral|utmcct=/browse/PLATFORM-892; utmb=27730403.3.10.1282164955; utmc=27730403') paste.parsed_querystring ([('all_fields', '1'), ('offset', '0'), ('limit', '20'), ('qjson', '<span class=')], 'all_fields=1&offset=0&limit=20&qjson=%3Cspan%20class=') paste.registry <paste.registry.Registry object at 0x130ed84c> paste.throw_errors True pylons.action_method <bound method RestController?.search of <ckan.controllers.rest.RestController? object at 0xe9bbe0c>> pylons.controller <ckan.controllers.rest.RestController? object at 0xe9bbe0c> pylons.environ_config {'session': 'beaker.session', 'cache': 'beaker.cache'} pylons.pylons <pylons.util.PylonsContext? object at 0xe9bbe8c> pylons.routes_dict {'action': u'search', 'controller': u'rest', 'register': u'resource'} repoze.who.logger <logging.Logger instance at 0xa3cb0cc> repoze.who.plugins {'openid': <OpenIdIdentificationPlugin? 170067148>, 'auth_tkt': <AuthTktCookiePlugin? 171739788>} routes.route <routes.route.Route object at 0xa102fac> routes.url <routes.util.URLGenerator object at 0x13a5a3cc> webob._parsed_query_vars (GET([('all_fields', '1'), ('offset', '0'), ('limit', '20'), ('qjson', '<span class=')]), 'all_fields=1&offset=0&limit=20&qjson=%3Cspan%20class=') webob.adhoc_attrs {'language': 'en-us'} wsgi process 'Multi process AND threads (?)' wsgi.file_wrapper <built-in method file_wrapper of mod_wsgi.Adapter object at 0x12f53530> wsgiorg.routing_args (<routes.util.URLGenerator object at 0x13a5a3cc>, {'action': u'search', 'controller': u'rest', 'register': u'resource'})

#437: Buildbot test failures - ascii codec

dread — Thu, 19 Aug 2010 13:14:00 GMT

On today's buildbot: http://buildbot.okfn.org/builders/buildbot-test/builds/201

2 failures about ascii (ignore other 2)

#459: Versions on branches are broken

johnbywater — Fri, 20 Aug 2010 10:26:13 GMT

#654: Harvest sources and jobs should return 404 when missing (not 500)

johnbywater — Wed, 22 Sep 2010 15:51:57 GMT

#349: The system shall support authorised users to edit packages on data.gov.uk

johnbywater — Mon, 14 Jun 2010 13:57:31 GMT

#432: Creating package over REST gives 500 error

dread — Tue, 17 Aug 2010 17:11:41 GMT

This occurs when CKAN is run with mod_wsgi (not under paster).

This is because of a unicode header being creating in rest.py.

#434: Logged in user to si.ckan.net causes genshi exception

dread — Wed, 18 Aug 2010 11:21:55 GMT

Here's the trace:

[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197] Error - <type 'exceptions.IndexError'>: list index out of range
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197] mod_wsgi (pid=6195): Exception occurred processing WSGI script '/home/okfn/var/srvc/si.ckan.net/py
env/bin/si.ckan.net.py'.
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197] Traceback (most recent call last):
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/paste/cascade.py", line
130, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return self.apps[-1](environ, start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/paste/registry.py", line
 375, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     app_iter = self.application(environ, start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/repoze/who/middleware.py
", line 107, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     app_iter = app(environ, wrapper.wrap_start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/pylons/middleware.py", l
ine 214, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     self.app, new_environ, catch_exc_info=True)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/pylons/util.py", line 94
, in call_wsgi_application
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     app_iter = application(environ, start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/weberror/errormiddleware
.py", line 156, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return self.application(environ, start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/beaker/middleware.py", l
ine 73, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return self.app(environ, start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/beaker/middleware.py", l
ine 152, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return self.wrap_app(environ, session_start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/routes/middleware.py", l
ine 130, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     response = self.app(environ, start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/pylons/wsgiapp.py", line
 125, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     response = self.dispatch(controller, environ, start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/pylons/wsgiapp.py", line
 324, in dispatch
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return controller(environ, start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/src/ckan/ckan/lib/base.py", line 73, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return WSGIController.__call__(self, environ, start_response)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/pylons/controllers/core.
py", line 221, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     response = self._dispatch_call()
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/pylons/controllers/core.
py", line 172, in _dispatch_call
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     response = self._inspect_call(func)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/pylons/controllers/core.
py", line 107, in _inspect_call
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     result = self._perform_call(func, args)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/pylons/controllers/core.
py", line 60, in _perform_call
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return func(**args)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/src/ckan/ckan/controllers/error.py", line 32, in doc
ument
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     ckan_template = render('error_document_template.html')
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/src/ckan/ckan/lib/base.py", line 38, in render
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return render_template()
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/src/ckan/ckan/lib/base.py", line 36, in render_templ
ate
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return literal(stream.render(method=method, encoding=None))
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/core.py", line 18
3, in render
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return encode(generator, method=method, encoding=encoding, out=out)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/output.py", line
57, in encode
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     return _encode(''.join(list(iterator)))
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/output.py", line 339, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for kind, data, pos in stream:
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/output.py", line 670, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for kind, data, pos in stream:
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/output.py", line 771, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for kind, data, pos in chain(stream, [(None, None, None)]):
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/output.py", line 586, in __call__
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for ev in stream:
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/core.py", line 288, in _ensure
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for event in stream:
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/template/base.py", line 618, in _include
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for event in tmpl.generate(ctxt, **vars):
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/template/base.py", line 618, in _include
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for event in tmpl.generate(ctxt, **vars):
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/template/base.py", line 605, in _include
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for event in stream:
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/template/markup.py", line 327, in _match
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for event in stream:
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/template/base.py", line 545, in _flatten
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for kind, data, pos in stream:
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/filters/i18n.py", line 177, in _generate
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     for event in msgbuf.translate(gettext(msgbuf.format())):
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/filters/i18n.py", line 1029, in translate
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     parts = parse_msg(string)
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]   File "/home/okfn/var/srvc/si.ckan.net/pyenv/lib/python2.5/site-packages/genshi/filters/i18n.py", line 1143, in parse_msg
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197]     parts.append((stack[-1], string))
[Wed Aug 18 11:10:19 2010] [error] [client 89.212.104.197] IndexError: list index out of range

ckan: 1e403927f2bb (metastable) genshi: 0.6 lang: slovenian

#500: Exception from diff

dread — Mon, 23 Aug 2010 08:43:53 GMT

Investigate exception occured occasionally in last couple of days on ckan.net:

WebApp Error: <type 'exceptions.AttributeError'>: 'NoneType' object has no attribute 'key'
URL: http://ckan.net/revision/diff/dbtune-audioscrobbler?diff=66a47b9e-232a-4838-8674-66fa1a5c76e1&oldid=a99c98be-767a-4e49-9025-2472b2d18b9c
Module weberror.errormiddleware:162 in __call__
<<              __traceback_supplement__ = Supplement, self, environ
                   sr_checker = ResponseStartChecker(start_response)
                   app_iter = self.application(environ, sr_checker)
                   return self.make_catching_iter(app_iter, environ, sr_checker)
               except:
>>  app_iter = self.application(environ, sr_checker)
Module beaker.middleware:73 in __call__
<<                                                     self.cache_manager)
               environ[self.environ_key] = self.cache_manager
               return self.app(environ, start_response)
>>  return self.app(environ, start_response)
Module beaker.middleware:152 in __call__
<<                          headers.append(('Set-cookie', cookie))
                   return start_response(status, headers, exc_info)
               return self.wrap_app(environ, session_start_response)
           def _get_session(self):
>>  return self.wrap_app(environ, session_start_response)
Module routes.middleware:130 in __call__
<<                  environ['SCRIPT_NAME'] = environ['SCRIPT_NAME'][:-1]
               response = self.app(environ, start_response)
               # Wrapped in try as in rare cases the attribute will be gone already
>>  response = self.app(environ, start_response)
Module pylons.wsgiapp:125 in __call__
<<
               controller = self.resolve(environ, start_response)
               response = self.dispatch(controller, environ, start_response)
               if 'paste.testing_variables' in environ and hasattr(response,
>>  response = self.dispatch(controller, environ, start_response)
Module pylons.wsgiapp:324 in dispatch
<<          if log_debug:
                   log.debug("Calling controller class with WSGI interface")
               return controller(environ, start_response)
           def load_test_env(self, environ):
>>  return controller(environ, start_response)
Module ckan.lib.base:73 in __call__
<<          # available in environ['pylons.routes_dict']
               try:
                   return WSGIController.__call__(self, environ, start_response)
               finally:
                   model.Session.remove()
>>  return WSGIController.__call__(self, environ, start_response)
Module pylons.controllers.core:221 in __call__
<<                  return response(environ, self.start_response)
               response = self._dispatch_call()
               if not start_response_called:
                   self.start_response = start_response
>>  response = self._dispatch_call()
Module pylons.controllers.core:172 in _dispatch_call
<<              req.environ['pylons.action_method'] = func
                   response = self._inspect_call(func)
               else:
                   if log_debug:
>>  response = self._inspect_call(func)
Module pylons.controllers.core:107 in _inspect_call
<<                        func.__name__, args)
               try:
                   result = self._perform_call(func, args)
               except HTTPException, httpe:
                   if log_debug:
>>  result = self._perform_call(func, args)
Module pylons.controllers.core:60 in _perform_call
<<          """Hide the traceback for everything above this method"""
               __traceback_hide__ = 'before_and_this'
               return func(**args)
           def _inspect_call(self, func):
>>  return func(**args)
Module ckan.controllers.revision:119 in diff
<<          c.revision_to = model.Session.query(model.Revision).get(
                   request.params.getone('diff'))
               diff = pkg.diff(c.revision_to, c.revision_from)
               c.diff = diff.items()
               c.diff.sort()
>>  diff = pkg.diff(c.revision_to, c.revision_from)
Module ckan.model.package:340 in diff
<<                              display_id = to_obj_rev.tag.name
                               elif obj_class.__name__ == 'PackageExtra':
                                   display_id = to_obj_rev.key
                               else:
                                   display_id = related_obj_id[:4]
>>  display_id = to_obj_rev.key
AttributeError: 'NoneType' object has no attribute 'key'
CGI Variables
DOCUMENT_ROOT	'/htdocs'
GATEWAY_INTERFACE	'CGI/1.1'
HTTP_ACCEPT	'*/*'
HTTP_ACCEPT_ENCODING	'gzip'
HTTP_ACCEPT_LANGUAGE	'zh-cn,zh-tw'
HTTP_CONNECTION	'close'
HTTP_HOST	'ckan.net'
HTTP_USER_AGENT	'Baiduspider+(+http://www.baidu.com/search/spider.htm)'
PATH	'/usr/local/bin:/usr/bin:/bin'
PATH_INFO	'/revision/diff/dbtune-audioscrobbler'
PATH_TRANSLATED	'/home/okfn/var/srvc/ckan.net/pyenv/bin/ckan.net.py/revision/diff/dbtune-audioscrobbler'
QUERY_STRING	'diff=66a47b9e-232a-4838-8674-66fa1a5c76e1&oldid=a99c98be-767a-4e49-9025-2472b2d18b9c'
REMOTE_ADDR	'123.125.66.32'
REMOTE_PORT	'63767'
REQUEST_METHOD	'GET'
REQUEST_URI	'/revision/diff/dbtune-audioscrobbler?diff=66a47b9e-232a-4838-8674-66fa1a5c76e1&oldid=a99c98be-767a-4e49-9025-2472b2d18b9c'
SCRIPT_FILENAME	'/home/okfn/var/srvc/ckan.net/pyenv/bin/ckan.net.py'
SCRIPT_URI	'http://ckan.net/revision/diff/dbtune-audioscrobbler'
SCRIPT_URL	'/revision/diff/dbtune-audioscrobbler'
SERVER_ADDR	'10.226.226.118'
SERVER_ADMIN	'[no address given]'
SERVER_NAME	'ckan.net'
SERVER_PORT	'80'
SERVER_PROTOCOL	'HTTP/1.1'
SERVER_SIGNATURE	'<address>Apache/2.2.9 (Debian) mod_wsgi/2.5 Python/2.5.2 Server at ckan.net Port 80</address>\n'
SERVER_SOFTWARE	'Apache/2.2.9 (Debian) mod_wsgi/2.5 Python/2.5.2'
WSGI Variables
application	<beaker.middleware.CacheMiddleware object at 0x9f603ec>
beaker.cache	<beaker.cache.CacheManager object at 0x9f6042c>
beaker.get_session	<bound method SessionMiddleware._get_session of <beaker.middleware.SessionMiddleware object at 0x9f602ac>>
beaker.session	{'_accessed_time': 1282385101.4243281, '_creation_time': 1282385101.4243281}
mod_wsgi.application_group	'ckan.net|'
mod_wsgi.callable_object	'application'
mod_wsgi.listener_host	''
mod_wsgi.listener_port	'80'
mod_wsgi.process_group	''
mod_wsgi.reload_mechanism	'0'
mod_wsgi.script_reloading	'1'
mod_wsgi.version	(2, 5)
paste.cookies	(<SimpleCookie: >, '')
paste.parsed_querystring	([('diff', '66a47b9e-232a-4838-8674-66fa1a5c76e1'), ('oldid', 'a99c98be-767a-4e49-9025-2472b2d18b9c')], 'diff=66a47b9e-232a-4838-8674-66fa1a5c76e1&oldid=a99c98be-767a-4e49-9025-2472b2d18b9c')
paste.registry	<paste.registry.Registry object at 0x104552ec>
paste.throw_errors	True
pylons.action_method	<bound method RevisionController.diff of <ckan.controllers.revision.RevisionController object at 0xfb17aec>>
pylons.controller	<ckan.controllers.revision.RevisionController object at 0xfb17aec>
pylons.environ_config	{'session': 'beaker.session', 'cache': 'beaker.cache'}
pylons.pylons	<pylons.util.PylonsContext object at 0x10286d4c>
pylons.routes_dict	{'action': u'diff', 'controller': u'revision', 'id': u'dbtune-audioscrobbler'}
repoze.who.logger	<logging.Logger instance at 0xa16e0cc>
repoze.who.plugins	{'openid': <OpenIdIdentificationPlugin 167584972>, 'auth_tkt': <AuthTktCookiePlugin 169253516>}
routes.route	<routes.route.Route object at 0x9f3690c>
routes.url	<routes.util.URLGenerator object at 0xfd8d7cc>
webob._parsed_query_vars	(GET([('diff', '66a47b9e-232a-4838-8674-66fa1a5c76e1'), ('oldid', 'a99c98be-767a-4e49-9025-2472b2d18b9c')]), 'diff=66a47b9e-232a-4838-8674-66fa1a5c76e1&oldid=a99c98be-767a-4e49-9025-2472b2d18b9c')
webob.adhoc_attrs	{'language': 'en-us'}
wsgi process	'Multi process AND threads (?)'
wsgi.file_wrapper	<built-in method file_wrapper of mod_wsgi.Adapter object at 0x103a5bf0>
wsgiorg.routing_args	(<routes.util.URLGenerator object at 0xfd8d7cc>, {'action': u'diff', 'controller': u'revision', 'id': u'dbtune-audioscrobbler'})

#514: Inconsistent use of 'location' header in API

dread — Wed, 25 Aug 2010 17:29:17 GMT

When you create a package then the 'location' header gets set. This doesn't happen for any other domain objects. I think this should be consistent - either none or all.

I've removed the info about the header in the docs in the meantime.

#516: Double-escaped text visible - revision page

dread — Thu, 26 Aug 2010 13:32:19 GMT

At: http://ckan.net/revision you can see: "Next »" on pager This doesn't seem to be a problem with earlier (1.0.1) version.

#532: Just a test - please ignore

johnbywater — Fri, 27 Aug 2010 11:00:42 GMT

#534: buildbot config - database machine moved

dread — Mon, 30 Aug 2010 10:52:48 GMT

Configure buildbot to use postgres db which has been moved to eu5.okfn.org

pudo said: Could you perhaps also comment on http://knowledgeforge.net/okfn/tasks/ticket/466 with a hint on how to update this properly? I only edited /home/buildslave/okfn/full/build/buildandsmoke/buildandsmoke.ini which was the only occurence of the DB credentials I could find there.

#535: genshi error when logged into sl.ckan.net

dread — Mon, 30 Aug 2010 10:56:14 GMT

Genshi exception when rendering the page whilst logged in to sl.ckan.net.

#558: Sort extras, relationships by key/object alphabetically

pudo — Fri, 03 Sep 2010 19:38:00 GMT

#561: Deleted packages are returned in the API

pudo — Mon, 06 Sep 2010 12:19:38 GMT

Anja is reporting this, severe bug, I think.

#562: OpenID Login fails using Yahoo and Google

pudo — Wed, 08 Sep 2010 10:18:39 GMT

reported by LOD folks.

subsidary bug: error messages from OpenID providers are not shown on the form return page.

#741: Extension / Plugin system for CKAN

ollyc — Mon, 25 Oct 2010 08:54:50 GMT

Create plugin system using <http://pypi.python.org/pypi/pyutilib.component.core/>

Specification in CEP-003 (browser:doc/cep/cep-003.txt) and <http://okfnpad.org/ckan-plugins>

#408: Integrate CMS via proxying

pudo — Mon, 09 Aug 2010 10:09:41 GMT

As sysadmin/user of CKAN I want to integrate a CMS to support easy addition of additional content ...
I want to integrate with navigation
As a minimum I want to be able to add content pages (more stuff would be e.g. comments, sidebars, main menu ...)

Why: CMS are great at creating content and have very good editors and theme support ...

Previous work

http://groups.google.com/group/pylons-discuss/browse_thread/thread/f8969e28b0a007c7

deliverance: retheming on the fly

http://knowledgeforge.net/shakespeare/hg/file/677eef9402e6/shakespeare/controllers/template.py

Rewrite urls for proxying
You need a theme to substitute into
pylons

Extreme version:

2 sources -> pylons/ckan app, cms
1 theme source: ckan or cms

1 frontend/proxy app which re-themes everything

Owned by CKAN:

package/
tag/
group/
api/
/

Everything else proxied.

#410: Latest comments widget on home page

pudo — Mon, 09 Aug 2010 10:12:13 GMT

This would be an extension to the disqus plugin that would patch itself into home/index.

#411: Deploy and test Varnish on eu3

pudo — Mon, 09 Aug 2010 10:21:15 GMT

Need to figure out cache invalidation for CKAN
Evaluate against an internal cache (i.e. memcached)

#428: Daily dump of DGU CKAN available

dread — Fri, 13 Aug 2010 12:40:51 GMT

End users want to get hold of a JSON & CSV dump of the DGU data.

#429: ONS data is imported daily to DGU

dread — Fri, 13 Aug 2010 12:41:34 GMT

#501: Read-only maintenance mode

pudo — Mon, 23 Aug 2010 09:10:17 GMT

CKAN should have a read-only maintenance mode with a nice little banner on all pages, appropriate REST messages etc. Bonus points if this is triggered via an environment variable and thus can be triggered by the surrounding apache.

#545: The system shall support creating packages via the Form API

johnbywater — Wed, 01 Sep 2010 11:08:30 GMT

#575: The system shall notify remote metadata harvesting errors

johnbywater — Thu, 09 Sep 2010 15:59:19 GMT

#652: Allow for addition and removal of groups on package/edit

pudo — Mon, 20 Sep 2010 15:55:34 GMT

Much more natural place to do this, even at the risk of overloading the package edit form.

#666: The system shall support updating packages from new versions of previously harvested dataset or service metadata

johnbywater — Mon, 27 Sep 2010 12:26:14 GMT

#690: The system shall support Gemini2 service records

johnbywater — Mon, 11 Oct 2010 18:43:33 GMT

Wish to fold service records into package resources. Trouble is that service records don't necessarily refer to a data record, so should this be a new package?

#693: Support ETag in cache decorator

wwaites — Mon, 11 Oct 2010 21:04:11 GMT

... and make cache-control header optional

#702: The system shall support changing package groups when editing a package

johnbywater — Mon, 18 Oct 2010 12:09:38 GMT

#454: Import ONS data via API

dread — Fri, 20 Aug 2010 10:24:11 GMT

#456: Daily dump

dread — Fri, 20 Aug 2010 10:25:17 GMT

#457: Dump database on a daily basis

dread — Fri, 20 Aug 2010 10:25:21 GMT

#462: Visitor downloads a daily database dump

johnbywater — Fri, 20 Aug 2010 11:55:05 GMT

#471: API user sends API key in correctly named header

johnbywater — Fri, 20 Aug 2010 13:48:08 GMT

#472: API user sends API key in incorrectly named header

johnbywater — Fri, 20 Aug 2010 13:48:28 GMT

#473: API user discovers correct header for sending API key

johnbywater — Fri, 20 Aug 2010 13:50:03 GMT

#491: Get form for creating harvest source entity

johnbywater — Sat, 21 Aug 2010 21:43:28 GMT

#492: Submit harvest source create form response to the API

johnbywater — Sat, 21 Aug 2010 21:44:02 GMT

#493: Get harvest source entity

johnbywater — Sat, 21 Aug 2010 21:44:43 GMT

#494: Get form for updating remote metadata entity

johnbywater — Sat, 21 Aug 2010 21:45:06 GMT

#495: Put form for updating remote metadata entity

johnbywater — Sat, 21 Aug 2010 21:45:50 GMT

#546: Get the package create form from the API

johnbywater — Wed, 01 Sep 2010 11:10:14 GMT

#547: Submit package create form response to the API

johnbywater — Wed, 01 Sep 2010 11:11:27 GMT

#567: Post new harvest job for given harvest source

johnbywater — Thu, 09 Sep 2010 13:45:43 GMT

#577: Get remote metadata harvest job errors

johnbywater — Sat, 11 Sep 2010 12:49:41 GMT

#578: Get remote metadata harvest job

johnbywater — Sat, 11 Sep 2010 12:50:07 GMT

#579: Delete remote metadata harvest job

johnbywater — Sat, 11 Sep 2010 12:50:40 GMT

#598: List remote metadata entities for given publisher

johnbywater — Sat, 11 Sep 2010 14:02:10 GMT

#650: WUI editing for Authorization Groups

pudo — Sun, 19 Sep 2010 09:54:33 GMT

Copy & adapt Group controller tests to suit AuthorizationGroups?
Copy & adapt Group controllers and templates for Authorization Groups

#655: Return status code 404 when harvest source is not found

johnbywater — Thu, 23 Sep 2010 14:33:19 GMT

#656: Return status code 404 when harvesting job is not found

johnbywater — Thu, 23 Sep 2010 14:33:49 GMT

#390: Remove a package for SB

johnbywater — Thu, 29 Jul 2010 15:50:25 GMT

#399: Import BIS data for SD

johnbywater — Thu, 05 Aug 2010 09:34:50 GMT

#400: Configure DGU Public API with TSO

johnbywater — Thu, 05 Aug 2010 09:35:56 GMT

Have TSO configure data.ov.uk/api/catalogue. Have TSO register catalogue.data.gov.uk.

#414: Change the Apache and Varnish ports

dread — Tue, 10 Aug 2010 09:13:59 GMT

Ask Paul for a new machine for testing. Then one for varnish-live and one for varnish-test.

#415: Chase Talis about loading RDF from CKAN.

dread — Tue, 10 Aug 2010 09:14:16 GMT

#416: Meet Evan at COI at 1 o'clock Tuesday 10th August 2010 (to progress Forms API)

dread — Tue, 10 Aug 2010 09:14:37 GMT

#419: Meet Paul at COI at 2 o'clock Tuesday 10th August 2010

johnbywater — Tue, 10 Aug 2010 09:16:34 GMT

#420: Release ckan 1.1

dread — Tue, 10 Aug 2010 09:28:38 GMT

#421: Upgrade test-hmg.ckan.net to latest ckan 1.1

dread — Tue, 10 Aug 2010 09:29:01 GMT

#424: Tell JF the format spec & examples

dread — Tue, 10 Aug 2010 10:35:34 GMT

#425: Document how licenses can be configured in CKAN

johnbywater — Tue, 10 Aug 2010 10:35:51 GMT

#426: Ask list for suggestions improving API docs

dread — Tue, 10 Aug 2010 10:36:14 GMT

#443: Get DGU daily dump working (local)

dread — Fri, 20 Aug 2010 10:10:46 GMT

Producing daily dump file
Publishing at ckan.net/dumps/ or better

#445: Add dumps to DGU SLA

johnbywater — Fri, 20 Aug 2010 10:16:20 GMT

Thinking about putting something like this:

Dumps Interface (this is NOT going in the API, it is another interface of the service):

resources:

Daily Dump Folder Daily Dump File

locations:

/dump/hmg.ckan.net-YYYY-MM-DD.[json|csv].zip

data format:

Dump Folder is a HTML document containing a table of links to Dump Files Dump File has either JSON or CSV format.

status codes (200, 404, 500)

#461: Change ONS data importing to work via API

johnbywater — Fri, 20 Aug 2010 11:23:31 GMT

Move script out to ckandgu repo
Change script to convert xml into package dicts
Test (against test.ckan.net, hmg.test.ckan.net)
Deploy

#468: Default config file has commented out configuration variable

johnbywater — Fri, 20 Aug 2010 13:24:19 GMT

#469: Unit test to check API key in correctly named HTTP header is accepted by server

johnbywater — Fri, 20 Aug 2010 13:26:00 GMT

#475: Unit test to check API key in incorrectly named HTTP header is not accepted by server

johnbywater — Fri, 20 Aug 2010 14:00:02 GMT

#478: Unit test for "interface resource" in API

johnbywater — Fri, 20 Aug 2010 14:37:47 GMT

We don't want /api to be an HTML page, but rather provide information about the API in a JSON structure.

#479: Suggest DGU presents database dumps through front end

johnbywater — Fri, 20 Aug 2010 17:12:45 GMT

#508: Create branch ultrastable

dread — Tue, 24 Aug 2010 15:27:43 GMT

Create a branch named ultrastable that represents latest version of 1.0 as found on dgu
- Going forward ultrastable will act as stable - 1 (and either ultrastable or stable will be on the rock-solid production systems such as dgu)
Do not deprecate the stable or metastable branch for the present
Ask for recommendations on mailing list about best practice for branching (e.g. versioned based versus a simple HEAD versus stable) and how this affects standard operations such as applying bugfixes.
- For useful discussion of how to do backporting and applying bugfix patches see: http://www.python.org/dev/peps/pep-0374/#backport
- NB: in general do not always have to apply in stable/ultrastable and merge upwards for bugfixes -- can use transplant extension as well.

#536: Extend base importer merging in resources

dread — Mon, 30 Aug 2010 13:01:53 GMT

Multiple ONS records can go into one package e.g. Stats for Jan, Stats for Feb, Stats for Mar etc.

We already identify an existing package by 'external_id'. Now need to find package by title and an extra field (department in ONS case) and allow it to merge in resource and tags. Alert if rest of meta data is different.

#548: Functional test for getting package create form from the API

johnbywater — Wed, 01 Sep 2010 11:13:18 GMT

#549: Method on API controller to return package create form

johnbywater — Wed, 01 Sep 2010 11:15:48 GMT

#550: Routing configuration for package create form API resource

johnbywater — Wed, 01 Sep 2010 11:16:25 GMT

#551: Documentation for getting package create form from API

johnbywater — Wed, 01 Sep 2010 11:17:25 GMT

#552: Method on API controller to accept valid package create form submission

johnbywater — Wed, 01 Sep 2010 11:20:09 GMT

#553: Functional test for submitting valid "package create form" response to the API

johnbywater — Wed, 01 Sep 2010 11:23:23 GMT

#554: Functional test for submitting invalid "package create form" response to the API

johnbywater — Wed, 01 Sep 2010 11:23:46 GMT

#555: Documentation for submitting package create form submission to API

johnbywater — Wed, 01 Sep 2010 11:24:50 GMT

#556: Method on API controller to accept invalid package create form submission

johnbywater — Wed, 01 Sep 2010 11:27:28 GMT

#581: Determine package name from metadata document

johnbywater — Sat, 11 Sep 2010 13:14:34 GMT

Need to establish procedure for automatically generating package names from metadata documents.

#582: Domain object class for harvest source

johnbywater — Sat, 11 Sep 2010 13:18:24 GMT

#583: Define fieldset object for harvest source forms

johnbywater — Sat, 11 Sep 2010 13:19:29 GMT

#584: Migration script to add harvest source table

johnbywater — Sat, 11 Sep 2010 13:20:04 GMT

#585: Method on API controller to return harvest source create form

johnbywater — Sat, 11 Sep 2010 13:21:42 GMT

#586: Routing configuration for harvest source create form API resource

johnbywater — Sat, 11 Sep 2010 13:22:16 GMT

#587: Documentation for getting harvest source create form from API

johnbywater — Sat, 11 Sep 2010 13:22:44 GMT

#588: Method on API controller to accept valid harvest source create form submission

johnbywater — Sat, 11 Sep 2010 13:26:15 GMT

#589: Method on API controller to reject invalid harvest source create form submission

johnbywater — Sat, 11 Sep 2010 13:27:00 GMT

#590: Documentation for submitting harvest source create form submission to API

johnbywater — Sat, 11 Sep 2010 13:27:39 GMT

#591: Routing configuration for harvest source entity API resource

johnbywater — Sat, 11 Sep 2010 13:34:42 GMT

#592: Methods to return harvest source entity as JSON

johnbywater — Sat, 11 Sep 2010 13:38:58 GMT

#593: Method on API controller to return remote metadata edit form

johnbywater — Sat, 11 Sep 2010 13:41:38 GMT

#594: Routing configuration for remote metadata edit form API resource

johnbywater — Sat, 11 Sep 2010 13:42:12 GMT

#595: Documentation for getting remote metadata edit form from API

johnbywater — Sat, 11 Sep 2010 13:42:52 GMT

#596: Method on API controller to accept valid remote metadata edit form submission

johnbywater — Sat, 11 Sep 2010 13:44:22 GMT

#597: Method on API controller to reject invalid remote metadata edit form submission

johnbywater — Sat, 11 Sep 2010 13:44:49 GMT