To reproduce:

1. paster db init
2. paster create-test-data
3. In Web UI: create new group 'tilo', which includes package 'annakarenina'
4. curl http://localhost:5000/api/search/package?groups=tilo results in 0 results (WRONG)
5. paster search-index rebuild
6. curl http://localhost:5000/api/search/package?groups=tilo results in 1 result

Need to cover pip-requirements.txt on different branches in the doc/deployment.rst.

Also could do with tidying up upgrade.txt into this document too.

Hi, I get a 500, Internal server error when I enable ckanext-stats. Flavio

Hello,

I have done a recent update to mu ckan install, and I am now getting this error for all my packages:

URL: http://ckan.emap.fgv.br/package/dengue-net
Module weberror.errormiddleware:162 in __call__
<<              __traceback_supplement__ = Supplement, self, environ
                   sr_checker = ResponseStartChecker(start_response)
                   app_iter = self.application(environ, sr_checker)
                   return self.make_catching_iter(app_iter, environ, sr_checker)
               except:
>>  app_iter = self.application(environ, sr_checker)
Module beaker.middleware:73 in __call__
<<                                                     self.cache_manager)
               environ[self.environ_key] = self.cache_manager
               return self.app(environ, start_response)
>>  return self.app(environ, start_response)
Module beaker.middleware:152 in __call__
<<                          headers.append(('Set-cookie', cookie))
                   return start_response(status, headers, exc_info)
               return self.wrap_app(environ, session_start_response)
          
           def _get_session(self):
>>  return self.wrap_app(environ, session_start_response)
Module routes.middleware:130 in __call__
<<                  environ['SCRIPT_NAME'] = environ['SCRIPT_NAME'][:-1]
              
               response = self.app(environ, start_response)
              
               # Wrapped in try as in rare cases the attribute will be gone already
>>  response = self.app(environ, start_response)
Module pylons.wsgiapp:125 in __call__
<<         
               controller = self.resolve(environ, start_response)
               response = self.dispatch(controller, environ, start_response)
              
               if 'paste.testing_variables' in environ and hasattr(response,
>>  response = self.dispatch(controller, environ, start_response)
Module pylons.wsgiapp:324 in dispatch
<<          if log_debug:
                   log.debug("Calling controller class with WSGI interface")
               return controller(environ, start_response)
          
           def load_test_env(self, environ):
>>  return controller(environ, start_response)
Module ckan.lib.base:118 in __call__
<<          # available in environ['pylons.routes_dict']   
               try:
                   return WSGIController.__call__(self, environ, start_response)
               finally:
                   model.Session.remove()
>>  return WSGIController.__call__(self, environ, start_response)
Module pylons.controllers.core:221 in __call__
<<                  return response(environ, self.start_response)
              
               response = self._dispatch_call()
               if not start_response_called:
                   self.start_response = start_response
>>  response = self._dispatch_call()
Module pylons.controllers.core:172 in _dispatch_call
<<              req.environ['pylons.action_method'] = func
                  
                   response = self._inspect_call(func)
               else:
                   if log_debug:
>>  response = self._inspect_call(func)
Module pylons.controllers.core:107 in _inspect_call
<<                        func.__name__, args)
               try:
                   result = self._perform_call(func, args)
               except HTTPException, httpe:
                   if log_debug:
>>  result = self._perform_call(func, args)
Module pylons.controllers.core:60 in _perform_call
<<          """Hide the traceback for everything above this method"""
               __traceback_hide__ = 'before_and_this'
               return func(**args)
          
           def _inspect_call(self, func):
>>  return func(**args)
Module ?:2 in read
Module ckan.lib.cache:167 in wrapper
<<      log = __import__("logging").getLogger("proxy_cache")
           def wrapper(func, *args, **kwargs):
               result = func(*args, **kwargs)
      
               pylons = get_pylons(args)
>>  result = func(*args, **kwargs)
Module ckan.controllers.package:208 in read
<<          #render the package
               PackageSaver().render_package(c.pkg)
               return render('package/read.html')
      
           def comments(self, id):
>>  return render('package/read.html')
Module ckan.lib.base:74 in render
<<     
           return cached_template(template_name, render_template, cache_key=cache_key,
                                  cache_type=cache_type, cache_expire=cache_expire)
                                  #, ns_options=('method'), method=method)
>>  cache_type=cache_type, cache_expire=cache_expire)
Module pylons.templating:249 in cached_template
<<          return content
           else:
               return render_func()
>>  return render_func()
Module ckan.lib.base:63 in render_template
<<              stream = item.filter(stream)
              
               return literal(stream.render(method=method, encoding=None))
          
           if 'Pragma' in response.headers:
>>  return literal(stream.render(method=method, encoding=None))
Module genshi.core:183 in render
<<              method = self.serializer or 'xml'
               generator = self.serialize(method=method, **kwargs)
               return encode(generator, method=method, encoding=encoding, out=out)
      
           def select(self, path, namespaces=None, variables=None):
>>  return encode(generator, method=method, encoding=encoding, out=out)
Module genshi.output:57 in encode
<<          _encode = lambda string: string
           if out is None:
               return _encode(''.join(list(iterator)))
           for chunk in iterator:
               out.write(_encode(chunk))
>>  return _encode(''.join(list(iterator)))
Module genshi.output:339 in __call__
<<          for filter_ in self.filters:
                   stream = filter_(stream)
               for kind, data, pos in stream:
                   cached = cache_get((kind, data))
                   if cached is not None:
>>  for kind, data, pos in stream:
Module genshi.output:670 in __call__
<<          _gen_prefix = _gen_prefix().next
      
               for kind, data, pos in stream:
                   output = cache_get((kind, data))
                   if output is not None:
>>  for kind, data, pos in stream:
Module genshi.output:771 in __call__
<<          push_text = textbuf.append
               pop_text = textbuf.pop
               for kind, data, pos in chain(stream, [(None, None, None)]):
      
                   if kind is TEXT:
>>  for kind, data, pos in chain(stream, [(None, None, None)]):
Module genshi.output:586 in __call__
<<      def __call__(self, stream):
               prev = (None, None, None)
               for ev in stream:
                   if prev[0] is START:
                       if ev[0] is END:
>>  for ev in stream:
Module genshi.core:288 in _ensure
<<      # unchanged
           yield event
           for event in stream:
               yield event
>>  for event in stream:
Module genshi.template.base:618 in _include
<<                      tmpl = self.loader.load(href, relative_to=event[2][0],
                                                   cls=cls or self.__class__)
                           for event in tmpl.generate(ctxt, **vars):
                               yield event
                       except TemplateNotFound:
>>  for event in tmpl.generate(ctxt, **vars):
Module genshi.template.base:618 in _include
<<                      tmpl = self.loader.load(href, relative_to=event[2][0],
                                                   cls=cls or self.__class__)
                           for event in tmpl.generate(ctxt, **vars):
                               yield event
                       except TemplateNotFound:
>>  for event in tmpl.generate(ctxt, **vars):
Module genshi.template.base:618 in _include
<<                      tmpl = self.loader.load(href, relative_to=event[2][0],
                                                   cls=cls or self.__class__)
                           for event in tmpl.generate(ctxt, **vars):
                               yield event
                       except TemplateNotFound:
>>  for event in tmpl.generate(ctxt, **vars):
Module genshi.template.base:605 in _include
<<          from genshi.template.loader import TemplateNotFound
      
               for event in stream:
                   if event[0] is INCLUDE:
                       href, cls, fallback = event[1]
>>  for event in stream:
Module genshi.template.markup:378 in _match
<<                      for event in self._match(self._flatten(template, ctxt,
                                                                  **vars),
                                                    ctxt, start=idx + 1, **vars):
                               yield event
>>  ctxt, start=idx + 1, **vars):
Module genshi.template.markup:327 in _match
<<                      break
      
               for event in stream:
      
                   # We (currently) only care about start and end events for matching
>>  for event in stream:
Module genshi.template.base:565 in _flatten
<<                  elif kind is EXPR:
                           result = _eval_expr(data, ctxt, vars)
                           if result is not None:
                               # First check for a string, otherwise the iterable test
>>  result = _eval_expr(data, ctxt, vars)
Module genshi.template.base:277 in _eval_expr
<<      if vars:
               ctxt.push(vars)
           retval = expr.evaluate(ctxt)
           if vars:
               ctxt.pop()
>>  retval = expr.evaluate(ctxt)
Module genshi.template.eval:178 in evaluate
<<          __traceback_hide__ = 'before_and_this'
               _globals = self._globals(data)
               return eval(self.code, _globals, {'__data__': data})
>>  return eval(self.code, _globals, {'__data__': data})
Module ?:16 in <Expression u"h.subnav_link(c, h.icon('comments') + _('Comments & Questions'), controller='package', action='comments', id=c.pkg.name)">
<<        </li>
             <li py:if="g.has_commenting">
                 ${h.subnav_link(c, h.icon('comments') + _('Comments &amp; Questions'), controller='package', action='comments', id=c.pkg.name)}
             </li>
             <li>${h.subnav_link(c, h.icon('page_white_stack') + _('History'), controller='package', action='history', id=c.pkg.name)}</li>
>>  ${h.subnav_link(c, h.icon('comments') + _('Comments &amp; Questions'), controller='package', action='comments', id=c.pkg.name)}
Module ckan.lib.helpers:126 in subnav_link
<<      return link_to(
               text,
               url_for(action=action, **kwargs),
               class_=('active' if c.action == action else '')
           )
>>  url_for(action=action, **kwargs),
Module routes.util:280 in url_for
<<          raise GenerationException(
                   "url_for could not generate URL. Called with args: %s %s" % \
                   (args, kargs))
           return url
>>  (args, kargs))
GenerationException: url_for could not generate URL. Called with args: () {'action': 'comments', 'controller': 'package', 'id': 'dengue-net'}
CGI Variables
AUTH_TYPE 	'cookie'
DOCUMENT_ROOT 	'/etc/apache2/htdocs'
GATEWAY_INTERFACE 	'CGI/1.1'
HTTP_ACCEPT 	'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
HTTP_ACCEPT_CHARSET 	'ISO-8859-1,utf-8;q=0.7,*;q=0.7'
HTTP_ACCEPT_ENCODING 	'gzip, deflate'
HTTP_ACCEPT_LANGUAGE 	'pt-br,pt;q=0.8,en-us;q=0.5,en;q=0.3'
HTTP_CONNECTION 	'keep-alive'
HTTP_COOKIE 	'__utma=4669863.1260802706.1295364409.1306935693.1306960483.86; __utmz=4669863.1295364409.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); SESS61faf7f7d2406929b2f9eb4cbfff17d2=a3306f5d1194cf0d8a51dbf7b42c1259; ckan=d4951177ef995d92f0f987d481bd9180aa0f89e4518921db9cd9b6ac75fdd3fab4a25d06; auth_tkt="c6c3c3e0b3fe7bb765356659992760b24da43221flavio!userid_type:unicode"; auth_tkt="c6c3c3e0b3fe7bb765356659992760b24da43221flavio!userid_type:unicode"; ckan_user="flavio"; ckan_display_name="Fl\xc3\xa1vio Code\xc3\xa7o Coelho"; ckan_apikey="c139718d-918f-4a2b-b219-e33cb05cbe23"; __utmc=4669863'
HTTP_HOST 	'ckan.emap.fgv.br'
HTTP_KEEP_ALIVE 	'115'
HTTP_REFERER 	'http://ckan.emap.fgv.br/'
HTTP_USER_AGENT 	'Mozilla/5.0 (X11; Linux x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'
PATH_INFO 	'/package/dengue-net'
PATH_TRANSLATED 	'/home/flavio/var/srvc/ckan.emap.fgv.br/pyenv/bin/ckan.emap.fgv.br.py/package/dengue-net'
REMOTE_ADDR 	'10.250.48.110'
REMOTE_PORT 	'47624'
REMOTE_USER 	u'flavio'
REMOTE_USER_DATA 	'userid_type:unicode'
REMOTE_USER_TOKENS 	['']
REQUEST_METHOD 	'GET'
REQUEST_URI 	'/package/dengue-net'
SCRIPT_FILENAME 	'/home/flavio/var/srvc/ckan.emap.fgv.br/pyenv/bin/ckan.emap.fgv.br.py'
SERVER_ADDR 	'10.252.2.60'
SERVER_ADMIN 	'[no address given]'
SERVER_NAME 	'ckan.emap.fgv.br'
SERVER_PORT 	'80'
SERVER_PROTOCOL 	'HTTP/1.1'
SERVER_SIGNATURE 	'<address>Apache/2.2.16 (Debian) Server at ckan.emap.fgv.br Port 80</address>\n'
SERVER_SOFTWARE 	'Apache/2.2.16 (Debian)'
WSGI Variables
application 	<beaker.middleware.CacheMiddleware object at 0x7f051c802690>
beaker.cache 	<beaker.cache.CacheManager object at 0x7f051c802750>
beaker.get_session 	<bound method SessionMiddleware._get_session of <beaker.middleware.SessionMiddleware object at 0x7f051c8026d0>>
beaker.session 	{'locale': u'pt_BR', '_accessed_time': 1307358102.6266389, '_creation_time': 1302605470.0245121}
mod_wsgi.application_group 	'dck093|'
mod_wsgi.callable_object 	'application'
mod_wsgi.handler_script 	''
mod_wsgi.input_chunked 	'0'
mod_wsgi.listener_host 	''
mod_wsgi.listener_port 	'80'
mod_wsgi.process_group 	'ckan'
mod_wsgi.request_handler 	'wsgi-script'
mod_wsgi.script_reloading 	'1'
mod_wsgi.version 	(3, 3)
paste.cookies 	(<SimpleCookie: SESS61faf7f7d2406929b2f9eb4cbfff17d2='a3306f5d1194cf0d8a51dbf7b42c1259' __utma='4669863.1260802706.1295364409.1306935693.1306960483.86' __utmc='4669863' __utmz='4669863.1295364409.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)' auth_tkt='c6c3c3e0b3fe7bb765356659992760b24da43221flavio!userid_type:unicode' ckan='d4951177ef995d92f0f987d481bd9180aa0f89e4518921db9cd9b6ac75fdd3fab4a25d06' ckan_apikey='c139718d-918f-4a2b-b219-e33cb05cbe23' ckan_display_name='Fl\xc3\xa1vio Code\xc3\xa7o Coelho' ckan_user='flavio'>, '__utma=4669863.1260802706.1295364409.1306935693.1306960483.86; __utmz=4669863.1295364409.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); SESS61faf7f7d2406929b2f9eb4cbfff17d2=a3306f5d1194cf0d8a51dbf7b42c1259; ckan=d4951177ef995d92f0f987d481bd9180aa0f89e4518921db9cd9b6ac75fdd3fab4a25d06; auth_tkt="c6c3c3e0b3fe7bb765356659992760b24da43221flavio!userid_type:unicode"; auth_tkt="c6c3c3e0b3fe7bb765356659992760b24da43221flavio!userid_type:unicode";... __utmc=4669863')
paste.registry 	<paste.registry.Registry object at 0x7f051d7e5710>
paste.throw_errors 	True
pylons.action_method 	<bound method PackageController.read of <ckan.controllers.package.PackageController object at 0x7f051d7e5b10>>
pylons.controller 	<ckan.controllers.package.PackageController object at 0x7f051d7e5b10>
pylons.environ_config 	{'session': 'beaker.session', 'cache': 'beaker.cache'}
pylons.pylons 	<pylons.util.PylonsContext object at 0x7f051d7e5a50>
pylons.routes_dict 	{'action': u'read', 'controller': u'package', 'id': u'dengue-net'}
repoze.who.identity 	<repoze.who identity (hidden, dict-like) at 139659946193344>
repoze.who.logger 	<logging.Logger instance at 0x7f051c801128>
repoze.who.plugins 	{'openid': <OpenIdIdentificationPlugin 139659931643152>, 'friendlyform': <FriendlyFormPlugin 139659929725584>, 'ckan.lib.authenticator:UsernamePasswordAuthenticator': <ckan.lib.authenticator.UsernamePasswordAuthenticator object at 0x7f051c9e7610>, 'auth_tkt': <AuthTktCookiePlugin 139659931643280>, 'ckan.lib.authenticator:OpenIDAuthenticator': <ckan.lib.authenticator.OpenIDAuthenticator object at 0x7f051c9dcfd0>}
routes.route 	<routes.route.Route object at 0x7f051c69fd50>
routes.url 	<routes.util.URLGenerator object at 0x7f051d7e57d0>
webob._parsed_cookies 	({'ckan': 'd4951177ef995d92f0f987d481bd9180aa0f89e4518921db9cd9b6ac75fdd3fab4a25d06', '__utmz': '4669863.1295364409.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)', 'auth_tkt': 'c6c3c3e0b3fe7bb765356659992760b24da43221flavio!userid_type:unicode', 'ckan_display_name': 'Fl\xc3\xa1vio Code\xc3\xa7o Coelho', 'ckan_apikey': 'c139718d-918f-4a2b-b219-e33cb05cbe23', 'ckan_user': 'flavio', '__utma': '4669863.1260802706.1295364409.1306935693.1306960483.86', '__utmc': '4669863', 'SESS61faf7f7d2406929b2f9eb4cbfff17d2': 'a3306f5d1194cf0d8a51dbf7b42c1259'}, '__utma=4669863.1260802706.1295364409.1306935693.1306960483.86; __utmz=4669863.1295364409.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); SESS61faf7f7d2406929b2f9eb4cbfff17d2=a3306f5d1194cf0d8a51dbf7b42c1259; ckan=d4951177ef995d92f0f987d481bd9180aa0f89e4518921db9cd9b6ac75fdd3fab4a25d06; auth_tkt="c6c3c3e0b3fe7bb765356659992760b24da43221flavio!userid_type:unicode"< wbr>; auth_tkt="c6c3c3e0b3fe7bb765356659992760b24da43221flavio!... __utmc=4669863')
webob._parsed_query_vars 	(GET([]), '')
webob.adhoc_attrs 	{'language': 'en-us'}
wsgi process 	'Multi process AND threads (?)'
wsgi.file_wrapper 	<built-in method file_wrapper of mod_wsgi.Adapter object at 0x7f051cc1ad50>
wsgi.version 	(1, 1)
wsgiorg.routing_args 	(<routes.util.URLGenerator object at 0x7f051d7e57d0>, {'action': u'read', 'controller': u'package', 'id': u'dengue-net'})

User can insert bad anchor tags into the User-About and Package-Notes fields and when you view them (web interface) it causes a 500 error.

Need to improve filtering for anchors in markdown.

<a href="http://xxxsex.com>nasty/website

Also check this related exception:

Module ckan.controllers.user:59 in read
<<          c.is_myself = user.name == c.user
               c.api_key = user.apikey
               c.about_formatted = self._format_about(user.about)
               revisions_q = model.Session.query(model.Revision
                       ).filter_by(author=user.name)
>>  c.about_formatted = self._format_about(user.about)
Module ckan.controllers.user:167 in _format_about
<<      def _format_about(self, about):
               about_formatted = ckan.misc.MarkdownFormat().to_html(about)
               return genshi.HTML(about_formatted) 
       
           def _get_form_password(self):
>>  return genshi.HTML(about_formatted)
WebApp Error: <class 'genshi.input.ParseError'>: junk characters in start tag: u'\u201dhttp://www.settingu': line 1, column 3

When a package has been deleted, say for spam moderation, comments still appear in the recent comments section.

This is a problem because non-admin users will be shown a warning that they're not authorised to view the package if they click on the link.

At CKAN.net currently, this affects the most recent comment.

Administrators have "View", "Edit" and "History" tabs. However, I can't see a way to delete a package from the web UI.

Version: CKAN.net as of today

1. Register a new user (/user/register)
2. It redirects you to user page and displays "Welcome back" even though you are a new user.

Markdown produces Debug logging, even when the pylons logging config is set to level of Info or Warning. This clogs up the logfile.

When upgrading database for cz.ckan.net (psql.okfn.org/cz.okfn.org) an error occurred (see below).

cz.ckan.net has been running on eu3 and this updrade was part of moving it to eu19, with a more recent version of ckan.

In the meantime I've reverted the db to the old version (pre-upgrade).

cz.ckan.net is not working properly until this is fixed.

okfn@eu19:~$ paster --plugin ckan db upgrade --config=/etc/ckan/cz/cz.ini
/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/url.py:105: SADeprecationWarning: The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'. The new URL format is postgresql[+driver]://<user>:<pass>@<host>/<dbname>
  module = __import__('sqlalchemy.dialects.%s' % (dialect, )).dialects
2011-06-29 09:07:41,294 INFO  [migrate.versioning.api] 28 -> 29... 
/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/reflection.py:46: SAWarning: Skipped unsupported reflection of expression-based index idx_pkg_lname
  ret = fn(self, con, *args, **kw)
/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/reflection.py:46: SAWarning: Skipped unsupported reflection of expression-based index idx_pkg_slname
  ret = fn(self, con, *args, **kw)
/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/reflection.py:46: SAWarning: Skipped unsupported reflection of expression-based index idx_pkg_suname
  ret = fn(self, con, *args, **kw)
/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/reflection.py:46: SAWarning: Skipped unsupported reflection of expression-based index idx_pkg_uname
  ret = fn(self, con, *args, **kw)
Traceback (most recent call last):
  File "/usr/bin/paster", line 18, in <module>
    command.run()
  File "/usr/lib/pymodules/python2.6/paste/script/command.py", line 84, in run
    invoke(command, command_name, options, args[1:])
  File "/usr/lib/pymodules/python2.6/paste/script/command.py", line 123, in invoke
    exit_code = runner.run(args)
  File "/usr/lib/pymodules/python2.6/paste/script/command.py", line 218, in run
    result = self.command()
  File "/usr/lib/pymodules/python2.6/ckan/lib/cli.py", line 84, in command
    model.repo.upgrade_db()
  File "/usr/lib/pymodules/python2.6/ckan/model/__init__.py", line 163, in upgrade_db
    mig.upgrade(self.metadata.bind, self.migrate_repository, version=version)
  File "/usr/lib/pymodules/python2.6/ckan_deps/migrate/versioning/api.py", line 185, in upgrade
    return _migrate(url, repository, version, upgrade=True, err=err, **opts)
  File "<string>", line 2, in _migrate
  File "/usr/lib/pymodules/python2.6/ckan_deps/migrate/versioning/util/__init__.py", line 160, in with_engine
    return f(*a, **kw)
  File "/usr/lib/pymodules/python2.6/ckan_deps/migrate/versioning/api.py", line 364, in _migrate
    schema.runchange(ver, change, changeset.step)
  File "/usr/lib/pymodules/python2.6/ckan_deps/migrate/versioning/schema.py", line 83, in runchange
    change.run(self.engine, step)
  File "/usr/lib/pymodules/python2.6/ckan_deps/migrate/versioning/script/py.py", line 140, in run
    script_func(engine)
  File "/usr/lib/pymodules/python2.6/ckan/migration/versions/029_version_groups.py", line 152, in upgrade
    migrate_engine.execute(q)
  File "/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/base.py", line 1788, in execute
    return connection.execute(statement, *multiparams, **params)
  File "/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/base.py", line 1191, in execute
    params)
  File "/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/base.py", line 1271, in _execute_clauseelement
    return self.__execute_context(context)
  File "/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/base.py", line 1302, in __execute_context
    context.parameters[0], context=context)
  File "/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/base.py", line 1401, in _cursor_execute
    context)
  File "/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/base.py", line 1394, in _cursor_execute
    context)
  File "/usr/lib/pymodules/python2.6/ckan_deps/sqlalchemy/engine/default.py", line 299, in do_execute
    cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (IntegrityError) insert or update on table "package_group_revision" violates foreign key constraint "package_group_revision_continuity_id_fkey"
DETAIL:  Key (continuity_id)=(7f373978-40a0-4848-bdc9-018ad213aa0b) is not present in table "package_group".
 'INSERT INTO package_group_revision (id, package_id, group_id, state, revision_id, continuity_id) VALUES (%(id)s, %(package_id)s, %(group_id)s, %(state)s, %(revision_id)s, %(continuity_id)s)' {'state': u'active', 'package_id': u'14d76e9e-028c-49c1-955b-ef30d4abb970', 'continuity_id': u'7f373978-40a0-4848-bdc9-018ad213aa0b', 'revision_id': u'24a11f58-2ff9-46f3-b113-0bf7f0baf254', 'group_id': u'4b832199-b8e3-4989-9b5d-de8dd6322612', 'id': u'7a3863bd-3054-40c8-bb3d-68b73096c108'}

Got broken due to the moderated edits changes #1141.

Need to make sure preview works.

Notes on the CKAN API from Aron:

• Server returns text/html for errors even when "Accept: application/json" header is set.
• DELETE package request requires a Content-Length? Should be 405 Method Not Allowed?
• Tag returned as a JSON object when updating but as a string when requesting. Same with "extras" content.
• How to remove key from "extras" object? Passing None as described in the docs is invalid JSON and using null doesn't work.
• Tags return package names rather than ids when querying GET /tag/{id}
• Doesn't return rendered_text property on package update.
• Relationships use "object" key rather than id or package_id.

Exception raised doing this particular diff.

​http://ckan.net/revision/diff/sudocfr?diff_entity=package&oldid=a18b2520-09e0-44be-80dd-4fea94ffe7d1&diff=3934ff16-a3a7-4481-b390-72944903b945

...
Module ckan.controllers.revision:142 in diff
<<          if c.diff_entity == 'package':
                   c.pkg = model.Package.by_name(id)
                   diff = c.pkg.diff(c.revision_to, c.revision_from)
               elif c.diff_entity == 'group':
                   c.group = model.Group.by_name(id)
>>  diff = c.pkg.diff(c.revision_to, c.revision_from)
Module ckan.model.package:449 in diff
<<                              display_id = to_obj_rev.tag.name
                               elif obj_class.__name__ == 'PackageExtra':
                                   display_id = to_obj_rev.key
                               else:
                                   display_id = related_obj_id[:4]
>>  display_id = to_obj_rev.key
AttributeError: 'NoneType' object has no attribute 'key'

Seems to be to do with this particular extra key:

[<PackageExtraRevision id=ba1f3b3e-f61f-4924-bae0-ec3dd2163ae4 package_id=febe4da9-5874-4411-a7da-b497fdebf739 key=namespace value=http://www.sudoc.fr/ state=active revision_id=5ec8e730-c73d-4f1c-989b-2e015eb3d315 continuity_id=ba1f3b3e-f61f-4924-bae0-ec3dd2163ae4 expired_id=None revision_timestamp=2011-07-08 13:48:11.027260 expired_timestamp=9999-12-31 00:00:00 current=True>]

Is it because it was created between the from_revision and to_revision?

• No tests.
• 500 error given when user is not found.

IndexError? exception in genshi.filters.i18n keeps popping up daily.

e.g.

...
Module ckan.controllers.authorization_group:28 in index
<<              items_per_page=20
               )
               return render('authorization_group/index.html')
       
           def read(self, id):
>>  return render('authorization_group/index.html')
Module ckan.lib.base:74 in render
<<      
           return cached_template(template_name, render_template, cache_key=cache_key, 
                                  cache_type=cache_type, cache_expire=cache_expire)
                                  #, ns_options=('method'), method=method)
>>  cache_type=cache_type, cache_expire=cache_expire)
Module pylons.templating:249 in cached_template
<<          return content
           else:
               return render_func()
>>  return render_func()
Module ckan.lib.base:63 in render_template
<<              stream = item.filter(stream)
               
               return literal(stream.render(method=method, encoding=None))
           
           if 'Pragma' in response.headers:
>>  return literal(stream.render(method=method, encoding=None))
Module genshi.core:183 in render
<<              method = self.serializer or 'xml'
               generator = self.serialize(method=method, **kwargs)
               return encode(generator, method=method, encoding=encoding, out=out)
       
           def select(self, path, namespaces=None, variables=None):
>>  return encode(generator, method=method, encoding=encoding, out=out)
Module genshi.output:57 in encode
<<          _encode = lambda string: string
           if out is None:
               return _encode(''.join(list(iterator)))
           for chunk in iterator:
               out.write(_encode(chunk))
>>  return _encode(''.join(list(iterator)))
Module genshi.output:339 in __call__
<<          for filter_ in self.filters:
                   stream = filter_(stream)
               for kind, data, pos in stream:
                   cached = cache_get((kind, data))
                   if cached is not None:
>>  for kind, data, pos in stream:
Module genshi.output:670 in __call__
<<          _gen_prefix = _gen_prefix().next
       
               for kind, data, pos in stream:
                   output = cache_get((kind, data))
                   if output is not None:
>>  for kind, data, pos in stream:
Module genshi.output:771 in __call__
<<          push_text = textbuf.append
               pop_text = textbuf.pop
               for kind, data, pos in chain(stream, [(None, None, None)]):
       
                   if kind is TEXT:
>>  for kind, data, pos in chain(stream, [(None, None, None)]):
Module genshi.output:586 in __call__
<<      def __call__(self, stream):
               prev = (None, None, None)
               for ev in stream:
                   if prev[0] is START:
                       if ev[0] is END:
>>  for ev in stream:
Module genshi.core:288 in _ensure
<<      # unchanged
           yield event
           for event in stream:
               yield event
>>  for event in stream:
Module genshi.core:288 in _ensure
<<      # unchanged
           yield event
           for event in stream:
               yield event
>>  for event in stream:
Module genshi.filters.transform:686 in _unmark
<<      def _unmark(self, stream):
               for mark, event in stream:
                   kind = event[0]
                   if not (kind is None or kind is ATTR or kind is BREAK):
>>  for mark, event in stream:
Module genshi.filters.transform:1145 in __call__
<<          :param stream: The marked event stream to filter
               """
               for mark, event in stream:
                   yield mark, event
                   if mark is ENTER:
>>  for mark, event in stream:
Module genshi.filters.transform:714 in __call__
<<          stream = iter(stream)
               next = stream.next
               for mark, event in stream:
                   if mark is None:
                       yield mark, event
>>  for mark, event in stream:
Module genshi.filters.transform:682 in _mark
<<      def _mark(self, stream):
               for event in stream:
                   yield OUTSIDE, event
>>  for event in stream:
Module genshi.core:288 in _ensure
<<      # unchanged
           yield event
           for event in stream:
               yield event
>>  for event in stream:
Module genshi.filters.transform:686 in _unmark
<<      def _unmark(self, stream):
               for mark, event in stream:
                   kind = event[0]
                   if not (kind is None or kind is ATTR or kind is BREAK):
>>  for mark, event in stream:
Module ckanext.googleanalytics.plugin:62 in download_adder
<<              (downloaded %s times)</span>'''
                   count = None
                   for mark, (kind, data, pos) in stream:
                       if mark and kind == START:
                           href = data[1].get('href')
>>  for mark, (kind, data, pos) in stream:
Module genshi.filters.transform:714 in __call__
<<          stream = iter(stream)
               next = stream.next
               for mark, event in stream:
                   if mark is None:
                       yield mark, event
>>  for mark, event in stream:
Module genshi.filters.transform:682 in _mark
<<      def _mark(self, stream):
               for event in stream:
                   yield OUTSIDE, event
>>  for event in stream:
Module genshi.core:288 in _ensure
<<      # unchanged
           yield event
           for event in stream:
               yield event
>>  for event in stream:
Module genshi.filters.transform:686 in _unmark
<<      def _unmark(self, stream):
               for mark, event in stream:
                   kind = event[0]
                   if not (kind is None or kind is ATTR or kind is BREAK):
>>  for mark, event in stream:
Module genshi.filters.transform:1175 in __call__
<<          """
               callable_value = hasattr(self.value, '__call__')
               for mark, (kind, data, pos) in stream:
                   if mark is ENTER:
                       if callable_value:
>>  for mark, (kind, data, pos) in stream:
Module genshi.filters.transform:714 in __call__
<<          stream = iter(stream)
               next = stream.next
               for mark, event in stream:
                   if mark is None:
                       yield mark, event
>>  for mark, event in stream:
Module genshi.filters.transform:682 in _mark
<<      def _mark(self, stream):
               for event in stream:
                   yield OUTSIDE, event
>>  for event in stream:
Module genshi.core:288 in _ensure
<<      # unchanged
           yield event
           for event in stream:
               yield event
>>  for event in stream:
Module genshi.filters.transform:686 in _unmark
<<      def _unmark(self, stream):
               for mark, event in stream:
                   kind = event[0]
                   if not (kind is None or kind is ATTR or kind is BREAK):
>>  for mark, event in stream:
Module genshi.filters.transform:1145 in __call__
<<          :param stream: The marked event stream to filter
               """
               for mark, event in stream:
                   yield mark, event
                   if mark is ENTER:
>>  for mark, event in stream:
Module genshi.filters.transform:714 in __call__
<<          stream = iter(stream)
               next = stream.next
               for mark, event in stream:
                   if mark is None:
                       yield mark, event
>>  for mark, event in stream:
Module genshi.filters.transform:682 in _mark
<<      def _mark(self, stream):
               for event in stream:
                   yield OUTSIDE, event
>>  for event in stream:
Module genshi.template.base:618 in _include
<<                      tmpl = self.loader.load(href, relative_to=event[2][0],
                                                   cls=cls or self.__class__)
                           for event in tmpl.generate(ctxt, **vars):
                               yield event
                       except TemplateNotFound:
>>  for event in tmpl.generate(ctxt, **vars):
Module genshi.template.base:618 in _include
<<                      tmpl = self.loader.load(href, relative_to=event[2][0],
                                                   cls=cls or self.__class__)
                           for event in tmpl.generate(ctxt, **vars):
                               yield event
                       except TemplateNotFound:
>>  for event in tmpl.generate(ctxt, **vars):
Module genshi.template.base:618 in _include
<<                      tmpl = self.loader.load(href, relative_to=event[2][0],
                                                   cls=cls or self.__class__)
                           for event in tmpl.generate(ctxt, **vars):
                               yield event
                       except TemplateNotFound:
>>  for event in tmpl.generate(ctxt, **vars):
Module genshi.template.base:605 in _include
<<          from genshi.template.loader import TemplateNotFound
       
               for event in stream:
                   if event[0] is INCLUDE:
                       href, cls, fallback = event[1]
>>  for event in stream:
Module genshi.template.markup:378 in _match
<<                      for event in self._match(self._flatten(template, ctxt,
                                                                  **vars),
                                                    ctxt, start=idx + 1, **vars):
                               yield event
>>  ctxt, start=idx + 1, **vars):
Module genshi.template.markup:327 in _match
<<                      break
       
               for event in stream:
       
                   # We (currently) only care about start and end events for matching
>>  for event in stream:
Module genshi.template.base:545 in _flatten
<<          while 1:
                   for kind, data, pos in stream:
       
                       if kind is START and data[1]:
>>  for kind, data, pos in stream:
Module genshi.filters.i18n:177 in _generate
<<                  msgbuf.append(*previous)
                       previous = None
                   for event in msgbuf.translate(gettext(msgbuf.format())):
                       yield event
                   if previous:
>>  for event in msgbuf.translate(gettext(msgbuf.format())):
Module genshi.filters.i18n:1029 in translate
<<                      )
       
               parts = parse_msg(string)
               parts_counter = {}
               for order, string in parts:
>>  parts = parse_msg(string)
Module genshi.filters.i18n:1143 in parse_msg
<<      if string:
               parts.append((stack[-1], string))
       
           return parts
>>  parts.append((stack[-1], string))
IndexError: list index out of range

See reports on list at:

• ​http://lists.okfn.org/pipermail/ckan-dev/2011-July/001103.html
• ​http://lists.okfn.org/pipermail/ckan-dev/2011-July/001107.html

Tracked this to issue in dictization/model_save whereby was ignoring case where Package Tag already there but in deleted state. Rather than describe at length see fix.

Exception received on test.ckan.net at 10.12 on 29/7/11

http://test.ckan.net/package?__cache=2759373&page=4
Module pylons.controllers.core:60 in _perform_call
<<          """Hide the traceback for everything above this method"""
               __traceback_hide__ = 'before_and_this'
               return func(**args)
           
           def _inspect_call(self, func):
>>  return func(**args)
Module ckan.controllers.package:153 in search
<<                          }
       
                   query = get.package_search(context,data_dict)
       
                   c.page = h.Page(
>>  query = get.package_search(context,data_dict)
Module ckan.logic.action.get:514 in package_search
<<      for package in query.results:
               result_dict = table_dictize(package, context)
               result_dict = _extend_package_dict(result_dict,context)
       
               results.append(result_dict)
>>  result_dict = _extend_package_dict(result_dict,context)
Module ckan.logic.action.get:537 in _extend_package_dict
<<      license_id = package_dict['license_id']
           if license_id:
               isopen = model.Package.get_license_register()[license_id].isopen()
               package_dict['isopen'] = isopen
           else:
>>  isopen = model.Package.get_license_register()[license_id].isopen()
Module ckan.model.license:59 in __getitem__
<<              return default
               else:
                   raise KeyError, "License not found: %s" % key
       
           def get(self, key, default=None):
>>  raise KeyError, "License not found: %s" % key
KeyError: u'License not found: None'

and a similar one:

 http://test.ckan.net/tag/barbecue
Module pylons.wsgiapp:324 in dispatch
<<          if log_debug:
                   log.debug("Calling controller class with WSGI interface")
               return controller(environ, start_response)
           
           def load_test_env(self, environ):
>>  return controller(environ, start_response)
Module ckan.lib.base:118 in __call__
<<          # available in environ['pylons.routes_dict']    
               try:
                   return WSGIController.__call__(self, environ, start_response)
               finally:
                   model.Session.remove()
>>  return WSGIController.__call__(self, environ, start_response)
Module pylons.controllers.core:221 in __call__
<<                  return response(environ, self.start_response)
               
               response = self._dispatch_call()
               if not start_response_called:
                   self.start_response = start_response
>>  response = self._dispatch_call()
Module pylons.controllers.core:172 in _dispatch_call
<<              req.environ['pylons.action_method'] = func
                   
                   response = self._inspect_call(func)
               else:
                   if log_debug:
>>  response = self._inspect_call(func)
Module pylons.controllers.core:107 in _inspect_call
<<                        func.__name__, args)
               try:
                   result = self._perform_call(func, args)
               except HTTPException, httpe:
                   if log_debug:
>>  result = self._perform_call(func, args)
Module pylons.controllers.core:60 in _perform_call
<<          """Hide the traceback for everything above this method"""
               __traceback_hide__ = 'before_and_this'
               return func(**args)
           
           def _inspect_call(self, func):
>>  return func(**args)
Module ckan.controllers.tag:2 in read
Module ckan.lib.cache:167 in wrapper
<<      log = __import__("logging").getLogger("proxy_cache")
           def wrapper(func, *args, **kwargs):
               result = func(*args, **kwargs)
       
               pylons = get_pylons(args)
>>  result = func(*args, **kwargs)
Module ckan.controllers.tag:64 in read
<<          data_dict = {'id':id}
               try:
                   c.tag = get.tag_show(context,data_dict)
               except NotFound:
                   abort(404, _('Tag not found'))
>>  c.tag = get.tag_show(context,data_dict)
Module ckan.logic.action.get:341 in tag_show
<<      extended_packages = []
           for package in tag_dict['packages']:
               extended_packages.append(_extend_package_dict(package,context))
       
           tag_dict['packages'] = extended_packages
>>  extended_packages.append(_extend_package_dict(package,context))
Module ckan.logic.action.get:537 in _extend_package_dict
<<      license_id = package_dict['license_id']
           if license_id:
               isopen = model.Package.get_license_register()[license_id].isopen()
               package_dict['isopen'] = isopen
           else:
>>  isopen = model.Package.get_license_register()[license_id].isopen()
Module ckan.model.license:59 in __getitem__
<<              return default
               else:
                   raise KeyError, "License not found: %s" % key
       
           def get(self, key, default=None):
>>  raise KeyError, "License not found: %s" % key
KeyError: u'License not found: CreativeCommonsAttributionCCBY25'

It seems that searching for extras_foo:value works with solr, but extras_foo.bar:value doesn't. No theory on why.

I'm not sure why some timestamps are missing microseconds in the ckan.net data, but plenty seem to. This causes this exception when viewing the package rss feed:

URL: http://ckan.net/package/history/sfk_publishing?format=atom&days=7
In Module ckan.lib.helpers:227 in date_str_to_datetime
WebApp Error: <type 'exceptions.ValueError'>: time data '2008-04-13T20:40:20' does not match format '%Y-%m-%dT%H:%M:%S.%f'

Only seen with ckan 1.4.3b

This overlaps with Sean's follow support (for activity streams) and so will be dealt with there.

After the ckan migrate process, sqlalcehmy reflect does not work and causes an error when reflecting indexes. A bug report needs to be put into sqlalchemy.

Revisions aren't assigned to packages. Where are they coming from? This seems to have been occurring since the upgrade to 1.4.3b a couple of weeks ago?

This was removed during the auth refactoring, but it's needed in the old controllers

The email sending functionality (e.g for password reset) does not work on thedatahub.org and IATI (and probably some other instances) when using an address which is not a okfn.org one.

Could not send reset link: SMTPRecipientsRefused({u'amercadero@…': (550, 'relay not permitted')},)

As I said, adria.mercader@… works fine. The SMTP server used mail.okfn.org

e.g. this particular revision on ckan.net: ​http://ckan.net/api/rest/revision/9601fbcb-03a7-4ebc-981d-5aa16df4852c (most others have not been approved) Causes 500 error.

In password reset, it gets confused if you have two similar users. This is because with the string the user provides, it searches several fields, not just name but also fullname and email address, allowing you to search for these. But only name is unique.

Specific problem: Ira searches for "Irina" then it finds both: <User name=irina fullname=Irina [email protected] ] and <User name=shevski fullname=Ira email=> (I think)

Maybe need to choose which field it searches?

When you reset a password, the flash message saying "check your email" sometimes doesn't display on the next page. You then click on the link in your email and it displays on that page (i.e. too late).

Seen this on thedatahub.org. To do with the proxy caching?

A possible bug in CKAN when I tried deleting users using "paster --plugin=ckan user delete" command.

To reproduce the bug do the following:

1. Create a user with an ID (which in my case was a user's full name)

that contains non-unicode caracters like Norwegian "æ", "ø", or "å".

2. Make sure that you can see something like the example below:

(pyenv) rm@mycomputer:$ paster --plugin=ckan user Users: name=Rustæm

3. Then try deleting the user with following command:

(pyenv) rm@mycomputer:$ paster --plugin=ckan user delete "Rustæm"

You should now get a python encoding error. I know that this is quite rare case, but in our case it caused some trouble. Could you guys have a look at this bug?

CKAN ver. 1.3.3.

To reproduce:

1. Create a package with two tags the same: "bulk bulk"
2. Click 'save'
3. 500 ERROR - 'Server Error'

On same cases this exception is thrown when logging in:

File '/home/adria/dev/envs/iati/src/ckan/ckan/controllers/user.py', line 252 in logged_in
  user_dict = get_action('user_show')(context,data_dict)
File '/home/adria/dev/envs/iati/src/ckan/ckan/logic/action/get.py', line 414 in user_show
  revision_dict = revision_show(context,{'id':revision.id})
File '/home/adria/dev/envs/iati/src/ckan/ckan/logic/action/get.py', line 332 in revision_show
  ref_package_by=ref_package_by)
File '/home/adria/dev/envs/iati/src/ckan/ckan/model/__init__.py', line 317 in revision_as_dict
  for pkg in revision.packages]
AttributeError: 'NoneType' object has no attribute 'name'

Instead of this:

extras: {
type: ""Primary source""
license_id: ""localauth-withrights""
publisher_frequency: ""2Pre\u00f1o \u20ac $ \u00e0dri\u00e0""
}

we should return this:

extras: {
type: "Primary source",
license_id: "localauth-withrights",
publisher_frequency: "2Preño € $ àdrià",
}

e.g. ​http://ckan.net/group/edit/history

Module ckan.controllers.group:135 in edit
<<          try:
                   old_data = get_action('group_show')(context, data_dict)
                   c.grouptitle = old_data.get('title')
                   c.groupname = old_data.get('name')
>>  old_data = get_action('group_show')(context, data_dict)
Module ckan.logic.action.get:347 in group_show
<<      if group is None:
               raise NotFound
       
           check_access('group_show',context, data_dict)
>>  raise NotFound

This stems from a feature to allow you to prefill fields in the authorizationgroup creation form.

e.g. ​http://ckan.net/authorizationgroup/new?name=science

It's not tested, not supported in the new logic layer and therefore broken. I don't think anyone uses it - can remove it.

I have no idea whether this was a deliberate decision or not, but there is a total absence of any UI with which to delete resources from the currently deployed version of thedatahub.org.

When paster db clean is run, the search index should also be cleared.

SOLR search doesn't support searching for part of an extra field, but it does for other fields.

i.e. title="One Two Three" matches q=one AND q=title:one and geographic_coverage="England Scotland" matches q=England BUT NOT q=geographic_coverage:England

This problem emerged when we went to SOLR in #1275 (CKAN 1.5a). Tests were skipped.

This is could be a problem for DGU and maybe elsewhere.

Either: thedatahub.org does not display 'add a dataset' top navigation link or the 'create dataset' main central link to logged in user - instead showing 'sign up' link in the central box. It does however display the 'my account' and 'logout' link in the top right correctly.

Navigating to another part of the site (e.g. search or about or my account) does bring the 'add a dataset' navigation link back and the functionality works.

Works fine on test.ckan.net so only on thedatahub.

Not obviously a caching problem, since I did try clearing cache.

OR: it displays as if you're logged in even when you log out. The links doesn't work and prompt you to log in though.

Integrating ckanext-admin into core has thrown up a number of problems:

• Look & feel does not match the rest of the site
• Tests are not passing
• On the trash page, clicking 'undelete' triggers a purge
• Using the purge functionality is dangerous; deleting and purging the latest revision will corrupt a dataset (& several corrupt datasets have been found on thedatahub.org)
• Trash page can contain nested form tags in certain cases (breaking test harness & form redirection)

Perhaps the importer deletes them before re-importing. We shouldn't have deleted resources, so let's investigate.

Needs to update on:

1. language - It's broken on thedatahub.org if you change to German and then click on the CKAN icon top-left to go to thedatahub.org again.

AND

2. whether signed in or not - #1373

BUT not latest revisions (which is what it was)

Or get rid of etag caching on this page and others?

(Editing a dataset) If the second resource contains any validation error then it says "Resources: Package resource(s) incomplete" and "Resource 1:".

if you want your ckan to reside not at ​http://yourhost.org/ but rather e.g. at ​http://yourhost.org/ckan/ you can use this hint ​http://wiki.pylonshq.com/display/pylonsdocs/Configuration+Files and set ckan.site_url accordingly in your .ini. However, there are some instances where internal links are created without the required prefix.

Graphs don't show any more at ​http://thedatahub.org/stats

This might well be a bot, but not POSTing the email field should not cause an exception - it should be 400 and then the sysadmin doesn't get these nags.

http://www.publicdata.eu/user/register
...
Module ckan.controllers.user:79 in register
<<              c.login = request.params.getone('login')
                   c.fullname = request.params.getone('fullname')
                   c.email = request.params.getone('email')
                   if not model.User.check_name_available(c.login):
                       h.flash_error(_("That username is not available."))
>>  c.email = request.params.getone('email')
Module webob.multidict:335 in getone
<<          values were found.
               """
               return self._decode_value(self.multi.getone(self._encode_key(key)))
       
           def mixed(self):
>>  return self._decode_value(self.multi.getone(self._encode_key(key)))
Module webob.multidict:99 in getone
<<          v = self.getall(key)
               if not v:
                   raise KeyError('Key not found: %r' % key)
               if len(v) > 1:
                   raise KeyError('Multiple values match %r: %r' % (key, v))
>>  raise KeyError('Key not found: %r' % key)
KeyError: "Key not found: 'email'"

HTTP_USER_AGENT	'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MRA 4.3 (build 01218); .NET CLR 1.1.4322)'
webob._parsed_post_vars	(MultiDict([('login', 'ahusernames9'), ('fullname', 'ahusernames9'), ('password1', '******'), ('password2', '******'), ('s', 'Sign up')]), <FakeCGIBody at 0xb9d7c5ec viewing MultiDict([('lo...p')])>)

e.g. cz.ckan.net has lang=cz but when you click on Home page link "English" it flashes message "Language has been set to: English" but the rest of the page is still Czech.

Validation error is incorrect arguments, so should be a "400 Bad Request" response. "409 Conflict" is for when there is a conflict creating two resources with the same identifier. Yet we get 409 for all validation errors.

On some occasions (apparently random), the documents indexed in a specific SOLR core get mixed with different site_ids.

E.g: We look for all documents in the testing.iatiregistry.org core, faceted by site_id. We would expect all documents to have site_id = iati_testing, but some of them have site_id = iatiregistry.org

​http://okfn-solr.fry-it.com:8080/solr/testing.iatiregistry.org/select?indent=on&version=2.2&q=*:*&fq=+state:active&facet=true&facet.field=site_id

<lst name="facet_fields">
<lst name="site_id">
<int name="iati_testing">265</int>
<int name="iatiregistry.org">255</int>
</lst>
</lst>

If we compare one of the records which disappeared from the "iati_testing" site_id in both the production and testing SOLR cores of the server, the records are exactly the same, including the indexed_ts property:

​http://okfn-solr.fry-it.com:8080/solr/iatiregistry.org/select?indent=on&version=2.2&q=id:97d1ab0d-b203-4757-8f4e-a0c84d2f759f&facet=true&facet.field=site_id

​http://okfn-solr.fry-it.com:8080/solr/testing.iatiregistry.org/select?indent=on&version=2.2&q=id:97d1ab0d-b203-4757-8f4e-a0c84d2f759f&facet=true&facet.field=site_id

Note that the response from the URLs shown may vary, as the testing site could have been reindexed.

paster db load ckan.pgdump will load in the database but not update the search index, no we're using SOLR.

implementation

We can't tack on 'rebuild' to this command, because it needs a 'db upgrade' first. I'm tempted to make 'db load' do the upgrade and rebuild all in one command.

If you try to GET a dataset, purge it then POST it again, it fails checking authz for the group:

Module ckan.controllers.api:278 in create
<<                  (register, subregister))
               try:
                   response_data = action(context, data_dict)
                   location = None
                   if "id" in data_dict:
>>  response_data = action(context, data_dict)
Module ckan.logic.action.create:240 in package_create_rest
<<      api = context.get('api_version') or '1'
       
           check_access('package_create_rest', context, data_dict)
       
           dictized_package = package_api_to_dict(data_dict, context)
>>  check_access('package_create_rest', context, data_dict)
Module ckan.logic:129 in check_access
<<          #    log.debug('Valid API key needed to make changes')
               #    raise NotAuthorized
               logic_authorization = new_authz.is_authorized(action, context, data_dict)
               if not logic_authorization['success']:
                   msg = logic_authorization.get('msg','')
>>  logic_authorization = new_authz.is_authorized(action, context, data_dict)
Module ckan.new_authz:16 in is_authorized
<<      auth_function = _get_auth_function(action)
           if auth_function:
               return auth_function(context, data_dict)
           else:
               raise ValueError(_('Authorization function not found: %s' % action))
>>  return auth_function(context, data_dict)
Module ckan.logic.auth.create:115 in package_create_rest
<<          return {'success': False, 'msg': _('Valid API key needed to create a package')}
       
           return package_create(context, data_dict)
       
       def group_create_rest(context, data_dict):
>>  return package_create(context, data_dict)
Module ckan.logic.auth.create:15 in package_create
<<      else:
               
               check2 = check_group_auth(context,data_dict)
               if not check2:
                   return {'success': False, 'msg': _('User %s not authorized to edit these groups') % str(user)}
>>  check2 = check_group_auth(context,data_dict)
Module ckan.logic.auth.create:90 in check_group_auth
<<      groups = set()
           for group_dict in group_dicts:
               id = group_dict.get('id')
               if not id:
                   continue
>>  id = group_dict.get('id')
AttributeError: 'unicode' object has no attribute 'get'

(using RESTful Model API)

The reason is the POST is expecting the group to be expressed as a dict, not the list of names that GET returns.