Plone/Dexterity schema.Choice not allowing Spanish characters - character-encoding

Question

In Plone 4.1.2 I created a myContentType with Dexterity. It has 3 zope.schema.Choice fields. Two of them take their values from a hardcoded vocabulary and the other one from a dynamic vocabulary. In both cases, if I choose a value that has Spanish accents, when I save the add form the selection is gone and doesn't show up in the view form (without showing any error message). But if I choose a non accented value everything works fine.
Any advise on how to solve this problem?
(David; I hope this is what you asked me for)
# -*- coding: utf-8 -*-
from five import grok
from zope import schema
from plone.directives import form, dexterity
from zope.component import getMultiAdapter
from plone.namedfile.interfaces import IImageScaleTraversable
from plone.namedfile.field import NamedBlobFile, NamedBlobImage
from plone.formwidget.contenttree import ObjPathSourceBinder
from zope.schema.vocabulary import SimpleVocabulary, SimpleTerm
from zope.schema.interfaces import IVocabularyFactory
from z3c.formwidget.query.interfaces import IQuerySource
from zope.component import queryUtility
from plone.formwidget.masterselect import (
_,
MasterSelectField,
MasterSelectBoolField,
)
from plone.app.textfield.interfaces import ITransformer
from plone.indexer import indexer
from oaxaca.newcontent import ContentMessageFactory as _
from oaxaca.newcontent.config import OAXACA
from types import UnicodeType
_default_encoding = 'utf-8'
def _encode(s, encoding=_default_encoding):
try:
return s.encode(encoding)
except (TypeError, UnicodeDecodeError, ValueError):
return s
def _decode(s, encoding=_default_encoding):
try:
return unicode(s, encoding)
except (TypeError, UnicodeDecodeError, ValueError):
return s
view = view.encode('utf-8')
def getSlaveVocab(master):
results = []
if master in OAXACA:
results = sorted(OAXACA[master])
return SimpleVocabulary.fromValues(results)
class IFicha(form.Schema, IImageScaleTraversable):
"""Describes a ficha
"""
tipoMenu = schema.Choice(
title=_(u"Tipo de evento"),
description=_(u"Marque la opción que aplique o "
"seleccione otro si ninguna aplica"),
values=(
u'Manifestación en lugar público',
u'Toma de instalaciones municipales',
u'Toma de instalaciones estatales',
u'Toma de instalaciones federales',
u'Bloqueo de carretera municipal',
u'Bloqueo de carretera estatal',
u'Bloqueo de carretera federal',
u'Secuestro de funcionario',
u'Otro',),
required=False,
)
tipoAdicional = schema.TextLine(
title=_(u"Registre un nuevo tipo de evento"),
description=_(u"Use este campo solo si marcó otro en el menú de arriba"),
required=False
)
fecha = schema.Date(
title=_(u"Fecha"),
description=_(u"Seleccione el día en que ocurrió el evento"),
required=False
)
municipio = MasterSelectField(
title=_(u"Municipio"),
description=_(u"Seleccione el municipio donde ocurrió el evento"),
required=False,
vocabulary="oaxaca.newcontent.municipios",
slave_fields=(
{'name': 'localidad',
'action': 'vocabulary',
'vocab_method': getSlaveVocab,
'control_param': 'master',
},
)
)
localidad = schema.Choice(
title=_(u"Localidad"),
description=_(u"Seleccione la localidad donde ocurrió el evento."),
values=[u'',],
required=False,
)
actores = schema.Text(
title=_(u"Actores"),
description=_(u"Liste las agrupaciones y los individuos que participaron en el evento"),
required=False,
)
demandas = schema.Text(
title=_(u"Demandas"),
description=_(u"Liste las demandas o exigencias de los participantes"),
required=False
)
depResponsable = schema.Text(
title=_(u"Dependencias"),
description=_(u"Liste las dependencias gubernamentales responsables de atender las demandas"),
required=False
)
seguimiento = schema.Text(
title=_(u"Acciones de seguimiento"),
description=_(u"Anote cualquier accion de seguimiento que se haya realizado"),
required=False
)
modulo = schema.Choice(
title=_(u"Informa"),
description=_(u"Seleccione el módulo que llena esta ficha"),
values=(
u'M1',
u'M2',
u'M3',
u'M4',
u'M5',
u'M6',
u'M7',
u'M8',
u'M9',
u'M10',
u'M11',
u'M12',
u'M13',
u'M14',
u'M15',
u'M16',
u'M17',
u'M18',
u'M19',
u'M20',
u'M21',
u'M22',
u'M23',
u'M24',
u'M25',
u'M26',
u'M27',
u'M28',
u'M29',
u'M30',),
required=False
)
imagen1 = NamedBlobImage(
title=_(u"Imagen 1"),
description=_(u"Subir imagen 1"),
required=False
)
imagen2 = NamedBlobImage(
title=_(u"Imagen 2"),
description=_(u"Subir imagen 2"),
required=False
)
anexo1 = NamedBlobFile(
title=_(u"Anexo 1"),
description=_(u"Subir archivo 1"),
required=False
)
anexo2 = NamedBlobFile(
title=_(u"Anexo 2"),
description=_(u"Subir archivo 2"),
required=False
)
#indexer(IFicha)
def textIndexer(obj):
"""SearchableText contains fechaFicha, actores, demandas, municipio and localidad as plain text.
"""
transformer = ITransformer(obj)
text = transformer(obj.text, 'text/plain')
return '%s %s %s %s %s' % (obj.fecha,
obj.actores,
obj.demandas,
obj.municipio,
obj.localidad)
grok.global_adapter(textIndexer, name='SearchableText')
class View(grok.View):
"""Default view (called "##view"") for a ficha.
The associated template is found in ficha_templates/view.pt.
"""
grok.context(IFicha)
grok.require('zope2.View')
grok.name('view')

Answer 1

I found the same problem some months ago on early development of collective.nitf.
The tokens on a vocabulary must be normalized; this is how I solved it:
# -*- coding: utf-8 -*-
import unicodedata
…
class SectionsVocabulary(object):
"""Creates a vocabulary with the sections stored in the registry; the
vocabulary is normalized to allow the use of non-ascii characters.
"""
grok.implements(IVocabularyFactory)
def __call__(self, context):
registry = getUtility(IRegistry)
settings = registry.forInterface(INITFSettings)
items = []
for section in settings.sections:
token = unicodedata.normalize('NFKD', section).encode('ascii', 'ignore').lower()
items.append(SimpleVocabulary.createTerm(section, token, section))
return SimpleVocabulary(items)
grok.global_utility(SectionsVocabulary, name=u'collective.nitf.Sections')

Answer 2

Plone uses gettext for internationalization. The bulletproof approach would be to implement your custom functionality in English and use locales for your specific language. Look at the relevant parts of the community manual on how this is done. Since you already setup a MessageFactory you could even use a tool like e.g. zettwerk.i18nduder for quick extraction of message strings.

Answer 3

I found a partial explanation/solution here. I can get the Spanish characters in the view form if i do:
-- coding: utf-8 --
from plone.directives import form
from five import grok
from zope import schema
from plone.directives import form, dexterity
from zope.schema.vocabulary import SimpleVocabulary
myVocabulary = SimpleVocabulary.fromItems((
(u"Foo", "id_foó"),
(u"Baroo", "id_baroó")))
class IPrueba(form.Schema):
tipoMenu = schema.Choice(
title=_(u"Tipo de evento"),
description=_(u"Marque la opción que aplique o "
"seleccione otro si ninguna aplica"),
vocabulary=myVocabulary,
required=False,
)