Marshmallow 3.10.0 ValidationError: Missing data for required field? - integration-testing

Question

I am new to Marshmallow (3.10.0) and I am trying to understand the following errors that I am getting:
Traceback (most recent call last):
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 779, in _run_validator
validator_func(output, partial=partial, many=many)
File "/usr/src/wazo-confd/wazo_confd/helpers/destination.py", line 351, in _validate_skill_rule_variables
['skill_rule_id'],
marshmallow.exceptions.ValidationError: Missing data for required field. When `skill_rule_variables` is defined
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/src/wazo-confd/wazo_confd/helpers/common.py", line 28, in wrapper
return func(*args, **kwargs)
File "/opt/venv/lib/python3.7/site-packages/xivo/auth_verifier.py", line 155, in wrapper
return func(*args, **kwargs)
File "/opt/venv/lib/python3.7/site-packages/xivo/auth_verifier.py", line 125, in wrapper
return func(*args, **kwargs)
File "/usr/src/wazo-confd/wazo_confd/plugins/call_filter_fallback/resource.py", line 36, in put
fallbacks = self.schema().load(request.get_json(), partial=True)
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 728, in load
data, many=many, partial=partial, unknown=unknown, postprocess=True
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 866, in _do_load
unknown=unknown,
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 674, in _deserialize
index=index,
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 496, in _call_and_store
value = getter_func(data)
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 667, in <lambda>
val, field_name, data, **d_kwargs
File "/opt/venv/lib/python3.7/site-packages/marshmallow/fields.py", line 356, in deserialize
output = self._deserialize(value, attr, data, **kwargs)
File "/usr/src/wazo-confd/wazo_confd/helpers/destination.py", line 544, in _deserialize
return fields.Nested(schema, **self.kwargs)._deserialize(value, attr, data)
File "/opt/venv/lib/python3.7/site-packages/marshmallow/fields.py", line 611, in _deserialize
return self._load(value, data, partial=partial)
File "/opt/venv/lib/python3.7/site-packages/marshmallow/fields.py", line 594, in _load
valid_data = self.schema.load(value, unknown=self.unknown, partial=partial)
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 728, in load
data, many=many, partial=partial, unknown=unknown, postprocess=True
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 891, in _do_load
field_errors=field_errors,
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 1194, in _invoke_schema_validators
partial=partial,
File "/opt/venv/lib/python3.7/site-packages/marshmallow/schema.py", line 781, in _run_validator
error_store.store_error(err.messages, err.field_name, index=index)
File "/opt/venv/lib/python3.7/site-packages/marshmallow/error_store.py", line 22, in store_error
messages = {field_name: messages}
TypeError: unhashable type: 'list'
The first one seems to occur in this file (destination.py):
# Copyright 2016-2022 The Wazo Authors (see the AUTHORS file)
# SPDX-License-Identifier: GPL-3.0-or-later
import json
from marshmallow import (
EXCLUDE,
Schema,
fields,
pre_dump,
post_load,
post_dump,
validates_schema,
validates,
)
from marshmallow.exceptions import ValidationError
from marshmallow.validate import Length, OneOf, Regexp, Predicate, Range
from xivo_dao.helpers import errors
from xivo_dao.helpers.exception import NotFoundError
from xivo_dao.resources.application import dao as application_dao
from xivo_dao.resources.conference import dao as conference_dao
from xivo_dao.resources.group import dao as group_dao
from xivo_dao.resources.ivr import dao as ivr_dao
from xivo_dao.resources.moh import dao as moh_dao
from xivo_dao.resources.outcall import dao as outcall_dao
from xivo_dao.resources.queue import dao as queue_dao
from xivo_dao.resources.skill_rule import dao as skill_rule_dao
from xivo_dao.resources.switchboard import dao as switchboard_dao
from xivo_dao.resources.user import dao as user_dao
from xivo_dao.resources.voicemail import dao as voicemail_dao
from wazo_confd.helpers.mallow import StrictBoolean
from wazo_confd.helpers.validator import GetResource, Validator
import logging
logger = logging.getLogger(__name__)
COMMAND_REGEX = r'^(?!(try)?system\()[a-zA-Z]{3,}\((.*)\)$'
CONTEXT_REGEX = r'^[a-zA-Z0-9_-]{1,39}$'
EXTEN_REGEX = r'^[0-9*#]{1,255}$'
SKILL_RULE_VARIABLE_REGEX = r'^[^[;\|]+$'
class BaseDestinationSchema(Schema):
class Meta:
unknown = EXCLUDE
type = fields.String(
validate=OneOf(
[
'application',
'conference',
'custom',
'extension',
'group',
'hangup',
'ivr',
'none',
'outcall',
'queue',
'sound',
'switchboard',
'user',
'voicemail',
]
),
required=True,
)
#post_dump
def convert_type_to_user(self, data, **kwargs):
if data['type'] == 'endcall':
data['type'] = 'hangup'
return data
#post_load
def convert_type_to_database(self, data, **kwargs):
if data['type'] == 'hangup':
data['type'] = 'endcall'
return data
class ApplicationDestinationSchema(BaseDestinationSchema):
application = fields.String(
validate=OneOf(
['callback_disa', 'custom', 'directory', 'disa', 'fax_to_mail', 'voicemail']
),
attribute='subtype',
required=True,
)
#post_dump
def convert_application_to_user(self, data, **kwargs):
if data['application'] == 'callbackdisa':
data['application'] = 'callback_disa'
elif data['application'] == 'faxtomail':
data['application'] = 'fax_to_mail'
elif data['application'] == 'voicemailmain':
data['application'] = 'voicemail'
return data
#post_load
def convert_application_to_database(self, data, **kwargs):
if data['subtype'] == 'callback_disa':
data['subtype'] = 'callbackdisa'
elif data['subtype'] == 'fax_to_mail':
data['subtype'] = 'faxtomail'
elif data['subtype'] == 'voicemail':
data['subtype'] = 'voicemailmain'
return data
class CallBackDISADestinationSchema(ApplicationDestinationSchema):
pin = fields.String(
validate=(Predicate('isdigit'), Length(max=40)),
allow_none=True,
attribute='actionarg1',
)
context = fields.String(
validate=Regexp(CONTEXT_REGEX), attribute='actionarg2', required=True
)
class CustomApplicationDestinationSchema(ApplicationDestinationSchema):
application_uuid = fields.UUID(attribute='actionarg1', required=True)
_application = fields.Nested(
'ApplicationSchema', only=['name'], attribute='application', dump_only=True
)
#post_dump
def make_application_fields_flat(self, data, **kwargs):
if data.get('_application'):
data['application_name'] = data['_application']['name']
data.pop('_application', None)
return data
class DISADestinationSchema(ApplicationDestinationSchema):
pin = fields.String(
validate=(Predicate('isdigit'), Length(max=40)),
allow_none=True,
attribute='actionarg1',
)
context = fields.String(
validate=Regexp(CONTEXT_REGEX), attribute='actionarg2', required=True
)
class DirectoryDestinationSchema(ApplicationDestinationSchema):
context = fields.String(
validate=Regexp(CONTEXT_REGEX), attribute='actionarg1', required=True
)
class FaxToMailDestinationSchema(ApplicationDestinationSchema):
email = fields.Email(validate=Length(max=80), attribute='actionarg1', required=True)
class VoicemailMainDestinationSchema(ApplicationDestinationSchema):
context = fields.String(
validate=Regexp(CONTEXT_REGEX), attribute='actionarg1', required=True
)
class ConferenceDestinationSchema(BaseDestinationSchema):
conference_id = fields.Integer(attribute='actionarg1', required=True)
conference = fields.Nested('ConferenceSchema', only=['name'], dump_only=True)
#post_dump
def make_conference_fields_flat(self, data, **kwargs):
if data.get('conference'):
data['conference_name'] = data['conference']['name']
data.pop('conference', None)
return data
class CustomDestinationSchema(BaseDestinationSchema):
command = fields.String(
validate=(Regexp(COMMAND_REGEX), Length(max=255)),
attribute='actionarg1',
required=True,
)
class ExtensionDestinationSchema(BaseDestinationSchema):
exten = fields.String(
validate=Regexp(EXTEN_REGEX), attribute='actionarg1', required=True
)
context = fields.String(
validate=Regexp(CONTEXT_REGEX), attribute='actionarg2', required=True
)
class GroupDestinationSchema(BaseDestinationSchema):
group_id = fields.Integer(attribute='actionarg1', required=True)
ring_time = fields.Float(
validate=Range(min=0), attribute='actionarg2', allow_none=True
)
group = fields.Nested('GroupSchema', only=['label', 'name'], dump_only=True)
#post_dump
def make_group_fields_flat(self, data, **kwargs):
if data.get('group'):
# TODO(pc-m): Label was added in 21.04 group_name should be remove when we remove
# the compatibility logic in group schema
data['group_name'] = data['group']['name']
data['group_label'] = data['group']['label']
data.pop('group', None)
return data
class HangupDestinationSchema(BaseDestinationSchema):
cause = fields.String(
validate=OneOf(['busy', 'congestion', 'normal']),
attribute='subtype',
missing='normal',
required=False,
)
#post_dump
def convert_cause_to_user(self, data, **kwargs):
if data['cause'] == 'hangup':
data['cause'] = 'normal'
return data
#post_load
def convert_cause_to_database(self, data, **kwargs):
if data['subtype'] == 'normal':
data['subtype'] = 'hangup'
return data
class BusyDestinationSchema(HangupDestinationSchema):
timeout = fields.Float(
attribute='actionarg1', validate=Range(min=0), allow_none=True
)
class CongestionDestinationSchema(HangupDestinationSchema):
timeout = fields.Float(
attribute='actionarg1', validate=Range(min=0), allow_none=True
)
class IVRDestinationSchema(BaseDestinationSchema):
ivr_id = fields.Integer(attribute='actionarg1', required=True)
ivr = fields.Nested('IvrSchema', only=['name'], dump_only=True)
#post_dump
def make_ivr_fields_flat(self, data, **kwargs):
if data.get('ivr'):
data['ivr_name'] = data['ivr']['name']
data.pop('ivr', None)
return data
class NormalDestinationSchema(HangupDestinationSchema):
pass
class NoneDestinationSchema(BaseDestinationSchema):
pass
class OutcallDestinationSchema(BaseDestinationSchema):
outcall_id = fields.Integer(attribute='actionarg1', required=True)
exten = fields.String(
validate=(Predicate('isdigit'), Length(max=255)),
attribute='actionarg2',
required=True,
)
class QueueDestinationSchema(BaseDestinationSchema):
queue_id = fields.Integer(attribute='actionarg1', required=True)
ring_time = fields.Float(validate=Range(min=0), allow_none=True)
skill_rule_id = fields.Integer(allow_none=True)
skill_rule_variables = fields.Dict(allow_none=True)
queue = fields.Nested('QueueSchema', only=['label'], dump_only=True)
#pre_dump
def separate_action(self, data, **kwargs):
options = data.actionarg2.split(';') if data.actionarg2 else []
data.ring_time = None
data.skill_rule_id = None
data.skill_rule_variables = None
_skill_rule_variables = None
if len(options) == 1:
data.ring_time = options[0] or None
elif len(options) == 2: # id is always bound with variables
data.skill_rule_id = options[0]
_skill_rule_variables = options[1] or None
elif len(options) == 3:
data.ring_time = options[0]
data.skill_rule_id = options[1]
_skill_rule_variables = options[2] or None
if _skill_rule_variables:
_skill_rule_variables = _skill_rule_variables.replace(
'|', ','
) # dialplan interpret comma ...
data.skill_rule_variables = json.loads(_skill_rule_variables)
return data
#post_load
def merge_action(self, data, **kwargs):
ring_time = data.pop('ring_time', None)
skill_rule_id = data.pop('skill_rule_id', None)
skill_rule_variables = data.pop('skill_rule_variables', None)
skill_rule_variables_str = (
json.dumps(skill_rule_variables).replace(',', '|')
if skill_rule_variables
else ''
)
data[
'actionarg2'
] = '{ring_time}{sep1}{skill_rule_id}{sep2}{skill_rule_variables}'.format(
ring_time=ring_time or '',
sep1=';' if ring_time and skill_rule_id else '',
skill_rule_id=skill_rule_id or '',
sep2=';' if skill_rule_id else '',
skill_rule_variables=skill_rule_variables_str,
)
return data
#post_dump
def make_queue_fields_flat(self, data, **kwargs):
if data.get('queue'):
data['queue_label'] = data['queue']['label']
data.pop('queue', None)
return data
#validates_schema
def _validate_skill_rule_variables(self, data, **kwargs):
logger.critical('------------------------------------------')
logger.critical(data)
logger.critical(kwargs)
logger.critical('------------------------------------------')
if not data.get('skill_rule_variables'):
return
if not data.get('skill_rule_id'):
raise ValidationError(
'Missing data for required field. When `skill_rule_variables` is defined',
['skill_rule_id'],
)
#validates('skill_rule_variables')
def _validate_skill_rule_variables_value(self, variables):
# with marshmallow 3.0 we can set this validator on the field declaration
if not variables:
return
validator = Regexp(SKILL_RULE_VARIABLE_REGEX)
for key, value in variables.items():
validator(key)
validator(value)
class SoundDestinationSchema(BaseDestinationSchema):
filename = fields.String(
validate=Length(max=255), attribute='actionarg1', required=True
)
skip = StrictBoolean()
no_answer = StrictBoolean()
#pre_dump
def separate_action(self, data, **kwargs):
options = data.actionarg2 if data.actionarg2 else ''
data.skip = True if 'skip' in options else False
data.no_answer = True if 'noanswer' in options else False
return data
#post_load
def merge_action(self, data, **kwargs):
data['actionarg2'] = '{skip}{noanswer}'.format(
skip='skip' if data.pop('skip', False) else '',
noanswer='noanswer' if data.pop('no_answer', False) else '',
)
return data
class SwitchboardDestinationSchema(BaseDestinationSchema):
switchboard_uuid = fields.UUID(attribute='actionarg1', required=True)
ring_time = fields.Float(
validate=Range(min=0), attribute='actionarg2', allow_none=True
)
switchboard = fields.Nested('SwitchboardSchema', only=['name'], dump_only=True)
#post_dump
def make_switchboard_fields_flat(self, data, **kwargs):
if data.get('switchboard'):
data['switchboard_name'] = data['switchboard']['name']
data.pop('switchboard', None)
return data
class UserDestinationSchema(BaseDestinationSchema):
user_id = fields.Integer(attribute='actionarg1', required=True)
ring_time = fields.Float(validate=Range(min=0), allow_none=True)
moh_uuid = fields.UUID(allow_none=True)
user = fields.Nested('UserSchema', only=['firstname', 'lastname'], dump_only=True)
#post_dump
def make_user_fields_flat(self, data, **kwargs):
if data.get('user'):
data['user_firstname'] = data['user']['firstname']
data['user_lastname'] = data['user']['lastname']
data.pop('user', None)
return data
#pre_dump
def separate_action(self, data, **kwargs):
options = data.actionarg2.split(';') if data.actionarg2 else []
data.ring_time = None
data.moh_uuid = None
if len(options) > 0:
data.ring_time = options[0] or None
if len(options) > 1: # id is always bound with variables
data.moh_uuid = options[1]
return data
#post_load
def merge_action(self, data, **kwargs):
ring_time = data.pop('ring_time', None)
moh_uuid = data.pop('moh_uuid', None)
actionarg2 = ''
if ring_time is not None:
actionarg2 += str(ring_time)
if moh_uuid is not None:
actionarg2 += ';{}'.format(moh_uuid)
data['actionarg2'] = actionarg2
return data
class VoicemailDestinationSchema(BaseDestinationSchema):
voicemail_id = fields.Integer(attribute='actionarg1', required=True)
skip_instructions = StrictBoolean()
greeting = fields.String(validate=OneOf(['busy', 'unavailable']), allow_none=True)
voicemail = fields.Nested('VoicemailSchema', only=['name'], dump_only=True)
#pre_dump
def separate_action(self, data, **kwargs):
options = data.actionarg2 if data.actionarg2 else ''
data.skip_instructions = True if 's' in options else False
data.greeting = (
'busy' if 'b' in options else 'unavailable' if 'u' in options else None
)
return data
#post_load
def merge_action(self, data, **kwargs):
greeting = data.pop('greeting', None)
data['actionarg2'] = '{}{}'.format(
'b' if greeting == 'busy' else 'u' if greeting == 'unavailable' else '',
's' if data.pop('skip_instructions', False) else '',
)
return data
#post_dump
def make_voicemail_fields_flat(self, data, **kwargs):
if data.get('voicemail'):
data['voicemail_name'] = data['voicemail']['name']
data.pop('voicemail', None)
return data
class DestinationField(fields.Nested):
application_schemas = {
'callback_disa': CallBackDISADestinationSchema,
'callbackdisa': CallBackDISADestinationSchema,
'custom': CustomApplicationDestinationSchema,
'directory': DirectoryDestinationSchema,
'disa': DISADestinationSchema,
'fax_to_mail': FaxToMailDestinationSchema,
'faxtomail': FaxToMailDestinationSchema,
'voicemail': VoicemailMainDestinationSchema,
'voicemailmain': VoicemailMainDestinationSchema,
}
hangup_schemas = {
'busy': BusyDestinationSchema,
'congestion': CongestionDestinationSchema,
'normal': NormalDestinationSchema,
'hangup': NormalDestinationSchema,
}
destination_schemas = {
'application': ApplicationDestinationSchema,
'conference': ConferenceDestinationSchema,
'custom': CustomDestinationSchema,
'extension': ExtensionDestinationSchema,
'group': GroupDestinationSchema,
'hangup': HangupDestinationSchema,
'endcall': HangupDestinationSchema,
'ivr': IVRDestinationSchema,
'none': NoneDestinationSchema,
'outcall': OutcallDestinationSchema,
'queue': QueueDestinationSchema,
'sound': SoundDestinationSchema,
'switchboard': SwitchboardDestinationSchema,
'user': UserDestinationSchema,
'voicemail': VoicemailDestinationSchema,
}
def __init__(self, **kwargs):
# FIXME(sileht): I'm not sure validation works here...
# This of dynamic nesterd stuffs should not done like this.
self.kwargs = kwargs
self.kwargs["unknown"] = EXCLUDE
super().__init__(BaseDestinationSchema, **self.kwargs)
def _deserialize(self, value, attr, data, **kwargs):
self.schema.context = self.context
base = super()._deserialize(value, attr, data, **kwargs)
schema = self.destination_schemas[base['type']]
if base['type'] == 'application':
base = fields.Nested(schema, **self.kwargs)._deserialize(value, attr, data)
schema = self.application_schemas[base['subtype']]
if base['type'] == 'endcall':
base = fields.Nested(schema, **self.kwargs)._deserialize(value, attr, data)
schema = self.hangup_schemas[base['subtype']]
return fields.Nested(schema, **self.kwargs)._deserialize(value, attr, data)
def _serialize(self, nested_obj, attr, obj):
base = super()._serialize(nested_obj, attr, obj)
if not base:
return base
schema = self.destination_schemas[base['type']]
if base['type'] == 'application':
base = fields.Nested(schema, **self.kwargs)._serialize(
nested_obj, attr, obj
)
schema = self.application_schemas[base['application']]
if base['type'] == 'hangup':
base = fields.Nested(schema, **self.kwargs)._serialize(
nested_obj, attr, obj
)
schema = self.hangup_schemas[base['cause']]
return fields.Nested(schema, **self.kwargs)._serialize(nested_obj, attr, obj)
class OptionalGetSkillRuleFromActionArg2Resource(Validator):
def __init__(self, dao_get):
self.dao_get = dao_get
def validate(self, model):
destination = QueueDestinationSchema().dump(model)
skill_rule_id = destination.get('skill_rule_id', None)
if not skill_rule_id:
return
try:
self.dao_get(skill_rule_id)
except NotFoundError:
metadata = {'skill_rule_id': skill_rule_id}
raise errors.param_not_found('skill_rule_id', 'SkillRule', **metadata)
class GetMohFromActionArg2Resource(Validator):
def __init__(self, dao_get):
self._dao_get = dao_get
def validate(self, model):
destination = UserDestinationSchema().dump(model)
moh_uuid = destination.get('moh_uuid', None)
if not moh_uuid:
return
try:
self._dao_get(moh_uuid)
except NotFoundError:
metadata = {'moh_uuid': moh_uuid}
raise errors.param_not_found('moh_uuid', 'MOH', **metadata)
class DestinationValidator:
_VALIDATORS = {
'application:callbackdisa': [],
'application:custom': [
GetResource('actionarg1', application_dao.get, 'Application')
],
'application:directory': [],
'application:disa': [],
'application:faxtomail': [],
'application:voicemailmain': [],
'conference': [GetResource('actionarg1', conference_dao.get, 'Conference')],
'custom': [],
'extension': [],
'group': [GetResource('actionarg1', group_dao.get, 'Group')],
'endcall:busy': [],
'endcall:congestion': [],
'endcall:hangup': [],
'ivr': [GetResource('actionarg1', ivr_dao.get, 'IVR')],
'none': [],
'outcall': [GetResource('actionarg1', outcall_dao.get, 'Outcall')],
'queue': [
GetResource('actionarg1', queue_dao.get, 'Queue'),
OptionalGetSkillRuleFromActionArg2Resource(skill_rule_dao.get),
],
'sound': [],
'switchboard': [GetResource('actionarg1', switchboard_dao.get, 'Switchboard')],
'user': [
GetResource('actionarg1', user_dao.get, 'User'),
GetMohFromActionArg2Resource(moh_dao.get),
],
'voicemail': [GetResource('actionarg1', voicemail_dao.get, 'Voicemail')],
}
def validate(self, destination):
for validator in self._VALIDATORS[destination.action]:
validator.validate(destination)
As for the second one, it seems to be caused within this file (resource.py):
# Copyright 2018-2020 The Wazo Authors (see the AUTHORS file)
# SPDX-License-Identifier: GPL-3.0-or-later
from flask import request
from wazo_confd.auth import required_acl
from wazo_confd.helpers.restful import ConfdResource
from .schema import CallFilterFallbackSchema
class CallFilterFallbackList(ConfdResource):
schema = CallFilterFallbackSchema
has_tenant_uuid = True
def __init__(self, service, call_filter_dao):
super().__init__()
self.service = service
self.call_filter_dao = call_filter_dao
#required_acl('confd.callfilters.{call_filter_id}.fallbacks.read')
def get(self, call_filter_id):
tenant_uuids = self._build_tenant_list({'recurse': True})
call_filter = self.call_filter_dao.get(
call_filter_id, tenant_uuids=tenant_uuids
)
return self.schema().dump(call_filter.fallbacks)
#required_acl('confd.callfilters.{call_filter_id}.fallbacks.update')
def put(self, call_filter_id):
tenant_uuids = self._build_tenant_list({'recurse': True})
call_filter = self.call_filter_dao.get(
call_filter_id, tenant_uuids=tenant_uuids
)
fallbacks = self.schema().load(request.get_json())
self.service.edit(call_filter, fallbacks)
return '', 204
I am not sure what causes these errors; tried to find online solutions but I could not fix the issue.