AssertionError on trying to add new entity using matcher on spaCy - named-entity-recognition

Question

I'm trying to match all e-mail like looking text in a bunch of documents and add it to custom NER label called 'EMAIL'.
Here is the code for a test case.
nlp = spacy.load('en_core_web_sm')
matcher = Matcher(nlp.vocab)
EMAIL = nlp.vocab.strings['EMAIL']
def add_email_ent(matcher, doc, i, matches):
match_id, start, end = matches[i]
doc.ents += ((EMAIL, start, end),)
matcher.add('EmailPII', add_email_ent, [{'LIKE_EMAIL': True}])
text = u"Hi, this is John. My email is john#ymail.com and an alternate is john#gmail.com"
doc = nlp(text)
matches = matcher(doc)
for i,[match_id, start, end] in enumerate(matches):
print (i+1, doc[start:end])
for ent in doc.ents:
print (ent.text, ent.label_)
Here's what I get when I run this code.
Traceback (most recent call last):
File "C:/Python27/emailpii.py", line 26, in <module>
matches = matcher(doc)
File "matcher.pyx", line 407, in spacy.matcher.Matcher.__call__
File "C:/Python27/emailpii.py", line 19, in add_event_ent
doc.ents += ((EMAIL, start, end),)
File "doc.pyx", line 415, in spacy.tokens.doc.Doc.ents.__get__
File "span.pyx", line 61, in spacy.tokens.span.Span.__cinit__
AssertionError: 17587345535198158200
However, on running a similar example
import spacy
print "*****************"
print(spacy.__version__)
print "*****************"
from spacy.matcher import Matcher
#from spacy import displacy
nlp = spacy.load('en_core_web_sm')
matcher = Matcher(nlp.vocab)
EVENT = nlp.vocab.strings['EVENT']
def add_event_ent(matcher, doc, i, matches):
match_id, start, end = matches[i]
doc.ents += ((EVENT, start, end),)
matcher.add('GoogleIO', add_event_ent,
[{'ORTH': 'Google'}, {'ORTH': 'I'}, {'ORTH': '/'}, {'ORTH': 'O'}],
[{'ORTH': 'Google'}, {'ORTH': 'I'}, {'ORTH': '/'}, {'ORTH': 'O'}, {'IS_DIGIT': True}])
text = u"Google I/O was great this year. See you all again in Google I/O 2018"
doc = nlp(text)
matches = matcher(doc)
for i,[match_id, start, end] in enumerate(matches):
print (i, doc[start:end])
for ent in doc.ents:
print (ent.text, ent.label_)
#displacy.serve(doc, style = 'ent')
I get the output as desired:
2.0.1
(0, Google I/O)
(1, Google I/O)
(2, Google I/O 2018)
(u'Google I/O', u'EVENT')
(u'this year', u'DATE')
(u'Google I/O 2018', u'EVENT')
Am I missing something here?

Answer 1

I believe your first code fails because you have not added an Entity label for 'EMAIL'. The second code works because EVENT is a pre-existing Entity type.
The documentation is not very clear on what the first argument of the matcher.add() method actually does, but it adds an Entity label for you. Here are two alternatives that should work and clear up the confusion:
Alternative 1:
import spacy
from spacy.matcher import Matcher
nlp = spacy.load('en_core_web_sm')
matcher = Matcher(nlp.vocab)
#EMAIL = nlp.vocab.strings['EMAIL'] #Not needed
def add_email_ent(matcher, doc, i, matches):
match_id, start, end = matches[i]
doc.ents += ((match_id, start, end),)
matcher.add('EMAIL', add_email_ent, [{'LIKE_EMAIL': True}])
text = u"Hi, this is John. My email is john#ymail.com and an alternate is john#gmail.com"
doc = nlp(text)
matches = matcher(doc)
for i,[match_id, start, end] in enumerate(matches):
print (i+1, doc[start:end])
for ent in doc.ents:
print (ent.text, ent.label_)
Alternative 2 (I'm not sure why you'd want to do it this way because you end up with two entity labels serving essentially the same purpose, but provided just for illustration purposes):
import spacy
from spacy.matcher import Matcher
from spacy.pipeline import EntityRecognizer
nlp = spacy.load('en_core_web_sm')
matcher = Matcher(nlp.vocab)
ner = EntityRecognizer(nlp.vocab)
ner.add_label('EMAIL')
EMAIL = nlp.vocab.strings['EMAIL']
def add_email_ent(matcher, doc, i, matches):
match_id, start, end = matches[i]
doc.ents += ((EMAIL, start, end),)
matcher.add('EmailPII', add_email_ent, [{'LIKE_EMAIL': True}])
text = u"Hi, this is John. My email is john#ymail.com and an alternate is john#gmail.com"
doc = nlp(text)
matches = matcher(doc)
for i,[match_id, start, end] in enumerate(matches):
print (i+1, doc[start:end])
for ent in doc.ents:
print (ent.text, ent.label_)