In the docs it's shown that an args_converter can be written as a regular function or as a lambda function, but i can't seem to get it to work with a non lambda one, and all examples i see are lambdas.
Here's a piece of my code:
def groupArgsConv(index, item):
res = {'text': item['title'],
'size_hint_y': None,
'heigth': '25dp'}
return res
self.ids.grouplist.adapter = ListAdapter(args_converter=groupArgsConv,
data=self.groups,
cls=ListItemLabel)
This doesn't work, but this does:
list_item_args_converter = lambda index, item: {'text': item['title'],
'size_hint_y': None,
'height': '25dp'}
self.ids.grouplist.adapter = ListAdapter(args_converter=list_item_args_converter,
data=self.groups,
cls=ListItemLabel)
Why? How do i get the 1st example to work?
This code works for me:
from kivy.adapters.listadapter import ListAdapter
from kivy.uix.listview import ListItemLabel, ListView
from kivy.base import runTouchApp
data = [{'title': str(i)} for i in range(100)]
def groupArgsConv(index, item):
res = {'text': item['title'],
'size_hint_y': None,
'height': '25dp'}
return res
list_adapter = ListAdapter(
data=data,
args_converter=groupArgsConv,
cls=ListItemLabel)
if __name__ == '__main__':
runTouchApp(ListView(adapter=list_adapter))
What do you mean by "doesn't work"? Do you have some kind of an exception or strange output in the log?
Related
I am trying loop the ChiSquare test and outcome is not shown as required, that is in Dataframe.
All columns are coming one row..
Please help
# Import the function
from scipy.stats import chi2_contingency
chi2_check = []
for i in df_clean.select_dtypes(['object']):
if chi2_contingency(pd.crosstab(df_clean['Final_Comments'], df_clean[i]))[1] < 0.05:
chi2_check.append('Reject Null Hypothesis')
else:
chi2_check.append('Fail to Reject Null Hypothesis')
res = pd.DataFrame(data = [df_clean.select_dtypes(['object']), chi2_check]
).T
res.columns = ['Column', 'Hypothesis']
print(res)
res.columns
I am trying to implement a custom aggregation using TFF by changing the code from this tutorial . I would like to rewrite next_fn so that all the client weights are placed at the server for further computations. As federated_collect was removed from tff-nightly, I am trying to do that using federated_aggregate.
This is what I have so far:
def accumulate(x, y):
x.append(y)
return x
def merge(x, y):
x.extend(y)
return y
#tff.federated_computation(federated_server_type, federated_dataset_type)
def next_fn(server_state, federated_dataset):
server_weights_at_client = tff.federated_broadcast(
server_state.trainable_weights)
client_deltas = tff.federated_map(
client_update_fn, (federated_dataset, server_weights_at_client))
z = []
agg_result = tff.federated_aggregate(client_deltas, z,
accumulate=tff.tf_computation(accumulate),
merge=tff.tf_computation(merge),
report=tff.tf_computation(lambda x: x))
new_weights = do_smth_with_result(agg_result)
server_state = tff.federated_map(
server_update_fn, (server_state, new_weights))
return server_state
However this results in the following Exception:
File "/home/yana/Documents/Uni/Thesis/grufedatt_try.py", line 351, in <module>
def next_fn(server_state, federated_dataset):
File "/home/yana/anaconda3/envs/fedenv/lib/python3.9/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 494, in __call__
wrapped_func = self._strategy(
File "/home/yana/anaconda3/envs/fedenv/lib/python3.9/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 222, in __call__
result = fn_to_wrap(*args, **kwargs)
File "/home/yana/Documents/Uni/Thesis/grufedatt_try.py", line 358, in next_fn
agg_result = tff.federated_aggregate(client_deltas, z,
File "/home/yana/anaconda3/envs/fedenv/lib/python3.9/site-packages/tensorflow_federated/python/core/impl/federated_context/intrinsics.py", line 140, in federated_aggregate
raise TypeError(
TypeError: Expected parameter `accumulate` to be of type (<<<float32[9999,96],float32[96,1024],float32[256,1024],float32[1024],float32[256,96],float32[96]>>,<float32[9999,96],float32[96,1024],float32[256,1024],float32[1024],float32[256,96],float32[96]>> -> <<float32[9999,96],float32[96,1024],float32[256,1024],float32[1024],float32[256,96],float32[96]>>), but received (<<>,<float32[9999,96],float32[96,1024],float32[256,1024],float32[1024],float32[256,96],float32[96]>> -> <<float32[9999,96],float32[96,1024],float32[256,1024],float32[1024],float32[256,96],float32[96]>>) instead.
Try using tff.aggregators.federated_sample with max_num_samples being equal to the number of clients you have.
That should be a simple drop-in replacement for how you would previously use tff.federated_collect.
In your accumulate, the issue is that you are changing number of tensors the accumulator would contain, so you get an error when accumulating more than a single accumuland. If you would want to go this way though, for a rank-1 accumuland with k elements, you could probably do something like the following instead:
#tff.tf_computation(tff.types.TensorType(tf.float32, [None, k]),
tff.types.TensorType(tf.float32, [k]))
def accumulate(accumulator, accumuland):
return tf.concat([accumulator, tf.expand_dims(accumuland, axis=0)], axis=0)
From the documentation, Number of allowed automatic retries if computing a result fails.
Does "result" refer to each individual task or the entire compute() call?
If it refers to the entire call, how to implement retries for each task in dask.delayed?
Also, I'm not sure if the retries are working at all, as per below code.
import dask
import random
#dask.delayed
def add(x, y):
return x + y
#dask.delayed
def divide(sum_i):
n = random.randint(0, 1)
result = sum_i / n
return result
tasks = []
for i in range(3):
sum_i = add(i, i+1)
divide_n = divide(sum_i)
tasks.append(divide_n)
dask.compute(*tasks, retries=1000)
Expected output is (1, 3, 5), actual is ZeroDivisionError.
If anyone is interested, we use a #retry decorator for tasks, like so:
#dask.delayed
#retry(Exception, tries=3, delay=5)
def my_func():
pass
Retry decorator:
from functools import wraps
def retry(exceptions, tries=4, delay=3, backoff=2, logger=None):
"""
Retry calling the decorated function using an exponential backoff.
Args:
exceptions: The exception to check. may be a tuple of
exceptions to check.
tries: Number of times to try (not retry) before giving up.
delay: Initial delay between retries in seconds.
backoff: Backoff multiplier (e.g. value of 2 will double the delay
each retry).
logger: Logger to use.
"""
if not logger:
logger = logging.getLogger(__name__)
def deco_retry(f):
#wraps(f)
def f_retry(*args, **kwargs):
mtries, mdelay = tries, delay
while mtries > 1:
try:
return f(*args, **kwargs)
except exceptions as e:
msg = f"{e}, \nRetrying in {mdelay} seconds..."
logger.warning(msg)
sleep(mdelay)
mtries -= 1
mdelay *= backoff
return f(*args, **kwargs)
return f_retry # true decorator
return deco_retry
I am trying to send a list of elements as a PipelineParameter to a lightweight component.
Here is a sample that reproduces the problem. Here is the function:
def my_func(my_list: list) -> bool:
print(f'my_list is {my_list}')
print(f'my_list is of type {type(my_list)}')
print(f'elem 0 is {my_list[0]}')
print(f'elem 1 is {my_list[1]}')
return True
And if I execute it with this:
test_data = ['abc', 'def']
my_func(test_data)
It behaves as expected:
my_list is ['abc', 'def']
my_list is of type <class 'list'>
elem 0 is abc
elem 1 is def
but if I wrap it in an op and and set up a pipeline:
import kfp
my_op = kfp.components.func_to_container_op(my_func)
#kfp.dsl.pipeline()
def my_pipeline(my_list: kfp.dsl.PipelineParam = kfp.dsl.PipelineParam('my_list', param_type=kfp.dsl.types.List())):
my_op(my_list)
kfp.compiler.Compiler().compile(my_pipeline, 'my_pipeline.zip')
And then run a pipeline:
client = kfp.Client()
experiment = client.create_experiment('Default')
client.run_pipeline(experiment.id, 'my job', 'my_pipeline.zip', params={'my_list': test_data})
Then it seems at some point my list was converted to a string!
my_list is ['abc', 'def']
my_list is of type <class 'str'>
elem 0 is [
elem 1 is '
Here is a workaround I discovered, serializing arguments as a json string. Not sure this is really the best way...
The bare function becomes:
def my_func(json_arg_str: str) -> bool:
import json
args = json.loads(json_arg_str)
my_list = args['my_list']
print(f'my_list is {my_list}')
print(f'my_list is of type {type(my_list)}')
print(f'elem 0 is {my_list[0]}')
print(f'elem 1 is {my_list[1]}')
return True
Which still works as long as you pass the args as a json string instead of a list:
test_data = '{"my_list":["abc", "def"]}'
my_func(test_data)
Which produces expected results:
my_list is ['abc', 'def']
my_list is of type <class 'list'>
elem 0 is abc
elem 1 is def
And now the pipeline is changed to accept a str instead of a PipelineParam of type kfp.dsl.types.List:
import kfp
my_op = kfp.components.func_to_container_op(my_func)
#kfp.dsl.pipeline()
def my_pipeline(json_arg_str: str):
my_op(json_arg_str)
kfp.compiler.Compiler().compile(my_pipeline, 'my_pipeline.zip')
Which, when executed like this:
client = kfp.Client()
experiment = client.create_experiment('Default')
client.run_pipeline(experiment.id, 'my job', 'my_pipeline.zip', params={'json_arg_str': test_data})
Produces the same result:
my_list is ['abc', 'def']
my_list is of type <class 'list'>
elem 0 is abc
elem 1 is def
Although it works, I nevertheless find this workaround annoying. What then is the point of kfp.dsl.types.List, if not for allowing a PipelineParam that is a List?
Currently the best option seems to be serializing the arguments. There is one issue related to this: https://github.com/kubeflow/pipelines/issues/1901
I'm trying to match all e-mail like looking text in a bunch of documents and add it to custom NER label called 'EMAIL'.
Here is the code for a test case.
nlp = spacy.load('en_core_web_sm')
matcher = Matcher(nlp.vocab)
EMAIL = nlp.vocab.strings['EMAIL']
def add_email_ent(matcher, doc, i, matches):
match_id, start, end = matches[i]
doc.ents += ((EMAIL, start, end),)
matcher.add('EmailPII', add_email_ent, [{'LIKE_EMAIL': True}])
text = u"Hi, this is John. My email is john#ymail.com and an alternate is john#gmail.com"
doc = nlp(text)
matches = matcher(doc)
for i,[match_id, start, end] in enumerate(matches):
print (i+1, doc[start:end])
for ent in doc.ents:
print (ent.text, ent.label_)
Here's what I get when I run this code.
Traceback (most recent call last):
File "C:/Python27/emailpii.py", line 26, in <module>
matches = matcher(doc)
File "matcher.pyx", line 407, in spacy.matcher.Matcher.__call__
File "C:/Python27/emailpii.py", line 19, in add_event_ent
doc.ents += ((EMAIL, start, end),)
File "doc.pyx", line 415, in spacy.tokens.doc.Doc.ents.__get__
File "span.pyx", line 61, in spacy.tokens.span.Span.__cinit__
AssertionError: 17587345535198158200
However, on running a similar example
import spacy
print "*****************"
print(spacy.__version__)
print "*****************"
from spacy.matcher import Matcher
#from spacy import displacy
nlp = spacy.load('en_core_web_sm')
matcher = Matcher(nlp.vocab)
EVENT = nlp.vocab.strings['EVENT']
def add_event_ent(matcher, doc, i, matches):
match_id, start, end = matches[i]
doc.ents += ((EVENT, start, end),)
matcher.add('GoogleIO', add_event_ent,
[{'ORTH': 'Google'}, {'ORTH': 'I'}, {'ORTH': '/'}, {'ORTH': 'O'}],
[{'ORTH': 'Google'}, {'ORTH': 'I'}, {'ORTH': '/'}, {'ORTH': 'O'}, {'IS_DIGIT': True}])
text = u"Google I/O was great this year. See you all again in Google I/O 2018"
doc = nlp(text)
matches = matcher(doc)
for i,[match_id, start, end] in enumerate(matches):
print (i, doc[start:end])
for ent in doc.ents:
print (ent.text, ent.label_)
#displacy.serve(doc, style = 'ent')
I get the output as desired:
2.0.1
(0, Google I/O)
(1, Google I/O)
(2, Google I/O 2018)
(u'Google I/O', u'EVENT')
(u'this year', u'DATE')
(u'Google I/O 2018', u'EVENT')
Am I missing something here?
I believe your first code fails because you have not added an Entity label for 'EMAIL'. The second code works because EVENT is a pre-existing Entity type.
The documentation is not very clear on what the first argument of the matcher.add() method actually does, but it adds an Entity label for you. Here are two alternatives that should work and clear up the confusion:
Alternative 1:
import spacy
from spacy.matcher import Matcher
nlp = spacy.load('en_core_web_sm')
matcher = Matcher(nlp.vocab)
#EMAIL = nlp.vocab.strings['EMAIL'] #Not needed
def add_email_ent(matcher, doc, i, matches):
match_id, start, end = matches[i]
doc.ents += ((match_id, start, end),)
matcher.add('EMAIL', add_email_ent, [{'LIKE_EMAIL': True}])
text = u"Hi, this is John. My email is john#ymail.com and an alternate is john#gmail.com"
doc = nlp(text)
matches = matcher(doc)
for i,[match_id, start, end] in enumerate(matches):
print (i+1, doc[start:end])
for ent in doc.ents:
print (ent.text, ent.label_)
Alternative 2 (I'm not sure why you'd want to do it this way because you end up with two entity labels serving essentially the same purpose, but provided just for illustration purposes):
import spacy
from spacy.matcher import Matcher
from spacy.pipeline import EntityRecognizer
nlp = spacy.load('en_core_web_sm')
matcher = Matcher(nlp.vocab)
ner = EntityRecognizer(nlp.vocab)
ner.add_label('EMAIL')
EMAIL = nlp.vocab.strings['EMAIL']
def add_email_ent(matcher, doc, i, matches):
match_id, start, end = matches[i]
doc.ents += ((EMAIL, start, end),)
matcher.add('EmailPII', add_email_ent, [{'LIKE_EMAIL': True}])
text = u"Hi, this is John. My email is john#ymail.com and an alternate is john#gmail.com"
doc = nlp(text)
matches = matcher(doc)
for i,[match_id, start, end] in enumerate(matches):
print (i+1, doc[start:end])
for ent in doc.ents:
print (ent.text, ent.label_)