I'm using pyflink and Flink 11.2 and I've defined my table like this:
def _create_sink_table(st_env):
# Create SINK table.
st_env.execute_sql(f"""
CREATE TABLE {"in"} (
`a` STRING,
`b` STRING,
`c` STRING,
`d` ROW(
`e` STRING,
`f` STRING
)
) WITH (
'connector' = 'kafka',
'topic' = '{SINK_TOPIC_NAME}',
'properties.bootstrap.servers' = '{BOOTSTRAP_SERVERS}',
'format' = 'json',
'json.fail-on-missing-field' = 'false',
'json.ignore-parse-errors' = 'true'
)
""")
I want to use table.where to filter for subfield:
def execute(st_env: StreamTableEnvironment):
table = st_env.from_path("in")
table = table.select("a, b, c, d")
table = table.where("d.e = 'fail'")
Everything I've tried so far resulted in an error, this is one of them:
➜ ./run.sh
Traceback (most recent call last):
File "/tmp/test.py", line 143, in <module>
main()
File "/tmp/test.py", line 139, in main
execute(st_env)
File "/tmp/test.py", line 109, in execute
table = table.where("d.e = 'fail'")
File "/opt/flink/opt/python/pyflink.zip/pyflink/table/table.py", line 140, in where
File "/opt/flink/opt/python/py4j-0.10.8.1-src.zip/py4j/java_gateway.py", line 1286, in __call__
File "/opt/flink/opt/python/pyflink.zip/pyflink/util/exceptions.py", line 147, in deco
File "/opt/flink/opt/python/py4j-0.10.8.1-src.zip/py4j/protocol.py", line 328, in get_return_value
py4j.protocol.Py4JJavaError: An error occurred while calling o39.where.
: org.apache.flink.table.api.ValidationException: Undefined function: e
at org.apache.flink.table.expressions.resolver.LookupCallResolver.lambda$visit$0(LookupCallResolver.java:49)
at java.util.Optional.orElseThrow(Optional.java:290)
at org.apache.flink.table.expressions.resolver.LookupCallResolver.visit(LookupCallResolver.java:49)
at org.apache.flink.table.expressions.resolver.LookupCallResolver.visit(LookupCallResolver.java:38)
at org.apache.flink.table.expressions.ApiExpressionVisitor.visit(ApiExpressionVisitor.java:37)
at org.apache.flink.table.expressions.LookupCallExpression.accept(LookupCallExpression.java:65)
at org.apache.flink.table.expressions.resolver.LookupCallResolver.lambda$resolveChildren$1(LookupCallResolver.java:64)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
It tries to execute the function e on d, instead of retrieving the subfield.
I think table.where("d.get(0) = 'fail'") maybe work
Related
Hey guys I am trying to test the commit of napalm, but it is not able to finding the cfg file, I also tried to change to "txt", but the same error.
Here my code:
import napalm
import json
driver = napalm.get_network_driver("ios")
device = driver(
hostname="10.0.0.254",
username="cisco",
password="cisco",
optional_args={"secret" : "cisco"}
)
device.open()
device.load_merge_candidate(filename="config.cfg")
device.commit_config()
device.close()
and this is the error,both files are in the same folder:
user#user-pc:~/Documents/python_files$ /usr/bin/python3
/home/user/Documents/python_files/network/config_compare.py Traceback
(most recent call last): File
"/home/user/Documents/python_files/network/config_compare.py", line
18, in
device.load_merge_candidate(filename="config.cfg") File "/home/user/.local/lib/python3.8/site-packages/napalm/ios/ios.py",
line 315, in load_merge_candidate
return_status, msg = self._load_candidate_wrapper( File "/home/user/.local/lib/python3.8/site-packages/napalm/ios/ios.py",
line 282, in _load_candidate_wrapper
(return_status, msg) = self._scp_file( File "/home/user/.local/lib/python3.8/site-packages/napalm/ios/ios.py",
line 620, in _scp_file
return self._xfer_file( File "/home/user/.local/lib/python3.8/site-packages/napalm/ios/ios.py",
line 670, in _xfer_file
with TransferClass(**kwargs) as transfer: File "/home/user/.local/lib/python3.8/site-packages/netmiko/ssh_dispatcher.py",
line 278, in FileTransfer
return FileTransferClass(*args, **kwargs) File "/home/user/.local/lib/python3.8/site-packages/netmiko/scp_handler.py",
line 80, in init
self.source_md5 = self.file_md5(source_file) File "/home/user/.local/lib/python3.8/site-packages/netmiko/scp_handler.py",
line 257, in file_md5
with open(file_name, "rb") as f: FileNotFoundError: [Errno 2] No such file or directory: 'config.c
tks for any help..
It looks like a problem with a filepath, are you sure that 'config.cfg' is in the same dir as your code?
Try the below to see if you can access the file (if you can't you need to modify the filepath):
import napalm
driver = napalm.get_network_driver("ios")
device = driver(
hostname="10.0.0.254",
username="cisco",
password="cisco",
optional_args={"secret" : "cisco"}
)
device.open()
print('This is a config to be added:\n')
with open('config.cfg','r') as f:
output = f.read()
device.load_merge_candidate(filename="config.cfg")
# I would add this to see what's you are merging:
print(device.compare_config())
confirm_config= input('Do you want to deploy the above config? Press Y to deploy \n')
if confirm_config == 'Y':
device.commit_config()
device.close()
else:
device.close()
1 issue at large
I am producing a iterative process via tff.learning.build_federated_averaging_process(). and receive the error:
Traceback (most recent call last):
File "B:\tools and software\Anaconda\envs\bookProjects\lib\site-packages\IPython\core\interactiveshell.py", line 3331, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-2-47998fd56829>", line 1, in <module>
runfile('B:/projects/openProjects/githubprojects/BotnetTrafficAnalysisFederaedLearning/anomaly-detection/train_v04.py', args=['--experiment_name=temp', '--client_batch_size=20', '--client_optimizer=sgd', '--client_learning_rate=0.2', '--server_optimizer=sgd', '--server_learning_rate=1.0', '--total_rounds=200', '--rounds_per_eval=1', '--rounds_per_checkpoint=50', '--rounds_per_profile=0', '--root_output_dir=B:/projects/openProjects/githubprojects/BotnetTrafficAnalysisFederaedLearning/anomaly-detection/logs/fed_out/'], wdir='B:/projects/openProjects/githubprojects/BotnetTrafficAnalysisFederaedLearning/anomaly-detection')
File "B:\tools and software\PyCharm 2020.1\plugins\python\helpers\pydev\_pydev_bundle\pydev_umd.py", line 197, in runfile
pydev_imports.execfile(filename, global_vars, local_vars) # execute the script
File "B:\tools and software\PyCharm 2020.1\plugins\python\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "B:/projects/openProjects/githubprojects/BotnetTrafficAnalysisFederaedLearning/anomaly-detection/train_v04.py", line 306, in <module>
app.run(main)
File "B:\tools and software\Anaconda\envs\bookProjects\lib\site-packages\absl\app.py", line 299, in run
_run_main(main, args)
File "B:\tools and software\Anaconda\envs\bookProjects\lib\site-packages\absl\app.py", line 250, in _run_main
sys.exit(main(argv))
File "B:/projects/openProjects/githubprojects/BotnetTrafficAnalysisFederaedLearning/anomaly-detection/train_v04.py", line 299, in main
train_main()
File "B:/projects/openProjects/githubprojects/BotnetTrafficAnalysisFederaedLearning/anomaly-detection/train_v04.py", line 262, in train_main
server_optimizer_fn=server_optimizer_fn,
File "B:\tools and software\Anaconda\envs\bookProjects\lib\site-packages\tensorflow_federated\python\learning\federated_averaging.py", line 211, in build_federated_averaging_process
stateful_delta_aggregate_fn, stateful_model_broadcast_fn)
File "B:\tools and software\Anaconda\envs\bookProjects\lib\site-packages\tensorflow_federated\python\learning\framework\optimizer_utils.py", line 498, in build_model_delta_optimizer_process
py_typecheck.check_callable(model_fn)
File "B:\tools and software\Anaconda\envs\bookProjects\lib\site-packages\tensorflow_federated\python\common_libs\py_typecheck.py", line 106, in check_callable
type_string(type(target))))
TypeError: Expected a callable, found non-callable tensorflow_federated.python.learning.model_utils.EnhancedModel.
highlighting:
in build_federated_averaging_process
stateful_delta_aggregate_fn, stateful_model_broadcast_fn)
and
TypeError: Expected a callable, found non-callable tensorflow_federated.python.learning.model_utils.EnhancedModel.
2 have tried
looked at another similar issue here have tried to make
model_fn a collection.abc Callable, model_fn=Callable[[], model_fn]
only creates a new error.
3 some code:
iterative process:
model_fn = model_builder(input_dim=sysarg,
input_spec=input_spec)
iterative_process = tff.learning.build_federated_averaging_process(
model_fn=model_fn,
client_optimizer_fn=client_optimizer_fn,
server_optimizer_fn=server_optimizer_fn,
)
iterative_process = compression_process_adapter.CompressionProcessAdapter(iterative_process)```
model bulder:
def model_builder(input_dim, input_spec):
model = create_model(input_dim)
return tff.learning.from_keras_model(keras_model=model,
loss=tf.keras.losses.MeanSquaredError(),
input_spec=input_spec,
metrics=[tf.keras.metrics.Accuracy()],
)
create model (for good measure)
def create_model(input_dim):
autoencoder = Sequential([
tf.keras.layers.Dense(int(0.75 * input_dim), activation="tanh", input_shape=(input_dim,)),
tf.keras.layers.Dense(int(0.5 * input_dim), activation="tanh"),
tf.keras.layers.Dense(int(0.33 * input_dim), activation="tanh"),
tf.keras.layers.Dense(int(0.25 * input_dim), activation="tanh"),
tf.keras.layers.Dense(int(0.33 * input_dim), activation="tanh"),
tf.keras.layers.Dense(int(0.5 * input_dim), activation="tanh"),
tf.keras.layers.Dense(int(0.75 * input_dim), activation="tanh"),
tf.keras.layers.Dense(input_dim)
])
The model_fn argument of tff.learning.build_federated_averaging_process needs to be a callable (What is a callable?) that takes no arguments and returns a tff.learning.Model.
From the code (reproduced here for readability):
def model_builder(input_dim, input_spec):
model = create_model(input_dim)
return tff.learning.from_keras_model(
keras_model=model,
loss=tf.keras.losses.MeanSquaredError(),
input_spec=input_spec,
metrics=[tf.keras.metrics.Accuracy()])
model_fn = model_builder(input_dim=sysarg, input_spec=input_spec)
iterative_process = tff.learning.build_federated_averaging_process(
model_fn=model_fn,
client_optimizer_fn=client_optimizer_fn,
server_optimizer_fn=server_optimizer_fn)
model_fn is actually an instance of tff.learning.Model, rather than a callable that constructs and returns a model.
model_builder should be passed to tff.learning.build_federated_averaging_process, but since it takes arguments this won't work as-is.
One alternative would be to use functools.partial:
iterative_process = tff.learning.build_federated_averaging_process(
model_fn=functools.partial(model_builder, input_dim=sysarg, input_spec=input_spec),
client_optimizer_fn=client_optimizer_fn,
server_optimizer_fn=server_optimizer_fn)
Or even use a no-arg lambda:
iterative_process = tff.learning.build_federated_averaging_process(
model_fn=lambda: model_builder(input_dim=sysarg, input_spec=input_spec),
client_optimizer_fn=client_optimizer_fn,
server_optimizer_fn=server_optimizer_fn)
The following question has a good discussion on the differences between lambdas and partials: Python: Why is functools.partial necessary?
I'm trying to write my results in two separate hdf5 files. My code works for one of them but produces an error for the other one, even though the code for both is identical. Here is my code:
def test(model, outfile_f3,outfile_f5):
dtype = torch.FloatTensor
path = args.imagefolder
filenames = sorted(glob.glob(os.path.join(path, '*.png')))
N = len(filenames)
f3_activations = []
f5_activations = []
for i in range(N):
fname = filenames[i]
print(fname)
v = Image.open(fname)
image = load_image(v, image_sizes['eig'])
image = torch.from_numpy(image).type(dtype).cuda()
image = image.unsqueeze(0)
att, f3, f5, latent = model(image, segment=args.segment, add_offset=args.addoffset and args.segment, test=True)
#latents.append(latent.detach()[0].cpu().numpy().flatten())
#attended.append(att.detach()[0].cpu().numpy().flatten())
f3_activations.append(f3.detach()[0].cpu().numpy().flatten())
f5_activations.append(f5.detach()[0].cpu().numpy().flatten())
file_f3 = h5py.File(outfile_f3, 'w')
file_f3.create_dataset('f3_activations', data=np.array(f3_activations))
file_f3.close()
file_f5 = h5py.File(outfile_f5, 'w')
file_f5.create_dataset('f5_activations', data=np.array(f5_activations))
file_f5.close()
#asciiList = [n.split('/')[-1][:-4].encode("ascii", "ignore") for n in filenames]
#f.create_dataset('filenames', (len(asciiList), 1), 'S10', data=asciiList)
outfile_f3 and outfile_f5 have previously been defined as:
outfile_f3 = os.path.join('./output', 'f3_activations.hdf5')
outfile_f5 = os.path.join('.output','f5_activations.hdf5')
When I try to run the script that uses test function I get the following error:
File "record_f3_and_f5_actvations.py", line 109, in test
file_f5 = h5py.File(outfile_f5, 'w')
File "/gpfs/milgram/project/yildirim/ms3542/conda_envs/env/lib/python3.6/site-packages/h5py/_hl/files.py", line 408, in init
swmr=swmr)
File "/gpfs/milgram/project/yildirim/ms3542/conda_envs/env/lib/python3.6/site-packages/h5py/_hl/files.py", line 179, in make_fid
fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl)
File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
File "h5py/h5f.pyx", line 108, in h5py.h5f.create
OSError: Unable to create file (unable to open file: name = '.output/f5_activations.hdf5', errno = 2, error message = 'No such file or directory', flags = 13, o_flags = 242)
It has no problem with creating, opening and writing in the file f3_activations.hdf5, but for some reason, it can't do the same with f5_activations.hdf5
transform=transforms.Compose([
transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
root = '/home/meet/CSRNet-pytorch/ShanghaiTech/'
part_A_train = os.path.join(root,'part_A/train_data','images')
part_A_test = os.path.join(root,'part_A/test_data','images')
part_B_train = os.path.join(root,'part_B/train_data','images')
part_B_test = os.path.join(root,'part_B/test_data','images')
path_sets = [part_A_test]
#defining the image path
img_paths = []
for path in path_sets:
for img_path in glob.glob(os.path.join(path, '*.jpg')):
img_paths.append(img_path)
model = CSRNet()
model = model.cuda()
On running model = CSRNet() I get an "Typeerror: super(type, obj): obj must be an instance or subtype of type"
I am referring to a tutorial to get the number of people count from crowd images.
This is the tutorial I am referring to:
https://www.analyticsvidhya.com/blog/2019/02/building-crowd-counting-model-python/
This is the kernal output window : -
model = CSRNet()
Downloading: "https://download.pytorch.org/models/vgg16-397923af.pth" to /home/meet/.torch/models/vgg16-397923af.pth
Traceback (most recent call last):
File "", line 1, in
model = CSRNet()
File "/home/meet/CSRNet-pytorch/model.py", line 16, in init
mod = models.vgg16(pretrained = True)
File "/home/meet/anaconda3/lib/python3.7/site-packages/torchvision/models/vgg.py", line 152, in vgg16
model.load_state_dict(model_zoo.load_url(model_urls['vgg16']))
File "/home/meet/anaconda3/lib/python3.7/site-packages/torch/utils/model_zoo.py", line 66, in load_url
_download_url_to_file(url, cached_file, hash_prefix, progress=progress)
File "/home/meet/anaconda3/lib/python3.7/site-packages/torch/utils/model_zoo.py", line 73, in _download_url_to_file
u = urlopen(url, stream=True)
File "/home/meet/anaconda3/lib/python3.7/site-packages/requests/api.py", line 70, in get
return request('get', url, params=params, **kwargs)
File "/home/meet/anaconda3/lib/python3.7/site-packages/requests/api.py", line 55, in request
with sessions.Session() as session:
File "/home/meet/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 346, in init
self.mount('https://', HTTPAdapter())
File "/home/meet/anaconda3/lib/python3.7/site-packages/requests/adapters.py", line 117, in init
super(HTTPAdapter, self).init()
I am creating all the attributes properly and I know that my request is complete because it is giving a TypeError when I add any extraneous fields that my web service does not accept. Now when I call the Method like client.service.Method(attributes...) I get a WebFault but there is no description for why the WebFault is occurring. Has anyone faced this problem before or does anyone know what this could mean?
The traceback is:
File "C:/Python25/sudsfirst.py", line 34, in <module>
result = client.service.Method(arg1,arg2)
File "build\bdist.win32\egg\suds\client.py", line 542 in __call__
return client.invoke(args, kwargs)
File "build\bdist.win32\egg\suds\client.py", line 602, in invoke
result = self.send(soapenv)
File "build\bdist.win32\egg\suds\client.py", line 649, in send
result = self.failed(binding, e)
File "build\bdist.win32\egg\suds\bindings\binding.py", line 265, in get_fault
raise WebFault(p, faultroot)
WebFault
The code is:
url = 'http://blah?wsdl'
client = Client(url)
arg1 = client.factory.create('type1')
arg1.type1 = ...
arg2 = client.factory.create('type2')
arg2.type1 = ...
client.service.Method(arg1, arg2)
Sorry, I can't mention more than this about the code.