Interactive brokers native API. - problem with client. disconnect() - interactive-brokers

I am a noob programmer without a computer science background. I learned to code by trial and error and with online help.
I wrote the following code to download hourly candles from IB TWS.
There is a problem with client.disconnet(). This causes "Error Code: 504 & Error Message: Not connected".
If I remove this line -client.disconnect() from the code and run it again, the error message disappears but API connection stays active. API connection staying active is ok but it seems inefficient way of doing it.
How can it overcome this error message
Thanks
Here is the full code and error message
from ibapi.client import EClient, Contract
from ibapi.wrapper import EWrapper
from ibapi.utils import iswrapper
import datetime
from datetime import date
from datetime import time
from threading import Thread
import time
import socket
import pandas as pd
import talib as ta
import os
class ReadTicker(EWrapper, EClient):
''' Serves as the client and the wrapper '''
def __init__(self, addr, port, client_id):
EClient.__init__(self, self)
# Initialize properties
self.usdpair = {'AUD':'14433401', 'EUR':'12087792', 'GBP':'12087797'}
self.candle_dict = {}
# Connect to TWS
self.connect('127.0.0.1', 7497, 304)
# Launch the client thread
thread = Thread(target=self.run)
thread.start()
#iswrapper
def historicalData(self, req_id, bar):
''' Callback to reqHistoricalData '''
# Add the instrument prices to the dictionary
self.candle_dict['Date'].append(bar.date)
self.candle_dict['Open'].append(bar.open)
self.candle_dict['High'].append(bar.high)
self.candle_dict['Low'].append(bar.low)
self.candle_dict['Close'].append(bar.close)
def error(self, req_id, code, msg):
print('Error Code: {} & Error Message: {}'.format(code, msg))
def main():
start = datetime.time(9, 0, 0)
end = datetime.time(22, 0, 0)
current = datetime.datetime.now().time()
if start <= current <= end:
# Make directory for csv file n create path
if not os.path.exists(r'C:\TWSAPI\samples\Python\Testbed\xlsfiles'):
os.makedirs(r'C:\TWSAPI\samples\Python\Testbed\xlsfiles')
root = r'C:\TWSAPI\samples\Python\Testbed\xlsfiles'
#Create the client and connect to TWS
client = ReadTicker('127.0.0.1', 7497, 304)
time.sleep(10) #Sleep interval to allow time for connection to server
reqId = 2001
for usdpair in client.usdpair:
# Define the contract
usdcontract = Contract()
usdcontract.symbol = usdpair
usdcontract.secType = 'CASH'
usdcontract.exchange = 'IDEALPRO'
usdcontract.currency = "USD"
usdcontract.conId = client.usdpair[usdpair]
print('OHLC Data download for {}USD started.'.format(usdpair))
for v in ['Date','Open', 'High', 'Low', 'Close']:
client.candle_dict[v] = []
now = datetime.datetime.now().strftime("%Y%m%d %H:%M:%S")
client.reqHistoricalData(reqId, usdcontract, now, '7 D', '1 hour', 'TRADES', False, 1, False, [])
reqId += 1
time.sleep(5)
df = pd.DataFrame(client.candle_dict, columns=['Date', 'Open', 'High', 'Low', 'Close'])
df['Date'] = pd.to_datetime(df['Date'])
df.to_csv(root + '/'+ usdpair + 'USD' + '.csv', encoding='utf-8', index= False)
client.candle_dict.clear()
client.disconnect()
else:
print('Trading time is over')
if __name__ == '__main__':
main()
Error Message: is as follows:
Error Code: 2104 & Error Message: Market data farm connection is OK:uscrypto
Error Code: 2104 & Error Message: Market data farm connection is OK:usfarm.nj
Error Code: 2104 & Error Message: Market data farm connection is OK:hfarm
Error Code: 2104 & Error Message: Market data farm connection is OK:cashfarm
Error Code: 2106 & Error Message: HMDS data farm connection is OK:euhmds
Error Code: 2106 & Error Message: HMDS data farm connection is OK:cashhmds
Error Code: 2106 & Error Message: HMDS data farm connection is OK:hkhmds
Error Code: 2106 & Error Message: HMDS data farm connection is OK:fundfarm
Error Code: 2106 & Error Message: HMDS data farm connection is OK:ushmds
Error Code: 2158 & Error Message: Sec-def data farm connection is OK:secdefhk
OHLC Data download for AUDUSD started.
Error Code: 162 & Error Message: Historical Market Data Service error message:No historical market data for AUD/CASH#FXSUBPIP Last 3600
OHLC Data download for EURUSD started.
Error Code: 504 & Error Message: Not connected
unhandled exception in EReader thread
Traceback (most recent call last):
File "C:\Users\venky\anaconda3\lib\site-packages\ibapi-9.76.1-py3.8.egg\ibapi\reader.py", line 34, in run
data = self.conn.recvMsg()
File "C:\Users\venky\anaconda3\lib\site-packages\ibapi-9.76.1-py3.8.egg\ibapi\connection.py", line 99, in recvMsg
buf = self._recvAllMsg()
File "C:\Users\venky\anaconda3\lib\site-packages\ibapi-9.76.1-py3.8.egg\ibapi\connection.py", line 119, in _recvAllMsg
buf = self.socket.recv(4096)
OSError: [WinError 10038] An operation was attempted on something that is not a socket
OHLC Data download for GBPUSD started.
Error Code: 504 & Error Message: Not connected
PS C:\TWSAPI\samples\Python\Testbed>

Related

How to connect to KaaIOT MQTT broker

I want to connect my KaaIOT cloud and subscribe a topic to show result from terminal. I do not know where to get the topic name to subscribe with. I had read the KaaIOT documentation but still cannot have a clear idea on it. Could someone help me with a sample code for me to reference?
KaaIOT Information
appVersion.name: c184ijqrqa51q5haskp0-v1
appVersion.registeredDate: 2021-03-16T05:59:54.185Z
createdDate: 2021-03-16T05:59:54.186Z
endpointId: fc2c5833-77c5-445a-89a0-9b0e7498c048
model: Raspberry Pi (192.168.0.171)
metadataUpdatedDate: 2021-03-17T09:13:01.809Z
Sample Code
import itertools
import json
import queue
import random
import string
import sys
import time
import paho.mqtt.client as mqtt
KPC_HOST = "mqtt.cloud.kaaiot.com" # Kaa Cloud plain MQTT host
KPC_PORT = 1883 # Kaa Cloud plain MQTT port
APPLICATION_VERSION = "" # Paste your application version
ENDPOINT_TOKEN = "" # Paste your endpoint token
class MetadataClient:
def __init__(self, client):
self.client = client
self.metadata_by_request_id = {}
self.global_request_id = itertools.count()
get_metadata_subscribe_topic = f'kp1/{APPLICATION_VERSION}/epmx/{ENDPOINT_TOKEN}/get/#'
self.client.message_callback_add(get_metadata_subscribe_topic, self.handle_metadata)
def handle_metadata(self, client, userdata, message):
request_id = int(message.topic.split('/')[-2])
if message.topic.split('/')[-1] == 'status' and request_id in self.metadata_by_request_id:
print(f'<--- Received metadata response on topic {message.topic}')
metadata_queue = self.metadata_by_request_id[request_id]
metadata_queue.put_nowait(message.payload)
else:
print(f'<--- Received bad metadata response on topic {message.topic}:\n{str(message.payload.decode("utf-8"))}')
def get_metadata(self):
request_id = next(self.global_request_id)
get_metadata_publish_topic = f'kp1/{APPLICATION_VERSION}/epmx/{ENDPOINT_TOKEN}/get/{request_id}'
metadata_queue = queue.Queue()
self.metadata_by_request_id[request_id] = metadata_queue
print(f'---> Requesting metadata by topic {get_metadata_publish_topic}')
self.client.publish(topic=get_metadata_publish_topic, payload=json.dumps({}))
try:
metadata = metadata_queue.get(True, 5)
del self.metadata_by_request_id[request_id]
return str(metadata.decode("utf-8"))
except queue.Empty:
print('Timed out waiting for metadata response from server')
sys.exit()
def patch_metadata_unconfirmed(self, metadata):
partial_metadata_udpate_publish_topic = f'kp1/{APPLICATION_VERSION}/epmx/{ENDPOINT_TOKEN}/update/keys'
print(f'---> Reporting metadata on topic {partial_metadata_udpate_publish_topic}\nwith payload {metadata}')
self.client.publish(topic=partial_metadata_udpate_publish_topic, payload=metadata)
def main():
# Initiate server connection
print(f'Connecting to Kaa server at {KPC_HOST}:{KPC_PORT} using application version {APPLICATION_VERSION} and endpoint token {ENDPOINT_TOKEN}')
client_id = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6))
client = mqtt.Client(client_id=client_id)
client.connect(KPC_HOST, KPC_PORT, 60)
client.loop_start()
metadata_client = MetadataClient(client)
# Fetch current endpoint metadata attributes
retrieved_metadata = metadata_client.get_metadata()
print(f'Retrieved metadata from server: {retrieved_metadata}')
# Do a partial endpoint metadata update
metadata_to_report = json.dumps({"model": "BFG 9000", "mac": "00-14-22-01-23-45"})
metadata_client.patch_metadata_unconfirmed(metadata_to_report)
time.sleep(5)
client.disconnect()
if __name__ == '__main__':
main()
I have established a communication with EMQX broker by using MQTT protocol. But I don't know about KaaIoT much but this might help you. As I went through your code, I didn't see the part where you have subscribed to the topic(correct me if I am wrong). You can refer this. I have implemented the sub pub model and below is the subscriber code which runs fine for EMQX broker. You can try it for KaaIoT.
import paho.mqtt.client as mqtt
import time
import logging
def on_connect(client, userdata, flags, rc):
logging.info("Connected flags"+str(flags)+"result code " + str(rc)+ "client1_id")
client.connected_flag=True
def on_message(client, userdata, message):
print("Received message: " ,str(message.payload.decode("utf-8")))
def on_disconnect(client, userdata, rc):
if rc != 0:
print("Unexpected MQTT disconnection. Will auto-reconnect")
client = mqtt.Client('''Your client id string''')
client.connect("mqtt.cloud.kaaiot.com", 1883, 60)
client.subscribe('''Your topic name (mentioned where data is published)''',qos=1)
client.on_connect = on_connect
client.on_message=on_message
client.on_disconnect = on_disconnect
client.loop_forever()

ThingsBoard IoT Gateway doesn't update MQTT values

I try to receive simple text values from external MQTT broker topics with IoT Gateway.
For this purpose I simplify the existing script (extensions/mqtt/custom_mqtt_uplink_converter.py):
from thingsboard_gateway.connectors.mqtt.mqtt_uplink_converter import MqttUplinkConverter, log
class CustomMqttUplinkConverter(MqttUplinkConverter):
def __init__(self, config):
self.__config = config.get('converter')
self.dict_result = {}
def convert(self, topic, body):
try:
log.debug("New data received: %s: %s" % (topic,body))
# if topic = '/devices/buzzer/controls/volume' device name will be 'buzzer'.
self.dict_result["deviceName"] = topic.split("/")[2]
# just hardcode this
self.dict_result["deviceType"] = "buzzer"
self.dict_result["telemetry"] = {"data": body}
log.debug("Result: %s" % (self.dict_result))
return self.dict_result
except ...
When I start gateway I see in his log that he successfully connected and read the values:
INFO ... MQTT Broker Connector connected to 10.1.1.2:1883 - successfully.'
DEBUG ... Client <paho.mqtt.client.Client object at 0x7fb42d19dd68>, userdata None, flags {'session present': 0}, extra_params ()'
DEBUG ... <module 'CustomMqttUplinkConverter' from '/var/lib/thingsboard_gateway/extensions/mqtt/custom_mqtt_uplink_converter.py'>'
DEBUG ... Import CustomMqttUplinkConverter from /var/lib/thingsboard_gateway/extensions/mqtt.'
DEBUG ... Converter CustomMqttUplinkConverter for topic /devices/buzzer/controls/volume - found!'
INFO ... Connector "MQTT Broker Connector" subscribe to /devices/buzzer/controls/volume'
DEBUG ... Received data: {}'
DEBUG ... (None,)'
INFO ... "MQTT Broker Connector" subscription success to topic /devices/buzzer/controls/volume, subscription message id = 1'
DEBUG ... New data received: /devices/buzzer/controls/volume: 66'
DEBUG ... Result: {'deviceName': 'buzzer', 'deviceType': 'buzzer', 'telemetry': {'data': 66}}'
But this values are the last values he can read. If I change volume one broker new values will not appear neither in the log nor in TB UI. (I control updates with mosquitto_sub.)
Seems this converter will never called again until gateway restarted. Is it correct behaveour?
How can I make sure that my code is correct if I don't see the result?
Hi I have tried your version of the custom converter, it didn't work, but when I changed
self.dict_result["telemetry"] = {"data": body}
to
self.dict_result["telemetry"] = [{"data": body}]
It sent data correctly.
The gateway requires an array of telemetry of attributes from the converter.

Slow results from Microsoft Graph's list_attachments API and occasionally returns an error

The list_attachments API is slow to respond and sometimes returns an error message with neither a code nor a description.
To time the API's response, I triggered it 20 times and then recorded the total time and time per API call, both in seconds.
PFB the python script used to test the list attachments API.
# Initialize variables to track error and success
error_count = 0
success_count = 0
# Allow token scope to not match requested scope. (Other auth libraries allow
# this, but Requests-OAuthlib raises an exception on scope mismatch by default.)
os.environ['OAUTHLIB_RELAX_TOKEN_SCOPE'] = '1'
# Define the method to fetch attachments for a message
def fetch_attachment():
global error_count
global success_count
API_ROOT = 'https://graph.microsoft.com/'
API_VERSION = 'v1.0'
endpoint = API_ROOT + API_VERSION + RESOURCE
response = msgraph_session.get(endpoint).json()
if 'error' in response:
error_count += 1
LOG.info("Response from the attachments API is %r", response)
else:
success_count += 1
# Declare the statement that you want to time
STMT = '''
from __main__ import fetch_attachment
fetch_attachment()
'''
# Run the method and time it
if __name__ == '__main__':
time = timeit.timeit(STMT, number=20) # We will run the statement 20 times
LOG.info("Total time : %s", time)
LOG.info("Time per call: %s", float(time)/20)
LOG.info("Error Count %s", error_count)
LOG.info("Success Count %s", success_count)
OUTPUT
2018-08-02 11:36:53,510 - INFO - __main__ - Total time : 407.309007168
2018-08-02 11:36:53,510 - INFO - __main__ - Time per call: 20.3654503584
2018-08-02 11:36:53,511 - INFO - __main__ - Error Count 4
2018-08-02 11:36:53,511 - INFO - __main__ - Success Count 16
Out of 20 times, 4 returned the below error message
{
"error":{
"innerError":{
"date":"2018-08-02T18:32:27",
"request-id":"a52a676c-20a6-46c8-a71f-24ce35b166d7"
},
"message":"",
"code":"UnknownError"
}
}
abhi.
According to your information, I suppose you want to call the API /me/messages/{id}/attachments. Based on my test, when I add one attachment into the mail, calling this api is not slow.
But when add ten attachments, it's so slow. Because it will response each of the attachments content, so if each of the file is too large, it will be slow.
We can find the part of response like this:
{
"#odata.type": "#microsoft.graph.fileAttachment",
"id": "{attachmentID}",
"lastModifiedDateTime": "2018-08-14T07:18:21Z",
"name": "{attachment name}",
"contentType": "{attachment type}",
"contentBytes": ""
}
I think we can use the query parameters to customize responses. We can add the query parameter so that its response does not contain the file content. We can get each attachment id by the API like this:
https://graph.microsoft.com/beta/me/messages/{message id}/attachments?$select=id,name,size
Then we can get the content of attachment by the attachment id.
https://graph.microsoft.com/beta/me/messages/{message id}/attachments/{attachment id}

Is there a way to verify that an iOS Registration Token is valid or Invalid?

I already understand the proper procedure for obtaining a push registration token from APNS when the app launches so that that you always have a valid token. But for troubleshooting, is there an API for checking the validity of a registration token similar to how you can validate a push cert?
Here's the solution using the python script below.
Credit solution to Dara Kong
Send test APNS notifications to device using cert
usage:
push_debug.py [-h] [-s] certificate device_id
positional arguments:
certificate path to push certificate
device_id device ID
optional arguments:
-h, --help show this help message and exit
-s, --sandbox Use APNS sandbox environment
Send notification through production environment
python push_debug.py my_certificate.pem c35124fd2676d646423705b0721004e3b8426d163e10dbf76b46347a4477f12b
Send notification through sandbox environment
python push_debug.py -s my_certificate.pem 3f3980225497e1846fb5c2db9e0b3510023402c7a772011106c702d2aec20cc5
Save this script to file named push_debug.py
import json
import logging
import os
import socket
import ssl
import struct
import sys
import time
import uuid
import argparse
APNS_HOST = 'gateway.push.apple.com'
APNS_HOST_SANDBOX = 'gateway.sandbox.push.apple.com'
APNS_PORT = 2195
APNS_ERRORS = {
1:'Processing error',
2:'Missing device token',
3:'missing topic',
4:'missing payload',
5:'invalid token size',
6:'invalid topic size',
7:'invalid payload size',
8:'invalid token',
255:'Unknown'
}
def push(cert_path, device, sandbox):
if not os.path.exists(cert_path):
logging.error("Invalid certificate path: %s" % cert_path)
sys.exit(1)
device = device.decode('hex')
expiry = time.time() + 3600
try:
sock = ssl.wrap_socket(
socket.socket(socket.AF_INET, socket.SOCK_STREAM),
certfile=cert_path
)
host = APNS_HOST_SANDBOX if sandbox else APNS_HOST
sock.connect((host, APNS_PORT))
sock.settimeout(1)
except Exception as e:
logging.error("Failed to connect: %s" % e)
sys.exit(1)
logging.info("Connected to APNS\n")
for ident in range(1,4):
logging.info("Sending %d of 3 push notifications" % ident)
payload = json.dumps({
'aps': {
'alert': 'Push Test %d: %s' % (ident, str(uuid.uuid4())[:8])
}
})
items = [1, ident, expiry, 32, device, len(payload), payload]
try:
sent = sock.write(struct.pack('!BIIH32sH%ds'%len(payload), *items))
if sent:
logging.info("Message sent\n")
else:
logging.error("Unable to send message\n")
except socket.error as e:
logging.error("Socket write error: %s", e)
# If there was an error sending, we will get a response on socket
try:
response = sock.read(6)
command, status, failed_ident = struct.unpack('!BBI',response[:6])
logging.info("APNS Error: %s\n", APNS_ERRORS.get(status))
sys.exit(1)
except socket.timeout:
pass
except ssl.SSLError:
pass
sock.close()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Send test APNS notifications to device using cert")
parser.add_argument("certificate", help="path to push certificate")
parser.add_argument("device_id", help="Device ID")
parser.add_argument("-s", "--sandbox", action="store_true", help="Use APNS sandbox environment")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
push(args.certificate, args.device_id, args.sandbox)
logging.info("Complete\n")

How to check whether IMAP-IDLE works?

I noticed that my IMAP server seems to support IDLE but notifications arrive late. So I was asking myself: How can I check whether IDLE works (or is it my mail client)?
Inspired by http://pymotw.com/2/imaplib/, you can use following Python scripts to check if and how fast push notification via IDLE work:
imaplib_connect.py
import imaplib
import ConfigParser
import os
def open_connection(verbose=False):
# Read the config file
config = ConfigParser.ConfigParser()
config.read([os.path.abspath('settings.ini')])
# Connect to the server
hostname = config.get('server', 'hostname')
if verbose: print 'Connecting to', hostname
connection = imaplib.IMAP4_SSL(hostname)
# Login to our account
username = config.get('account', 'username')
password = config.get('account', 'password')
if verbose: print 'Logging in as', username
connection.login(username, password)
return connection
if __name__ == '__main__':
c = open_connection(verbose=True)
try:
print c
finally:
c.logout()
print "logged out"
imaplib_idlewait.py
import imaplib
import pprint
import imaplib_connect
imaplib.Debug = 4
c = imaplib_connect.open_connection()
try:
c.select('INBOX', readonly=True)
c.send("%s IDLE\r\n"%(c._new_tag()))
print ">>> waiting for new mail..."
while True:
line = c.readline().strip();
if line.startswith('* BYE ') or (len(line) == 0):
print ">>> leaving..."
break
if line.endswith('EXISTS'):
print ">>> NEW MAIL ARRIVED!"
finally:
try:
print ">>> closing..."
c.close()
except:
pass
c.logout()
settings.ini
[server]
hostname: yourserver.com
[account]
username: yourmail#yourserver.com
password: yoursecretpassword
After creating those files, just call
python imaplib_idlewait.py
Please note, that this scripts does not close gracefully if you press CTRL+C (readline() is blocking and is not terminated by close()), however, for testing it should be good enough.
Also note, that most mail server terminate the connection after 30 minutes. After that you have to re-open the connection, e.g. like demonstrated here: http://blog.mister-muffin.de/2013/06/05/reliable-imap-synchronization-with-idle-support

Resources