How to convert a pygame image to Open CV image? - opencv

Question

I am currently getting a real time RGB video from a Kinect2 camera using Pygame and pykinect2. I want to convert it into an open cv image so that it would be helpful for me in my further Computations.
import pykinect2
import pygame
import cv2
import ctypes
from pykinect2 import PyKinectV2
from pykinect2 import PyKinectRuntime
kinectcam = PyKinectRuntime.PyKinectRuntime(PyKinectV2.FrameSourceTypes_Color)
def draw_color_frame(frame, target_surface):
target_surface.lock()
address = kinectcam.surface_as_array(target_surface.get_buffer())
ctypes.memmove(address, frame.ctypes.data, frame.size)
del address
target_surface.unlock()
pygame.init()
frame_surface = pygame.Surface((kinectcam.color_frame_desc.Width, kinectcam.color_frame_desc.Height), 0, 32)
clock = pygame.time.Clock()
pygame.display.set_caption("Kinect View")
infoObject = pygame.display.Info()
screen = pygame.display.set_mode((infoObject.current_w >> 1, infoObject.current_h >> 1),
pygame.HWSURFACE|pygame.DOUBLEBUF|pygame.RESIZABLE, 32)
clock = pygame.time.Clock()
done = False
while not done:
for event in pygame.event.get(): # User did something
if event.type == pygame.QUIT: # If user clicked close
done = True # Flag that we are done so we exit this loop
elif event.type == pygame.VIDEORESIZE: # window resized
screen = pygame.display.set_mode(event.dict['size'],
pygame.HWSURFACE|pygame.DOUBLEBUF|pygame.RESIZABLE, 32)
if kinectcam.has_new_color_frame():
frame = kinectcam.get_last_color_frame()
draw_color_frame(frame, frame_surface)
frame = None
h_to_w = float(frame_surface.get_height()) / frame_surface.get_width()
target_height = int(h_to_w * screen.get_width())
surface_to_draw = pygame.transform.scale(frame_surface, (screen.get_width(), target_height));
screen.blit(surface_to_draw, (0,0))
surface_to_draw = None
pygame.display.update()
pygame.display.flip()
clock.tick(60)
pygame.quit()
kinectcam.close()

Answer 1

I assume your are trying to convert the image you are blitting (surface_to_draw). To convert pygame.Surface object to opencv image:
# create a copy of the surface
view = pygame.surfarray.array3d(surface_to_draw)
# convert from (width, height, channel) to (height, width, channel)
view = view.transpose([1, 0, 2])
# convert from rgb to bgr
img_bgr = cv2.cvtColor(view, cv2.COLOR_RGB2BGR)
Update: I also assumed your pygame image is color image.

Answer 2

Using pyKinect2 library, you can create an acquisitionClass.py which defines different methods and properties required for Kinect frame processing. Then call the acquisitionClass.py which contains the get_color_frame() that does the conversion, from your main script (Run.py) to use and display the converted color frame to opencv frame as follows:
Note: I'm answering this assuming you do not want to use the pyGame library but OpenCV instead.
Run.py
import cv2
from pykinect2 import PyKinectV2
from pykinect2.PyKinectV2 import *
from pykinect2 import PyKinectRuntime
from acquisitionKinect import AcquisitionKinect
from frame import Frame
if __name__ == '__main__':
kinect = AcquisitionKinect()
frame = Frame()
while True:
kinect.get_frame(frame)
kinect.get_color_frame()
image = kinect._frameRGB
#OpenCv uses RGB image, kinect returns type RGBA, remove extra dim.
image = cv2.cvtColor(image, cv2.COLOR_RGBA2RGB)
if not image is None:
cv2.imshow("Output-Keypoints",image)
cv2.waitKey(30)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
acquisitionKinect.py
import ctypes
import _ctypes
import sys
if sys.hexversion >= 0x03000000:
import _thread as thread
else:
import thread
class AcquisitionKinect():
#Create a constructor to initialize different types of array and frame objects
def __init__(self, resolution_mode=1.0):
self.resolution_mode = resolution_mode
self._done = False
# Kinect runtime object, we want only color and body frames
self._kinect = PyKinectRuntime.PyKinectRuntime(PyKinectV2.FrameSourceTypes_Color | PyKinectV2.FrameSourceTypes_Body | PyKinectV2.FrameSourceTypes_Depth)
# here we will store skeleton data
self._bodies = None
self.body_tracked = False
self.joint_points = np.array([])
self.joint_points3D = np.array([])
self.joint_points_RGB = np.array([])
self.joint_state = np.array([])
self._frameRGB = None
self._frameDepth = None
self._frameDepthQuantized = None
self._frameSkeleton = None
self.frameNum = 0
def get_frame(self, frame):
self.acquireFrame()
frame.ts = int(round(time.time() * 1000))
self.frameNum += 1
frame.frameRGB = self._frameRGB
frame.frameDepth = self._frameDepth
frame.frameDepthQuantized = self._frameDepthQuantized
frame.frameSkeleton = self._frameSkeleton
#Get a color frame object
def get_color_frame(self):
self._frameRGB = self._kinect.get_last_color_frame()
self._frameRGB = self._frameRGB.reshape((1080, 1920,-1)).astype(np.uint8)
self._frameRGB = cv2.resize(self._frameRGB, (0,0), fx=1/self.resolution_mode, fy=1/self.resolution_mode)
#Acquire the type of frame required
def acquireFrame(self):
if self._kinect.has_new_color_frame():
self.get_color_frame()
def close(self):
self._kinect.close()
self._frameDepth = None
self._frameRGB = None
self._frameSkeleton = None
Frame.py
class Frame():
frameRGB = None
frameDepth = None
frameDepthQuantized = None
frameSkeleton = None
frame_num = 0
shoulder_orientation_euler = None
shoulder_orientation_quat = None