Interfacing an I2S microphone with BeagleBone Black - beagleboneblack

I am trying to interface an I2S microphone (https://www.adafruit.com/product/3421) with BeagleBone Black. I followed the article (http://www.ti.com/lit/an/sprac97/sprac97.pdf), and able to update the device tree, and the Linux kernel as suggested in the article.
The I2S component (for microphone) of the device tree is included as a dtsi in the main device tree source. The content of the dtsi is as below
&am33xx_pinmux {
mcasp1_pins: mcasp1_pins {
pinctrl-single,pins = <
/* sink must enable receivers */
0x1a0 0x23
/* P9_42 mcasp1_aclkx - bit clock */
0x1a4 0x23
/* P9_27 mcasp1_fsx - frame sync */
0x1a8 0x23
/* P9_41 mcasp1_axr0 - i2s input */
>;
};
};
&mcasp1 {
#sound-dai-cells = <0>;
pinctrl-names = "default";
pinctrl-0 = <&mcasp1_pins>;
status = "okay";
op-mode = <0>;
tdm-slots = <2>;
num-serializer = <4>;
serial-dir = < /* 1 TX 2 RX 0 unused */
2 0 0 0
>;
rx-num-evt = <1>;
tx-num-evt = <1>;
};
/ {
pcm5102a: pcm5102a {
#sound-dai-cells = <0>;
compatible = "ti,pcm5102a";
status = "okay";
};
sound1: sound#1 {
compatible = "simple-audio-card";
simple-audio-card,name = "PCM5102a";
simple-audio-card,format = "i2s";
simple-audio-card,bitclock-master = <&sound1_master>;
simple-audio-card,frame-master = <&sound1_master>;
simple-audio-card,bitclock-inversion;
simple-audio-card,cpu {
sound-dai = <&mcasp1>;
};
sound1_master: simple-audio-card,codec {
#sound-dai-cells = <0>;
sound-dai = <&pcm5102a>;
clocks = <&mcasp1_fck>;
clock-names = "mclk";
};
};
};
The final device tree (decompiled from am335x-boneblack.dtb) is attached here. McASP entries (mcasp1_pins), including pins as specified in the TI document above are on line no 1077.
I have also compiled the kernel with a new driver pcm5102 as suggested in the document. Finally, I see the driver listed in the output of the arecord command.
root#arm:/sys/class/gpio# arecord -l
**** List of CAPTURE Hardware Devices ****
card 0: PCM5102a [PCM5102a], device 0: davinci-mcasp.0-pcm5102a-hifi pcm5102a-hifi-0 []
Subdevices: 1/1
Subdevice #0: subdevice #0
However whenever I try to record audio, I am not getting any audio data. The audio file is formed, but the file size is always 44 bytes irrespective of how long I try to record audio for. Clearly no data is there in the file.
Recording using arecord command gives error as below
arecord -d 10 -Dhw:0,0 -f dat audio.wav
Recording WAVE 'audio.wav' : Signed 16 bit Little Endian, Rate 48000 Hz, Stereo
arecord: pcm_read:2032: read error: Input/output error
Moreover, whenever I try to connect the clock (MCASP1_ACLKR (Bit Clock) - P9_42) with my oscilloscope, I don't see any pulses. Looks like there is no proper clock signal on this pin. What can I do to debug, and fix this?
Any ideas?
Parag

The PCM1864 board used in the link you provided generates its own clock and operates in master mode. The microphone you are using, however, requires the bus master to generate both the bit clock and frame sync signals to operate.
The McASP module of the am33xx processor can generate these signals, although some modifications to the driver/device tree might be necessary.
The mcasp signals can be mapped to the following pins on the BeagleBone board (generated by TI's pinmux tool). Note, the device used here is mcasp0, not mcasp1. I assume TI was using a different version of the beagle bone which had mcasp1 connected to the IO header.
pinctrl-single,pins = <
AM33XX_IOPAD(0x9ac, PIN_INPUT_PULLDOWN | MUX_MODE0) /* (A14) mcasp0_ahclkx.mcasp0_ahclkx */
AM33XX_IOPAD(0x99c, PIN_INPUT_PULLDOWN | MUX_MODE0) /* (C12) mcasp0_ahclkr.mcasp0_ahclkr */
AM33XX_IOPAD(0x990, PIN_INPUT_PULLDOWN | MUX_MODE0) /* (A13) mcasp0_aclkx.mcasp0_aclkx */
AM33XX_IOPAD(0x994, PIN_INPUT_PULLDOWN | MUX_MODE0) /* (B13) mcasp0_fsx.mcasp0_fsx */
AM33XX_IOPAD(0x9a0, PIN_INPUT_PULLDOWN | MUX_MODE0) /* (B12) mcasp0_aclkr.mcasp0_aclkr */
AM33XX_IOPAD(0x9a4, PIN_INPUT_PULLDOWN | MUX_MODE0) /* (C13) mcasp0_fsr.mcasp0_fsr */
AM33XX_IOPAD(0x998, PIN_INPUT_PULLDOWN | MUX_MODE0) /* (D12) mcasp0_axr0.mcasp0_axr0 */
AM33XX_IOPAD(0x9a8, PIN_INPUT_PULLDOWN | MUX_MODE0) /* (D13) mcasp0_axr1.mcasp0_axr1 */
>;
According to the datasheet of the processor, the clock signals of the receiver (ahclkr, aclkr, fsr) can be setup to run independently of of in sync with the clock signals of the transmitter and dividers can be specified as necessary. The pins which are used to provide the clock signals will need to be set to PIN_OUTPUT.
It seems to me that the overlay provided by beagle board here, when used in conjunction with the default clock device tree insert here, derives the clocks from the system clock. You might want to experiment with this.

Related

I have a problem reading ADC with DMA on STM32 F767zi. When I look into the buffer, all I see are zeros, and I do not know why?

The size of my buffer is 4096 and I want to fill the buffer with the adc values that are read.
#define ADC_BUF_LEN 4096
uint16_t adc_buf[ADC_BUF_LEN];
I am using this function to start the reading process of the ADC using DMA
HAL_ADC_Start_DMA(&hadc2, (uint32_t*)adc_buf, ADC_BUF_LEN);
Here are my configurations for the ADC
hadc2.Instance = ADC2;
hadc2.Init.ClockPrescaler = ADC_CLOCK_SYNC_PCLK_DIV6;
hadc2.Init.Resolution = ADC_RESOLUTION_12B;
hadc2.Init.ScanConvMode = ADC_SCAN_DISABLE;
hadc2.Init.ContinuousConvMode = ENABLE;
hadc2.Init.DiscontinuousConvMode = DISABLE;
hadc2.Init.ExternalTrigConvEdge = ADC_EXTERNALTRIGCONVEDGE_NONE;
hadc2.Init.ExternalTrigConv = ADC_SOFTWARE_START;
hadc2.Init.DataAlign = ADC_DATAALIGN_RIGHT;
hadc2.Init.NbrOfConversion = 1;
hadc2.Init.DMAContinuousRequests = ENABLE;
hadc2.Init.EOCSelection = ADC_EOC_SINGLE_CONV;
if (HAL_ADC_Init(&hadc2) != HAL_OK)
{
Error_Handler();
}
This my DMA configuration
/**
* Enable DMA controller clock
*/
static void MX_DMA_Init(void)
{
/* DMA controller clock enable */
__HAL_RCC_DMA2_CLK_ENABLE();
/* DMA interrupt init */
/* DMA2_Stream2_IRQn interrupt configuration */
HAL_NVIC_SetPriority(DMA2_Stream2_IRQn, 0, 0);
HAL_NVIC_EnableIRQ(DMA2_Stream2_IRQn);
}
These are my interrupt functions. I set a toggle breakpoint on the first interrupt and when I debug, the LED does not turn on which suggests that the DMA isn't working?
/* USER CODE BEGIN 4 */
//Called when first half of buffer is filled
void HAL_ADC_ConvHalfCpltCallback(ADC_HandleTypeDef* hadc) {
//HAL_ADC_GetValue(&hadc2);
HAL_GPIO_WritePin(LD3_GPIO_Port, LD3_Pin, GPIO_PIN_SET);
}
//Called when buffer is completely filled
void HAL_ADC_ConvCpltCallback(ADC_HandleTypeDef* hadc){
HAL_GPIO_WritePin(LD3_GPIO_Port, LD3_Pin, GPIO_PIN_RESET);
}
I set the prescaled to be divided by 6 if this information helps.
Please give me some advice!
Thank you in advance.

why use cpu_to_le32 for the DMA address but not for length? (in an example code in a book)

I was reading the book Essential Linux Device Driver (by Sreekrishnan Venkateswaran) and in Chapter 10 Listing 10.5. Setting Up DMA Descriptors and Buffers, I see
/* Device-specific data structure for the Ethernet Function */
struct device_data {
struct pci_dev *pdev; /* The PCI Device structure */
struct net_device *ndev; /* The Net Device structure */
void *dma_buffer_rx; /* Kernel virtual address of the receive descriptor */
dma_addr_t dma_bus_rx; /* Bus address of the receive descriptor */
void *dma_buffer_tx; /* Kernel virtual address of the transmit descriptor */
dma_addr_t dma_bus_tx; /* Bus address of the transmit descriptor */
/* ... */
spin_lock_t device_lock; /* Serialize */
} *mydev_data;
/* On-card registers related to DMA */
#define DMA_RX_REGISTER_OFFSET 0x0 /* Offset of the register holding the bus address of the RX descriptor */
#define DMA_TX_REGISTER_OFFSET 0x4 /* Offset of the register holding the bus address of the TX descriptor */
#define CONTROL_REGISTER 0x8 /* Offset of the control register */
/* Control Register Defines */
#define INITIATE_XMIT 0x1
/* Descriptor control word definitions */
#define FREE_FLAG 0x1 /* Free Descriptor */
#define INTERRUPT_FLAG 0x2 /* Assert interrupt after DMA */
/* Invoked from Listing 10.3 */
static void dma_descriptor_setup(struct pci_dev *pdev)
{
/* Allocate receive DMA descriptors and buffers */
mydev_data->dma_buffer_rx = pci_alloc_consistent(pdev, 3096, &mydev_data->dma_bus_rx);
/* Fill the two receive descriptors as shown in Figure 10.2 */
/* RX descriptor 1 */
mydev_data->dma_buffer_rx[0] = cpu_to_le32((unsigned long)(mydev_data->dma_bus_rx + 24)); /* Buffer address */
mydev_data->dma_buffer_rx[1] = 1536; /* Buffer length */
mydev_data->dma_buffer_rx[2] = FREE_FLAG; /* Descriptor is free */
/* RX descriptor 2 */
mydev_data->dma_buffer_rx[3] = cpu_to_le32((unsigned long)(mydev_data->dma_bus_rx + 1560)); /* Buffer address */
mydev_data->dma_buffer_rx[4] = 1536; /* Buffer length */
mydev_data->dma_buffer_rx[5] = FREE_FLAG; /* Descriptor is free */
wmb(); /* Write Memory Barrier */
/* Write the address of the receive descriptor to the appropriate register in the card. The I/O base address, ioaddr, was populated in Listing 10.3 */
outl(cpu_to_le32((unsigned long)mydev_data->dma_bus_rx), ioaddr + DMA_RX_REGISTER_OFFSET);
/* Allocate transmit DMA descriptors and buffers */
mydev_data->dma_buffer_tx = pci_alloc_consistent(pdev, 3096, &mydev_data->dma_bus_tx);
/* Fill the two transmit descriptors as shown in Figure 10.2 */
/* TX descriptor 1 */
mydev_data->dma_buffer_tx[0] = cpu_to_le32((unsigned long)(mydev_data->dma_bus_tx + 24)); /* Buffer address */ <---- line A
mydev_data->dma_buffer_tx[1] = 1536; /* Buffer length */ <---- line B
/* Valid descriptor. Generate an interrupt after completing the DMA */
mydev_data->dma_buffer_tx[2] = (FREE_FLAG | INTERRUPT_FLAG);
/* TX descriptor 2 */
mydev_data->dma_buffer_tx[3] = cpu_to_le32((unsigned long)(mydev_data->dma_bus_tx + 1560)); /* Buffer address */
mydev_data->dma_buffer_tx[4] = 1536; /* Buffer length */
mydev_data->dma_buffer_tx[5] = (FREE_FLAG | INTERRUPT_FLAG);
wmb(); /* Write Memory Barrier */
/* Write the address of the transmit descriptor to the appropriate register in the card. The I/O base, ioaddr, was populated in Listing 10.3 */
outl(cpu_to_le32((unsigned long)mydev_data->dma_bus_tx), ioaddr + DMA_TX_REGISTER_OFFSET);
}
/* Invoked from Listing 10.3 */
static void dma_descriptor_release(struct pci_dev *pdev)
{
pci_free_consistent(pdev, 3096, mydev_data->dma_bus_tx);
pci_free_consistent(pdev, 3096, mydev_data->dma_bus_rx);
}
In the code, the driver prepares a buffer for the DMA descriptors and DMA buffers using pci_alloc_consistent() and sets them up and passes the buffer address (bus address) to the hardware making sure it's in little endian format using cpu_to_le32(). So I understood the H/W sees the buffer descriptor. But in the descriptor, why did it use cpu_to_le32() for the descriptor address (line A above) and not for the following buffer length (line B above)? Does the H/W see only the buffer address and not the size? Or is it an error in the book? By the way, this is for a fictitious PCI ethernet chip driver.
In practice such approach indeed looks like a mistake. But theoretically it's possible that one field (address) is always little endian no matter what, while another (length) is in native mode. Nowadays, with help of FPGA, I guess one even may try to implement this theoretical case.
So, in the given context, especially taking into consideration your remark that this is for a fictitious PCI ethernet chip, it is a bit hard to say what was author's intention here.

Beagle Bone Black LED's physical address

Is there anyway we can find the physical address of the BBB's LEDs?
I am new to this, but I am trying to write a device driver that access hardware directly.
Thank you :).
Yes, you can consult the BBB schematics to find the pins connecting LEDs here. Let me help you with that.
And here are the actual chipset pins.
So, you need to access and control the pins GPIO1_21, GPIO1_22, GPIO1_23 and GPIO1_24. For this, you need to make sure no other driver is configuring/using these pins. Most probably these pins are already being controlled by the existing "gpio-leds" driver. See the device tree node for gpio-led driver below (source)
leds {
pinctrl-names = "default";
pinctrl-0 = <&user_leds_s0>;
compatible = "gpio-leds";
led#2 {
label = "beaglebone:green:heartbeat";
gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>;
linux,default-trigger = "heartbeat";
default-state = "off";
};
led#3 {
label = "beaglebone:green:mmc0";
gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>;
linux,default-trigger = "mmc0";
default-state = "off";
};
led#4 {
label = "beaglebone:green:usr2";
gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>;
linux,default-trigger = "cpu0";
default-state = "off";
};
led#5 {
label = "beaglebone:green:usr3";
gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>;
linux,default-trigger = "mmc1";
default-state = "off";
};
};
You need to either disable or override the above node to use your own driver. Also make sure to select the correct pinmux setting for your driver so these pins are in gpio mode. You can reuse the default "user_leds_s0" pin group shown below.
&am33xx_pinmux {
pinctrl-names = "default";
pinctrl-0 = <&clkout2_pin>;
user_leds_s0: user_leds_s0 {
pinctrl-single,pins = <
0x54 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a5.gpio1_21 */
0x58 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a6.gpio1_22 */
0x5c (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a7.gpio1_23 */
0x60 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a8.gpio1_24 */
>;
};
Finally, instead of using the physical address of the GPIO pin directly, you should use Linux's GPIO interface as it has already taken control of the GPIO registers. Your final devicetree node for your custom driver could look something like this:
leds {
pinctrl-names = "default";
pinctrl-0 = <&user_leds_s0>;
compatible = "my-gpio-led-driver";
userled-gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>, /* USR0 */
<&gpio1 22 GPIO_ACTIVE_HIGH>, /* USR1 */
<&gpio1 23 GPIO_ACTIVE_HIGH>; /* USR2 */
<&gpio1 24 GPIO_ACTIVE_HIGH>; /* USR3 */
}
In the driver code, you can refer to these GPIO pins as below
struct gpio_desc *usr_led0, *usr_led1, *usr_led2, *usr_led3;
usr_led0 = gpiod_get_index(dev, "userled", 0, GPIOD_OUT_HIGH);
usr_led1 = gpiod_get_index(dev, "userled", 1, GPIOD_OUT_HIGH);
usr_led2 = gpiod_get_index(dev, "userled", 2, GPIOD_OUT_HIGH);
usr_led3 = gpiod_get_index(dev, "userled", 3, GPIOD_OUT_HIGH);
Then you can use linux gpio consumer interface (gpiod_* API) to control the leds.
Follow these links for more details:
Requesting the GPIO pins required in the device tree.
Getting and controlling the GPIO pins from kernel driver.

PIC32 USB CDC is slow to write and raises erroneous flag

I am currently attempting to write code for a pic32MZ2048EFH100 which receives and responds to messages from a custom program on a PC through the USB CDC drivers in harmony.
Currently, the pic is able to both send and receive messages. However, whenever it sends a message to the PC, it always raises the buffer 80% full flag (EV_RX80FULL) incorrectly. In addition to this, the PIC32 sends messages relatively slowly. I am porting the firmware over from a TERN board which works correctly. There are times when the PC sends two messages very quickly, the TERN will send out an acknowledgement for one, receive the next, and then acknowledge that one. The PIC, on the other hand, will receive both and then send out two acknowledgements which will cause problems with the program on the PC side.
The largest difference between the two boards is that the drivers are different which is where we think the problem is originating. The PIC uses a driver from Microchip and the .inf is as follows
;---------------------------------------------------------------------------------
;Note: When the driver package is signed, any modifications to this .inf file will
;break the signature, and the driver package will need to be re-signed.
;---------------------------------------------------------------------------------
; Modified Windows USB CDC Abstract Control Model Serial Driver Setup File
; Copyright (C) 2012 Microchip Technology Inc.
[Version]
Signature="$Windows NT$"
Class=Ports
ClassGuid={4D36E978-E325-11CE-BFC1-08002BE10318}
Provider=%MFGNAME%
CatalogFile=%MFGFILENAME%.cat
DriverVer=10/06/2014,5.1.2600.9
[Manufacturer]
%MFGNAME%=DeviceList,NTamd64
;------------------------------------------------------------------------------
; Vendor and Product ID Definitions
;------------------------------------------------------------------------------
; When developing your USB device, the VID and PID used in the PC side
; application program and the firmware on the microcontroller must match.
; The VID and PID can be changed in the USB device descriptor and on the below
; lines in this file. If you modify this .inf file to customize it for your
; device, please remove all existing Microchip (VID 0x04D8) entries from
; the device lists.
;------------------------------------------------------------------------------
[DeviceList]
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_000A
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0205
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0206
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_00DF&MI_00
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0057&MI_01
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_006E&MI_01
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0208&MI_00
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0208&MI_02
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_00DD&MI_00
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0207&MI_00
[DeviceList.NTamd64]
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_000A
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0205
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0206
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_00DF&MI_00
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0057&MI_01
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_006E&MI_01
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0208&MI_00
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0208&MI_02
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_00DD&MI_00
%DESCRIPTION%=DriverInstall,USB\VID_04D8&PID_0207&MI_00
;------------------------------------------------------------------------------
; Windows 32bit OSes Section
;------------------------------------------------------------------------------
[DriverInstall.nt]
include=mdmcpq.inf
CopyFiles=FakeModemCopyFileSection
AddReg=DriverInstall.nt.AddReg
[DriverInstall.nt.AddReg]
HKR,,DevLoader,,*ntkern
HKR,,NTMPDriver,,%DRIVERFILENAME%.sys
HKR,,EnumPropPages32,,"MsPorts.dll,SerialPortPropPageProvider"
[DriverInstall.NT.Services]
include=mdmcpq.inf
AddService=usbser, 0x00000002, LowerFilter_Service_Inst
;------------------------------------------------------------------------------
; Windows 64bit OSes Section
;------------------------------------------------------------------------------
[DriverInstall.NTamd64]
include=mdmcpq.inf
CopyFiles=FakeModemCopyFileSection
AddReg=DriverInstall.NTamd64.AddReg
[DriverInstall.NTamd64.AddReg]
HKR,,DevLoader,,*ntkern
HKR,,NTMPDriver,,%DRIVERFILENAME%.sys
HKR,,EnumPropPages32,,"MsPorts.dll,SerialPortPropPageProvider"
[DriverInstall.NTamd64.Services]
include=mdmcpq.inf
AddService=usbser, 0x00000002, LowerFilter_Service_Inst
;------------------------------------------------------------------------------
; Common Sections
;------------------------------------------------------------------------------
[DestinationDirs]
DefaultDestDir=12
[SourceDisksNames]
[SourceDisksFiles]
[FakeModemCopyFileSection]
[LowerFilter_Service_Inst]
DisplayName= %SERVICE%
ServiceType= 1
StartType = 3
ErrorControl = 0
ServiceBinary = %12%\usbser.sys
;------------------------------------------------------------------------------
; String Definitions
;------------------------------------------------------------------------------
; These strings can be modified to customize your device
;------------------------------------------------------------------------------
[Strings]
MFGFILENAME="mchpcdc"
DRIVERFILENAME ="usbser"
MFGNAME="Microchip Technology, Inc." ;This name shows up in the device manager properties for the device
DESCRIPTION="USB Serial Port" ;This is the "friendly name" that shows up in the device manager
SERVICE="USB to Serial Driver"
Here is the USB configuration on the PIC:
/**************************************************
* USB Device Function Driver Init Data
**************************************************/
const USB_DEVICE_CDC_INIT cdcInit0 =
{
.queueSizeRead = 1,
.queueSizeWrite = 1,
.queueSizeSerialStateNotification = 1
};
/**************************************************
* USB Device Layer Function Driver Registration
* Table
**************************************************/
const USB_DEVICE_FUNCTION_REGISTRATION_TABLE funcRegistrationTable[1] =
{
/* Function 1 */
{
.configurationValue = 1, /* Configuration value */
.interfaceNumber = 0, /* First interfaceNumber of this function */
.speed = USB_SPEED_HIGH|USB_SPEED_FULL, /* Function Speed */
.numberOfInterfaces = 2, /* Number of interfaces */
.funcDriverIndex = 0, /* Index of CDC Function Driver */
.driver = (void*)USB_DEVICE_CDC_FUNCTION_DRIVER, /* USB CDC function data exposed to device layer */
.funcDriverInit = (void*)&cdcInit0 /* Function driver init data */
},
};
/*******************************************
* USB Device Layer Descriptors
*******************************************/
/*******************************************
* USB Device Descriptor
*******************************************/
const USB_DEVICE_DESCRIPTOR deviceDescriptor =
{
0x12, // Size of this descriptor in bytes
USB_DESCRIPTOR_DEVICE, // DEVICE descriptor type
0x0200, // USB Spec Release Number in BCD format
USB_CDC_CLASS_CODE, // Class Code
USB_CDC_SUBCLASS_CODE, // Subclass code
0x00, // Protocol code
USB_DEVICE_EP0_BUFFER_SIZE, // Max packet size for EP0, see system_config.h
0x04D8, // Vendor ID
0x000A, // Product ID
0x0100, // Device release number in BCD format
0x01, // Manufacturer string index
0x02, // Product string index
0x00, // Device serial number string index
0x01 // Number of possible configurations
};
/*******************************************
* USB Device Qualifier Descriptor for this
* demo.
*******************************************/
const USB_DEVICE_QUALIFIER deviceQualifierDescriptor1 =
{
0x0A, // Size of this descriptor in bytes
USB_DESCRIPTOR_DEVICE_QUALIFIER, // Device Qualifier Type
0x0200, // USB Specification Release number
USB_CDC_CLASS_CODE, // Class Code
USB_CDC_SUBCLASS_CODE, // Subclass code
0x00, // Protocol code
USB_DEVICE_EP0_BUFFER_SIZE, // Maximum packet size for endpoint 0
0x01, // Number of possible configurations
0x00 // Reserved for future use.
};
/*******************************************
* USB High Speed Configuration Descriptor
*******************************************/
const uint8_t highSpeedConfigurationDescriptor[]=
{
/* Configuration Descriptor */
0x09, // Size of this descriptor in bytes
USB_DESCRIPTOR_CONFIGURATION, // Descriptor Type
67,0, //(67 Bytes)Size of the Config descriptor.e
2, // Number of interfaces in this cfg
0x01, // Index value of this configuration
0x00, // Configuration string index
USB_ATTRIBUTE_DEFAULT | USB_ATTRIBUTE_SELF_POWERED, // Attributes
50, // Max power consumption (2X mA)
/* Descriptor for Function 1 - CDC */
/* Interface Descriptor */
0x09, // Size of this descriptor in bytes
USB_DESCRIPTOR_INTERFACE, // Descriptor Type
0, // Interface Number
0x00, // Alternate Setting Number
0x01, // Number of endpoints in this interface
USB_CDC_COMMUNICATIONS_INTERFACE_CLASS_CODE, // Class code
USB_CDC_SUBCLASS_ABSTRACT_CONTROL_MODEL, // Subclass code
USB_CDC_PROTOCOL_AT_V250, // Protocol code
0x00, // Interface string index
/* CDC Class-Specific Descriptors */
sizeof(USB_CDC_HEADER_FUNCTIONAL_DESCRIPTOR), // Size of the descriptor
USB_CDC_DESC_CS_INTERFACE, // CS_INTERFACE
USB_CDC_FUNCTIONAL_HEADER, // Type of functional descriptor
0x20,0x01, // CDC spec version
sizeof(USB_CDC_ACM_FUNCTIONAL_DESCRIPTOR), // Size of the descriptor
USB_CDC_DESC_CS_INTERFACE, // CS_INTERFACE
USB_CDC_FUNCTIONAL_ABSTRACT_CONTROL_MANAGEMENT, // Type of functional descriptor
USB_CDC_ACM_SUPPORT_LINE_CODING_LINE_STATE_AND_NOTIFICATION,// bmCapabilities of ACM
sizeof(USB_CDC_UNION_FUNCTIONAL_DESCRIPTOR_HEADER) + 1, // Size of the descriptor
USB_CDC_DESC_CS_INTERFACE, // CS_INTERFACE
USB_CDC_FUNCTIONAL_UNION, // Type of functional descriptor
0, // com interface number
1,
sizeof(USB_CDC_CALL_MANAGEMENT_DESCRIPTOR), // Size of the descriptor
USB_CDC_DESC_CS_INTERFACE, // CS_INTERFACE
USB_CDC_FUNCTIONAL_CALL_MANAGEMENT, // Type of functional descriptor
0x00, // bmCapabilities of CallManagement
1, // Data interface number
/* Interrupt Endpoint (IN)Descriptor */
0x07, // Size of this descriptor
USB_DESCRIPTOR_ENDPOINT, // Endpoint Descriptor
1| USB_EP_DIRECTION_IN, // EndpointAddress ( EP1 IN INTERRUPT)
USB_TRANSFER_TYPE_INTERRUPT, // Attributes type of EP (INTERRUPT)
0x10,0x00, // Max packet size of this EP
0x02, // Interval (in ms)
/* Interface Descriptor */
0x09, // Size of this descriptor in bytes
USB_DESCRIPTOR_INTERFACE, // INTERFACE descriptor type
1, // Interface Number
0x00, // Alternate Setting Number
0x02, // Number of endpoints in this interface
USB_CDC_DATA_INTERFACE_CLASS_CODE, // Class code
0x00, // Subclass code
USB_CDC_PROTOCOL_NO_CLASS_SPECIFIC, // Protocol code
0x00, // Interface string index
/* Bulk Endpoint (OUT)Descriptor */
0x07, // Size of this descriptor
USB_DESCRIPTOR_ENDPOINT, // Endpoint Descriptor
2|USB_EP_DIRECTION_OUT, // EndpointAddress ( EP2 OUT)
USB_TRANSFER_TYPE_BULK, // Attributes type of EP (BULK)
0x00, 0x02, // Max packet size of this EP
0x00, // Interval (in ms)
/* Bulk Endpoint (IN)Descriptor */
0x07, // Size of this descriptor
USB_DESCRIPTOR_ENDPOINT, // Endpoint Descriptor
2|USB_EP_DIRECTION_IN, // EndpointAddress ( EP2 IN )
0x02, // Attributes type of EP (BULK)
0x00, 0x02, // Max packet size of this EP
0x00, // Interval (in ms)
};
/******************************************************
* USB Driver Initialization
******************************************************/
const DRV_USBHS_INIT drvUSBInit =
{
/* Interrupt Source for USB module */
.interruptSource = INT_SOURCE_USB_1,
/* Interrupt Source for USB module */
.interruptSourceUSBDma = INT_SOURCE_USB_1_DMA,
/* System module initialization */
.moduleInit = {SYS_MODULE_POWER_RUN_FULL},
.operationMode = DRV_USBHS_OPMODE_DEVICE,
.operationSpeed = USB_SPEED_HIGH,
/* Stop in idle */
.stopInIdle = false,
/* Suspend in sleep */
.suspendInSleep = false,
/* Identifies peripheral (PLIB-level) ID */
.usbID = 0,
};

IP camera capture

I am trying to capture the stream of two IP cameras directly connected to a mini PCIe dual gigabit expansion card in a nVidia Jetson TK1.
I achieved to capture the stream of both cameras using gstreamer with the next command:
gst-launch-0.10 rtspsrc location=rtsp://admin:123456#192.168.0.123:554/mpeg4cif latency=0 ! decodebin ! ffmpegcolorspace ! autovideosink rtspsrc location=rtsp://admin:123456#192.168.2.254:554/mpeg4cif latency=0 ! decodebin ! ffmpegcolorspace ! autovideosink
It displays one window per camera, but gives this output just when the capture starts:
WARNING: from element /GstPipeline:pipeline0/GstAutoVideoSink:autovideosink1/GstXvImageSink:autovideosink1-actual-sink-xvimage: A lot of buffers are being dropped.
Additional debug info:
gstbasesink.c(2875): gst_base_sink_is_too_late (): /GstPipeline:pipeline0/GstAutoVideoSink:autovideosink1/GstXvImageSink:autovideosink1-actual-sink-xvimage:
There may be a timestamping problem, or this computer is too slow.
---> TVMR: Video-conferencing detected !!!!!!!!!
The stream is played good, with "good" synchronization also between cameras, but after a while, suddenly one of the cameras stops, and usually few seconds later the other one stops too. Using an interface snifer like Wireshark I can check that the rtsp packets are still sending from the cameras.
My purpose is to use this cameras to use them as a stereo camera using openCV. I am able to capture the stream with OpenCV with the following function:
camera[0].open("rtsp://admin:123456#192.168.2.254:554/mpeg4cif");//right
camera[1].open("rtsp://admin:123456#192.168.0.123:554/mpeg4cif");//left
It randomnly starts the capture good or bad, synchronized or not, with delay or not, but after a while is impossible to use the captured images as you can observe in the image:
And the output while running the openCV program usually is this: (I have copied the most complete one)
[h264 # 0x1b9580] slice type too large (2) at 0 23
[h264 # 0x1b9580] decode_slice_header error
[h264 # 0x1b1160] left block unavailable for requested intra mode at 0 6
[h264 # 0x1b1160] error while decoding MB 0 6, bytestream (-1)
[h264 # 0x1b1160] mmco: unref short failure
[h264 # 0x1b9580] too many reference frames
[h264 # 0x1b1160] pps_id (-1) out of range
The used cameras are two SIP-1080J modules.
Anyone knows how to achieve a good capture using openCV? First of all get rid of those h264 messages and have stable images while the program executes.
If not, how can I improve the pipelines and buffers using gstreamer to have a good capture without the sudden stop of the stream?. Although I never captured through openCV using gstreamer, perhaps some day I will know how to do it and solve this problem.
Thanks a lot.
After some days of deep search and some attempts, I turned on directly to use the gstreamer-0.10 API. First I learned how to use it with the tutorials from http://docs.gstreamer.com/pages/viewpage.action?pageId=327735
For most of the tutorials, you just need to install libgstreamer0.10-dev and some other packages. I installed all by:
sudo apt-get install libgstreamer0*
Then copy the code of the example you want to try into a .c file and type from the terminal in the folder where the .c file is located (In some examples you have to add more libs to pkg-config):
gcc basic-tutorial-1.c $(pkg-config --cflags --libs gstreamer-0.10) -o basic-tutorial-1.c
After that I did not feel lost I started to try to mix some c and c++ code. You can compile it using a proper g++ command, or with a CMakeLists.txt or the way you want to... As I am developing with a nVidia Jetson TK1, I use Nsight Eclipse Edition and I need to configure the project properties properly to be able to use the gstreamer-0.10 libs and the openCV libs.
Mixing some code, finally I am able to capture the streams of my two IP cameras in real time without appreciable delay, without bad decoding in any frame and both streams synchronized. The only thing left that I have not solved yet is the obtaining of frames in color and not in gray scale when (I have tried with other CV_ values with "segmentation fault" result):
v = Mat(Size(640, 360),CV_8U, (char*)GST_BUFFER_DATA(gstImageBuffer));
The complete code is next where I capture using gstreamer, transform the capture to a openCV Mat object and then show it. The code is for just a capture of one IP camera. You can replicate the objects and methods for capture multiple cameras at the same time.
#include <opencv2/core/core.hpp>
#include <opencv2/contrib/contrib.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/video/video.hpp>
#include <gst/gst.h>
#include <gst/app/gstappsink.h>
#include <gst/app/gstappbuffer.h>
#include <glib.h>
#define DEFAULT_LATENCY_MS 1
using namespace cv;
typedef struct _vc_cfg_data {
char server_ip_addr[100];
} vc_cfg_data;
typedef struct _vc_gst_data {
GMainLoop *loop;
GMainContext *context;
GstElement *pipeline;
GstElement *rtspsrc,*depayloader, *decoder, *converter, *sink;
GstPad *recv_rtp_src_pad;
} vc_gst_data;
typedef struct _vc_data {
vc_gst_data gst_data;
vc_cfg_data cfg;
} vc_data;
/* Global data */
vc_data app_data;
static void vc_pad_added_handler (GstElement *src, GstPad *new_pad, vc_data *data);
#define VC_CHECK_ELEMENT_ERROR(e, name) \
if (!e) { \
g_printerr ("Element %s could not be created. Exiting.\n", name); \
return -1; \
}
/*******************************************************************************
Gstreamer pipeline creation and init
*******************************************************************************/
int vc_gst_pipeline_init(vc_data *data)
{
GstStateChangeReturn ret;
// Template
GstPadTemplate* rtspsrc_pad_template;
// Create a new GMainLoop
data->gst_data.loop = g_main_loop_new (NULL, FALSE);
data->gst_data.context = g_main_loop_get_context(data->gst_data.loop);
// Create gstreamer elements
data->gst_data.pipeline = gst_pipeline_new ("videoclient");
VC_CHECK_ELEMENT_ERROR(data->gst_data.pipeline, "pipeline");
//RTP UDP Source - for received RTP messages
data->gst_data.rtspsrc = gst_element_factory_make ("rtspsrc", "rtspsrc");
VC_CHECK_ELEMENT_ERROR(data->gst_data.rtspsrc,"rtspsrc");
printf("URL: %s\n",data->cfg.server_ip_addr);
g_print ("Setting RTSP source properties: \n");
g_object_set (G_OBJECT (data->gst_data.rtspsrc), "location", data->cfg.server_ip_addr, "latency", DEFAULT_LATENCY_MS, NULL);
//RTP H.264 Depayloader
data->gst_data.depayloader = gst_element_factory_make ("rtph264depay","depayloader");
VC_CHECK_ELEMENT_ERROR(data->gst_data.depayloader,"rtph264depay");
//ffmpeg decoder
data->gst_data.decoder = gst_element_factory_make ("ffdec_h264", "decoder");
VC_CHECK_ELEMENT_ERROR(data->gst_data.decoder,"ffdec_h264");
data->gst_data.converter = gst_element_factory_make ("ffmpegcolorspace", "converter");
VC_CHECK_ELEMENT_ERROR(data->gst_data.converter,"ffmpegcolorspace");
// i.MX Video sink
data->gst_data.sink = gst_element_factory_make ("appsink", "sink");
VC_CHECK_ELEMENT_ERROR(data->gst_data.sink,"appsink");
gst_app_sink_set_max_buffers((GstAppSink*)data->gst_data.sink, 1);
gst_app_sink_set_drop ((GstAppSink*)data->gst_data.sink, TRUE);
g_object_set (G_OBJECT (data->gst_data.sink),"sync", FALSE, NULL);
//Request pads from rtpbin, starting with the RTP receive sink pad,
//This pad receives RTP data from the network (rtp-udpsrc).
rtspsrc_pad_template = gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (data->gst_data.rtspsrc),"recv_rtp_src_0");
// Use the template to request the pad
data->gst_data.recv_rtp_src_pad = gst_element_request_pad (data->gst_data.rtspsrc, rtspsrc_pad_template,
"recv_rtp_src_0", NULL);
// Print the name for confirmation
g_print ("A new pad %s was created\n",
gst_pad_get_name (data->gst_data.recv_rtp_src_pad));
// Add elements into the pipeline
g_print(" Adding elements to pipeline...\n");
gst_bin_add_many (GST_BIN (data->gst_data.pipeline),
data->gst_data.rtspsrc,
data->gst_data.depayloader,
data->gst_data.decoder,
data->gst_data.converter,
data->gst_data.sink,
NULL);
// Link some of the elements together
g_print(" Linking some elements ...\n");
if(!gst_element_link_many (data->gst_data.depayloader, data->gst_data.decoder, data->gst_data.converter, data->gst_data.sink, NULL))
g_print("Error: could not link all elements\n");
// Connect to the pad-added signal for the rtpbin. This allows us to link
//the dynamic RTP source pad to the depayloader when it is created.
if(!g_signal_connect (data->gst_data.rtspsrc, "pad-added",
G_CALLBACK (vc_pad_added_handler), data))
g_print("Error: could not add signal handler\n");
// Set the pipeline to "playing" state
g_print ("Now playing A\n");
ret = gst_element_set_state (data->gst_data.pipeline, GST_STATE_PLAYING);
if (ret == GST_STATE_CHANGE_FAILURE) {
g_printerr ("Unable to set the pipeline A to the playing state.\n");
gst_object_unref (data->gst_data.pipeline);
return -1;
}
return 0;
}
static void vc_pad_added_handler (GstElement *src, GstPad *new_pad, vc_data *data) {
GstPad *sink_pad = gst_element_get_static_pad (data->gst_data.depayloader, "sink");
GstPadLinkReturn ret;
GstCaps *new_pad_caps = NULL;
GstStructure *new_pad_struct = NULL;
const gchar *new_pad_type = NULL;
g_print ("Received new pad '%s' from '%s':\n", GST_PAD_NAME (new_pad), GST_ELEMENT_NAME (src));
/* Check the new pad's name */
if (!g_str_has_prefix (GST_PAD_NAME (new_pad), "recv_rtp_src_")) {
g_print (" It is not the right pad. Need recv_rtp_src_. Ignoring.\n");
goto exit;
}
/* If our converter is already linked, we have nothing to do here */
if (gst_pad_is_linked (sink_pad)) {
g_print (" Sink pad from %s already linked. Ignoring.\n", GST_ELEMENT_NAME (src));
goto exit;
}
/* Check the new pad's type */
new_pad_caps = gst_pad_get_caps (new_pad);
new_pad_struct = gst_caps_get_structure (new_pad_caps, 0);
new_pad_type = gst_structure_get_name (new_pad_struct);
/* Attempt the link */
ret = gst_pad_link (new_pad, sink_pad);
if (GST_PAD_LINK_FAILED (ret)) {
g_print (" Type is '%s' but link failed.\n", new_pad_type);
} else {
g_print (" Link succeeded (type '%s').\n", new_pad_type);
}
exit:
/* Unreference the new pad's caps, if we got them */
if (new_pad_caps != NULL)
gst_caps_unref (new_pad_caps);
/* Unreference the sink pad */
gst_object_unref (sink_pad);
}
int vc_gst_pipeline_clean(vc_data *data) {
GstStateChangeReturn ret;
GstStateChangeReturn ret2;
/* Cleanup Gstreamer */
if(!data->gst_data.pipeline)
return 0;
/* Send the main loop a quit signal */
g_main_loop_quit(data->gst_data.loop);
g_main_loop_unref(data->gst_data.loop);
ret = gst_element_set_state (data->gst_data.pipeline, GST_STATE_NULL);
if (ret == GST_STATE_CHANGE_FAILURE) {
g_printerr ("Unable to set the pipeline A to the NULL state.\n");
gst_object_unref (data->gst_data.pipeline);
return -1;
}
g_print ("Deleting pipeline\n");
gst_object_unref (GST_OBJECT (data->gst_data.pipeline));
/* Zero out the structure */
memset(&data->gst_data, 0, sizeof(vc_gst_data));
return 0;
}
void handleKey(char key)
{
switch (key)
{
case 27:
break;
}
}
int vc_mainloop(vc_data* data)
{
GstBuffer *gstImageBuffer;
Mat v;
namedWindow("view",WINDOW_NORMAL);
while (1) {
gstImageBuffer = gst_app_sink_pull_buffer((GstAppSink*)data->gst_data.sink);
if (gstImageBuffer != NULL )
{
v = Mat(Size(640, 360),CV_8U, (char*)GST_BUFFER_DATA(gstImageBuffer));
imshow("view", v);
handleKey((char)waitKey(3));
gst_buffer_unref(gstImageBuffer);
}else{
g_print("gsink buffer didn't return buffer.");
}
}
return 0;
}
int main (int argc, char *argv[])
{
setenv("DISPLAY", ":0", 0);
strcpy(app_data.cfg.server_ip_addr, "rtsp://admin:123456#192.168.0.123:554/mpeg4cif");
gst_init (&argc, &argv);
if(vc_gst_pipeline_init(&app_data) == -1) {
printf("Gstreamer pipeline creation and init failed\n");
goto cleanup;
}
vc_mainloop(&app_data);
printf ("Returned, stopping playback\n");
cleanup:
return vc_gst_pipeline_clean(&app_data);
return 0;
}
I hope this helps!! ;)
uri = 'rtsp://admin:123456#192.168.0.123:554/mpeg4cif'
gst_str = ("rtspsrc location={} latency={} ! rtph264depay ! h264parse ! omxh264dec ! nvvidconv ! video/x-raw, width=(int){}, height=(int){}, format=(string)BGRx ! videoconvert ! appsink sync=false").format(uri, 200, 3072, 2048)
cap= cv2.VideoCapture(gst_str,cv2.CAP_GSTREAMER)
while(True):
_,frame = cap.read()
if frame is None:
break
cv2.imshow("",frame)
cv2.waitKey(0)
cap.release()
cv2.destroyAllWindows()

Resources