Related
I need a way to segment each arrow alone. I tried OpenCv findContours but it broke it or add it to multiple shapes and arrows as the share the boundaries of shapes. I tried OpenCV connected components but this arrows almost in some graph connected all of it. Plus having trouble as the boundaries almost have the same color as the arrow. And in these kind of images each arrow contains different colors. Any opinion about this problem.
This is a sample diagram. I have to deal with harder diagrams like this.
Ok, work with new picture.
1. Binarization the arrows (and shapes):
cv::Mat imgCl = cv::imread("62uoU.jpg", cv::IMREAD_COLOR);
cv::Mat img;
cv::cvtColor(imgCl, img, cv::COLOR_BGR2GRAY);
cv::Mat mask1;
cv::threshold(img, mask1, 30, 255, cv::THRESH_BINARY_INV);
cv::Mat mask2;
cv::threshold(img, mask2, 120, 255, cv::THRESH_BINARY_INV);
cv::Mat diff;
cv::absdiff(mask1, mask2, diff);
cv::imshow("diff1", diff);
Result 1:
Remove rectangle shapes:
cv::Rect objRect(0, 0, diff.cols, diff.rows);
cv::Size minSize(objRect.width / 100, objRect.height / 100);
cv::Mat bin = cv::Mat(diff, objRect).clone();
for (;;)
{
cv::Rect cutRect;
if (!PosRefinement(bin, cutRect, 0.9f, minSize))
{
break;
}
cv::rectangle(bin, cutRect, cv::Scalar(0, 0, 0), cv::FILLED);
cv::rectangle(diff, cutRect, cv::Scalar(0, 0, 0), cv::FILLED);
objRect.x += cutRect.x;
objRect.y += cutRect.y;
objRect.width = cutRect.width;
objRect.height = cutRect.height;
}
cv::imshow("diff", diff);
Result 2:
Find lines:
std::vector<cv::Vec4i> linesP;
cv::HoughLinesP(diff, linesP, 1, CV_PI / 180, 20, 10, 5);
for (size_t i = 0; i < linesP.size(); i++)
{
cv::Vec4i l = linesP[i];
cv::line(imgCl, cv::Point(l[0], l[1]), cv::Point(l[2], l[3]), cv::Scalar(0, 0, 255), 3, cv::LINE_AA);
}
cv::imshow("img", imgCl);
Result 3:
Black arrows was founded. It can to improve this solution: find and delete text areas from image (tesseract or cv::text::ERFilter). And add a little morphology for draw arrow tips with Hough lines.
P.S. Utility function:
bool PosRefinement(
cv::Mat bin,
cv::Rect& cutRect,
double kThreshold,
cv::Size minSize
)
{
const double areaThreshold = 100;
const int radius = 5;
const int maxIters = 100;
std::vector<std::vector<cv::Point>> contours;
std::vector<cv::Vec4i> hierarchy;
cv::findContours(bin, contours, hierarchy, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE, cv::Point());
size_t bestCont = contours.size();
double maxArea = 0;
for (size_t i = 0; i < contours.size(); i++)
{
double area = cv::contourArea(contours[i]);
if (area > maxArea)
{
maxArea = area;
bestCont = i;
}
}
if (maxArea < areaThreshold)
{
return false;
}
cv::Moments m = cv::moments(contours[bestCont]);
cv::Point mc(cvRound(m.m10 / m.m00), cvRound(m.m01 / m.m00));
cv::Rect currRect(mc.x - radius / 2, mc.y - radius / 2, radius, radius);
auto Clamp = [](int v, int hi) -> bool
{
if (v < 0)
{
v = 0;
return true;
}
else if (hi && v > hi - 1)
{
v = hi - 1;
return true;
}
return false;
};
auto RectClamp = [&](cv::Rect& r, int w, int h) -> bool
{
return Clamp(r.x, w) || Clamp(r.x + r.width, w) || Clamp(r.y, h) || Clamp(r.y + r.height, h);
};
int stepL = radius / 2;
int stepR = radius / 2;
int stepT = radius / 2;
int stepB = radius / 2;
double k = 0;
struct State
{
double k = 0;
int stepL = 0;
int stepR = 0;
int stepT = 0;
int stepB = 0;
cv::Rect currRect;
State() = default;
State(double k_, int stepL_, int stepR_, int stepT_, int stepB_, cv::Rect currRect_)
:
k(k_),
stepL(stepL_),
stepR(stepR_),
stepT(stepT_),
stepB(stepB_),
currRect(currRect_)
{
}
bool operator==(const State& st) const
{
return (st.k == k) && (st.stepL == stepL) && (st.stepR == stepR) && (st.stepT == stepT) && (st.stepB == stepB) && (st.currRect == currRect);
}
};
const size_t statesCount = 2;
State prevStates[statesCount];
size_t stateInd = 0;
for (int it = 0; it < maxIters; ++it)
{
cv::Rect rleft(currRect.x - stepL, currRect.y, currRect.width + stepL, currRect.height);
cv::Rect rright(currRect.x, currRect.y, currRect.width + stepR, currRect.height);
cv::Rect rtop(currRect.x, currRect.y - stepT, currRect.width, currRect.height + stepT);
cv::Rect rbottom(currRect.x, currRect.y, currRect.width, currRect.height + stepB);
double kleft = 0;
double kright = 0;
double ktop = 0;
double kbottom = 0;
if (!RectClamp(rleft, bin.cols, bin.rows))
{
cv::Rect rstep(currRect.x - stepL, currRect.y, stepL, currRect.height);
if (cv::sum(bin(rstep))[0] / (255.0 * rstep.area()) > kThreshold / 2)
{
kleft = cv::sum(bin(rleft))[0] / (255.0 * rleft.area());
}
}
if (!RectClamp(rright, bin.cols, bin.rows))
{
cv::Rect rstep(currRect.x + currRect.width, currRect.y, stepR, currRect.height);
if (cv::sum(bin(rstep))[0] / (255.0 * rstep.area()) > kThreshold / 2)
{
kright = cv::sum(bin(rright))[0] / (255.0 * rright.area());
}
}
if (!RectClamp(rtop, bin.cols, bin.rows))
{
cv::Rect rstep(currRect.x, currRect.y - stepT, currRect.width, stepT);
if (cv::sum(bin(rstep))[0] / (255.0 * rstep.area()) > kThreshold / 2)
{
ktop = cv::sum(bin(rtop))[0] / (255.0 * rtop.area());
}
}
if (!RectClamp(rbottom, bin.cols, bin.rows))
{
cv::Rect rstep(currRect.x, currRect.y + currRect.height, currRect.width, stepB);
if (cv::sum(bin(rstep))[0] / (255.0 * rstep.area()) > kThreshold / 2)
{
kbottom = cv::sum(bin(rbottom))[0] / (255.0 * rbottom.area());
}
}
bool wasEnlargeX = false;
if (kleft > kThreshold)
{
currRect.x -= stepL;
currRect.width += stepL;
wasEnlargeX = true;
if (kleft > k)
{
++stepL;
}
}
else
{
if (stepL > 1)
{
--stepL;
}
currRect.x += 1;
currRect.width -= 1;
}
if (kright > kThreshold)
{
currRect.width += stepR;
wasEnlargeX = true;
if (kright > k)
{
++stepR;
}
}
else
{
if (stepR > 1)
{
--stepR;
}
currRect.width -= 1;
}
bool wasEnlargeY = false;
if (ktop > kThreshold)
{
currRect.y -= stepT;
currRect.height += stepT;
wasEnlargeY = true;
if (ktop > k)
{
++stepT;
}
}
else
{
if (stepT > 1)
{
--stepT;
}
currRect.y += 1;
currRect.height -= 1;
}
if (kbottom > kThreshold)
{
currRect.height += stepB;
wasEnlargeY = true;
if (kbottom > k)
{
++stepB;
}
}
else
{
if (stepB > 1)
{
--stepB;
}
currRect.height -= 1;
}
k = cv::sum(bin(currRect))[0] / (255.0 * currRect.area());
State currState(k, stepL, stepR, stepT, stepB, currRect);
bool repState = false;
for (size_t i = 0; i < statesCount; ++i)
{
if (prevStates[i] == currState)
{
repState = true;
break;
}
}
if (repState)
{
break;
}
else
{
prevStates[stateInd] = currState;
stateInd = (stateInd + 1 < statesCount) ? (stateInd + 1) : 0;
}
if (k < kThreshold && (stepL + stepR + stepT + stepB == 4) && !wasEnlargeX && !wasEnlargeY)
{
break;
}
}
cutRect.x = std::max(0, currRect.x - 1);
cutRect.width = currRect.width + 2;
cutRect.y = std::max(0, currRect.y - 1);
cutRect.height = currRect.height + 2;
return (cutRect.width >= minSize.width) && (cutRect.height >= minSize.height);
}
For your example it might be simple. The picture (png) has 4 channels and 4th channel is transparent mask. It can work only with transparent channel and filter arrows with moments:
cv::Mat img = cv::imread("voXFs.png", cv::IMREAD_UNCHANGED);
std::cout << "imsize = " << img.size() << ", chans = " << img.channels() << std::endl;
cv::imshow("img", img);
std::vector<cv::Mat> chans;
cv::split(img, chans);
cv::imshow("transp", chans.back());
cv::Mat mask;
cv::threshold(chans.back(), mask, 50, 255, cv::THRESH_BINARY | cv::THRESH_OTSU);
std::vector<std::vector<cv::Point> > contours;
cv::findContours(mask, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
cv::Mat draw;
cv::cvtColor(mask, draw, cv::COLOR_GRAY2BGR);
for (size_t i = 0; i < contours.size(); ++i)
{
double area = cv::contourArea(contours[i]);
double len = cv::arcLength(contours[i], false);
double k = len / area;
if (area > 10 && len > 60 && k > 2)
{
std::cout << "area = " << area << ", len = " << len << ", k = " << k << std::endl;
cv::drawContours(draw, contours, i, cv::Scalar(255, 0, 0), 1);
}
}
cv::imshow("mask", mask);
cv::imshow("draw", draw);
cv::waitKey(0);
But for more robust result:
Find and delete text areas from image (tesseract or cv::text::ERFilter).
Erode mask, find all shapes by contours, draw and dilate they. Bitwise and operation for mask and result.
The end!
I have made a robotic arm. I need it to capture video and detect blue and red color. And move the blue object to left and red object to right. But the problem is, whenever it is finding a color, it is supposed to send a character '1' (for blue) and '2' for red to the serial port. My problem is,
1. First of all it is taking a lot of time to react.
2. If it detects red. sends '2' to the serial port. But it repeats the action. Even if I change object.
3. Or If it detects red. sends '1' to the serial port. But it repeats the action. Even if I change object.
Color is being detected successfully.
Please someone help me. I've got stuck for a long time.
My Processing Code:
import processing.video.*;
import processing.serial.*;
Serial myPort; // Create object from Serial class
String portName;
Capture video;
color trackColorgreen,trackColorblue,trackColoryellow;
float threshold = 20;
float distThreshold = 75;
ArrayList<Blob> blobgreen = new ArrayList<Blob>();
ArrayList<Blob> blobblue = new ArrayList<Blob>();
ArrayList<Blob> blobyellow = new ArrayList<Blob>();
void findcolorgreen()
{
for (int x = 0; x < video.width; x++ ) {
for (int y = 0; y < video.height; y++ ) {
int loc = x + y * video.width;
// What is current color
color currentColor = video.pixels[loc];
float r1 = red(currentColor);
float g1 = green(currentColor);
float b1 = blue(currentColor);
float r2 = red(trackColorgreen);
float g2 = green(trackColorgreen);
float b2 = blue(trackColorgreen);
float d = distSq(r1, g1, b1, r2, g2, b2);
if (d < threshold*threshold) {
boolean found = false;
for (Blob b : blobgreen) {
if (b.isNear(x, y)) {
b.add(x, y);
found = true;
break;
}
}
if (!found) {
Blob b = new Blob(x, y);
blobgreen.add(b);
}
}
}
}
}
void findcolorblue()
{
for (int x = 0; x < video.width; x++ ) {
for (int y = 0; y < video.height; y++ ) {
int loc = x + y * video.width;
// What is current color
color currentColor = video.pixels[loc];
float r1 = red(currentColor);
float g1 = green(currentColor);
float b1 = blue(currentColor);
float r2 = red(trackColorblue);
float g2 = green(trackColorblue);
float b2 = blue(trackColorblue);
float d = distSq(r1, g1, b1, r2, g2, b2);
if (d < threshold*threshold) {
boolean found = false;
for (Blob b : blobblue) {
if (b.isNear(x, y)) {
b.add(x, y);
found = true;
break;
}
}
if (!found) {
Blob b = new Blob(x, y);
blobblue.add(b);
}
}
}
}
}
void setup() {
size(640, 360);
String[] cameras = Capture.list();
printArray(cameras);
video = new Capture(this, cameras[3]);
video.start();
//trackColorgreen = color(11,77,28);
trackColorgreen = color(159,211,207);
//trackColorblue = color( 22,135,217);
trackColorblue = color(195,173,175);
//trackColoryellow = color(179,136,9);
// trackColoryellow = color(221,228,125);
portName = Serial.list()[0]; //change the 0 to a 1 or 2 etc. to match your port
myPort = new Serial(this, portName, 9600);
}
void captureEvent(Capture video) {
video.read();
}
void draw() {
video.loadPixels();
image(video, 0, 0);
blobgreen.clear();
blobyellow.clear();
blobblue.clear();
//threshold = map(mouseX, 0, width, 0, 100);
threshold = 80;
// Begin loop to walk through every pixel
findcolorgreen();
findcolorblue();
if(blobgreen.size()==0 && blobblue.size()==0 && blobyellow.size()==0)myPort.write('0');
for (Blob b : blobblue) {
if (b.size() > 800) {
b.show();
myPort.write('1');
delay(2000);
println("blue");
}
}
for (Blob b : blobgreen) {
if (b.size() > 800) {
b.show();
myPort.write('2');
delay(2000);
println("green");
}
}
}
float distSq(float x1, float y1, float x2, float y2) {
float d = (x2-x1)*(x2-x1) + (y2-y1)*(y2-y1);
return d;
}
float distSq(float x1, float y1, float z1, float x2, float y2, float z2) {
float d = (x2-x1)*(x2-x1) + (y2-y1)*(y2-y1) +(z2-z1)*(z2-z1);
return d;
}
void mousePressed() {
// Save color where the mouse is clicked in trackColor variable
int loc = mouseX + mouseY*video.width;
color trackColor = video.pixels[loc];
}
class Blob {
float minx;
float miny;
float maxx;
float maxy;
Blob(float x, float y) {
minx = x;
miny = y;
maxx = x;
maxy = y;
}
void show() {
stroke(0);
fill(255);
strokeWeight(2);
rectMode(CORNERS);
rect(minx, miny, maxx, maxy);
}
void add(float x, float y) {
minx = min(minx, x);
miny = min(miny, y);
maxx = max(maxx, x);
maxy = max(maxy, y);
}
float size() {
return (maxx-minx)*(maxy-miny);
}
boolean isNear(float x, float y) {
float cx = (minx + maxx) / 2;
float cy = (miny + maxy) / 2;
float d = distSq(cx, cy, x, y);
if (d < distThreshold*distThreshold) {
return true;
} else {
return false;
}
}
}
Arduino Code:
#include <Servo.h>
Servo s2,s3,s4,s5,s6;
int pos;
char val;
void setup() {
s2.attach(3);
s3.attach(4);
s4.attach(5);
s5.attach(6);
s6.attach(7);
s2.write(90);
s3.write(10);
s4.write(100);
s5.write(70);
s6.write(15);
}
void grab()
{
for(pos=15;pos<=85;pos+=1)
{
s6.write(pos);
s5.write(85-pos);
delay(20);
}
delay(50);
// moving s3 forward
for (pos = 10; pos <= 60; pos += 1) {
s3.write(pos);
delay(20);
}
delay(50);
// moving s4 forward
for (pos = 100; pos >= 80; pos -= 1) {
s4.write(pos);
delay(20);
}
delay(20);
// moving s3 forward
for (pos = 60; pos <= 70; pos += 1) {
s3.write(pos);
delay(20);
}
for(pos=85;pos>=15;pos-=1)
{
s6.write(pos);
int x=-(pos-85);
s5.write(x);
delay(20);
}
s5.write(80);
delay(20);
//s3 initial
// moving s3 forward
for (pos = 70; pos >= 10; pos -= 1) {
s3.write(pos);
delay(20);
}
delay(20);
// moving s4 forward
for (pos = 80; pos <= 100; pos += 1) {
s4.write(pos);
delay(20);
}
delay(20);
}
void move_left()
{
for (pos = 90; pos <= 180; pos += 1) {
s2.write(pos);
delay(20);
}
delay(20);
// moving s3 forward
for (pos = 10; pos <= 60; pos += 1) {
s3.write(pos);
delay(20);
}
delay(20);
// moving s4 forward
for (pos = 100; pos >= 70; pos -= 1) {
s4.write(pos);
delay(20);
}
delay(50);
// moving s3 forward
for (pos = 60; pos <= 70; pos += 1) {
s3.write(pos);
delay(20);
}
//expanding claw
for(pos=15;pos<=85;pos+=1)
{
s6.write(pos);
s5.write(85-pos);
delay(20);
}
delay(20);
// moving s4 up
for (pos = 70; pos <=110; pos += 1) {
s4.write(pos);
delay(20);
}
delay(20);
//s3 initial
// moving s3 forward
for (pos = 70; pos >= 10; pos -= 1) {
s3.write(pos);
delay(20);
}
delay(20);
// moving s4 forward
for (pos = 110; pos >= 100; pos -= 1) {
s4.write(pos);
delay(20);
}
delay(20);
//For servo 1 rotation
for (pos = 180; pos >= 90; pos -= 1) {
s2.write(pos);
delay(20);
}
delay(20);
for(pos=85;pos>=15;pos-=1)
{
s6.write(pos);
int x=-(pos-85);
s5.write(x);
delay(20);
}
s5.write(80);
delay(20);
}
void move_right()
{
for (pos = 90; pos >= 0; pos -= 1) {
s2.write(pos);
delay(50);
}
delay(50);
delay(50);
// moving s3 forward
for (pos = 10; pos <= 60; pos += 1) {
s3.write(pos);
delay(50);
}
delay(50);
// moving s4 forward
for (pos = 100; pos >= 70; pos -= 1) {
s4.write(pos);
delay(50);
}
delay(50);
// moving s3 forward
for (pos = 60; pos <= 70; pos += 1) {
s3.write(pos);
delay(50);
}
//expanding claw
for(pos=15;pos<=85;pos+=1)
{
s6.write(pos);
s5.write(85-pos);
delay(20);
}
delay(50);
// moving s4 up
for (pos = 70; pos <=110; pos += 1) {
s4.write(pos);
delay(50);
}
delay(50);
// s3.write(10);
//delay(500);
//s4.write(100);
//s3 initial
// moving s3 forward
for (pos = 70; pos >= 10; pos -= 1) {
s3.write(pos);
delay(50);
}
delay(50);
// moving s4 forward
for (pos = 110; pos >= 100; pos -= 1) {
s4.write(pos);
delay(50);
}
delay(50);
//For servo 1 rotation
for (pos = 0; pos <= 90; pos += 1) {
s2.write(pos);
delay(50);
}
delay(50);
for(pos=85;pos>=15;pos-=1)
{
s6.write(pos);
int x=-(pos-85);
s5.write(x);
delay(20);
}
s5.write(80);
delay(50);
}
void loop() {
if (Serial.available())
{
val = Serial.read(); // read it and store it in val
if(val=='1')
{
grab();
delay(200);
move_left();
delay(200);
}
else if(val=='2')
{
grab();
delay(200);
move_right();
delay(200);
}
}
}
My robot:
https://drive.google.com/open?id=0B9JjLX3LHgUSSU40U3YzOU1LYXc
I tried to create Fill and Eraser Tools by GDI but this way is too slow for windows phone devices and work with large photos with this method is too hard for this divices.
I search for Alternative soulation for Image Processing and find Win2D and Sharpdx but not sure These Api can help me for create these tools.
this is Fill tool in winrtxamltoolkit
public static void FloodFill(this WriteableBitmap target, int x, int y, int outlineColor, int fillColor, byte maxdiff)
{
var width = target.PixelWidth;
var height = target.PixelHeight;
var queue = new List<Pnt>();
using (var context = target.GetBitmapContext(ReadWriteMode.ReadWrite))
{
queue.Add(new Pnt { X = x, Y = y });
while (queue.Count > 0)
{
var p = queue[queue.Count - 1];
queue.RemoveAt(queue.Count - 1);
if (p.X == -1) continue;
if (p.X == width) continue;
if (p.Y == -1) continue;
if (p.Y == height) continue;
if (context.Pixels[width * p.Y + p.X] == outlineColor) continue;
if (context.Pixels[width * p.Y + p.X] == fillColor) continue;
if (context.Pixels[width * p.Y + p.X].MaxDiff(outlineColor) > maxdiff)
{
context.Pixels[width * p.Y + p.X] = fillColor;
}
else
{
continue;
}
context.Pixels[width * p.Y + p.X] = fillColor;
queue.Add(new Pnt { X = p.X, Y = p.Y - 1 });
queue.Add(new Pnt { X = p.X + 1, Y = p.Y });
queue.Add(new Pnt { X = p.X, Y = p.Y + 1 });
queue.Add(new Pnt { X = p.X - 1, Y = p.Y });
}
target.Invalidate();
}
}
and this is Ereaser tool in WriteableBitmapEx
public static void FillEllipseCenteredTrsnceparent(this WriteableBitmap bmp, int xc, int yc, int xr, int yr, Color color)
{
using (BitmapContext context = bmp.GetBitmapContext())
{
Func<Color, int> toInt32 = c =>
{
var i = ((((c.A << 0x18) | (((c.R * c.A + 1) >> 8) << 0x10)) | (((c.G * c.A + 1) >> 8) << 8)) | ((c.B * c.A + 1) >> 8));
return i;
};
int[] pixels = context.Pixels;
int width = context.Width;
int height = context.Height;
if ((xr >= 1) && (yr >= 1))
{
int num3;
int num4;
int num5;
int num6;
int num7;
int num8;
int num9 = xr;
int num10 = 0;
int num11 = (xr * xr) << 1;
int num12 = (yr * yr) << 1;
int num13 = (yr * yr) * (1 - (xr << 1));
int num14 = xr * xr;
int num15 = 0;
int num16 = num12 * xr;
int num17 = 0;
//int sa = (color >> 0x18) & 0xff;
//int sr = (color >> 0x10) & 0xff;
//int sg = (color >> 8) & 0xff;
//int sb = color & 0xff;
//bool flag = !doAlphaBlend || (sa == 0xff);
while (num16 >= num17)
{
num5 = yc + num10;
num6 = yc - num10;
if (num5 < 0)
{
num5 = 0;
}
if (num5 >= height)
{
num5 = height - 1;
}
if (num6 < 0)
{
num6 = 0;
}
if (num6 >= height)
{
num6 = height - 1;
}
num3 = num5 * width;
num4 = num6 * width;
num8 = xc + num9;
num7 = xc - num9;
if (num8 < 0)
{
num8 = 0;
}
if (num8 >= width)
{
num8 = width - 1;
}
if (num7 < 0)
{
num7 = 0;
}
if (num7 >= width)
{
num7 = width - 1;
}
for (int i = num7; i <= num8; i++)
{
pixels[i + num3] = toInt32(color);
pixels[i + num4] = toInt32(color);
}
num10++;
num17 += num11;
num15 += num14;
num14 += num11;
if ((num13 + (num15 << 1)) > 0)
{
num9--;
num16 -= num12;
num15 += num13;
num13 += num12;
}
}
num9 = 0;
num10 = yr;
num5 = yc + num10;
num6 = yc - num10;
if (num5 < 0)
{
num5 = 0;
}
if (num5 >= height)
{
num5 = height - 1;
}
if (num6 < 0)
{
num6 = 0;
}
if (num6 >= height)
{
num6 = height - 1;
}
num3 = num5 * width;
num4 = num6 * width;
num13 = yr * yr;
num14 = (xr * xr) * (1 - (yr << 1));
num15 = 0;
num16 = 0;
num17 = num11 * yr;
while (num16 <= num17)
{
num8 = xc + num9;
num7 = xc - num9;
if (num8 < 0)
{
num8 = 0;
}
if (num8 >= width)
{
num8 = width - 1;
}
if (num7 < 0)
{
num7 = 0;
}
if (num7 >= width)
{
num7 = width - 1;
}
for (int j = num7; j <= num8; j++)
{
pixels[j + num3] = toInt32(color);
pixels[j + num4] = toInt32(color);
}
num9++;
num16 += num12;
num15 += num13;
num13 += num12;
if ((num14 + (num15 << 1)) > 0)
{
num10--;
num5 = yc + num10;
num6 = yc - num10;
if (num5 < 0)
{
num5 = 0;
}
if (num5 >= height)
{
num5 = height - 1;
}
if (num6 < 0)
{
num6 = 0;
}
if (num6 >= height)
{
num6 = height - 1;
}
num3 = num5 * width;
num4 = num6 * width;
num17 -= num11;
num15 += num14;
num14 += num11;
}
}
}
}
}
Is there a way to conver this code to win2d or Sharpdx?
Can anyone help me out with this?
I am trying to calculate gradient orientation using the Sobel operator in OpenCV for gradient in x and y direction. I am using the atan2 function for computing the tangent in radians, which I later convert to degrees, but all the angles I am getting are between 0 and 90 degrees.
My expectation is to get angles between 0 and 360 degrees. The image I am using is grayscale. The code segment is here below.
Mat PeripheralArea;
Mat grad_x, grad_y; // this is the matrix for the gradients in x and y directions
int off_set_y = 0, off_set_x = 0;
int scale = 1, num_bins = 8, bin = 0;
int delta=-1 ;
int ddepth = CV_16S;
GaussianBlur(PeripheralArea, PeripheralArea, Size(3, 3), 0, 0, BORDER_DEFAULT);
Sobel(PeripheralArea, grad_y, ddepth, 0, 1,3,scale, delta, BORDER_DEFAULT);
Sobel(PeripheralArea, grad_x, ddepth, 1, 0,3, scale, delta, BORDER_DEFAULT);
for (int row_y1 = 0, row_y2 = 0; row_y1 < grad_y.rows / 5, row_y2 < grad_x.rows / 5; row_y1++, row_y2++) {
for (int col_x1 = 0, col_x2 = 0; col_x1 < grad_y.cols / 5, col_x2 < grad_x.cols / 5; col_x1++, col_x2++) {
gradient_direction_radians = (double) atan2((double) grad_y.at<uchar>(row_y1 + off_set_y, col_x1 + off_set_x), (double) grad_x.at<uchar>(row_y2 + off_set_y, col_x2 + off_set_x));
gradient_direction_degrees = (int) (180 * gradient_direction_radians / 3.1415);
gradient_direction_degrees = gradient_direction_degrees < 0
? gradient_direction_degrees+360
: gradient_direction_degrees;
}
}
Note the off_set_x and off_set_y variable are not part of the computation
but to offset to different square blocks for which I eventually want to
compute an histogram feature vector
You have specified that the destination depth of Sobel() is CV_16S.
Yet, when you access grad_x and grad_y, you use .at<uchar>(), implying that their elements are 8 bit unsigned quantities, when in fact they are 16 bit signed. You could use .at<short>() instead, but to me it looks like there a number of issues with your code, not the least of which is that there is an OpenCV function that does exactly what you want.
Use cv::phase(), and replace your for loops with
cv::Mat gradient_angle_degrees;
bool angleInDegrees = true;
cv::phase(grad_x, grad_y, gradient_angle_degrees, angleInDegrees);
I solved this need when I dived into doing some edge detection using C++.
For orientation of gradient I use artan2(), this standard API defines its +y and +x same as how we usually traverse a 2D image.
Plot it to show you my understanding.
///////////////////////////////
// Quadrants of image:
// 3(-dx,-dy) | 4(+dx,-dy) [-pi,0]
// ------------------------->+x
// 2(-dx,+dy) | 1(+dx,+dy) [0,pi]
// v
// +y
///////////////////////////////
// Definition of arctan2():
// -135(-dx,-dy) | -45(+dx,-dy)
// ------------------------->+x
// 135(-dx,+dy) | +45(+dx,+dy)
// v
// +y
///////////////////////////////
How I do for gradient:
bool gradient(double*& magnitude, double*& orientation, double* src, int width, int height, string file) {
if (src == NULL)
return false;
if (width <= 0 || height <= 0)
return false;
double gradient_x_correlation[3*3] = {-0.5, 0.0, 0.5,
-0.5, 0.0, 0.5,
-0.5, 0.0, 0.5};
double gradient_y_correlation[3*3] = {-0.5,-0.5,-0.5,
0.0, 0.0, 0.0,
0.5, 0.5, 0.5};
double *Gx = NULL;
double *Gy = NULL;
this->correlation(Gx, src, gradient_x_correlation, width, height, 3);
this->correlation(Gy, src, gradient_y_correlation, width, height, 3);
if (Gx == NULL || Gy == NULL)
return false;
//magnitude
magnitude = new double[sizeof(double)*width*height];
if (magnitude == NULL)
return false;
memset(magnitude, 0, sizeof(double)*width*height);
double gx = 0.0;
double gy = 0.0;
double gm = 0.0;
for (int j=0; j<height; j++) {
for (int i=0; i<width; i++) {
gx = pow(Gx[i+j*width],2);
gy = pow(Gy[i+j*width],2);
gm = sqrt(pow(Gx[i+j*width],2)+pow(Gy[i+j*width],2));
if (gm >= 255.0) {
return false;
}
magnitude[i+j*width] = gm;
}
}
//orientation
orientation = new double[sizeof(double)*width*height];
if (orientation == NULL)
return false;
memset(orientation, 0, sizeof(double)*width*height);
double ori = 0.0;
double dtmp = 0.0;
double ori_normalized = 0.0;
for (int j=0; j<height; j++) {
for (int i=0; i<width; i++) {
gx = (Gx[i+j*width]);
gy = (Gy[i+j*width]);
ori = atan2(Gy[i+j*width], Gx[i+j*width])/PI*(180.0); //[-pi,+pi]
if (gx >= 0 && gy >= 0) { //[Qudrant 1]:[0,90] to be [0,63]
if (ori < 0) {
printf("[Err1QUA]ori:%.1f\n", ori);
return false;
}
ori_normalized = (ori)*255.0/360.0;
if (ori != 0.0 && dtmp != ori) {
printf("[Qudrant 1]orientation: %.1f to be %.1f(%d)\n", ori, ori_normalized, (uint8_t)ori_normalized);
dtmp = ori;
}
}
else if (gx >= 0 && gy < 0) { //[Qudrant 4]:[270,360) equal to [-90, 0) to be [191,255]
if (ori > 0) {
printf("[Err4QUA]orientation:%.1f\n", ori);
return false;
}
ori_normalized = (360.0+ori)*255.0/360.0;
if (ori != 0.0 && dtmp != ori) {
printf("[Qudrant 4]orientation:%.1f to be %.1f(%d)\n", ori, ori_normalized, (uint8_t)ori_normalized);
dtmp = ori;
}
}
else if (gx < 0 && gy >= 0) { //[Qudrant 2]:(90,180] to be [64,127]
if (ori < 0) {
printf("[Err2QUA]orientation:%.1f\n", ori);
return false;
}
ori_normalized = (ori)*255.0/360.0;
if (ori != 0.0 && dtmp != ori) {
printf("[Qudrant 2]orientation: %.1f to be %.1f(%d)\n", ori, ori_normalized, (uint8_t)ori_normalized);
dtmp = ori;
}
}
else if (gx < 0 && gy < 0) { //[Qudrant 3]:(180,270) equal to (-180, -90) to be [128,190]
if (ori > 0) {
printf("[Err3QUA]orientation:%.1f\n", ori);
return false;
}
ori_normalized = (360.0+ori)*255.0/360.0;
if (ori != 0.0 && dtmp != ori) {
printf("[Qudrant 3]orientation:%.1f to be %.1f(%d)\n", ori, ori_normalized, (uint8_t)ori_normalized);
dtmp = ori;
}
}
else {
printf("[EXCEPTION]orientation:%.1f\n", ori);
return false;
}
orientation[i+j*width] = ori_normalized;
}
}
return true;
}
How I do for cross correlation:
bool correlation(double*& dst, double* src, double* kernel, int width, int height, int window) {
if (src == NULL || kernel == NULL)
return false;
if (width <= 0 || height <= 0 || width < window || height < window )
return false;
dst = new double[sizeof(double)*width*height];
if (dst == NULL)
return false;
memset(dst, 0, sizeof(double)*width*height);
int ii = 0;
int jj = 0;
int nn = 0;
int mm = 0;
double max = std::numeric_limits<double>::min();
double min = std::numeric_limits<double>::max();
double range = std::numeric_limits<double>::max();
for (int j=0; j<height; j++) {
for (int i=0; i<width; i++) {
for (int m=0; m<window; m++) {
for (int n=0; n<window; n++) {
ii = i+(n-window/2);
jj = j+(m-window/2);
nn = n;
mm = m;
if (ii >=0 && ii<width && jj>=0 && jj<height) {
dst[i+j*width] += src[ii+jj*width]*kernel[nn+mm*window];
}
else {
dst[i+j*width] += 0;
}
}
}
if (dst[i+j*width] > max)
max = dst[i+j*width];
else if (dst[i+j*width] < min)
min = dst[i+j*width];
}
}
//normalize double matrix to be an uint8_t matrix
range = max - min;
double norm = 0.0;
printf("correlated matrix max:%.1f, min:%.1f, range:%.1f\n", max, min, range);
for (int j=0; j<height; j++) {
for (int i=0; i<width; i++) {
norm = dst[i+j*width];
norm = 255.0*norm/range;
dst[i+j*width] = norm;
}
}
return true;
}
For me, I use an image like a hollow rectangle, you can download it on my sample.
The orientation of gradient of the hollow rectangle part of my sample image would move from 0 to 360 clockwise (Quadrant 1 to 2 to 3 to 4).
Here is my print which describes the trace of orientation:
[Qudrant 1]orientation: 45.0 to be 31.9(31)
[Qudrant 1]orientation: 90.0 to be 63.8(63)
[Qudrant 2]orientation: 135.0 to be 95.6(95)
[Qudrant 2]orientation: 180.0 to be 127.5(127)
[Qudrant 3]orientation:-135.0 to be 159.4(159)
[Qudrant 3]orientation:-116.6 to be 172.4(172)
[Qudrant 4]orientation:-90.0 to be 191.2(191)
[Qudrant 4]orientation:-63.4 to be 210.1(210)
[Qudrant 4]orientation:-45.0 to be 223.1(223)
You can see more source code about digital image processing on my GitHub :)
I want to track objects using the Camshift algorithm. I tried to correct the bugs that I found in the JavaCV translation of the original OpenCV Camshift file.
Here is my code:
package objectTracking;
import com.googlecode.javacv.CanvasFrame;
import com.googlecode.javacv.FrameGrabber;
import com.googlecode.javacv.OpenCVFrameGrabber;
import static com.googlecode.javacv.cpp.opencv_core.*;
import com.googlecode.javacv.cpp.opencv_core.CvBox2D;
import com.googlecode.javacv.cpp.opencv_core.CvPoint;
import com.googlecode.javacv.cpp.opencv_core.CvRect;
import com.googlecode.javacv.cpp.opencv_core.CvScalar;
import com.googlecode.javacv.cpp.opencv_core.IplImage;
import com.googlecode.javacv.cpp.opencv_core.IplImageArray;
import static com.googlecode.javacv.cpp.opencv_imgproc.*;
import com.googlecode.javacv.cpp.opencv_imgproc.CvConnectedComp;
import com.googlecode.javacv.cpp.opencv_imgproc.CvHistogram;
import static com.googlecode.javacv.cpp.opencv_video.*;
import com.sun.jna.ptr.FloatByReference;
import java.awt.event.MouseEvent;
import java.awt.event.MouseListener;
public class CamShifter implements MouseListener{
final static int MOUSE_PRESSED = 1;
final static int MOUSE_RELEASED = 2;
IplImage image, frame, hsv, hue, mask, backproject, histimg;
IplImageArray hueArray;
CvHistogram hist;
CanvasFrame histogram = new CanvasFrame("Histogram"), camshiftDemo = new CanvasFrame("CamshiftDemo");
boolean backproject_mode = false;
boolean select_object = false;
int track_object = 0;
boolean show_hist = true;
boolean paused = false;
CvPoint origin = new CvPoint();
CvRect selection = new CvRect();
OpenCVFrameGrabber grabber = new OpenCVFrameGrabber(0);
CvRect track_window = new CvRect();
CvBox2D track_box = new CvBox2D();
CvConnectedComp track_comp = new CvConnectedComp();
int hsize = 16;
float hranges[] = {0,180};
int[] hdims = {hsize};
float hranges_arr[][] = {hranges};
int vmin = 10, vmax = 256, smin = 30;
public static void main(String args[]) throws Exception {
CamShifter csh = new CamShifter();
csh.work(csh);
System.out.println("CamShiftDetector trial");
}
public CamShifter() throws FrameGrabber.Exception {
grabber.start();
camshiftDemo.getCanvas().addMouseListener(this);
}
public void onMouse(int event, int x, int y) {
if (select_object) {
//get selection
int selX, selY, selW, selH;
selX = Math.min(x, origin.x());
selY = Math.min(y, origin.y());
selW = selX + Math.abs(x - origin.x());
selH = selY + Math.abs(y - origin.y());
selection = cvRect(selX, selY, selW, selH);
System.out.println("Selection : \n("+selX+", "+selY+")\n("+selW+", "+selH+")");
//ensure that selection is enclosed within the image
selX = Math.max(selection.x(), 0);
selY = Math.max(selection.y(), 0);
selW = Math.min(selection.width(), image.width());
selH = Math.min(selection.height(), image.height());
selection = cvRect(selX, selY, selW - selX, selH - selY);
System.out.println("ensure that selection is enclosed within the image");
System.out.println("Selection : \n("+selX+", "+selY+")\n("+selW+", "+selH+")");
}
switch (event) {
case MOUSE_PRESSED:
origin = cvPoint(x, y);
selection = cvRect(x, y, 0, 0);
select_object = true;
break;
case MOUSE_RELEASED:
select_object = false;
if (selection.width() > 0 && selection.height() > 0) {
track_object = -1;
}
break;
}
}
CvScalar hsv2rgb(float hue) {
int[] rgb = new int[3];
int p, sector;
int[][] sector_data = {{0, 2, 1}, {1, 2, 0}, {1, 0, 2}, {2, 0, 1}, {2, 1, 0}, {0, 1, 2}};
hue *= 0.033333333333333333333333333333333f;
sector = (int) Math.floor(hue);
p = Math.round(255 * (hue - sector));
p = p ^ 1;
int temp = 0;
if ((sector & 1) == 1) {
temp = 255;
} else {
temp = 0;
}
p ^= temp;
rgb[sector_data[sector][0]] = 255;
rgb[sector_data[sector][1]] = 0;
rgb[sector_data[sector][2]] = p;
return cvScalar(rgb[2], rgb[1], rgb[0], 0);
}
String coffee;
public void work(CamShifter csh) throws Exception {
IplImage capture = grabber.grab();
System.out.println("paused = "+paused);
if (capture == null) {
System.out.println("Could not initialize capturing...\n");
return;
}
while (true) {
int bin_w;
if (!paused) {
frame = grabber.grab();
if (frame == null) {
return;
}
}
if (image == null) {
image = cvCreateImage(frame.cvSize(), 8, 3);
hsv = cvCreateImage(frame.cvSize(), 8, 3);
hue = cvCreateImage(frame.cvSize(), 8, 1);
mask = cvCreateImage(frame.cvSize(), 8, 1);
backproject = cvCreateImage(frame.cvSize(), 8, 1);
histimg = cvCreateImage(cvSize(320, 200), 8, 3);
cvZero(histimg);
hist = cvCreateHist( 1, hdims, CV_HIST_ARRAY, hranges_arr, 1 );
}
cvCopy(frame, image);
if (!paused)
{
cvCvtColor(image, hsv, CV_BGR2HSV);
if (track_object != 0) {
int _vmin = vmin, _vmax = vmax;
cvInRangeS(hsv, cvScalar(0, smin, Math.min(_vmin, _vmax), 0), cvScalar(180, 256, Math.max(_vmin, _vmax), 0), mask);
cvSplit(hsv, hue, null, null, null);
hueArray = new IplImageArray(hue);
if (track_object < 0) {
float max_val = 0.f;
cvSetImageROI(hue, selection);
cvSetImageROI(mask, selection);
cvCalcHist(hueArray, hist, 0, null);
if (max_val != 0) { // TODO: entier non null == true en C, à vérifier
max_val = (float) 255. / max_val;
} else {
max_val = 0;
}
FloatByReference fl_ref = new FloatByReference(max_val);
cvConvertScale(hist.bins(), hist.bins(), Float.parseFloat(fl_ref.toString()), 0);
cvResetImageROI(hue);
cvResetImageROI(mask);
track_window = selection;
track_object = 1;
cvZero(histimg);
bin_w = histimg.width() / hsize;
for (int i = 0; i < hsize; i++) {
int val = Math.round((int) (cvGetReal1D(hist.bins(), i) * histimg.height() / 255.));
CvScalar color = hsv2rgb(i * 180.f / hsize);
cvRectangle(histimg, cvPoint(i * bin_w, histimg.height()), cvPoint((i + 1) * bin_w, histimg.height() - val), color, -1, 8, 0);
}
}
cvCalcBackProject(hueArray, backproject, hist);
cvAnd(backproject, mask, backproject, null);
cvCamShift(backproject, track_window, cvTermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1), track_comp, track_box);
track_window = track_comp.rect();
// if (track_window.width()*track_window.height()<=1)
// {
// int cols = backproject.width(), rows = backproject.height(), r = (Math.min(cols, rows)+5)/6;
// track_window = cvRect(
// Math.max(track_window.x()-r,0),
// Math.max(track_window.y()-r,0),
// Math.min(track_window.x()+r,cols),
// Math.min(track_window.y()+r,rows));
// }
if (backproject_mode) {
cvCvtColor(backproject, image, CV_GRAY2BGR);
}
if (image.origin() == 0) {
track_box = track_box.angle(-track_box.angle());
cvEllipseBox(image, track_box, cvScalar(0, 0, 255, 0), 3, CV_AA, 0);
}
}
} else if (track_object < 0) {
paused = false;
}
if (select_object && selection.width() > 0 && selection.height() > 0) {
cvSetImageROI(image, selection);
cvXorS(image, cvScalarAll(255), image, null);
cvResetImageROI(image);
}
camshiftDemo.showImage(image);
histogram.showImage(histimg);
}
}
#Override
public void mouseClicked(MouseEvent e) {
// System.out.println("Mouse Clicked !");
}
#Override
public void mousePressed(MouseEvent e) {
this.onMouse(MOUSE_PRESSED, e.getX(), e.getY());
System.out.println("Mouse Pressed !");
System.out.println("\t e.getX(): "+e.getX());
System.out.println("\t e.getY(): "+e.getY());
}
#Override
public void mouseReleased(MouseEvent e) {
this.onMouse(MOUSE_RELEASED, e.getX(), e.getY());
System.out.println("Mouse Released !");
System.out.println("\t e.getX(): "+e.getX());
System.out.println("\t e.getY(): "+e.getY());
}
#Override
public void mouseEntered(MouseEvent e) {
// System.out.println("Mouse Entered !");
}
#Override
public void mouseExited(MouseEvent e) {
// System.out.println("Mouse Exited !");
}
}
Everytime I run the application, and after I select my object to track, the JVM crashes when getting to the line containing cvCalcHist.
Can anyone please tell me what's wrong with my code. I've been on it since days and I can't figure out the problem with it :(
Thank you very much.