Implementation
Now, let's take a stab at implementing everything we defined in the Header.
Let's start with the constructor. A lot of this is initializing the Luxonis pipelines, but we're also setting up the Smart Capture Client.
LuxonisDemo::LuxonisDemo(const string &trigger_file_path, const string &blob_path) {
// ------------ Luxonis Pipeline Prep
// Define sources and outputs
auto camRgb = pipeline.create<dai::node::ColorCamera>();
auto nn = pipeline.create<dai::node::MobileNetDetectionNetwork>();
auto xoutRgb = pipeline.create<dai::node::XLinkOut>();
auto nnOut = pipeline.create<dai::node::XLinkOut>();
auto nnNetworkOut = pipeline.create<dai::node::XLinkOut>();
xoutRgb->setStreamName("rgb");
nnOut->setStreamName("nn");
nnNetworkOut->setStreamName("nnNetwork");
// Properties
camRgb->setPreviewSize(300, 300); // NN input
camRgb->setInterleaved(false);
camRgb->setFps(40);
// Define a neural network that will make predictions based on the source frames
nn->setConfidenceThreshold(0.5);
nn->setBlobPath(blob_path);
nn->setNumInferenceThreads(2);
nn->input.setBlocking(false);
// Linking
if (syncNN) {
nn->passthrough.link(xoutRgb->input);
} else {
camRgb->preview.link(xoutRgb->input);
}
camRgb->preview.link(nn->input);
nn->out.link(nnOut->input);
nn->outNetwork.link(nnNetworkOut->input);
// Connect to device and start pipeline
device = std::make_shared<dai::Device>(pipeline);
// Output queues will be used to get the grayscale / depth frames and nn data from the outputs defined above
qRgb = device->getOutputQueue("rgb", 4, false);
qDet = device->getOutputQueue("nn", 4, false);
qNN = device->getOutputQueue("nnNetwork", 4, false);
// ------------ Luxonis Pipeline Prep Complete
// ------------ Smart Capture Setup
this->client = Client::make_client(trigger_file_path).second;
this->trigger_file_path = trigger_file_path;
// This means we're logging the Smart Capture Log Output to the
// specified file.
Logger::log_location = LOGLOCATION::FILE;
Logger::log_file = "/home/scale/scale_ws/smart-capture-sdk/sc_log.log";
// The following would set it back to Console output for easier debugging.
// Logger::log_location = LOGLOCATION::CONSOLE;
// ------------ Smart Capture Setup Complete
}And now let's put together the run_demo method. Much of this is pipeline management code for interfacing with the Luxonis and processing the frames and object detection output. The Smart Capture specific code is highlighted with comments.
void LuxonisDemo::run_demo(int frame_count) {
bool printOutputLayersOnce = true;
int frames = 0;
while (frames < frame_count) {
// ----------- Luxonis Pipeline Code
std::shared_ptr <dai::ImgFrame> inRgb;
std::shared_ptr <dai::ImgDetections> inDet;
std::shared_ptr <dai::NNData> inNN;
if (syncNN) {
inRgb = qRgb->get<dai::ImgFrame>();
inDet = qDet->get<dai::ImgDetections>();
inNN = qNN->get<dai::NNData>();
} else {
inRgb = qRgb->tryGet<dai::ImgFrame>();
inDet = qDet->tryGet<dai::ImgDetections>();
inNN = qNN->tryGet<dai::NNData>();
}
counter++;
auto currentTime = system_clock::now();
auto elapsed = duration_cast < duration < float >> (currentTime - startTime);
if (elapsed > seconds(1)) {
fps = counter / elapsed.count();
counter = 0;
startTime = currentTime;
}
elapsed = duration_cast < duration < float >> (currentTime - reload_time);
if (elapsed > seconds(10)) {
client->reload(trigger_file_path, "test");
reload_time = currentTime;
}
if (inRgb) {
frame = inRgb->getCvFrame();
std::stringstream fpsStr;
fpsStr << "NN fps: " << std::fixed << std::setprecision(2) << fps;
cv::putText(frame,
fpsStr.str(),
cv::Point(2, inRgb->getHeight() - 4),
cv::FONT_HERSHEY_TRIPLEX,
0.4,
cv::Scalar(255, 255, 255));
}
if (inDet) {
detections = inDet->detections;
}
if (printOutputLayersOnce && inNN) {
std::cout << "Output layer names: ";
for (const auto &ten: inNN->getAllLayerNames()) {
std::cout << ten << ", ";
}
std::cout << std::endl;
printOutputLayersOnce = false;
}
// ------------ Luxonis Pipeline Code Complete
// ------------ Smart Capture State Update and Evaluation
update_detection_state();
auto results = client->evaluate_all();
elapsed = duration_cast < duration < float >> (currentTime - print_time);
if (elapsed > seconds(5)) {
for (auto result: results) {
printf("Counter: %u, Trigger id: %s. Result: %u. Evaluated: %u\n",
counter,
result->trigger_id.c_str(),
result->result,
result->evaluated);
}
print_time = currentTime;
}
// ------------ Smart Capture Code Complete
// ------------ OpenCV Detection Display Code
if (!frame.empty()) {
show("video", frame, detections);
}
int key = cv::waitKey(1);
if (key == 'q' || key == 'Q') {
return;
}
++frames;
// ------------ OpenCV Detection Display Code Complete
}
}
Next, let's define that update_detection_state() method used in run_demo()
void LuxonisDemo::update_detection_state() {
client->update_state<int>("model_objects_detected", detections.size());
float min_conf = 100;
for (auto &detection: detections) {
if (detection.confidence < min_conf) { min_conf = detection.confidence; }
}
client->update_state<float>("model_confidence", min_conf);
}For completeness sake, let's put together the helper show() function.
void LuxonisDemo::show(std::string name,
cv::Mat frame,
std::vector <dai::ImgDetection> &detections) {
// Add bounding boxes and text to the frame and show it to the user
auto color = cv::Scalar(255, 19, 20);
// nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height
for (auto &detection: detections) {
int x1 = detection.xmin * frame.cols;
int y1 = detection.ymin * frame.rows;
int x2 = detection.xmax * frame.cols;
int y2 = detection.ymax * frame.rows;
uint32_t labelIndex = detection.label;
std::string labelStr = to_string(labelIndex);
if (labelIndex < labelMap.size()) {
labelStr = labelMap[labelIndex];
}
cv::putText(frame, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
std::stringstream confStr;
confStr << std::fixed << std::setprecision(2) << detection.confidence * 100;
cv::putText(frame, confStr.str(), cv::Point(x1 + 10, y1 + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
cv::rectangle(frame, cv::Rect(cv::Point(x1, y1), cv::Point(x2, y2)), color, cv::FONT_HERSHEY_SIMPLEX);
}
// Show the frame
cv::imshow(name, frame);
}The final implementation, in its entirety below. Note the header import and namespacing in the full file.
#include <luxonis_orin.hpp>
using namespace std;
namespace SmartCapture {
LuxonisDemo::LuxonisDemo(const string &trigger_file_path,
const string &blob_path) {
// Define sources and outputs
auto camRgb = pipeline.create<dai::node::ColorCamera>();
auto nn = pipeline.create<dai::node::MobileNetDetectionNetwork>();
auto xoutRgb = pipeline.create<dai::node::XLinkOut>();
auto nnOut = pipeline.create<dai::node::XLinkOut>();
auto nnNetworkOut = pipeline.create<dai::node::XLinkOut>();
xoutRgb->setStreamName("rgb");
nnOut->setStreamName("nn");
nnNetworkOut->setStreamName("nnNetwork");
// Properties
camRgb->setPreviewSize(300, 300); // NN input
camRgb->setInterleaved(false);
camRgb->setFps(40);
// Define a neural network that will make predictions based
// on the source frames
nn->setConfidenceThreshold(0.5);
nn->setBlobPath(blob_path);
nn->setNumInferenceThreads(2);
nn->input.setBlocking(false);
// Linking
if (syncNN) {
nn->passthrough.link(xoutRgb->input);
} else {
camRgb->preview.link(xoutRgb->input);
}
camRgb->preview.link(nn->input);
nn->out.link(nnOut->input);
nn->outNetwork.link(nnNetworkOut->input);
// Connect to device and start pipeline
device = std::make_shared<dai::Device>(pipeline);
// Output queues will be used to get the grayscale /
// depth frames and nn data from the outputs defined above
qRgb = device->getOutputQueue("rgb", 4, false);
qDet = device->getOutputQueue("nn", 4, false);
qNN = device->getOutputQueue("nnNetwork", 4, false);
this->client = Client::make_client(trigger_file_path).second;
this->trigger_file_path = trigger_file_path;
Logger::log_location = LOGLOCATION::FILE;
Logger::log_file = "/home/scale/scale_ws/smart-capture-sdk/sc_log.log";
}
void LuxonisDemo::show(std::string name,
cv::Mat frame,
std::vector <dai::ImgDetection> &detections) {
// Add bounding boxes and text to the frame and show it to the user
auto color = cv::Scalar(255, 19, 20);
// nn data, being the bounding box locations,
// are in <0..1> range - they need to be normalized with frame width/height
for (auto &detection: detections) {
int x1 = detection.xmin * frame.cols;
int y1 = detection.ymin * frame.rows;
int x2 = detection.xmax * frame.cols;
int y2 = detection.ymax * frame.rows;
uint32_t labelIndex = detection.label;
std::string labelStr = to_string(labelIndex);
if (labelIndex < labelMap.size()) {
labelStr = labelMap[labelIndex];
}
cv::putText(frame,
labelStr,
cv::Point(x1 + 10, y1 + 20),
cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
std::stringstream confStr;
confStr << std::fixed <<
std::setprecision(2) << detection.confidence * 100;
cv::putText(frame,
confStr.str(),
cv::Point(x1 + 10, y1 + 40),
cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
cv::rectangle(frame,
cv::Rect(cv::Point(x1, y1),
cv::Point(x2, y2)),
color, cv::FONT_HERSHEY_SIMPLEX);
}
// Show the frame
cv::imshow(name, frame);
}
void LuxonisDemo::update_detection_state() {
client->update_state<int>("model_objects_detected", detections.size());
float min_conf = 100;
for (auto &detection: detections) {
if (detection.confidence < min_conf) {
min_conf = detection.confidence;
}
}
client->update_state<float>("model_confidence", min_conf);
}
void LuxonisDemo::run_demo(int frame_count) {
bool printOutputLayersOnce = true;
int frames = 0;
while (frames < frame_count) {
std::shared_ptr <dai::ImgFrame> inRgb;
std::shared_ptr <dai::ImgDetections> inDet;
std::shared_ptr <dai::NNData> inNN;
if (syncNN) {
inRgb = qRgb->get<dai::ImgFrame>();
inDet = qDet->get<dai::ImgDetections>();
inNN = qNN->get<dai::NNData>();
} else {
inRgb = qRgb->tryGet<dai::ImgFrame>();
inDet = qDet->tryGet<dai::ImgDetections>();
inNN = qNN->tryGet<dai::NNData>();
}
counter++;
auto currentTime = system_clock::now();
auto elapsed = duration_cast < duration < float >> (currentTime - startTime);
if (elapsed > seconds(1)) {
fps = counter / elapsed.count();
counter = 0;
startTime = currentTime;
}
elapsed = duration_cast < duration < float >> (currentTime - reload_time);
if (elapsed > seconds(10)) {
client->reload(trigger_file_path, "test");
reload_time = currentTime;
}
if (inRgb) {
frame = inRgb->getCvFrame();
std::stringstream fpsStr;
fpsStr << "NN fps: " << std::fixed << std::setprecision(2) << fps;
cv::putText(frame,
fpsStr.str(),
cv::Point(2, inRgb->getHeight() - 4),
cv::FONT_HERSHEY_TRIPLEX,
0.4,
cv::Scalar(255, 255, 255));
}
if (inDet) {
detections = inDet->detections;
}
if (printOutputLayersOnce && inNN) {
std::cout << "Output layer names: ";
for (const auto &ten: inNN->getAllLayerNames()) {
std::cout << ten << ", ";
}
std::cout << std::endl;
printOutputLayersOnce = false;
}
// SC stuff
update_detection_state();
auto results = client->evaluate_all();
elapsed = duration_cast < duration < float >> (currentTime - print_time);
if (elapsed > seconds(5)) {
for (auto result: results) {
printf("Counter: %u, Trigger id: %s. Result: %u. Evaluated: %u\n",
counter,
result->trigger_id.c_str(),
result->result,
result->evaluated);
}
print_time = currentTime;
}
if (!frame.empty()) {
show("video", frame, detections);
}
int key = cv::waitKey(1);
if (key == 'q' || key == 'Q') {
return;
}
++frames;
}
}
}
Finally, in order to actually run the tutorial in a main file, we have the following in main.cpp adjacent to the implementation in luxonis_orin.cpp
#include <luxonis_orin.hpp>
int main(int argc, char **argv) {
printf("Running Luxonis Demo\n");
const string trigger_file = "/home/scale/scale_ws/smart-capture-sdk/luxonis_orin.json";
const string blob_path = "/home/scale/.hunter/_Base/PrivateData/4f4506726e3083981064938a0faaf9af6180d2c6/4f45067/raw/mobilenet-ssd_openvino_2021.4_6shave.blob";
SmartCapture::LuxonisDemo demo(trigger_file, blob_path);
demo.run_demo(200);
}
Updated 10 months ago
Did this page help you?