How can I restart synchronous reading from webcamera every time it fails? - ms-media-foundation

I am trying to read from a web camera in synchronous mode using Microsoft Media Foundation:
#include <iostream>
#include <vector>
#include <sstream>
#include <atlbase.h>
#include <mfidl.h>
#include <Mfapi.h>
#include <Mfreadwrite.h>
#pragma comment(lib,"Mfplat.lib")
#pragma comment(lib,"Mf.lib")
#pragma comment(lib,"Mfreadwrite.lib")
#pragma comment(lib,"mfuuid.lib")
#pragma comment(lib,"shlwapi.lib")
int main()
{
HRESULT hr;
DWORD dwCoInit = COINIT_MULTITHREADED;
hr = CoInitializeEx(NULL, dwCoInit);
if (FAILED(hr))
return -1;
hr = MFStartup(MF_VERSION);
if (FAILED(hr))
return -2;
CComBSTR name("M034-WDR");
CComPtr<IMFMediaSource> pMediaSource;
CComPtr<IMFSourceReader> pSourceReader;
CComPtr<IMFMediaType> pMediaType;
// Find all camera sources:
CComPtr<IMFAttributes> pAttributes;
MFCreateAttributes(&pAttributes, 1);
pAttributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID); // camera source
CComHeapPtr<IMFActivate*> ppActivates;
UINT32 nActivateCount = 0;
MFEnumDeviceSources(pAttributes, &ppActivates, &nActivateCount);
// Loop over the camera sources, find the one with correct name:
CComPtr<IMFActivate> pActivate;
for (UINT32 nIndex = 0; nIndex < nActivateCount; nIndex++) {
CComHeapPtr<WCHAR> name_i;
UINT32 length;
ppActivates[nIndex]->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME, &name_i, &length);
if (wcscmp(name_i, name) == 0) {
pActivate = ppActivates[nIndex];
}
reinterpret_cast<CComPtr<IMFActivate>&>(ppActivates[nIndex]).Release();
}
if (!pActivate) { // camera with name not found
return -3;
}
IMFMediaSource* source;
hr = pActivate->ActivateObject(__uuidof(IMFMediaSource), (VOID**)&source);
if (FAILED(hr))
return -4;
pMediaSource = source;
hr = MFCreateSourceReaderFromMediaSource(pMediaSource, NULL, &pSourceReader);
if (FAILED(hr))
return -5;
IMFMediaType* mediaType;
hr = pSourceReader->GetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, &mediaType);
if (FAILED(hr))
return -6;
unsigned int width, height;
hr = MFGetAttributeSize(mediaType, MF_MT_FRAME_SIZE, &width, &height);
if (FAILED(hr))
return -7;
while (true) {
CComPtr<IMFSample> pSample;
DWORD dwControlFlags = 0;
DWORD* pdwActualStreamIndex = NULL;
DWORD pdwStreamFlags;
LONGLONG timestamp;
OutputDebugStringA("MF:Before ReadSample()");
hr = pSourceReader->ReadSample(0,//dwStreamIndex,
dwControlFlags,
pdwActualStreamIndex,
&pdwStreamFlags,
&timestamp,
&pSample);
OutputDebugStringA("MF:After ReadSample()");
if (FAILED(hr)) {
std::stringstream ss;
ss << "MF:ReadSample() failed. hr = " << std::hex << hr << ".";
OutputDebugStringA(ss.str().c_str());
}
if (pSample == NULL) {
OutputDebugStringA("MF:ReadSample(): got null sample.");
}
}
}
The code above works fine on my laptop with one such webcam.
On another PC with another such webcam however; it runs fine for ca 1 minute before it starts to fail:
00000920 54.00323105 [7008] MF:Before ReadSample()
00000921 54.00324631 [7008] MF:After ReadSample()
00000922 54.00325012 [7008] MF:ReadSample() failed. hr = c00d3ea2.
00000923 54.00326538 [7008] MF:ReadSample(): got null sample.
Looking up the error code:
C:\temp>Err_6.4.5.exe c00d3ea2
# for hex 0xc00d3ea2 / decimal -1072873822
MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED mferror.h
# The video recording device is no longer present.%0
Anyways the Microsoft Camera application also experiences problems. Every minute ca. the image goes black for 1 sec ca. before the camera appears to be restarted.
How can I restart my code like Microsoft Camera does?

Related

confusing pointer error while implementing linked list

#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#define MALLOC(p,s) {\
if (!((p) = malloc(s))) { \
fprintf(stderr, "insufficient memory");\
exit(EXIT_FAILURE);\
}\
}
#define IS_EMPTY(first) (!first)
typedef struct listNode* listPointer;
typedef struct listNode {
int data;
listPointer link;
}listNode;
void printList(listPointer first);
int main(void)
{
int x;
int tmpData;
listPointer first = NULL;
listPointer tmpLink = NULL;
FILE* fp = NULL;
if (!(fp = fopen("in.txt", "r"))) {
fprintf(stderr, "cannot open the file");
exit(EXIT_FAILURE);
}
while (!feof(fp)) {
fscanf(fp, "%d", &tmpData);
MALLOC(tmpLink, sizeof(listNode));
if (IS_EMPTY(first)) {
MALLOC(first, sizeof(listNode));
*tmpLink = *first;
}
tmpLink->data = tmpData;
tmpLink = tmpLink->link;
}
printList(first);
}
void printList(listPointer first)
{
for (; first; first = first->link) {
printf("%d ", first->data);
}
printf("\n");
}
We know that we can implement the insert function.
But I'm really curious about why this doesn't work.
What "first" refers to and what "tmpLink" refers to is the same
After implementing the link list while updating tmpLink,
I'm going to use "first" to print later.
I've spent almost a day just thinking about this, and I've tried debugging it, but I don't know why.

How to use Intel realsense camera's (ZR300) with pre-made video or image instead live capture

I want to use pre-made files instead of live capture in the following program to track the person.
what is realsense SDK API used to load pre-made files and catch the frame by frame?
Is it possible to use to detect/track person any general video/image files which captured using any other camera's ?
Example Program:
Example Source Link
Source
#include <thread>
#include <iostream>
#include <signal.h>
#include "version.h"
#include "pt_utils.hpp"
#include "pt_console_display.hpp"
#include "pt_web_display.hpp"
#include "or_console_display.hpp"
#include "or_web_display.hpp"
using namespace std;
using namespace rs::core;
using namespace rs::object_recognition;
// Version number of the samples
extern constexpr auto rs_sample_version = concat("VERSION: ",RS_SAMPLE_VERSION_STR);
// Doing the OR processing for a frame can take longer than the frame interval, so we
// keep track of whether or not we are still processing the last frame.
bool is_or_processing_frame = false;
unique_ptr<web_display::pt_web_display> pt_web_view;
unique_ptr<web_display::or_web_display> or_web_view;
unique_ptr<console_display::pt_console_display> pt_console_view;
unique_ptr<console_display::or_console_display> or_console_view;
void processing_OR(correlated_sample_set or_sample_set, or_video_module_impl* impl, or_data_interface* or_data,
or_configuration_interface* or_configuration)
{
rs::core::status st;
// Declare data structure and size for results
rs::object_recognition::localization_data* localization_data = nullptr;
//Run object localization processing
st = impl->process_sample_set(or_sample_set);
if (st != rs::core::status_no_error)
{
is_or_processing_frame = false;
return;
}
// Retrieve recognition data from the or_data object
int array_size = 0;
st = or_data->query_localization_result(&localization_data, array_size);
if (st != rs::core::status_no_error)
{
is_or_processing_frame = false;
return;
}
//Send OR data to ui
if (localization_data && array_size != 0)
{
or_console_view->on_object_localization_data(localization_data, array_size, or_configuration);
or_web_view->on_object_localization_data(localization_data, array_size, or_configuration);
}
is_or_processing_frame = false;
}
int main(int argc,char* argv[])
{
rs::core::status st;
pt_utils pt_utils;
rs::core::image_info colorInfo,depthInfo;
rs::core::video_module_interface::actual_module_config actualModuleConfig;
rs::person_tracking::person_tracking_video_module_interface* ptModule = nullptr;
rs::object_recognition::or_video_module_impl impl;
rs::object_recognition::or_data_interface* or_data = nullptr;
rs::object_recognition::or_configuration_interface* or_configuration = nullptr;
cout << endl << "Initializing Camera, Object Recognition and Person Tracking modules" << endl;
if(pt_utils.init_camera(colorInfo,depthInfo,actualModuleConfig,impl,&or_data,&or_configuration) != rs::core::status_no_error)
{
cerr << "Error: Device is null." << endl << "Please connect a RealSense device and restart the application" << endl;
return -1;
}
pt_utils.init_person_tracking(&ptModule);
//Person Tracking Configuration. Set tracking mode to 0
ptModule->QueryConfiguration()->QueryTracking()->Enable();
ptModule->QueryConfiguration()->QueryTracking()->SetTrackingMode((Intel::RealSense::PersonTracking::PersonTrackingConfiguration::TrackingConfiguration::TrackingMode)0);
if(ptModule->set_module_config(actualModuleConfig) != rs::core::status_no_error)
{
cerr<<"error : failed to set the enabled module configuration" << endl;
return -1;
}
//Object Recognition Configuration
//Set mode to localization
or_configuration->set_recognition_mode(rs::object_recognition::recognition_mode::LOCALIZATION);
//Set the localization mechnizm to use CNN
or_configuration->set_localization_mechanism(rs::object_recognition::localization_mechanism::CNN);
//Ignore all objects under 0.7 probabilty (confidence)
or_configuration->set_recognition_confidence(0.7);
//Enabling object center feature
or_configuration->enable_object_center_estimation(true);
st = or_configuration->apply_changes();
if (st != rs::core::status_no_error)
return st;
//Launch GUI
string sample_name = argv[0];
// Create console view
pt_console_view = move(console_display::make_console_pt_display());
or_console_view = move(console_display::make_console_or_display());
// Create and start remote(Web) view
or_web_view = move(web_display::make_or_web_display(sample_name, 8000, true));
pt_web_view = move(web_display::make_pt_web_display(sample_name, 8000, true));
cout << endl << "-------- Press Esc key to exit --------" << endl << endl;
while (!pt_utils.user_request_exit())
{
//Get next frame
rs::core::correlated_sample_set* sample_set = pt_utils.get_sample_set(colorInfo,depthInfo);
rs::core::correlated_sample_set* sample_set_pt = pt_utils.get_sample_set(colorInfo,depthInfo);
//Increment reference count of images at sample set
for (int i = 0; i < static_cast<uint8_t>(rs::core::stream_type::max); ++i)
{
if (sample_set_pt->images[i] != nullptr)
{
sample_set_pt->images[i]->add_ref();
}
}
//Draw Color frames
auto colorImage = (*sample_set)[rs::core::stream_type::color];
pt_web_view->on_rgb_frame(10, colorImage->query_info().width, colorImage->query_info().height, colorImage->query_data());
//Run OR in a separate thread. Update GUI with the result
if (!is_or_processing_frame) // If we aren't already processing or for a frame:
{
is_or_processing_frame = true;
std::thread recognition_thread(processing_OR, *sample_set,
&impl, or_data, or_configuration);
recognition_thread.detach();
}
//Run Person Tracking
if (ptModule->process_sample_set(*sample_set_pt) != rs::core::status_no_error)
{
cerr << "error : failed to process sample" << endl;
continue;
}
//Update GUI with PT result
pt_console_view->on_person_info_update(ptModule);
pt_web_view->on_PT_tracking_update(ptModule);
}
pt_utils.stop_camera();
actualModuleConfig.projection->release();
return 0;
}
After installing the Realsense SKD, check the realsense_playback_device_sample for how to load the RSSDK capture file.
The short answer is not really. Beside the images that are captured from the other camera, you also need to supply the camera intrinsic and extrinsic settings in order to calculate the depth of and object and call the person tracking module.

QAudioOutput buffer underflow

Getting a message "Got a buffer underflow!" after each write in this simple program.
Beep.hpp:
#pragma once
#include <QTimer>
#include <QAudioOutput>
class Beep: public QObject
{
Q_OBJECT
public:
explicit Beep();
virtual ~Beep();
void onTimer();
private:
QAudioOutput m_out;
QIODevice *m_outDev;
QTimer m_timer;
};
Beep.cpp:
#include "Beep.hpp"
int ms = 100;
const QAudioFormat defaultAudioFormat = []()
{
QAudioFormat format;
format.setSampleRate(8000);
format.setChannelCount(1);
format.setSampleSize(16);
format.setCodec("audio/pcm");
format.setByteOrder(QAudioFormat::LittleEndian);
format.setSampleType(QAudioFormat::SignedInt);
return format;
}();
Beep::Beep() :
m_out(defaultAudioFormat),
m_outDev()
{
m_out.setBufferSize(16 * ms);
m_outDev = m_out.start();
QObject::connect(&m_timer, &QTimer::timeout, this, &Beep::onTimer);
m_timer.setSingleShot(false);
m_timer.start(ms);
}
Beep::~Beep()
{
}
void Beep::onTimer()
{
std::vector<uint8_t> samples(16 * ms);
m_outDev->write((char*) &samples.front(), samples.size());
}
main.cpp:
#include <QCoreApplication>
#include "Beep.hpp"
int main(int argc, char *argv[])
{
QCoreApplication app(argc, argv);
Beep beep;
return app.exec();
}
This test program is just writing buffers with zeros. With real data there are cracking sounds.
Writing more data or changing timings makes it worse. What's wrong with this code?
Using a Timer is the wrong way to do it.
Use the notify() signal
void AudioManager::init_audio(AudioManager *mgr) {
if (mgr->stream_id == -1) return;
mgr->audio_format.setSampleRate(mgr->context->time_base.den);
mgr->audio_format.setSampleSize(16);
mgr->audio_format.setChannelCount(2);
mgr->audio_format.setCodec("audio/pcm");
mgr->audio_format.setSampleType(QAudioFormat::SignedInt);
QAudioDeviceInfo info(QAudioDeviceInfo::defaultOutputDevice());
if (!info.isFormatSupported(mgr->audio_format)) {
mgr->audio_format = info.nearestFormat(mgr->audio_format);
}
mgr->audio_out = new QAudioOutput(mgr->audio_format, nullptr);
mgr->audio_out->setNotifyInterval(15);
mgr->audio_out->setBufferSize(mgr->context->time_base.den * 4); // 1 second worth of stereo data
connect(mgr->audio_out, SIGNAL(notify()), mgr, SLOT(audio_out_notify()));
connect(mgr->audio_out, SIGNAL(stateChanged(QAudio::State)), mgr, SLOT(audio_out_state_changed(QAudio::State)));
qreal volume_out = (qreal)parent->volume / 100.0f;
mgr->audio_out->setVolume(volume_out);
mgr->audio_out_device = mgr->audio_out->start();
}
This will be called when the audio playback requires more data
void AudioManager::audio_out_notify() {
qDebug() << "Audio notify";
check_audio_playback();
}
Most of the below code will be irrelevant but it is also called is audio has stopped playing.
void AudioManager::check_audio_playback() {
if (stream_id == -1) return;
pthread_mutex_lock(&audio_mutex);
if (!audio_out->state() == QAudio::State::IdleState) {
pthread_mutex_unlock(&audio_mutex);
return;
}
if (parent->pts_start_time < 0.0) {
if (parent->Video.stream_id == -1 && decode_pos > 65) { // start playback
parent->pts_start_time = buffers[0].frame_time;
parent->sys_start_time = (double)parent->timer.elapsed() / 1000.0;
qDebug() << "Audio playback started";
} else {
pthread_mutex_unlock(&audio_mutex);
return;
}
}
if (playback_pos == decode_pos) {
pthread_mutex_unlock(&audio_mutex);
return;
}
AudioBuffer *buffer = nullptr;
double current_sys_time = ((double)parent->timer.elapsed() / 1000.0) - parent->sys_start_time;
bool bounds = false;
int skipped = 0;
while (!bounds) {
if (playback_pos == decode_pos) bounds = true;
else {
AudioBuffer *temp_buffer = &buffers[playback_pos];
double temp_time = temp_buffer->frame_time - parent->pts_start_time;
if (temp_time < current_sys_time ) {
if (buffer) {
buffer->used = false;
skipped++;
}
buffer = temp_buffer;
playback_pos++; playback_pos %= MAX_AUD_BUFFERS;
} else {
bounds = true;
}
}
}
if (skipped > 0) qDebug("Skipped %d audio buffers on playback", skipped);
if (buffer) {
audio_out_device->write((const char *)buffer->data, buffer->buffer_size);
buffer->used = false;
}
pthread_mutex_unlock(&audio_mutex);
}
The example on the Qt website wasn't that obvious http://qt.apidoc.info/5.1.1/qtmultimedia/audiooutput.html at first but when I put it in to test it wasn't too bad.
The reason was that the source of audio data wasn't a "production-quality module" (it's a dummy testing class): the timer was drifting because its real interval was 10ms plus the processing time.
Other observations:
make QAudioOutput::setBufferSize() bigger
do QAudioInput::read() and QAudioOutput::write() in chunks with size that matches QAudioInput::periodSize() and QAudioOutput::periodSize()

Trouble with creating an empty file using C programming language in UNIX environment

I have recently started programming in UNIX environment. I need to write a program which creates an empty file with name and size given in the terminal using this commands
gcc foo.c -o foo.o
./foo.o result.txt 1000
Here result.txt means the name of the newly created file, and 1000 means the size of the file in bytes.
I know for sure that lseek function moves the file offset, but the trouble is that whenever I run the program it creates a file with a given name, however the size of the file is 0.
Here is the code of my small program.
#include <unistd.h>
#include <stdio.h>
#include <fcntl.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/param.h>
#include <sys/stat.h>
int main(int argc, char **argv)
{
int fd;
char *file_name;
off_t bytes;
mode_t mode;
if (argc < 3)
{
perror("There is not enough command-line arguments.");
//return 1;
}
file_name = argv[1];
bytes = atoi(argv[2]);
mode = S_IWUSR | S_IWGRP | S_IWOTH;
if ((fd = creat(file_name, mode)) < 0)
{
perror("File creation error.");
//return 1;
}
if (lseek(fd, bytes, SEEK_SET) == -1)
{
perror("Lseek function error.");
//return 1;
}
close(fd);
return 0;
}
If you aren't allowed to use any other functions to assist in creating a "blank" text file, why not change your file mode on creat() then loop-and-write:
int fd = creat(file_name, 0666);
for (int i=0; i < bytes; i++) {
int wbytes = write(fd, " ", 1);
if (wbytes < 0) {
perror("write error")
return 1;
}
}
You'll want to have some additional checks here but, that would be the general idea.
I don't know whats acceptable in your situation but, possibly adding just the write() call after lseek() even:
// XXX edit to include write
if ((fd = creat(file_name, 0666)) < 0) {
perror("File creation error");
//return 1;
}
// XXX seek to bytes - 1
if (lseek(fd, bytes - 1, SEEK_SET) == -1) {
perror("lseek() error");
//return 1;
}
// add this call to write a single byte # position set by lseek
if (write(fd, " ", 1) == -1) {
perror("write() error");
//return 1;
}
close(fd);
return 0;

Http Streaming from ffmpeg, How to get a sequenced packet?

I am trying to make http streaming program.
So I follow this code at this.
However, when i decode, only one frame is decoded.
I think I need call back function.
Do you know how to make a call back function?
I know 'asf' packet's call back function is like int read_data(void *opaque, char *buf, int buf_size)
But the other formats(mp3, ogg, aac, ..) doesn't work..
Please help me.
Any advice or comment are very appreciated.
#include <stdio.h>
#include <stdlib.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavdevice/avdevice.h>
int main(int argc, char **argv)
{
static AVInputFormat *file_iformat;
static AVFormatContext *pFormatCtx;
AVFormatParameters params;
AVCodecContext *pCodecCtx;
AVCodec *pCodec;
const char url[] = "http://listen.radionomy.com/feelingfloyd";
avcodec_register_all();
avdevice_register_all();
av_register_all();
av_log_set_level(AV_LOG_VERBOSE);
file_iformat = av_find_input_format("mp3"); /* mp3 demuxer */
if (!file_iformat)
{
fprintf(stderr, "Unknown input format: %s\n", &url[0]);
exit(1);
}
//file_iformat->flags |= AVFMT_NOFILE; /* ??? */
params.prealloced_context = 0;
if (av_open_input_file(&pFormatCtx, &url[0], file_iformat, 0, &params) != 0)
{
fprintf(stderr, "err 1\n");
exit(2);
}
/* poulates AVFormatContex structure */
if (av_find_stream_info(pFormatCtx) < 0)
{
fprintf(stderr, "err 2\n");
}
/* sanity check (1 stream) */
if (pFormatCtx->nb_streams != 1 &&
pFormatCtx->streams[0]->codec->codec_type != AVMEDIA_TYPE_AUDIO)
{
fprintf(stderr, "err 3\n");
}
pCodecCtx = pFormatCtx->streams[0]->codec;
/* find decoder for input audio stream */
pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if (pCodec == NULL)
{
fprintf(stderr, "err 4: unsupported codec\n");
}
if (pCodec->capabilities & CODEC_CAP_TRUNCATED)
pCodecCtx->flags |= CODEC_FLAG_TRUNCATED;
if (avcodec_open(pCodecCtx, pCodec) < 0)
{
fprintf(stderr, "err 5\n");
}
{
uint8_t *pAudioBuffer;
AVPacket pkt;
int ret;
int data_size = 2 * AVCODEC_MAX_AUDIO_FRAME_SIZE;
av_init_packet(&pkt);
//pkt.data=NULL;
//pkt.size=0;
//pkt.stream_index = 0;
pAudioBuffer = av_malloc(data_size * sizeof(int16_t));
while (av_read_frame(pFormatCtx, &pkt) == 0) {
//data_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
ret = avcodec_decode_audio3(pFormatCtx->streams[pkt.stream_index]->codec,
(int16_t *)pAudioBuffer, &data_size, &pkt);
/* got an error (-32) here */
if (ret < 0) {
av_strerror(ret, (char *)pAudioBuffer, data_size);
fprintf(stderr, "err 6 (%s)\n", pAudioBuffer);
break;
}
printf("size=%d, stream_index=%d |ret=%d data_size=%d\n",
pkt.size, pkt.stream_index, ret, data_size);
av_free_packet(&pkt);
}
av_free(pAudioBuffer);
}
avcodec_close(pCodecCtx);
av_close_input_file(pFormatCtx);
return 0;
}
I figure out this problem by using av_open_input_file.
I got a this problem when I made a iphone app that play http audio streaming. And the above code didn't work. only played just some of audio frame. it also means there are so many buffering.
However After using iphone audio callback function and large audio buffer, it works fine.
Those who are curious about the final code send me a messege.

Resources