Sequential off-screen rendering / screen capture without windowing system using OpenSceneGraph - c++

Question

I am working currently on an off-screen renderer so that I can do Mutual Information Registration for real-world scenes. I use OpenSceneGraph to cope with the large data and automatic loading. I am having trouble getting a framebuffer capture within a sequential, single-threaded program.
Well, I have this class (header):
#include <osg/ref_ptr>
#include <osg/Array>
#include <osg/ImageUtils>
#include <osgGA/StateSetManipulator>
#include <osgViewer/Viewer>
#include <osg/GraphicsContext>
#include <osg/Texture2D>
#include <osg/FrameBufferObject>
#include <osgDB/WriteFile>
#include <osg/Referenced>
#include <osg/Vec3>
#include <osg/Image>
#include <osg/State>
#include <string>
#include <chrono>
#include <thread>
#include <assert.h>
#include "ImagingPrimitives.h"
class BoundRenderScene {
public:
BoundRenderScene();
virtual ~BoundRenderScene();
void NextFrame(void);
inline OpenThreads::Mutex* GetMutexObject(void) { return &_mutex; }
inline osg::Image* GetFrame(void)
{
OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_mutex);
return _frame.get();
}
inline void GetFrame(osg::Image* img)
{
OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_mutex);
if(_frame.valid() && (img!=NULL) && img->valid())
{
glReadBuffer(GL_BACK);
img->readPixels(0,0,_camera_configuration->GetSX(),_camera_configuration->GetSY(), GL_RGB,GL_UNSIGNED_BYTE);
uint w = img->s(), h = img->t(), d = img->r(), c = uint(img->getPixelSizeInBits()/8);
/*
* bare testing write op
* osgDB::writeImageFile(const_cast<const osg::Image&>(*img), "/tmp/testimg.png");
*/
}
}
inline void SetCameraConfiguration(CameraConfiguration* configuration) { _camera_configuration = configuration; }
inline void SetCameraMatrix(osg::Matrixd camera_matrix) { _camera_matrix = camera_matrix; }
inline void SetScene(osg::Node* scene) { _scene = scene; }
inline void Initialize(void) {
if(!_initialized)
_init();
else
_re_init();
}
protected:
osgViewer::Viewer _viewer;
osg::Matrixd _camera_matrix;
osg::ref_ptr<osg::Texture2D> _tex;
osg::ref_ptr<osg::FrameBufferObject> _fbo;
mutable osg::ref_ptr<osg::Image> _frame;
osg::ref_ptr<osg::Node> _scene;
osg::ref_ptr<osg::GraphicsContext::Traits> _traits;
osg::ref_ptr<osg::GraphicsContext> _gc;
CameraConfiguration* _camera_configuration;
SnapshotCallback* cb;
std::string _filepath;
private:
void _init(void);
void _re_init(void);
bool _initialized;
mutable OpenThreads::Mutex _mutex;
osg::Matrixd pre_transform;
osg::Matrixd transformation;
};
Also, because many examples within offscreen-rendering and for screen capture work with Post/FinalDrawCallaback's, I copied the callback structure from the "osgdistortion" example, but added the mutex for synchronisation:
struct SnapshotCallback : public osg::Camera::DrawCallback
{
public:
inline SnapshotCallback(OpenThreads::Mutex* mtx_obj, std::string filepath, int width, int height) : _filepath(filepath), _output_to_file(false), _mutex(mtx_obj)
{
_image = new osg::Image();
_image->allocateImage(width, height, 1, GL_RGB, GL_UNSIGNED_BYTE);
if(filepath!="")
_output_to_file = true;
}
inline virtual void operator() (osg::RenderInfo& renderInfo) const
{
OpenThreads::ScopedLock<OpenThreads::Mutex> lock(*_mutex);
osg::Camera* camera = renderInfo.getCurrentCamera();
osg::Viewport* viewport = camera ? camera->getViewport() : 0;
if(viewport && _image.valid())
{
glReadBuffer(GL_BACK);
_image->readPixels(int(viewport->x()),int(viewport->y()),int(viewport->width()),int(viewport->height()), GL_RGB, GL_UNSIGNED_BYTE);
if(_output_to_file)
{
osgDB::writeImageFile(*_image, _filepath);
}
}
}
inline virtual void operator() (const osg::Camera& camera) const
{
OpenThreads::ScopedLock<OpenThreads::Mutex> lock(*_mutex);
osg::Viewport* viewport = camera.getViewport();
if(viewport && _image.valid())
{
glReadBuffer(GL_BACK);
_image->readPixels(int(viewport->x()),int(viewport->y()),int(viewport->width()),int(viewport->height()), GL_RGB, GL_UNSIGNED_BYTE);
if(_output_to_file)
{
osgDB::writeImageFile(*_image, _filepath);
}
}
}
std::string _filepath;
bool _output_to_file;
mutable OpenThreads::Mutex* _mutex;
mutable osg::ref_ptr<osg::Image> _image;
};
I initialize and render the scene as follows:
#include "BoundRenderScene.h"
void BoundRenderScene::_init(void)
{
if(_camera!=NULL)
_viewer.setDone(true);
_traits->x = 0;
_traits->y = 0;
_traits->width = _camera_configuration->GetSX();
_traits->height = _camera_configuration->GetSY();
_traits->red = 8;
_traits->green = 8;
_traits->blue = 8;
_traits->alpha = 0;
_traits->depth = 24;
_traits->windowDecoration = false;
_traits->pbuffer = true;
_traits->doubleBuffer = true;
_traits->sharedContext = 0x0;
if(_gc.get()!=NULL)
{
bool release_success = _gc->releaseContext();
if(!release_success)
std::cerr << "Error releasing Graphics Context.";
}
_gc = osg::GraphicsContext::createGraphicsContext(_traits.get());
_viewer.getCamera()->setGraphicsContext(_gc.get());
_viewer.setThreadingModel(osgViewer::Viewer::SingleThreaded);
_viewer.setUpThreading();
_viewer.realize();
_frame->allocateImage(_camera_configuration->GetSX(), _camera_configuration->GetSY(), 1, GL_RGB, GL_UNSIGNED_BYTE);
_viewer.getCamera()->getOrCreateStateSet();
_viewer.getCamera()->setRenderTargetImplementation(osg::Camera::PIXEL_BUFFER);
cb = new SnapshotCallback(&_mutex,_filepath, _camera_configuration->GetSX(), _camera_configuration->GetSY());
//_viewer.getCamera()->setPostDrawCallback( cb );
//Clear colour "black" for representing "no information" => background elimination in natural image, pls.
_viewer.getCamera()->setClearColor(osg::Vec4f(0.25f, 0.25f, 0.25f, 1.0f));
_viewer.getCamera()->setClearMask(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT);
_viewer.getCamera()->setDrawBuffer(GL_BACK);
_viewer.getCamera()->setReadBuffer(GL_BACK);
_viewer.getCamera()->setViewport(0,0,_camera_configuration->GetSX(),_camera_configuration->GetSY());
_viewer.getCamera()->setProjectionMatrix(osg::Matrixd::perspective(osg::RadiansToDegrees(_camera_configuration->GetFoV()), _camera_configuration->GetAspectRatio(), 0.1, 150.0));
//looking in geo-coord system
_viewer.getCamera()->setViewMatrix(osg::Matrixd::lookAt(osg::Vec3d(0.0, 0.0, -1.0), osg::Vec3d(0.0, 0.0, 1.0), osg::Vec3d(0.0, 1.0, 0.0)));
_viewer.getCamera()->attach(osg::Camera::COLOR_BUFFER, _frame.get());
_viewer.getCamera()->setRenderTargetImplementation(osg::Camera::FRAME_BUFFER_OBJECT);
_tex->setTextureSize(_camera_configuration->GetSX(), _camera_configuration->GetSY());
_tex->setInternalFormat(GL_RGB);
_tex->setFilter(osg::Texture::MIN_FILTER, osg::Texture::LINEAR);
_tex->setFilter(osg::Texture::MAG_FILTER, osg::Texture::LINEAR);
_tex->setWrap(osg::Texture::WRAP_S, osg::Texture::CLAMP_TO_EDGE);
_tex->setWrap(osg::Texture::WRAP_T, osg::Texture::CLAMP_TO_EDGE);
_tex->setResizeNonPowerOfTwoHint(false);
_tex->setImage(0,_frame.get());
_fbo->setAttachment(osg::Camera::COLOR_BUFFER, osg::FrameBufferAttachment(_tex.get()));
_viewer.setDone(false);
_viewer.setSceneData(_scene.get());
_viewer.setCameraManipulator(0x0);
}
void BoundRenderScene::NextFrame(void)
{
OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_mutex);
if(_frame.valid() && !_viewer.done())
{
osg::Matrixd inverse_cam = osg::Matrixd::inverse(_camera_matrix);
transformation = inverse_cam * pre_transform;
_viewer.getCamera()->setViewMatrix(transformation);
_viewer.updateTraversal();
_viewer.frame();
}
else
std::cout << "Viewer or Camera invalid." << std::endl;
}
The main workflow looks like this (simplified):
BoundRenderScene renderer;
std::vector<osg::Matrixd> poses;
/*
* setting initial parameters
* fill poses with camera positions to render, for regsitration
*/
renderer._init();
for(uint i = 0; i < poses.size(); i++)
{
renderer.SetCameraMatrix(poses.at(i));
renderer.NextImage();
sleep(0.04); // to get the 25fps frame limit
osg::Image* reg_image = renderer.GetImage();
/*
* Do further processing
*/
}
Now comes the crux: the OpenSceneGraph example "osgprenderer" (included in OSG) does off-screen rendering using an osg::Camera::DrawCallback, as my SnapshotCallback. Unfortunately, the operator()-function in my case never get's called in my scenegraph, so that way of screen capture doesn't work for me. It's also rather inconvenient as the rest of the Mutual Information procedure is a rather sequential pipeline.
Other wrappers (https://github.com/xarray/osgRecipes/blob/master/integrations/osgberkelium/osgberkelium.cpp) use methods similar to my "void GetFrame(osg::Image* img)" method, where the image is actively read using "readPixels". That is very convenient for my workflow, but the method always returns a blank image. It doesn't crash, but it doesn't do it's job either.
The method that does work is "osg: and :Image* GetFrame(void)", which returns the bound/attached FBO image. It is similar to the "osgdistortion" example. It does work for rendering one- to two images, but after some time, rendering and processing get out of sync and the application crashes as follows:
[---FIRST FRAME---]
GraphicsCostEstimator::calibrate(..)
cull_draw() 0x1998ca0
ShaderComposer::~ShaderComposer() 0x35a4d40
Renderer::compile()
OpenGL extension 'GL_ARB_vertex_buffer_object' is supported.
OpenGL extension 'GL_EXT_secondary_color' is supported.
OpenGL extension 'GL_EXT_fog_coord' is supported.
OpenGL extension '' is not supported.
OpenGL extension 'GL_EXT_packed_depth_stencil' is supported.
Setting up osg::Camera::FRAME_BUFFER_OBJECT
end cull_draw() 0x1998ca0
[processing]
[ SECOND FRAME ]
cull_draw() 0x1998ca0
OpenGL extension 'GL_ARB_fragment_program' is supported.
OpenGL extension 'GL_ARB_vertex_program' is supported.
OpenGL extension 'GL_ARB_shader_objects' is supported.
OpenGL extension 'GL_ARB_vertex_shader' is supported.
OpenGL extension 'GL_ARB_fragment_shader' is supported.
OpenGL extension 'GL_ARB_shading_language_100' is supported.
OpenGL extension 'GL_EXT_geometry_shader4' is supported.
OpenGL extension 'GL_EXT_gpu_shader4' is supported.
OpenGL extension 'GL_ARB_tessellation_shader' is supported.
OpenGL extension 'GL_ARB_uniform_buffer_object' is supported.
OpenGL extension 'GL_ARB_get_program_binary' is supported.
OpenGL extension 'GL_ARB_gpu_shader_fp64' is supported.
OpenGL extension 'GL_ARB_shader_atomic_counters' is supported.
glVersion=4.5, isGlslSupported=YES, glslLanguageVersion=4.5
Warning: detected OpenGL error 'invalid operation' at end of SceneView::draw()
end cull_draw() 0x1998ca0
[-FROM 3rd FRAME ONWARDS-]
[workload, matrix setup]
[_viewer.frame()]
cull_draw() 0x1998ca0
Warning: detected OpenGL error 'invalid operation' at start of State::apply()
end cull_draw() 0x1998ca0
[next frame]
[BREAKING]
cull_draw() 0x1998ca0
Warning: detected OpenGL error 'invalid operation' at start of State::apply()
end cull_draw() 0x1998ca0
[more work]
Segmentation fault (core dumped)
So, the question is:
I had a look into the source files from osg for the Viewer-related classes, but I was not able to determine where the error
Warning: detected OpenGL error 'invalid operation' at start of State::apply()
comes from. Any idea where to start looking for it ?
For sequential rendering and screen capture, which method is the best to use within OSG ?
How can I obtain the mutex of the normal osg::Viewer, so to sync the renderer with the rest of py pipeline ? (Renderer is single-threaded)
Any other suggestions from experiences OpenSceneGraph off-screen
renderers and screen captures ?

Answer 1

As deeper research turned out, releasing the graphics context in the class destructor freed the OpenGL pipeline, BUT: it also disallocated stateset-bound textures of the loaded scene/model, although the model itself was not suspended (as given in the question: it is re-used in the following passes). So, in further render passes, the render pipeline wanted to access OSG assets which have been released via releasing the GL context.
in code it changed from:
BoundRenderScene::~BoundRenderScene() {
// TODO Auto-generated destructor stub
_viewer.setDone(true);
_viewer.setReleaseContextAtEndOfFrameHint(true);
_gc->releaseContext();
#ifdef DEBUG
std::cout << "BoundRenderScene deleted." << std::endl;
#endif
}
to:
BoundRenderScene::~BoundRenderScene() {
// TODO Auto-generated destructor stub
_viewer.setDone(true);
_viewer.setReleaseContextAtEndOfFrameHint(true);
#ifdef DEBUG
std::cout << "BoundRenderScene deleted." << std::endl;
#endif
}
This resolved the OpenSceneGraph-internal error messages. Now, in order to solve the frame capture problem itself, I implemented the callback from osgprenderer:
struct SnapshotCallback : public osg::Camera::DrawCallback
{
public:
inline SnapshotCallback(std::string filepath) : _filepath(filepath), _output_to_file(false), _image(NULL)
{
if(filepath!="")
_output_to_file = true;
_image = new osg::Image();
}
inline virtual void operator() (osg::RenderInfo& renderInfo) const
{
osg::Camera* camera = renderInfo.getCurrentCamera();
osg::Viewport* viewport = camera ? camera->getViewport() : 0;
if(viewport)
{
glReadBuffer(camera->getDrawBuffer());
_image->allocateImage(int(viewport->width()), int(viewport->height()), 1, GL_RGB, GL_UNSIGNED_BYTE);
_image->readPixels(int(viewport->x()),int(viewport->y()),int(viewport->width()),int(viewport->height()), GL_RGB, GL_UNSIGNED_BYTE);
if(_output_to_file)
{
osgDB::writeImageFile(*reinterpret_cast<osg::Image*>(_image->clone(osg::CopyOp::DEEP_COPY_ALL)), _filepath);
}
}
}
inline virtual void operator() (const osg::Camera& camera) const
{
osg::Viewport* viewport = camera.getViewport();
if(viewport)
{
glReadBuffer(camera.getDrawBuffer());
_image->allocateImage(int(viewport->width()), int(viewport->height()), 1, GL_RGB, GL_UNSIGNED_BYTE);
_image->readPixels(int(viewport->x()),int(viewport->y()),int(viewport->width()),int(viewport->height()), GL_RGB, GL_UNSIGNED_BYTE);
if(_output_to_file)
{
osgDB::writeImageFile(*reinterpret_cast<osg::Image*>(_image->clone(osg::CopyOp::DEEP_COPY_ALL)), _filepath);
}
}
}
inline osg::Image* GetImage(void)
{
return reinterpret_cast<osg::Image*>(_image->clone(osg::CopyOp::DEEP_COPY_ALL));
}
protected:
std::string _filepath;
bool _output_to_file;
mutable osg::ref_ptr<osg::Image> _image;
};
Now, with the cloned buffer instead of the actual image buffer (idea taken over from osgscreencapture example), I do get the real image without memory errors.
For double-buffered rendering, I though have to somehow render the scene twice for the right buffer to contain the objects' images, but this is for my use case currently less of an issue (I/O-bound rendering, not operation-bound).
so, the main function looks like follows:
BoundRenderScene renderer;
std::vector<osg::Matrixd> poses;
/*
* setting initial parameters
* fill poses with camera positions to render, for registration
*/
renderer._init();
for(uint i = 0; i < poses.size(); i++)
{
renderer.SetCameraMatrix(poses.at(i));
renderer.NextImage();
renderer.NextImage();
osg::Image* reg_image = renderer.GetImage();
/*
* Do further processing
*/
}