Commit a14fcb2f authored by Locatelli's avatar Locatelli
Browse files

do some memory test calling garbage collector for python side

parent 7512f75e
......@@ -2,6 +2,8 @@
* See possible using of CUDA acceleration
* Create for offscreen empty spies, remove multiplot files at the start of server
* Multiclient options
- no offscreen log, webspy plots
- offscreen plots for spy and multiplot will be regenerated or taken form the nomad server computer
......@@ -12,7 +14,7 @@
* Bugs:
- Plot1D crash after several time using it, see core file, pb QT event
- Problem to pass from empty data to others
- Problem to pass from empty plot to others
- Problem close a plot take a time in cameo, so if restarting a new before finishing previous does not worked
......@@ -127,9 +127,19 @@ test_LDADD = $(LIBS) $(RM_LIBS) \
$(BOOST_THREAD_LIB)
test2_SOURCES = \
tests/Counter.cpp \
tests/moc_Counter.cpp \
maintest2.cpp
maintest2.cpp \
manager/RequestDealerManager.cpp \
manager/ServerSubscriberManager.cpp \
manager/ServerRequesterManager.cpp \
manager/PlotManager.cpp \
manager/OffScreenPlotManager.cpp \
manager/mpl/MplEventManager.cpp \
$(PLOT) \
$(PROTOBUF) \
$(DATACONT) \
$(MPLCPP) \
$(QT) \
$(JSON)
tests/moc_%.cpp: tests/%.h
moc $(DEFINES) $(INCPATH) $< -o $@
......
This diff is collapsed.
......@@ -29,7 +29,6 @@
#include "view/mpl/MplLock.h"
#include "view/mpl/MplFigure.h"
#include <iostream>
#include <thread>
#include <fstream>
#include <vector>
#include <chrono>
......@@ -40,6 +39,8 @@
#include "plot/offscreen/OffScreenEmptyPlot.h"
#include "plot/offscreen/OffScreenPlot1D.h"
#include "plot/offscreen/OffScreenPlot2D.h"
#include <boost/thread.hpp>
#include <thread>
using namespace std;
namespace bp = boost::python;
......@@ -71,10 +72,9 @@ public:
* \param[in] path the path directory of the pb file
*/
SavePlot(view::mpl::Mpl* mpl, const std::string& pbfilename, const std::string& path);
SavePlot();
/*!
* \brief do save action
*/
~SavePlot();
void start();
void stop();
void run();
private:
......@@ -82,45 +82,9 @@ private:
view::mpl::Mpl* m_Mpl; //! Mpl object (main class which managed the matplotib api)
std::string m_Pbfilename; //! the plot binary file name from server
std::string m_Path; //! The path directory of the pb file
boost::thread mThread;
};
unordered_map<string, thread> tm_;
unordered_map<string, SavePlot> spm_;
static std::mutex m_Mutex;
bool intm(const string& id) {
unordered_map<string, thread>::iterator it = tm_.find(id);
return (it != tm_.end());
}
void addtm(view::mpl::Mpl* mpl, const string& id, const string& path) {
std::lock_guard<std::mutex> lock(m_Mutex);
if (!intm(id)) {
SavePlot sp1(mpl, id, path);
thread t1(bind(&SavePlot::run, sp1));
t1.detach();
tm_[id] = std::move(t1);
spm_[id] = std::move(sp1);
cout << "add id " << id << " " << tm_.size() << endl;
}
}
void deltm(const string& id) {
string lid = std::move(id);
std::lock_guard<std::mutex> lock(m_Mutex);
unordered_map<string, thread>::iterator it = tm_.find(lid);
if (it != tm_.end()) {
it->second.std::thread::~thread();
tm_.erase(lid);
cout << "del id " << lid << " " << tm_.size() << endl;
}
unordered_map<string, SavePlot>::iterator it2 = spm_.find(lid);
if (it2 != spm_.end()) {
spm_.erase(lid);
cout << "del id " << lid << " " << spm_.size() << endl;
}
}
/*
* savePlot
*/
......@@ -129,106 +93,110 @@ SavePlot::SavePlot(view::mpl::Mpl* mpl, const std::string& pbfilename, const std
m_Pbfilename = pbfilename;
m_Path = path;
}
/*
* savePlot
*/
SavePlot::SavePlot() {
m_Mpl = nullptr;
SavePlot::~SavePlot() {
}
/*
* savePlot
*/
void SavePlot::run() {
DBGMSG("SavePlot::savePlot file : " << m_Pbfilename);
view::mpl::MplFigure* figure = new view::mpl::MplFigure(m_Mpl, false);
DBGMSG("SavePlot::savePlot 2 file : " << m_Pbfilename);
boost::filesystem::path pbfile = m_Path;
pbfile /= m_Pbfilename;
pbfile.replace_extension("pb");
ifstream inputMessage(pbfile.string().c_str(), fstream::binary);
if (inputMessage.is_open() == false) {
Error("OffScreenPlotManager", "savePlot", "Failed to open temporary file which contains proto::PlotPropertyDataMessage",
m_Pbfilename);
} else {
uint32 size = (uint32) boost::filesystem::file_size(pbfile);
char buffer[size];
inputMessage.read(buffer, size);
buffer::Data data;
data.ParseFromArray(buffer, size);
DBGMSG("type = " << data.type());
DBGMSG("buffer = " << size);
DBGMSG("numor = " << data.numor());
DBGMSG("dataxArray = " << data.xdata_size());
DBGMSG("datayArray = " << data.ydata_size());
DBGMSG("datazArray = " << data.zdata_size());
inputMessage.close();
ostringstream plotkey;
plotkey << m_Pbfilename;
uint32 dpi = 100;
float64 pad = 1;
if (data.type() == buffer::Data::Spy) {
dpi = 30;
pad = 0;
}
else if (data.type() == buffer::Data::Multiplot) {
dpi = 30;
}
if ((data.xdata_size() > 0) && (data.ydata_size() > 0) && (data.zdata_size() > 0)) {
// 2D
for (int32 i = 0; i < data.xdata_size(); ++i) {
try {
plot::offscreen::OffScreenPlot2D plot(figure, plotkey.str(), m_Mpl, &data);
plot.display();
plot.save(pbfile.parent_path().parent_path().string(), dpi, pad);
} catch (Error& e) {
Error("OffScreenPlotManager", "savePlot", "Failed create 2d plot", plotkey.str());
}
}
} else if ((data.xdata_size() > 0) && (data.ydata_size() > 0)) {
// 1D
for (int32 i = 0; i < data.xdata_size(); ++i) {
try {
plot::offscreen::OffScreenPlot1D plot(figure, plotkey.str(), m_Mpl, &data);
plot.display();
plot.save(pbfile.parent_path().parent_path().string(), dpi, pad);
} catch (Error& e) {
Error("OffScreenPlotManager", "savePlot", "Failed create 1d plot", plotkey.str());
}
}
} else {
try {
plot::offscreen::OffScreenEmptyPlot plot(figure, plotkey.str(), m_Mpl, &data);
plot.display();
plot.save(pbfile.parent_path().parent_path().string(), dpi, pad);
} catch (Error& e) {
Error("OffScreenPlotManager", "savePlot", "Failed create empty plot", plotkey.str());
}
}
void SavePlot::start()
{
mThread = boost::thread(&SavePlot::run, this);
mThread.detach();
}
void SavePlot::stop()
{
mThread.interrupt();
mThread.join();
}
// if (data.type() == buffer::Data::Log) {
// // delete file
// boost::filesystem::remove(pbfile);
void SavePlot::run() {
try {
unique_ptr<view::mpl::MplFigure> figure(new view::mpl::MplFigure(m_Mpl, false));
// boost::filesystem::path pbfile = m_Path;
// pbfile /= m_Pbfilename;
// pbfile.replace_extension("pb");
// ifstream inputMessage(pbfile.string().c_str(), fstream::binary);
// if (inputMessage.is_open() == false) {
// Error("OffScreenPlotManager", "savePlot", "Failed to open temporary file which contains proto::PlotPropertyDataMessage",
// m_Pbfilename);
// } else {
// uint32 size = (uint32) boost::filesystem::file_size(pbfile);
// char buffer[size];
// inputMessage.read(buffer, size);
// buffer::Data data;
// data.ParseFromArray(buffer, size);
//
// DBGMSG("type = " << data.type());
// DBGMSG("buffer = " << size);
// DBGMSG("numor = " << data.numor());
// DBGMSG("dataxArray = " << data.xdata_size());
// DBGMSG("datayArray = " << data.ydata_size());
// DBGMSG("datazArray = " << data.zdata_size());
// inputMessage.close();
//
// ostringstream plotkey;
// plotkey << m_Pbfilename;
//
// // Send image to log database system
// json::LogSender sender(&data);
// sender.postMessage();
// boost::filesystem::path pngfile = m_Path;
// pngfile = pngfile.parent_path().parent_path();
// pngfile /= m_Pbfilename;
// DBGMSG("remove -> " << pngfile.string());
// // delete image file
// boost::filesystem::remove(pngfile);
// uint32 dpi = 100;
// float64 pad = 1;
// if (data.type() == buffer::Data::Spy) {
// dpi = 30;
// pad = 0;
// }
// else if (data.type() == buffer::Data::Multiplot) {
// dpi = 30;
// }
// if ((data.xdata_size() > 0) && (data.ydata_size() > 0) && (data.zdata_size() > 0)) {
// // 2D
// for (int32 i = 0; i < data.xdata_size(); ++i) {
// try {
// plot::offscreen::OffScreenPlot2D plot(figure.get(), plotkey.str(), m_Mpl, &data);
// plot.display();
// plot.save(pbfile.parent_path().parent_path().string(), dpi, pad);
// } catch (Error& e) {
// Error("OffScreenPlotManager", "savePlot", "Failed create 2d plot", plotkey.str());
// }
// }
// } else if ((data.xdata_size() > 0) && (data.ydata_size() > 0)) {
// // 1D
// for (int32 i = 0; i < data.xdata_size(); ++i) {
// try {
// plot::offscreen::OffScreenPlot1D plot(figure.get(), plotkey.str(), m_Mpl, &data);
// plot.display();
// plot.save(pbfile.parent_path().parent_path().string(), dpi, pad);
// } catch (Error& e) {
// Error("OffScreenPlotManager", "savePlot", "Failed create 1d plot", plotkey.str());
// }
// }
// } else {
// try {
// plot::offscreen::OffScreenEmptyPlot plot(figure.get(), plotkey.str(), m_Mpl, &data);
// plot.display();
// plot.save(pbfile.parent_path().parent_path().string(), dpi, pad);
// } catch (Error& e) {
// Error("OffScreenPlotManager", "savePlot", "Failed create empty plot", plotkey.str());
// }
// }
//
//
// // if (data.type() == buffer::Data::Log) {
// // // delete file
// // boost::filesystem::remove(pbfile);
// //
// // // Send image to log database system
// // json::LogSender sender(&data);
// // sender.postMessage();
// // boost::filesystem::path pngfile = m_Path;
// // pngfile = pngfile.parent_path().parent_path();
// // pngfile /= m_Pbfilename;
// // DBGMSG("remove -> " << pngfile.string());
// // // delete image file
// // boost::filesystem::remove(pngfile);
// // }
// }
}
delete figure;
cout << "delete this" << endl;
cout << "delete this ok" << endl;
deltm(m_Pbfilename);
// cout << "delete this" << endl;
} catch(boost::thread_interrupted& interrupt) {
}
delete this;
}
......@@ -239,21 +207,33 @@ void test(view::mpl::Mpl* mpl) {
uint32 i = 0;
while (true) {
addtm(mpl, "plot#detector_plot_data0#Detector.png", "/users/locatelli/offscreenImages/web/pb");
addtm(mpl, "ScanSpy#scanSpyPlot0.png", "/users/locatelli/offscreenImages/web/pb");
addtm(mpl, "plot#detector_plot_data.0.0#0.png", "/users/locatelli/offscreenImages/multiplot/pb");
addtm(mpl, "plot#detector_plot_data.0.0#1.png", "/users/locatelli/offscreenImages/multiplot/pb");
addtm(mpl, "plot#detector_plot_data.0.0#2.png", "/users/locatelli/offscreenImages/multiplot/pb");
addtm(mpl, "plot#detector_plot_data.0.0#3.png", "/users/locatelli/offscreenImages/multiplot/pb");
addtm(mpl, "plot#detector_plot_data.0.0#4.png", "/users/locatelli/offscreenImages/multiplot/pb");
addtm(mpl, "plot#detector_plot_data.0.0#5.png", "/users/locatelli/offscreenImages/multiplot/pb");
addtm(mpl, "plot#detector_plot_data.0.0#6.png", "/users/locatelli/offscreenImages/multiplot/pb");
addtm(mpl, "plot#detector_plot_data.0.0#7.png", "/users/locatelli/offscreenImages/multiplot/pb");
addtm(mpl, "plot#detector_plot_data.0.0#8.png", "/users/locatelli/offscreenImages/multiplot/pb");
addtm(mpl, "plot#detector_plot_data.0.0#9.png", "/users/locatelli/offscreenImages/multiplot/pb");
//break;
if (++i > 1000) break;
SavePlot* sp1 = new SavePlot(mpl, "plot#detector_plot_data0#Detector.png", "/users/locatelli/offscreenImages/web/pb");
sp1->start();
SavePlot* sp2 = new SavePlot(mpl, "ScanSpy#scanSpyPlot0.png", "/users/locatelli/offscreenImages/web/pb");
sp2->start();
SavePlot* sp3 = new SavePlot(mpl, "plot#detector_plot_data.0.0#0.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp3->start();
SavePlot* sp4 = new SavePlot(mpl, "plot#detector_plot_data.0.0#1.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp4->start();
SavePlot* sp5 = new SavePlot(mpl, "plot#detector_plot_data.0.0#2.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp5->start();
SavePlot* sp6 = new SavePlot(mpl, "plot#detector_plot_data.0.0#3.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp6->start();
SavePlot* sp7 = new SavePlot(mpl, "plot#detector_plot_data.0.0#4.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp7->start();
SavePlot* sp8 = new SavePlot(mpl, "plot#detector_plot_data.0.0#5.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp8->start();
SavePlot* sp9 = new SavePlot(mpl, "plot#detector_plot_data.0.0#6.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp9->start();
SavePlot* sp10 = new SavePlot(mpl, "plot#detector_plot_data.0.0#7.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp10->start();
SavePlot* sp11 = new SavePlot(mpl, "plot#detector_plot_data.0.0#8.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp11->start();
SavePlot* sp12 = new SavePlot(mpl, "plot#detector_plot_data.0.0#9.png", "/users/locatelli/offscreenImages/multiplot/pb");
sp12->start();
if (++i > 10000000) break;
// break;
this_thread::sleep_for(chrono::milliseconds(500));
}
this_thread::sleep_for(chrono::seconds(10));
......@@ -335,10 +315,8 @@ int main() {
thread td(bind(test, &mpl));
td.join();
std::cout << "Number of threads = "
std::cout << "Number of threads = "
<< std::thread::hardware_concurrency() << std::endl;
std::cout << "tm_ size = "
<< tm_.size() << std::endl;
cin >> txt;
return EXIT_SUCCESS;
......
#include "tests/Counter.h"
/*
* Nomad Instrument Control Software
*
* Copyright 2011 Institut Laue-Langevin
*
* Licensed under the EUPL, Version 1.1 only (the "License");
* You may not use this work except in compliance with the Licence.
* You may obtain a copy of the Licence at:
*
* http://joinup.ec.europa.eu/software/page/eupl
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the Licence is distributed on an "AS IS" basis,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the Licence for the specific language governing permissions and
* limitations under the Licence.
*/
#include <iostream>
#include <QApplication>
#include <termios.h>
#include <unistd.h>
#include <fstream>
#include <cstdlib>
#include <signal.h>
#include <thread>
#include "Common.h"
#include "Trace.h"
#include "Error.h"
#include "view/mpl/Mpl.h"
#include "view/mpl/MplFigure.h"
#include "protobuf/generated/AcquisitionSerializer.pb.h"
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/operations.hpp>
#include "plot/offscreen/OffScreenEmptyPlot.h"
#include "plot/offscreen/OffScreenPlot1D.h"
#include "plot/offscreen/OffScreenPlot2D.h"
using namespace std;
int main(int argc, char *argv[]) {
QApplication app(argc, argv);
qRegisterMetaType<Counter>();
Counter a, b;
QObject::connect(&a, SIGNAL(valueChanged(int)),
&b, SLOT(setValue(int)));
/*!
* \brief main program function
* \return error code
*/
int main(int32 argc, char* argv[]) {
string txt;
cin >> txt;
// Need to create Mpl in main thread
view::mpl::Mpl* m_Mpl = new view::mpl::Mpl();
string m_Pbfilename = "plot#detector_plot_data0#Detector.png";
string m_Path = "/users/locatelli/offscreenImages/web/pb";
uint32 i = 0;
while(true) {
DBGMSG("SavePlot::savePlot file : " << m_Pbfilename);
view::mpl::MplFigure* figure = new view::mpl::MplFigure(m_Mpl, false);
DBGMSG("SavePlot::savePlot 2 file : " << m_Pbfilename);
boost::filesystem::path pbfile = m_Path;
pbfile /= m_Pbfilename;
pbfile.replace_extension("pb");
ifstream inputMessage(pbfile.string().c_str(), fstream::binary);
if (inputMessage.is_open() == false) {
Error("OffScreenPlotManager", "savePlot", "Failed to open temporary file which contains proto::PlotPropertyDataMessage",
m_Pbfilename);
} else {
uint32 size = (uint32) boost::filesystem::file_size(pbfile);
char buffer[size];
inputMessage.read(buffer, size);
buffer::Data data;
data.ParseFromArray(buffer, size);
DBGMSG("type = " << data.type());
DBGMSG("buffer = " << size);
DBGMSG("numor = " << data.numor());
DBGMSG("dataxArray = " << data.xdata_size());
DBGMSG("datayArray = " << data.ydata_size());
DBGMSG("datazArray = " << data.zdata_size());
inputMessage.close();
ostringstream plotkey;
plotkey << m_Pbfilename;
uint32 dpi = 100;
float64 pad = 1;
if (data.type() == buffer::Data::Spy) {
dpi = 30;
pad = 0;
}
else if (data.type() == buffer::Data::Multiplot) {
dpi = 30;
}
if ((data.xdata_size() > 0) && (data.ydata_size() > 0) && (data.zdata_size() > 0)) {
// 2D
for (int32 i = 0; i < data.xdata_size(); ++i) {
try {
plot::offscreen::OffScreenPlot2D plot(figure, plotkey.str(), m_Mpl, &data);
plot.display();
plot.save(pbfile.parent_path().parent_path().string(), dpi, pad);
} catch (Error& e) {
Error("OffScreenPlotManager", "savePlot", "Failed create 2d plot", plotkey.str());
}
}
} else if ((data.xdata_size() > 0) && (data.ydata_size() > 0)) {
// 1D
for (int32 i = 0; i < data.xdata_size(); ++i) {
try {
plot::offscreen::OffScreenPlot1D plot(figure, plotkey.str(), m_Mpl, &data);
plot.display();
plot.save(pbfile.parent_path().parent_path().string(), dpi, pad);
} catch (Error& e) {
Error("OffScreenPlotManager", "savePlot", "Failed create 1d plot", plotkey.str());
}
}
} else {
try {
plot::offscreen::OffScreenEmptyPlot plot(figure, plotkey.str(), m_Mpl, &data);
plot.display();
plot.save(pbfile.parent_path().parent_path().string(), dpi, pad);
} catch (Error& e) {
Error("OffScreenPlotManager", "savePlot", "Failed create empty plot", plotkey.str());
}
}
b.setValue(112); // a.value() == 12, b.value() == 12
a.setValue(12); // a.value() == 12, b.value() == 12
cout << "a = " << a.value() << endl;
cout << "b = " << b.value() << endl;
// if (data.type() == buffer::Data::Log) {
// // delete file
// boost::filesystem::remove(pbfile);
//
// // Send image to log database system
// json::LogSender sender(&data);
// sender.postMessage();
// boost::filesystem::path pngfile = m_Path;
// pngfile = pngfile.parent_path().parent_path();
// pngfile /= m_Pbfilename;
// DBGMSG("remove -> " << pngfile.string());
// // delete image file
// boost::filesystem::remove(pngfile);
// }
}
delete figure;
if (++i > 10000000) {
break;
}
}
b.setValue(48); // a.value() == 12, b.value() == 48
delete m_Mpl;
cout << "a = " << a.value() << endl;
cout << "b = " << b.value() << endl;
return EXIT_SUCCESS;
}
//#include <python3.5m/Python.h>
//#include <python3.5m/sip.h>
//#include <QApplication>
//#include <QtWidgets>
//#include "tests/mainwindow.h"
//#include "Trace.h"
//#include <stdexcept>
//#include <iostream>
//#include <vector>
//#include <thread>
//#include <boost/python/object.hpp>
//#include <boost/python/errors.hpp>
//#include <boost/python/call_method.hpp>
//#include <boost/python/list.hpp>
//#include <boost/python/tuple.hpp>
//#include <boost/python/dict.hpp>
//#include <boost/python/args.hpp>
//#include <boost/python/extract.hpp>
//#include <boost/python/import.hpp>
//#include <boost/python/type_id.hpp>
//#include <boost/python/str.hpp>
//
//#include "view/mpl/MplLock.h"
//
//namespace bp = boost::python;
//using namespace std;
//
////template<typename T = PyObject> using Handle = boost::python::handle<T>;
//
////// Helper to create an Object from a new reference to a raw PyObject*
////Object NewRef(PyObject *obj) {
//// return Object(Handle<>(obj));
////}
//
//std::vector<double> x = { 1, 2, 3, 4 };
//std::vector<double> y = { 1, 2, 4, 5 };
//bp::object axis;
//bp::object line;
//bp::object figurecanvas;
//MainWindow* window;
//bp::object pyplotModule;
//
//int counter = 0;
//
//void refresh() {
// while (true) {
// sleep(1);
// std::ostringstream buf;
// buf << counter++;
// window->set(buf.str());
// {
// view::mpl::MplLock lock;
// x.push_back(x.size() + 1);
// y.push_back(1);
// try {
//// axis.attr("clear")();
//// axis.attr("plot")(*args);
// std::cout << "-->1" << std::endl;
// boost::python::list xarray;
// for (double val : x) {
// xarray.append(val);
// }
// auto args = boost::python::make_tuple(xarray);
// line.attr("set_xdata")(*args);
// std::cout << "-->2" << std::endl;
// boost::python::list yarray;
// for (double val : y) {
// yarray.append(val);
// }
// args = boost::python::make_tuple(yarray);
// line.attr("set_ydata")(*args);
// std::cout << "-->3" << std::endl;