Skip to content

Commit

Permalink
improved the 4d image reader
Browse files Browse the repository at this point in the history
  • Loading branch information
Install authored and Install committed Aug 30, 2022
1 parent 23ba5ea commit 78b47ac
Show file tree
Hide file tree
Showing 18 changed files with 138 additions and 37 deletions.
137 changes: 106 additions & 31 deletions fluorender/FluoRender/DataManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2539,6 +2539,12 @@ int VolumeData::GetCurChannel()
//time sequence
void VolumeData::SetCurTime(int time)
{
if (time < 0) time = 0;
if (m_reader)
{
if (time >= m_reader->GetTimeNum())
time = m_reader->GetTimeNum() - 1;
}
m_time = time;
}

Expand Down Expand Up @@ -4044,7 +4050,7 @@ wxThread::ExitCode NrrdScaleThread::Entry()
size_t dim = m_src->dim;

if (dim <= 2)
return nullptr;
return (wxThread::ExitCode)1;

std::vector<size_t> ssize(dim);
std::vector<double> sspc(dim);
Expand Down Expand Up @@ -4235,6 +4241,7 @@ wxThread::ExitCode NrrdScaleThread::Entry()
}
}
}
return (wxThread::ExitCode)0;
}


Expand Down Expand Up @@ -8046,7 +8053,8 @@ Color DataManager::GetWavelengthColor(double wavelength)
VolumeDecompressorThread::VolumeDecompressorThread(VolumeLoader *vl)
: wxThread(wxTHREAD_DETACHED), m_vl(vl)
{

wxCriticalSectionLocker enter(*m_vl->ms_pThreadCS);
m_vl->m_running_decomp_th++;
}

VolumeDecompressorThread::~VolumeDecompressorThread()
Expand All @@ -8063,10 +8071,6 @@ wxThread::ExitCode VolumeDecompressorThread::Entry()
if (!m_vl->ms_pThreadCS)
return (wxThread::ExitCode)0;

m_vl->ms_pThreadCS->Enter();
m_vl->m_running_decomp_th++;
m_vl->ms_pThreadCS->Leave();

while(1)
{
m_vl->ms_pThreadCS->Enter();
Expand Down Expand Up @@ -8148,24 +8152,30 @@ wxThread::ExitCode VolumeDecompressorThread::Entry()
else
{
BaseReader* reader = q.vd->GetReader();
wstring nrrd_idstring;
std::wstringstream wss;
wss << reader->GetPathName() << L" " << q.chid << L" " << q.frameid;
nrrd_idstring = wss.str();
Nrrd* nrrd = reader->Convert_ThreadSafe(q.frameid, q.chid, false);
if (nrrd)
{
wstring nrrd_idstring;
std::wstringstream wss;
wss << reader->GetPathName() << L" " << q.chid << L" " << q.frameid;
nrrd_idstring = wss.str();

m_vl->ms_pThreadCS->Enter();
VolumeLoaderImage tmp;
tmp.vd = q.vd;
tmp.chid = q.chid;
tmp.frameid = q.frameid;
tmp.vlnrrd = make_shared<VL_Nrrd>(nrrd);
m_vl->m_used_memory += tmp.vlnrrd->getDatasize();
m_vl->m_loading_files.erase(nrrd_idstring);
m_vl->m_loaded_files[nrrd_idstring] = tmp;
m_vl->ms_pThreadCS->Leave();
}
else
{
m_vl->ms_pThreadCS->Enter();
m_vl->m_loading_files.erase(nrrd_idstring);
m_vl->ms_pThreadCS->Leave();
}
}
}

Expand Down Expand Up @@ -8195,6 +8205,18 @@ VolumeLoaderThread::~VolumeLoaderThread()
delete[] m_vl->m_decomp_queues[i].in_data;
m_vl->m_used_memory -= m_vl->m_decomp_queues[i].datasize;
}
else
{
BaseReader* reader = m_vl->m_decomp_queues[i].vd->GetReader();
if (reader)
{
wstring nrrd_idstring;
std::wstringstream wss;
wss << reader->GetPathName() << L" " << m_vl->m_decomp_queues[i].chid << L" " << m_vl->m_decomp_queues[i].frameid;
nrrd_idstring = wss.str();
m_vl->m_loading_files.erase(nrrd_idstring);
}
}
}
m_vl->m_decomp_queues.clear();
}
Expand Down Expand Up @@ -8233,6 +8255,22 @@ wxThread::ExitCode VolumeLoaderThread::Entry()

m_vl->ms_pThreadCS->Leave();

long long submitted_frame_size = 0;
auto it = m_vl->m_queues.begin();
while (it != m_vl->m_queues.end())
{
if (!it->vd->isBrxml() && it->vd && it->vd->GetReader())
{
submitted_frame_size += it->vd->GetDataSize();
if (submitted_frame_size >= m_vl->m_memory_limit)
it = m_vl->m_queues.erase(it);
else
it++;
}
else
it++;
}

while(1)
{
if (TestDestroy())
Expand Down Expand Up @@ -8333,7 +8371,6 @@ wxThread::ExitCode VolumeLoaderThread::Entry()
m_vl->m_loaded[b.brick] = b;

//OutputDebugStringA("vl-1: leave\n");

m_vl->ms_pThreadCS->Leave();

dthread->Run();
Expand Down Expand Up @@ -8429,19 +8466,21 @@ wxThread::ExitCode VolumeLoaderThread::Entry()
continue;
}

m_vl->m_queues.erase(m_vl->m_queues.begin());
m_vl->m_queued.push_back(b);

wstring nrrd_idstring;
std::wstringstream wss;
wss << b.vd->GetReader()->GetPathName() << L" " << b.chid << L" " << b.frameid;
nrrd_idstring = wss.str();

m_vl->m_loading_files.erase(nrrd_idstring);
m_vl->m_queues.erase(m_vl->m_queues.begin());
m_vl->m_queued.push_back(b);
m_vl->ms_pThreadCS->Leave();

if (m_vl->m_loaded_files.find(nrrd_idstring) != m_vl->m_loaded_files.end() ||
m_vl->m_loading_files.find(nrrd_idstring) != m_vl->m_loading_files.end())
{
m_vl->ms_pThreadCS->Leave();
continue;
}
m_vl->ms_pThreadCS->Leave();

if (m_vl->m_used_memory >= m_vl->m_memory_limit)
{
Expand Down Expand Up @@ -8470,10 +8509,10 @@ wxThread::ExitCode VolumeLoaderThread::Entry()

bool decomp_in_this_thread = false;

if (m_vl->m_max_decomp_th == 0)
if (m_vl->m_frame_reader_max_decomp_th == 0)
decomp_in_this_thread = true;
else if (m_vl->m_max_decomp_th < 0 ||
m_vl->m_running_decomp_th < m_vl->m_max_decomp_th)
else if (m_vl->m_frame_reader_max_decomp_th < 0 ||
m_vl->m_running_decomp_th < m_vl->m_frame_reader_max_decomp_th)
{
VolumeDecompressorThread* dthread = new VolumeDecompressorThread(m_vl);
if (dthread->Create() == wxTHREAD_NO_ERROR)
Expand All @@ -8487,6 +8526,9 @@ wxThread::ExitCode VolumeLoaderThread::Entry()
}
else
{
if (dthread)
delete dthread;
dthread = nullptr;
if (m_vl->m_running_decomp_th <= 0)
decomp_in_this_thread = true;
else
Expand Down Expand Up @@ -8552,6 +8594,7 @@ VolumeLoader::VolumeLoader()
m_max_decomp_th = wxThread::GetCPUCount()-1;
if (m_max_decomp_th < 0)
m_max_decomp_th = -1;
m_frame_reader_max_decomp_th = 6;
m_memory_limit = 10000000LL;
m_used_memory = 0LL;
}
Expand Down Expand Up @@ -8662,6 +8705,7 @@ void VolumeLoader::CheckMemoryCache()
void VolumeLoader::CleanupLoadedBrick()
{
long long required = 0;
long long b_required = 0;

for(int i = 0; i < m_queues.size(); i++)
{
Expand All @@ -8675,20 +8719,25 @@ void VolumeLoader::CleanupLoadedBrick()
m_loaded[b] = m_queues[i];
}
else
required += (size_t)b->nx()*(size_t)b->ny()*(size_t)b->nz()*(size_t)b->nb(0);
b_required += (size_t)b->nx() * (size_t)b->ny() * (size_t)b->nz() * (size_t)b->nb(0);
}
else if (m_queues[i].vd)
{
required += m_queues[i].vd->GetDataSize();
if (!IsNrrdLoaded(m_queues[i].vd, m_queues[i].chid, m_queues[i].frameid))
required += m_queues[i].vd->GetDataSize();
}
}

required += b_required;

if (required > 3LL*1024LL*1024LL*1024LL)
required = 3LL*1024LL*1024LL*1024LL;

m_used_memory = 0;
for (auto& elem : m_memcached_data)
m_used_memory += elem.second->getSize();
for (auto& elem : m_loaded_files)
m_used_memory += elem.second.vlnrrd->getDatasize();

if (required > 0 || m_used_memory >= m_memory_limit)
{
Expand Down Expand Up @@ -8737,7 +8786,7 @@ void VolumeLoader::CleanupLoadedBrick()
required -= datasize;
m_used_memory -= datasize;
m_loaded_files.erase(it->key);
loaded_image_keys.erase((++it).base());
it = decltype(it)(loaded_image_keys.erase(next(it).base()));
}
else
it++;
Expand Down Expand Up @@ -8773,9 +8822,10 @@ void VolumeLoader::CleanupLoadedBrick()
{
long long datasize = m_loaded_files[it->key].vlnrrd->getDatasize();
required -= datasize;
b_required -= datasize;
m_used_memory -= datasize;
m_loaded_files.erase(it->key);
loaded_image_keys.erase((++it).base());
it = decltype(it)(loaded_image_keys.erase(next(it).base()));
}
else
it++;
Expand Down Expand Up @@ -8873,7 +8923,7 @@ void VolumeLoader::CleanupLoadedBrick()
for(int i = m_queues.size()-1; i >= 0; i--)
{
TextureBrick *b = m_queues[i].brick;
if (b->isLoaded() && !b->is_brickdata_locked() && m_loaded.find(b) != m_loaded.end())
if (b && b->isLoaded() && !b->is_brickdata_locked() && m_loaded.find(b) != m_loaded.end())
{
bool skip = false;
for (int j = m_queued.size()-1; j >= 0; j--)
Expand Down Expand Up @@ -8909,7 +8959,7 @@ void VolumeLoader::CleanupLoadedBrick()
for (int i = 0; i < b_locked.size(); i++)
{
TextureBrick *b = b_locked[i].brick;
if (!b->isLoaded())
if (!b || !b->isLoaded())
continue;
bool skip = false;
for (int j = m_queued.size()-1; j >= 0; j--)
Expand Down Expand Up @@ -9244,18 +9294,43 @@ void VolumeLoader::PreloadLevel(VolumeData *vd, int lv, bool lock)

std::shared_ptr<VL_Nrrd> VolumeLoader::GetLoadedNrrd(VolumeData* vd, int ch, int frame)
{
BaseReader* reader = vd->GetReader();
wstring nrrd_idstring;
std::wstringstream wss;
wss << reader->GetPathName() << L" " << ch << L" " << frame;
nrrd_idstring = wss.str();
wstring nrrd_idstring = GetTimeDataKeyString(vd, ch, frame);

if (m_loaded_files.find(nrrd_idstring) != m_loaded_files.end())
return m_loaded_files[nrrd_idstring].vlnrrd;

return nullptr;
}

bool VolumeLoader::IsNrrdLoaded(VolumeData* vd, int ch, int frame)
{
wstring nrrd_idstring = GetTimeDataKeyString(vd, ch, frame);

if (m_loaded_files.find(nrrd_idstring) != m_loaded_files.end())
return true;

return false;
}

bool VolumeLoader::IsNrrdLoading(VolumeData* vd, int ch, int frame)
{
wstring nrrd_idstring = GetTimeDataKeyString(vd, ch, frame);

if (m_loading_files.find(nrrd_idstring) != m_loading_files.end())
return true;

return false;
}

wstring VolumeLoader::GetTimeDataKeyString(VolumeData* vd, int ch, int frame)
{
BaseReader* reader = vd->GetReader();
wstring nrrd_idstring;
std::wstringstream wss;
wss << reader->GetPathName() << L" " << ch << L" " << frame;
return wss.str();
}

void VolumeLoader::AddLoadedNrrd(const std::shared_ptr<VL_Nrrd> &nrrd, VolumeData* vd, int ch, int frame)
{
VolumeLoaderImage vli;
Expand Down
8 changes: 8 additions & 0 deletions fluorender/FluoRender/DataManager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1837,6 +1837,7 @@ class EXPORT_API VolumeLoader
void Join();
bool Run();
void SetMaxThreadNum(int num) {m_max_decomp_th = num;}
void SetFrameReaderMaxThreadNum(int num) { m_frame_reader_max_decomp_th = num; }
void SetMemoryLimitByte(long long limit) {m_memory_limit = limit;}
void CleanupLoadedBrick();
void TryToFreeMemory(long long req=-1);
Expand All @@ -1847,9 +1848,15 @@ class EXPORT_API VolumeLoader
void GetPalams(long long &used_mem, int &running_decomp_th, int &queue_num, int &decomp_queue_num);
void PreloadLevel(VolumeData *vd, int lv, bool lock=false);
std::shared_ptr<VL_Nrrd> GetLoadedNrrd(VolumeData* vd, int ch, int frame);
bool IsNrrdLoaded(VolumeData* vd, int ch, int frame);
bool IsNrrdLoading(VolumeData* vd, int ch, int frame);
std::wstring GetTimeDataKeyString(VolumeData* vd, int ch, int frame);
void AddLoadedNrrd(const std::shared_ptr<VL_Nrrd> &nrrd, VolumeData* vd, int ch, int frame);
//void DeleteLoadedNrrd(Nrrd* nrrd, VolumeData* vd, int ch, int frame);

int GetMaxThreadNum() { return m_max_decomp_th; }
int GetFrameReaderMaxThreadNum() { return m_frame_reader_max_decomp_th; }

long long GetAvailableMemory() { return m_memory_limit - m_used_memory; }
long long GetMemoryLimitByte() { return m_memory_limit; }
long long GetUsedMemory() { return m_used_memory; }
Expand All @@ -1873,6 +1880,7 @@ class EXPORT_API VolumeLoader
unordered_map<wstring, VolumeLoaderImage> m_loaded_files;
int m_running_decomp_th;
int m_max_decomp_th;
int m_frame_reader_max_decomp_th;
bool m_valid;

long long m_memory_limit;
Expand Down
Loading

0 comments on commit 78b47ac

Please sign in to comment.