From 67bb69f0c3a815cd0fff0d9ed67e2a092b4dba92 Mon Sep 17 00:00:00 2001
From: Thomas Liao <tliao@carboncopies.org>
Date: Mon, 24 Mar 2025 19:27:42 -0700
Subject: [PATCH 1/5] Add contention mechanism for image processor pool
---
Source/Core/CMakeLists.txt | 1 +
Source/Core/VSDA/EM/EMRenderer.cpp | 2 +-
.../VSDA/EM/VoxelSubsystem/EMSubRegion.cpp | 46 +++++++++----------
.../ImageProcessorPool/ImageProcessorPool.cpp | 2 +-
4 files changed, 25 insertions(+), 26 deletions(-)
diff --git a/Source/Core/CMakeLists.txt b/Source/Core/CMakeLists.txt
index e69e19947..6bf6f521d 100644
--- a/Source/Core/CMakeLists.txt
+++ b/Source/Core/CMakeLists.txt
@@ -6,6 +6,7 @@ set(MAIN_SOURCES
${SRC_DIR}/Core/Main.h
+
${SRC_DIR}/Core/Config/ConfigurationManager.cpp
${SRC_DIR}/Core/Config/ConfigurationManager.h
${SRC_DIR}/Core/Config/ConfigFileParser.cpp
diff --git a/Source/Core/VSDA/EM/EMRenderer.cpp b/Source/Core/VSDA/EM/EMRenderer.cpp
index e3473408d..8731dbd07 100644
--- a/Source/Core/VSDA/EM/EMRenderer.cpp
+++ b/Source/Core/VSDA/EM/EMRenderer.cpp
@@ -44,7 +44,7 @@ bool ExecuteSubRenderOperations(Config::Config* _Config, BG::Common::Logger::Log
}
_Simulation->VSDAData_.State_ = VSDA_RENDER_IN_PROGRESS;
- _Logger->Log("Executing Render Job For Requested Simulation", 4);
+ _Logger->Log("Executing Render Job For Requested Simulation " + std::to_string(_Simulation->ID), 4);
// Unpack Variables For Easier Access
diff --git a/Source/Core/VSDA/EM/VoxelSubsystem/EMSubRegion.cpp b/Source/Core/VSDA/EM/VoxelSubsystem/EMSubRegion.cpp
index 897516712..f1918bcc7 100644
--- a/Source/Core/VSDA/EM/VoxelSubsystem/EMSubRegion.cpp
+++ b/Source/Core/VSDA/EM/VoxelSubsystem/EMSubRegion.cpp
@@ -114,8 +114,29 @@ bool EMRenderSubRegion(BG::Common::Logger::LoggingSystem* _Logger, SubRegion* _S
_Logger->Log("This EM Render Operation Desires " + std::to_string(VSDAData_->Params_.SliceThickness_um) + "um Slice Thickness", 5);
_Logger->Log("Therefore, we are using " + std::to_string(NumVoxelsPerSlice) + "vox per slice at " + std::to_string(VSDAData_->Params_.VoxelResolution_um) + "um per vox", 5);
_Logger->Log("We Will Render A Total Of " + std::to_string(NumZSlices) + " Slices", 5);
+
+
+ // Force us to wait for any other renders using the image processor pool
+ while (_ImageProcessorPool->GetQueueSize() > 0) {
+
+ // Update Current Slice Information (Account for slice numbers not starting at 0)
+ VSDAData_->CurrentOperation_ = "Image Processing Enqueued";
+ VSDAData_->TotalSliceImages_ = 0;
+ VSDAData_->CurrentSliceImage_ = 0;
+ VSDAData_->VoxelQueueLength_ = 0;
+ VSDAData_->TotalVoxelQueueLength_ = 0;
+ VSDAData_->TotalSlices_ = 0;
+ VSDAData_->CurrentSlice_ = 0;
+ VSDAData_->CurrentSlice_ = VSDAData_->TotalSlices_ - _ImageProcessorPool->GetQueueSize();
+
+ _Logger->Log("Waiting for ImageProcessorPool to become available; '" + std::to_string(_ImageProcessorPool->GetQueueSize()) + "' items remaining", 1);
+
+ // Now wait a while so we don't spam the console
+ std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+ }
+
// Update Status Bar
VSDAData_->CurrentOperation_ = "Image Processing";
VSDAData_->TotalSliceImages_ = 0;
@@ -147,36 +168,13 @@ bool EMRenderSubRegion(BG::Common::Logger::LoggingSystem* _Logger, SubRegion* _S
// Ensure All Tasks Are Finished
while (_ImageProcessorPool->GetQueueSize() > 0) {
- // // Calculate Desired Image Size
- // // In order for us to deal with multiple different pixel/voxel setting, we create an image of start size where one pixel = 1 voxel
- // // then later on, we resample it to be the right size (for the target image)
- // int VoxelsPerStepX = ceil(VSDAData_->Params_.ImageWidth_px / VSDAData_->Params_.NumPixelsPerVoxel_px);
- // int VoxelsPerStepY = ceil(VSDAData_->Params_.ImageHeight_px / VSDAData_->Params_.NumPixelsPerVoxel_px);
- // int NumChannels = 3;
- // float CameraStepSizeX_um = VoxelsPerStepX * VSDAData_->Params_.VoxelResolution_um;
- // float CameraStepSizeY_um = VoxelsPerStepY * VSDAData_->Params_.VoxelResolution_um;
-
- // double TotalSliceWidth = abs((double)VSDAData_->Array_->GetBoundingBox().bb_point1[0] - (double)VSDAData_->Array_->GetBoundingBox().bb_point2[0]);
- // double TotalSliceHeight = abs((double)VSDAData_->Array_->GetBoundingBox().bb_point1[1] - (double)VSDAData_->Array_->GetBoundingBox().bb_point2[1]);
- // int TotalXSteps = ceil(TotalSliceWidth / CameraStepSizeX_um);
- // int TotalYSteps = ceil(TotalSliceHeight / CameraStepSizeY_um);
- // TotalXSteps = std::min(TotalXSteps, _SubRegion->MaxImagesX);
- // TotalYSteps = std::min(TotalYSteps, _SubRegion->MaxImagesY);
-
- // int ImagesPerSlice = TotalXSteps * TotalYSteps;
-
-
// Update Current Slice Information (Account for slice numbers not starting at 0)
VSDAData_->CurrentSlice_ = VSDAData_->TotalSlices_ - _ImageProcessorPool->GetQueueSize();
- // VSDAData_->TotalSliceImages_ = ImagesPerSlice;
- // VSDAData_->CurrentSliceImage_ = _ImageProcessorPool->GetQueueSize() % ImagesPerSlice;
- // Log Queue Size
_Logger->Log("ImageProcessorPool Queue Length '" + std::to_string(_ImageProcessorPool->GetQueueSize()) + "'", 1);
-
// Now wait a while so we don't spam the console
- std::this_thread::sleep_for(std::chrono::milliseconds(500));
+ std::this_thread::sleep_for(std::chrono::milliseconds(1000));
}
diff --git a/Source/Core/VSDA/EM/VoxelSubsystem/ImageProcessorPool/ImageProcessorPool.cpp b/Source/Core/VSDA/EM/VoxelSubsystem/ImageProcessorPool/ImageProcessorPool.cpp
index 6ff42463a..758aa6b6f 100644
--- a/Source/Core/VSDA/EM/VoxelSubsystem/ImageProcessorPool/ImageProcessorPool.cpp
+++ b/Source/Core/VSDA/EM/VoxelSubsystem/ImageProcessorPool/ImageProcessorPool.cpp
@@ -199,7 +199,7 @@ void ImageProcessorPool::EncoderThreadMainFunction(int _ThreadNumber) {
Task->IsDone_ = true;
// Logging
- Logger_->Log("Compressed segmentation layer " + std::to_string(Task->VoxelZ) + " to " + Task->OutputPath_, 1);
+ //Logger_->Log("Compressed segmentation layer " + std::to_string(Task->VoxelZ) + " to " + Task->OutputPath_, 1);
continue;
}
--
GitLab
From c5031d56cc686c6fee2cd688816e8a026af969a1 Mon Sep 17 00:00:00 2001
From: Thomas Liao <tliao@carboncopies.org>
Date: Mon, 24 Mar 2025 19:27:55 -0700
Subject: [PATCH 2/5] Add memory usage check
---
Source/Core/VSDA/EM/EMRenderer.cpp | 33 +++++++++++++------
.../ImageProcessorPool/ImageProcessorPool.h | 4 ++-
2 files changed, 26 insertions(+), 11 deletions(-)
diff --git a/Source/Core/VSDA/EM/EMRenderer.cpp b/Source/Core/VSDA/EM/EMRenderer.cpp
index 8731dbd07..df35657be 100644
--- a/Source/Core/VSDA/EM/EMRenderer.cpp
+++ b/Source/Core/VSDA/EM/EMRenderer.cpp
@@ -51,16 +51,6 @@ bool ExecuteSubRenderOperations(Config::Config* _Config, BG::Common::Logger::Log
MicroscopeParameters* Params = &_Simulation->VSDAData_.Params_;
ScanRegion* BaseRegion = &_Simulation->VSDAData_.Regions_[_Simulation->VSDAData_.ActiveRegionID_];
- // // -- Phase -1 --
- // // We need to backpropagate the ids of the neurons to the compartments, then to us.
- // for (auto Neuron : _Simulation->Neurons) {
- // if (auto SCNeuron = dynamic_cast<Simulation::SCNeuron>(Neuron.get())) {
- // RegisterNeuronUIDToCompartments(_N.SomaCompartmentIDs, _N.ID + 1);
- // RegisterNeuronUIDToCompartments(_N.DendriteCompartmentIDs, _N.ID + 1);
- // RegisterNeuronUIDToCompartments(_N.AxonCompartmentIDs, _N.ID + 1);
- // }
- // }
-
// -- Phase 0 --
// Here, we detect how much memory this machine has and then use that to make an educated guess as to the max size of the voxel array.
@@ -153,6 +143,29 @@ bool ExecuteSubRenderOperations(Config::Config* _Config, BG::Common::Logger::Log
BaseRegion->RegionIndexInfo_ = Info;
+ // Now we need to check that we have enough memory right now to begin rendering, if we dont, we need to wait until we do
+ // this actually just looks for other renders that are happening right now and checks its status
+ double CurrentRendererUsage = _ImageProcessorPool->TotalConsumedMemory_MB.load();
+
+ while (((_ImageProcessorPool->TotalConsumedMemory_MB.load() + MemorySize_MB) / SystemRAM_MB) > 0.9) {
+
+ // Update Current Slice Information (Account for slice numbers not starting at 0)
+ _Simulation->VSDAData_.CurrentOperation_ = "Waiting for free RAM";
+ _Simulation->VSDAData_.TotalSliceImages_ = 0;
+ _Simulation->VSDAData_.CurrentSliceImage_ = 0;
+ _Simulation->VSDAData_.VoxelQueueLength_ = 0;
+ _Simulation->VSDAData_.TotalVoxelQueueLength_ = 0;
+ _Simulation->VSDAData_.TotalSlices_ = 0;
+ _Simulation->VSDAData_.CurrentSlice_ = 0;
+
+ _Logger->Log("Waiting for enough free RAM to become available before starting render", 1);
+
+ // Now wait a while so we don't spam the console
+ std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+
+ }
+
+
// Now, we go through all of the steps in each direction that we identified, and calculate the bounding boxes for each
std::vector<SubRegion> SubRegions;
diff --git a/Source/Core/VSDA/EM/VoxelSubsystem/ImageProcessorPool/ImageProcessorPool.h b/Source/Core/VSDA/EM/VoxelSubsystem/ImageProcessorPool/ImageProcessorPool.h
index 6df35589c..7f7e6d7d6 100644
--- a/Source/Core/VSDA/EM/VoxelSubsystem/ImageProcessorPool/ImageProcessorPool.h
+++ b/Source/Core/VSDA/EM/VoxelSubsystem/ImageProcessorPool/ImageProcessorPool.h
@@ -78,7 +78,6 @@ private:
std::atomic_bool ThreadControlFlag_; /**Bool that signals threads to exit*/
-
/**
* @brief Thread safe enqueue function.
*
@@ -109,6 +108,9 @@ private:
public:
+ std::atomic<double> TotalConsumedMemory_MB; /**Counter for emrenderer to keep track of how much memory is being used by renders*/
+
+
/**
* @brief Initializes the imageprocessorpool with the given number of threads requested.
*
--
GitLab
From af942ae69a04f1f37f9fb2f9f1e932efe8a83dde Mon Sep 17 00:00:00 2001
From: Thomas Liao <tliao@carboncopies.org>
Date: Mon, 24 Mar 2025 19:29:20 -0700
Subject: [PATCH 3/5] Fix memory usage counter
---
Source/Core/VSDA/EM/EMRenderer.cpp | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/Source/Core/VSDA/EM/EMRenderer.cpp b/Source/Core/VSDA/EM/EMRenderer.cpp
index df35657be..2bf9bc21d 100644
--- a/Source/Core/VSDA/EM/EMRenderer.cpp
+++ b/Source/Core/VSDA/EM/EMRenderer.cpp
@@ -165,6 +165,9 @@ bool ExecuteSubRenderOperations(Config::Config* _Config, BG::Common::Logger::Log
}
+ // Incriment counter of memory usage
+ _ImageProcessorPool->TotalConsumedMemory_MB += MemorySize_MB;
+
// Now, we go through all of the steps in each direction that we identified, and calculate the bounding boxes for each
@@ -259,6 +262,11 @@ bool ExecuteSubRenderOperations(Config::Config* _Config, BG::Common::Logger::Log
_Simulation->VSDAData_.Array_ = std::make_unique<VoxelArray>(_Logger, Empty, 999.);
_Simulation->VSDAData_.State_ = VSDA_RENDER_DONE;
+
+ // Decrement memory usage counter
+ _ImageProcessorPool->TotalConsumedMemory_MB -= MemorySize_MB;
+
+
return true;
}
--
GitLab
From 58b58f3d8dcef6913fe3116abd3fdc01c2ac14d0 Mon Sep 17 00:00:00 2001
From: Thomas Liao <tliao@carboncopies.org>
Date: Mon, 24 Mar 2025 19:34:35 -0700
Subject: [PATCH 4/5] Bump memory consumption
---
Source/Data/NES.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Source/Data/NES.yaml b/Source/Data/NES.yaml
index 6a159af40..6ec7d6ca6 100644
--- a/Source/Data/NES.yaml
+++ b/Source/Data/NES.yaml
@@ -1,5 +1,5 @@
Network_NES_API_Port: 8001
Network_NES_API_Host: 0.0.0.0
-VSDA_EM_PercentOfSysteMemoryLimit: 45
+VSDA_EM_PercentOfSysteMemoryLimit: 70
VSDA_EM_MaxVoxelArraySize: 5000
\ No newline at end of file
--
GitLab
From f5bba4f2cb1a15f908484a2b718ba1d5051f35c0 Mon Sep 17 00:00:00 2001
From: Thomas Liao <tliao@carboncopies.org>
Date: Mon, 24 Mar 2025 19:56:40 -0700
Subject: [PATCH 5/5] Update config
---
ThirdParty/NetmorphCMake | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ThirdParty/NetmorphCMake b/ThirdParty/NetmorphCMake
index b98b2cc54..66bb13e45 160000
--- a/ThirdParty/NetmorphCMake
+++ b/ThirdParty/NetmorphCMake
@@ -1 +1 @@
-Subproject commit b98b2cc5480fcfdcf19dff2857b9177d886648e5
+Subproject commit 66bb13e457d52db985be86e262690ec7dc053b79
--
GitLab