VirtualFluids 0.2.0
Parallel CFD LBM Solver
Loading...
Searching...
No Matches
PrecursorWriter.cu
Go to the documentation of this file.
1//=======================================================================================
2// ____ ____ __ ______ __________ __ __ __ __
3// \ \ | | | | | _ \ |___ ___| | | | | / \ | |
4// \ \ | | | | | |_) | | | | | | | / \ | |
5// \ \ | | | | | _ / | | | | | | / /\ \ | |
6// \ \ | | | | | | \ \ | | | \__/ | / ____ \ | |____
7// \ \ | | |__| |__| \__\ |__| \________/ /__/ \__\ |_______|
8// \ \ | | ________________________________________________________________
9// \ \ | | | ______________________________________________________________|
10// \ \| | | | __ __ __ __ ______ _______
11// \ | | |_____ | | | | | | | | | _ \ / _____)
12// \ | | _____| | | | | | | | | | | \ \ \_______
13// \ | | | | |_____ | \_/ | | | | |_/ / _____ |
14// \ _____| |__| |________| \_______/ |__| |______/ (_______/
15//
16// This file is part of VirtualFluids. VirtualFluids is free software: you can
17// redistribute it and/or modify it under the terms of the GNU General Public
18// License as published by the Free Software Foundation, either version 3 of
19// the License, or (at your option) any later version.
20//
21// VirtualFluids is distributed in the hope that it will be useful, but WITHOUT
22// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
23// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
24// for more details.
25//
26// SPDX-License-Identifier: GPL-3.0-or-later
27// SPDX-FileCopyrightText: Copyright © VirtualFluids Project contributors, see AUTHORS.md in root folder
28//
29//! \addtogroup gpu_Samplers Samplers
30//! \ingroup gpu_core core
31//! \{
32//! \author Henrik Asmuth, Henry Korb
33//======================================================================================
34#include "PrecursorWriter.h"
35
36#include <algorithm>
37#include <limits>
38#include <memory>
39#include <thread>
40
41#include "basics/writer/WbWriterVtkXmlImageBinary.h"
42#include "logger/Logger.h"
43
44#include "Cuda/CudaMemoryManager.h"
45#include "Cuda/CudaStreamManager.h"
46#include "DataStructureInitializer/GridProvider.h"
47#include "Output/FilePartCalculator.h"
48#include "Parameter/Parameter.h"
49#include "StringUtilities/StringUtil.h"
50#include "Utilities/KernelUtilities.h"
51#include "cuda_helper/CudaGrid.h"
52#include "cuda_helper/CudaIndexCalculation.h"
53#include "helper_cuda.h"
54
55using namespace vf::lbm::dir;
56
57namespace vf::gpu {
58
59uint index1d(uint y, uint z, uint ny, uint nz)
60{
61 return y + ny * z;
62}
63
64constexpr uint linearIdx(const uint component, const uint node, const uint timeStep, const uint numberOfComponents,
65 const uint numberOfNodes)
66{
67 return node + numberOfNodes * (component + numberOfComponents * timeStep);
68}
69
70constexpr uint linearIdx(const uint component, const uint node, const uint numberOfNodes)
71{
72 return node + component * numberOfNodes;
73}
74
75////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
76__global__ void fillArrayVelocities(const uint numberOfPrecursorNodes,
77 const uint* indices,
78 real* precursorData,
79 const real* vx,
80 const real* vy,
81 const real* vz,
82 const real velocityRatio)
83
84{
85 const uint nodeIndex = vf::cuda::get1DIndexFrom2DBlock();
86
87 if (nodeIndex >= numberOfPrecursorNodes)
88 return;
89
90 precursorData[linearIdx(0U, nodeIndex, numberOfPrecursorNodes)] = vx[indices[nodeIndex]] * velocityRatio;
91 precursorData[linearIdx(1U, nodeIndex, numberOfPrecursorNodes)] = vy[indices[nodeIndex]] * velocityRatio;
92 precursorData[linearIdx(2U, nodeIndex, numberOfPrecursorNodes)] = vz[indices[nodeIndex]] * velocityRatio;
93}
94
95////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
96__global__ void fillArrayDistributions(const uint numberOfPrecursorNodes,
97 const uint* indices,
98 real* precursorData,
99 real* distributions,
100 const uint* neighborX,
101 const uint* neighborY,
102 const uint* neighborZ,
103 const bool isEvenTimeStep,
104 const unsigned long numberOfLBnodes)
105{
106 const uint nodeIndex = vf::cuda::get1DIndexFrom2DBlock();
107
108 if (nodeIndex >= numberOfPrecursorNodes)
109 return;
110
111 Distributions27 dist;
112 getPointersToDistributions(dist, distributions, numberOfLBnodes, isEvenTimeStep);
113
114 ////////////////////////////////////////////////////////////////////////////////
115 // ! - Set neighbor indices (necessary for indirect addressing)
116 const uint k_000 = indices[nodeIndex];
117 // uint k_M00 = neighborX[k_000];
118 const uint k_0M0 = neighborY[k_000];
119 const uint k_00M = neighborZ[k_000];
120 // uint k_MM0 = neighborY[k_M00];
121 // uint k_M0M = neighborZ[k_M00];
122 const uint k_0MM = neighborZ[k_0M0];
123 // uint k_MMM = neighborZ[k_MM0];
124
125 ////////////////////////////////////////////////////////////////////////////////////
126 //! - Get local distributions in PX directions
127 //!
128 precursorData[linearIdx(PrecursorWriter::dP00, nodeIndex, numberOfPrecursorNodes)] = (dist.f[dP00])[k_000];
129 precursorData[linearIdx(PrecursorWriter::dPP0, nodeIndex, numberOfPrecursorNodes)] = (dist.f[dPP0])[k_000];
130 precursorData[linearIdx(PrecursorWriter::dPM0, nodeIndex, numberOfPrecursorNodes)] = (dist.f[dPM0])[k_0M0];
131 precursorData[linearIdx(PrecursorWriter::dP0P, nodeIndex, numberOfPrecursorNodes)] = (dist.f[dP0P])[k_000];
132 precursorData[linearIdx(PrecursorWriter::dP0M, nodeIndex, numberOfPrecursorNodes)] = (dist.f[dP0M])[k_00M];
133 precursorData[linearIdx(PrecursorWriter::dPPP, nodeIndex, numberOfPrecursorNodes)] = (dist.f[dPPP])[k_000];
134 precursorData[linearIdx(PrecursorWriter::dPMP, nodeIndex, numberOfPrecursorNodes)] = (dist.f[dPMP])[k_0M0];
135 precursorData[linearIdx(PrecursorWriter::dPPM, nodeIndex, numberOfPrecursorNodes)] = (dist.f[dPPM])[k_00M];
136 precursorData[linearIdx(PrecursorWriter::dPMM, nodeIndex, numberOfPrecursorNodes)] = (dist.f[dPMM])[k_0MM];
137}
138__global__ void fillArrayVelocitiesTemperature(uint numberOfPrecursorNodes,
139 const uint* indices,
140 real* precursorData,
141 const real* vx,
142 const real* vy,
143 const real* vz,
144 const real* temperature,
145 const real velocityRatio)
146{
147 const uint nodeIndex = vf::cuda::get1DIndexFrom2DBlock();
148
149 if (nodeIndex >= numberOfPrecursorNodes)
150 return;
151
152 precursorData[linearIdx(0U, nodeIndex, numberOfPrecursorNodes)] = vx[indices[nodeIndex]] * velocityRatio;
153 precursorData[linearIdx(1U, nodeIndex, numberOfPrecursorNodes)] = vy[indices[nodeIndex]] * velocityRatio;
154 precursorData[linearIdx(2U, nodeIndex, numberOfPrecursorNodes)] = vz[indices[nodeIndex]] * velocityRatio;
155 precursorData[linearIdx(3U, nodeIndex, numberOfPrecursorNodes)] = temperature[indices[nodeIndex]];
156}
157
158__global__ void fillArrayVelocityAndTemperatureDistributions(
159 const uint numberOfPrecursorNodes,
160 const uint* indices,
161 real* precursorData,
162 real* velocityDistributions,
163 real* temperatureDistributions,
164 const uint* neighborX,
165 const uint* neighborY,
166 const uint* neighborZ,
167 const bool isEvenTimeStep,
168 const unsigned long numberOfLBnodes)
169{
170 const uint nodeIndex = vf::cuda::get1DIndexFrom2DBlock();
171
172 if (nodeIndex >= numberOfPrecursorNodes)
173 return;
174
175 Distributions27 velDist, tempDist;
176 getPointersToDistributions(velDist, velocityDistributions, numberOfLBnodes, isEvenTimeStep);
177 getPointersToDistributions(tempDist, temperatureDistributions, numberOfLBnodes, isEvenTimeStep);
178
179 const uint k_000 = indices[nodeIndex];
180 const uint k_0M0 = neighborY[k_000];
181 const uint k_00M = neighborZ[k_000];
182 const uint k_0MM = neighborZ[k_0M0];
183
184 // Velocity distributions (offset 0-8)
185 precursorData[linearIdx(PrecursorWriter::dP00, nodeIndex, numberOfPrecursorNodes)] = (velDist.f[dP00])[k_000];
186 precursorData[linearIdx(PrecursorWriter::dPP0, nodeIndex, numberOfPrecursorNodes)] = (velDist.f[dPP0])[k_000];
187 precursorData[linearIdx(PrecursorWriter::dPM0, nodeIndex, numberOfPrecursorNodes)] = (velDist.f[dPM0])[k_0M0];
188 precursorData[linearIdx(PrecursorWriter::dP0P, nodeIndex, numberOfPrecursorNodes)] = (velDist.f[dP0P])[k_000];
189 precursorData[linearIdx(PrecursorWriter::dP0M, nodeIndex, numberOfPrecursorNodes)] = (velDist.f[dP0M])[k_00M];
190 precursorData[linearIdx(PrecursorWriter::dPPP, nodeIndex, numberOfPrecursorNodes)] = (velDist.f[dPPP])[k_000];
191 precursorData[linearIdx(PrecursorWriter::dPMP, nodeIndex, numberOfPrecursorNodes)] = (velDist.f[dPMP])[k_0M0];
192 precursorData[linearIdx(PrecursorWriter::dPPM, nodeIndex, numberOfPrecursorNodes)] = (velDist.f[dPPM])[k_00M];
193 precursorData[linearIdx(PrecursorWriter::dPMM, nodeIndex, numberOfPrecursorNodes)] = (velDist.f[dPMM])[k_0MM];
194
195 // Temperature distributions (offset 9-17)
196 precursorData[linearIdx(PrecursorWriter::dP00 + 9, nodeIndex, numberOfPrecursorNodes)] = (tempDist.f[dP00])[k_000];
197 precursorData[linearIdx(PrecursorWriter::dPP0 + 9, nodeIndex, numberOfPrecursorNodes)] = (tempDist.f[dPP0])[k_000];
198 precursorData[linearIdx(PrecursorWriter::dPM0 + 9, nodeIndex, numberOfPrecursorNodes)] = (tempDist.f[dPM0])[k_0M0];
199 precursorData[linearIdx(PrecursorWriter::dP0P + 9, nodeIndex, numberOfPrecursorNodes)] = (tempDist.f[dP0P])[k_000];
200 precursorData[linearIdx(PrecursorWriter::dP0M + 9, nodeIndex, numberOfPrecursorNodes)] = (tempDist.f[dP0M])[k_00M];
201 precursorData[linearIdx(PrecursorWriter::dPPP + 9, nodeIndex, numberOfPrecursorNodes)] = (tempDist.f[dPPP])[k_000];
202 precursorData[linearIdx(PrecursorWriter::dPMP + 9, nodeIndex, numberOfPrecursorNodes)] = (tempDist.f[dPMP])[k_0M0];
203 precursorData[linearIdx(PrecursorWriter::dPPM + 9, nodeIndex, numberOfPrecursorNodes)] = (tempDist.f[dPPM])[k_00M];
204 precursorData[linearIdx(PrecursorWriter::dPMM + 9, nodeIndex, numberOfPrecursorNodes)] = (tempDist.f[dPMM])[k_0MM];
205}
206
207////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
208void PrecursorWriter::init()
209{
210 VF_LOG_INFO("PrecursorWriter: Start initializing...");
211 VF_LOG_INFO("Writing yz-planes at x={}m every {}. timestep, starting at t={}", this->xPos, this->tSave,
212 this->tStartWritingOutput);
213 VF_LOG_INFO("Writing to {} with prefix {}", outputPath, probeName);
214
215 precursorStructs.resize(para->getMaxLevel() + 1);
216 for (int level = 0; level <= para->getMaxLevel(); level++) {
217 const real deltaX = para->getScaledLengthRatio(level);
218
219 real lowestY = std::numeric_limits<real>::max();
220 real highestY = std::numeric_limits<real>::min();
221
222 real lowestZ = std::numeric_limits<real>::max();
223 real highestZ = std::numeric_limits<real>::min();
224
225 std::vector<uint> indicesOnGrid;
226 std::vector<real> coordY, coordZ;
227
228 for (size_t pos = 1; pos < para->getParH(level)->numberOfNodes; pos++) {
229 const real pointCoordX = para->getParH(level)->coordinateX[pos];
230 const real pointCoordY = para->getParH(level)->coordinateY[pos];
231 const real pointCoordZ = para->getParH(level)->coordinateZ[pos];
232 if (para->getParH(level)->typeOfGridNode[pos] == GEO_FLUID && pointCoordX < (deltaX + xPos) &&
233 pointCoordX >= xPos && pointCoordY <= yMax && pointCoordY >= yMin && pointCoordZ <= zMax &&
234 pointCoordZ >= zMin) {
235 highestY = std::max(highestY, pointCoordY);
236 highestZ = std::max(highestZ, pointCoordZ);
237
238 lowestY = std::min(lowestY, pointCoordY);
239 lowestZ = std::min(lowestZ, pointCoordZ);
240 indicesOnGrid.push_back((uint)pos);
241 coordY.push_back(pointCoordY);
242 coordZ.push_back(pointCoordZ);
243 }
244 }
245 if (indicesOnGrid.size() == 0)
246 throw std::runtime_error("PrecursorWriter did not find any points on the grid");
247
248 const uint ny = uint(std::round((highestY - lowestY) / deltaX)) + 1;
249 const uint nz = uint(std::round((highestZ - lowestZ) / deltaX)) + 1;
250
251 precursorStructs[level].indicesOnPlane.reserve(indicesOnGrid.size());
252
253 for (uint i = 0; i < indicesOnGrid.size(); i++) {
254 const uint idxY = uint(std::round((coordY[i] - lowestY) / deltaX));
255 const uint idxZ = uint(std::round((coordZ[i] - lowestZ) / deltaX));
256 precursorStructs[level].indicesOnPlane.push_back(index1d(idxY, idxZ, ny, nz));
257 }
258
259 uint numberOfQuantities{};
260 if (outputVariable == OutputVariable::Velocities)
261 numberOfQuantities += 3;
262 else if (outputVariable == OutputVariable::Distributions)
263 numberOfQuantities += 9;
264
265 if (tempOutputVariable == OutputVariableTemperature::Temperature)
266 numberOfQuantities += 1;
267 else if (tempOutputVariable == OutputVariableTemperature::Distributions)
268 numberOfQuantities += 9;
269 precursorStructs[level].numberOfQuantities = numberOfQuantities;
270 precursorStructs[level].numberOfPointsInBC = (uint)indicesOnGrid.size();
271 precursorStructs[level].spacing = makeUbTuple(deltaX, deltaX, tSave * para->getScaledTimeRatio(level));
272 precursorStructs[level].origin = makeUbTuple(lowestY, lowestZ);
273 precursorStructs[level].extent = makeUbTuple(0, (int)ny - 1, 0, (int)nz - 1);
274 precursorStructs[level].numberOfPointsInData = ny * nz;
275 const uint maxNodes = FilePartCalculator::limitOfNodesForVTK / (ny * nz);
276 const uint maxOffsets = std::numeric_limits<int>::max() / (numberOfQuantities * ny * nz * (uint)sizeof(double));
277 precursorStructs[level].numberOfTimeStepsPerFile = std::min(std::min(maxNodes, maxTimestepsPerFile), maxOffsets);
278 precursorStructs[level].numberOfFilesWritten = 0;
279 precursorStructs[level].numberOfTimeStepsBuffered = 0;
280 precursorStructs[level].streamIndex =
281 para->getStreamManager()->registerAndLaunchStream(CudaStreamIndex::PrecursorWriter);
282
283 cudaMemoryManager->cudaAllocPrecursorWriter(this, level);
284
285 std::copy(indicesOnGrid.begin(), indicesOnGrid.end(), precursorStructs[level].indicesH);
286
287 cudaMemoryManager->cudaCopyPrecursorWriterIndicesHtoD(this, level);
288
289 VF_LOG_INFO("Found {} points in precursor plane on level {}. {} timesteps per file.",
290 precursorStructs[level].numberOfPointsInBC, level, precursorStructs[level].numberOfTimeStepsPerFile);
291 }
292 VF_LOG_INFO("PrecursorWriter: Done initializing.");
293}
294
295void PrecursorWriter::sample(int level, uint t)
296{
297 const uint t_level = para->getTimeStep(level, t, true);
298 const uint tStartOut_level = tStartWritingOutput * std::exp2(level);
299 auto* levelStruct = getPrecursorStruct(level);
300
301 if (t_level < tStartOut_level || ((t_level - tStartOut_level) % tSave) != 0) return;
302
303 vf::cuda::CudaGrid grid(para->getParH(level)->numberofthreads, levelStruct->numberOfPointsInBC);
304 auto* stream = para->getStreamManager()->getStream(CudaStreamIndex::PrecursorWriter, levelStruct->streamIndex);
305 if (outputVariable == OutputVariable::Velocities &&
306 tempOutputVariable == OutputVariableTemperature::Temperature) {
307 fillArrayVelocitiesTemperature<<<grid.grid, grid.threads, 0, stream>>>(levelStruct->numberOfPointsInBC, levelStruct->indicesD,
308 levelStruct->bufferD, para->getParD(level)->velocityX,
309 para->getParD(level)->velocityY,
310 para->getParD(level)->velocityZ, para->getParD(level)->concentration,
311 para->getVelocityRatio());
312 } else if (outputVariable == OutputVariable::Distributions &&
313 tempOutputVariable == OutputVariableTemperature::Distributions) {
314 fillArrayVelocityAndTemperatureDistributions<<<grid.grid, grid.threads, 0, stream>>>(
315 levelStruct->numberOfPointsInBC, levelStruct->indicesD, levelStruct->bufferD,
316 para->getParD(level)->distributions.f[0], para->getParD(level)->distributionsAD.f[0],
317 para->getParD(level)->neighborX, para->getParD(level)->neighborY,
318 para->getParD(level)->neighborZ, para->getEvenOrOdd(level),
319 para->getParD(level)->numberOfNodes);
320 getLastCudaError("In PrecursorWriter::interact fillArrayVelocityAndTemperatureDistributions execution failed");
321 } else if (outputVariable == OutputVariable::Velocities && tempOutputVariable == OutputVariableTemperature::None) {
322 fillArrayVelocities<<<grid.grid, grid.threads, 0, stream>>>(levelStruct->numberOfPointsInBC, levelStruct->indicesD,
323 levelStruct->bufferD, para->getParD(level)->velocityX,
324 para->getParD(level)->velocityY,
325 para->getParD(level)->velocityZ, para->getVelocityRatio());
326 getLastCudaError("In PrecursorWriter::interact fillArrayVelocities execution failed");
327 } else if (outputVariable == OutputVariable::Distributions && tempOutputVariable == OutputVariableTemperature::None) {
328 fillArrayDistributions<<<grid.grid, grid.threads, 0, stream>>>(
329 levelStruct->numberOfPointsInBC, levelStruct->indicesD, levelStruct->bufferD,
330 para->getParD(level)->distributions.f[0], para->getParD(level)->neighborX, para->getParD(level)->neighborY,
331 para->getParD(level)->neighborZ, para->getEvenOrOdd(level), para->getParD(level)->numberOfNodes);
332 getLastCudaError("In PrecursorWriter::interact fillArrayDistributions execution failed");
333 }
334 cudaMemoryManager->cudaCopyPrecursorWriterOutputVariablesDtoH(this, level);
335
336 // switch device buffer and data pointer so precursor data is gathered in buffer and copied from bufferD to bufferH
337 std::swap(levelStruct->bufferD, levelStruct->dataD);
338
339 levelStruct->numberOfTimeStepsBuffered++;
340
341 if (levelStruct->numberOfTimeStepsBuffered >= levelStruct->numberOfTimeStepsPerFile || t == para->getTimestepEnd()) {
342 // switch host buffer and data pointer so precursor data is copied in buffer and written from data
343 std::swap(levelStruct->bufferH, levelStruct->dataH);
344 if(writingThread.joinable())
345 writingThread.join();
346 writingThread = std::thread([this](uint level, uint timeSteps) { this->write(level, timeSteps); }, level,
347 levelStruct->numberOfTimeStepsBuffered);
348 levelStruct->numberOfTimeStepsBuffered = 0;
349 }
350}
351
352PrecursorWriter::~PrecursorWriter()
353{
354 if(writingThread.joinable())
355 writingThread.join();
356 for (int level = 0; level <= para->getMaxLevel(); level++) {
357 auto* levelStruct = getPrecursorStruct(level);
358 std::swap(levelStruct->bufferH, levelStruct->dataH);
359 if (getPrecursorStruct(level)->numberOfTimeStepsBuffered > 0)
360 write(level, levelStruct->numberOfTimeStepsBuffered);
361
362 cudaMemoryManager->cudaFreePrecursorWriter(this, level);
363 }
364}
365
366void PrecursorWriter::write(int level, uint numberOfTimeStepsBuffered)
367{
368 auto* levelStruct = getPrecursorStruct(level);
369
370 cudaStreamSynchronize(para->getStreamManager()->getStream(CudaStreamIndex::PrecursorWriter, levelStruct->streamIndex));
371 const std::string fileName = makeFileName(level, levelStruct->numberOfFilesWritten);
372
373 const std::size_t numberOfPointsInData = levelStruct->numberOfPointsInData;
374
375 const int startTime = levelStruct->numberOfFilesWritten * levelStruct->numberOfTimeStepsPerFile;
376
377 UbTupleInt6 extent = makeUbTuple(val<1>(levelStruct->extent), val<2>(levelStruct->extent),
378 val<3>(levelStruct->extent), val<4>(levelStruct->extent),
379 startTime, startTime + (int)numberOfTimeStepsBuffered - 1);
380
381 UbTupleFloat3 origin = makeUbTuple(static_cast<float>(val<1>(levelStruct->origin)),
382 static_cast<float>(val<2>(levelStruct->origin)), 0.F);
383
384 std::vector<std::vector<double>> nodeData;
385
386 for (uint quant = 0; quant < levelStruct->numberOfQuantities; quant++) {
387 std::vector<double> doubleArr(numberOfPointsInData * numberOfTimeStepsBuffered, NAN);
388 for (uint timeStep = 0; timeStep < numberOfTimeStepsBuffered; timeStep++) {
389 for (uint pos = 0; pos < levelStruct->numberOfPointsInBC; pos++) {
390 const std::size_t indexOnPlane = levelStruct->indicesOnPlane[pos] + timeStep * numberOfPointsInData;
391 doubleArr[indexOnPlane] = double(
392 levelStruct->dataH[linearIdx(quant, pos, timeStep, levelStruct->numberOfQuantities,
393 levelStruct->numberOfPointsInBC)]);
394 }
395 }
396 nodeData.push_back(doubleArr);
397 }
398
399 std::vector<std::vector<double>> cellData;
400 getWriter()->writeData(fileName, nodeDataNames, cellDataNames, nodeData, cellData, extent, origin,
401 levelStruct->spacing, extent, this->writePrecision);
402 VF_LOG_INFO("PrecursorWriter: Wrote file {}", fileName);
403 levelStruct->numberOfFilesWritten++;
404}
405
406std::string PrecursorWriter::makeFileName(int level, uint numberOfFilesWritten)
407{
408 return outputPath + probeName + "_lev_" + StringUtil::toString<int>(level) + "_ID_" +
409 StringUtil::toString<int>(para->getMyProcessID()) + "_File_" + StringUtil::toString<int>(numberOfFilesWritten) +
410 getWriter()->getFileExtension();
411}
412
413void PrecursorWriter::getTaggedFluidNodes(GridProvider* gridProvider)
414{
415 for (uint level = 0; level < (uint)para->getMaxLevel(); level++) {
416 if (outputVariable == OutputVariable::Velocities || tempOutputVariable == OutputVariableTemperature::Temperature) {
417 std::vector<uint> indices(precursorStructs[level].indicesH,
418 precursorStructs[level].indicesH + precursorStructs[level].numberOfPointsInBC);
419 gridProvider->tagFluidNodeIndices(indices, CollisionTemplate::WriteMacroVars, level);
420 }
421 }
422}
423
424}
425
426//! \}