Merge pull request #455 from dangglxxx/upstream

ODE GPU solver is accessed by DeepFlame!

Author: maorz1998

applications/solvers/dfLowMachFoam/Make/options

@@ -28,7 +28,9 @@ EXE_INC = -std=c++14 \
     $(PYTHON_INC_DIR) \
     $(if $(AMGX_DIR), -I$(DF_ROOT)/src_gpu,) \
     $(if $(AMGX_DIR), -I/usr/local/cuda/include,) \
-    $(if $(AMGX_DIR), -I$(AMGX_DIR)/include,)
+    $(if $(AMGX_DIR), -I$(AMGX_DIR)/include,) \
+    $(if $(ODE_GPU_SOLVER), -I$(OPENCC_PATH)/include,) \
+    $(if $(ODE_GPU_SOLVER), -DODE_GPU_SOLVER,)
 
 EXE_LIBS = \
     -lcompressibleTransportModels \
@@ -52,4 +54,5 @@ EXE_LIBS = \
     $(if $(AMGX_DIR), /usr/local/cuda/lib64/libcudart.so,) \
 	$(if $(AMGX_DIR), /usr/local/cuda/lib64/libnccl.so,) \
     $(if $(AMGX_DIR), $(DF_ROOT)/src_gpu/build/libdfMatrix.so,) \
-    $(if $(AMGX_DIR), $(AMGX_DIR)/build/libamgxsh.so,)
+    $(if $(AMGX_DIR), $(AMGX_DIR)/build/libamgxsh.so,) \
+    $(if $(ODE_GPU_SOLVER), $(ODE_GPU_SOLVER)/lib/libopencc.so,)

applications/solvers/dfLowMachFoam/YEqn.H

@@ -85,6 +85,24 @@ if(flag_mpi_init) MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM);
             start1 = std::clock();
             tmp<volScalarField> DEff = chemistry->rhoD(i) + turbulence->mut()/Sct;
 
+            #ifdef ODE_GPU_SOLVER
+            fvScalarMatrix YiEqn
+            (
+                fvm::ddt(rho, Yi)
+            +
+                (
+                    turbName == "laminar"
+                    ?  (mvConvection->fvmDiv(phi, Yi) + mvConvection->fvmDiv(phiUc, Yi))
+                    :   mvConvection->fvmDiv(phi, Yi)
+                )
+            ==
+                (
+                    splitting
+                    ?   fvm::laplacian(DEff(), Yi)
+                    :  (fvm::laplacian(DEff(), Yi) + RR_GPU[i])
+                    )
+            );
+            #else
             fvScalarMatrix YiEqn
             (
                 fvm::ddt(rho, Yi)
@@ -101,6 +119,7 @@ if(flag_mpi_init) MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM);
                     :  (fvm::laplacian(DEff(), Yi) + combustion->R(Yi))
                     )
             );
+            #endif
 
             end1 = std::clock();
             time_monitor_YEqn_mtxAssembly += double(end1 - start1) / double(CLOCKS_PER_SEC);
@@ -264,7 +283,43 @@ if(flag_mpi_init) MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM);
     if (!splitting)
     {
         std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now();
+        
+        #ifdef ODE_GPU_SOLVER
+	scalar dt = runTime.deltaTValue();
+
+        memcpy(h_T, &T[0], num_cells * sizeof(double));
+        memcpy(h_p, &p[0], num_cells * sizeof(double));
+
+        forAll(Y, speciesI) {
+            volScalarField& Yi = Y[speciesI];
+            memcpy(h_y + speciesI * num_cells, &Yi[0], num_cells * sizeof(double));
+        }
+
+        for (int i = 0; i < num_cells; i++) {
+            for (int j = 0; j < sp_num; j++) {
+                h_y_t[j + i*sp_num] = h_y[i + j*num_cells];
+            }
+        }
+
+        opencc_ode_all(h_T, h_p, h_y_t, 1e-10, dt, CPU);
+
+        for (int i = 0; i < num_cells; i++) {
+            for (int j = 0; j < sp_num; j++) {
+                Ynew[i + j*num_cells] = h_y_t[j + i*sp_num];
+            }
+        }
+
+        QdotGPU = Zero;
+        forAll(QdotGPU,celli)
+        {
+            for (int sp = 0; sp < sp_num; sp++)
+            {
+                RRGPU[sp][celli] = (Ynew[sp*num_cells+celli]-Y[sp][celli])*rho[celli]/dt;
+            }
+        }
+        #else
         combustion->correct();
+        #endif
         //label flag_mpi_init;
         //MPI_Initialized(&flag_mpi_init);
         if(flag_mpi_init) MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM);

applications/solvers/dfLowMachFoam/createFields_GPU.H

@@ -0,0 +1,71 @@
+/*******************************************************************************************
+ * This file is used to add OpenCC calls to OpenFoam, 
+ * initialize OpenCC scopes and request GPU space.
+ * 
+ * @author Lynn Dang
+ ******************************************************************************************/
+
+volScalarField QdotGPU
+(
+    IOobject
+    (
+        "QdotGPU",
+        mesh.time().timeName(),
+        mesh,
+        IOobject::READ_IF_PRESENT,
+        IOobject::AUTO_WRITE
+    ),
+    mesh,
+    dimensionedScalar(dimEnergy/dimVolume/dimTime, 0)
+);
+
+PtrList<volScalarField::Internal> RRGPU(Y.size());
+forAll(RRGPU, fieldi)
+{
+    RRGPU.set
+    (
+        fieldi,
+        new volScalarField::Internal
+        (
+            IOobject
+            (
+                "RRGPU." + Y[fieldi].name(),
+                mesh.time().timeName(),
+                mesh,
+                IOobject::NO_READ,
+                IOobject::NO_WRITE
+            ),
+            mesh,
+            dimensionedScalar(dimMass/dimVolume/dimTime, 0)
+        )
+    );
+}
+
+int num_cells = T.size();
+int num_species = Y.size();
+
+double* h_T   = new double[num_cells];
+double* h_p   = new double[num_cells];
+double* h_y   = new double[num_cells * num_species];
+double* h_y_t = new double[num_cells * num_species];
+int* h_size = new int[1];
+
+memcpy(h_T, &T[0], num_cells * sizeof(double));
+memcpy(h_p, &p[0], num_cells * sizeof(double));
+
+forAll(Y, speciesI) {
+    volScalarField& Yi = Y[speciesI];
+    memcpy(h_y + speciesI * num_cells, &Yi[0], num_cells * sizeof(double));
+}
+
+h_size[0] = num_cells;
+
+int sp_num = num_species;
+
+string mechanismFile = CanteraTorchProperties.lookupOrDefault("CanteraMechanismFile", string(""));
+char target_mechanismFile[100]; 
+std::strcpy(target_mechanismFile, mechanismFile.c_str()); 
+
+opencc_ode_init(target_mechanismFile, num_cells, h_T, h_p, h_y);
+
+double* Ynew = new double[num_cells * num_species];

applications/solvers/dfLowMachFoam/dfLowMachFoam.C

@@ -49,6 +49,10 @@ Description
 #include "DNNInferencer.H"
 #endif
 
+#ifdef ODE_GPU_SOLVER
+#include "opencc.h"
+#endif
+
 #include "fvCFD.H"
 #include "fluidThermo.H"
 #include "turbulentFluidThermoModel.H"
@@ -137,6 +141,10 @@ int main(int argc, char *argv[])
     #include "createFields.H"
     #include "createRhoUfIfPresent.H"
 
+    #ifdef ODE_GPU_SOLVER
+    #include "createFields_GPU.H"
+    #endif
+
     double time_monitor_init = 0;
 
     double time_monitor_other = 0;

bashrc.in

@@ -9,11 +9,12 @@ export CANTERA_DATA=$CANTERA_ROOT/share/cantera/data
 export LD_LIBRARY_PATH=$LIBTORCH_ROOT/lib:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=$CANTERA_ROOT/lib:$LD_LIBRARY_PATH
 export AMGX_DIR=@AMGX_DIR@
+export ODE_GPU_SOLVER=@ODE_GPU_SOLVER@
 
 export DF_APPBIN=pwd/platforms/$WM_OPTIONS/bin
 export DF_LIBBIN=pwd/platforms/$WM_OPTIONS/lib
 export PATH=$DF_APPBIN:$PATH
 export LD_LIBRARY_PATH=$DF_LIBBIN:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=$DF_ROOT/src_gpu/build:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=$AMGX_DIR/build:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=$DF_ROOT/src/dfChemistryModel/DNNInferencer/build:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$DF_ROOT/src/dfChemistryModel/DNNInferencer/build:$LD_LIBRARY_PATH

configure.sh

@@ -9,17 +9,19 @@ unset PYTORCH_INC
 unset PYTORCH_LIB
 unset USE_GPUSOLVER
 unset AMGX_DIR
+unset ODE_GPU_SOLVER
 
 print_usage() {
     echo "Usage: . install.sh --libtorch_no (default) | --libtorch_dir _path_to_libtorch | --libtorch_autodownload | --use_pytorch | --libcantera_dir _path_to_libcantera
-        | --amgx_dir _path_to_amgx"
+        | --amgx_dir _path_to_amgx | --use_ode_gpu_solver"
 }
 
 # default
 LIBTORCH_AUTO=false
 USE_LIBTORCH=false
 USE_PYTORCH=false
 USE_GPUSOLVER=false
+USE_ODE_GPU_SOLVER=false
 
 while test $# -gt 0; do
     case "$1" in
@@ -75,6 +77,10 @@ while test $# -gt 0; do
             fi
             shift
             ;;
+        --use_ode_gpu_solver)
+            shift
+            USE_ODE_GPU_SOLVER=true
+            ;;
         -h|--help)
             shift
             print_usage
@@ -163,6 +169,9 @@ fi
 if [ $USE_GPUSOLVER = true ]; then
     echo AMGX_DIR=$AMGX_DIR
 fi
+if [ $USE_ODE_GPU_SOLVER = true ]; then
+    echo ODE_GPU_SOLVER=$OPENCC_PATH
+fi
 
 cp bashrc.in bashrc
 sed -i "s#pwd#$PWD#g" ./bashrc
@@ -171,6 +180,7 @@ sed -i "s#PYTORCH_INC#$PYTORCH_INC#g" ./bashrc
 sed -i "s#PYTORCH_LIB#$PYTORCH_LIB#g" ./bashrc
 sed -i "s#LIBCANTERA_DIR#$LIBCANTERA_DIR#g" ./bashrc
 sed -i "s#@AMGX_DIR@#$AMGX_DIR#g" ./bashrc
+sed -i "s#@ODE_GPU_SOLVER@#$OPENCC_PATH#g" ./bashrc
 
 
 
@@ -190,4 +200,4 @@ else
     cp -r $FOAM_SRC/lagrangian/turbulence src_orig/lagrangian
     cp -r $FOAM_SRC/regionModels/surfaceFilmModels src_orig/regionModels
     cp -r $FOAM_SRC/functionObjects/field src_orig/functionObjects
-fi
+fi