diff --git a/.gitignore b/.gitignore
index 4d26a1431b..54fa8c3806 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 # Configured outputs
 Makefile
 src/defs.hpp
+configure.log
 
 # Object and binary directories
 obj/
diff --git a/Makefile.in b/Makefile.in
index 6f0d122be3..b8730a4df3 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -30,6 +30,7 @@ SRC_FILES := $(wildcard src/*.cpp) \
 	     $(wildcard src/bvals/cc/fft_grav/*.cpp) \
 	     $(wildcard src/bvals/cc/hydro/*.cpp) \
 	     $(wildcard src/bvals/cc/mg/*.cpp) \
+             $(wildcard src/bvals/cc/nr_radiation/*.cpp) \
 	     $(wildcard src/bvals/fc/*.cpp) \
 	     $(wildcard src/bvals/orbital/*.cpp) \
 	     $(wildcard src/bvals/utils/*.cpp) \
@@ -45,6 +46,12 @@ SRC_FILES := $(wildcard src/*.cpp) \
 	     $(wildcard src/hydro/*.cpp) \
 	     $(wildcard src/hydro/srcterms/*.cpp) \
 	     $(wildcard src/hydro/hydro_diffusion/*.cpp) \
+             $(wildcard src/nr_radiation/*.cpp) \
+             $(wildcard src/nr_radiation/integrators/*.cpp) \
+             $(wildcard src/nr_radiation/integrators/srcterms/*.cpp) \
+             $(wildcard src/nr_radiation/implicit/*.cpp) \
+             $(wildcard src/cr/*.cpp) \
+             $(wildcard src/cr/integrators/*.cpp) \
 	     src/hydro/rsolvers/$(RSOLVER_DIR)$(RSOLVER_FILE) \
 	     $(wildcard src/inputs/*.cpp) \
 	     $(wildcard src/mesh/*.cpp) \
diff --git a/README.md b/README.md
index ab68957d2c..d8853eb4af 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ athena
 	  <img width="345" height="345" src="https://user-images.githubusercontent.com/1410981/115276281-759d8580-a108-11eb-9fc9-833480b97f95.png">
 </p>
 
-Athena++ GRMHD code and adaptive mesh refinement (AMR) framework
+Athena++ radiation GRMHD code and adaptive mesh refinement (AMR) framework
 
 Please read [our contributing guidelines](./CONTRIBUTING.md) for details on how to participate.
 
diff --git a/configure.py b/configure.py
index 80a5244240..bbf88bf9c9 100755
--- a/configure.py
+++ b/configure.py
@@ -7,36 +7,39 @@
 # Makefile.in and src/defs.hpp.in respectively.
 #
 # The following options are implememted:
-#   -h  --help        help message
-#   --prob=name       use src/pgen/name.cpp as the problem generator
-#   --coord=xxx       use xxx as the coordinate system
-#   --eos=xxx         use xxx as the equation of state
-#   --flux=xxx        use xxx as the Riemann solver
-#   --nghost=xxx      set NGHOST=xxx
-#   --nscalars=xxx    set NSCALARS=xxx
-#   -eos_table        enable EOS table
-#   -b                enable magnetic fields
-#   -s                enable special relativity
-#   -g                enable general relativity
-#   -t                enable interface frame transformations for GR
-#   -debug            enable debug flags (-g -O0); override other compiler options
-#   -coverage         enable compiler-dependent code coverage flags
-#   -float            enable single precision (default is double)
-#   -mpi              enable parallelization with MPI
-#   -omp              enable parallelization with OpenMP
-#   -hdf5             enable HDF5 output (requires the HDF5 library)
-#   --hdf5_path=path  path to HDF5 libraries (requires the HDF5 library)
-#   -fft              enable FFT (requires the FFTW library)
-#   --fftw_path=path  path to FFTW libraries (requires the FFTW library)
-#   --grav=xxx        use xxx as the self-gravity solver
-#   --cxx=xxx         use xxx as the C++ compiler (works w/ or w/o -mpi)
-#   --ccmd=name       use name as the command to call the (non-MPI) C++ compiler
-#   --mpiccmd=name    use name as the command to call the MPI C++ compiler
-#   --gcovcmd=name    use name as the command to call the gcov utility
-#   --cflag=string    append string whenever invoking compiler/linker
-#   --include=path    use -Ipath when compiling
-#   --lib_path=path   use -Lpath when linking
-#   --lib=xxx         use -lxxx when linking
+#   -h  --help           help message
+#   --prob=name          use src/pgen/name.cpp as the problem generator
+#   --coord=xxx          use xxx as the coordinate system
+#   --eos=xxx            use xxx as the equation of state
+#   --flux=xxx           use xxx as the Riemann solver
+#   --nghost=xxx         set NGHOST=xxx
+#   --nscalars=xxx       set NSCALARS=xxx
+#   -eos_table           enable EOS table
+#   -b                   enable magnetic fields
+#   -s                   enable special relativity
+#   -g                   enable general relativity
+#   -t                   enable interface frame transformations for GR
+#   -debug               enable debug flags (-g -O0); override other compiler options
+#   -coverage            enable compiler-dependent code coverage flags
+#   -float               enable single precision (default is double)
+#   -mpi                 enable parallelization with MPI
+#   -omp                 enable parallelization with OpenMP
+#   -hdf5                enable HDF5 output (requires the HDF5 library)
+#   --hdf5_path=path     path to HDF5 libraries (requires the HDF5 library)
+#   -fft                 enable FFT (requires the FFTW library)
+#   --fftw_path=path     path to FFTW libraries (requires the FFTW library)
+#   --grav=xxx           use xxx as the self-gravity solver
+#   --cxx=xxx            use xxx as the C++ compiler (works w/ or w/o -mpi)
+#   --ccmd=name          use name as the command to call the (non-MPI) C++ compiler
+#   --mpiccmd=name       use name as the command to call the MPI C++ compiler
+#   --gcovcmd=name       use name as the command to call the gcov utility
+#   --cflag=string       append string whenever invoking compiler/linker
+#   --include=path       use -Ipath when compiling
+#   --lib_path=path      use -Lpath when linking
+#   --lib=xxx            use -lxxx when linking
+#   -nr_radiation        turn on non-relativistic radiation transport
+#   -implicit_radiation  implicit radiation transport module
+#   -cr                  enable cosmic ray transport
 # ----------------------------------------------------------------------------------------
 
 # Modules
@@ -205,6 +208,24 @@
                     default='',
                     help='path to HDF5 libraries')
 
+# -nr_radiation argument
+parser.add_argument('-nr_radiation',
+                    action='store_true',
+                    default=False,
+                    help='enable non-relativistic radiative transfer')
+
+# -implicit_radiation argument
+parser.add_argument('-implicit_radiation',
+                    action='store_true',
+                    default=False,
+                    help='enable radiative transfer')
+
+# -cosmic ray argument
+parser.add_argument('-cr',
+                    action='store_true',
+                    default=False,
+                    help='enable cosmic ray transport')
+
 # The main choices for --cxx flag, using "ctype[-suffix]" formatting, where "ctype" is the
 # major family/suite/group of compilers and "suffix" may represent variants of the
 # compiler version and/or predefined sets of compiler options. The C++ compiler front ends
@@ -350,6 +371,14 @@ def c_to_cpp(arg):
         raise SystemExit('### CONFIGURE ERROR: '
                          + 'General EOS is incompatible with flux ' + args['flux'])
 
+if args['g'] and (args['nr_radiation'] or args['implicit_radiation']):
+    raise SystemExit('### CONFIGURE ERROR: '
+                     + ' GR is incompatible with nr_radiation or implicit_radiation')
+
+if args['nr_radiation'] and args['implicit_radiation']:
+    raise SystemExit('### CONFIGURE ERROR: '
+                     + ' nr_radiation and implicit_radiation cannot be used together')
+
 # --- Step 3. Set definitions and Makefile options based on above argument
 
 # Prepare dictionaries of substitutions to be made
@@ -444,6 +473,31 @@ def c_to_cpp(arg):
     if not args['t']:
         makefile_options['RSOLVER_FILE'] += '_no_transform'
 
+
+# -radiation argument
+definitions['NRAD_VARIABLES'] = '0'
+
+if args['nr_radiation']:
+    definitions['NR_RADIATION_ENABLED'] = '1'
+    definitions['NRAD_VARIABLES'] = '14'
+else:
+    definitions['NR_RADIATION_ENABLED'] = '0'
+
+if args['implicit_radiation']:
+    definitions['IM_RADIATION_ENABLED'] = '1'
+    definitions['NRAD_VARIABLES'] = '14'
+else:
+    definitions['IM_RADIATION_ENABLED'] = '0'
+
+# -cr argument
+definitions['NCR_VARIABLES'] = '0'
+if args['cr']:
+    definitions['CR_ENABLED'] = '1'
+    definitions['NCR_VARIABLES'] = '4'
+else:
+    definitions['CR_ENABLED'] = '0'
+
+
 # --cxx=[name] argument
 if args['cxx'] == 'g++':
     # GCC is C++11 feature-complete since v4.8.1 (2013-05-31)
@@ -811,30 +865,52 @@ def c_to_cpp(arg):
 elif args['grav'] == 'mg':
     self_grav_string = 'Multigrid'
 
-print('Your Athena++ distribution has now been configured with the following options:')
-print('  Problem generator:          ' + args['prob'])
-print('  Coordinate system:          ' + args['coord'])
-print('  Equation of state:          ' + args['eos'])
-print('  Riemann solver:             ' + args['flux'])
-print('  Magnetic fields:            ' + ('ON' if args['b'] else 'OFF'))
-print('  Number of scalars:          ' + args['nscalars'])
-print('  Special relativity:         ' + ('ON' if args['s'] else 'OFF'))
-print('  General relativity:         ' + ('ON' if args['g'] else 'OFF'))
-print('  Frame transformations:      ' + ('ON' if args['t'] else 'OFF'))
-print('  Self-Gravity:               ' + self_grav_string)
-print('  Super-Time-Stepping:        ' + ('ON' if args['sts'] else 'OFF'))
-print('  Debug flags:                ' + ('ON' if args['debug'] else 'OFF'))
-print('  Code coverage flags:        ' + ('ON' if args['coverage'] else 'OFF'))
-print('  Linker flags:               ' + makefile_options['LINKER_FLAGS'] + ' '
-      + makefile_options['LIBRARY_FLAGS'])
-print('  Floating-point precision:   ' + ('single' if args['float'] else 'double'))
-print('  Number of ghost cells:      ' + args['nghost'])
-print('  MPI parallelism:            ' + ('ON' if args['mpi'] else 'OFF'))
-print('  OpenMP parallelism:         ' + ('ON' if args['omp'] else 'OFF'))
-print('  FFT:                        ' + ('ON' if args['fft'] else 'OFF'))
-print('  HDF5 output:                ' + ('ON' if args['hdf5'] else 'OFF'))
+
+def output_config(opt_descr, opt_choice, filehandle=None):
+    first_col_width = 32
+    first_col_indent = 2
+    descr_len = len(opt_descr)
+    right_pad_len = first_col_width - (descr_len + first_col_indent + 2)  # include colon
+    right_pad = right_pad_len*' ' if right_pad_len >= 0 else ''
+    line_str = first_col_indent*' ' + opt_descr + ': ' + right_pad + opt_choice
+    print(line_str)
+    if (filehandle is not None):
+        filehandle.write(line_str + '\n')
+
+
+# write the configuration optitions into a log file
+flog = open('./configure.log', 'w')
+
+output_config('Your Athena++ distribution has now been configured with the following options', '', flog)  # noqa
+output_config('Problem generator', args['prob'], flog)
+output_config('Coordinate system', args['coord'], flog)
+output_config('Equation of state', args['eos'], flog)
+output_config('Riemann solver', args['flux'], flog)
+output_config('Magnetic fields', ('ON' if args['b'] else 'OFF'), flog)
+output_config('Number of scalars', args['nscalars'], flog)
+output_config('Special relativity', ('ON' if args['s'] else 'OFF'), flog)
+output_config('General relativity', ('ON' if args['g'] else 'OFF'), flog)
+output_config('Radiative Transfer', ('ON' if args['nr_radiation'] else 'OFF'), flog)
+output_config('Implicit Radiation', ('ON' if args['implicit_radiation'] else 'OFF'), flog)
+output_config('Cosmic Ray Transport', ('ON' if args['cr'] else 'OFF'), flog)
+output_config('Frame transformations', ('ON' if args['t'] else 'OFF'), flog)
+output_config('Self-Gravity', self_grav_string, flog)
+output_config('Super-Time-Stepping', ('ON' if args['sts'] else 'OFF'), flog)
+output_config('Debug flags', ('ON' if args['debug'] else 'OFF'), flog)
+output_config('Code coverage flags', ('ON' if args['coverage'] else 'OFF'), flog)
+output_config('Linker flags', makefile_options['LINKER_FLAGS'] + ' '
+              + makefile_options['LIBRARY_FLAGS'], flog)
+output_config('Floating-point precision', ('single' if args['float'] else 'double'), flog)
+output_config('Number of ghost cells', args['nghost'], flog)
+output_config('MPI parallelism', ('ON' if args['mpi'] else 'OFF'), flog)
+output_config('OpenMP parallelism', ('ON' if args['omp'] else 'OFF'), flog)
+output_config('FFT', ('ON' if args['fft'] else 'OFF'), flog)
+output_config('HDF5 output', ('ON' if args['hdf5'] else 'OFF'), flog)
 if args['hdf5']:
-    print('  HDF5 precision:             ' + ('double' if args['h5double'] else 'single'))
-print('  Compiler:                   ' + args['cxx'])
-print('  Compilation command:        ' + makefile_options['COMPILER_COMMAND'] + ' '
-      + makefile_options['PREPROCESSOR_FLAGS'] + ' ' + makefile_options['COMPILER_FLAGS'])
+    output_config('HDF5 precision', ('double' if args['h5double'] else 'single'), flog)
+output_config('Compiler', args['cxx'], flog)
+output_config('Compilation command', makefile_options['COMPILER_COMMAND'] + ' '
+              + makefile_options['PREPROCESSOR_FLAGS'] + ' '
+              + makefile_options['COMPILER_FLAGS'], flog)
+
+flog.close()
diff --git a/inputs/cosmic_ray/athinput.cr_diffusion b/inputs/cosmic_ray/athinput.cr_diffusion
new file mode 100644
index 0000000000..92b4d31297
--- /dev/null
+++ b/inputs/cosmic_ray/athinput.cr_diffusion
@@ -0,0 +1,54 @@
+<comment>
+problem   = cosmic ray diffusion
+reference =
+configure = --prob=cr_diffusion
+
+<job>
+problem_id = cr  # problem ID: basename of output filenames
+
+
+<time>
+cfl_number = 0.3  # The Courant, Friedrichs, & Lewy (CFL) Number
+nlim       = -1   # cycle limit
+tlim       = 0.2  # time limit
+ncycle_out = 1    # interval for stdout summary info
+
+<mesh>
+nx1    = 16        # Number of zones in X1-direction
+x1min  = -1.0      # minimum value of X1
+x1max  = 1.0       # maximum value of X1
+ix1_bc = periodic  # inner-X1 boundary flag
+ox1_bc = periodic  # inner-X1 boundary flag
+
+nx2    = 256      # Number of zones in X2-direction
+x2min  = -1.0     # minimum value of X2
+x2max  = 1.0      # maximum value of X2
+ix2_bc = outflow  # inner-X2 boundary flag
+ox2_bc = outflow  # inner-X2 boundary flag
+
+nx3    = 1         # Number of zones in X3-direction
+x3min  = -1.0      # minimum value of X3
+x3max  = 1.0       # maximum value of X3
+ix3_bc = periodic  # inner-X3 boundary flag
+ox3_bc = periodic  # inner-X3 boundary flag
+
+
+<meshblock>
+nx1 = 16
+nx2 = 256
+nx3 = 1
+
+
+<hydro>
+gamma  = 1.6666666666667  # gamma = C_p/C_v
+dfloor = 1.e-8
+pfloor = 1.e-7
+
+<cr>
+vmax     = 100
+src_flag = 0
+
+<problem>
+v0        = 0
+sigma     = 1.e3
+direction = 1
diff --git a/inputs/radiation/athinput.rad_linearwave b/inputs/radiation/athinput.rad_linearwave
new file mode 100644
index 0000000000..29856b5e69
--- /dev/null
+++ b/inputs/radiation/athinput.rad_linearwave
@@ -0,0 +1,58 @@
+<comment>
+problem   = Radiation linear wave test
+reference =
+configure = --prob=rad_linearwave -nr_radiation
+or
+configure = --prob=rad_linearwave -implicit_radiation
+
+<job>
+problem_id = radwave  # problem ID: basename of output filenames
+
+<output1>
+file_type = hst   # History data dump
+dt        = 0.01  # time increment between outputs
+
+
+<time>
+cfl_number = 0.4                 # The Courant, Friedrichs, & Lewy (CFL) Number
+nlim       = 100000              # cycle limit
+tlim       = 0.7745966144169111  # time limit
+ncycle_out = 1                   # interval for stdout summary info
+
+<mesh>
+nx1        = 256       # Number of zones in X1-direction
+x1min      = 0.0       # minimum value of X1
+x1max      =  1.0      # maximum value of X1
+ix1_bc     = periodic  # inner-X1 boundary flag
+ox1_bc     = periodic  # inner-X1 boundary flag
+
+nx2        = 8         # Number of zones in X2-direction
+x2min      = -0.5      # minimum value of X2
+x2max      = 0.5       # maximum value of X2
+ix2_bc     = periodic  # inner-X2 boundary flag
+ox2_bc     = periodic  # inner-X2 boundary flag
+
+nx3        = 1         # Number of zones in X3-direction
+x3min      = -0.5      # minimum value of X3
+x3max      = 0.5       # maximum value of X3
+ix3_bc     = periodic  # inner-X3 boundary flag
+ox3_bc     = periodic  # inner-X3 boundary flag
+
+<meshblock>
+nx1 = 64
+nx2 = 8
+nx3 = 1
+
+<hydro>
+gamma = 1.6666666666667  # gamma = C_p/C_v
+
+<radiation>
+nmu         = 1
+prat        = 0.01
+crat        = 10
+error_limit = 1.e-12
+taucell     = 5
+
+<problem>
+regime        = 1
+compute_error = true
diff --git a/inputs/radiation/athinput.rad_linearwave_amr b/inputs/radiation/athinput.rad_linearwave_amr
new file mode 100644
index 0000000000..0d6476f76d
--- /dev/null
+++ b/inputs/radiation/athinput.rad_linearwave_amr
@@ -0,0 +1,63 @@
+<comment>
+problem   = Radiation linear wave test
+reference =
+configure = --prob=rad_linearwave -nr_radiation
+or
+configure = --prob=rad_linearwave -implicit_radiation
+
+
+<job>
+problem_id = radwave  # problem ID: basename of output filenames
+
+<output1>
+file_type = hst   # History data dump
+dt        = 0.01  # time increment between outputs
+
+
+<time>
+cfl_number = 0.4                 # The Courant, Friedrichs, & Lewy (CFL) Number
+nlim       = 100000              # cycle limit
+tlim       = 0.7745966144169111  # time limit
+ncycle_out = 1                   # interval for stdout summary info
+
+<mesh>
+nx1        = 128       # Number of zones in X1-direction
+x1min      = 0.0       # minimum value of X1
+x1max      =  1.0      # maximum value of X1
+ix1_bc     = periodic  # inner-X1 boundary flag
+ox1_bc     = periodic  # inner-X1 boundary flag
+
+nx2        = 8         # Number of zones in X2-direction
+x2min      = -0.5      # minimum value of X2
+x2max      = 0.5       # maximum value of X2
+ix2_bc     = periodic  # inner-X2 boundary flag
+ox2_bc     = periodic  # inner-X2 boundary flag
+
+nx3        = 1         # Number of zones in X3-direction
+x3min      = -0.5      # minimum value of X3
+x3max      = 0.5       # maximum value of X3
+ix3_bc     = periodic  # inner-X3 boundary flag
+ox3_bc     = periodic  # inner-X3 boundary flag
+
+refinement     = adaptive  # AMR
+derefine_count = 5         # allow derefinement after 5 steps
+numlevel       = 2         # number of AMR levels
+
+<meshblock>
+nx1 = 8
+nx2 = 8
+nx3 = 1
+
+<hydro>
+gamma = 1.6666666666667  # gamma = C_p/C_v
+
+<radiation>
+nmu         = 1
+prat        = 0.01
+crat        = 10
+error_limit = 1.e-12
+taucell     = 5
+
+<problem>
+regime        = 1
+compute_error = true
diff --git a/inputs/radiation/athinput.thermal_multigroup b/inputs/radiation/athinput.thermal_multigroup
new file mode 100644
index 0000000000..f35388e3c9
--- /dev/null
+++ b/inputs/radiation/athinput.thermal_multigroup
@@ -0,0 +1,65 @@
+<comment>
+problem   = Radiation thermal relaxation with multi-group
+reference =
+configure = --prob=thermal_multigroup -nr_radiation
+or
+configure = --prob=thermal_multigroup -implicit_radiation
+
+<job>
+problem_id = thermal  # problem ID: basename of output filenames
+
+
+<time>
+cfl_number = 0.4     # The Courant, Friedrichs, & Lewy (CFL) Number
+nlim       = 100000  # cycle limit
+tlim       = 2.0     # time limit
+ncycle_out = 1       # interval for stdout summary info
+
+<mesh>
+nx1        = 32        # Number of zones in X1-direction
+x1min      = 0.0       # minimum value of X1
+x1max      = 1.0       # maximum value of X1
+ix1_bc     = periodic  # inner-X1 boundary flag
+ox1_bc     = periodic  # inner-X1 boundary flag
+
+nx2        = 1         # Number of zones in X2-direction
+x2min      = 0.0       # minimum value of X2
+x2max      = 1.0       # maximum value of X2
+ix2_bc     = periodic  # inner-X2 boundary flag
+ox2_bc     = periodic  # inner-X2 boundary flag
+
+nx3        = 1         # Number of zones in X3-direction
+x3min      = 0.0       # minimum value of X3
+x3max      = 1.0       # maximum value of X3
+ix3_bc     = periodic  # inner-X3 boundary flag
+ox3_bc     = periodic  # inner-X3 boundary flag
+
+<meshblock>
+
+nx1 = 32
+nx2 = 1
+
+<hydro>
+gamma = 1.6666666666667  # gamma = C_p/C_v
+
+<radiation>
+nmu           = 4
+prat          = 1.0
+crat          = 10
+error_limit   = 1.e-12
+n_frequency   = 3
+frequency_min = -4.0
+frequency_max = -8.0
+
+nlimit        = 500
+
+<problem>
+er_1       = 10.0
+er_2       = 20
+er_3       = 30
+tgas       = 1.0
+sigma_1    = 100.0
+sigma_2    = 200.0
+sigma_3    = 300.0
+kappa_es   = 0
+black_body = 0
diff --git a/inputs/radiation/athinput.thermal_relaxation b/inputs/radiation/athinput.thermal_relaxation
new file mode 100644
index 0000000000..281bde1b69
--- /dev/null
+++ b/inputs/radiation/athinput.thermal_relaxation
@@ -0,0 +1,57 @@
+<comment>
+problem   = Radiation thermal relaxation test
+reference =
+configure = --prob=thermal_relaxation -nr_radiation
+or
+configure = --prob=thermal_relaxation -implicit_radiation
+
+<job>
+problem_id = thermal  # problem ID: basename of output filenames
+
+<output1>
+file_type = hst  # History data dump
+dt        = 1.9  # time increment between outputs
+
+
+<time>
+cfl_number = 0.4     # The Courant, Friedrichs, & Lewy (CFL) Number
+nlim       = 100000  # cycle limit
+tlim       = 2.0     # time limit
+ncycle_out = 1       # interval for stdout summary info
+
+<mesh>
+nx1        = 32        # Number of zones in X1-direction
+x1min      = 0.0       # minimum value of X1
+x1max      = 1.0       # maximum value of X1
+ix1_bc     = periodic  # inner-X1 boundary flag
+ox1_bc     = periodic  # inner-X1 boundary flag
+
+nx2        = 32        # Number of zones in X2-direction
+x2min      = 0.0       # minimum value of X2
+x2max      = 1.0       # maximum value of X2
+ix2_bc     = periodic  # inner-X2 boundary flag
+ox2_bc     = periodic  # inner-X2 boundary flag
+
+nx3        = 1         # Number of zones in X3-direction
+x3min      = 0.0       # minimum value of X3
+x3max      = 1.0       # maximum value of X3
+ix3_bc     = periodic  # inner-X3 boundary flag
+ox3_bc     = periodic  # inner-X3 boundary flag
+
+<meshblock>
+nx1 = 32
+nx2 = 32
+
+<hydro>
+gamma = 1.6666666666667  # gamma = C_p/C_v
+
+<radiation>
+nmu         = 4
+prat        = 1.0
+crat        = 10
+error_limit = 1.e-12
+
+<problem>
+er    = 10.0
+tgas  = 1.0
+sigma = 100.0
diff --git a/src/athena.hpp b/src/athena.hpp
index 6226979c09..28bfae80c8 100644
--- a/src/athena.hpp
+++ b/src/athena.hpp
@@ -51,8 +51,10 @@ class ParameterInput;
 class HydroDiffusion;
 class FieldDiffusion;
 struct MGCoordinates;
-
 class OrbitalAdvection;
+class NRRadiation;
+class IMRadiation;
+class CosmicRay;
 
 //--------------------------------------------------------------------------------------
 //! \struct LogicalLocation
@@ -167,7 +169,7 @@ enum CoordinateDirection {X1DIR=0, X2DIR=1, X3DIR=2};
 enum class BoundaryQuantity {cc, fc, cc_flcor, fc_flcor, mggrav,
                              mggrav_f, orbital_cc, orbital_fc};
 enum class HydroBoundaryQuantity {cons, prim};
-enum class BoundaryCommSubset {mesh_init, gr_amr, all, orbital};
+enum class BoundaryCommSubset {mesh_init, gr_amr, all, orbital, radiation, radhydro};
 // TODO(felker): consider generalizing/renaming to QuantityFormulation
 enum class FluidFormulation {evolve, background, disabled}; // rename background -> fixed?
 enum class TaskType {op_split_before, main_int, op_split_after};
@@ -213,5 +215,27 @@ using MGSourceMaskFunc = void (*)(AthenaArray<Real> &src,
     int is, int ie, int js, int je, int ks, int ke, const MGCoordinates &coord);
 using OrbitalVelocityFunc = Real (*)(
     OrbitalAdvection *porb, Real x1, Real x2, Real x3);
+using RadBoundaryFunc = void (*)(
+     MeshBlock *pmb, Coordinates *pco, NRRadiation *prad,
+     const AthenaArray<Real> &w, FaceField &b,
+     AthenaArray<Real> &ir,
+     Real time, Real dt, int is, int ie, int js, int je, int ks, int ke, int ngh);
+using OpacityFunc = void (*)(MeshBlock *pmb, AthenaArray<Real> &prim);
+using FrequencyFunc = void (*)(NRRadiation *prad);
+using EmissionFunc = void(*)(NRRadiation *prad, Real tgas);
+using CROpacityFunc = void (*)(MeshBlock *pmb, AthenaArray<Real> &u_cr,
+                      AthenaArray<Real> &prim, AthenaArray<Real> &bcc);
+using CRStreamingFunc = void (*)(MeshBlock *pmb, AthenaArray<Real> &u_cr,
+                      AthenaArray<Real> &prim, AthenaArray<Real> &bcc,
+                      AthenaArray<Real> &grad_pc, int k, int j, int is, int ie);
+using SRJFunc = void (*)(IMRadiation *pimrad);
+
+using CRBoundaryFunc = void (*)(
+     MeshBlock *pmb, Coordinates *pco, CosmicRay *pcr,
+     const AthenaArray<Real> &w, FaceField &b, AthenaArray<Real> &u_cr,
+     Real time, Real dt, int is, int ie, int js, int je, int ks, int ke, int ngh);
+using CRSrcTermFunc = void (*)(
+    MeshBlock *pmb, const Real time, const Real dt,
+    const AthenaArray<Real> &prim, FaceField &b, AthenaArray<Real> &u_cr);
 
 #endif // ATHENA_HPP_
diff --git a/src/bvals/bvals.cpp b/src/bvals/bvals.cpp
index c547424926..185f68e931 100644
--- a/src/bvals/bvals.cpp
+++ b/src/bvals/bvals.cpp
@@ -27,6 +27,7 @@
 #include "../athena.hpp"
 #include "../athena_arrays.hpp"
 #include "../coordinates/coordinates.hpp"
+#include "../cr/cr.hpp"
 #include "../eos/eos.hpp"
 #include "../field/field.hpp"
 #include "../globals.hpp"
@@ -35,6 +36,7 @@
 #include "../mesh/mesh.hpp"
 #include "../mesh/mesh_refinement.hpp"
 #include "../multigrid/multigrid.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../orbital_advection/orbital_advection.hpp"
 #include "../parameter_input.hpp"
 #include "../scalars/scalars.hpp"
@@ -265,7 +267,6 @@ void BoundaryValues::SetupPersistentMPI() {
   // initialize the shearing block lists
   if (shearing_box != 0) {
     MeshBlock *pmb = pmy_block_;
-    int nbtotal = pmy_mesh_->nbtotal;
     int *ranklist = pmy_mesh_->ranklist;
     int *nslist = pmy_mesh_->nslist;
     LogicalLocation *loclist = pmy_mesh_->loclist;
@@ -333,6 +334,27 @@ void BoundaryValues::CheckUserBoundaries() {
             << "is not enrolled in direction " << i  << " (in [0,6])." << std::endl;
         ATHENA_ERROR(msg);
       }
+
+      if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+        if (pmy_mesh_->RadBoundaryFunc_[i] == nullptr) {
+          std::stringstream msg;
+          msg << "### FATAL ERROR in BoundaryValues::CheckBoundary" << std::endl
+              << "A user-defined boundary is specified but the actual RadBoundaryFunc_ "
+              << "is not enrolled in direction " << i  << " (in [0,6])."
+              << std::endl;
+          ATHENA_ERROR(msg);
+        }
+      }
+      if (CR_ENABLED) {
+        if (pmy_mesh_->CRBoundaryFunc_[i] == nullptr) {
+          std::stringstream msg;
+          msg << "### FATAL ERROR in BoundaryValues::CheckBoundary" << std::endl
+              << "A user-defined boundary is specified but the actual CRBoundaryFunc_ "
+              << "is not enrolled in direction " << i  << " (in [0,6])."
+              << std::endl;
+          ATHENA_ERROR(msg);
+        }
+      }
     }
   }
   return;
@@ -356,6 +378,9 @@ void BoundaryValues::StartReceivingSubset(BoundaryCommSubset phase,
     switch (phase) {
       case BoundaryCommSubset::mesh_init:
         break;
+      case BoundaryCommSubset::radiation:
+        break;
+      case BoundaryCommSubset::radhydro:
       case BoundaryCommSubset::all:
       case BoundaryCommSubset::orbital:
         for (auto bvars_it = bvars_subset.begin(); bvars_it != bvars_subset.end();
@@ -448,12 +473,29 @@ void BoundaryValues::ApplyPhysicalBoundaries(const Real time, const Real dt,
     ps = pmb->pscalars;
   }
 
+
+  NRRadiation *prad = nullptr;
+  RadBoundaryVariable *pradbvar = nullptr;
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    prad = pmb->pnrrad;
+    pradbvar = &(prad->rad_bvar);
+  }
+
+  CosmicRay *pcr = nullptr;
+  //CellCenteredBoundaryVariable *pcrbvar = nullptr;
+  if (CR_ENABLED) {
+    pcr = pmb->pcr;
+    //pcrbvar = &(pcr->cr_bvar);
+  }
+
+
+
   // Apply boundary function on inner-x1 and update W,bcc (if not periodic)
   if (apply_bndry_fn_[BoundaryFace::inner_x1]) {
     DispatchBoundaryFunctions(pmb, pco, time, dt,
                               pmb->is, pmb->ie, bjs, bje, bks, bke, NGHOST,
-                              ph->w, pf->b, BoundaryFace::inner_x1,
-                              bvars_subset);
+                              ph->w, pf->b, prad->ir, pcr->u_cr,
+                              BoundaryFace::inner_x1, bvars_subset);
     // KGF: COUPLING OF QUANTITIES (must be manually specified)
     if (MAGNETIC_FIELDS_ENABLED) {
       pmb->pfield->CalculateCellCenteredField(pf->b, pf->bcc, pco,
@@ -472,8 +514,8 @@ void BoundaryValues::ApplyPhysicalBoundaries(const Real time, const Real dt,
   if (apply_bndry_fn_[BoundaryFace::outer_x1]) {
     DispatchBoundaryFunctions(pmb, pco, time, dt,
                               pmb->is, pmb->ie, bjs, bje, bks, bke, NGHOST,
-                              ph->w, pf->b, BoundaryFace::outer_x1,
-                              bvars_subset);
+                              ph->w, pf->b, prad->ir, pcr->u_cr,
+                              BoundaryFace::outer_x1, bvars_subset);
     // KGF: COUPLING OF QUANTITIES (must be manually specified)
     if (MAGNETIC_FIELDS_ENABLED) {
       pmb->pfield->CalculateCellCenteredField(pf->b, pf->bcc, pco,
@@ -493,8 +535,8 @@ void BoundaryValues::ApplyPhysicalBoundaries(const Real time, const Real dt,
     if (apply_bndry_fn_[BoundaryFace::inner_x2]) {
       DispatchBoundaryFunctions(pmb, pco, time, dt,
                                 bis, bie, pmb->js, pmb->je, bks, bke, NGHOST,
-                                ph->w, pf->b, BoundaryFace::inner_x2,
-                                bvars_subset);
+                                ph->w, pf->b, prad->ir, pcr->u_cr,
+                                BoundaryFace::inner_x2, bvars_subset);
       // KGF: COUPLING OF QUANTITIES (must be manually specified)
       if (MAGNETIC_FIELDS_ENABLED) {
         pmb->pfield->CalculateCellCenteredField(pf->b, pf->bcc, pco,
@@ -509,12 +551,19 @@ void BoundaryValues::ApplyPhysicalBoundaries(const Real time, const Real dt,
       }
     }
 
+    if ((NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) &&
+           (block_bcs[BoundaryFace::inner_x2] != BoundaryFlag::block)) {
+      if (prad->rotate_theta == 1) {
+        pradbvar->RotateHPi_InnerX2(time, dt, bis, bie, pmb->js, bks, bke, NGHOST);
+      }
+    }// end radiation
+
     // Apply boundary function on outer-x2 and update W,bcc (if not periodic)
     if (apply_bndry_fn_[BoundaryFace::outer_x2]) {
       DispatchBoundaryFunctions(pmb, pco, time, dt,
                                 bis, bie, pmb->js, pmb->je, bks, bke, NGHOST,
-                                ph->w, pf->b, BoundaryFace::outer_x2,
-                                bvars_subset);
+                                ph->w, pf->b, prad->ir, pcr->u_cr,
+                                BoundaryFace::outer_x2, bvars_subset);
       // KGF: COUPLING OF QUANTITIES (must be manually specified)
       if (MAGNETIC_FIELDS_ENABLED) {
         pmb->pfield->CalculateCellCenteredField(pf->b, pf->bcc, pco,
@@ -538,8 +587,8 @@ void BoundaryValues::ApplyPhysicalBoundaries(const Real time, const Real dt,
     if (apply_bndry_fn_[BoundaryFace::inner_x3]) {
       DispatchBoundaryFunctions(pmb, pco, time, dt,
                                 bis, bie, bjs, bje, pmb->ks, pmb->ke, NGHOST,
-                                ph->w, pf->b, BoundaryFace::inner_x3,
-                                bvars_subset);
+                                ph->w, pf->b, prad->ir, pcr->u_cr,
+                                BoundaryFace::inner_x3, bvars_subset);
       // KGF: COUPLING OF QUANTITIES (must be manually specified)
       if (MAGNETIC_FIELDS_ENABLED) {
         pmb->pfield->CalculateCellCenteredField(pf->b, pf->bcc, pco,
@@ -554,12 +603,21 @@ void BoundaryValues::ApplyPhysicalBoundaries(const Real time, const Real dt,
       }
     }
 
+    if ((NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) &&
+           (block_bcs[BoundaryFace::inner_x3] != BoundaryFlag::block)) {
+      if (prad->rotate_phi == 1) {
+        pradbvar->RotateHPi_InnerX3(time, dt, bis, bie, bjs, bje, pmb->ks, NGHOST);
+      } else if (prad->rotate_phi == 2) {
+        pradbvar->RotatePi_InnerX3(time, dt, bis, bie, bjs, bje, pmb->ks,NGHOST);
+      }
+    }
+
     // Apply boundary function on outer-x3 and update W,bcc (if not periodic)
     if (apply_bndry_fn_[BoundaryFace::outer_x3]) {
       DispatchBoundaryFunctions(pmb, pco, time, dt,
                                 bis, bie, bjs, bje, pmb->ks, pmb->ke, NGHOST,
-                                ph->w, pf->b, BoundaryFace::outer_x3,
-                                bvars_subset);
+                                ph->w, pf->b, prad->ir, pcr->u_cr,
+                                BoundaryFace::outer_x3, bvars_subset);
       // KGF: COUPLING OF QUANTITIES (must be manually specified)
       if (MAGNETIC_FIELDS_ENABLED) {
         pmb->pfield->CalculateCellCenteredField(pf->b, pf->bcc, pco,
@@ -573,6 +631,14 @@ void BoundaryValues::ApplyPhysicalBoundaries(const Real time, const Real dt,
             ps->r, ph->u, ps->s, pco, bis, bie, bjs, bje, pmb->ke+1, pmb->ke+NGHOST);
       }
     }
+    if ((NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) &&
+           (block_bcs[BoundaryFace::outer_x3] != BoundaryFlag::block)) {
+      if (prad->rotate_phi == 1) {
+        pradbvar->RotateHPi_OuterX3(time, dt, bis, bie, bjs, bje, pmb->ke, NGHOST);
+      } else if (prad->rotate_phi == 2) {
+        pradbvar->RotatePi_OuterX3(time, dt, bis, bie, bjs, bje, pmb->ke, NGHOST);
+      }
+    }
   }
   return;
 }
@@ -584,11 +650,23 @@ void BoundaryValues::ApplyPhysicalBoundaries(const Real time, const Real dt,
 void BoundaryValues::DispatchBoundaryFunctions(
     MeshBlock *pmb, Coordinates *pco, Real time, Real dt,
     int il, int iu, int jl, int ju, int kl, int ku, int ngh,
-    AthenaArray<Real> &prim, FaceField &b, BoundaryFace face,
+    AthenaArray<Real> &prim, FaceField &b, AthenaArray<Real> &ir,
+    AthenaArray<Real> &u_cr,  BoundaryFace face,
     std::vector<BoundaryVariable *> bvars_subset) {
   if (block_bcs[face] ==  BoundaryFlag::user) {  // user-enrolled BCs
     pmy_mesh_->BoundaryFunction_[face](pmb, pco, prim, b, time, dt,
                                        il, iu, jl, ju, kl, ku, NGHOST);
+
+    // user-defined  boundary for radiation
+    if ((NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)) {
+      pmy_mesh_->RadBoundaryFunc_[face](pmb,pco,pmb->pnrrad,prim,b, ir,time,dt,
+                                             il,iu,jl,ju,kl,ku,NGHOST);
+    }
+
+    if (CR_ENABLED) {
+      pmy_mesh_->CRBoundaryFunc_[face](pmb,pco,pmb->pcr,prim, b, u_cr,time,dt,
+                                              il,iu,jl,ju,kl,ku,NGHOST);
+    }
   }
   // KGF: this is only to silence the compiler -Wswitch warnings about not handling the
   // "undef" case when considering all possible BoundaryFace enumerator values. If "undef"
@@ -653,6 +731,30 @@ void BoundaryValues::DispatchBoundaryFunctions(
             break;
         }
         break;
+      case BoundaryFlag::vacuum: // special boundary condition type for radiation
+        switch (face) {
+          case BoundaryFace::undef:
+            ATHENA_ERROR(msg);
+          case BoundaryFace::inner_x1:
+            (*bvars_it)->VacuumInnerX1(time, dt, il, jl, ju, kl, ku, NGHOST);
+            break;
+          case BoundaryFace::outer_x1:
+            (*bvars_it)->VacuumOuterX1(time, dt, iu, jl, ju, kl, ku, NGHOST);
+            break;
+          case BoundaryFace::inner_x2:
+            (*bvars_it)->VacuumInnerX2(time, dt, il, iu, jl, kl, ku, NGHOST);
+            break;
+          case BoundaryFace::outer_x2:
+            (*bvars_it)->VacuumOuterX2(time, dt, il, iu, ju, kl, ku, NGHOST);
+            break;
+          case BoundaryFace::inner_x3:
+            (*bvars_it)->VacuumInnerX3(time, dt, il, iu, jl, ju, kl, NGHOST);
+            break;
+          case BoundaryFace::outer_x3:
+            (*bvars_it)->VacuumOuterX3(time, dt, il, iu, jl, ju, ku, NGHOST);
+            break;
+        }
+        break;
       case BoundaryFlag::polar_wedge:
         switch (face) {
           case BoundaryFace::undef:
diff --git a/src/bvals/bvals.hpp b/src/bvals/bvals.hpp
index 106183b8c2..fef2db1e81 100644
--- a/src/bvals/bvals.hpp
+++ b/src/bvals/bvals.hpp
@@ -220,7 +220,8 @@ class BoundaryValues : public BoundaryBase, //public BoundaryPhysics,
   void DispatchBoundaryFunctions(
       MeshBlock *pmb, Coordinates *pco, Real time, Real dt,
       int il, int iu, int jl, int ju, int kl, int ku, int ngh,
-      AthenaArray<Real> &prim, FaceField &b, BoundaryFace face,
+      AthenaArray<Real> &prim, FaceField &b, AthenaArray<Real> &ir,
+      AthenaArray<Real> &u_cr, BoundaryFace face,
       std::vector<BoundaryVariable *> bvars_subset);
 
   void CheckPolarBoundaries();  // called in BoundaryValues() ctor
@@ -237,5 +238,6 @@ class BoundaryValues : public BoundaryBase, //public BoundaryPhysics,
   //! - consider removing these friendship designations:
   friend class CellCenteredBoundaryVariable;
   friend class HydroBoundaryVariable;  // needed for shearing box quantities
+  friend class RadBoundaryVariable; // needed for radiation boundary condition
 };
 #endif // BVALS_BVALS_HPP_
diff --git a/src/bvals/bvals_interfaces.hpp b/src/bvals/bvals_interfaces.hpp
index 795cf19a2e..cb23a0ec1d 100644
--- a/src/bvals/bvals_interfaces.hpp
+++ b/src/bvals/bvals_interfaces.hpp
@@ -68,7 +68,8 @@ enum {BLOCK_BNDRY __attribute__((deprecated)) = -1,
 #else
 enum {FACE_UNDEF = -1, INNER_X1, OUTER_X1, INNER_X2, OUTER_X2, INNER_X3, OUTER_X3};
 enum {BLOCK_BNDRY = -1, BNDRY_UNDEF, REFLECTING_BNDRY, OUTFLOW_BNDRY, USER_BNDRY,
-      PERIODIC_BNDRY, POLAR_BNDRY, POLAR_BNDRY_WEDGE, SHEAR_PERIODIC_BNDRY};
+      PERIODIC_BNDRY, VACUUM_BNDRY, POLAR_BNDRY, POLAR_BNDRY_WEDGE,
+      SHEAR_PERIODIC_BNDRY};
 #endif
 
 //! identifiers for all 6 faces of a MeshBlock
@@ -81,7 +82,7 @@ enum BoundaryFace {undef=-1, inner_x1=0, outer_x1=1, inner_x2=2, outer_x2=3,
 
 //! identifiers for boundary conditions
 enum class BoundaryFlag {block=-1, undef, reflect, outflow, user, periodic,
-                         polar, polar_wedge, shear_periodic,
+                         polar, polar_wedge, vacuum, shear_periodic,
                          mg_zerograd, mg_zerofixed, mg_multipole};
 
 //! identifiers for types of neighbor blocks (connectivity with current MeshBlock)
@@ -298,6 +299,19 @@ class BoundaryPhysics {
   virtual void OutflowOuterX3(Real time, Real dt,
                               int il, int iu, int jl, int ju, int ku, int ngh) = 0;
 
+  virtual void VacuumInnerX1(Real time, Real dt,
+                              int il, int jl, int ju, int kl, int ku, int ngh) = 0;
+  virtual void VacuumOuterX1(Real time, Real dt,
+                              int iu, int jl, int ju, int kl, int ku, int ngh) = 0;
+  virtual void VacuumInnerX2(Real time, Real dt,
+                              int il, int iu, int jl, int kl, int ku, int ngh) = 0;
+  virtual void VacuumOuterX2(Real time, Real dt,
+                              int il, int iu, int ju, int kl, int ku, int ngh) = 0;
+  virtual void VacuumInnerX3(Real time, Real dt,
+                              int il, int iu, int jl, int ju, int kl, int ngh) = 0;
+  virtual void VacuumOuterX3(Real time, Real dt,
+                              int il, int iu, int jl, int ju, int ku, int ngh) = 0;
+
   virtual void PolarWedgeInnerX2(Real time, Real dt,
                                  int il, int iu, int jl, int kl, int ku, int ngh) = 0;
   virtual void PolarWedgeOuterX2(Real time, Real dt,
diff --git a/src/bvals/bvals_refine.cpp b/src/bvals/bvals_refine.cpp
index 4eb5a7b30f..21e0fffff9 100644
--- a/src/bvals/bvals_refine.cpp
+++ b/src/bvals/bvals_refine.cpp
@@ -4,7 +4,7 @@
 // Licensed under the 3-clause BSD License, see LICENSE file for details
 //========================================================================================
 //! \file bvals_refine.cpp
-//! \brief functions for prolongation at level boundaries
+//! \brief functions for prolongation and restriction at level boundaries
 //!
 //! SWITCHING BETWEEN PRIMITIVE VS. CONSERVED AND STANDARD VS. COARSE BUFFERS HERE:
 //! -----------
@@ -61,12 +61,14 @@
 // Athena++ headers
 #include "../athena.hpp"
 #include "../athena_arrays.hpp"
+#include "../cr/cr.hpp"
 #include "../eos/eos.hpp"
 #include "../field/field.hpp"
 #include "../hydro/hydro.hpp"
 #include "../mesh/mesh.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../scalars/scalars.hpp"
-#include "bvals.hpp"
+#include "./bvals.hpp"
 #include "cc/hydro/bvals_hydro.hpp"
 #include "fc/bvals_fc.hpp"
 
@@ -114,7 +116,7 @@ void BoundaryValues::ProlongateBoundaries(const Real time, const Real dt,
       dynamic_cast<HydroBoundaryVariable *>(bvars_main_int[0]);
   Hydro *ph = pmb->phydro;
 
-  CellCenteredBoundaryVariable *psbvar = nullptr;
+  //CellCenteredBoundaryVariable *psbvar = nullptr;
   PassiveScalars *ps = nullptr;
   if (NSCALARS > 0) {
     ps = pmb->pscalars;
@@ -129,6 +131,21 @@ void BoundaryValues::ProlongateBoundaries(const Real time, const Real dt,
     pfbvar = dynamic_cast<FaceCenteredBoundaryVariable *>(bvars_main_int[1]);
   }
 
+  // hard core assume that radiation boundary is just after hydro/MHD
+  NRRadiation *pnrrad = nullptr;
+  RadBoundaryVariable *pradbvar = nullptr;
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    pnrrad = pmb->pnrrad;
+    pradbvar = &(pnrrad->rad_bvar);
+  }
+
+  CosmicRay *pcr=nullptr;
+  CellCenteredBoundaryVariable *pcrbvar = nullptr;
+  if (CR_ENABLED) {
+    pcr = pmb->pcr;
+    pcrbvar = &(pcr->cr_bvar);
+  }
+
   // For each finer neighbor, to prolongate a boundary we need to fill one more cell
   // surrounding the boundary zone to calculate the slopes ("ghost-ghost zone"). 3x steps:
   for (int n=0; n<nneighbor; n++) {
@@ -208,6 +225,12 @@ void BoundaryValues::ProlongateBoundaries(const Real time, const Real dt,
       ps->sbvar.var_cc = &(ps->coarse_r_);
     }
 
+    if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED))
+      pradbvar->var_cc = &(pnrrad->coarse_ir_);
+
+    if (CR_ENABLED)
+      pcrbvar->var_cc = &(pcr->coarse_cr_);
+
     // Step 2. Re-apply physical boundaries on the coarse boundary:
     ApplyPhysicalBoundariesOnCoarseLevel(nb, time, dt, si, ei, sj, ej, sk, ek,
                                          bvars_subset);
@@ -222,6 +245,12 @@ void BoundaryValues::ProlongateBoundaries(const Real time, const Real dt,
       ps->sbvar.var_cc = &(ps->r);
     }
 
+    if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED))
+      pradbvar->var_cc = &(pnrrad->ir);
+
+    if (CR_ENABLED)
+      pcrbvar->var_cc = &(pcr->u_cr);
+
     // Step 3. Finally, the ghost-ghost zones are ready for prolongation:
     ProlongateGhostCells(nb, si, ei, sj, ej, sk, ek);
   } // end loop over nneighbor
@@ -322,6 +351,15 @@ void BoundaryValues::RestrictGhostCellsOnSameLevel(const NeighborBlock& nb, int
       }
     }
   } // end loop over pvars_fc_
+
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    NRRadiation *pnrrad=pmb->pnrrad;
+    AthenaArray<Real> *var_cc = pnrrad->rad_bvar.var_cc;
+    AthenaArray<Real> *coarse_cc = pnrrad->rad_bvar.coarse_buf;
+    int nu= var_cc->GetDim1() - 1;
+    pmr->RestrictCellCenteredValues(*var_cc, *coarse_cc, -1, 0, nu,
+                                         ris, rie, rjs, rje, rks, rke);
+  }
   return;
 }
 
@@ -348,6 +386,18 @@ void BoundaryValues::ApplyPhysicalBoundariesOnCoarseLevel(
     pf = pmb->pfield;
   }
 
+  NRRadiation *pnrrad = nullptr;
+  RadBoundaryVariable *pradbvar = nullptr;
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    pnrrad = pmb->pnrrad;
+    pradbvar = &(pnrrad->rad_bvar);
+  }
+
+  CosmicRay *pcr = nullptr;
+  if (CR_ENABLED)
+    pcr = pmb->pcr;
+
+
   // convert the ghost zone and ghost-ghost zones into primitive variables
   // this includes cell-centered field calculation
   int f1m = 0, f1p = 0, f2m = 0, f2p = 0, f3m = 0, f3p = 0;
@@ -402,13 +452,17 @@ void BoundaryValues::ApplyPhysicalBoundariesOnCoarseLevel(
     if (apply_bndry_fn_[BoundaryFace::inner_x1]) {
       DispatchBoundaryFunctions(pmb, pmr->pcoarsec, time, dt,
                                 pmb->cis, pmb->cie, sj, ej, sk, ek, 1,
-                                ph->coarse_prim_, pf->coarse_b_, BoundaryFace::inner_x1,
+                                ph->coarse_prim_, pf->coarse_b_,
+                                pnrrad->coarse_ir_, pcr->coarse_cr_,
+                                BoundaryFace::inner_x1,
                                 bvars_subset);
     }
     if (apply_bndry_fn_[BoundaryFace::outer_x1]) {
       DispatchBoundaryFunctions(pmb, pmr->pcoarsec, time, dt,
                                 pmb->cis, pmb->cie, sj, ej, sk, ek, 1,
-                                ph->coarse_prim_, pf->coarse_b_, BoundaryFace::outer_x1,
+                                ph->coarse_prim_, pf->coarse_b_,
+                                pnrrad->coarse_ir_, pcr->coarse_cr_,
+                                BoundaryFace::outer_x1,
                                 bvars_subset);
     }
   }
@@ -416,29 +470,67 @@ void BoundaryValues::ApplyPhysicalBoundariesOnCoarseLevel(
     if (apply_bndry_fn_[BoundaryFace::inner_x2]) {
       DispatchBoundaryFunctions(pmb, pmr->pcoarsec, time, dt,
                                 si, ei, pmb->cjs, pmb->cje, sk, ek, 1,
-                                ph->coarse_prim_, pf->coarse_b_, BoundaryFace::inner_x2,
+                                ph->coarse_prim_, pf->coarse_b_,
+                                pnrrad->coarse_ir_, pcr->coarse_cr_,
+                                BoundaryFace::inner_x2,
                                 bvars_subset);
     }
+    if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED) &&
+           (block_bcs[BoundaryFace::inner_x2] != BoundaryFlag::block)) {
+      if (pnrrad->rotate_theta == 1) {
+        pradbvar->RotateHPi_InnerX2(time, dt, si, ei, pmb->cjs, sk, ek, 1);
+      }
+    }// end radiation
+
     if (apply_bndry_fn_[BoundaryFace::outer_x2]) {
       DispatchBoundaryFunctions(pmb, pmr->pcoarsec, time, dt,
                                 si, ei, pmb->cjs, pmb->cje, sk, ek, 1,
-                                ph->coarse_prim_, pf->coarse_b_, BoundaryFace::outer_x2,
+                                ph->coarse_prim_, pf->coarse_b_,
+                                pnrrad->coarse_ir_, pcr->coarse_cr_,
+                                BoundaryFace::outer_x2,
                                 bvars_subset);
     }
+    if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED) &&
+           (block_bcs[BoundaryFace::outer_x2] != BoundaryFlag::block)) {
+      if (pnrrad->rotate_theta == 1) {
+        pradbvar->RotateHPi_OuterX2(time, dt, si, ei, pmb->cje, sk, ek, 1);
+      }
+    }
   }
   if (nb.ni.ox3 == 0 && pmb->block_size.nx3 > 1) {
     if (apply_bndry_fn_[BoundaryFace::inner_x3]) {
       DispatchBoundaryFunctions(pmb, pmr->pcoarsec, time, dt,
                                 si, ei, sj, ej, pmb->cks, pmb->cke, 1,
-                                ph->coarse_prim_, pf->coarse_b_, BoundaryFace::inner_x3,
+                                ph->coarse_prim_, pf->coarse_b_,
+                                pnrrad->coarse_ir_, pcr->coarse_cr_,
+                                BoundaryFace::inner_x3,
                                 bvars_subset);
     }
+    if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED) &&
+        (block_bcs[BoundaryFace::inner_x3] != BoundaryFlag::block)) {
+      if (pnrrad->rotate_phi == 1) {
+        pradbvar->RotateHPi_InnerX3(time, dt, si, ei, sj, ej, pmb->cks, 1);
+      } else if (pnrrad->rotate_phi == 2) {
+        pradbvar->RotatePi_InnerX3(time, dt, si, ei, sj, ej, pmb->cks,1);
+      }
+    }// end radiation
+
     if (apply_bndry_fn_[BoundaryFace::outer_x3]) {
       DispatchBoundaryFunctions(pmb, pmr->pcoarsec, time, dt,
                                 si, ei, sj, ej, pmb->cks, pmb->cke, 1,
-                                ph->coarse_prim_, pf->coarse_b_, BoundaryFace::outer_x3,
+                                ph->coarse_prim_, pf->coarse_b_,
+                                pnrrad->coarse_ir_, pcr->coarse_cr_,
+                                BoundaryFace::outer_x3,
                                 bvars_subset);
     }
+    if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED) &&
+        (block_bcs[BoundaryFace::outer_x3] != BoundaryFlag::block)) {
+      if (pnrrad->rotate_phi == 1) {
+        pradbvar->RotateHPi_OuterX3(time, dt, si, ei, sj, ej, pmb->cke, 1);
+      } else if (pnrrad->rotate_phi == 2) {
+        pradbvar->RotatePi_OuterX3(time, dt, si, ei, sj, ej, pmb->cke, 1);
+      }
+    }
   }
   return;
 }
@@ -471,6 +563,17 @@ void BoundaryValues::ProlongateGhostCells(const NeighborBlock& nb,
     pmr->ProlongateCellCenteredValues(*coarse_cc, *var_cc, 0, nu,
                                       si, ei, sj, ej, sk, ek);
   }
+
+  // do radiation quantities separately
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    NRRadiation *pnrrad=pmb->pnrrad;
+    AthenaArray<Real> *var_cc = pnrrad->rad_bvar.var_cc;
+    AthenaArray<Real> *coarse_cc = pnrrad->rad_bvar.coarse_buf;
+    int nu= var_cc->GetDim1() - 1;
+    pmr->ProlongateCellCenteredValues(*coarse_cc, *var_cc, -1, 0, nu,
+                                      si, ei, sj, ej, sk, ek);
+  }
+
   // swap back MeshRefinement ptrs to standard/coarse conserved variable arrays:
   pmr->SetHydroRefinement(HydroBoundaryQuantity::cons);
   if (NSCALARS > 0) {
diff --git a/src/bvals/cc/bvals_cc.cpp b/src/bvals/cc/bvals_cc.cpp
index c923ef1f2a..eb6f603918 100644
--- a/src/bvals/cc/bvals_cc.cpp
+++ b/src/bvals/cc/bvals_cc.cpp
@@ -145,6 +145,92 @@ CellCenteredBoundaryVariable::~CellCenteredBoundaryVariable() {
   }
 }
 
+// override constructor
+// for radiation quantites with different order
+
+CellCenteredBoundaryVariable::CellCenteredBoundaryVariable(
+    MeshBlock *pmb, AthenaArray<Real> *var, AthenaArray<Real> *coarse_var,
+    AthenaArray<Real> *var_flux, bool fflux, int flag)
+    : BoundaryVariable(pmb, fflux), var_cc(var), coarse_buf(coarse_var),
+      x1flux(var_flux[X1DIR]), x2flux(var_flux[X2DIR]), x3flux(var_flux[X3DIR]),
+      nl_(0), nu_(var->GetDim1() -1), flip_across_pole_(nullptr) {
+  //! \note
+  //! CellCenteredBoundaryVariable should only be used w/ 4D or 3D (nx4=1) AthenaArray
+  //! For now, assume that full span of 4th dim of input AthenaArray should be used:
+  //! ---> get the index limits directly from the input AthenaArray
+  //! <=nu_ (inclusive), <nx4 (exclusive)
+  if (nu_ < 0) {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in CellCenteredBoundaryVariable constructor" << std::endl
+        << "An 'AthenaArray<Real> *var' of nx4_ = " << var->GetDim4() << " was passed\n"
+        << "Should be nx4 >= 1 (likely uninitialized)." << std::endl;
+    ATHENA_ERROR(msg);
+  }
+
+  // KT: fflux is a flag and it is true (false) when flux correction is (not) needed.
+  //     I have not implemented it for shearing box, leaving it to Tomohiro.
+
+
+  InitBoundaryData(bd_var_, BoundaryQuantity::cc);
+#ifdef MPI_PARALLEL
+  // KGF: dead code, leaving for now:
+  // cc_phys_id_ = pbval_->ReserveTagVariableIDs(1);
+  cc_phys_id_ = pbval_->bvars_next_phys_id_;
+#endif
+  if (fflux_ && ((pmy_mesh_->multilevel)
+      || (pbval_->shearing_box != 0))) { // SMR or AMR or SHEARING_BOX
+    fflux_ = true;
+    InitBoundaryData(bd_var_flcor_, BoundaryQuantity::cc_flcor);
+#ifdef MPI_PARALLEL
+    cc_flx_phys_id_ = cc_phys_id_ + 1;
+#endif
+  } else {
+    fflux_ = false;
+  }
+
+  if (pbval_->shearing_box != 0) {
+#ifdef MPI_PARALLEL
+    shear_cc_phys_id_ = cc_phys_id_ + 2;
+    shear_flx_phys_id_ = shear_cc_phys_id_ + 1;
+#endif
+    int nc2 = pmb->ncells2;
+    int nc3 = pmb->ncells3;
+    int nx3 = pmb->block_size.nx3;
+    int &xgh = pbval_->xgh_;
+    for (int upper=0; upper<2; upper++) {
+      if (pbval_->is_shear[upper]) {
+        shear_cc_[upper].NewAthenaArray(nc3, NGHOST, nc2+2*xgh+1, nu_+1);
+        shear_var_flx_[upper].NewAthenaArray(nc3, nc2,nu_+1);
+        shear_map_flx_[upper].NewAthenaArray(nc3, 1, nc2+2*xgh+1, nu_+1);
+
+        // TODO(KGF): the rest of this should be a part of InitBoundaryData()
+
+        int bsize = pmb->block_size.nx2*pbval_->ssize_*(nu_ + 1);
+        int fsize = pmb->block_size.nx2*nx3*(nu_ + 1);
+        for (int n=0; n<4; n++) {
+          shear_bd_var_[upper].send[n] = new Real[bsize];
+          shear_bd_var_[upper].recv[n] = new Real[bsize];
+          shear_bd_var_[upper].flag[n] = BoundaryStatus::waiting;
+#ifdef MPI_PARALLEL
+          shear_bd_var_[upper].req_send[n] = MPI_REQUEST_NULL;
+          shear_bd_var_[upper].req_recv[n] = MPI_REQUEST_NULL;
+#endif
+        }
+        for (int n=0; n<3; n++) {
+          shear_bd_flux_[upper].send[n] = new Real[fsize];
+          shear_bd_flux_[upper].recv[n] = new Real[fsize];
+          shear_bd_flux_[upper].flag[n] = BoundaryStatus::waiting;
+#ifdef MPI_PARALLEL
+          shear_bd_flux_[upper].req_send[n] = MPI_REQUEST_NULL;
+          shear_bd_flux_[upper].req_recv[n] = MPI_REQUEST_NULL;
+#endif
+        }
+      } // end "if is a shearing boundary"
+    }  // end loop over inner, outer shearing boundaries
+  } // end shearing box component
+}
+
+
 //----------------------------------------------------------------------------------------
 //! \fn int CellCenteredBoundaryVariable::ComputeVariableBufferSize(
 //!     const NeighborIndexes& ni, int cng)
diff --git a/src/bvals/cc/bvals_cc.hpp b/src/bvals/cc/bvals_cc.hpp
index 8e5172087e..3c959bcf2d 100644
--- a/src/bvals/cc/bvals_cc.hpp
+++ b/src/bvals/cc/bvals_cc.hpp
@@ -33,6 +33,10 @@ class CellCenteredBoundaryVariable : public BoundaryVariable {
   CellCenteredBoundaryVariable(MeshBlock *pmb,
                                AthenaArray<Real> *var, AthenaArray<Real> *coarse_var,
                                AthenaArray<Real> *var_flux, bool fflux);
+    //override function for arrays need different initialization of nu_
+  CellCenteredBoundaryVariable(MeshBlock *pmb,
+                               AthenaArray<Real> *var, AthenaArray<Real> *coarse_var,
+                               AthenaArray<Real> *var_flux, bool fflux, int flag);
   ~CellCenteredBoundaryVariable();
 
   //! \note
@@ -116,6 +120,19 @@ class CellCenteredBoundaryVariable : public BoundaryVariable {
   void OutflowOuterX3(Real time, Real dt,
                       int il, int iu, int jl, int ju, int ku, int ngh) override;
 
+  void VacuumInnerX1(Real time, Real dt,
+                     int il, int jl, int ju, int kl, int ku, int ngh) override {return;};
+  void VacuumOuterX1(Real time, Real dt,
+                     int iu, int jl, int ju, int kl, int ku, int ngh) override {return;};
+  void VacuumInnerX2(Real time, Real dt,
+                     int il, int iu, int jl, int kl, int ku, int ngh) override {return;};
+  void VacuumOuterX2(Real time, Real dt,
+                     int il, int iu, int ju, int kl, int ku, int ngh) override {return;};
+  void VacuumInnerX3(Real time, Real dt,
+                     int il, int iu, int jl, int ju, int kl, int ngh) override {return;};
+  void VacuumOuterX3(Real time, Real dt,
+                     int il, int iu, int jl, int ju, int ku, int ngh) override {return;};
+
   void PolarWedgeInnerX2(Real time, Real dt,
                          int il, int iu, int jl, int kl, int ku, int ngh) override;
   void PolarWedgeOuterX2(Real time, Real dt,
@@ -180,6 +197,8 @@ class CellCenteredBoundaryVariable : public BoundaryVariable {
                                             Real *buf, int nb);
   void SetFluxShearingBoxBoundarySameLevel(AthenaArray<Real> &src,
                                            Real *buf, const int nb);
+  friend class ParticleMeshBoundaryVariable;
+  friend class RadBoundaryVariable;
 };
 
 #endif // BVALS_CC_BVALS_CC_HPP_
diff --git a/src/bvals/cc/nr_radiation/bvals_rad.cpp b/src/bvals/cc/nr_radiation/bvals_rad.cpp
new file mode 100644
index 0000000000..ea51de9373
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/bvals_rad.cpp
@@ -0,0 +1,400 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file bvals_rad.cpp
+//  \brief implements boundary functions for Radiation variables and utilities to manage
+//  primitive/conservative variable relationship in a derived class of the
+//  CellCenteredBoundaryVariable base class.
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../coordinates/coordinates.hpp"
+#include "../../../globals.hpp"
+#include "../../../hydro/hydro.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../nr_radiation/radiation.hpp"
+#include "bvals_rad.hpp"
+
+//----------------------------------------------------------------------------------------
+//! \class RadiationBoundaryFunctions
+
+RadBoundaryVariable::RadBoundaryVariable(MeshBlock *pmb,
+    AthenaArray<Real> *var_rad, AthenaArray<Real> *coarse_var,
+    AthenaArray<Real> *var_flux) :
+    CellCenteredBoundaryVariable(pmb, var_rad, coarse_var, var_flux, true, 1) {
+    // the radiation array is (k,j,i,n)
+    // the number of variables is GetDim1
+    // All the other shared functions are initialized in CellCenteredBoundaryVariable
+    // radiation specific functions need to be defined here
+    azimuthal_shift_rad_.NewAthenaArray(pmb->ke + NGHOST + 2,nu_+1);
+    ir_cm_.NewAthenaArray(pmb->nfre_ang);
+    ir_lab_.NewAthenaArray(pmb->nfre_ang);
+    if (pbval_->shearing_box != 0) {
+      int pnum = pmb->block_size.nx2+2*NGHOST+1;
+      pflux_.NewAthenaArray(pnum,pmb->nfre_ang);
+    }
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn int RadBoundaryVariable::LoadBoundaryBufferSameLevel(Real *buf,
+//                                                                const NeighborBlock& nb)
+//  \brief Set cell-centered boundary buffers for sending to a block on the same level
+
+int RadBoundaryVariable::LoadBoundaryBufferSameLevel(Real *buf, const NeighborBlock& nb) {
+  MeshBlock *pmb = pmy_block_;
+  int si, sj, sk, ei, ej, ek;
+
+  si = (nb.ni.ox1 > 0) ? (pmb->ie - NGHOST + 1) : pmb->is;
+  ei = (nb.ni.ox1 < 0) ? (pmb->is + NGHOST - 1) : pmb->ie;
+  sj = (nb.ni.ox2 > 0) ? (pmb->je - NGHOST + 1) : pmb->js;
+  ej = (nb.ni.ox2 < 0) ? (pmb->js + NGHOST - 1) : pmb->je;
+  sk = (nb.ni.ox3 > 0) ? (pmb->ke - NGHOST + 1) : pmb->ks;
+  ek = (nb.ni.ox3 < 0) ? (pmb->ks + NGHOST - 1) : pmb->ke;
+  int p = 0;
+  AthenaArray<Real> &var = *var_cc;
+  BufferUtility::PackData(var, buf, sk, ek, nl_, nu_, si, ei, sj, ej, p);
+
+  return p;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn int RadBoundaryVariable::LoadBoundaryBufferToCoarser(Real *buf,
+//                                                                const NeighborBlock& nb)
+//  \brief Set cell-centered boundary buffers for sending to a block on the coarser level
+
+int RadBoundaryVariable::LoadBoundaryBufferToCoarser(Real *buf, const NeighborBlock& nb) {
+  MeshBlock *pmb = pmy_block_;
+  MeshRefinement *pmr = pmb->pmr;
+  int si, sj, sk, ei, ej, ek;
+  int cn = NGHOST - 1;
+  AthenaArray<Real> &var = *var_cc;
+  AthenaArray<Real> &coarse_var = *coarse_buf;
+
+  si = (nb.ni.ox1 > 0) ? (pmb->cie - cn) : pmb->cis;
+  ei = (nb.ni.ox1 < 0) ? (pmb->cis + cn) : pmb->cie;
+  sj = (nb.ni.ox2 > 0) ? (pmb->cje - cn) : pmb->cjs;
+  ej = (nb.ni.ox2 < 0) ? (pmb->cjs + cn) : pmb->cje;
+  sk = (nb.ni.ox3 > 0) ? (pmb->cke - cn) : pmb->cks;
+  ek = (nb.ni.ox3 < 0) ? (pmb->cks + cn) : pmb->cke;
+
+  int p = 0;
+  // function overload to do restriction for radiation variabbles
+  pmr->RestrictCellCenteredValues(var, coarse_var, -1, nl_, nu_, si, ei, sj, ej, sk, ek);
+  BufferUtility::PackData(coarse_var, buf, sk, ek, nl_, nu_, si, ei, sj, ej, p);
+  return p;
+}
+
+
+//----------------------------------------------------------------------------------------
+//! \fn int CellCenteredBoundaryVariable::LoadBoundaryBufferToFiner(Real *buf,
+//                                                                const NeighborBlock& nb)
+//  \brief Set cell-centered boundary buffers for sending to a block on the finer level
+
+int RadBoundaryVariable::LoadBoundaryBufferToFiner(Real *buf, const NeighborBlock& nb) {
+  MeshBlock *pmb = pmy_block_;
+  int si, sj, sk, ei, ej, ek;
+  int cn = pmb->cnghost - 1;
+  AthenaArray<Real> &var = *var_cc;
+
+  si = (nb.ni.ox1 > 0) ? (pmb->ie - cn) : pmb->is;
+  ei = (nb.ni.ox1 < 0) ? (pmb->is + cn) : pmb->ie;
+  sj = (nb.ni.ox2 > 0) ? (pmb->je - cn) : pmb->js;
+  ej = (nb.ni.ox2 < 0) ? (pmb->js + cn) : pmb->je;
+  sk = (nb.ni.ox3 > 0) ? (pmb->ke - cn) : pmb->ks;
+  ek = (nb.ni.ox3 < 0) ? (pmb->ks + cn) : pmb->ke;
+
+  // send the data first and later prolongate on the target block
+  // need to add edges for faces, add corners for edges
+  if (nb.ni.ox1 == 0) {
+    if (nb.ni.fi1 == 1)   si += pmb->block_size.nx1/2 - pmb->cnghost;
+    else            ei -= pmb->block_size.nx1/2 - pmb->cnghost;
+  }
+  if (nb.ni.ox2 == 0 && pmb->block_size.nx2 > 1) {
+    if (nb.ni.ox1 != 0) {
+      if (nb.ni.fi1 == 1) sj += pmb->block_size.nx2/2 - pmb->cnghost;
+      else          ej -= pmb->block_size.nx2/2 - pmb->cnghost;
+    } else {
+      if (nb.ni.fi2 == 1) sj += pmb->block_size.nx2/2 - pmb->cnghost;
+      else          ej -= pmb->block_size.nx2/2 - pmb->cnghost;
+    }
+  }
+  if (nb.ni.ox3 == 0 && pmb->block_size.nx3 > 1) {
+    if (nb.ni.ox1 != 0 && nb.ni.ox2 != 0) {
+      if (nb.ni.fi1 == 1) sk += pmb->block_size.nx3/2 - pmb->cnghost;
+      else          ek -= pmb->block_size.nx3/2 - pmb->cnghost;
+    } else {
+      if (nb.ni.fi2 == 1) sk += pmb->block_size.nx3/2 - pmb->cnghost;
+      else          ek -= pmb->block_size.nx3/2 - pmb->cnghost;
+    }
+  }
+
+  int p = 0;
+  BufferUtility::PackData(var, buf, sk, ek, nl_, nu_, si, ei, sj, ej, p);
+  return p;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void BoundaryVariable::SetBoundaries()
+//  \brief set the boundary data
+
+void RadBoundaryVariable::SetBoundaries() {
+  MeshBlock *pmb = pmy_block_;
+  int mylevel = pmb->loc.level;
+  for (int n=0; n<pbval_->nneighbor; n++) {
+    NeighborBlock& nb = pbval_->neighbor[n];
+    if (nb.snb.level == mylevel)
+      SetBoundarySameLevel(bd_var_.recv[nb.bufid], nb);
+    else if (nb.snb.level < mylevel) // only sets the prolongation buffer
+      SetBoundaryFromCoarser(bd_var_.recv[nb.bufid], nb);
+    else
+      SetBoundaryFromFiner(bd_var_.recv[nb.bufid], nb);
+    bd_var_.flag[nb.bufid] = BoundaryStatus::completed; // completed
+  }
+
+  if (pbval_->block_bcs[BoundaryFace::inner_x2] == BoundaryFlag::polar ||
+      pbval_->block_bcs[BoundaryFace::outer_x2] == BoundaryFlag::polar)
+    PolarBoundarySingleAzimuthalBlock();
+
+  return;
+}
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::SetBoundarySameLevel(Real *buf,
+//                                                              const NeighborBlock& nb)
+//  \brief Set cell-centered boundary received from a block on the same level
+
+void RadBoundaryVariable::SetBoundarySameLevel(Real *buf,
+                                                        const NeighborBlock& nb) {
+  MeshBlock *pmb = pmy_block_;
+  int si, sj, sk, ei, ej, ek;
+  AthenaArray<Real> &var = *var_cc;
+
+  if (nb.ni.ox1 == 0)     si = pmb->is,        ei = pmb->ie;
+  else if (nb.ni.ox1 > 0) si = pmb->ie + 1,      ei = pmb->ie + NGHOST;
+  else              si = pmb->is - NGHOST, ei = pmb->is - 1;
+  if (nb.ni.ox2 == 0)     sj = pmb->js,        ej = pmb->je;
+  else if (nb.ni.ox2 > 0) sj = pmb->je + 1,      ej = pmb->je + NGHOST;
+  else              sj = pmb->js - NGHOST, ej = pmb->js - 1;
+  if (nb.ni.ox3 == 0)     sk = pmb->ks,        ek = pmb->ke;
+  else if (nb.ni.ox3 > 0) sk = pmb->ke + 1,      ek = pmb->ke + NGHOST;
+  else              sk = pmb->ks - NGHOST, ek = pmb->ks - 1;
+
+  int p = 0;
+  // no need to flip for radiation
+  if (nb.polar) {
+    for (int k=sk; k<=ek; ++k) {
+      for (int j=ej; j>=sj; --j) {
+        for (int i=si; i<=ei; ++i) {
+#pragma omp simd linear(p)
+          for (int n=nl_; n<=nu_; ++n) {
+            var(k,j,i,n) = buf[p++];
+          }
+        }
+      }
+    }
+  } else {
+    BufferUtility::UnpackData(buf, var, sk, ek, nl_, nu_, si, ei, sj, ej, p);
+  }
+  return;
+}
+
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::SetBoundaryFromCoarser(Real *buf,
+//                                                                const NeighborBlock& nb)
+//  \brief Set cell-centered prolongation buffer received from a block on a coarser level
+
+void RadBoundaryVariable::SetBoundaryFromCoarser(Real *buf,
+                                                          const NeighborBlock& nb) {
+  MeshBlock *pmb = pmy_block_;
+  int si, sj, sk, ei, ej, ek;
+  int cng = pmb->cnghost;
+  AthenaArray<Real> &coarse_var = *coarse_buf;
+
+  if (nb.ni.ox1 == 0) {
+    si = pmb->cis, ei = pmb->cie;
+    if ((pmb->loc.lx1 & 1LL) == 0LL) ei += cng;
+    else                             si -= cng;
+  } else if (nb.ni.ox1 > 0)  {
+    si = pmb->cie + 1,   ei = pmb->cie + cng;
+  } else {
+    si = pmb->cis - cng, ei = pmb->cis - 1;
+  }
+  if (nb.ni.ox2 == 0) {
+    sj = pmb->cjs, ej = pmb->cje;
+    if (pmb->block_size.nx2 > 1) {
+      if ((pmb->loc.lx2 & 1LL) == 0LL) ej += cng;
+      else                             sj -= cng;
+    }
+  } else if (nb.ni.ox2 > 0) {
+    sj = pmb->cje + 1,   ej = pmb->cje + cng;
+  } else {
+    sj = pmb->cjs - cng, ej = pmb->cjs - 1;
+  }
+  if (nb.ni.ox3 == 0) {
+    sk = pmb->cks, ek = pmb->cke;
+    if (pmb->block_size.nx3 > 1) {
+      if ((pmb->loc.lx3 & 1LL) == 0LL) ek += cng;
+      else                             sk -= cng;
+    }
+  } else if (nb.ni.ox3 > 0)  {
+    sk = pmb->cke + 1,   ek = pmb->cke + cng;
+  } else {
+    sk = pmb->cks - cng, ek = pmb->cks - 1;
+  }
+
+  int p = 0;
+  if (nb.polar) {
+    for (int k=sk; k<=ek; ++k) {
+      for (int j=ej; j>=sj; --j) {
+        for (int i=si; i<=ei; ++i) {
+#pragma omp simd linear(p)
+          for (int n=nl_; n<=nu_; ++n) {
+            coarse_var(k,j,i,n) = buf[p++];
+          }
+        }
+      }
+    }
+  } else {
+    BufferUtility::UnpackData(buf, coarse_var, sk, ek, nl_, nu_, si, ei, sj, ej, p);
+  }
+  return;
+}
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::SetBoundaryFromFiner(Real *buf,
+//                                                              const NeighborBlock& nb)
+//  \brief Set cell-centered boundary received from a block on a finer level
+
+void RadBoundaryVariable::SetBoundaryFromFiner(Real *buf, const NeighborBlock& nb) {
+  MeshBlock *pmb = pmy_block_;
+  AthenaArray<Real> &var = *var_cc;
+  // receive already restricted data
+  int si, sj, sk, ei, ej, ek;
+
+  if (nb.ni.ox1 == 0) {
+    si = pmb->is, ei = pmb->ie;
+    if (nb.ni.fi1 == 1)   si += pmb->block_size.nx1/2;
+    else            ei -= pmb->block_size.nx1/2;
+  } else if (nb.ni.ox1 > 0) {
+    si = pmb->ie + 1,      ei = pmb->ie + NGHOST;
+  } else {
+    si = pmb->is - NGHOST, ei = pmb->is - 1;
+  }
+  if (nb.ni.ox2 == 0) {
+    sj = pmb->js, ej = pmb->je;
+    if (pmb->block_size.nx2 > 1) {
+      if (nb.ni.ox1 != 0) {
+        if (nb.ni.fi1 == 1) sj += pmb->block_size.nx2/2;
+        else          ej -= pmb->block_size.nx2/2;
+      } else {
+        if (nb.ni.fi2 == 1) sj += pmb->block_size.nx2/2;
+        else          ej -= pmb->block_size.nx2/2;
+      }
+    }
+  } else if (nb.ni.ox2 > 0) {
+    sj = pmb->je + 1,      ej = pmb->je + NGHOST;
+  } else {
+    sj = pmb->js - NGHOST, ej = pmb->js - 1;
+  }
+  if (nb.ni.ox3 == 0) {
+    sk = pmb->ks, ek = pmb->ke;
+    if (pmb->block_size.nx3 > 1) {
+      if (nb.ni.ox1 != 0 && nb.ni.ox2 != 0) {
+        if (nb.ni.fi1 == 1) sk += pmb->block_size.nx3/2;
+        else          ek -= pmb->block_size.nx3/2;
+      } else {
+        if (nb.ni.fi2 == 1) sk += pmb->block_size.nx3/2;
+        else          ek -= pmb->block_size.nx3/2;
+      }
+    }
+  } else if (nb.ni.ox3 > 0) {
+    sk = pmb->ke + 1,      ek = pmb->ke + NGHOST;
+  } else {
+    sk = pmb->ks - NGHOST, ek = pmb->ks - 1;
+  }
+
+  int p = 0;
+  if (nb.polar) {
+//     Real sign=1.0;
+//      if (flip_across_pole_ != nullptr) sign = flip_across_pole_[n] ? -1.0 : 1.0;
+    for (int k=sk; k<=ek; ++k) {
+      for (int j=ej; j>=sj; --j) {
+        for (int i=si; i<=ei; ++i) {
+#pragma omp simd linear(p)
+          for (int n=nl_; n<=nu_; ++n) {
+            var(k,j,i,n) = buf[p++];
+          }
+        }
+      }
+    }
+  } else {
+    BufferUtility::UnpackData(buf, var,  sk, ek, nl_, nu_, si, ei, sj, ej, p);
+  }
+  return;
+}
+
+
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::PolarBoundarySingleAzimuthalBlock()
+// \brief polar boundary edge-case: single MeshBlock spans the entire azimuthal (x3) range
+
+void RadBoundaryVariable::PolarBoundarySingleAzimuthalBlock() {
+  MeshBlock *pmb = pmy_block_;
+
+  if (pmb->loc.level  ==  pmy_mesh_->root_level && pmy_mesh_->nrbx3 == 1
+      && pmb->block_size.nx3 > 1) {
+    AthenaArray<Real> &var = *var_cc;
+    if (pbval_->block_bcs[BoundaryFace::inner_x2] == BoundaryFlag::polar) {
+      int nx3_half = (pmb->ke - pmb->ks + 1) / 2;
+      for (int j=pmb->js-NGHOST; j<=pmb->js-1; ++j) {
+        for (int i=pmb->is-NGHOST; i<=pmb->ie+NGHOST; ++i) {
+          for (int k=pmb->ks-NGHOST; k<=pmb->ke+NGHOST; ++k) {
+            for (int n=nl_; n<=nu_; ++n) {
+              azimuthal_shift_rad_(k,n) = var(k,j,i,n);
+            }
+          }
+          for (int k=pmb->ks-NGHOST; k<=pmb->ke+NGHOST; ++k) {
+            int k_shift = k;
+            k_shift += (k < (nx3_half + NGHOST) ? 1 : -1) * nx3_half;
+            for (int n=nl_; n<=nu_; ++n) {
+              var(k,j,i,n) = azimuthal_shift_rad_(k_shift,n);
+            }
+          }
+        }
+      }
+    }
+
+    if (pbval_->block_bcs[BoundaryFace::outer_x2] == BoundaryFlag::polar) {
+      int nx3_half = (pmb->ke - pmb->ks + 1) / 2;
+      for (int j=pmb->je+1; j<=pmb->je+NGHOST; ++j) {
+        for (int i=pmb->is-NGHOST; i<=pmb->ie+NGHOST; ++i) {
+          for (int k=pmb->ks-NGHOST; k<=pmb->ke+NGHOST; ++k) {
+            for (int n=nl_; n<=nu_; ++n) {
+              azimuthal_shift_rad_(k,n) = var(k,j,i,n);
+            }
+          }
+          for (int k=pmb->ks-NGHOST; k<=pmb->ke+NGHOST; ++k) {
+            int k_shift = k;
+            k_shift += (k < (nx3_half + NGHOST) ? 1 : -1) * nx3_half;
+            for(int n=nl_; n<=nu_; ++n) {
+              var(k,j,i,n) = azimuthal_shift_rad_(k_shift,n);
+            }
+          }
+        }
+      }
+    }
+  }
+  return;
+}
diff --git a/src/bvals/cc/nr_radiation/bvals_rad.hpp b/src/bvals/cc/nr_radiation/bvals_rad.hpp
new file mode 100644
index 0000000000..0d8d5bd27c
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/bvals_rad.hpp
@@ -0,0 +1,152 @@
+#ifndef BVALS_CC_NR_RADIATION_BVALS_RAD_HPP_
+#define BVALS_CC_NR_RADIATION_BVALS_RAD_HPP_
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file bvals_rad.hpp
+//  \brief
+//========================================================================================
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../utils/buffer_utils.hpp"
+#include "../bvals_cc.hpp"
+
+
+//----------------------------------------------------------------------------------------
+//! \class CellCenteredBoundaryVariable
+//  \brief
+
+class RadBoundaryVariable : public CellCenteredBoundaryVariable {
+ public:
+  RadBoundaryVariable(MeshBlock *pmb,
+                      AthenaArray<Real> *var_rad, AthenaArray<Real> *coarse_var,
+                      AthenaArray<Real> *var_flux);
+  virtual ~RadBoundaryVariable() = default;
+
+  // functions unique implementation to radiation class
+  void SendFluxCorrection() override;
+  bool ReceiveFluxCorrection() override;
+
+
+  void SetBoundaries() override;
+
+
+  // function for shearing box
+  void AddRadShearForInit();
+  void ShearQuantities(AthenaArray<Real> &shear_cc_, bool upper) override;
+  void SetShearingBoxBoundaryBuffers();
+  void SetFluxShearingBoxBoundaryBuffers();
+  void SendFluxShearingBoxBoundaryBuffers();
+  bool ReceiveFluxShearingBoxBoundaryBuffers();
+  bool ReceiveShearingBoxBoundaryBuffers();
+  void SendShearingBoxBoundaryBuffers();
+
+  // BoundaryPhysics: need to rotate the intensity
+  void ReflectInnerX1(Real time, Real dt,
+                      int il, int jl, int ju, int kl, int ku, int ngh) override;
+  void ReflectOuterX1(Real time, Real dt,
+                      int iu, int jl, int ju, int kl, int ku, int ngh) override;
+  void ReflectInnerX2(Real time, Real dt,
+                      int il, int iu, int jl, int kl, int ku, int ngh) override;
+  void ReflectOuterX2(Real time, Real dt,
+                      int il, int iu, int ju, int kl, int ku, int ngh) override;
+  void ReflectInnerX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int kl, int ngh) override;
+  void ReflectOuterX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int ku, int ngh) override;
+
+  void VacuumInnerX1(Real time, Real dt,
+                      int il, int jl, int ju, int kl, int ku, int ngh) override;
+  void VacuumOuterX1(Real time, Real dt,
+                      int iu, int jl, int ju, int kl, int ku, int ngh) override;
+  void VacuumInnerX2(Real time, Real dt,
+                      int il, int iu, int jl, int kl, int ku, int ngh) override;
+  void VacuumOuterX2(Real time, Real dt,
+                      int il, int iu, int ju, int kl, int ku, int ngh) override;
+  void VacuumInnerX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int kl, int ngh) override;
+  void VacuumOuterX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int ku, int ngh) override;
+
+
+  void OutflowInnerX1(Real time, Real dt,
+                      int il, int jl, int ju, int kl, int ku, int ngh) override;
+  void OutflowOuterX1(Real time, Real dt,
+                      int iu, int jl, int ju, int kl, int ku, int ngh) override;
+  void OutflowInnerX2(Real time, Real dt,
+                      int il, int iu, int jl, int kl, int ku, int ngh) override;
+  void OutflowOuterX2(Real time, Real dt,
+                      int il, int iu, int ju, int kl, int ku, int ngh) override;
+  void OutflowInnerX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int kl, int ngh) override;
+  void OutflowOuterX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int ku, int ngh) override;
+
+  // rotate the specific intensity by some amount
+  // these functions are typically called with periodic boundary together
+  void RotateHPi_InnerX2(Real time, Real dt,
+                      int il, int iu, int jl, int kl, int ku, int ngh);
+
+  void RotateHPi_OuterX2(Real time, Real dt,
+                      int il, int iu, int ju, int kl, int ku, int ngh);
+
+  void RotateHPi_InnerX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int kl, int ngh);
+
+  void RotateHPi_OuterX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int ku, int ngh);
+
+  void RotatePi_InnerX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int kl, int ngh);
+
+  void RotatePi_OuterX3(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int ku, int ngh);
+
+  void PolarWedgeInnerX2(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int ku, int ngh) override;
+
+  void PolarWedgeOuterX2(Real time, Real dt,
+                      int il, int iu, int jl, int ju, int ku, int ngh) override;
+
+ private:
+  AthenaArray<Real>  azimuthal_shift_rad_;
+
+  // override function for flux correction
+  void SetFluxBoundaryFromFiner(Real *buf, const NeighborBlock& nb);
+  void SetFluxBoundarySameLevel(Real *buf, const NeighborBlock& nb);
+
+  int LoadFluxBoundaryBufferToCoarser(Real *buf, const NeighborBlock& nb);
+  int LoadFluxBoundaryBufferSameLevel(Real *buf, const NeighborBlock& nb) override;
+
+  void SetBoundaryFromCoarser(Real *buf, const NeighborBlock& nb) override;
+  void SetBoundaryFromFiner(Real *buf, const NeighborBlock& nb) override;
+
+  int LoadBoundaryBufferSameLevel(Real *buf, const NeighborBlock& nb) override;
+  int LoadBoundaryBufferToCoarser(Real *buf, const NeighborBlock& nb) override;
+  int LoadBoundaryBufferToFiner(Real *buf, const NeighborBlock& nb) override;
+
+  void SetBoundarySameLevel(Real *buf, const NeighborBlock& nb) override;
+
+
+  void PolarBoundarySingleAzimuthalBlock() override;
+
+  // shearing box functions
+  void LoadShearingBoxBoundarySameLevel(AthenaArray<Real> &src, Real *buf, int nb);
+  void SetShearingBoxBoundarySameLevel(AthenaArray<Real> &src,
+                                       Real *buf, const int nb);
+  void LoadFluxShearingBoxBoundarySameLevel(AthenaArray<Real> &src,
+                                            Real *buf, int nb);
+  void SetFluxShearingBoxBoundarySameLevel(AthenaArray<Real> &src,
+                                           Real *buf, const int nb);
+
+  AthenaArray<Real> ir_cm_, ir_lab_, pflux_; // co-moving frame specific inteisites
+};
+
+#endif //  BVALS_CC_NR_RADIATION_BVALS_RAD_HPP_
diff --git a/src/bvals/cc/nr_radiation/bvals_shear_rad.cpp b/src/bvals/cc/nr_radiation/bvals_shear_rad.cpp
new file mode 100644
index 0000000000..f8138a0fa4
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/bvals_shear_rad.cpp
@@ -0,0 +1,359 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file bvals_shear_cc.cpp
+//! \brief functions that apply shearing box BCs for cell-centered radiation variables
+//========================================================================================
+
+// C headers
+
+// C++ headers
+#include <algorithm>  // min
+#include <cmath>
+#include <cstdlib>
+#include <cstring>    // memcpy
+#include <iomanip>
+#include <iostream>   // endl
+#include <sstream>    // stringstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../coordinates/coordinates.hpp"
+#include "../../../globals.hpp"
+#include "../../../hydro/hydro.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../nr_radiation/integrators/rad_integrators.hpp"
+#include "../../../nr_radiation/radiation.hpp"
+#include "../../../orbital_advection/orbital_advection.hpp"
+#include "../../../parameter_input.hpp"
+#include "../../../utils/buffer_utils.hpp"
+#include "../../bvals.hpp"
+#include "../../bvals_interfaces.hpp"
+#include "./bvals_rad.hpp"
+
+// MPI header
+#ifdef MPI_PARALLEL
+#include <mpi.h>
+#endif
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::AddHydroShearForInit()
+//! \brief Send shearing box boundary buffers for radiation variables
+//shearing periodic boundary condition for radiation:
+// Specific intensities in the co-moving frame is periodic
+// Then boost back to the lab frame according to the new local velocity
+
+void RadBoundaryVariable::AddRadShearForInit() {
+  MeshBlock *pmb = pmy_block_;
+  Mesh *pmesh = pmb->pmy_mesh;
+  AthenaArray<Real> &var = *var_cc;
+  Hydro *phydro = pmb->phydro;
+  NRRadiation *prad = pmb->pnrrad;
+
+  int jl = pmb->js - NGHOST;
+  int ju = pmb->je + NGHOST;
+  int kl = pmb->ks;
+  int ku = pmb->ke;
+  if (pmesh->mesh_size.nx3 > 1) {
+    kl -= NGHOST;
+    ku += NGHOST;
+  }
+
+  Real qomL = pbval_->qomL_;
+
+  int sign[2]{1, -1};
+  int ib[2]{pmb->is - NGHOST, pmb->ie + 1};
+
+  // could call modified ShearQuantities(src=shear_cc_, dst=var, upper), by first loading
+  // shear_cc_=var for IDN, IM2 so that order of IM2, IEN update to var doesn't matter.
+  // Would need to reassign src=shear_cc_ to updated dst=var for IM2 after? Is it used?
+  for (int upper=0; upper<2; upper++) {
+    if (pbval_->is_shear[upper]) {
+      // step 1. -- add shear to the periodic boundary values
+      for (int k=kl; k<=ku; k++) {
+        for (int j=jl; j<=ju; j++) {
+          for (int i=0; i<NGHOST; i++) {
+            // add shear to conservative
+            int ii = ib[upper] + i;
+            // get flow velocity in local cell
+            // this requires that shearing periodic boundary has been applied to hydro
+            Real vx = phydro->u(IM1,k,j,ii)/phydro->u(IDN,k,j,ii);
+            Real vy = phydro->u(IM2,k,j,ii)/phydro->u(IDN,k,j,ii);
+            Real vz = phydro->u(IM3,k,j,ii)/phydro->u(IDN,k,j,ii);
+            // the original velocity in the active zones of the other side is
+            //vx, vy-sign[upper]*qomL, vz
+            Real vy_ori = vy - sign[upper]*qomL;
+            // get co-moving frame Intensities for vx, vy_ori, vz
+            Real *mux = &(prad->mu(0,k,j,ii,0));
+            Real *muy = &(prad->mu(1,k,j,ii,0));
+            Real *muz = &(prad->mu(2,k,j,ii,0));
+            Real *ir_lab = &(var(k,j,ii,0));
+            prad->pradintegrator->LabToCom(vx,vy_ori,vz,mux,muy,muz,ir_lab,ir_cm_);
+            // now set lab frame intensities for vx, vy, vz
+            prad->pradintegrator->ComToLab(vx,vy,vz,mux,muy,muz,ir_cm_,ir_lab);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+void RadBoundaryVariable::ShearQuantities(AthenaArray<Real> &shear_cc_, bool upper) {
+  MeshBlock *pmb = pmy_block_;
+  Mesh *pmesh = pmb->pmy_mesh;
+  Hydro *phydro = pmb->phydro;
+  NRRadiation *prad = pmb->pnrrad;
+  int &xgh = pbval_->xgh_;
+  int jl = pmb->js - NGHOST;
+  int ju = pmb->je + NGHOST+2*xgh+1;
+  int kl = pmb->ks;
+  int ku = pmb->ke;
+  if (pmesh->mesh_size.nx3 > 1) {
+    kl -= NGHOST;
+    ku += NGHOST;
+  }
+
+  Real qomL = pbval_->qomL_;
+  int sign[2]{1, -1};
+  int ib[2]{pmb->is - NGHOST, pmb->ie + 1};
+
+  for (int k=kl; k<=ku; k++) {
+    for (int i=0; i<NGHOST; i++) {
+      int ii = ib[upper]+i;
+      for (int j=jl; j<=ju; j++) {
+        Real vx = phydro->w(IVX,k,j,ii);
+        Real vy = phydro->w(IVY,k,j,ii);
+        Real vz = phydro->w(IVZ,k,j,ii);
+        Real vy_ori = vy - sign[upper]*qomL;
+
+        Real *mux = &(prad->mu(0,k,j,ii,0));
+        Real *muy = &(prad->mu(1,k,j,ii,0));
+        Real *muz = &(prad->mu(2,k,j,ii,0));
+        Real *ir_lab = &(shear_cc_(k,i,j,0));
+        prad->pradintegrator->LabToCom(vx,vy_ori,vz,mux,muy,muz,ir_lab,ir_cm_);
+        // now set lab frame intensities for vx, vy, vz
+        prad->pradintegrator->ComToLab(vx,vy,vz,mux,muy,muz,ir_cm_,ir_lab);
+      }
+    }
+  }
+  return;
+}
+
+
+
+
+//--------------------------------------------------------------------------------------
+//! \fn int RadBoundaryVariable::LoadShearingBoxBoundarySameLevel(
+//!                                       AthenaArray<Real> &src, Real *buf, int nb)
+//! \brief Set CC shearing boundary buffers for sending to a block on the same level
+
+void RadBoundaryVariable::LoadShearingBoxBoundarySameLevel(
+                                 AthenaArray<Real> &src, Real *buf, int nb) {
+  MeshBlock *pmb = pmy_block_;
+  Mesh *pmesh = pmb->pmy_mesh;
+  int si, sj, sk, ei, ej, ek;
+  int jo = pbval_->joverlap_;
+  int *jmin1 = pbval_->sb_data_[0].jmin_send;
+  int *jmin2 = pbval_->sb_data_[1].jmin_send;
+  int *jmax1 = pbval_->sb_data_[0].jmax_send;
+  int *jmax2 = pbval_->sb_data_[1].jmax_send;
+  sk = pmb->ks;        ek = pmb->ke;
+  if (pmesh->mesh_size.nx3 > 1) ek += NGHOST, sk -= NGHOST;
+
+  if (nb<4) { // inner boundary
+    si = pmb->is-NGHOST;
+    ei = pmb->is-1;
+    sj = jmin1[nb]-jo-1;
+    ej = jmax1[nb]-jo-1;
+  } else if (nb<8) { //outer boundary
+    si = pmb->ie+1;
+    ei = pmb->ie+NGHOST;
+    sj = jmin2[nb-4]+jo;
+    ej = jmax2[nb-4]+jo;
+  } else {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in CellCenteredBoundaryVariable::"
+        << "ShearingBoxBoundarySameLevel"<<std::endl
+        << "nb = " << nb << " not valid" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  int p = 0;
+  for (int k=sk; k<=ek; k++) {
+    for (int i=si; i<=ei; i++) {
+      for (int j=sj; j<=ej; j++) {
+        for (int n=nl_; n<=nu_; ++n) {
+          buf[p++] = src(k,j,i,n);
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+
+// --------------------------------------------------------------------------------------
+//! \fn void CellCenteredBoundaryVariable::SetShearingBoxBoundarySameLevel(
+//!                        AthenaArray<Real> &src, Real *buf, const int nb)
+//! \brief Set CC shearing boundary data received from a block on the same level
+
+void RadBoundaryVariable::SetShearingBoxBoundarySameLevel(
+                    AthenaArray<Real> &src, Real *buf,const int nb) {
+  MeshBlock *pmb = pmy_block_;
+  Mesh *pmesh = pmb->pmy_mesh;
+  int si, sj, sk, ei, ej, ek;
+  int &xgh = pbval_->xgh_;
+  si = pmb->is-NGHOST; ei = pmb->is-1;
+  sk = pmb->ks;        ek = pmb->ke;
+  if (pmesh->mesh_size.nx3 > 1)  ek += NGHOST, sk -= NGHOST;
+
+  int *jmin1 = pbval_->sb_data_[0].jmin_recv;
+  int *jmin2 = pbval_->sb_data_[1].jmin_recv;
+  int *jmax1 = pbval_->sb_data_[0].jmax_recv;
+  int *jmax2 = pbval_->sb_data_[1].jmax_recv;
+
+  if (nb<4) {
+    sj = jmin1[nb]+xgh;     ej = jmax1[nb]+xgh;
+  } else if (nb<8) {
+    sj = jmin2[nb-4]+xgh;   ej = jmax2[nb-4]+xgh;
+  } else {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in CellCenteredBoundaryVariable::"
+        << "SetShearingBoxBoundarySameLevel"<<std::endl
+        << "nb = " << nb << " not valid" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  int p = 0;
+  BufferUtility::UnpackData(buf, src, sk, ek, nl_, nu_,
+                            sj, ej, si, ei, p);
+  return;
+}
+
+
+
+//----------------------------------------------------------------------------------------
+//! \fn bool CellCenteredBoundaryVariable::SetShearingBoxBoundaryBuffers()
+//! \brief Update CC shearing box boundary variables
+
+void RadBoundaryVariable::SetShearingBoxBoundaryBuffers() {
+  MeshBlock *pmb = pmy_block_;
+  Mesh *pmesh = pmb->pmy_mesh;
+  OrbitalAdvection *porb = pmb->porb;
+  AthenaArray<Real> &var = *var_cc;
+  AthenaArray<Real> &pflux = pflux_;
+  int &xgh = pbval_->xgh_;
+  int &xorder = pmb->pnrrad->pradintegrator->rad_xorder;
+  int ib[2]{pmb->is - NGHOST, pmb->ie + 1};
+  int js = pmb->js, je = pmb->je;
+  int kl = pmb->ks, ku = pmb->ke;
+  if (pmesh->mesh_size.nx3 > 1) {
+    kl -= NGHOST;
+    ku += NGHOST;
+  }
+  int jl = js-NGHOST;
+  int ju = je+NGHOST;
+
+  for (int upper=0; upper<2; upper++) {
+    if (pbval_->is_shear[upper]) { // check inner boundaries
+      // step 1 -- (optionally) appy shear to shear_cc_ (does nothing by default)
+      if (!porb->orbital_advection_defined)
+        ShearQuantities(shear_cc_[upper], upper);  // Hydro overrides this
+      // step 2. -- calculating remapping flux and update var
+      Real eps = (1.0-2*upper)*pbval_->eps_;
+      for (int k=kl; k<=ku; k++) {
+        for (int i=0; i<NGHOST; i++) {
+          int ii = ib[upper]+i;
+          if (xorder<=2) {
+            porb->RemapFluxPlm(pflux, shear_cc_[upper], eps, 1-upper, k,
+                               i, jl, ju+1, nl_, nu_, xgh);
+          } else {
+            printf("Not supported yet!\n");
+            // porb->RemapFluxPpm(pflux, pbuf, eps, 1-upper, k, i, jl, ju+1, xgh);
+          }
+          const int shift = xgh+1-upper;
+          for (int j=jl; j<=ju; j++) {
+            for (int n=nl_; n<=nu_; n++) {
+              var(k,j,ii,n) = shear_cc_[upper](k,i,j+shift,n)
+                              - (pflux(j+1,n) - pflux(j,n));
+            }
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+void RadBoundaryVariable::SendShearingBoxBoundaryBuffers() {
+  AthenaArray<Real> &var = *var_cc;
+  int ssize = nu_ + 1;
+  int offset[2]{0, 4};
+  for (int upper=0; upper<2; upper++) {
+    if (pbval_->is_shear[upper]) {
+      for (int n=0; n<4; n++) {
+        SimpleNeighborBlock& snb = pbval_->sb_data_[upper].send_neighbor[n];
+        if (snb.rank != -1) {
+          LoadShearingBoxBoundarySameLevel(var, shear_bd_var_[upper].send[n],
+                                           n+offset[upper]);
+          if (snb.rank == Globals::my_rank) {// on the same process
+            CopyShearBufferSameProcess(snb, shear_send_count_cc_[upper][n]*ssize, n,
+                                       upper);
+          } else { // MPI
+#ifdef MPI_PARALLEL
+            int tag = pbval_->CreateBvalsMPITag(snb.lid, n+offset[upper],
+                                                shear_cc_phys_id_);
+            MPI_Isend(shear_bd_var_[upper].send[n], shear_send_count_cc_[upper][n]*ssize,
+                      MPI_ATHENA_REAL, snb.rank, tag, MPI_COMM_WORLD,
+                      &shear_bd_var_[upper].req_send[n]);
+#endif
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+bool RadBoundaryVariable::ReceiveShearingBoxBoundaryBuffers() {
+  bool flag[2]{true, true};
+  int nb_offset[2]{0, 4};
+
+  for (int upper=0; upper<2; upper++) {
+    if (pbval_->is_shear[upper]) { // check inner boundaries
+      for (int n=0; n<4; n++) {
+        if (shear_bd_var_[upper].flag[n] == BoundaryStatus::completed) continue;
+        if (shear_bd_var_[upper].flag[n] == BoundaryStatus::waiting) {
+          // on the same process
+          if (pbval_->sb_data_[upper].recv_neighbor[n].rank == Globals::my_rank) {
+            flag[upper] = false;
+            continue;
+          } else { // MPI boundary
+#ifdef MPI_PARALLEL
+            int test;
+            MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &test,
+                       MPI_STATUS_IGNORE);
+            MPI_Test(&shear_bd_var_[upper].req_recv[n], &test, MPI_STATUS_IGNORE);
+            if (!static_cast<bool>(test)) {
+              flag[upper] = false;
+              continue;
+            }
+            shear_bd_var_[upper].flag[n] = BoundaryStatus::arrived;
+#endif
+          }
+        }
+        // set var if boundary arrived
+        SetShearingBoxBoundarySameLevel(shear_cc_[upper], shear_bd_var_[upper].recv[n],
+                                        n+nb_offset[upper]);
+        shear_bd_var_[upper].flag[n] = BoundaryStatus::completed; // completed
+      }  // loop over recv[0] to recv[3]
+    }  // if boundary is shearing
+  }  // loop over inner/outer boundaries
+  return (flag[0] && flag[1]);
+}
diff --git a/src/bvals/cc/nr_radiation/bvals_shear_rad_flux.cpp b/src/bvals/cc/nr_radiation/bvals_shear_rad_flux.cpp
new file mode 100644
index 0000000000..43e8fea0f7
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/bvals_shear_rad_flux.cpp
@@ -0,0 +1,234 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file bvals_shear_flux.cpp
+//  \brief functions that apply shearing box BCs for radiation fluxes
+//========================================================================================
+
+// C headers
+
+// C++ headers
+#include <algorithm>  // min
+#include <cmath>
+#include <cstdlib>
+#include <cstring>    // memcpy
+#include <iomanip>
+#include <iostream>   // endl
+#include <sstream>    // stringstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../coordinates/coordinates.hpp"
+#include "../../../eos/eos.hpp"
+#include "../../../field/field.hpp"
+#include "../../../globals.hpp"
+#include "../../../hydro/hydro.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../nr_radiation/integrators/rad_integrators.hpp"
+#include "../../../nr_radiation/radiation.hpp"
+#include "../../../orbital_advection/orbital_advection.hpp"
+#include "../../../parameter_input.hpp"
+#include "../../../utils/buffer_utils.hpp"
+#include "../../bvals.hpp"
+#include "../../bvals_interfaces.hpp"
+#include "./bvals_rad.hpp"
+
+// MPI header
+#ifdef MPI_PARALLEL
+#include <mpi.h>
+#endif
+
+
+//--------------------------------------------------------------------------------------
+//! \fn void CellCenteredBoundaryVariable::LoadFluxShearingBoxBoundarySameLevel(
+//!                               AthenaArray<Real> &src, Real *buf, int nb)
+//! \brief Set shearing boundary flux buffers for sending to a block on the same level
+
+void RadBoundaryVariable::LoadFluxShearingBoxBoundarySameLevel(
+                               AthenaArray<Real> &src, Real *buf, int nb) {
+  MeshBlock *pmb = pmy_block_;
+  int sj, sk, ej, ek;
+  int jo = pbval_->joverlap_flux_;
+  int *jmin1 = pbval_->sb_flux_data_[0].jmin_send;
+  int *jmin2 = pbval_->sb_flux_data_[1].jmin_send;
+  int *jmax1 = pbval_->sb_flux_data_[0].jmax_send;
+  int *jmax2 = pbval_->sb_flux_data_[1].jmax_send;
+  sk = pmb->ks;        ek = pmb->ke;
+
+  if (nb<3) { // inner boundary
+    sj = jmin1[nb]-jo-1;
+    ej = jmax1[nb]-jo-1;
+  } else if (nb<6) { //outer boundary
+    sj = jmin2[nb-3]+jo;
+    ej = jmax2[nb-3]+jo;
+  } else {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in CellCenteredBoundaryVariable:"
+        << "LoadFluxShearingBoxBoundarySameLevel"     << std::endl
+        << "nb = " << nb << " not valid" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  int p = 0;
+  BufferUtility::PackData(src, buf, nl_, nu_, sj, ej, sk, ek, p);
+
+  return;
+}
+
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void CellCenteredBoundaryVariable::SetShearingBoxBoundarySameLevel(
+//!                        AthenaArray<Real> &src, Real *buf, const int nb)
+//! \brief Set shearing box boundary flux data received from a block on the same level
+
+void RadBoundaryVariable::SetFluxShearingBoxBoundarySameLevel(
+                       AthenaArray<Real> &src, Real *buf, const int nb) {
+  MeshBlock *pmb = pmy_block_;
+  int sj, sk, ej, ek;
+  int &xgh = pbval_->xgh_;
+  sk = pmb->ks;        ek = pmb->ke;
+
+  int *jmin1 = pbval_->sb_flux_data_[0].jmin_recv;
+  int *jmin2 = pbval_->sb_flux_data_[1].jmin_recv;
+  int *jmax1 = pbval_->sb_flux_data_[0].jmax_recv;
+  int *jmax2 = pbval_->sb_flux_data_[1].jmax_recv;
+
+  if (nb<3) {
+    sj = jmin1[nb]+xgh;     ej = jmax1[nb]+xgh;
+  } else if (nb<6) {
+    sj = jmin2[nb-3]+xgh;   ej = jmax2[nb-3]+xgh;
+  } else {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in CellCenteredBoundaryVariable::"
+        << "SetFluxShearingBoxBoundarySameLevel"<<std::endl
+        << "nb = " << nb << " not valid" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  int p = 0;
+
+  for (int k=sk; k<=ek; k++) {
+    for (int j=sj; j<=ej; j++) {
+#pragma omp simd
+      for (int n=nl_; n<=nu_; ++n) {
+        src(k,0,j,n) = buf[p++];
+      }
+    }
+  }
+  return;
+}
+
+
+//----------------------------------------------------------------------------------------
+//! \fn bool CellCenteredBoundaryVariable::SetFluxShearingBoxBoundaryBuffers()
+//! \brief Update shearing box boundary fluxes
+
+void RadBoundaryVariable::SetFluxShearingBoxBoundaryBuffers() {
+  MeshBlock *pmb = pmy_block_;
+  OrbitalAdvection *porb = pmb->porb;
+  AthenaArray<Real> &pflux = pflux_;
+  int &xgh = pbval_->xgh_;
+  int &xorder = pmb->pnrrad->pradintegrator->rad_xorder;
+  int is = pmb->is, ie = pmb->ie;
+  int js = pmb->js, je = pmb->je;
+  int ks = pmb->ks, ke = pmb->ke;
+
+  int ib[2]{is, ie+1};
+  for (int upper=0; upper<2; upper++) {
+    if (pbval_->is_shear[upper]) { // check inner boundaries
+      Real eps = (1.0-2*upper)*pbval_->eps_flux_;
+      int ii = ib[upper];
+      for (int k=ks; k<=ke; k++) {
+        if (xorder<=2) {
+          porb->RemapFluxPlm(pflux, shear_map_flx_[upper], eps, 1-upper,
+                                          k, 0, js, je+1, nl_, nu_, xgh);
+        } else {
+           printf("3 order shearing box Not supported yet!\n");
+           // porb->RemapFluxPpm(pflux, pbuf, eps, 1-upper, k, 0, js, je+1, xgh);
+        }
+        const int shift = xgh+1-upper;
+        for (int j=js; j<=je; j++) {
+          for (int n=nl_; n<=nu_; n++) {
+            x1flux(k,j,ii,n) = 0.5*(x1flux(k,j,ii,n)+shear_map_flx_[upper](k,0,j+shift,n)
+                               - (pflux(j+1,n) - pflux(j,n)));
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+void RadBoundaryVariable::SendFluxShearingBoxBoundaryBuffers() {
+  int ssize = nu_ + 1;
+  int offset[2]{0, 3};
+  for (int upper=0; upper<2; upper++) {
+    if (pbval_->is_shear[upper]) {
+      for (int n=0; n<3; n++) {
+        SimpleNeighborBlock& snb = pbval_->sb_flux_data_[upper].send_neighbor[n];
+        if (snb.rank != -1) {
+          LoadFluxShearingBoxBoundarySameLevel(shear_var_flx_[upper],
+                                   shear_bd_flux_[upper].send[n], n+offset[upper]);
+          if (snb.rank == Globals::my_rank) {// on the same process
+            CopyShearFluxSameProcess(snb, shear_send_count_flx_[upper][n]*ssize, n,
+                                     upper);
+          } else { // MPI
+#ifdef MPI_PARALLEL
+            int tag = pbval_->CreateBvalsMPITag(snb.lid, n+offset[upper],
+                                                shear_flx_phys_id_);
+            MPI_Isend(shear_bd_flux_[upper].send[n],
+                      shear_send_count_flx_[upper][n]*ssize,
+                      MPI_ATHENA_REAL, snb.rank, tag, MPI_COMM_WORLD,
+                      &shear_bd_flux_[upper].req_send[n]);
+#endif
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+
+bool RadBoundaryVariable::ReceiveFluxShearingBoxBoundaryBuffers() {
+  bool flag[2]{true, true};
+  int nb_offset[2]{0, 3};
+  for (int upper=0; upper<2; upper++) {
+    if (pbval_->is_shear[upper]) { // check inner boundaries
+      for (int n=0; n<3; n++) {
+        if (shear_bd_flux_[upper].flag[n] == BoundaryStatus::completed) continue;
+        if (shear_bd_flux_[upper].flag[n] == BoundaryStatus::waiting) {
+          // on the same process
+          if (pbval_->sb_flux_data_[upper].recv_neighbor[n].rank == Globals::my_rank) {
+            flag[upper] = false;
+            continue;
+          } else { // MPI boundary
+#ifdef MPI_PARALLEL
+            int test;
+            MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &test,
+                       MPI_STATUS_IGNORE);
+            MPI_Test(&shear_bd_flux_[upper].req_recv[n], &test, MPI_STATUS_IGNORE);
+            if (!static_cast<bool>(test)) {
+              flag[upper] = false;
+              continue;
+            }
+            shear_bd_flux_[upper].flag[n] = BoundaryStatus::arrived;
+#endif
+          }
+        }
+        // set var if boundary arrived
+        SetFluxShearingBoxBoundarySameLevel(shear_map_flx_[upper],
+                                            shear_bd_flux_[upper].recv[n],
+                                            n+nb_offset[upper]);
+        shear_bd_flux_[upper].flag[n] = BoundaryStatus::completed; // completed
+      }
+    }
+  }
+  return (flag[0] && flag[1]);
+}
diff --git a/src/bvals/cc/nr_radiation/flux_correction_radiation.cpp b/src/bvals/cc/nr_radiation/flux_correction_radiation.cpp
new file mode 100644
index 0000000000..1fd7c6b79a
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/flux_correction_radiation.cpp
@@ -0,0 +1,350 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file flux_correction_radiation.cpp
+//! \brief
+//========================================================================================
+
+// C headers
+
+// C++ headers
+#include <algorithm>  // min
+#include <cmath>
+#include <cstdlib>
+#include <cstring>    // std::memcpy
+#include <iomanip>
+#include <iostream>   // endl
+#include <sstream>    // stringstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../coordinates/coordinates.hpp"
+#include "../../../eos/eos.hpp"
+#include "../../../field/field.hpp"
+#include "../../../globals.hpp"
+#include "../../../hydro/hydro.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../nr_radiation/integrators/rad_integrators.hpp"
+#include "../../../nr_radiation/radiation.hpp"
+#include "../../../parameter_input.hpp"
+#include "../../../utils/buffer_utils.hpp"
+#include "../bvals_cc.hpp"
+#include "./bvals_rad.hpp"
+
+// MPI header
+#ifdef MPI_PARALLEL
+#include <mpi.h>
+#endif
+
+//----------------------------------------------------------------------------------------
+//! \fn int RadBoundaryVariable::LoadFluxBoundaryBufferSameLevel(Real *buf,
+//!                                                  const NeighborBlock& nb)
+//! \brief Set surface flux buffers for sending to a block on the same level
+
+int RadBoundaryVariable::LoadFluxBoundaryBufferSameLevel(Real *buf,
+                                                         const NeighborBlock& nb) {
+  MeshBlock *pmb=pmy_block_;
+  NRRadiation *prad = pmb->pnrrad;
+  Real qomL = pbval_->qomL_;
+  int p = 0;
+  if (pbval_->shearing_box == 1 && nb.shear
+      && (nb.fid == BoundaryFace::inner_x1 || nb.fid == BoundaryFace::outer_x1)) {
+    int i;
+    int sign;
+    if (nb.fid == BoundaryFace::inner_x1) {
+      i = pmb->is;
+      sign = -1;
+    } else {
+      i = pmb->ie + 1;
+      sign =  1;
+    }
+    // pack x1flux
+    for (int k=pmb->ks; k<=pmb->ke; k++) {
+      for (int j=pmb->js; j<=pmb->je; j++) {
+        // convert flux due to velocity difference
+        Real vx = pmb->phydro->w(IVX,k,j,i);
+        Real vy = pmb->phydro->w(IVY,k,j,i);
+        Real vz = pmb->phydro->w(IVZ,k,j,i);
+        Real *mux = &(prad->mu(0,k,j,i,0));
+        Real *muy = &(prad->mu(1,k,j,i,0));
+        Real *muz = &(prad->mu(2,k,j,i,0));
+        Real *flux_lab = &(x1flux(k,j,i,0));
+        prad->pradintegrator->LabToCom(vx,vy,vz,mux,muy,muz,flux_lab,ir_cm_);
+        flux_lab = &(ir_lab_(0));
+        prad->pradintegrator->ComToLab(vx,vy+sign*qomL,vz,mux,muy,muz,ir_cm_,flux_lab);
+        for (int nn=nl_; nn<=nu_; nn++) {
+          buf[p++] = flux_lab[nn];
+        }
+      }
+    }
+  }
+  return p;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn int RadBoundaryVariable::LoadFluxBoundaryBufferToCoarser(Real *buf,
+//!                                                        const NeighborBlock& nb)
+//! \brief Set surface flux buffers for sending to a block on the coarser level
+
+int RadBoundaryVariable::LoadFluxBoundaryBufferToCoarser(Real *buf,
+                                                         const NeighborBlock& nb) {
+  MeshBlock *pmb=pmy_block_;
+  Coordinates *pco = pmb->pcoord;
+  // cache pointers to surface area arrays (BoundaryBase protected variable)
+  AthenaArray<Real> &sarea0 = pbval_->sarea_[0];
+  AthenaArray<Real> &sarea1 = pbval_->sarea_[1];
+  int p = 0;
+  // x1 direction
+  if (nb.fid == BoundaryFace::inner_x1 || nb.fid == BoundaryFace::outer_x1) {
+    int i = pmb->is + (pmb->ie-pmb->is + 1)*nb.fid;
+    if (pmb->block_size.nx3>1) { // 3D
+      for (int k=pmb->ks; k<=pmb->ke; k+=2) {
+        for (int j=pmb->js; j<=pmb->je; j+=2) {
+          Real amm = pco->GetFace1Area(k,   j,   i);
+          Real amp = pco->GetFace1Area(k,   j+1, i);
+          Real apm = pco->GetFace1Area(k+1, j,   i);
+          Real app = pco->GetFace1Area(k+1, j+1, i);
+          Real tarea = amm + amp + apm + app;
+          for (int nn=nl_; nn<=nu_; nn++) {
+            buf[p++] = (x1flux(k  , j  , i, nn)*amm
+                       + x1flux(k  , j+1, i, nn)*amp
+                       + x1flux(k+1, j  , i, nn)*apm
+                       + x1flux(k+1, j+1, i, nn)*app)/tarea;
+          }
+        }
+      }
+    } else if (pmb->block_size.nx2>1) { // 2D
+      int k = pmb->ks;
+      for (int j=pmb->js; j<=pmb->je; j+=2) {
+        Real am = pco->GetFace1Area(k, j,   i);
+        Real ap = pco->GetFace1Area(k, j+1, i);
+        Real tarea = am + ap;
+        for (int nn=nl_; nn<=nu_; nn++) {
+          buf[p++] = (x1flux(k, j  , i, nn)*am + x1flux(k, j+1, i, nn)*ap)/tarea;
+        }
+      }
+    } else { // 1D
+      int k = pmb->ks, j = pmb->js;
+      for (int nn=nl_; nn<=nu_; nn++)
+        buf[p++] = x1flux(k, j, i, nn);
+    }
+    // x2 direction
+  } else if (nb.fid == BoundaryFace::inner_x2 || nb.fid == BoundaryFace::outer_x2) {
+    int j = pmb->js + (pmb->je-pmb->js + 1)*(nb.fid & 1);
+    if (pmb->block_size.nx3>1) { // 3D
+      for (int k=pmb->ks; k<=pmb->ke; k+=2) {
+        pco->Face2Area(k  , j, pmb->is, pmb->ie, sarea0);
+        pco->Face2Area(k+1, j, pmb->is, pmb->ie, sarea1);
+        for (int i=pmb->is; i<=pmb->ie; i+=2) {
+          Real tarea = sarea0(i) + sarea0(i+1) + sarea1(i) + sarea1(i+1);
+          for (int nn=nl_; nn<=nu_; nn++) {
+            buf[p++] = (x2flux(k  , j, i  , nn)*sarea0(i  )
+                       + x2flux(k  , j, i+1, nn)*sarea0(i+1)
+                       + x2flux(k+1, j, i  , nn)*sarea1(i  )
+                       + x2flux(k+1, j, i+1, nn)*sarea1(i+1))/tarea;
+          }
+        }
+      }
+    } else if (pmb->block_size.nx2>1) { // 2D
+      int k = pmb->ks;
+      pco->Face2Area(0, j, pmb->is ,pmb->ie, sarea0);
+      for (int i=pmb->is; i<=pmb->ie; i+=2) {
+        Real tarea = sarea0(i) + sarea0(i+1);
+        for (int nn=nl_; nn<=nu_; nn++) {
+          buf[p++] = (x2flux(k, j, i  , nn)*sarea0(i  )
+                     + x2flux(k, j, i+1, nn)*sarea0(i+1))/tarea;
+        }
+      }
+    }
+    // x3 direction - 3D onl_y
+  } else if (nb.fid == BoundaryFace::inner_x3 || nb.fid == BoundaryFace::outer_x3) {
+    int k = pmb->ks + (pmb->ke-pmb->ks + 1)*(nb.fid & 1);
+      for (int j=pmb->js; j<=pmb->je; j+=2) {
+      pco->Face3Area(k, j,   pmb->is, pmb->ie, sarea0);
+      pco->Face3Area(k, j+1, pmb->is, pmb->ie, sarea1);
+      for (int i=pmb->is; i<=pmb->ie; i+=2) {
+        Real tarea = sarea0(i) + sarea0(i+1) + sarea1(i) + sarea1(i+1);
+        for (int nn=nl_; nn<=nu_; nn++) {
+          buf[p++] = (x3flux(k, j  , i, nn )*sarea0(i  )
+                     + x3flux(k, j  , i+1, nn)*sarea0(i+1)
+                     + x3flux(k, j+1, i, nn)*sarea1(i  )
+                     + x3flux(k, j+1, i+1, nn)*sarea1(i+1))/tarea;
+        }
+      }
+    }
+  }
+  return p;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void CellCenteredBoundaryVariable::SendFluxCorrection()
+//! \brief Send surface flux buffers
+
+void RadBoundaryVariable::SendFluxCorrection() {
+  MeshBlock *pmb=pmy_block_;
+  for (int n=0; n<pbval_->nneighbor; n++) {
+    NeighborBlock& nb = pbval_->neighbor[n];
+    if (nb.ni.type != NeighborConnect::face) break;
+    if (bd_var_flcor_.sflag[nb.bufid] == BoundaryStatus::completed) continue;
+    int p = 0;
+    if (nb.snb.level == pmb->loc.level) { // to same level
+      p = LoadFluxBoundaryBufferSameLevel(bd_var_flcor_.send[nb.bufid],nb);
+    } else if (nb.snb.level < pmb->loc.level) { // to coaser
+      p = LoadFluxBoundaryBufferToCoarser(bd_var_flcor_.send[nb.bufid],nb);
+    }
+    // else { // to finer
+    // }
+
+    if (p>0) {
+      if (nb.snb.rank == Globals::my_rank) // on the same node
+        CopyFluxCorrectionBufferSameProcess(nb, p);
+#ifdef MPI_PARALLEL
+      else
+        MPI_Start(&(bd_var_flcor_.req_send[nb.bufid]));
+#endif
+    } else {
+      if (nb.snb.rank == Globals::my_rank) // on the same node
+        SetCompletedFlagSameProcess(nb);
+    }
+    bd_var_flcor_.sflag[nb.bufid] = BoundaryStatus::completed;
+  }
+  return;
+}
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::SetFluxBoundarySameLevel(Real *buf,
+//!                                                               const NeighborBlock& nb)
+//! \brief Set surface flux data received from a block on the same level
+
+void RadBoundaryVariable::SetFluxBoundarySameLevel(Real *buf,
+                                                           const NeighborBlock& nb) {
+  MeshBlock *pmb = pmy_block_;
+  int p = 0;
+
+  if (nb.fid == BoundaryFace::inner_x1) {
+    for (int k=pmb->ks; k<=pmb->ke; k++) {
+      for (int j=pmb->js; j<=pmb->je; j++) {
+        for (int nn=nl_; nn<=nu_; nn++) {
+          shear_var_flx_[0](k,j,nn) = buf[p++];
+        }
+      }
+    }
+  } else {
+    for (int k=pmb->ks; k<=pmb->ke; k++) {
+      for (int j=pmb->js; j<=pmb->je; j++) {
+        for (int nn=nl_; nn<=nu_; nn++) {
+          shear_var_flx_[1](k,j,nn) = buf[p++];
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void FaceCenteredBoundaryVariable::SetFluxBoundaryFromFiner(Real *buf,
+//!                                                               const NeighborBlock& nb)
+//! \brief Set surface flux data received from a block on the finer level
+
+void RadBoundaryVariable::SetFluxBoundaryFromFiner(Real *buf,
+                                                           const NeighborBlock& nb) {
+  MeshBlock *pmb = pmy_block_;
+  int p = 0;
+  if (nb.fid == BoundaryFace::inner_x1 || nb.fid == BoundaryFace::outer_x1) {
+    int il = pmb->is + (pmb->ie - pmb->is)*nb.fid+nb.fid;
+    int jl = pmb->js, ju = pmb->je, kl = pmb->ks, ku = pmb->ke;
+    if (nb.ni.fi1 == 0) ju -= pmb->block_size.nx2/2;
+    else          jl += pmb->block_size.nx2/2;
+    if (nb.ni.fi2 == 0) ku -= pmb->block_size.nx3/2;
+    else          kl += pmb->block_size.nx3/2;
+
+    for (int k=kl; k<=ku; k++) {
+      for (int j=jl; j<=ju; j++)
+        for (int nn=nl_; nn<=nu_; nn++) {
+          x1flux(k,j,il, nn) = buf[p++];
+      }
+    }
+  } else if (nb.fid == BoundaryFace::inner_x2 || nb.fid == BoundaryFace::outer_x2) {
+    int jl = pmb->js + (pmb->je - pmb->js)*(nb.fid & 1) + (nb.fid & 1);
+    int il = pmb->is, iu = pmb->ie, kl = pmb->ks, ku = pmb->ke;
+    if (nb.ni.fi1 == 0) iu -= pmb->block_size.nx1/2;
+    else          il += pmb->block_size.nx1/2;
+    if (nb.ni.fi2 == 0) ku -= pmb->block_size.nx3/2;
+    else          kl += pmb->block_size.nx3/2;
+
+    for (int k=kl; k<=ku; k++) {
+      for (int i=il; i<=iu; i++)
+        for (int nn=nl_; nn<=nu_; nn++) {
+          x2flux(k,jl,i, nn) = buf[p++];
+      }
+    }
+  } else if (nb.fid == BoundaryFace::inner_x3 || nb.fid == BoundaryFace::outer_x3) {
+    int kl = pmb->ks + (pmb->ke - pmb->ks)*(nb.fid & 1) + (nb.fid & 1);
+    int il = pmb->is, iu = pmb->ie, jl = pmb->js, ju = pmb->je;
+    if (nb.ni.fi1 == 0) iu -= pmb->block_size.nx1/2;
+    else          il += pmb->block_size.nx1/2;
+    if (nb.ni.fi2 == 0) ju -= pmb->block_size.nx2/2;
+    else          jl += pmb->block_size.nx2/2;
+
+    for (int j=jl; j<=ju; j++) {
+      for (int i=il; i<=iu; i++)
+        for (int nn=nl_; nn<=nu_; nn++) {
+          x3flux(kl,j,i, nn) = buf[p++];
+      }
+    }
+  }
+  return;
+}
+
+
+bool RadBoundaryVariable::ReceiveFluxCorrection() {
+  MeshBlock *pmb = pmy_block_;
+  bool flag=true;
+
+  for (int n=0; n<pbval_->nneighbor; n++) {
+    NeighborBlock& nb = pbval_->neighbor[n];
+    if (nb.ni.type != NeighborConnect::face) break;
+    if (bd_var_flcor_.flag[nb.bufid] == BoundaryStatus::completed) continue;
+    // receive data
+    if (bd_var_flcor_.flag[nb.bufid] == BoundaryStatus::waiting) {
+      if (nb.snb.rank == Globals::my_rank) {// on the same process
+        flag = false;
+        continue;
+      }
+#ifdef MPI_PARALLEL
+      else { // NOLINT
+        int test;
+        // probe MPI communications.  This is a bit of black magic that seems to promote
+        // communications to top of stack and gets them to complete more quickly
+        MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &test,
+                   MPI_STATUS_IGNORE);
+        MPI_Test(&(bd_var_flcor_.req_recv[nb.bufid]), &test, MPI_STATUS_IGNORE);
+        if (!static_cast<bool>(test)) {
+          flag = false;
+          continue;
+        }
+        bd_var_flcor_.flag[nb.bufid] = BoundaryStatus::arrived;
+      }
+#endif
+    }
+    // set data
+    if (bd_var_flcor_.flag[nb.bufid] == BoundaryStatus::arrived) {
+      if (nb.snb.level==pmb->loc.level)      // from same level
+        SetFluxBoundarySameLevel(bd_var_flcor_.recv[nb.bufid],nb);
+      else if (nb.snb.level>pmb->loc.level)  // from finer
+        SetFluxBoundaryFromFiner(bd_var_flcor_.recv[nb.bufid],nb);
+      // else                                   // from coarser
+      //   nothing to do
+
+      bd_var_flcor_.flag[nb.bufid] = BoundaryStatus::completed;
+    }
+  }
+
+  return flag;
+}
diff --git a/src/bvals/cc/nr_radiation/outflow_rad.cpp b/src/bvals/cc/nr_radiation/outflow_rad.cpp
new file mode 100644
index 0000000000..45ab9b7407
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/outflow_rad.cpp
@@ -0,0 +1,187 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file outflow_rad.cpp
+//  \brief implementation of outflow BCs in each dimension
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../nr_radiation/radiation.hpp"
+#include "./bvals_rad.hpp"
+
+
+// The angular grid can change
+// The angular octant is
+//   1  |  0       5  |  4
+//   -------      ---------
+//   3  |  2       7  |  6
+// in radiatin class, n_ang is angles per octant, noct is the number of octant
+// radiation relfection means set specific intensitiy corresponding
+// the value in the opposite octant
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::OutflowInnerX1(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief outflow boundary conditions, inner x1 boundary
+
+void RadBoundaryVariable::OutflowInnerX1(Real time, Real dt, int il, int jl, int ju,
+                                         int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  int &nang = pmy_block_->pnrrad->nang; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=1; i<=ngh; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for (int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            (*var_cc)(k,j,il-i,ang) = (*var_cc)(k,j,il,ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::OutflowOuterX1(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief Outflow boundary conditions, outer x1 boundary
+
+void RadBoundaryVariable::OutflowOuterX1(Real time, Real dt, int iu, int jl, int ju,
+                                         int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  int &nang = pmy_block_->pnrrad->nang; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=1; i<=ngh; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for (int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            (*var_cc)(k,j,iu+i,ang) = (*var_cc)(k,j,iu,ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::OutflowInnerX2(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief Outflow boundary conditions, inner x2 boundary
+
+void RadBoundaryVariable::OutflowInnerX2(Real time, Real dt, int il, int iu, int jl,
+                                         int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  int nang = pmy_block_->pnrrad->nang;
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for (int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            (*var_cc)(k,jl-j,i,ang) = (*var_cc)(k,jl,i,ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::OutflowOuterX2(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief Outflow boundary conditions, outer x2 boundary
+
+void RadBoundaryVariable::OutflowOuterX2(Real time, Real dt, int il, int iu, int ju,
+                                         int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  int &nang = pmy_block_->pnrrad->nang; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for(int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            (*var_cc)(k,ju+j,i,ang) = (*var_cc)(k,ju,i,ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::OutflowInnerX3(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+
+void RadBoundaryVariable::OutflowInnerX3(Real time, Real dt, int il, int iu, int jl,
+                                         int ju, int kl, int ngh) {
+  // copy radiation variables into ghost zones,
+
+  int &nang = pmy_block_->pnrrad->nang;
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for (int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            (*var_cc)(kl-k,j,i,ang) = (*var_cc)(kl,j,i,ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::OutflowInnerX3(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief Outflow boundary conditions, outer x3 boundary
+
+void RadBoundaryVariable::OutflowOuterX3(Real time, Real dt, int il, int iu, int jl,
+                                         int ju, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+
+  int &nang = pmy_block_->pnrrad->nang; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for (int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            (*var_cc)(ku+k,j,i,ang) = (*var_cc)(ku,j,i,ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
diff --git a/src/bvals/cc/nr_radiation/polarwedge_rad.cpp b/src/bvals/cc/nr_radiation/polarwedge_rad.cpp
new file mode 100644
index 0000000000..c3b265c065
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/polarwedge_rad.cpp
@@ -0,0 +1,59 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file polarwedge_rad.cpp
+//  \brief implementation of polar wedge radiation BCs in x2 direction
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../nr_radiation/radiation.hpp"
+#include "bvals_rad.hpp"
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::PolarWedgeInnerX2(
+//          Real time, Real dt, int il, int iu, int jl, int kl, int ku, int ngh)
+//  \brief polar wedge boundary conditions, inner x2 boundary
+
+// do not flip the sign of specific intensity
+void RadBoundaryVariable::PolarWedgeInnerX2(
+    Real time, Real dt, int il, int iu, int jl, int kl, int ku, int ngh) {
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+#pragma omp simd
+        for (int n=0; n<=nu_; ++n) {
+          (*var_cc)(k,jl-j,i,n) = (*var_cc)(k,jl+j-1,i,n);
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::PolarWedgeOuterX2(
+//          Real time, Real dt, int il, int iu, int ju, int kl, int ku, int ngh)
+//  \brief polar wedge boundary conditions, outer x2 boundary
+
+void RadBoundaryVariable::PolarWedgeOuterX2(
+    Real time, Real dt, int il, int iu, int ju, int kl, int ku, int ngh) {
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+#pragma omp simd
+        for (int n=0; n<=nu_; ++n) {
+          (*var_cc)(k,ju+j,i,n) = (*var_cc)(k,ju-j+1,i,n);
+        }
+      }
+    }
+  }
+  return;
+}
diff --git a/src/bvals/cc/nr_radiation/reflect_rad.cpp b/src/bvals/cc/nr_radiation/reflect_rad.cpp
new file mode 100644
index 0000000000..958e3ce1a7
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/reflect_rad.cpp
@@ -0,0 +1,248 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file reflect.cpp
+//  \brief implementation of reflecting BCs in each dimension
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../nr_radiation/radiation.hpp"
+#include "./bvals_rad.hpp"
+
+
+// The angular grid can change
+// The angular octant is
+//   1  |  0       5  |  4
+//   -------      ---------
+//   3  |  2       7  |  6
+// in radiatin class, n_ang is angles per octant, noct is the number of octant
+// radiation relfection means set specific intensitiy corresponding
+// the value in the opposite octant
+
+// Temporary function to copy intensity
+void CopyIntensity(Real *iri, Real *iro, int li, int lo, int n_ang) {
+  // here ir is only intensity for each cell and each frequency band
+  for (int n=0; n<n_ang; ++n) {
+    int angi = li * n_ang + n;
+    int ango = lo * n_ang + n;
+    iro[angi] = iri[ango];
+    iro[ango] = iri[angi];
+  }
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::ReflectInnerX1(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief REFLECTING boundary conditions, inner x1 boundary
+
+void RadBoundaryVariable::ReflectInnerX1(Real time, Real dt, int il, int jl, int ju,
+                                         int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=1; i<=ngh; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *iri = &var(k,j,(il+i-1),ifr*pmy_block_->pnrrad->nang);
+          Real *iro = &var(k,j, il-i, ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity(iri, iro, 0, 1, n_ang);
+          if (noct > 2) {
+            CopyIntensity(iri, iro, 2, 3, n_ang);
+          }
+          if (noct > 3) {
+            CopyIntensity(iri, iro, 4, 5, n_ang);
+            CopyIntensity(iri, iro, 6, 7, n_ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::ReflectOuterX1(
+//          Real time, Real dt, int iu, int jl, int ju, int kl, int ku, int ngh)
+//  \brief REFLECTING boundary conditions, outer x1 boundary
+
+void RadBoundaryVariable::ReflectOuterX1(
+    Real time, Real dt, int iu, int jl, int ju, int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=1; i<=ngh; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *iri = &var(k,j,(iu-i+1),ifr*pmy_block_->pnrrad->nang);
+          Real *iro = &var(k,j, iu+i, ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity(iri, iro, 0, 1, n_ang);
+          if (noct > 2) {
+            CopyIntensity(iri, iro, 2, 3, n_ang);
+          }
+          if (noct > 3) {
+            CopyIntensity(iri, iro, 4, 5, n_ang);
+            CopyIntensity(iri, iro, 6, 7, n_ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::ReflectInnerX2(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief REFLECTING boundary conditions, inner x2 boundary
+
+void RadBoundaryVariable::ReflectInnerX2(Real time, Real dt, int il, int iu, int jl,
+                                         int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *iri = &var(k,jl+j-1,i,ifr*pmy_block_->pnrrad->nang);
+          Real *iro = &var(k,jl-j,i, ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity(iri, iro, 0, 2, n_ang);
+          CopyIntensity(iri, iro, 1, 3, n_ang);
+
+          if (noct > 3) {
+            CopyIntensity(iri, iro, 4, 6, n_ang);
+            CopyIntensity(iri, iro, 5, 7, n_ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::ReflectOuterX2(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief REFLECTING boundary conditions, outer x2 boundary
+
+void RadBoundaryVariable::ReflectOuterX2(Real time, Real dt, int il, int iu, int ju,
+                                         int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *iri = &var(k,ju-j+1,i,ifr*pmy_block_->pnrrad->nang);
+          Real *iro = &var(k,ju+j,i, ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity(iri, iro, 0, 2, n_ang);
+          CopyIntensity(iri, iro, 1, 3, n_ang);
+
+          if (noct > 3) {
+            CopyIntensity(iri, iro, 4, 6, n_ang);
+            CopyIntensity(iri, iro, 5, 7, n_ang);
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::ReflectInnerX3(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief REFLECTING boundary conditions, inner x3 boundary
+
+void RadBoundaryVariable::ReflectInnerX3(Real time, Real dt, int il, int iu, int jl,
+                                         int ju, int kl, int ngh) {
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *iri = &var(kl+k-1,j,i,ifr*pmy_block_->pnrrad->nang);
+          Real *iro = &var(kl-k,j,i, ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity(iri, iro, 0, 4, n_ang);
+          CopyIntensity(iri, iro, 1, 5, n_ang);
+          CopyIntensity(iri, iro, 2, 6, n_ang);
+          CopyIntensity(iri, iro, 3, 7, n_ang);
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::ReflectOuterX3(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief REFLECTING boundary conditions, outer x3 boundary
+
+void RadBoundaryVariable::ReflectOuterX3(Real time, Real dt, int il, int iu, int jl,
+                                         int ju, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *iri = &var(ku-k+1,j,i,ifr*pmy_block_->pnrrad->nang);
+          Real *iro = &var(ku+k,j,i, ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity(iri, iro, 0, 4, n_ang);
+          CopyIntensity(iri, iro, 1, 5, n_ang);
+          CopyIntensity(iri, iro, 2, 6, n_ang);
+          CopyIntensity(iri, iro, 3, 7, n_ang);
+        }
+      }
+    }
+  }
+  return;
+}
diff --git a/src/bvals/cc/nr_radiation/rotate_rad.cpp b/src/bvals/cc/nr_radiation/rotate_rad.cpp
new file mode 100644
index 0000000000..b20f4274f2
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/rotate_rad.cpp
@@ -0,0 +1,335 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file rotate_rad.cpp
+//  \brief implementation of rotating radiation BCs in each dimension
+//======================================================================================
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../nr_radiation/radiation.hpp"
+#include "./bvals_rad.hpp"
+
+// The angular octant ( in x-y plane) is
+//   1  |  0       5  |  4
+//   -------      ---------
+//   3  |  2       7  |  6
+// in X-Z plane, it is
+//   5  |  4       7  |  6
+//   -------      ---------
+//   1  |  0       3  |  2
+
+
+// in radiatin class, n_ang is angles per octant, noct is the number of octant
+
+
+// Temporary function to copy intensity
+// swap 0 and 3, 1 and 2
+// change sign of both Fx and Fy
+// only do this for 2D spherical polar
+void CopyIntensity2(Real *ir, int n_ang, int direction) {
+  // here ir is only intensity for each cell and each frequency band
+  for (int n=0; n<n_ang; ++n) {
+    // from 0 to 4, 4 to 5, 5 to 1, 1 to 0
+    int ang1 = 0 * n_ang + n;
+    int ang2 = 3 * n_ang + n;
+
+    int ang3 = 1 * n_ang + n;
+    int ang4 = 2 * n_ang + n;
+
+    Real temp = ir[ang1];
+    ir[ang1] = ir[ang2];
+    ir[ang2] = temp;
+
+
+    temp = ir[ang3];
+    ir[ang3] = ir[ang4];
+    ir[ang4] = temp;
+  }
+  return;
+}
+
+void CopyIntensity3(Real *ir, int n_ang, int direction) {
+  // here ir is only intensity for each cell and each frequency band
+  for (int n=0; n<n_ang; ++n) {
+    // switch 1 and 2, switch 3 and 0
+    int ang1 = 0 * n_ang + n;
+    int ang2 = 3 * n_ang + n;
+    int ang3 = 1 * n_ang + n;
+    int ang4 = 2 * n_ang + n;
+    Real temp = ir[ang1];
+    ir[ang1] = ir[ang2];
+    ir[ang2] = temp;
+
+    temp = ir[ang4];
+    ir[ang4] = ir[ang3];
+    ir[ang3] = temp;
+  }
+  for (int n=0; n<n_ang; ++n) {
+    // switch 5 and 6, switch 4 and 7
+    int ang1 = 5 * n_ang + n;
+    int ang2 = 6 * n_ang + n;
+    int ang3 = 4 * n_ang + n;
+    int ang4 = 7 * n_ang + n;
+    Real temp = ir[ang1];
+    ir[ang1] = ir[ang2];
+    ir[ang2] = temp;
+    temp = ir[ang4];
+    ir[ang4] = ir[ang3];
+    ir[ang3] = temp;
+  }
+  return;
+}
+
+// The angular octant ( in x-y plane) is
+//   1  |  0       5  |  4
+//   -------      ---------
+//   3  |  2       7  |  6
+
+void CopyIntensity4(Real *ir, int n_ang, int direction) {
+  // here ir is only intensity for each cell and each frequency band
+  for (int n=0; n<n_ang; ++n) {
+    // from 0 to 1, 1 to 3, 3 to 2, 2 to 0
+    int ang1 = 0 * n_ang + n;
+    int ang2 = 1 * n_ang + n;
+    int ang3 = 3 * n_ang + n;
+    int ang4 = 2 * n_ang + n;
+    Real temp = ir[ang1];
+    ir[ang1] = ir[ang4];
+    ir[ang4] = ir[ang3];
+    ir[ang3] = ir[ang2];
+    ir[ang2] = temp;
+  }
+  for (int n=0; n<n_ang; ++n) {
+    // from 4 to 5, 5 to 7, 7 to 6, 6 to 1
+    int ang1 = 4 * n_ang + n;
+    int ang2 = 5 * n_ang + n;
+    int ang3 = 7 * n_ang + n;
+    int ang4 = 6 * n_ang + n;
+
+    Real temp = ir[ang1];
+    ir[ang1] = ir[ang4];
+    ir[ang4] = ir[ang3];
+    ir[ang3] = ir[ang2];
+    ir[ang2] = temp;
+  }
+  return;
+}
+
+void CopyIntensity5(Real *ir, int n_ang, int direction) {
+  // here ir is only intensity for each cell and each frequency band
+  for (int n=0; n<n_ang; ++n) {
+    // switch 0 and 4, switch 5 and 1
+    int ang1 = 0 * n_ang + n;
+    int ang2 = 4 * n_ang + n;
+    int ang3 = 5 * n_ang + n;
+    int ang4 = 1 * n_ang + n;
+    Real temp = ir[ang1];
+    ir[ang1] = ir[ang2];
+    ir[ang2] = temp;
+
+    temp = ir[ang4];
+    ir[ang4] = ir[ang3];
+    ir[ang3] = temp;
+  }
+  for (int n=0; n<n_ang; ++n) {
+    // from 4 to 5, 5 to 7, 7 to 6, 6 to 4
+    int ang1 = 2 * n_ang + n;
+    int ang2 = 6 * n_ang + n;
+    int ang3 = 7 * n_ang + n;
+    int ang4 = 3 * n_ang + n;
+    Real temp = ir[ang1];
+    ir[ang1] = ir[ang2];
+    ir[ang2] = temp;
+
+    temp = ir[ang4];
+    ir[ang4] = ir[ang3];
+    ir[ang3] = temp;
+  }
+  return;
+}
+
+//--------------------------------------------------------------------------------------
+//! \fn void RotateHPi_InnerX2(MeshBlock *pmb, Coordinates *pco, AthenaArray<Real> &a,
+//                          int is, int ie, int js, int je, int ks, int ke)
+//  \brief ROTATE boundary conditions for x2, inner x2 boundary by Pi/2
+
+// anti-clockwise rotation
+// This function is used after periodic copy
+
+void RadBoundaryVariable::RotateHPi_InnerX2(
+    Real time, Real dt, int il, int iu, int jl, int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *ir = &var(k,jl-j,i,ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity5(ir, n_ang, 1);
+        }
+      }
+    }
+  }
+  return;
+}
+
+//--------------------------------------------------------------------------------------
+//! \fn void RotateHPi_OuterX2(MeshBlock *pmb, Coordinates *pco, AthenaArray<Real> &a,
+//                          int is, int ie, int js, int je, int ks, int ke)
+//  \brief  ROTATE boundary radiation conditions, outer x2 boundary
+
+void RadBoundaryVariable::RotateHPi_OuterX2(
+    Real time, Real dt, int il, int iu, int ju, int kl, int ku, int ngh) {
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang =pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *ir = &var(k,ju+j,i,ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity5(ir, n_ang, -1);
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+//--------------------------------------------------------------------------------------
+//! \fn void RotateHPi_InnerX3((MeshBlock *pmb, Coordinates *pco, AthenaArray<Real> &a,
+//                     int is, int ie, int js, int je, int ks, int ke)
+//  \brief ROTATE boundary conditions for x3, inner x3 boundary by Pi/2
+
+// anti-clockwise rotation
+// This function is used after periodic copy
+void RadBoundaryVariable::RotateHPi_InnerX3(
+    Real time, Real dt, int il, int iu, int jl, int ju, int kl, int ngh) {
+  // copy radiation variables into ghost zones
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq =pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *ir = &var(kl-k,j,i,ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity4(ir, n_ang, 1);
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+//--------------------------------------------------------------------------------------
+//! \fn void RotateHPi_OuterX3((MeshBlock *pmb, Coordinates *pco, AthenaArray<Real> &a,
+//                     int is, int ie, int js, int je, int ks, int ke)
+//  \brief ROTATE boundary conditions for x3, outer x3 boundary by Pi/2
+
+// anti-clockwise rotation
+// This function is used after periodic copy
+void RadBoundaryVariable::RotateHPi_OuterX3(
+    Real time, Real dt, int il, int iu, int jl, int ju, int ku, int ngh) {
+  // copy radiation variables into ghost zones
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *ir = &var(ku+k,j,i,ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity4(ir, n_ang, 1);
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+//--------------------------------------------------------------------------------------
+//! \fn void RotatePi_InnerX3((MeshBlock *pmb, Coordinates *pco, AthenaArray<Real> &a,
+//                     int is, int ie, int js, int je, int ks, int ke)
+//  \brief ROTATE boundary conditions for x3, inner x3 boundary by Pi
+
+// anti-clockwise rotation
+// This function is used after periodic copy
+
+void RadBoundaryVariable::RotatePi_InnerX3(
+    Real time, Real dt, int il, int iu, int jl, int ju, int kl, int ngh) {
+  // copy radiation variables into ghost zones
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *ir = &var(kl-k,j,i,ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity3(ir, n_ang, 1);
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+//--------------------------------------------------------------------------------------
+//! \fn void RotatePi_OuterX3((MeshBlock *pmb, Coordinates *pco, AthenaArray<Real> &a,
+//                     int is, int ie, int js, int je, int ks, int ke)
+//  \brief ROTATE boundary conditions for x3, outer x3 boundary by Pi
+
+// anti-clockwise rotation
+// This function is used after periodic copy
+void RadBoundaryVariable::RotatePi_OuterX3(
+    Real time, Real dt, int il, int iu, int jl, int ju, int ku, int ngh) {
+  // copy radiation variables into ghost zones
+  int &noct = pmy_block_->pnrrad->noct;
+  int n_ang = pmy_block_->pnrrad->nang/noct; // angles per octant
+  int &nfreq = pmy_block_->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          AthenaArray<Real> &var = *var_cc;
+          Real *ir = &var(ku+k,j,i,ifr*pmy_block_->pnrrad->nang);
+          CopyIntensity3(ir, n_ang, 1);
+        }
+      }
+    }
+  }
+  return;
+}
diff --git a/src/bvals/cc/nr_radiation/vacuum_rad.cpp b/src/bvals/cc/nr_radiation/vacuum_rad.cpp
new file mode 100644
index 0000000000..d0bb995676
--- /dev/null
+++ b/src/bvals/cc/nr_radiation/vacuum_rad.cpp
@@ -0,0 +1,240 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file vacuum_rad.cpp
+//  \brief implementation of vacuum BCs in each dimension
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../nr_radiation/radiation.hpp"
+#include "./bvals_rad.hpp"
+
+
+// The angular grid can change
+// The angular octant is
+//   1  |  0       5  |  4
+//   -------      ---------
+//   3  |  2       7  |  6
+// in radiatin class, n_ang is angles per octant, noct is the number of octant
+// radiation relfection means set specific intensitiy corresponding
+// the value in the opposite octant
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::VacuumInnerX1(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief VACUUM boundary conditions, inner x1 boundary
+
+void RadBoundaryVariable::VacuumInnerX1(Real time, Real dt, int il, int jl, int ju,
+                                        int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+  MeshBlock *pmb=pmy_block_;
+  int &nang = pmb->pnrrad->nang; // angles per octant
+  int &nfreq = pmb->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=1; i<=ngh; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for(int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            Real& miux=pmb->pnrrad->mu(0,k,j,il,n);
+            if (miux < 0.0) {
+              (*var_cc)(k,j,il-i,ang) = (*var_cc)(k,j,il,ang);
+            } else {
+              (*var_cc)(k,j,il-i,ang) = 0.0;
+            }
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::VacuumOuterX1(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief VACUUM boundary conditions, outer x1 boundary
+
+void RadBoundaryVariable::VacuumOuterX1(
+    Real time, Real dt, int iu, int jl, int ju, int kl, int ku, int ngh) {
+
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+  MeshBlock *pmb=pmy_block_;
+  int &nang = pmb->pnrrad->nang; // angles per octant
+  int &nfreq = pmb->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=1; i<=ngh; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for(int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            Real& miux=pmb->pnrrad->mu(0,k,j,iu,n);
+            if (miux > 0.0) {
+              (*var_cc)(k,j,iu+i,ang) = (*var_cc)(k,j,iu,ang);
+            } else {
+              (*var_cc)(k,j,iu+i,ang) = 0.0;
+            }
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::VacuumInnerX2(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief REFLECTING boundary conditions, inner x2 boundary
+
+void RadBoundaryVariable::VacuumInnerX2(
+    Real time, Real dt, int il, int iu, int jl, int kl, int ku, int ngh) {
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+  MeshBlock *pmb=pmy_block_;
+  int &nang = pmb->pnrrad->nang; // angles per octant
+  int &nfreq = pmb->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for(int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            Real& miuy=pmb->pnrrad->mu(1,k,jl,i,n);
+            if (miuy < 0.0) {
+              (*var_cc)(k,jl-j,i,ang) = (*var_cc)(k,jl,i,ang);
+            } else {
+              (*var_cc)(k,jl-j,i,ang) = 0.0;
+            }
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::VacuumOuterX2(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief VACUUM boundary conditions, outer x2 boundary
+
+void RadBoundaryVariable::VacuumOuterX2(
+    Real time, Real dt, int il, int iu, int ju,  int kl, int ku, int ngh) {
+
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+  MeshBlock *pmb=pmy_block_;
+  int &nang = pmb->pnrrad->nang; // angles per octant
+  int &nfreq = pmb->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for(int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            Real& miuy=pmb->pnrrad->mu(1,k,ju,i,n);
+            if (miuy > 0.0) {
+              (*var_cc)(k,ju+j,i,ang) = (*var_cc)(k,ju,i,ang);
+            } else {
+              (*var_cc)(k,ju+j,i,ang) = 0.0;
+            }
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+
+
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::VacuumInnerX2(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief VACUUM boundary conditions, inner x2 boundary
+
+void RadBoundaryVariable::VacuumInnerX3(
+    Real time, Real dt, int il, int iu, int jl,  int ju, int kl, int ngh) {
+
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+  MeshBlock *pmb=pmy_block_;
+  int &nang = pmb->pnrrad->nang; // angles per octant
+  int &nfreq = pmb->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for(int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            Real& miuz=pmb->pnrrad->mu(2,kl,j,i,n);
+            if (miuz < 0.0) {
+              (*var_cc)(kl-k,j,i,ang) = (*var_cc)(kl,j,i,ang);
+            } else {
+              (*var_cc)(kl-k,j,i,ang) = 0.0;
+            }
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void RadBoundaryVariable::VacuumOuterX3(
+//          Real time, Real dt, int il, int jl, int ju, int kl, int ku, int ngh)
+//  \brief VACUUM boundary conditions, outer X3 boundary
+
+void RadBoundaryVariable::VacuumOuterX3(
+    Real time, Real dt, int il, int iu, int jl,  int ju, int ku, int ngh) {
+
+  // copy radiation variables into ghost zones,
+  // reflect rays along angles with opposite nx
+  MeshBlock *pmb=pmy_block_;
+  int &nang = pmb->pnrrad->nang; // angles per octant
+  int &nfreq = pmb->pnrrad->nfreq; // number of frequency bands
+
+  for (int k=1; k<=ngh; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      for (int i=il; i<=iu; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for(int n=0; n<nang; ++n) {
+            int ang=ifr*nang+n;
+            Real& miuz=pmb->pnrrad->mu(2,ku,j,i,n);
+            if (miuz > 0.0) {
+              (*var_cc)(ku+k,j,i,ang) = (*var_cc)(ku,j,i,ang);
+            } else {
+              (*var_cc)(ku+k,j,i,ang) = 0.0;
+            }
+          }
+        }
+      }
+    }
+  }
+  return;
+}
diff --git a/src/bvals/fc/bvals_fc.hpp b/src/bvals/fc/bvals_fc.hpp
index 5811800cc2..0cb7ba6ffe 100644
--- a/src/bvals/fc/bvals_fc.hpp
+++ b/src/bvals/fc/bvals_fc.hpp
@@ -116,6 +116,21 @@ class FaceCenteredBoundaryVariable : public BoundaryVariable {
   void OutflowOuterX3(Real time, Real dt,
                       int il, int iu, int jl, int ju, int ku, int ngh) override;
 
+
+  void VacuumInnerX1(Real time, Real dt,
+                     int il, int jl, int ju, int kl, int ku, int ngh) override {return;};
+  void VacuumOuterX1(Real time, Real dt,
+                     int iu, int jl, int ju, int kl, int ku, int ngh) override {return;};
+  void VacuumInnerX2(Real time, Real dt,
+                     int il, int iu, int jl, int kl, int ku, int ngh) override {return;};
+  void VacuumOuterX2(Real time, Real dt,
+                     int il, int iu, int ju, int kl, int ku, int ngh) override {return;};
+  void VacuumInnerX3(Real time, Real dt,
+                     int il, int iu, int jl, int ju, int kl, int ngh) override {return;};
+  void VacuumOuterX3(Real time, Real dt,
+                     int il, int iu, int jl, int ju, int ku, int ngh) override {return;};
+
+
   void PolarWedgeInnerX2(Real time, Real dt,
                          int il, int iu, int jl, int kl, int ku, int ngh) override;
   void PolarWedgeOuterX2(Real time, Real dt,
diff --git a/src/coordinates/coordinates.cpp b/src/coordinates/coordinates.cpp
index 9baa4f78b2..c11dd078a0 100644
--- a/src/coordinates/coordinates.cpp
+++ b/src/coordinates/coordinates.cpp
@@ -17,6 +17,7 @@
 #include "../bvals/bvals.hpp"
 #include "../hydro/hydro.hpp"
 #include "../mesh/mesh.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../parameter_input.hpp"
 #include "coordinates.hpp"
 
@@ -123,7 +124,6 @@ Coordinates::Coordinates(MeshBlock *pmb, ParameterInput *pin, bool flag) :
                   << std::endl;
       }
     }
-
   } else {
     // uniform grid: use UniformMeshGeneratorX1()
     Real dx = (block_size.x1max - block_size.x1min)/(iu-il+1);
@@ -579,13 +579,13 @@ void Coordinates::LaplacianX1(const AthenaArray<Real> &s, AthenaArray<Real> &del
                               const int il, const int iu) {
   if (pmy_block->block_size.nx3 > 1) {
 #pragma omp simd
-    for(int i=il; i<=iu; ++i) {
+    for (int i=il; i<=iu; ++i) {
       delta_s(i) = (s(n,k,j-1,i) - 2.0*s(n,k,j,i) + s(n,k,j+1,i)) / (dx2f(j)*dx2f(j))
                    + (s(n,k-1,j,i) - 2.0*s(n,k,j,i) + s(n,k+1,j,i)) / (dx3f(k)*dx3f(k));
     }
   } else if (pmy_block->block_size.nx2 > 1) {
 #pragma omp simd
-    for(int i=il; i<=iu; ++i) {
+    for (int i=il; i<=iu; ++i) {
       delta_s(i) = (s(n,k,j-1,i) - 2.0*s(n,k,j,i) + s(n,k,j+1,i)) / (dx2f(j)*dx2f(j));
     }
   } else {
@@ -635,13 +635,13 @@ void Coordinates::LaplacianX2(const AthenaArray<Real> &s, AthenaArray<Real> &del
                               const int il, const int iu) {
   if (pmy_block->block_size.nx3 > 1) {
 #pragma omp simd
-    for(int i=il; i<=iu; ++i) {
+    for (int i=il; i<=iu; ++i) {
       delta_s(i) = (s(n,k,j,i-1) - 2.0*s(n,k,j,i) + s(n,k,j,i+1)) / (dx1f(i)*dx1f(i))
                    + (s(n,k-1,j,i) - 2.0*s(n,k,j,i) + s(n,k+1,j,i)) / (dx3f(k)*dx3f(k));
     }
   } else {
 #pragma omp simd
-    for(int i=il; i<=iu; ++i) {
+    for (int i=il; i<=iu; ++i) {
       delta_s(i) = (s(n,k,j,i-1) - 2.0*s(n,k,j,i) + s(n,k,j,i+1)) / (dx1f(i)*dx1f(i));
     }
   }
@@ -685,13 +685,13 @@ void Coordinates::LaplacianX3(const AthenaArray<Real> &s, AthenaArray<Real> &del
                               const int il, const int iu) {
   if (pmy_block->block_size.nx2 > 1) {
 #pragma omp simd
-    for(int i=il; i<=iu; ++i) {
+    for (int i=il; i<=iu; ++i) {
       delta_s(i) = (s(n,k,j,i-1) - 2.0*s(n,k,j,i) + s(n,k,j,i+1)) / (dx1f(i)*dx1f(i))
                    + (s(n,k,j-1,i) - 2.0*s(n,k,j,i) + s(n,k,j+1,i)) / (dx2f(j)*dx2f(j));
     }
   } else {
 #pragma omp simd
-    for(int i=il; i<=iu; ++i) {
+    for (int i=il; i<=iu; ++i) {
       delta_s(i) = (s(n,k,j,i-1) - 2.0*s(n,k,j,i) + s(n,k,j,i+1)) / (dx1f(i)*dx1f(i));
     }
   }
@@ -739,6 +739,108 @@ void Coordinates::AddCoordTermsDivergence(
   return;
 }
 
+void Coordinates::AddCRCoordTermsDivergence(
+          const AthenaArray<Real> &u_cr1, AthenaArray<Real> &u_cr) {
+  return;
+}
+
+void Coordinates::CRGradPcCoordTermsDivergence(const AthenaArray<Real> &u_cr,
+                                          AthenaArray<Real> &grad_pc) {
+  return;
+}
+
+void Coordinates::ConvertAngle(MeshBlock *pmb, const int nang, AthenaArray<Real> &mu) {
+  return;
+}
+
+void Coordinates::AxisDirection(int *axisx, int *axisy, int *axisz) {
+  return;
+}
+
+void Coordinates::ZetaArea(NRRadiation *prad, AthenaArray<Real> &area) {
+  if (prad->angle_flag == 1) {
+    int nzeta = prad->nzeta;
+    int npsi = prad->npsi;
+    if (npsi * nzeta > 0) {
+      for (int m=0; m<2*npsi; ++m) {
+        for (int n=0; n<2*nzeta+1; ++n) {
+          Real sinzeta_sq= 1.0 - prad->coszeta_f(n) * prad->coszeta_f(n);
+          area(m,n) = sinzeta_sq * prad->len_psi(m);
+        }// end psi
+      }// end zeta
+    } else if (nzeta > 0) {
+      for (int n=0; n<2*nzeta+1; ++n) {
+        area(n) = 1.0 - prad->coszeta_f(n) * prad->coszeta_f(n);
+      }
+    }
+  }
+}
+
+void Coordinates::PsiArea(NRRadiation *prad, AthenaArray<Real> &area) {
+  if (prad->angle_flag == 1) {
+    int nzeta = prad->nzeta;
+    int npsi = prad->npsi;
+    if (npsi * nzeta > 0) {
+      for (int n=0; n<2*nzeta; ++n) {
+        for (int m=0; m<2*npsi+1; ++m) {
+          area(n,m) = prad->len_zeta(n);
+        }
+      }
+    } else if (npsi > 0) {
+      for (int m=0; m<2*npsi+1; ++m) {
+        area(m) = 1.0;
+      }
+    }
+  }
+}
+
+void Coordinates::AngularVol(NRRadiation *prad, AthenaArray<Real> &vol) {
+  if (prad->angle_flag == 1) {
+    int nzeta = prad->nzeta;
+    int npsi = prad->npsi;
+    if (npsi * nzeta > 0) {
+      for (int n=0; n<2*nzeta; ++n) {
+        for (int m=0; m<2*npsi; ++m) {
+          int ang_num = n*(2*npsi)+m;
+          vol(ang_num) = prad->len_zeta(n) * prad->len_psi(m);
+        }
+      }
+    } else if (npsi > 0) {
+      for (int m=0; m<2*npsi; ++m)
+        vol(m) = prad->len_psi(m);
+    } else if (nzeta > 0) {
+      for (int n=0; n<2*nzeta; ++n)
+        vol(n) = prad->len_zeta(n);
+    }
+  }
+}
+
+void Coordinates::GetGeometryZeta(NRRadiation *prad, const int k, const int j,
+                                  const int i, AthenaArray<Real> &g_zeta) {
+  int &nzeta = prad->nzeta;
+  for (int n=0; n<nzeta*2+1; ++n) {
+    g_zeta(n) = 1.0;
+  }
+}
+
+void Coordinates::GetGeometryPsi(NRRadiation *prad, const int k, const int j,
+                                 const int i, const int n_zeta,
+                                 AthenaArray<Real> &g_psi) {
+  int &npsi = prad->npsi;
+  for (int n=0; n<2*npsi+1; ++n) {
+    g_psi(n) = 1.0;
+  }
+}
+
+// function overwirte in case nzeta = 0
+void Coordinates::GetGeometryPsi(NRRadiation *prad, const int k, const int j,
+                                 const int i, AthenaArray<Real> &g_psi) {
+  int &npsi = prad->npsi;
+  for (int n=0; n<2*npsi+1; ++n) {
+    g_psi(n) = 1.0;
+  }
+}
+
 //----------------------------------------------------------------------------------------
 // Coordinate (Geometric) source term function for STS
 void Coordinates::AddCoordTermsDivergence_STS(
diff --git a/src/coordinates/coordinates.hpp b/src/coordinates/coordinates.hpp
index 91ee66283c..d431882f0b 100644
--- a/src/coordinates/coordinates.hpp
+++ b/src/coordinates/coordinates.hpp
@@ -126,10 +126,36 @@ class Coordinates {
   virtual void AddCoordTermsDivergence(const Real dt, const AthenaArray<Real> *flux,
                              const AthenaArray<Real> &prim, const AthenaArray<Real> &bcc,
                              AthenaArray<Real> &u);
+
+  // coordinate source term for cosmic rays
+  virtual void AddCRCoordTermsDivergence(const AthenaArray<Real> &u_cr,
+                                       AthenaArray<Real> &coord_src);
+
+  // Coordinate (Geometric) source term for Grad Pc
+  virtual void CRGradPcCoordTermsDivergence(const AthenaArray<Real> &u_cr,
+                                       AthenaArray<Real> &grad_pc);
+
   virtual void AddCoordTermsDivergence_STS(const Real dt, int stage,
                              const AthenaArray<Real> *flux,
                              AthenaArray<Real> &u, AthenaArray<Real> &flux_div);
 
+
+    // ...define the direction of radiation angle with respect to the local axis
+  virtual void AxisDirection(int *axisx, int *axisy, int *axisz);
+  virtual void ConvertAngle(MeshBlock *pmb, const int nang, AthenaArray<Real> &mu);
+
+// Functions for angular flux
+  virtual void ZetaArea(NRRadiation *prad, AthenaArray<Real> &area);
+  virtual void PsiArea(NRRadiation *prad, AthenaArray<Real> &area);
+  virtual void AngularVol(NRRadiation *prad, AthenaArray<Real> &vol);
+  virtual void GetGeometryZeta(NRRadiation *prad, const int k, const int j,
+                               const int i, AthenaArray<Real> &g_zeta);
+  virtual void GetGeometryPsi(NRRadiation *prad, const int k, const int j,
+                              const int i, const int n_zeta, AthenaArray<Real> &g_psi);
+  // function overwirte in case nzeta = 0
+  virtual void GetGeometryPsi(NRRadiation *prad, const int k, const int j,
+                              const int i, AthenaArray<Real> &g_psi);
+
   // ...to determine if index is a pole
   bool IsPole(int j);
 
@@ -337,9 +363,23 @@ class Cylindrical : public Coordinates {
   void AddCoordTermsDivergence(const Real dt, const AthenaArray<Real> *flux,
                      const AthenaArray<Real> &prim, const AthenaArray<Real> &bcc,
                      AthenaArray<Real> &u) final;
+
+  // coordinate source term for cosmic rays
+  void AddCRCoordTermsDivergence(const AthenaArray<Real> &u_cr,
+                               AthenaArray<Real> &coord_src) final;
+
+// Coordinate (Geometric) source term for Grad Pc
+  void CRGradPcCoordTermsDivergence(const AthenaArray<Real> &u_cr,
+                               AthenaArray<Real> &grad_pc) final;
+
   void AddCoordTermsDivergence_STS(const Real dt, int stage,
-                     const AthenaArray<Real> *flux,
-                     AthenaArray<Real> &u, AthenaArray<Real> &flux_div) final;
+                                   const AthenaArray<Real> *flux,
+                                   AthenaArray<Real> &u,
+                                   AthenaArray<Real> &flux_div) final;
+
+  // for radiation functions...
+  void AxisDirection(int *axisx, int *axisy, int *axisz) final;
+  void ConvertAngle(MeshBlock *pmb, const int nang, AthenaArray<Real> &mu) final;
 };
 
 //----------------------------------------------------------------------------------------
@@ -396,11 +436,33 @@ class SphericalPolar : public Coordinates {
 
   // ...to compute geometrical source terms
   void AddCoordTermsDivergence(const Real dt, const AthenaArray<Real> *flux,
-                     const AthenaArray<Real> &prim, const AthenaArray<Real> &bcc,
-                     AthenaArray<Real> &u) final;
+                               const AthenaArray<Real> &prim,
+                               const AthenaArray<Real> &bcc, AthenaArray<Real> &u) final;
+
+  // coordinate source term for cosmic rays
+  void AddCRCoordTermsDivergence(const AthenaArray<Real> &u_cr,
+                               AthenaArray<Real> &coord_src) final;
+
+// Coordinate (Geometric) source term for Grad Pc
+  void CRGradPcCoordTermsDivergence(const AthenaArray<Real> &u_cr,
+                               AthenaArray<Real> &grad_pc) final;
+
+  //for radiation functions...
+  void AxisDirection(int *axisx, int *axisy, int *axisz) final;
+  void ConvertAngle(MeshBlock *pmb, const int nang, AthenaArray<Real> &mu) final;
+
+  void GetGeometryZeta(NRRadiation *prad, const int k, const int j,
+                       const int i, AthenaArray<Real> &g_zeta) final;
+  void GetGeometryPsi(NRRadiation *prad, const int k, const int j,
+                      const int i, const int n_zeta, AthenaArray<Real> &g_psi) final;
+
+  void GetGeometryPsi(NRRadiation *prad, const int k, const int j,
+                      const int i, AthenaArray<Real> &g_psi) final;
+
+
   void AddCoordTermsDivergence_STS(const Real dt, int stage,
-                     const AthenaArray<Real> *flux,
-                     AthenaArray<Real> &u, AthenaArray<Real> &flux_div) final;
+                                   const AthenaArray<Real> *flux, AthenaArray<Real> &u,
+                                   AthenaArray<Real> &flux_div) final;
 };
 
 //----------------------------------------------------------------------------------------
@@ -512,6 +574,7 @@ class Schwarzschild : public Coordinates {
                      const AthenaArray<Real> &prim, const AthenaArray<Real> &bcc,
                      AthenaArray<Real> &u) final;
 
+
   // In GR, functions...
   // ...to compute metric
   void CellMetric(const int k, const int j, const int il, const int iu,
diff --git a/src/coordinates/cylindrical.cpp b/src/coordinates/cylindrical.cpp
index 30ef257516..d583a17bcc 100644
--- a/src/coordinates/cylindrical.cpp
+++ b/src/coordinates/cylindrical.cpp
@@ -15,6 +15,7 @@
 // Athena++ headers
 #include "../athena.hpp"
 #include "../athena_arrays.hpp"
+#include "../cr/cr.hpp"
 #include "../eos/eos.hpp"
 #include "../hydro/hydro.hpp"
 #include "../hydro/hydro_diffusion/hydro_diffusion.hpp"
@@ -322,7 +323,6 @@ void Cylindrical::AddCoordTermsDivergence(
       }
     }
   }
-
   return;
 }
 
@@ -364,3 +364,90 @@ void Cylindrical::AddCoordTermsDivergence_STS(
   }
   return;
 }
+
+//----------------------------------------------------------------------------------------
+// Coordinate (Geometric) source term function for cosmic rays
+
+void Cylindrical::AddCRCoordTermsDivergence(
+  const AthenaArray<Real> &u_input, AthenaArray<Real> &coord_src) {
+  if (CR_ENABLED) {
+    CosmicRay *pcr=pmy_block->pcr;
+    for (int k=pmy_block->ks; k<=pmy_block->ke; ++k) {
+      for (int j=pmy_block->js; j<=pmy_block->je; ++j) {
+#pragma omp simd
+        for (int i=pmy_block->is; i<=pmy_block->ie; ++i) {
+          // src_1 = <M_{phi phi}><1/r>
+          Real m_pp =  u_input(CRE,k,j,i)/3.0;
+          coord_src(CRF1,k,j,i) =  pcr->vmax * coord_src1_i_(i)*m_pp;
+          // 0 for other components
+          coord_src(CRE,k,j,i) = 0.0;
+          coord_src(CRF2,k,j,i) = 0.0;
+          coord_src(CRF3,k,j,i) = 0.0;
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+//----------------------------------------------------------------------------------------
+// Coordinate (Geometric) source term for Grad Pc
+
+void Cylindrical::CRGradPcCoordTermsDivergence(
+    const AthenaArray<Real> &u_cr, AthenaArray<Real> &grad_pc) {
+  // Go through cellscosmicay
+  if (CR_ENABLED) {
+    //CosmicRay *pcr=pmy_block->pcr;
+    for (int k=pmy_block->ks; k<=pmy_block->ke; ++k) {
+      for (int j=pmy_block->js; j<=pmy_block->je; ++j) {
+        for (int i=pmy_block->is; i<=pmy_block->ie; ++i) {
+          // src_1 = <M_{phi phi}><1/r>
+          Real m_ii = u_cr(CRE,k,j,i)/3.0;
+          grad_pc(0,k,j,i) -= coord_src1_i_(i)*m_ii;
+        }
+      }// end j
+    }// end k
+  }// ENd CR_ENABLED
+  return;
+}
+
+//--------------------------------------------------------------------------
+// For radiation angles
+void Cylindrical::AxisDirection(int *axisx, int *axisy, int *axisz) {
+  *axisx = 0;
+  *axisy = 1;
+  *axisz = 2;
+}
+
+void Cylindrical::ConvertAngle(MeshBlock *pmb, const int nang,
+                               AthenaArray<Real> &mu) {
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    int n1z = pmb->ncells1, n2z = pmb->ncells2, n3z = pmb->ncells3;
+    // int ndim=1;
+    // if (n2z > 1) ndim = 2;
+    // if (n3z > 1) ndim = 3;
+
+    for (int k=0; k<n3z; ++k) {
+      for (int j=0; j<n2z; ++j) {
+        Real& x2 = x2v(j);
+        Real cosx2 = cos(x2);
+        Real sinx2 = sin(x2);
+        if (n2z == 1) {
+          cosx2 = 1.0;
+          sinx2 = 0.0;
+        }
+        for (int i=0; i<n1z; ++i) {
+          Real *miux = &(mu(0,k,j,i,0));
+          Real *miuy = &(mu(1,k,j,i,0));
+          for (int mi=0; mi<nang; ++mi) {
+            Real miux0 = miux[mi];
+            Real miuy0 = miuy[mi];
+            miux[mi] = miux0 * cosx2 + miuy0 * sinx2;
+            miuy[mi] = miuy0 * cosx2 - miux0 * sinx2;
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/src/coordinates/spherical_polar.cpp b/src/coordinates/spherical_polar.cpp
index fcd530e501..68e50751dc 100644
--- a/src/coordinates/spherical_polar.cpp
+++ b/src/coordinates/spherical_polar.cpp
@@ -19,10 +19,12 @@
 // Athena++ headers
 #include "../athena.hpp"
 #include "../athena_arrays.hpp"
+#include "../cr/cr.hpp"
 #include "../eos/eos.hpp"
 #include "../hydro/hydro.hpp"
 #include "../hydro/hydro_diffusion/hydro_diffusion.hpp"
 #include "../mesh/mesh.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../parameter_input.hpp"
 #include "coordinates.hpp"
 
@@ -529,7 +531,6 @@ void SphericalPolar::AddCoordTermsDivergence(const Real dt, const AthenaArray<Re
 void SphericalPolar::AddCoordTermsDivergence_STS(const Real dt, int stage,
                                    const AthenaArray<Real> *flux,
                                    AthenaArray<Real> &u, AthenaArray<Real> &flux_div) {
-  Real iso_cs = pmy_block->peos->GetIsoSoundSpeed();
   bool use_x2_fluxes = pmy_block->block_size.nx2 > 1;
 
   HydroDiffusion &hd = pmy_block->phydro->hdif;
@@ -612,3 +613,152 @@ void SphericalPolar::AddCoordTermsDivergence_STS(const Real dt, int stage,
   }
   return;
 }
+
+//----------------------------------------------------------------------------------------
+// Coordinate (Geometric) source term function for cosmic rays
+
+void SphericalPolar::AddCRCoordTermsDivergence(
+  const AthenaArray<Real> &u_input, AthenaArray<Real> &coord_src) {
+  // Go through cellscosmicay
+  if (CR_ENABLED) {
+    CosmicRay *pcr=pmy_block->pcr;
+    for (int k=pmy_block->ks; k<=pmy_block->ke; ++k) {
+      for (int j=pmy_block->js; j<=pmy_block->je; ++j) {
+        for (int i=pmy_block->is; i<=pmy_block->ie; ++i) {
+          // src_1 = < M_{theta theta} + M_{phi phi} ><1/r>
+          Real m_ii = (2.0/3.0) * u_input(CRE,k,j,i);
+          coord_src(CRF1,k,j,i) = pcr->vmax * coord_src1_i_(i)*m_ii;
+          // src_2 = < M_{phi phi} ><cot theta/r>
+          Real m_pp=(1.0/3.0) * u_input(CRE,k,j,i);
+          coord_src(CRF2,k,j,i) = pcr->vmax * coord_src1_i_(i)*coord_src1_j_(j)*m_pp;
+          // set 0 for other components
+          coord_src(CRE,k,j,i) = 0.0;
+          coord_src(CRF3,k,j,i) = 0.0;
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+// Coordinate (Geometric) source term for Grad Pc
+
+void SphericalPolar::CRGradPcCoordTermsDivergence(
+  const AthenaArray<Real> &u_cr, AthenaArray<Real> &grad_pc) {
+  // Go through cellscosmicay
+  if (CR_ENABLED) {
+    for (int k=pmy_block->ks; k<=pmy_block->ke; ++k) {
+      for (int j=pmy_block->js; j<=pmy_block->je; ++j) {
+        for (int i=pmy_block->is; i<=pmy_block->ie; ++i) {
+          // src_1 = < M_{theta theta} + M_{phi phi} ><1/r>
+          Real m_ii = (2.0/3.0) * u_cr(CRE,k,j,i);
+          grad_pc(0,k,j,i) -= coord_src1_i_(i)*m_ii;
+        }
+      }
+    }
+  }
+  return;
+}
+
+//----------------------------------------------------------------------------------------
+// For radiation angles
+void SphericalPolar::AxisDirection(int *axisx, int *axisy, int *axisz) {
+  *axisx = 1;
+  *axisy = 2;
+  *axisz = 0;
+  return;
+}
+
+
+
+void SphericalPolar::ConvertAngle(MeshBlock *pmb, const int nang,
+                                  AthenaArray<Real> &mu) {
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    int n1z = pmb->ncells1, n2z = pmb->ncells2, n3z = pmb->ncells3;
+
+    for(int k=0; k<n3z; ++k) {
+      Real x3 = x3v(k);
+      Real cosx3 = cos(x3);
+      Real sinx3 = sin(x3);
+      if (n3z == 1) {
+        cosx3 = 1.0;
+        sinx3 = 0.0;
+      }
+      for(int j=0; j<n2z; ++j) {
+        Real x2 = x2v(j);
+        Real cosx2 = cos(x2);
+        Real sinx2 = sin(x2);
+        if (n2z == 1) {
+          cosx2 = 1.0;
+          sinx2 = 0.0;
+        }
+        for(int i=0; i<n1z; ++i) {
+          Real *miur = &(mu(0,k,j,i,0));
+          Real *miutheta = &(mu(1,k,j,i,0));
+          Real *miuphi = &(mu(2,k,j,i,0));
+          // now rotate angles
+          for(int mi=0; mi<nang; ++mi) {
+            Real miuz0 = miur[mi];
+            Real miux0 = miutheta[mi];
+            Real miuy0 = miuphi[mi];
+            miur[mi]     = sinx2*cosx3*miux0 + sinx2*sinx3*miuy0 + cosx2*miuz0;
+            miutheta[mi] = cosx2*cosx3*miux0 + cosx2*sinx3*miuy0 - sinx2*miuz0;
+            miuphi[mi]   = -sinx3*miux0      + cosx3*miuy0;
+          }
+        }
+      }
+    }
+  }
+}
+
+
+// get the geometry factor for zeta flux
+// this needs to go throug all the nzeta
+void SphericalPolar::GetGeometryZeta(NRRadiation *prad, const int k, const int j,
+                                     const int i, AthenaArray<Real> &g_zeta) {
+  int &nzeta = prad->nzeta;
+  Real radius = x1v(i);
+  for(int n=0; n<nzeta*2+1; ++n) {
+    g_zeta(n) = 1./radius;
+  }
+}
+
+// get the geometry factor for psi flux
+// this needs to go throug all the nzeta
+void SphericalPolar::GetGeometryPsi(NRRadiation *prad, const int k, const int j,
+                                    const int i, const int n_zeta,
+                                    AthenaArray<Real> &g_psi) {
+  int &npsi = prad->npsi;
+  Real radius = x1v(i);
+  Real sinzeta_v = 1.0 - prad->coszeta_v(n_zeta) * prad->coszeta_v(n_zeta);
+  sinzeta_v = std::sqrt(sinzeta_v);
+  Real &cottheta = prad->cot_theta(j);
+  if (npsi == 1) {
+    for(int n=0; n<2*npsi+1; ++n) {
+      g_psi(n) = 0.0;
+    }
+  } else {
+    for(int n=0; n<2*npsi+1; ++n) {
+      g_psi(n) = sinzeta_v * cottheta * prad->sin_psi_f(n)/radius;
+    }
+  }
+}
+
+
+void SphericalPolar::GetGeometryPsi(NRRadiation *prad, const int k, const int j,
+                        const int i, AthenaArray<Real> &g_psi) {
+  int &npsi = prad->npsi;
+  Real radius = x1v(i);
+
+  Real &cottheta = prad->cot_theta(j);
+  if (npsi == 1) {
+    for(int n=0; n<2*npsi+1; ++n) {
+      g_psi(n) = 0.0;
+    }
+  } else {
+    for(int n=0; n<2*npsi+1; ++n) {
+      g_psi(n) = cottheta * prad->sin_psi_f(n)/radius;
+    }
+  }
+}
diff --git a/src/cr/cr.cpp b/src/cr/cr.cpp
new file mode 100644
index 0000000000..cd71b650cc
--- /dev/null
+++ b/src/cr/cr.cpp
@@ -0,0 +1,280 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file cr.cpp
+//  \brief implementation of functions in cosmic ray class
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <cstdio>  // fopen and fwrite
+#include <iostream>  // cout
+#include <sstream>  // msg
+#include <stdexcept> // runtime erro
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
+#include "../coordinates/coordinates.hpp"
+#include "../globals.hpp"
+#include "../mesh/mesh.hpp"
+#include "../parameter_input.hpp"
+#include "cr.hpp"
+#include "integrators/cr_integrators.hpp"
+
+// constructor, initializes data structures and parameters
+
+// The default opacity function.
+
+// This function also needs to set the streaming velocity
+// This is needed to calculate the work term
+inline void DefaultOpacity(MeshBlock *pmb, AthenaArray<Real> &u_cr,
+              AthenaArray<Real> &prim, AthenaArray<Real> &bcc) {
+  // set the default opacity to be a large value in the default hydro case
+  CosmicRay *pcr=pmb->pcr;
+  int kl=pmb->ks, ku=pmb->ke;
+  int jl=pmb->js, ju=pmb->je;
+  int il=pmb->is-1, iu=pmb->ie+1;
+  if (pmb->block_size.nx2 > 1) {
+    jl -= 1;
+    ju += 1;
+  }
+  if (pmb->block_size.nx3 > 1) {
+    kl -= 1;
+    ku += 1;
+  }
+  Real invlim = 1.0/pcr->vmax;
+  for(int k=kl; k<=ku; ++k) {
+    for(int j=jl; j<=ju; ++j) {
+#pragma omp simd
+      for(int i=il; i<=iu; ++i) {
+        pcr->sigma_diff(0,k,j,i) = pcr->max_opacity;
+        pcr->sigma_diff(1,k,j,i) = pcr->max_opacity;
+        pcr->sigma_diff(2,k,j,i) = pcr->max_opacity;
+      }
+    }
+  }
+
+  // Need to calculate the rotation matrix
+  // We need this to determine the direction of rotation velocity
+  if (MAGNETIC_FIELDS_ENABLED && (pcr->stream_flag > 0)) {
+    //First, calculate B_dot_grad_Pc
+    for(int k=kl; k<=ku; ++k) {
+      for(int j=jl; j<=ju; ++j) {
+      // diffusion coefficient is calculated with respect to B direction
+      // Use a simple estimate of Grad Pc
+
+        // x component
+        pmb->pcoord->CenterWidth1(k,j,il-1,iu+1,pcr->cwidth);
+        for(int i=il; i<=iu; ++i) {
+          Real distance = 0.5*(pcr->cwidth(i-1) + pcr->cwidth(i+1))
+                         + pcr->cwidth(i);
+          Real dprdx=(u_cr(CRE,k,j,i+1) - u_cr(CRE,k,j,i-1))/3.0;
+          dprdx /= distance;
+          pcr->sigma_adv(0,k,j,i) = dprdx;
+        }
+        // y component
+        pmb->pcoord->CenterWidth2(k,j-1,il,iu,pcr->cwidth1);
+        pmb->pcoord->CenterWidth2(k,j,il,iu,pcr->cwidth);
+        pmb->pcoord->CenterWidth2(k,j+1,il,iu,pcr->cwidth2);
+
+        for(int i=il; i<=iu; ++i) {
+          Real distance = 0.5*(pcr->cwidth1(i) + pcr->cwidth2(i))
+                         + pcr->cwidth(i);
+          Real dprdy=(u_cr(CRE,k,j+1,i) - u_cr(CRE,k,j-1,i))/3.0;
+          dprdy /= distance;
+          pcr->sigma_adv(1,k,j,i) = dprdy;
+        }
+        // z component
+        pmb->pcoord->CenterWidth3(k-1,j,il,iu,pcr->cwidth1);
+        pmb->pcoord->CenterWidth3(k,j,il,iu,pcr->cwidth);
+        pmb->pcoord->CenterWidth3(k+1,j,il,iu,pcr->cwidth2);
+
+        for(int i=il; i<=iu; ++i) {
+          Real distance = 0.5*(pcr->cwidth1(i) + pcr->cwidth2(i))
+                          + pcr->cwidth(i);
+          Real dprdz=(u_cr(CRE,k+1,j,i) -  u_cr(CRE,k-1,j,i))/3.0;
+          dprdz /= distance;
+          pcr->sigma_adv(2,k,j,i) = dprdz;
+        }
+
+        for(int i=il; i<=iu; ++i) {
+          // Now calculate the angles of B
+          Real bxby = std::sqrt(bcc(IB1,k,j,i)*bcc(IB1,k,j,i) +
+                           bcc(IB2,k,j,i)*bcc(IB2,k,j,i));
+          Real btot = std::sqrt(bcc(IB1,k,j,i)*bcc(IB1,k,j,i) +
+                           bcc(IB2,k,j,i)*bcc(IB2,k,j,i) +
+                           bcc(IB3,k,j,i)*bcc(IB3,k,j,i));
+
+          if (btot > TINY_NUMBER) {
+            pcr->b_angle(0,k,j,i) = bxby/btot;
+            pcr->b_angle(1,k,j,i) = bcc(IB3,k,j,i)/btot;
+          } else {
+            pcr->b_angle(0,k,j,i) = 1.0;
+            pcr->b_angle(1,k,j,i) = 0.0;
+          }
+          if (bxby > TINY_NUMBER) {
+            pcr->b_angle(2,k,j,i) = bcc(IB2,k,j,i)/bxby;
+            pcr->b_angle(3,k,j,i) = bcc(IB1,k,j,i)/bxby;
+          } else {
+            pcr->b_angle(2,k,j,i) = 0.0;
+            pcr->b_angle(3,k,j,i) = 1.0;
+          }
+
+          Real va = std::sqrt(btot*btot/prim(IDN,k,j,i));
+          if (va < TINY_NUMBER) {
+            pcr->sigma_adv(0,k,j,i) = pcr->max_opacity;
+          } else {
+            Real b_grad_pc = bcc(IB1,k,j,i) * pcr->sigma_adv(0,k,j,i)
+                           + bcc(IB2,k,j,i) * pcr->sigma_adv(1,k,j,i)
+                           + bcc(IB3,k,j,i) * pcr->sigma_adv(2,k,j,i);
+            pcr->sigma_adv(0,k,j,i) = std::abs(b_grad_pc)/(btot * va * (1.0 + 1.0/3.0)
+                                               * invlim * u_cr(CRE,k,j,i));
+          }
+          pcr->sigma_adv(1,k,j,i) = pcr->max_opacity;
+          pcr->sigma_adv(2,k,j,i) = pcr->max_opacity;
+        }
+      }
+    }
+  } else {
+    for(int k=kl; k<=ku; ++k) {
+      for(int j=jl; j<=ju; ++j) {
+#pragma omp simd
+        for(int i=il; i<=iu; ++i) {
+          pcr->sigma_adv(0,k,j,i) = pcr->max_opacity;
+          pcr->sigma_adv(1,k,j,i) = pcr->max_opacity;
+          pcr->sigma_adv(2,k,j,i) = pcr->max_opacity;
+
+          pcr->v_adv(0,k,j,i) = 0.0;
+          pcr->v_adv(1,k,j,i) = 0.0;
+          pcr->v_adv(2,k,j,i) = 0.0;
+        }
+      }
+    }
+  }
+}
+
+inline void DefaultStreaming(MeshBlock *pmb, AthenaArray<Real> &u_cr,
+             AthenaArray<Real> &prim, AthenaArray<Real> &bcc,
+             AthenaArray<Real> &grad_pc, int k, int j, int is, int ie) {
+  CosmicRay *pcr=pmb->pcr;
+  Real invlim = 1.0/pcr->vmax;
+
+  for(int i=is; i<=ie; ++i) {
+    Real inv_sqrt_rho = 1.0/std::sqrt(prim(IDN,k,j,i));
+    Real bsq = bcc(IB1,k,j,i)*bcc(IB1,k,j,i)
+              +bcc(IB2,k,j,i)*bcc(IB2,k,j,i)
+              +bcc(IB3,k,j,i)*bcc(IB3,k,j,i);
+
+    Real b_grad_pc = bcc(IB1,k,j,i) * grad_pc(0,k,j,i)
+                   + bcc(IB2,k,j,i) * grad_pc(1,k,j,i)
+                   + bcc(IB3,k,j,i) * grad_pc(2,k,j,i);
+
+    Real va1 = bcc(IB1,k,j,i) * inv_sqrt_rho;
+    Real va2 = bcc(IB2,k,j,i) * inv_sqrt_rho;
+    Real va3 = bcc(IB3,k,j,i) * inv_sqrt_rho;
+
+    Real va = std::sqrt(bsq) * inv_sqrt_rho;
+    Real dpc_sign = 0.0;
+
+    if (b_grad_pc > TINY_NUMBER) dpc_sign = 1.0;
+    else if (-b_grad_pc > TINY_NUMBER) dpc_sign = -1.0;
+
+    if (pcr->stream_flag > 0) {
+      pcr->v_adv(0,k,j,i) = -va1 * dpc_sign;
+      pcr->v_adv(1,k,j,i) = -va2 * dpc_sign;
+      pcr->v_adv(2,k,j,i) = -va3 * dpc_sign;
+      if (va > TINY_NUMBER) {
+        pcr->sigma_adv(0,k,j,i) = std::abs(b_grad_pc)/(std::sqrt(bsq) * va *
+                               (4.0/3.0) * invlim * u_cr(CRE,k,j,i));
+        pcr->sigma_adv(1,k,j,i) = pcr->max_opacity;
+        pcr->sigma_adv(2,k,j,i) = pcr->max_opacity;
+      }
+    } else {
+      pcr->v_adv(0,k,j,i) = 0.0;
+      pcr->v_adv(1,k,j,i) = 0.0;
+      pcr->v_adv(2,k,j,i) = 0.0;
+      pcr->sigma_adv(0,k,j,i)  = pcr->max_opacity;
+      pcr->sigma_adv(1,k,j,i)  = pcr->max_opacity;
+      pcr->sigma_adv(2,k,j,i)  = pcr->max_opacity;
+    }
+  }
+}
+
+CosmicRay::CosmicRay(MeshBlock *pmb, ParameterInput *pin):
+    pmy_block(pmb), u_cr(NCR,pmb->ncells3,pmb->ncells2,pmb->ncells1),
+    u_cr1(NCR,pmb->ncells3,pmb->ncells2,pmb->ncells1),
+    coarse_cr_(NCR,pmb->ncc3, pmb->ncc2, pmb->ncc1,
+               (pmb->pmy_mesh->multilevel ? AthenaArray<Real>::DataStatus::allocated :
+                AthenaArray<Real>::DataStatus::empty)),
+    sigma_diff(3,pmb->ncells3,pmb->ncells2,pmb->ncells1),
+    sigma_adv(3,pmb->ncells3,pmb->ncells2,pmb->ncells1),
+    v_adv(3,pmb->ncells3,pmb->ncells2,pmb->ncells1),
+    v_diff(3,pmb->ncells3,pmb->ncells2,pmb->ncells1),
+    // constructor overload resolution of non-aggregate class type AthenaArray<Real>
+    flux{ {NCR, pmb->ncells3, pmb->ncells2, pmb->ncells1+1},
+      {NCR,pmb->ncells3, pmb->ncells2+1, pmb->ncells1,
+       (pmb->pmy_mesh->f2 ? AthenaArray<Real>::DataStatus::allocated :
+        AthenaArray<Real>::DataStatus::empty)},
+      {NCR,pmb->ncells3+1, pmb->ncells2, pmb->ncells1,
+       (pmb->pmy_mesh->f3 ? AthenaArray<Real>::DataStatus::allocated :
+        AthenaArray<Real>::DataStatus::empty)}
+    },
+    cr_bvar(pmb, &u_cr, &coarse_cr_, flux, true),
+    UserSourceTerm_{} {
+  Mesh *pm = pmy_block->pmy_mesh;
+  pmb->RegisterMeshBlockData(u_cr);
+  // "Enroll" in S/AMR by adding to vector of tuples of pointers in MeshRefinement class
+  if (pm->multilevel) {
+    refinement_idx = pmy_block->pmr->AddToRefinement(&u_cr, &coarse_cr_);
+  }
+  cr_source_defined = false;
+  cr_bvar.bvar_index = pmb->pbval->bvars.size();
+  pmb->pbval->bvars.push_back(&cr_bvar);
+  pmb->pbval->bvars_main_int.push_back(&cr_bvar);
+
+  vmax = pin->GetOrAddReal("cr", "vmax", 1.0);
+  vlim = pin->GetOrAddReal("cr", "vlim", 0.9);
+  max_opacity = pin->GetOrAddReal("cr", "max_opacity", BIG_NUMBER);
+  stream_flag = pin->GetOrAddInteger("cr", "vs_flag", 1);
+  src_flag = pin->GetOrAddInteger("cr", "src_flag", 1);
+
+  int nc1 = pmb->ncells1, nc2 = pmb->ncells2, nc3 = pmb->ncells3;
+  b_grad_pc.NewAthenaArray(nc3, nc2, nc1);
+  b_angle.NewAthenaArray(4, nc3, nc2, nc1);
+
+  cwidth.NewAthenaArray(nc1);
+  cwidth1.NewAthenaArray(nc1);
+  cwidth2.NewAthenaArray(nc1);
+
+  // set a default opacity function
+  UpdateOpacity = DefaultOpacity;
+  UpdateStreaming = DefaultStreaming;
+  pcrintegrator = new CRIntegrator(this, pin);
+}
+
+void CosmicRay::EnrollOpacityFunction(CROpacityFunc MyOpacityFunction) {
+  UpdateOpacity = MyOpacityFunction;
+}
+
+void CosmicRay::EnrollStreamingFunction(CRStreamingFunc MyStreamingFunction) {
+  UpdateStreaming = MyStreamingFunction;
+}
+
+void CosmicRay::EnrollUserCRSource(CRSrcTermFunc my_func) {
+  UserSourceTerm_ = my_func;
+  cr_source_defined = true;
+}
diff --git a/src/cr/cr.hpp b/src/cr/cr.hpp
new file mode 100644
index 0000000000..f64172d0ba
--- /dev/null
+++ b/src/cr/cr.hpp
@@ -0,0 +1,81 @@
+#ifndef CR_CR_HPP_
+#define CR_CR_HPP_
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+// See LICENSE file for full public license information.
+//======================================================================================
+//! \file cr.hpp
+//  \brief definitions for cosmic ray class
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <string>
+
+// Athena++ classes headers
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
+#include "../bvals/cc/bvals_cc.hpp"
+
+class MeshBlock;
+class ParameterInput;
+class CRIntegrator;
+
+// Array indices for  moments
+enum {CRE=0, CRF1=1, CRF2=2, CRF3=3};
+
+class CosmicRay {
+  friend class CRIntegrator;
+  friend class BoundaryValues;
+ public:
+  CosmicRay(MeshBlock *pmb, ParameterInput *pin);
+  //  ~CosmicRay();
+
+  MeshBlock* pmy_block;    // ptr to MeshBlock containing this Fluid
+  AthenaArray<Real> u_cr, u_cr1, u_cr2; //cosmic ray energy density and flux
+  AthenaArray<Real> coarse_cr_;
+
+  //   diffusion coefficients for both normal diffusion term, and advection term
+  AthenaArray<Real> sigma_diff, sigma_adv;
+
+  AthenaArray<Real> v_adv; // streaming velocity
+  AthenaArray<Real> v_diff; // the diffuion velocity, need to calculate the flux
+
+  int refinement_idx{-1};
+
+  AthenaArray<Real> flux[3]; // store transport flux, also need for refinement
+
+  Real vmax; // the maximum velocity (effective speed of light)
+  Real vlim;
+  Real max_opacity;
+
+  CellCenteredBoundaryVariable cr_bvar;
+  CRIntegrator *pcrintegrator;
+
+  // Function in problem generators to update opacity
+  void EnrollOpacityFunction(CROpacityFunc MyOpacityFunction);
+  void EnrollStreamingFunction(CRStreamingFunc MyStreamingFunction);
+
+  void EnrollUserCRSource(CRSrcTermFunc my_func);
+  bool cr_source_defined;
+
+  // The function pointer for the diffusion coefficient
+  CROpacityFunc UpdateOpacity;
+  CRStreamingFunc UpdateStreaming;
+
+  AthenaArray<Real> cwidth;
+  AthenaArray<Real> cwidth1;
+  AthenaArray<Real> cwidth2;
+  AthenaArray<Real> b_grad_pc; // array to store B\dot Grad Pc
+  AthenaArray<Real> b_angle; //sin\theta,cos\theta,sin\phi,cos\phi of B direction
+
+  int stream_flag; // flag to include streaming or not
+  int src_flag; // flag to
+
+ private:
+  CRSrcTermFunc UserSourceTerm_;
+};
+
+#endif // CR_CR_HPP_
diff --git a/src/cr/integrators/cr_flux.cpp b/src/cr/integrators/cr_flux.cpp
new file mode 100644
index 0000000000..6c6ca79bdf
--- /dev/null
+++ b/src/cr/integrators/cr_flux.cpp
@@ -0,0 +1,97 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file dc.cpp
+//  \brief piecewise constant (donor cell) reconstruction
+
+// Athena++ headers
+#include <iostream>   // endl
+#include <sstream>    // stringstream
+#include <stdexcept>  // runtime_error
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../cr.hpp"
+#include "./cr_integrators.hpp"
+
+//----------------------------------------------------------------------------------------
+//! \fn ReIntegrator::CRFlux()
+//  \brief HLLE flux for Cosmic Ray Transport
+
+void CRIntegrator::CRFlux(int fdir, int il, int iu,
+      AthenaArray<Real> &w_l, AthenaArray<Real> &w_r,
+      AthenaArray<Real> &vdiff_l, AthenaArray<Real> &vdiff_r,
+                                      AthenaArray<Real> &flx) {
+  // First, get the photon diffusion speed
+  Real vmax = pmy_cr->vmax;
+  Real eddl = 1.0/3.0;
+  Real eddr = 1.0/3.0;
+  Real edd12 = 0.0;
+  Real edd13 = 0.0;
+  Real edd23 = 0.0;
+  // use sigma_l, sigma_r to get diffusion speed
+  for (int i=il; i<=iu; ++i) {
+    Real vl = w_l(NCR,i);
+    Real vr = w_r(NCR,i);
+
+    Real meanadv=0.5*(vl + vr);
+    Real meandiffv = 0.5*(vdiff_l(i)+vdiff_r(i));
+
+    Real al = std::min((meanadv - meandiffv),(vl-vdiff_l(i)));
+    Real ar = std::max((meanadv + meandiffv),(vr+vdiff_r(i)));
+    ar = std::min(ar,vmax * std::sqrt(eddr));
+    al = std::max(al,-vmax * std::sqrt(eddl));
+
+    Real bp = ar > 0.0 ? ar : 0.0;
+    Real bm = al < 0.0 ? al : 0.0;
+
+    // computer L/R fluxes along lines
+    // F_L - (S_L)U_L
+    // F_R - (S_R)U_R
+
+    Real fl_e = vmax * w_l(fdir,i) - bm * w_l(CRE,i);
+    Real fr_e = vmax * w_r(fdir,i) - bp * w_r(CRE,i);
+    Real fl_f1, fr_f1, fl_f2, fr_f2, fl_f3, fr_f3;
+    if (fdir == CRF1) {
+      fl_f1 = vmax * eddl * w_l(CRE,i) - bm * w_l(CRF1,i);
+      fr_f1 = vmax * eddr * w_r(CRE,i) - bp * w_r(CRF1,i);
+
+      fl_f2 = vmax * edd12 * w_l(CRE,i) - bm * w_l(CRF2,i);
+      fr_f2 = vmax * edd12 * w_r(CRE,i) - bp * w_r(CRF2,i);
+
+      fl_f3 = vmax * edd13 * w_l(CRE,i) - bm * w_l(CRF3,i);
+      fr_f3 = vmax * edd13 * w_r(CRE,i) - bp * w_r(CRF3,i);
+    } else if (fdir == CRF2) {
+      fl_f1 = vmax * edd12 * w_l(CRE,i) - bm * w_l(CRF1,i);
+      fr_f1 = vmax * edd12 * w_r(CRE,i) - bp * w_r(CRF1,i);
+
+      fl_f2 = vmax * eddl * w_l(CRE,i) - bm * w_l(CRF2,i);
+      fr_f2 = vmax * eddr * w_r(CRE,i) - bp * w_r(CRF2,i);
+
+      fl_f3 = vmax * edd23 * w_l(CRE,i) - bm * w_l(CRF3,i);
+      fr_f3 = vmax * edd23 * w_r(CRE,i) - bp * w_r(CRF3,i);
+    } else {
+      fl_f1 = vmax * edd13 * w_l(CRE,i) - bm * w_l(CRF1,i);
+      fr_f1 = vmax * edd13 * w_r(CRE,i) - bp * w_r(CRF1,i);
+
+      fl_f2 = vmax * edd23 * w_l(CRE,i) - bm * w_l(CRF2,i);
+      fr_f2 = vmax * edd23 * w_r(CRE,i) - bp * w_r(CRF2,i);
+
+      fl_f3 = vmax * eddl * w_l(CRE,i) - bm * w_l(CRF3,i);
+      fr_f3 = vmax * eddr * w_r(CRE,i) - bp * w_r(CRF3,i);
+    }
+    // calculate the HLLE flux
+    Real tmp = 0.0;
+    if (std::abs(bm-bp) > TINY_NUMBER) {
+      tmp = 0.5*(bp + bm)/(bp - bm);
+    }
+    flx(CRE,i) = 0.5*(fl_e + fr_e) + (fl_e - fr_e) * tmp;
+    flx(CRF1,i) = 0.5*(fl_f1 + fr_f1) + (fl_f1 - fr_f1) * tmp;
+    flx(CRF2,i) = 0.5*(fl_f2 + fr_f2) + (fl_f2 - fr_f2) * tmp;
+    flx(CRF3,i) = 0.5*(fl_f3 + fr_f3) + (fl_f3 - fr_f3) * tmp;
+  }
+  return;
+}
diff --git a/src/cr/integrators/cr_integrators.cpp b/src/cr/integrators/cr_integrators.cpp
new file mode 100644
index 0000000000..5d8938d18d
--- /dev/null
+++ b/src/cr/integrators/cr_integrators.cpp
@@ -0,0 +1,124 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file cr_integrators.cpp
+//  \brief implementation of cosmic ray integrators
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <sstream>
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../parameter_input.hpp"
+#include "../cr.hpp"
+#include "cr_integrators.hpp"
+
+
+CRIntegrator::CRIntegrator(CosmicRay *pcr, ParameterInput *pin) {
+  pmy_cr = pcr;
+  MeshBlock *pmb=pcr->pmy_block;
+
+  cr_xorder = pin->GetOrAddInteger("time","cr_xorder",2);
+  if (cr_xorder == 3) {
+    if (NGHOST < 3) {
+      std::stringstream msg;
+      msg << "### FATAL ERROR in cosmic ray reconstruction constructor" << std::endl
+          << "cr_xorder=" << cr_xorder <<
+          " (PPM) reconstruction selected, but nghost=" << NGHOST << std::endl
+          << "Reconfigure with --nghost=3  " <<std::endl;
+      ATHENA_ERROR(msg);
+    }
+  }
+
+  int ncells1 = pmb->ncells1, ncells2 = pmb->ncells2,
+  ncells3 = pmb->ncells3;
+
+  x1face_area_.NewAthenaArray(ncells1+1);
+  if(ncells2 > 1) {
+    x2face_area_.NewAthenaArray(ncells1);
+    x2face_area_p1_.NewAthenaArray(ncells1);
+  }
+  if(ncells3 > 1) {
+    x3face_area_.NewAthenaArray(ncells1);
+    x3face_area_p1_.NewAthenaArray(ncells1);
+  }
+  cell_volume_.NewAthenaArray(ncells1);
+
+
+  cwidth2_.NewAthenaArray(ncells1);
+  cwidth3_.NewAthenaArray(ncells1);
+
+  dflx_.NewAthenaArray(NCR,ncells1);
+
+  // arrays for spatial recontruction
+  //NCR+1 represent Ecr, Fcr1, Fcr2, Fcr3 and vel
+  ucr_l_.NewAthenaArray(NCR+1,ncells1);
+  ucr_lb_.NewAthenaArray(NCR+1,ncells1);
+
+  ucr_r_.NewAthenaArray(NCR+1,ncells1);
+  ucr_vel_.NewAthenaArray(NCR+1,ncells3,ncells2,ncells1);
+
+  vdiff_l_.NewAthenaArray(ncells1);
+  vdiff_r_.NewAthenaArray(ncells1);
+
+
+  taufact_ = pin->GetOrAddReal("cr","taucell",1.0);
+  vel_flx_flag_ = pin->GetOrAddInteger("cr","vflx",0);
+
+  new_sol_.NewAthenaArray(NCR,ncells1);
+
+  grad_pc_.NewAthenaArray(3,ncells3,ncells2,ncells1);
+  ec_source_.NewAthenaArray(ncells3,ncells2,ncells1);
+  coord_source_.NewAthenaArray(NCR,ncells3,ncells2,ncells1);
+}
+
+// destructor
+
+CRIntegrator::~CRIntegrator() {
+  x1face_area_.DeleteAthenaArray();
+  if (pmy_cr->pmy_block->ncells2 > 1) {
+    x2face_area_.DeleteAthenaArray();
+    x2face_area_p1_.DeleteAthenaArray();
+  }
+  if (pmy_cr->pmy_block->ncells3 > 1) {
+    x3face_area_.DeleteAthenaArray();
+    x3face_area_p1_.DeleteAthenaArray();
+  }
+  cell_volume_.DeleteAthenaArray();
+
+  cwidth2_.DeleteAthenaArray();
+  cwidth3_.DeleteAthenaArray();
+  dflx_.DeleteAthenaArray();
+  ucr_l_.DeleteAthenaArray();
+  ucr_r_.DeleteAthenaArray();
+  ucr_lb_.DeleteAthenaArray();
+  vdiff_l_.DeleteAthenaArray();
+  vdiff_r_.DeleteAthenaArray();
+
+  new_sol_.DeleteAthenaArray();
+
+  grad_pc_.DeleteAthenaArray();
+  ec_source_.DeleteAthenaArray();
+  coord_source_.DeleteAthenaArray();
+  ucr_vel_.DeleteAthenaArray();
+}
diff --git a/src/cr/integrators/cr_integrators.hpp b/src/cr/integrators/cr_integrators.hpp
new file mode 100644
index 0000000000..d58f826b9b
--- /dev/null
+++ b/src/cr/integrators/cr_integrators.hpp
@@ -0,0 +1,60 @@
+#ifndef CR_INTEGRATORS_CR_INTEGRATORS_HPP_
+#define CR_INTEGRATORS_CR_INTEGRATORS_HPP_
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+// See LICENSE file for full public license information.
+//======================================================================================
+//! \file radiation.hpp
+//  \brief definitions for Radiation class
+//======================================================================================
+
+// Athena++ classes headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../task_list/task_list.hpp"
+#include "../cr.hpp"
+
+class MeshBlock;
+class ParameterInput;
+class CosmicRay;
+
+//! \class RadIntegrator
+//  \brief integrate algorithm for radiative transfer
+class CRIntegrator {
+  friend class CosmicRay;
+ public:
+  CRIntegrator(CosmicRay *pcr, ParameterInput *pin);
+  ~CRIntegrator();
+
+  CosmicRay *pmy_cr;
+
+  void FluxDivergence(const Real wght, AthenaArray<Real> &cr_out);
+  void CalculateFluxes(AthenaArray<Real> &w,
+          AthenaArray<Real> &bcc, AthenaArray<Real> &cr, const int order);
+
+  void CRFlux(int fdir, int il, int iu,
+              AthenaArray<Real> &w_l, AthenaArray<Real> &w_r,
+              AthenaArray<Real> &vdiff_l, AthenaArray<Real> &vdiff_r,
+              AthenaArray<Real> &flx);
+  void AddSourceTerms(MeshBlock *pmb, const Real dt, AthenaArray<Real> &u,
+        AthenaArray<Real> &w, AthenaArray<Real> &bcc, AthenaArray<Real> &ucr);
+  int cr_xorder;
+
+ private:
+  AthenaArray<Real> new_sol_;
+  AthenaArray<Real> ucr_l_, ucr_r_, ucr_lb_; // for reconstruction
+  AthenaArray<Real> vdiff_l_, vdiff_r_;
+
+  AthenaArray<Real> grad_pc_, ec_source_, coord_source_;
+  AthenaArray<Real> ucr_vel_; //array for reconstruction
+
+  // temporary array to store the flux
+  Real taufact_;
+  int vel_flx_flag_;
+  AthenaArray<Real> x1face_area_, x2face_area_, x3face_area_;
+  AthenaArray<Real> x2face_area_p1_, x3face_area_p1_;
+  AthenaArray<Real> cell_volume_, dflx_, cwidth2_, cwidth3_;
+};
+
+#endif // CR_INTEGRATORS_CR_INTEGRATORS_HPP_
diff --git a/src/cr/integrators/cr_source.cpp b/src/cr/integrators/cr_source.cpp
new file mode 100644
index 0000000000..2474a5c065
--- /dev/null
+++ b/src/cr/integrators/cr_source.cpp
@@ -0,0 +1,212 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file cr_source.cpp
+//  \brief Add cosmic ray source terms to  gas
+//======================================================================================
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../eos/eos.hpp"
+#include "../../field/field.hpp"
+#include "../../hydro/hydro.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../parameter_input.hpp"
+#include "../../utils/utils.hpp"
+#include "../cr.hpp"
+
+// class header
+#include "cr_integrators.hpp"
+
+// MPI/OpenMP header
+#ifdef MPI_PARALLEL
+#include <mpi.h>
+#endif
+
+
+// OpenMP header
+#ifdef OPENMP_PARALLEL
+#include <omp.h>
+#endif
+
+// add the source terms implicitly
+void CRIntegrator::AddSourceTerms(MeshBlock *pmb, const Real dt, AthenaArray<Real> &u,
+        AthenaArray<Real> &w, AthenaArray<Real> &bcc,
+        AthenaArray<Real> &u_cr) {
+  CosmicRay *pcr=pmb->pcr;
+  Real vlim = pcr->vmax;
+  Real invlim = 1.0/vlim;
+  Real rho_floor = pmb->peos->GetDensityFloor();
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+         // Real fxx = 1.0/3.0;
+         // Real fyy = 1.0/3.0;
+         // Real fzz = 1.0/3.0;
+         // Real fxy = 0.0;
+         // Real fxz = 0.0;
+         // Real fyz = 0.0;
+
+         Real *ec = &(u_cr(CRE,k,j,0));
+         Real *fc1 = &(u_cr(CRF1,k,j,0));
+         Real *fc2 = &(u_cr(CRF2,k,j,0));
+         Real *fc3 = &(u_cr(CRF3,k,j,0));
+
+         // The angle of B
+         Real *sint_b = &(pcr->b_angle(0,k,j,0));
+         Real *cost_b = &(pcr->b_angle(1,k,j,0));
+         Real *sinp_b = &(pcr->b_angle(2,k,j,0));
+         Real *cosp_b = &(pcr->b_angle(3,k,j,0));
+
+         // adv1 is dPc/dx, adv2 is dPc/dy, adv3 is dPc/dz
+
+      for (int i=is; i<=ie; ++i) {
+         Real rho = u(IDN,k,j,i);
+         rho = std::max(rho,rho_floor);
+         Real v1 = u(IM1,k,j,i)/rho;
+         Real v2 = u(IM2,k,j,i)/rho;
+         Real v3 = u(IM3,k,j,i)/rho;
+         Real vtot1 = v1;
+         Real vtot2 = v2;
+         Real vtot3 = v3;
+
+         // add the streaming velocity
+         if (pcr->stream_flag) {
+           vtot1 += pcr->v_adv(0,k,j,i);
+           vtot2 += pcr->v_adv(1,k,j,i);
+           vtot3 += pcr->v_adv(2,k,j,i);
+         }
+
+         Real fr1 = fc1[i];
+         Real fr2 = fc2[i];
+         Real fr3 = fc3[i];
+
+         // in the case with magnetic field
+        // rotate the vectors to oriante to the B direction
+         if (MAGNETIC_FIELDS_ENABLED) {
+           // Apply rotation of the vectors
+           RotateVec(sint_b[i],cost_b[i],sinp_b[i],cosp_b[i],v1,v2,v3);
+
+           RotateVec(sint_b[i],cost_b[i],sinp_b[i],cosp_b[i],fr1,fr2,fr3);
+
+           RotateVec(sint_b[i],cost_b[i],sinp_b[i],cosp_b[i],vtot1,vtot2,vtot3);
+
+           // perpendicular energy source term is already added via ec_source_
+           // we will still need to include this in general for diffusion case
+           vtot2 = 0.0;
+           vtot3 = 0.0;
+         }
+
+         Real sigma_x = pcr->sigma_diff(0,k,j,i);
+         Real sigma_y = pcr->sigma_diff(1,k,j,i);
+         Real sigma_z = pcr->sigma_diff(2,k,j,i);
+
+         if (pcr->stream_flag) {
+           sigma_x = 1.0/(1.0/pcr->sigma_diff(0,k,j,i) +
+                             1.0/pcr->sigma_adv(0,k,j,i));
+
+           sigma_y = 1.0/(1.0/pcr->sigma_diff(1,k,j,i) +
+                             1.0/pcr->sigma_adv(1,k,j,i));
+
+           sigma_z = 1.0/(1.0/pcr->sigma_diff(2,k,j,i) +
+                             1.0/pcr->sigma_adv(2,k,j,i));
+         }
+
+         // Now update the momentum equation
+         //\partial F/\partial t=-V_m\sigma (F-v(E+Pc_)/v_m))
+         // And the energy equation
+         //\partial E_c/\partial t = -vtot sigma (F- v(E_c+P_c)/v_m)
+
+         Real rhs1 = ec[i];
+         Real rhs2 = fr1;
+         Real rhs3 = fr2;
+         Real rhs4 = fr3;
+
+         Real coef_11 = 1.0 - dt * sigma_x * vtot1 * v1 * invlim * 4.0/3.0
+                            - dt * sigma_y * vtot2 * v2 * invlim * 4.0/3.0
+                            - dt * sigma_z * vtot3 * v3 * invlim * 4.0/3.0;
+         Real coef_12 = dt * sigma_x * vtot1;
+         Real coef_13 = dt * sigma_y * vtot2;
+         Real coef_14 = dt * sigma_z * vtot3;
+
+         Real coef_21 = -dt * v1 * sigma_x * 4.0/3.0;
+         Real coef_22 = 1.0 + dt * vlim * sigma_x;
+
+         Real coef_31 = -dt * v2 * sigma_y * 4.0/3.0;
+         Real coef_33 = 1.0 + dt * vlim * sigma_y;
+
+         Real coef_41 = -dt * v3 * sigma_z * 4.0/3.0;
+         Real coef_44 = 1.0 + dt * vlim * sigma_z;
+
+        //newfr1 = (rhs2 - coef21 * newEc)/coef22
+        // newfr2= (rhs3 - coef31 * newEc)/coef33
+        // newfr3 = (rhs4 - coef41 * newEc)/coef44
+        // coef11 - coef21 * coef12 /coef22 - coef13 * coef31 /
+         //              coef33 - coef41 * coef14 /coef44)* newec
+        //    =rhs1 - coef12 *rhs2/coef22 - coef13 * rhs3/coef33 - coef14 * rhs4/coef44
+
+        Real e_coef = coef_11 - coef_12 * coef_21/coef_22 - coef_13 * coef_31/coef_33
+                       - coef_14 * coef_41/coef_44;
+        Real new_ec = rhs1 - coef_12 * rhs2/coef_22 - coef_13 * rhs3/coef_33
+                      - coef_14 * rhs4/coef_44;
+        new_ec /= e_coef;
+
+        Real newfr1 = (rhs2 - coef_21 * new_ec)/coef_22;
+        Real newfr2 = (rhs3 - coef_31 * new_ec)/coef_33;
+        Real newfr3 = (rhs4 - coef_41 * new_ec)/coef_44;
+
+
+        // Now apply the invert rotation
+        if (MAGNETIC_FIELDS_ENABLED) {
+         // Apply rotation of the vectors
+          InvRotateVec(sint_b[i],cost_b[i],sinp_b[i],cosp_b[i],
+                                         newfr1,newfr2,newfr3);
+          new_ec += dt * ec_source_(k,j,i);
+        }
+
+         // Add the energy source term
+         if (NON_BAROTROPIC_EOS && (pcr->src_flag > 0)) {
+           Real new_eg = u(IEN,k,j,i) - (new_ec - ec[i]);
+           if (new_eg < 0.0) new_eg = u(IEN,k,j,i);
+           u(IEN,k,j,i) = new_eg;
+         }
+
+         if (new_ec < 0.0) new_ec = ec[i];
+
+         if (pcr->src_flag > 0) {
+           u(IM1,k,j,i) += (-(newfr1 - fc1[i]) * invlim);
+           u(IM2,k,j,i) += (-(newfr2 - fc2[i]) * invlim);
+           u(IM3,k,j,i) += (-(newfr3 - fc3[i]) * invlim);
+         }
+         u_cr(CRE,k,j,i) = new_ec;
+         u_cr(CRF1,k,j,i) = newfr1;
+         u_cr(CRF2,k,j,i) = newfr2;
+         u_cr(CRF3,k,j,i) = newfr3;
+      }
+    }
+  }
+  // Add user defined source term for cosmic rays
+  if (pcr->cr_source_defined)
+    pcr->UserSourceTerm_(pmb, pmb->pmy_mesh->time, dt, w,pmb->pfield->b, u_cr);
+}
diff --git a/src/cr/integrators/cr_transport.cpp b/src/cr/integrators/cr_transport.cpp
new file mode 100644
index 0000000000..0b722a0fe9
--- /dev/null
+++ b/src/cr/integrators/cr_transport.cpp
@@ -0,0 +1,524 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file cr_transport.cpp
+//  \brief implementation of cosmic ray transport
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <algorithm>   // min,max
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../parameter_input.hpp"
+#include "../../reconstruct/reconstruction.hpp"
+#include "../../utils/utils.hpp"
+#include "../cr.hpp"
+
+// class header
+#include "./cr_integrators.hpp"
+
+// MPI/OpenMP header
+#ifdef MPI_PARALLEL
+#include <mpi.h>
+#endif
+
+
+// OpenMP header
+#ifdef OPENMP_PARALLEL
+#include <omp.h>
+#endif
+
+static constexpr Real tau_asymptotic_lim=1.0e-3;
+
+void CRIntegrator::CalculateFluxes(
+    AthenaArray<Real> &w, AthenaArray<Real> &bcc,
+    AthenaArray<Real> &cr, const int order) {
+  CosmicRay *pcr=pmy_cr;
+  MeshBlock *pmb=pcr->pmy_block;
+  Coordinates *pco = pmb->pcoord;
+  Real invlim = 1.0/pcr->vmax;
+
+  int ncells1 = pmb->ncells1, ncells2 = pmb->ncells2,
+      ncells3 = pmb->ncells3;
+
+  AthenaArray<Real> &x1flux=pcr->flux[X1DIR];
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+  int il, iu, jl, ju, kl, ku;
+  jl = js, ju=je, kl=ks, ku=ke;
+
+  if (ncells2 > 1) {
+    if (ncells3 == 1) {
+      jl=js-1, ju=je+1, kl=ks, ku=ke;
+    } else {
+      jl=js-1, ju=je+1, kl=ks-1, ku=ke+1;
+    }
+  }
+
+  //---------------------------------------------------------------------
+  for (int k=0; k<ncells3; ++k) {
+    for (int j=0; j<ncells2; ++j) {
+      // diffusion velocity along the direction of sigma vector
+      // We first assume B is along x coordinate
+      // Then rotate according to B direction to the actual acooridnate
+      for (int i=0; i<ncells1; ++i) {
+        Real eddxx=1.0/3.0;
+        Real totsigma = pcr->sigma_diff(0,k,j,i);
+        if (pcr->stream_flag)
+          totsigma = 1.0/(1.0/pcr->sigma_diff(0,k,j,i)
+                          + 1.0/pcr->sigma_adv(0,k,j,i));
+        Real taux = taufact_ * totsigma * pco->dx1f(i);
+        taux = taux * taux/(2.0 * eddxx);
+        Real diffv = 1.0;
+
+        if (taux < tau_asymptotic_lim) {
+          diffv = std::sqrt((1.0 - 0.5* taux));
+        } else {
+          diffv = std::sqrt((1.0 - std::exp(-taux)) / taux);
+        }
+        pcr->v_diff(0,k,j,i) = pcr->vmax * std::sqrt(eddxx) * diffv;
+      }
+
+      // y direction
+      if (ncells2 >1) {
+        pco->CenterWidth2(k,j,0,ncells1-1,cwidth2_);
+        // get the optical depth across the cell
+        for (int i=0; i<ncells1; ++i) {
+          Real eddyy=1.0/3.0;
+          Real totsigma = pcr->sigma_diff(1,k,j,i);
+          if (pcr->stream_flag)
+            totsigma = 1.0/(1.0/pcr->sigma_diff(1,k,j,i)
+                            + 1.0/pcr->sigma_adv(1,k,j,i));
+          Real tauy = taufact_ * totsigma * cwidth2_(i);
+          tauy = tauy * tauy/(2.0 * eddyy);
+
+          Real diffv = 1.0;
+
+          if (tauy < tau_asymptotic_lim) {
+            diffv = std::sqrt((1.0 - 0.5* tauy));
+          } else {
+            diffv = std::sqrt((1.0 - std::exp(-tauy)) / tauy);
+          }
+          pcr->v_diff(1,k,j,i) = pcr->vmax * std::sqrt(eddyy) * diffv;
+        }
+      } else {
+        for (int i=0; i<ncells1; ++i)
+          pcr->v_diff(1,k,j,i) = 0.0;
+      }
+
+      // z direction
+      if (ncells3 > 1) {
+        pco->CenterWidth3(k,j,0,ncells1-1,cwidth3_);
+        // get the optical depth across the cell
+        for (int i=0; i<ncells1; ++i) {
+          Real eddzz=1.0/3.0;
+          Real totsigma = pcr->sigma_diff(2,k,j,i);
+          if (pcr->stream_flag)
+            totsigma = 1.0/(1.0/pcr->sigma_diff(2,k,j,i)
+                            + 1.0/pcr->sigma_adv(2,k,j,i));
+          Real tauz = taufact_ * totsigma * cwidth3_(i);
+          tauz = tauz * tauz/(2.0 * eddzz);
+
+          Real diffv = 1.0;
+
+          if (tauz < tau_asymptotic_lim) {
+            diffv = std::sqrt((1.0 - 0.5* tauz));
+          } else {
+            diffv = std::sqrt((1.0 - std::exp(-tauz)) / tauz);
+          }
+          pcr->v_diff(2,k,j,i) = pcr->vmax * std::sqrt(eddzz) * diffv;
+        }
+      } else {
+        for (int i=0; i<ncells1; ++i)
+          pcr->v_diff(2,k,j,i) = 0.0;
+      }
+
+      // rotate the v_diff vector to the local coordinate
+      if (MAGNETIC_FIELDS_ENABLED) {
+        for (int i=0; i<ncells1; ++i) {
+          InvRotateVec(pcr->b_angle(0,k,j,i),pcr->b_angle(1,k,j,i),
+                       pcr->b_angle(2,k,j,i),pcr->b_angle(3,k,j,i),
+                       pcr->v_diff(0,k,j,i),pcr->v_diff(1,k,j,i),
+                       pcr->v_diff(2,k,j,i));
+          // take the absolute value
+          // Also add the Alfven velocity for the streaming flux
+          pcr->v_diff(0,k,j,i) = std::abs(pcr->v_diff(0,k,j,i));
+
+          pcr->v_diff(1,k,j,i) = std::abs(pcr->v_diff(1,k,j,i));
+
+          pcr->v_diff(2,k,j,i) = std::abs(pcr->v_diff(2,k,j,i));
+        }
+      }
+      // need to add additional sound speed for stability
+      for (int i=0; i<ncells1; ++i) {
+        Real cr_sound_x = vel_flx_flag_ * std::sqrt((4.0/9.0)
+                                                    * cr(CRE,k,j,i)/w(IDN,k,j,i));
+        pcr->v_diff(0,k,j,i) += cr_sound_x;
+        if (ncells2 > 1) {
+          Real cr_sound_y = vel_flx_flag_ * std::sqrt((4.0/9.0)
+                                                      * cr(CRE,k,j,i)/w(IDN,k,j,i));
+          pcr->v_diff(1,k,j,i) += cr_sound_y;
+        }
+
+        if (ncells3 > 1) {
+          Real cr_sound_z = vel_flx_flag_ * std::sqrt((4.0/9.0)
+                                                      * cr(CRE,k,j,i)/w(IDN,k,j,i));
+
+          pcr->v_diff(2,k,j,i) += cr_sound_z;
+        }
+      }
+    }
+  }
+  // prepare Array for reconstruction
+  for (int n=0; n<NCR; ++n) {
+    for (int k=0; k<ncells3; ++k) {
+      for (int j=0; j<ncells2; ++j) {
+        for (int i=0; i<ncells1; ++i) {
+          ucr_vel_(n,k,j,i) = cr(n,k,j,i);
+        }
+      }
+    }
+  }
+
+  //--------------------------------------------------------------------------------------
+  // i-direction
+
+  // add vx velocity
+  for (int k=0; k<ncells3; ++k) {
+    for (int j=0; j<ncells2; ++j) {
+      for (int i=0; i<ncells1; ++i) {
+        ucr_vel_(NCR,k,j,i) = w(IVX,k,j,i);
+      }
+    }
+  }
+
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      // First, need to do reconstruction
+      // to reconstruct Ec, Fc, vel, v_a and
+      // return Ec,Fc and signal speed at left and right state
+      if (order == 1) {
+        pmb->precon->DonorCellX1(k, j, is-1, ie+1, ucr_vel_, ucr_l_, ucr_r_);
+      } else if (order == 2) {
+        pmb->precon->PiecewiseLinearX1(k, j, is-1, ie+1, ucr_vel_, ucr_l_, ucr_r_);
+      } else {
+        pmb->precon->PiecewiseParabolicX1(k, j, is-1, ie+1, ucr_vel_, ucr_l_, ucr_r_);
+      }
+
+      // get the optical depth across the cell
+#pragma omp simd
+      for (int i=is; i<=ie+1; ++i) {
+        vdiff_l_(i) = pcr->v_diff(0,k,j,i-1);
+        vdiff_r_(i) = pcr->v_diff(0,k,j,i);
+      }
+
+      // calculate the flux
+      CRFlux(CRF1, is, ie+1, ucr_l_, ucr_r_, vdiff_l_, vdiff_r_, dflx_);
+      // store the flux
+      for (int n=0; n<NCR; ++n) {
+#pragma omp simd
+        for (int i=is; i<=ie+1; ++i) {
+          x1flux(n,k,j,i) = dflx_(n,i);
+        }
+      }
+    }
+  }
+
+  //--------------------------------------------------------------------------------------
+  // j-direction
+  if (pmb->pmy_mesh->f2) {
+    AthenaArray<Real> &x2flux=pcr->flux[X2DIR];
+    // add vy velocity
+    for (int k=0; k<ncells3; ++k) {
+      for (int j=0; j<ncells2; ++j) {
+        for (int i=0; i<ncells1; ++i) {
+          ucr_vel_(NCR,k,j,i) = w(IVY,k,j,i);
+        }
+      }
+    }
+
+    il=is-1; iu=ie+1; kl=ks; ku=ke;
+    if (ncells3 ==  1) // 2D
+      kl = ks, ku = ke;
+    else // 3D
+      kl = ks-1, ku = ke+1;
+    for (int k=kl; k<=ku; ++k) {
+      //reconstruct the first row
+      if (order == 1) {
+        pmb->precon->DonorCellX2(k, js-1, il, iu, ucr_vel_, ucr_l_, ucr_r_);
+      } else if (order == 2) {
+        pmb->precon->PiecewiseLinearX2(k, js-1, il, iu, ucr_vel_, ucr_l_, ucr_r_);
+      } else {
+        pmb->precon->PiecewiseParabolicX2(k, js-1, il, iu, ucr_vel_, ucr_l_, ucr_r_);
+      }
+
+
+      for (int j=js; j<=je+1; ++j) {
+        if (order == 1) {
+          pmb->precon->DonorCellX2(k, j, il, iu, ucr_vel_, ucr_lb_, ucr_r_);
+        } else if (order == 2) {
+          pmb->precon->PiecewiseLinearX2(k, j, il, iu, ucr_vel_, ucr_lb_, ucr_r_);
+        } else {
+          pmb->precon->PiecewiseParabolicX2(k, j, il, iu, ucr_vel_, ucr_lb_, ucr_r_);
+        }
+
+        // get the optical depth across the cell
+#pragma omp simd
+        for (int i=il; i<=iu; ++i) {
+          vdiff_l_(i) = pcr->v_diff(1,k,j-1,i);
+          vdiff_r_(i) = pcr->v_diff(1,k,j,i);
+        }
+        // calculate the flux
+        CRFlux(CRF2, il, iu, ucr_l_, ucr_r_, vdiff_l_, vdiff_r_, dflx_);
+        // store the flux
+        for (int n=0; n<NCR; ++n) {
+#pragma omp simd
+          for (int i=il; i<=iu; ++i) {
+            x2flux(n,k,j,i) = dflx_(n,i);
+          }
+        }
+        // swap the array for next cycle
+        ucr_l_.SwapAthenaArray(ucr_lb_);
+      }
+    }
+  }
+  //  k-direction
+  if (pmb->pmy_mesh->f3) {
+    AthenaArray<Real> &x3flux=pcr->flux[X3DIR];
+    il =is-1, iu=ie+1, jl=js-1, ju=je+1;
+    // add vz velocity
+    for (int k=0; k<ncells3; ++k) {
+      for (int j=0; j<ncells2; ++j) {
+        for (int i=0; i<ncells1; ++i) {
+          ucr_vel_(NCR,k,j,i) = w(IVZ,k,j,i);
+        }// end i
+      }// end j
+    }// end k
+
+    for (int j=jl; j<=ju; ++j) {
+      if (order == 1) {
+        pmb->precon->DonorCellX3(ks-1, j, il, iu, ucr_vel_, ucr_l_, ucr_r_);
+      } else if (order == 2) {
+        pmb->precon->PiecewiseLinearX3(ks-1, j, il, iu, ucr_vel_, ucr_l_, ucr_r_);
+      } else {
+        pmb->precon->PiecewiseParabolicX3(ks-1, j, il, iu, ucr_vel_, ucr_l_, ucr_r_);
+      }
+
+      for (int k=ks; k<=ke+1; ++k) {
+        // reconstruct L/R states at k
+        if (order == 1) {
+          pmb->precon->DonorCellX3(k, j, il, iu, ucr_vel_, ucr_lb_, ucr_r_);
+        } else if (order == 2) {
+          pmb->precon->PiecewiseLinearX3(k, j, il, iu, ucr_vel_, ucr_lb_, ucr_r_);
+        } else {
+          pmb->precon->PiecewiseParabolicX3(k, j, il, iu, ucr_vel_, ucr_lb_, ucr_r_);
+        }
+
+#pragma omp simd
+        for (int i=il; i<=iu; ++i) {
+          vdiff_l_(i) = pcr->v_diff(2,k-1,j,i);
+          vdiff_r_(i) = pcr->v_diff(2,k,j,i);
+        }
+        // calculate the flux
+        CRFlux(CRF3, il, iu, ucr_l_, ucr_r_, vdiff_l_, vdiff_r_, dflx_);
+        for (int n=0; n<NCR; ++n) {
+#pragma omp simd
+          for (int i=il; i<=iu; ++i) {
+            x3flux(n,k,j,i) = dflx_(n,i);
+          }
+        }
+        // swap the array for next cycle
+        ucr_l_.SwapAthenaArray(ucr_lb_);
+      }
+    }
+  }
+
+  //---------------------------------------------------------------------------------------
+  // Now calculate Grad Pc and the associated heating term
+  // the flux divergence term is Grad P_c for \partial F_c/\partial t
+  // only do this for the MHD case and along direction perpendicular
+  // to the magnetic field
+  if (MAGNETIC_FIELDS_ENABLED) {
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        pmb->pcoord->Face1Area(k,j,is,ie+1,x1face_area_);
+        pmb->pcoord->CellVolume(k,j,is,ie,cell_volume_);
+        // x1 direction
+        for (int n=0; n<3; ++n) {
+#pragma omp simd
+          for (int i=is; i<=ie; ++i) {
+            grad_pc_(n,k,j,i) = (x1face_area_(i+1)*x1flux(CRF1+n,k,j,i+1)
+                                 - x1face_area_(i)*x1flux(CRF1+n,k,j,i))/cell_volume_(i);
+          }
+        }
+
+        if (pmb->block_size.nx2 > 1) {
+          AthenaArray<Real> &x2flux=pcr->flux[X2DIR];
+          pmb->pcoord->Face2Area(k,j  ,is,ie,x2face_area_   );
+          pmb->pcoord->Face2Area(k,j+1,is,ie,x2face_area_p1_);
+          for (int n=0; n<3; ++n) {
+#pragma omp simd
+            for (int i=is; i<=ie; ++i) {
+              grad_pc_(n,k,j,i) += (x2face_area_p1_(i)*x2flux(CRF1+n,k,j+1,i)
+                                    - x2face_area_(i)*x2flux(CRF1+n,k,j,i))
+                                   /cell_volume_(i);
+            }
+          }
+        }
+        if (pmb->block_size.nx3 > 1) {
+          AthenaArray<Real> &x3flux=pcr->flux[X3DIR];
+          pmb->pcoord->Face3Area(k  ,j,is,ie,x3face_area_);
+          pmb->pcoord->Face3Area(k+1,j,is,ie,x3face_area_p1_);
+          for (int n=0; n<3; ++n) {
+#pragma omp simd
+            for (int i=is; i<=ie; ++i) {
+              grad_pc_(n,k,j,i) += (x3face_area_p1_(i) *x3flux(CRF1+n,k+1,j,i)
+                                    - x3face_area_(i)*x3flux(CRF1+n,k,j,i))
+                                   /cell_volume_(i);
+            }
+          }
+        }
+        for (int n=0; n<3; ++n) {
+#pragma omp simd
+          for (int i=is; i<=ie; ++i) {
+            grad_pc_(n,k,j,i) *= invlim;
+          }
+        }
+
+        // need to subtract the coordinate source term to get the actual grad Pc for c
+        // curlinear coordinate system
+        pmb->pcoord->CRGradPcCoordTermsDivergence(cr, grad_pc_);
+
+        // update streaming with grad_pc
+
+        pcr->UpdateStreaming(pmb, cr,w,bcc,grad_pc_,k,j,is,ie);
+
+        // calculate streaming velocity with magnetic field
+        for (int i=is; i<=ie; ++i) {
+          Real v1 = w(IVX,k,j,i);
+          Real v2 = w(IVY,k,j,i);
+          Real v3 = w(IVZ,k,j,i);
+
+          Real dpcdx = grad_pc_(0,k,j,i);
+          Real dpcdy = grad_pc_(1,k,j,i);
+          Real dpcdz = grad_pc_(2,k,j,i);
+
+          RotateVec(pcr->b_angle(0,k,j,i),pcr->b_angle(1,k,j,i),
+                    pcr->b_angle(2,k,j,i),pcr->b_angle(3,k,j,i),dpcdx,dpcdy,dpcdz);
+
+          RotateVec(pcr->b_angle(0,k,j,i),pcr->b_angle(1,k,j,i),
+                    pcr->b_angle(2,k,j,i),pcr->b_angle(3,k,j,i),v1,v2,v3);
+
+          // only calculate v_dot_gradpc perpendicular to B
+          // perpendicular direction only has flow velocity, no streaming velocity
+          Real v_dot_gradpc = v2 * dpcdy + v3 * dpcdz;
+
+          ec_source_(k,j,i) = v_dot_gradpc;
+        }
+      }
+    }
+  }
+  //-----------------------------------------------------------------------
+  // calculate coordinate source terms for Cosmic ray
+  pco->AddCRCoordTermsDivergence(cr,coord_source_);
+}
+
+
+void CRIntegrator::FluxDivergence(const Real wght, AthenaArray<Real> &cr_out) {
+  CosmicRay *pcr=pmy_cr;
+  MeshBlock *pmb = pcr->pmy_block;
+  //  Coordinates *pco = pmb->pcoord;
+
+  AthenaArray<Real> &x1flux=pcr->flux[X1DIR];
+  AthenaArray<Real> &x2flux=pcr->flux[X2DIR];
+  AthenaArray<Real> &x3flux=pcr->flux[X3DIR];
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+  AthenaArray<Real> &x1area = x1face_area_, &x2area = x2face_area_,
+                 &x2area_p1 = x2face_area_p1_, &x3area = x3face_area_,
+                 &x3area_p1 = x3face_area_p1_, &vol = cell_volume_, &dflx = dflx_;
+
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      // calculate x1-flux divergence
+      pmb->pcoord->Face1Area(k,j,is,ie+1,x1area);
+
+      for (int n=0; n<NCR; ++n) {
+#pragma omp simd
+        for (int i=is; i<=ie; ++i) {
+          dflx(n,i) = (x1area(i+1) *x1flux(n,k,j,i+1) - x1area(i)*x1flux(n,k,j,i));
+        }// end n
+      }// End i
+
+      // calculate x2-flux
+      if (pmb->block_size.nx2 > 1) {
+        pmb->pcoord->Face2Area(k,j  ,is,ie,x2area   );
+        pmb->pcoord->Face2Area(k,j+1,is,ie,x2area_p1);
+        for (int n=0; n<NCR; ++n) {
+#pragma omp simd
+          for (int i=is; i<=ie; ++i) {
+            dflx(n,i) += (x2area_p1(i)*x2flux(n,k,j+1,i) - x2area(i)*x2flux(n,k,j,i));
+          }
+        }
+      }// end nx2
+
+      // calculate x3-flux divergence
+      if (pmb->block_size.nx3 > 1) {
+        pmb->pcoord->Face3Area(k  ,j,is,ie,x3area   );
+        pmb->pcoord->Face3Area(k+1,j,is,ie,x3area_p1);
+
+        for (int n=0; n<NCR; ++n) {
+#pragma omp simd
+          for (int i=is; i<=ie; ++i) {
+            dflx(n,i) += (x3area_p1(i)*x3flux(n,k+1,j,i) - x3area(i)*x3flux(n,k,j,i));
+          }
+        }
+      }// end nx3
+      // update variable with flux divergence
+      pmb->pcoord->CellVolume(k,j,is,ie,vol);
+      for (int n=0; n<NCR; ++n) {
+#pragma omp simd
+        for (int i=is; i<=ie; ++i) {
+          cr_out(n,k,j,i) -= wght*dflx(n,i)/vol(i);
+        }
+      }
+    }
+  }
+
+  // Add coordinate source term
+  for (int n=0; n<NCR; ++n)
+    for (int k=ks; k<=ke; ++k)
+      for (int j=js; j<=je; ++j)
+#pragma omp simd
+        for (int i=is; i<=ie; ++i)
+          cr_out(n,k,j,i) += wght * coord_source_(n,k,j,i);
+
+  // check Ec is positive
+  for (int k=ks; k<=ke; ++k)
+    for (int j=js; j<=je; ++j)
+#pragma omp simd
+      for (int i=is; i<=ie; ++i) {
+        if (cr_out(CRE,k,j,i) < TINY_NUMBER)
+          cr_out(CRE,k,j,i) = TINY_NUMBER;
+      }
+}
diff --git a/src/defs.hpp.in b/src/defs.hpp.in
index 79cd87f3f6..565b272dba 100644
--- a/src/defs.hpp.in
+++ b/src/defs.hpp.in
@@ -14,6 +14,7 @@
 // macros which define physics and algorithms
 
 // configure.py dict(definitions) string values:
+
 // problem generator
 #define PROBLEM_GENERATOR "@PROBLEM@"
 
@@ -24,6 +25,7 @@
 #define RIEMANN_SOLVER "@RSOLVER@"
 
 // configure.py dict(definitions) Boolean values:
+
 // Equation of state
 #define EQUATION_OF_STATE "@EQUATION_OF_STATE@"
 
@@ -45,8 +47,14 @@
 // include self gravity? default=0 (false)
 #define SELF_GRAVITY_ENABLED @SELF_GRAVITY_ENABLED@
 
-// include radiative transfer? default=0 (false)
-#define RADIATION_ENABLED 0
+// include nonrelativistic radiative transfer? default=0 (false)
+#define NR_RADIATION_ENABLED @NR_RADIATION_ENABLED@
+
+// include radiative transfer evolved with implicit method? default=0 (false)
+#define IM_RADIATION_ENABLED @IM_RADIATION_ENABLED@
+
+// include cosmic ray transport? default=0 (false)
+#define CR_ENABLED @CR_ENABLED@
 
 // enable special or general relativity? default=0 (false)
 #define RELATIVISTIC_DYNAMICS @RELATIVISTIC_DYNAMICS@
@@ -66,6 +74,7 @@
 
 // configure.py dict(definitions) Boolean string macros:
 // (these options have the latter (false) option as defaults, unless noted otherwise)
+
 // make use of FFT? (FFT or NO_FFT)
 #define @FFT_OPTION@
 
@@ -95,6 +104,9 @@
 #define NWAVE @NWAVE_VALUE@
 #define NSCALARS @NUMBER_PASSIVE_SCALARS@
 #define NGHOST @NUMBER_GHOST_CELLS@
+#define NGRAV @NGRAV_VARIABLES@
+#define NCR @NCR_VARIABLES@   // cosmic ray transport module variable
+#define NRAD @NRAD_VARIABLES@  // Radiation variables for history output
 #define MAX_NSTAGE 6     // maximum number of stages per cycle for time-integrator
 #define MAX_NREGISTER 3  // maximum number of (u, b) register pairs for time-integrator
 
@@ -109,9 +121,12 @@
 #define ONE_3RD 0.33333333333333333
 #define TWO_3RD 0.66666666666666667
 #define TINY_NUMBER 1.0e-20
+#define BIG_NUMBER 1.0e+10
 #define HUGE_NUMBER 1.0e+36
 #define SQR(x) ( (x)*(x) )
 #define SIGN(x) ( ((x) < 0.0) ? -1.0 : 1.0 )
+#define PI_FOUR_POWER 97.409091034002415
+#define ONE_PI_FOUR_POWER 0.010265982254684
 
 #ifdef ENABLE_EXCEPTIONS
 #define ATHENA_ERROR(x) throw std::runtime_error(x.str().c_str())
diff --git a/src/hydro/new_blockdt.cpp b/src/hydro/new_blockdt.cpp
index b8f90fabf7..d510006ed1 100644
--- a/src/hydro/new_blockdt.cpp
+++ b/src/hydro/new_blockdt.cpp
@@ -17,10 +17,13 @@
 #include "../athena.hpp"
 #include "../athena_arrays.hpp"
 #include "../coordinates/coordinates.hpp"
+#include "../cr/cr.hpp"
 #include "../eos/eos.hpp"
 #include "../field/field.hpp"
 #include "../field/field_diffusion/field_diffusion.hpp"
 #include "../mesh/mesh.hpp"
+#include "../nr_radiation/implicit/radiation_implicit.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../orbital_advection/orbital_advection.hpp"
 #include "../scalars/scalars.hpp"
 #include "hydro.hpp"
@@ -59,6 +62,12 @@ void Hydro::NewBlockTimeStep() {
   Real min_dt_parabolic  = real_max;
   Real min_dt_user  = real_max;
 
+  Real cspeed = 0.0;
+  if(NR_RADIATION_ENABLED)
+    cspeed = pmb->pnrrad->reduced_c;
+  if(CR_ENABLED)
+    cspeed = std::max(cspeed,pmb->pcr->vmax);
+
   // TODO(felker): skip this next loop if pm->fluid_setup == FluidFormulation::disabled
   FluidFormulation fluid_status = pmb->pmy_mesh->fluid_setup;
   for (int k=ks; k<=ke; ++k) {
@@ -82,24 +91,30 @@ void Hydro::NewBlockTimeStep() {
               wi[IBY] = bcc(IB2,k,j,i);
               wi[IBZ] = bcc(IB3,k,j,i);
               Real cf = pmb->peos->FastMagnetosonicSpeed(wi,bx);
-              dt1(i) /= (std::abs(wi[IVX]) + cf);
+              Real speed = std::max(cspeed,(std::abs(wi[IVX]) + cf));
+              dt1(i) /= (speed);
 
               wi[IBY] = bcc(IB3,k,j,i);
               wi[IBZ] = bcc(IB1,k,j,i);
               bx = bcc(IB2,k,j,i) + std::abs(b_x2f(k,j,i) - bcc(IB2,k,j,i));
               cf = pmb->peos->FastMagnetosonicSpeed(wi,bx);
-              dt2(i) /= (std::abs(wi[IVY]) + cf);
+              speed = std::max(cspeed,(std::abs(wi[IVY]) + cf));
+              dt2(i) /= (speed);
 
               wi[IBY] = bcc(IB1,k,j,i);
               wi[IBZ] = bcc(IB2,k,j,i);
               bx = bcc(IB3,k,j,i) + std::abs(b_x3f(k,j,i) - bcc(IB3,k,j,i));
               cf = pmb->peos->FastMagnetosonicSpeed(wi,bx);
-              dt3(i) /= (std::abs(wi[IVZ]) + cf);
+              speed = std::max(cspeed,(std::abs(wi[IVZ]) + cf));
+              dt3(i) /= (speed);
             } else {
               Real cs = pmb->peos->SoundSpeed(wi);
-              dt1(i) /= (std::abs(wi[IVX]) + cs);
-              dt2(i) /= (std::abs(wi[IVY]) + cs);
-              dt3(i) /= (std::abs(wi[IVZ]) + cs);
+              Real speed1 = std::max(cspeed, (std::abs(wi[IVX]) + cs));
+              Real speed2 = std::max(cspeed, (std::abs(wi[IVY]) + cs));
+              Real speed3 = std::max(cspeed, (std::abs(wi[IVZ]) + cs));
+              dt1(i) /= (speed1);
+              dt2(i) /= (speed2);
+              dt3(i) /= (speed3);
             }
           } else { // FluidFormulation::background or disabled. Assume scalar advection:
             dt1(i) /= (std::abs(wi[IVX]));
@@ -161,6 +176,11 @@ void Hydro::NewBlockTimeStep() {
   // in the future (with default = cfl_number).
   min_dt_parabolic *= pmb->pmy_mesh->cfl_number;
 
+  if (IM_RADIATION_ENABLED) {
+    min_dt_hyperbolic *= pmb->pmy_mesh->pimrad->cfl_rad;
+    min_dt_parabolic  *= pmb->pmy_mesh->pimrad->cfl_rad;
+  }
+
   // For orbital advection, give a restriction on dt_hyperbolic.
   if (pmb->porb->orbital_advection_active) {
     Real min_dt_orb = pmb->porb->NewOrbitalAdvectionDt();
diff --git a/src/main.cpp b/src/main.cpp
index 04d74465b3..0c62818271 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -39,6 +39,8 @@
 #include "gravity/fft_gravity.hpp"
 #include "gravity/mg_gravity.hpp"
 #include "mesh/mesh.hpp"
+#include "nr_radiation/implicit/radiation_implicit.hpp"
+#include "nr_radiation/radiation.hpp"
 #include "outputs/io_wrapper.hpp"
 #include "outputs/outputs.hpp"
 #include "parameter_input.hpp"
@@ -466,6 +468,9 @@ int main(int argc, char *argv[]) {
         else if (SELF_GRAVITY_ENABLED == 2) // multigrid
           pmesh->pmgrd->Solve(stage);
       }
+      if (IM_RADIATION_ENABLED) {
+        pmesh->pimrad->Iteration(pmesh,ptlist,stage);
+      }
     }
 
     if (STS_ENABLED && pmesh->sts_integrator == "rkl2") {
diff --git a/src/mesh/amr_loadbalance.cpp b/src/mesh/amr_loadbalance.cpp
index bd1d363561..19364164e6 100644
--- a/src/mesh/amr_loadbalance.cpp
+++ b/src/mesh/amr_loadbalance.cpp
@@ -21,6 +21,7 @@
 #include "../field/field.hpp"
 #include "../globals.hpp"
 #include "../hydro/hydro.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../utils/buffer_utils.hpp"
 #include "mesh.hpp"
 #include "mesh_refinement.hpp"
@@ -111,14 +112,6 @@ void Mesh::CalculateLoadBalance(double *clist, int *rlist, int *slist, int *nlis
   }
   nlist[j] = nb-slist[j];
 
-  // if (Globals::my_rank == 0) {
-  //   for (int i=0; i<Globals::nranks; i++) {
-  //     double rcost = 0.0;
-  //     for(int n=slist[i]; n<slist[i]+nlist[i]; n++)
-  //       rcost += clist[n];
-  //   }
-  // }
-
 #ifdef MPI_PARALLEL
   if (nb % (Globals::nranks * num_mesh_threads_) != 0
       && !adaptive && !lb_flag_ && maxcost == mincost && Globals::my_rank == 0) {
@@ -177,9 +170,9 @@ void Mesh::UpdateCostList() {
 
 void Mesh::UpdateMeshBlockTree(int &nnew, int &ndel) {
   // compute nleaf= number of leaf MeshBlocks per refined block
-  int nleaf = 2, dim = 1;
-  if (mesh_size.nx2 > 1) nleaf = 4, dim = 2;
-  if (mesh_size.nx3 > 1) nleaf = 8, dim = 3;
+  int nleaf = 2;
+  if (mesh_size.nx2 > 1) nleaf = 4;
+  if (mesh_size.nx3 > 1) nleaf = 8;
 
   // collect refinement flags from all the meshblocks
   // count the number of the blocks to be (de)refined
@@ -449,6 +442,10 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, int ntot) {
   for (AthenaArray<Real> &var_cc : my_blocks(0)->vars_cc_) {
     nx4_tot += var_cc.GetDim4();
   }
+  // radiation variables are not included in vars_cc as they need different order
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    nx4_tot += my_blocks(0)->pnrrad->ir.GetDim1();
+  }
 
   // cell-centered quantities enrolled in SMR/AMR
   int bssame = bnx1*bnx2*bnx3*nx4_tot;
@@ -474,8 +471,8 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, int ntot) {
     int rb_idx = 0;     // recv buffer index
     for (int n=nbs; n<=nbe; n++) {
       int on = newtoold[n];
-      LogicalLocation &oloc = loclist[on];
-      LogicalLocation &nloc = newloc[n];
+      LogicalLocation const &oloc = loclist[on];
+      LogicalLocation const &nloc = newloc[n];
       if (oloc.level > nloc.level) { // f2c
         for (int l=0; l<nleaf; l++) {
           if (ranklist[on+l] == Globals::my_rank) continue;
@@ -512,7 +509,7 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, int ntot) {
     for (int n=gids_; n<=gide_; n++) {
       int nn = oldtonew[n];
       LogicalLocation &oloc = loclist[n];
-      LogicalLocation &nloc = newloc[nn];
+      LogicalLocation const &nloc = newloc[nn];
       MeshBlock* pb = FindMeshBlock(n);
       if (nloc.level == oloc.level) { // same level
         if (newrank[nn] == Globals::my_rank) continue;
@@ -605,8 +602,8 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, int ntot) {
     int rb_idx = 0;     // recv buffer index
     for (int n=nbs; n<=nbe; n++) {
       int on = newtoold[n];
-      LogicalLocation &oloc = loclist[on];
-      LogicalLocation &nloc = newloc[n];
+      LogicalLocation const &oloc = loclist[on];
+      LogicalLocation const &nloc = newloc[n];
       MeshBlock *pb = FindMeshBlock(n);
       if (oloc.level == nloc.level) { // same
         if (ranklist[on] == Globals::my_rank) continue;
@@ -690,6 +687,12 @@ void Mesh::PrepareSendSameLevel(MeshBlock* pb, Real *sendbuf) {
     BufferUtility::PackData(var_cc, sendbuf, 0, nu,
                             pb->is, pb->ie, pb->js, pb->je, pb->ks, pb->ke, p);
   }
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    int nu = pb->pnrrad->ir.GetDim1() - 1;
+    BufferUtility::PackData(pb->pnrrad->ir, sendbuf, pb->ks, pb->ke,
+                            0,  nu, pb->is, pb->ie, pb->js, pb->je, p);
+  }
+
   for (FaceField &var_fc : pb->vars_fc_) {
     BufferUtility::PackData(var_fc.x1f, sendbuf,
                             pb->is, pb->ie+1, pb->js, pb->je, pb->ks, pb->ke, p);
@@ -732,6 +735,13 @@ void Mesh::PrepareSendCoarseToFineAMR(MeshBlock* pb, Real *sendbuf,
     BufferUtility::PackData(*var_cc, sendbuf, 0, nu,
                             il, iu, jl, ju, kl, ku, p);
   }
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    int nu = pb->pnrrad->ir.GetDim1() - 1;
+    BufferUtility::PackData(pb->pnrrad->ir, sendbuf, kl, ku,
+                            0,  nu, il, iu, jl, ju, p);
+  }
+
+
   for (auto fc_pair : pb->pmr->pvars_fc_) {
     FaceField *var_fc = std::get<0>(fc_pair);
     BufferUtility::PackData((*var_fc).x1f, sendbuf,
@@ -766,6 +776,24 @@ void Mesh::PrepareSendFineToCoarseAMR(MeshBlock* pb, Real *sendbuf) {
                             pb->cjs, pb->cje,
                             pb->cks, pb->cke, p);
   }
+
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    AthenaArray<Real> &var_cc = pb->pnrrad->ir;
+    AthenaArray<Real> &coarse_cc = pb->pnrrad->coarse_ir_;
+    int nu = var_cc.GetDim1() - 1;
+
+    pmr->RestrictCellCenteredValues(var_cc, coarse_cc,-1,
+                                    0, nu,
+                                    pb->cis, pb->cie,
+                                    pb->cjs, pb->cje,
+                                    pb->cks, pb->cke);
+    BufferUtility::PackData(coarse_cc, sendbuf,
+                            pb->cks, pb->cke,
+                            0, nu,
+                            pb->cis, pb->cie,
+                            pb->cjs, pb->cje, p);
+  }
+
   for (auto fc_pair : pb->pmr->pvars_fc_) {
     FaceField *var_fc = std::get<0>(fc_pair);
     FaceField *coarse_fc = std::get<1>(fc_pair);
@@ -824,7 +852,7 @@ void Mesh::FillSameRankFineToCoarseAMR(MeshBlock* pob, MeshBlock* pmb,
                                     pob->cjs, pob->cje,
                                     pob->cks, pob->cke);
     // copy from old/original/other MeshBlock (pob) to newly created block (pmb)
-    AthenaArray<Real> &src = *coarse_cc;
+    AthenaArray<Real> const &src = *coarse_cc;
     AthenaArray<Real> &dst = *std::get<0>(*pmb_cc_it); // pmb->phydro->u;
     for (int nv=0; nv<=nu; nv++) {
       for (int k=kl, fk=pob->cks; fk<=pob->cke; k++, fk++) {
@@ -837,6 +865,33 @@ void Mesh::FillSameRankFineToCoarseAMR(MeshBlock* pob, MeshBlock* pmb,
     pmb_cc_it++;
   }
 
+
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    // restrict from pob block
+    AthenaArray<Real> &var_cc = pob->pnrrad->ir;
+    AthenaArray<Real> &coarse_cc = pob->pnrrad->coarse_ir_;
+    int nu = var_cc.GetDim1() - 1;
+
+    pmr->RestrictCellCenteredValues(var_cc, coarse_cc,-1,
+                                    0, nu,
+                                    pob->cis, pob->cie,
+                                    pob->cjs, pob->cje,
+                                    pob->cks, pob->cke);
+
+    // now copy from pob to new pmb
+    AthenaArray<Real> const &src = coarse_cc;
+    AthenaArray<Real> &dst = pmb->pnrrad->ir;
+
+    for (int k=kl, fk=pob->cks; fk<=pob->cke; k++, fk++) {
+      for (int j=jl, fj=pob->cjs; fj<=pob->cje; j++, fj++) {
+        for (int i=il, fi=pob->cis; fi<=pob->cie; i++, fi++) {
+          for (int nv=0; nv<=nu; nv++)
+            dst(k, j, i, nv) = src(fk, fj, fi, nv);
+        }
+      }
+    }
+  }
+
   auto pmb_fc_it = pmb->pmr->pvars_fc_.begin();
   for (auto fc_pair : pmr->pvars_fc_) {
     FaceField *var_fc = std::get<0>(fc_pair);
@@ -913,7 +968,7 @@ void Mesh::FillSameRankCoarseToFineAMR(MeshBlock* pob, MeshBlock* pmb,
     AthenaArray<Real> *coarse_cc = std::get<1>(cc_pair);
     int nu = var_cc->GetDim4() - 1;
 
-    AthenaArray<Real> &src = *std::get<0>(*pob_cc_it);
+    AthenaArray<Real> const &src = *std::get<0>(*pob_cc_it);
     AthenaArray<Real> &dst = *coarse_cc;
     // fill the coarse buffer
     for (int nv=0; nv<=nu; nv++) {
@@ -929,6 +984,30 @@ void Mesh::FillSameRankCoarseToFineAMR(MeshBlock* pob, MeshBlock* pmb,
         pob->cis, pob->cie, pob->cjs, pob->cje, pob->cks, pob->cke);
     pob_cc_it++;
   }
+
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    // copy from pmb block
+    AthenaArray<Real> &var_cc = pmb->pnrrad->ir;
+    AthenaArray<Real> &coarse_cc = pmb->pnrrad->coarse_ir_;
+    int nu = var_cc.GetDim1() - 1;
+
+    // fill the coarse buffer
+    AthenaArray<Real> &src = pob->pnrrad->ir;
+    AthenaArray<Real> &dst = coarse_cc;
+
+    for (int k=kl, ck=cks; k<=ku; k++, ck++) {
+      for (int j=jl, cj=cjs; j<=ju; j++, cj++) {
+        for (int i=il, ci=cis; i<=iu; i++, ci++) {
+          for (int nv=0; nv<=nu; nv++)
+            dst(k, j, i, nv) = src(ck, cj, ci, nv);
+        }
+      }
+    }
+    pmr->ProlongateCellCenteredValues(
+        dst, var_cc, -1, 0, nu,
+        pob->cis, pob->cie, pob->cjs, pob->cje, pob->cks, pob->cke);
+  }
+
   auto pob_fc_it = pob->pmr->pvars_fc_.begin();
   // iterate MeshRefinement std::vectors on new pmb
   for (auto fc_pair : pmr->pvars_fc_) {
@@ -982,6 +1061,13 @@ void Mesh::FinishRecvSameLevel(MeshBlock *pb, Real *recvbuf) {
     BufferUtility::UnpackData(recvbuf, var_cc, 0, nu,
                               pb->is, pb->ie, pb->js, pb->je, pb->ks, pb->ke, p);
   }
+
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    int nu = pb->pnrrad->ir.GetDim1() - 1;
+    BufferUtility::UnpackData(recvbuf, pb->pnrrad->ir, pb->ks, pb->ke, 0, nu,
+                              pb->is, pb->ie, pb->js, pb->je, p);
+  }
+
   for (FaceField &var_fc : pb->vars_fc_) {
     BufferUtility::UnpackData(recvbuf, var_fc.x1f,
                               pb->is, pb->ie+1, pb->js, pb->je, pb->ks, pb->ke, p);
@@ -1033,6 +1119,13 @@ void Mesh::FinishRecvFineToCoarseAMR(MeshBlock *pb, Real *recvbuf,
     BufferUtility::UnpackData(recvbuf, *var_cc, 0, nu,
                               il, iu, jl, ju, kl, ku, p);
   }
+
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    int nu = pb->pnrrad->ir.GetDim1() - 1;
+    BufferUtility::UnpackData(recvbuf, pb->pnrrad->ir, kl, ku, 0, nu,
+                              il, iu, jl, ju, p);
+  }
+
   for (auto fc_pair : pb->pmr->pvars_fc_) {
     FaceField *var_fc = std::get<0>(fc_pair);
     FaceField &dst_b = *var_fc;
@@ -1075,6 +1168,22 @@ void Mesh::FinishRecvCoarseToFineAMR(MeshBlock *pb, Real *recvbuf) {
         *coarse_cc, *var_cc, 0, nu,
         pb->cis, pb->cie, pb->cjs, pb->cje, pb->cks, pb->cke);
   }
+
+
+  if ((NR_RADIATION_ENABLED|| IM_RADIATION_ENABLED)) {
+    // copy from pmb block
+    AthenaArray<Real> &var_cc = pb->pnrrad->ir;
+    AthenaArray<Real> &coarse_cc = pb->pnrrad->coarse_ir_;
+    int nu = var_cc.GetDim1() - 1;
+
+    BufferUtility::UnpackData(recvbuf, coarse_cc, kl, ku,
+                              0, nu, il, iu, jl, ju, p);
+
+    pmr->ProlongateCellCenteredValues(
+        coarse_cc, var_cc, -1, 0, nu,
+        pb->cis, pb->cie, pb->cjs, pb->cje, pb->cks, pb->cke);
+  }
+
   for (auto fc_pair : pb->pmr->pvars_fc_) {
     FaceField *var_fc = std::get<0>(fc_pair);
     FaceField *coarse_fc = std::get<1>(fc_pair);
@@ -1112,5 +1221,5 @@ void Mesh::FinishRecvCoarseToFineAMR(MeshBlock *pb, Real *recvbuf) {
 
 int Mesh::CreateAMRMPITag(int lid, int ox1, int ox2, int ox3) {
   // former "AthenaTagMPI" AthenaTagMPI::amr=8 redefined to 0
-  return (lid<<8) | (ox1<<7)| (ox2<<6) | (ox3<<5) | 0;
+  return (lid<<8) | (ox1<<7)| (ox2<<6) | (ox3<<5);
 }
diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp
index 8d7fbe05a4..5b4430e1eb 100644
--- a/src/mesh/mesh.cpp
+++ b/src/mesh/mesh.cpp
@@ -30,6 +30,7 @@
 #include "../athena_arrays.hpp"
 #include "../bvals/bvals.hpp"
 #include "../coordinates/coordinates.hpp"
+#include "../cr/cr.hpp"
 #include "../eos/eos.hpp"
 #include "../fft/athena_fft.hpp"
 #include "../fft/turbulence.hpp"
@@ -42,6 +43,8 @@
 #include "../hydro/hydro.hpp"
 #include "../hydro/hydro_diffusion/hydro_diffusion.hpp"
 #include "../multigrid/multigrid.hpp"
+#include "../nr_radiation/implicit/radiation_implicit.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../orbital_advection/orbital_advection.hpp"
 #include "../outputs/io_wrapper.hpp"
 #include "../parameter_input.hpp"
@@ -525,6 +528,10 @@ Mesh::Mesh(ParameterInput *pin, int mesh_test) :
     pmgrd = new MGGravityDriver(this, pin);
   }
 
+  if (IM_RADIATION_ENABLED) {
+    pimrad = new IMRadiation(this, pin);
+  }
+
   // create MeshBlock list for this process
   gids_ = nslist[Globals::my_rank];
   gide_ = gids_ + nblist[Globals::my_rank] - 1;
@@ -845,6 +852,11 @@ Mesh::Mesh(ParameterInput *pin, IOWrapper& resfile, int mesh_test) :
     pmgrd = new MGGravityDriver(this, pin);
   }
 
+  if (IM_RADIATION_ENABLED) {
+    pimrad = new IMRadiation(this, pin);
+  }
+
+
   // allocate data buffer
   int nbmin = nblist[0];
   for (int n = 1; n < Globals::nranks; ++n) {
@@ -882,11 +894,21 @@ Mesh::Mesh(ParameterInput *pin, IOWrapper& resfile, int mesh_test) :
   }
   delete [] mbdata;
   // check consistency
-  if (datasize != my_blocks(0)->GetBlockSizeInBytes()) {
-    msg << "### FATAL ERROR in Mesh constructor" << std::endl
-        << "The restart file is broken or input parameters are inconsistent."
-        << std::endl;
-    ATHENA_ERROR(msg);
+  if ( (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) &&
+                    my_blocks(0)->pnrrad->restart_from_gray > 0) {
+    if (datasize != my_blocks(0)->GetBlockSizeInBytesGray()) {
+        msg << "### FATAL ERROR in Mesh constructor" << std::endl
+            << "The restart file is broken or input parameters are inconsistent."
+            << std::endl;
+        ATHENA_ERROR(msg);
+    }
+  } else {
+    if (datasize != my_blocks(0)->GetBlockSizeInBytes()) {
+        msg << "### FATAL ERROR in Mesh constructor" << std::endl
+            << "The restart file is broken or input parameters are inconsistent."
+            << std::endl;
+        ATHENA_ERROR(msg);
+    }
   }
 
   ResetLoadBalanceVariables();
@@ -911,6 +933,7 @@ Mesh::~Mesh() {
   delete [] loclist;
   if (SELF_GRAVITY_ENABLED == 1) delete pfgrd;
   else if (SELF_GRAVITY_ENABLED == 2) delete pmgrd;
+  if (IM_RADIATION_ENABLED) delete pimrad;
   if (turb_flag > 0) delete ptrbd;
   if (adaptive) { // deallocate arrays for AMR
     delete [] nref;
@@ -1182,6 +1205,59 @@ void Mesh::EnrollUserMGGravitySourceMaskFunction(MGSourceMaskFunc srcmask) {
   return;
 }
 
+
+//----------------------------------------------------------------------------------------
+// radiation related boundaries
+
+void Mesh::EnrollUserRadBoundaryFunction(BoundaryFace dir, RadBoundaryFunc my_bc) {
+  std::stringstream msg;
+  if (dir < 0 || dir > 5) {
+    msg << "### FATAL ERROR in EnrollUserRadBoundaryCondition function" << std::endl
+        << "dirName = " << dir << " not valid" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  if (mesh_bcs[dir] != BoundaryFlag::user) {
+    msg << "### FATAL ERROR in EnrollUserRadBoundaryFunction" << std::endl
+        << "The boundary condition flag must be set to the string 'user' in the "
+        << " <mesh> block in the input file to use user-enrolled BCs" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  RadBoundaryFunc_[static_cast<int>(dir)]=my_bc;
+  return;
+}
+
+//! * provide trivial overloads for old-style BoundaryFace enum argument
+void Mesh::EnrollUserRadBoundaryFunction(int dir, RadBoundaryFunc my_bc) {
+  EnrollUserRadBoundaryFunction(static_cast<BoundaryFace>(dir), my_bc);
+  return;
+}
+
+
+void Mesh::EnrollUserCRBoundaryFunction(BoundaryFace dir, CRBoundaryFunc my_bc) {
+  std::stringstream msg;
+  if (dir < 0 || dir > 5) {
+    msg << "### FATAL ERROR in EnrollUserCRBoundaryCondition function" << std::endl
+        << "dirName = " << dir << " not valid" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  if (mesh_bcs[dir] != BoundaryFlag::user) {
+    msg << "### FATAL ERROR in EnrollUserCRBoundaryFunction" << std::endl
+        << "The boundary condition flag must be set to the string 'user' in the "
+        << " <mesh> block in the input file to use user-enrolled BCs" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  CRBoundaryFunc_[static_cast<int>(dir)]=my_bc;
+  return;
+}
+
+void Mesh::EnrollUserCRBoundaryFunction(int dir, CRBoundaryFunc my_bc) {
+  EnrollUserCRBoundaryFunction(static_cast<BoundaryFace>(dir), my_bc);
+  return;
+}
+
+
+
+
 //----------------------------------------------------------------------------------------
 //! \fn void Mesh::EnrollUserRefinementCondition(AMRFlagFunc amrflag)
 //! \brief Enroll a user-defined function for checking refinement criteria
@@ -1404,6 +1480,8 @@ void Mesh::Initialize(int res_flag, ParameterInput *pin) {
       if (SELF_GRAVITY_ENABLED == 1
         || (SELF_GRAVITY_ENABLED == 2 && pmb->pgrav->fill_ghost))
         pmb->pgrav->gbvar.SetupPersistentMPI();
+      if (IM_RADIATION_ENABLED)
+        pmb->pnrrad->rad_bvar.SetupPersistentMPI();
     }
 
     // solve gravity for the first time
@@ -1426,6 +1504,8 @@ void Mesh::Initialize(int res_flag, ParameterInput *pin) {
         }
         pbval->StartReceivingSubset(BoundaryCommSubset::mesh_init,
                                     pbval->bvars_main_int);
+        if (IM_RADIATION_ENABLED)
+          pmb->pnrrad->rad_bvar.StartReceiving(BoundaryCommSubset::radiation);
       }
 
       // send conserved variables
@@ -1445,8 +1525,15 @@ void Mesh::Initialize(int res_flag, ParameterInput *pin) {
           }
           pmb->pscalars->sbvar.SendBoundaryBuffers();
         }
-      }
 
+
+        if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+          pmb->pnrrad->rad_bvar.SendBoundaryBuffers();
+        }
+        if (CR_ENABLED) {
+          pmb->pcr->cr_bvar.SendBoundaryBuffers();
+        }
+      }
       // wait to receive conserved variables
 #pragma omp for private(pmb,pbval)
       for (int i=0; i<nblocal; ++i) {
@@ -1456,11 +1543,22 @@ void Mesh::Initialize(int res_flag, ParameterInput *pin) {
           pmb->pfield->fbvar.ReceiveAndSetBoundariesWithWait();
         if (NSCALARS > 0)
           pmb->pscalars->sbvar.ReceiveAndSetBoundariesWithWait();
+
+        if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+          pmb->pnrrad->rad_bvar.ReceiveAndSetBoundariesWithWait();
+        if (CR_ENABLED)
+          pmb->pcr->cr_bvar.ReceiveAndSetBoundariesWithWait();
+
         if (shear_periodic && orbital_advection==0) {
           pmb->phydro->hbvar.AddHydroShearForInit();
+
+          if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+            pmb->pnrrad->rad_bvar.AddRadShearForInit();
         }
         pbval->ClearBoundarySubset(BoundaryCommSubset::mesh_init,
                                    pbval->bvars_main_int);
+        if (IM_RADIATION_ENABLED)
+          pmb->pnrrad->rad_bvar.ClearBoundary(BoundaryCommSubset::radiation);
       }
 
       // With AMR/SMR GR send primitives to enable cons->prim before prolongation
@@ -1607,6 +1705,25 @@ void Mesh::Initialize(int res_flag, ParameterInput *pin) {
 
         pbval->ApplyPhysicalBoundaries(time, 0.0, pbval->bvars_main_int);
       }
+      // for radiation, calculate opacity and moments
+      if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+        for (int i=0; i<nblocal; ++i) {
+          pmb = my_blocks(i); ph = pmb->phydro;
+          NRRadiation *prad = pmb->pnrrad;
+          prad->UserFrequency(prad);
+          prad->CalculateMoment(prad->ir);
+          prad->UpdateOpacity(pmb,ph->w);
+        }
+      }
+      // calculate opacity
+      if (CR_ENABLED) {
+        for(int i=0; i<nblocal; ++i) {
+          pmb=my_blocks(i); ph=pmb->phydro;
+          CosmicRay *pcr = pmb->pcr;
+          pf=pmb->pfield;
+          pcr->UpdateOpacity(pmb,pcr->u_cr,ph->w,pf->bcc);
+        }
+      }
 
       // Calc initial diffusion coefficients
 #pragma omp for private(pmb,ph,pf)
@@ -1626,7 +1743,7 @@ void Mesh::Initialize(int res_flag, ParameterInput *pin) {
           my_blocks(i)->pmr->CheckRefinementCondition();
         }
       }
-    } // omp parallel
+    }
 
     if (!res_flag && adaptive) {
       iflag = false;
@@ -1842,6 +1959,9 @@ void Mesh::CorrectMidpointInitialCondition() {
     // no need to re-SetupPersistentMPI() the MPI requests for boundary values
     pbval->StartReceivingSubset(BoundaryCommSubset::mesh_init,
                                 pbval->bvars_main_int);
+
+    if (IM_RADIATION_ENABLED)
+      pmb->pnrrad->rad_bvar.StartReceiving(BoundaryCommSubset::radiation);
   }
 
 #pragma omp for private(pmb,pbval)
@@ -1860,6 +1980,11 @@ void Mesh::CorrectMidpointInitialCondition() {
       }
       pmb->pscalars->sbvar.SendBoundaryBuffers();
     }
+
+    if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+      pmb->pnrrad->rad_bvar.SendBoundaryBuffers();
+    if (CR_ENABLED)
+      pmb->pcr->cr_bvar.SendBoundaryBuffers();
   }
 
   // wait to receive conserved variables
@@ -1878,11 +2003,25 @@ void Mesh::CorrectMidpointInitialCondition() {
       }
       pmb->pscalars->sbvar.ReceiveAndSetBoundariesWithWait();
     }
+
+
+    if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+      pmb->pnrrad->rad_bvar.ReceiveAndSetBoundariesWithWait();
+    if (CR_ENABLED)
+      pmb->pcr->cr_bvar.ReceiveAndSetBoundariesWithWait();
+
+
     if (shear_periodic && orbital_advection==0) {
       pmb->phydro->hbvar.AddHydroShearForInit();
+      if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+        pmb->pnrrad->rad_bvar.AddRadShearForInit();
+      }
     }
     pbval->ClearBoundarySubset(BoundaryCommSubset::mesh_init,
                                pbval->bvars_main_int);
+    if (IM_RADIATION_ENABLED) {
+      pmb->pnrrad->rad_bvar.ClearBoundary(BoundaryCommSubset::radiation);
+    }
   } // end second exchange of ghost cells
   return;
 }
@@ -1934,6 +2073,12 @@ void Mesh::ReserveMeshBlockPhysIDs() {
   if (NSCALARS > 0) {
     ReserveTagPhysIDs(CellCenteredBoundaryVariable::max_phys_id);
   }
+
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+    ReserveTagPhysIDs(RadBoundaryVariable::max_phys_id);
+  if (CR_ENABLED)
+    ReserveTagPhysIDs(CellCenteredBoundaryVariable::max_phys_id);
+
 #endif
   return;
 }
diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp
index de23f1c1f2..0db75cefab 100644
--- a/src/mesh/mesh.hpp
+++ b/src/mesh/mesh.hpp
@@ -25,6 +25,7 @@
 #include "../bvals/bvals.hpp"
 #include "../outputs/io_wrapper.hpp"
 #include "../parameter_input.hpp"
+#include "../task_list/im_rad_task_list.hpp"
 #include "../task_list/task_list.hpp"
 #include "../utils/interp_table.hpp"
 #include "mesh_refinement.hpp"
@@ -39,11 +40,16 @@ class BoundaryValues;
 class CellCenteredBoundaryVariable;
 class FaceCenteredBoundaryVariable;
 class TaskList;
+class IMRadTaskList;
 struct TaskStates;
 class Coordinates;
 class Reconstruction;
 class Hydro;
+class NRRadiation;
+class IMRadiation;
+class CosmicRay;
 class Field;
+class Particles;
 class PassiveScalars;
 class Gravity;
 class MGGravity;
@@ -68,6 +74,7 @@ class MeshBlock {
   friend class Mesh;
   friend class Hydro;
   friend class TaskList;
+  friend class IMRadTaskList;
 #ifdef HDF5OUTPUT
   friend class ATHDF5Output;
 #endif
@@ -89,9 +96,11 @@ class MeshBlock {
   int ncells1, ncells2, ncells3;
   // on 1x coarser level MeshBlock (i.e. ncc2=nx2/2 + 2*NGHOST, if nx2>1)
   int ncc1, ncc2, ncc3;
+  int nfre_ang; //total number of frequency x total number of angles
   int is, ie, js, je, ks, ke;
   int gid, lid;
   int cis, cie, cjs, cje, cks, cke, cnghost;
+  int gflag;
 
   // user output variables for analysis
   int nuser_out_var;
@@ -110,6 +119,8 @@ class MeshBlock {
 
   // physics-related objects (possibly containing their derived bvals classes)
   Hydro *phydro;
+  NRRadiation *pnrrad;
+  CosmicRay *pcr;
   Field *pfield;
   Gravity *pgrav;
   MGGravity* pmg;
@@ -117,8 +128,10 @@ class MeshBlock {
   EquationOfState *peos;
   OrbitalAdvection *porb;
 
+
   // functions
   std::size_t GetBlockSizeInBytes();
+  std::size_t GetBlockSizeInBytesGray();
   int GetNumberOfMeshBlockCells() {
     return block_size.nx1*block_size.nx2*block_size.nx3; }
   void SearchAndSetNeighbors(MeshBlockTree &tree, int *ranklist, int *nslist);
@@ -126,6 +139,11 @@ class MeshBlock {
                   AthenaArray<Real> &u_in1, AthenaArray<Real> &u_in2,
                   AthenaArray<Real> &u_in3, AthenaArray<Real> &u_in4,
                   const Real wght[5]);
+
+  // weightedAve for radiation variable
+  void WeightedAve(AthenaArray<Real> &u_out, AthenaArray<Real> &u_in1,
+                   AthenaArray<Real> &u_in2, const Real wght[3], int flag);
+
   void WeightedAve(FaceField &b_out,
                    FaceField &b_in1, FaceField &b_in2,
                    FaceField &b_in3, FaceField &b_in4,
@@ -185,11 +203,15 @@ class Mesh {
   friend class BoundaryValues;
   friend class CellCenteredBoundaryVariable;
   friend class FaceCenteredBoundaryVariable;
+  friend class RadBoundaryVariable;
   friend class MGBoundaryValues;
   friend class Coordinates;
   friend class MeshRefinement;
   friend class HydroSourceTerms;
   friend class Hydro;
+  friend class NRRadiation;
+  friend class IMRadiation;
+  friend class CosmicRay;
   friend class FFTDriver;
   friend class FFTGravityDriver;
   friend class TurbulenceDriver;
@@ -199,6 +221,7 @@ class Mesh {
   friend class HydroDiffusion;
   friend class FieldDiffusion;
   friend class OrbitalAdvection;
+  friend class Particles;
 #ifdef HDF5OUTPUT
   friend class ATHDF5Output;
 #endif
@@ -242,6 +265,9 @@ class Mesh {
   FFTGravityDriver *pfgrd;
   MGGravityDriver *pmgrd;
 
+  // implicit radiation iteration
+  IMRadiation *pimrad;
+
   AthenaArray<Real> *ruser_mesh_data;
   AthenaArray<int> *iuser_mesh_data;
 
@@ -307,6 +333,10 @@ class Mesh {
   // functions
   MeshGenFunc MeshGenerator_[3];
   BValFunc BoundaryFunction_[6];
+  // radiation boundaries
+  RadBoundaryFunc RadBoundaryFunc_[6];
+  CRBoundaryFunc CRBoundaryFunc_[6];
+
   AMRFlagFunc AMRFlag_;
   SrcTermFunc UserSourceTerm_;
   TimeStepFunc UserTimeStep_;
@@ -356,9 +386,15 @@ class Mesh {
   void EnrollUserBoundaryFunction(BoundaryFace face, BValFunc my_func);
   void EnrollUserMGGravityBoundaryFunction(BoundaryFace dir, MGBoundaryFunc my_bc);
   void EnrollUserMGGravitySourceMaskFunction(MGSourceMaskFunc srcmask);
+
+  void EnrollUserRadBoundaryFunction(BoundaryFace face, RadBoundaryFunc my_func);
+  void EnrollUserCRBoundaryFunction(BoundaryFace face, CRBoundaryFunc my_func);
+
   //! \deprecated (felker):
   //! * provide trivial overload for old-style BoundaryFace enum argument
   void EnrollUserBoundaryFunction(int face, BValFunc my_func);
+  void EnrollUserRadBoundaryFunction(int face, RadBoundaryFunc my_func);
+  void EnrollUserCRBoundaryFunction(int face, CRBoundaryFunc my_func);
 
   void EnrollUserRefinementCondition(AMRFlagFunc amrflag);
   void EnrollUserMeshGenerator(CoordinateDirection dir, MeshGenFunc my_mg);
diff --git a/src/mesh/mesh_refinement.cpp b/src/mesh/mesh_refinement.cpp
index c424769e16..ce682c0621 100644
--- a/src/mesh/mesh_refinement.cpp
+++ b/src/mesh/mesh_refinement.cpp
@@ -121,7 +121,6 @@ void MeshRefinement::RestrictCellCenteredValues(
           pco->CellVolume(k,j+1,si,ei,fvol_[0][1]);
           pco->CellVolume(k+1,j,si,ei,fvol_[1][0]);
           pco->CellVolume(k+1,j+1,si,ei,fvol_[1][1]);
-#pragma ivdep
           for (int ci=csi; ci<=cei; ci++) {
             int i = (ci - pmb->cis)*2 + pmb->is;
             // KGF: add the off-centered quantities first to preserve FP symmetry
@@ -147,7 +146,6 @@ void MeshRefinement::RestrictCellCenteredValues(
         int j = (cj - pmb->cjs)*2 + pmb->js;
         pco->CellVolume(0,j  ,si,ei,fvol_[0][0]);
         pco->CellVolume(0,j+1,si,ei,fvol_[0][1]);
-#pragma ivdep
         for (int ci=csi; ci<=cei; ci++) {
           int i = (ci - pmb->cis)*2 + pmb->is;
           // KGF: add the off-centered quantities first to preserve FP symmetry
@@ -166,7 +164,6 @@ void MeshRefinement::RestrictCellCenteredValues(
     int j = pmb->js, cj = pmb->cjs, k = pmb->ks, ck = pmb->cks;
     for (int n=sn; n<=en; ++n) {
       pco->CellVolume(k,j,si,ei,fvol_[0][0]);
-#pragma ivdep
       for (int ci=csi; ci<=cei; ci++) {
         int i = (ci - pmb->cis)*2 + pmb->is;
         Real tvol = fvol_[0][0](i) + fvol_[0][0](i+1);
@@ -177,6 +174,86 @@ void MeshRefinement::RestrictCellCenteredValues(
   }
 }
 
+
+// over load function  for radiation variables
+
+void MeshRefinement::RestrictCellCenteredValues(
+    const AthenaArray<Real> &fine, AthenaArray<Real> &coarse, int array_order,
+    int sn, int en, int csi, int cei, int csj, int cej, int csk, int cek) {
+  MeshBlock *pmb = pmy_block_;
+  Coordinates *pco = pmb->pcoord;
+  int si = (csi - pmb->cis)*2 + pmb->is, ei = (cei - pmb->cis)*2 + pmb->is + 1;
+
+  // reverse order for radiation variables
+  if (array_order < 0) {
+    // store the restricted data in the prolongation buffer for later use
+    if (pmb->block_size.nx3>1) { // 3D
+      for (int ck=csk; ck<=cek; ck++) {
+        int k = (ck - pmb->cks)*2 + pmb->ks;
+        for (int cj=csj; cj<=cej; cj++) {
+          int j = (cj - pmb->cjs)*2 + pmb->js;
+          pco->CellVolume(k,j,si,ei,fvol_[0][0]);
+          pco->CellVolume(k,j+1,si,ei,fvol_[0][1]);
+          pco->CellVolume(k+1,j,si,ei,fvol_[1][0]);
+          pco->CellVolume(k+1,j+1,si,ei,fvol_[1][1]);
+          for (int ci=csi; ci<=cei; ci++) {
+            int i = (ci - pmb->cis)*2 + pmb->is;
+            Real tvol = ((fvol_[0][0](i) + fvol_[0][1](i))
+                      + (fvol_[0][0](i+1) + fvol_[0][1](i+1)))
+                      + ((fvol_[1][0](i) + fvol_[1][1](i))
+                      + (fvol_[1][0](i+1) + fvol_[1][1](i+1)));
+
+            for (int n=sn; n<=en; ++n) {
+              // KGF: add the off-centered quantities first to preserve FP symmetry
+              // KGF: add the off-centered quantities first to preserve FP symmetry
+              coarse(ck,cj,ci,n) =
+                  (((fine(k  ,j  ,i,n)*fvol_[0][0](i) + fine(k  ,j+1,i,n)*fvol_[0][1](i))
+                    + (fine(k  ,j  ,i+1,n)*fvol_[0][0](i+1) +
+                       fine(k  ,j+1,i+1,n)*fvol_[0][1](i+1)))
+                   + ((fine(k+1,j  ,i,n)*fvol_[1][0](i)
+                       + fine(k+1,j+1,i,n)*fvol_[1][1](i))
+                      + (fine(k+1,j  ,i+1,n)*fvol_[1][0](i+1) +
+                         fine(k+1,j+1,i+1,n)*fvol_[1][1](i+1)))) / tvol;
+            }
+          }// end i
+        }
+      }
+    } else if (pmb->block_size.nx2>1) { // 2D
+      for (int cj=csj; cj<=cej; cj++) {
+        int j = (cj - pmb->cjs)*2 + pmb->js;
+        pco->CellVolume(0,j  ,si,ei,fvol_[0][0]);
+        pco->CellVolume(0,j+1,si,ei,fvol_[0][1]);
+        for (int ci=csi; ci<=cei; ci++) {
+          int i = (ci - pmb->cis)*2 + pmb->is;
+            // KGF: add the off-centered quantities first to preserve FP symmetry
+          Real tvol = (fvol_[0][0](i) + fvol_[0][1](i)) +
+                        (fvol_[0][0](i+1) + fvol_[0][1](i+1));
+          for (int n=sn; n<=en; ++n) {
+            // KGF: add the off-centered quantities first to preserve FP symmetry
+            coarse(0,cj,ci,n) =
+                ((fine(0,j  ,i,n)*fvol_[0][0](i) + fine(0,j+1,i,n)*fvol_[0][1](i))
+                 + (fine(0,j ,i+1,n)*fvol_[0][0](i+1)
+                    + fine(0,j+1,i+1,n)*fvol_[0][1](i+1)))
+                /tvol;
+          }
+        }
+      }
+    } else { // 1D
+      int j = pmb->js, cj = pmb->cjs, k = pmb->ks, ck = pmb->cks;
+
+      pco->CellVolume(k,j,si,ei,fvol_[0][0]);
+      for (int ci=csi; ci<=cei; ci++) {
+        int i = (ci - pmb->cis)*2 + pmb->is;
+        Real tvol = fvol_[0][0](i) + fvol_[0][0](i+1);
+        for (int n=sn; n<=en; ++n) {
+          coarse(ck,cj,ci,n)
+              = (fine(k,j,i,n)*fvol_[0][0](i) + fine(k,j,i+1,n)*fvol_[0][0](i+1))/tvol;
+        }
+      }
+    }
+  }
+}
+
 //----------------------------------------------------------------------------------------
 //! \fn void MeshRefinement::RestrictFieldX1(const AthenaArray<Real> &fine
 //!     AthenaArray<Real> &coarse, int csi, int cei, int csj, int cej, int csk, int cek)
@@ -199,7 +276,6 @@ void MeshRefinement::RestrictFieldX1(
         pco->Face1Area(k,   j+1, si, ei, sarea_x1_[0][1]);
         pco->Face1Area(k+1, j,   si, ei, sarea_x1_[1][0]);
         pco->Face1Area(k+1, j+1, si, ei, sarea_x1_[1][1]);
-#pragma ivdep
         for (int ci=csi; ci<=cei; ci++) {
           int i = (ci - pmb->cis)*2 + pmb->is;
           Real tarea = sarea_x1_[0][0](i) + sarea_x1_[0][1](i) +
@@ -217,7 +293,6 @@ void MeshRefinement::RestrictFieldX1(
       int j = (cj - pmb->cjs)*2 + pmb->js;
       pco->Face1Area(k,  j,   si, ei, sarea_x1_[0][0]);
       pco->Face1Area(k,  j+1, si, ei, sarea_x1_[0][1]);
-#pragma ivdep
       for (int ci=csi; ci<=cei; ci++) {
         int i = (ci - pmb->cis)*2 + pmb->is;
         Real tarea = sarea_x1_[0][0](i) + sarea_x1_[0][1](i);
@@ -226,7 +301,6 @@ void MeshRefinement::RestrictFieldX1(
       }
     }
   } else { // 1D - no restriction, just copy
-#pragma ivdep
     for (int ci=csi; ci<=cei; ci++) {
       int i = (ci - pmb->cis)*2 + pmb->is;
       coarse(csk,csj,ci) = fine(pmb->ks,pmb->js,i);
@@ -265,7 +339,6 @@ void MeshRefinement::RestrictFieldX2(
             sarea_x2_[1][0](i) = pco->dx1f(i);
           }
         }
-#pragma ivdep
         for (int ci=csi; ci<=cei; ci++) {
           int i = (ci - pmb->cis)*2 + pmb->is;
           Real tarea = sarea_x2_[0][0](i) + sarea_x2_[0][0](i+1) +
@@ -290,7 +363,6 @@ void MeshRefinement::RestrictFieldX2(
           sarea_x2_[0][0](i) = pco->dx1f(i);
         }
       }
-#pragma ivdep
       for (int ci=csi; ci<=cei; ci++) {
         int i = (ci - pmb->cis)*2 + pmb->is;
         Real tarea = sarea_x2_[0][0](i) + sarea_x2_[0][0](i+1);
@@ -301,7 +373,6 @@ void MeshRefinement::RestrictFieldX2(
   } else { // 1D
     int k = pmb->ks, j = pmb->js;
     pco->Face2Area(k, j, si, ei, sarea_x2_[0][0]);
-#pragma ivdep
     for (int ci=csi; ci<=cei; ci++) {
       int i = (ci - pmb->cis)*2 + pmb->is;
       Real tarea = sarea_x2_[0][0](i) + sarea_x2_[0][0](i+1);
@@ -333,7 +404,6 @@ void MeshRefinement::RestrictFieldX3(
         int j = (cj - pmb->cjs)*2 + pmb->js;
         pco->Face3Area(k,   j,  si, ei, sarea_x3_[0][0]);
         pco->Face3Area(k, j+1,  si, ei, sarea_x3_[0][1]);
-#pragma ivdep
         for (int ci=csi; ci<=cei; ci++) {
           int i = (ci - pmb->cis)*2 + pmb->is;
           Real tarea = sarea_x3_[0][0](i) + sarea_x3_[0][0](i+1) +
@@ -351,7 +421,6 @@ void MeshRefinement::RestrictFieldX3(
       int j = (cj - pmb->cjs)*2 + pmb->js;
       pco->Face3Area(k,   j, si, ei, sarea_x3_[0][0]);
       pco->Face3Area(k, j+1, si, ei, sarea_x3_[0][1]);
-#pragma ivdep
       for (int ci=csi; ci<=cei; ci++) {
         int i = (ci - pmb->cis)*2 + pmb->is;
         Real tarea = sarea_x3_[0][0](i) + sarea_x3_[0][0](i+1) +
@@ -365,7 +434,6 @@ void MeshRefinement::RestrictFieldX3(
   } else { // 1D
     int k = pmb->ks, j = pmb->js;
     pco->Face3Area(k, j, si, ei, sarea_x3_[0][0]);
-#pragma ivdep
     for (int ci=csi; ci<=cei; ci++) {
       int i = (ci - pmb->cis)*2 + pmb->is;
       Real tarea = sarea_x3_[0][0](i) + sarea_x3_[0][0](i+1);
@@ -412,7 +480,6 @@ void MeshRefinement::ProlongateCellCenteredValues(
           const Real& fx2p = pco->x2v(fj+1);
           Real dx2fm = x2c - fx2m;
           Real dx2fp = fx2p - x2c;
-#pragma ivdep
           for (int i=si; i<=ei; i++) {
             int fi = (i - pmb->cis)*2 + pmb->is;
             const Real& x1m = pcoarsec->x1v(i-1);
@@ -468,7 +535,6 @@ void MeshRefinement::ProlongateCellCenteredValues(
         const Real& fx2p = pco->x2v(fj+1);
         Real dx2fm = x2c - fx2m;
         Real dx2fp = fx2p - x2c;
-#pragma ivdep
         for (int i=si; i<=ei; i++) {
           int fi = (i - pmb->cis)*2 + pmb->is;
           const Real& x1m = pcoarsec->x1v(i-1);
@@ -504,7 +570,6 @@ void MeshRefinement::ProlongateCellCenteredValues(
   } else { // 1D
     int k = pmb->cks, fk = pmb->ks, j = pmb->cjs, fj = pmb->js;
     for (int n=sn; n<=en; n++) {
-#pragma ivdep
       for (int i=si; i<=ei; i++) {
         int fi = (i - pmb->cis)*2 + pmb->is;
         const Real& x1m = pcoarsec->x1v(i-1);
@@ -533,6 +598,166 @@ void MeshRefinement::ProlongateCellCenteredValues(
   return;
 }
 
+
+
+//----------------------------------------------------------------------------------------
+//! \fn void MeshRefinement::ProlongateCellCenteredValues(
+//        const AthenaArray<Real> &coarse,AthenaArray<Real> &fine,
+//      int array_order, int sn, int en,,
+//        int si, int ei, int sj, int ej, int sk, int ek)
+//  \brief Prolongate cell centered values for radiation variables
+
+void MeshRefinement::ProlongateCellCenteredValues(
+    const AthenaArray<Real> &coarse, AthenaArray<Real> &fine,
+    int array_order,
+    int sn, int en, int si, int ei, int sj, int ej, int sk, int ek) {
+  MeshBlock *pmb = pmy_block_;
+  Coordinates *pco = pmb->pcoord;
+
+  if (array_order < 0) {
+    if (pmb->block_size.nx3 > 1) {
+      for (int k=sk; k<=ek; k++) {
+        int fk = (k - pmb->cks)*2 + pmb->ks;
+        const Real& x3m = pcoarsec->x3v(k-1);
+        const Real& x3c = pcoarsec->x3v(k);
+        const Real& x3p = pcoarsec->x3v(k+1);
+        Real dx3m = x3c - x3m;
+        Real dx3p = x3p - x3c;
+        const Real& fx3m = pco->x3v(fk);
+        const Real& fx3p = pco->x3v(fk+1);
+        Real dx3fm =  x3c - fx3m;
+        Real dx3fp =  fx3p - x3c;
+        for (int j = sj; j<=ej; j++) {
+          int fj = (j - pmb->cjs)*2 + pmb->js;
+          const Real& x2m = pcoarsec->x2v(j-1);
+          const Real& x2c = pcoarsec->x2v(j);
+          const Real& x2p = pcoarsec->x2v(j+1);
+          Real dx2m = x2c - x2m;
+          Real dx2p = x2p - x2c;
+          const Real& fx2m = pco->x2v(fj);
+          const Real& fx2p = pco->x2v(fj+1);
+          Real dx2fm = x2c - fx2m;
+          Real dx2fp = fx2p - x2c;
+          for (int i=si; i<=ei; i++) {
+            int fi = (i - pmb->cis)*2 + pmb->is;
+            const Real& x1m = pcoarsec->x1v(i-1);
+            const Real& x1c = pcoarsec->x1v(i);
+            const Real& x1p = pcoarsec->x1v(i+1);
+            Real dx1m = x1c - x1m;
+            Real dx1p = x1p - x1c;
+            const Real& fx1m = pco->x1v(fi);
+            const Real& fx1p = pco->x1v(fi+1);
+            Real dx1fm = x1c - fx1m;
+            Real dx1fp = fx1p - x1c;
+
+            for (int n=sn; n<=en; n++) {
+              Real ccval = coarse(k,j,i,n);
+              // calculate 3D gradients using the minmod limiter
+              Real gx1m = (ccval - coarse(k,j,i-1,n))/dx1m;
+              Real gx1p = (coarse(k,j,i+1,n) - ccval)/dx1p;
+              Real gx1c = 0.5*(SIGN(gx1m) + SIGN(gx1p))*
+                          std::min(std::abs(gx1m), std::abs(gx1p));
+              Real gx2m = (ccval - coarse(k,j-1,i,n))/dx2m;
+              Real gx2p = (coarse(k,j+1,i,n) - ccval)/dx2p;
+              Real gx2c = 0.5*(SIGN(gx2m) + SIGN(gx2p))*
+                          std::min(std::abs(gx2m), std::abs(gx2p));
+              Real gx3m = (ccval - coarse(k-1,j,i,n))/dx3m;
+              Real gx3p = (coarse(k+1,j,i,n) - ccval)/dx3p;
+              Real gx3c = 0.5*(SIGN(gx3m) + SIGN(gx3p))*
+                          std::min(std::abs(gx3m), std::abs(gx3p));
+
+              // KGF: add the off-centered quantities first to preserve FP symmetry
+              // interpolate onto the finer grid
+              fine(fk  ,fj  ,fi  ,n) = ccval - (gx1c*dx1fm + gx2c*dx2fm + gx3c*dx3fm);
+              fine(fk  ,fj  ,fi+1,n) = ccval + (gx1c*dx1fp - gx2c*dx2fm - gx3c*dx3fm);
+              fine(fk  ,fj+1,fi  ,n) = ccval - (gx1c*dx1fm - gx2c*dx2fp + gx3c*dx3fm);
+              fine(fk  ,fj+1,fi+1,n) = ccval + (gx1c*dx1fp + gx2c*dx2fp - gx3c*dx3fm);
+              fine(fk+1,fj  ,fi  ,n) = ccval - (gx1c*dx1fm + gx2c*dx2fm - gx3c*dx3fp);
+              fine(fk+1,fj  ,fi+1,n) = ccval + (gx1c*dx1fp - gx2c*dx2fm + gx3c*dx3fp);
+              fine(fk+1,fj+1,fi  ,n) = ccval - (gx1c*dx1fm - gx2c*dx2fp - gx3c*dx3fp);
+              fine(fk+1,fj+1,fi+1,n) = ccval + (gx1c*dx1fp + gx2c*dx2fp + gx3c*dx3fp);
+            }
+          }
+        }
+      }
+    } else if (pmb->block_size.nx2 > 1) {
+      int k = pmb->cks, fk = pmb->ks;
+      for (int j=sj; j<=ej; j++) {
+        int fj = (j - pmb->cjs)*2 + pmb->js;
+        const Real& x2m = pcoarsec->x2v(j-1);
+        const Real& x2c = pcoarsec->x2v(j);
+        const Real& x2p = pcoarsec->x2v(j+1);
+        Real dx2m = x2c - x2m;
+        Real dx2p = x2p - x2c;
+        const Real& fx2m = pco->x2v(fj);
+        const Real& fx2p = pco->x2v(fj+1);
+        Real dx2fm = x2c - fx2m;
+        Real dx2fp = fx2p - x2c;
+        for (int i=si; i<=ei; i++) {
+          int fi = (i - pmb->cis)*2 + pmb->is;
+          const Real& x1m = pcoarsec->x1v(i-1);
+          const Real& x1c = pcoarsec->x1v(i);
+          const Real& x1p = pcoarsec->x1v(i+1);
+          Real dx1m = x1c - x1m;
+          Real dx1p = x1p - x1c;
+          const Real& fx1m = pco->x1v(fi);
+          const Real& fx1p = pco->x1v(fi+1);
+          Real dx1fm = x1c - fx1m;
+          Real dx1fp = fx1p - x1c;
+          for (int n=sn; n<=en; n++) {
+            Real ccval = coarse(k,j,i,n);
+
+            // calculate 2D gradients using the minmod limiter
+            Real gx1m = (ccval - coarse(k,j,i-1,n))/dx1m;
+            Real gx1p = (coarse(k,j,i+1,n) - ccval)/dx1p;
+            Real gx1c = 0.5*(SIGN(gx1m) + SIGN(gx1p))*
+                        std::min(std::abs(gx1m), std::abs(gx1p));
+            Real gx2m = (ccval - coarse(k,j-1,i,n))/dx2m;
+            Real gx2p = (coarse(k,j+1,i,n) - ccval)/dx2p;
+            Real gx2c = 0.5*(SIGN(gx2m) + SIGN(gx2p))*
+                        std::min(std::abs(gx2m), std::abs(gx2p));
+
+            // KGF: add the off-centered quantities first to preserve FP symmetry
+            // interpolate onto the finer grid
+            fine(fk  ,fj  ,fi  ,n) = ccval - (gx1c*dx1fm + gx2c*dx2fm);
+            fine(fk  ,fj  ,fi+1,n) = ccval + (gx1c*dx1fp - gx2c*dx2fm);
+            fine(fk  ,fj+1,fi  ,n) = ccval - (gx1c*dx1fm - gx2c*dx2fp);
+            fine(fk  ,fj+1,fi+1,n) = ccval + (gx1c*dx1fp + gx2c*dx2fp);
+          }
+        }
+      }
+    } else { // 1D
+      int k = pmb->cks, fk = pmb->ks, j = pmb->cjs, fj = pmb->js;
+
+      for (int i=si; i<=ei; i++) {
+        int fi = (i - pmb->cis)*2 + pmb->is;
+        const Real& x1m = pcoarsec->x1v(i-1);
+        const Real& x1c = pcoarsec->x1v(i);
+        const Real& x1p = pcoarsec->x1v(i+1);
+        Real dx1m = x1c - x1m;
+        Real dx1p = x1p - x1c;
+        const Real& fx1m = pco->x1v(fi);
+        const Real& fx1p = pco->x1v(fi+1);
+        Real dx1fm = x1c - fx1m;
+        Real dx1fp = fx1p - x1c;
+        for (int n=sn; n<=en; n++) {
+          Real ccval = coarse(k,j,i,n);
+          // calculate 1D gradient using the min-mod limiter
+          Real gx1m = (ccval - coarse(k,j,i-1,n))/dx1m;
+          Real gx1p = (coarse(k,j,i+1,n) - ccval)/dx1p;
+          Real gx1c = 0.5*(SIGN(gx1m) + SIGN(gx1p))*std::min(std::abs(gx1m),
+                                                             std::abs(gx1p));
+          // interpolate on to the finer grid
+          fine(fk  ,fj  ,fi  ,n) = ccval - gx1c*dx1fm;
+          fine(fk  ,fj  ,fi+1,n) = ccval + gx1c*dx1fp;
+        }
+      }
+    }
+  }
+  return;
+}
+
+
 //----------------------------------------------------------------------------------------
 //! \fn void MeshRefinement::ProlongateSharedFieldX1(const AthenaArray<Real> &coarse,
 //!     AthenaArray<Real> &fine, int si, int ei, int sj, int ej, int sk, int ek)
@@ -562,7 +787,6 @@ void MeshRefinement::ProlongateSharedFieldX1(
         Real dx2p = x2p - x2c;
         const Real& fx2m = pco->x2s1(fj);
         const Real& fx2p = pco->x2s1(fj+1);
-#pragma ivdep
         for (int i=si; i<=ei; i++) {
           int fi = (i - pmb->cis)*2 + pmb->is;
           Real ccval = coarse(k,j,i);
@@ -594,7 +818,6 @@ void MeshRefinement::ProlongateSharedFieldX1(
       Real dx2p = x2p - x2c;
       const Real& fx2m = pco->x2s1(fj);
       const Real& fx2p = pco->x2s1(fj+1);
-#pragma ivdep
       for (int i=si; i<=ei; i++) {
         int fi = (i - pmb->cis)*2 + pmb->is;
         Real ccval = coarse(k,j,i);
@@ -609,7 +832,6 @@ void MeshRefinement::ProlongateSharedFieldX1(
       }
     }
   } else { // 1D
-#pragma ivdep
     for (int i=si; i<=ei; i++) {
       int fi = (i - pmb->cis)*2 + pmb->is;
       fine(0,0,fi) = coarse(0,0,i);
@@ -640,7 +862,6 @@ void MeshRefinement::ProlongateSharedFieldX2(
       const Real& fx3p = pco->x3s2(fk+1);
       for (int j=sj; j<=ej; j++) {
         int fj = (j - pmb->cjs)*2 + pmb->js;
-#pragma ivdep
         for (int i=si; i<=ei; i++) {
           int fi = (i - pmb->cis)*2 + pmb->is;
           const Real& x1m = pcoarsec->x1s2(i-1);
@@ -672,7 +893,6 @@ void MeshRefinement::ProlongateSharedFieldX2(
     int k = pmb->cks, fk = pmb->ks;
     for (int j=sj; j<=ej; j++) {
       int fj = (j - pmb->cjs)*2 + pmb->js;
-#pragma ivdep
       for (int i=si; i<=ei; i++) {
         int fi = (i - pmb->cis)*2 + pmb->is;
         const Real& x1m = pcoarsec->x1s2(i-1);
@@ -692,7 +912,6 @@ void MeshRefinement::ProlongateSharedFieldX2(
       }
     }
   } else {
-#pragma ivdep
     for (int i=si; i<=ei; i++) {
       int fi = (i - pmb->cis)*2 + pmb->is;
       Real gxm = (coarse(0,0,i) - coarse(0,0,i-1))
@@ -732,7 +951,6 @@ void MeshRefinement::ProlongateSharedFieldX3(
         Real dx2p = x2p - x2c;
         const Real& fx2m = pco->x2s3(fj);
         const Real& fx2p = pco->x2s3(fj+1);
-#pragma ivdep
         for (int i=si; i<=ei; i++) {
           int fi = (i - pmb->cis)*2 + pmb->is;
           const Real& x1m = pcoarsec->x1s3(i-1);
@@ -773,7 +991,6 @@ void MeshRefinement::ProlongateSharedFieldX3(
       const Real& fx2p = pco->x2s3(fj+1);
       Real dx2fm = x2c - fx2m;
       Real dx2fp = fx2p - x2c;
-#pragma ivdep
       for (int i=si; i<=ei; i++) {
         int fi = (i - pmb->cis)*2 + pmb->is;
         const Real& x1m = pcoarsec->x1s3(i-1);
@@ -805,7 +1022,6 @@ void MeshRefinement::ProlongateSharedFieldX3(
       }
     }
   } else {
-#pragma ivdep
     for (int i=si; i<=ei; i++) {
       int fi = (i - pmb->cis)*2 + pmb->is;
       Real gxm = (coarse(0,0,i)   - coarse(0,0,i-1))
@@ -854,7 +1070,6 @@ void MeshRefinement::ProlongateInternalField(
         pco->Face3Area(fk+1, fj+1, fsi, fei,   sarea_x3_[1][1]);
         pco->Face3Area(fk+2, fj,   fsi, fei,   sarea_x3_[2][0]);
         pco->Face3Area(fk+2, fj+1, fsi, fei,   sarea_x3_[2][1]);
-#pragma ivdep
         for (int i=si; i<=ei; i++) {
           int fi = (i - pmb->cis)*2 + pmb->is;
           Real Uxx = 0.0, Vyy = 0.0, Wzz = 0.0;
@@ -956,7 +1171,6 @@ void MeshRefinement::ProlongateInternalField(
       pco->Face2Area(fk,   fj,   fsi, fei,   sarea_x2_[0][0]);
       pco->Face2Area(fk,   fj+1, fsi, fei,   sarea_x2_[0][1]);
       pco->Face2Area(fk,   fj+2, fsi, fei,   sarea_x2_[0][2]);
-#pragma ivdep
       for (int i=si; i<=ei; i++) {
         int fi = (i - pmb->cis)*2 + pmb->is;
         Real tmp1 = 0.25*(fine.x2f(fk,fj+2,fi+1)*sarea_x2_[0][2](fi+1)
@@ -987,7 +1201,6 @@ void MeshRefinement::ProlongateInternalField(
     }
   } else {
     pco->Face1Area(0, 0, fsi, fei+1, sarea_x1_[0][0]);
-#pragma ivdep
     for (int i=si; i<=ei; i++) {
       int fi = (i - pmb->cis)*2 + pmb->is;
       Real ph = sarea_x1_[0][0](fi)*fine.x1f(0,0,fi);
diff --git a/src/mesh/mesh_refinement.hpp b/src/mesh/mesh_refinement.hpp
index a618793ca7..f68a9da932 100644
--- a/src/mesh/mesh_refinement.hpp
+++ b/src/mesh/mesh_refinement.hpp
@@ -52,6 +52,12 @@ class MeshRefinement {
   void RestrictCellCenteredValues(const AthenaArray<Real> &fine,
                                   AthenaArray<Real> &coarse, int sn, int en,
                                   int csi, int cei, int csj, int cej, int csk, int cek);
+
+  // over-write function for radiation variables
+  void RestrictCellCenteredValues(
+      const AthenaArray<Real> &fine, AthenaArray<Real> &coarse, int array_order,
+      int sn, int en, int csi, int cei, int csj, int cej, int csk, int cek);
+
   void RestrictFieldX1(const AthenaArray<Real> &fine, AthenaArray<Real> &coarse,
                        int csi, int cei, int csj, int cej, int csk, int cek);
   void RestrictFieldX2(const AthenaArray<Real> &fine, AthenaArray<Real> &coarse,
@@ -61,6 +67,12 @@ class MeshRefinement {
   void ProlongateCellCenteredValues(const AthenaArray<Real> &coarse,
                                     AthenaArray<Real> &fine, int sn, int en,
                                     int si, int ei, int sj, int ej, int sk, int ek);
+
+  void ProlongateCellCenteredValues(const AthenaArray<Real> &coarse,
+                                    AthenaArray<Real> &fine, int array_order,
+                                    int sn, int en, int si, int ei, int sj, int ej,
+                                    int sk, int ek);
+
   void ProlongateSharedFieldX1(const AthenaArray<Real> &coarse, AthenaArray<Real> &fine,
                                int si, int ei, int sj, int ej, int sk, int ek);
   void ProlongateSharedFieldX2(const AthenaArray<Real> &coarse, AthenaArray<Real> &fine,
diff --git a/src/mesh/meshblock.cpp b/src/mesh/meshblock.cpp
index 6031356692..f5ae19132b 100644
--- a/src/mesh/meshblock.cpp
+++ b/src/mesh/meshblock.cpp
@@ -24,6 +24,7 @@
 #include "../athena_arrays.hpp"
 #include "../bvals/bvals.hpp"
 #include "../coordinates/coordinates.hpp"
+#include "../cr/cr.hpp"
 #include "../eos/eos.hpp"
 #include "../fft/athena_fft.hpp"
 #include "../field/field.hpp"
@@ -31,6 +32,9 @@
 #include "../gravity/gravity.hpp"
 #include "../gravity/mg_gravity.hpp"
 #include "../hydro/hydro.hpp"
+#include "../nr_radiation/implicit/radiation_implicit.hpp"
+#include "../nr_radiation/integrators/rad_integrators.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../orbital_advection/orbital_advection.hpp"
 #include "../parameter_input.hpp"
 #include "../reconstruct/reconstruction.hpp"
@@ -57,11 +61,13 @@ MeshBlock::MeshBlock(int igid, int ilid, LogicalLocation iloc, RegionSize input_
 
   ncells1 = block_size.nx1 + 2*NGHOST;
   ncc1 = block_size.nx1/2 + 2*NGHOST;
+  int ndim = 1;
   if (pmy_mesh->f2) {
     js = NGHOST;
     je = js + block_size.nx2 - 1;
     ncells2 = block_size.nx2 + 2*NGHOST;
     ncc2 = block_size.nx2/2 + 2*NGHOST;
+    ndim = 2;
   } else {
     js = je = 0;
     ncells2 = 1;
@@ -73,6 +79,7 @@ MeshBlock::MeshBlock(int igid, int ilid, LogicalLocation iloc, RegionSize input_
     ke = ks + block_size.nx3 - 1;
     ncells3 = block_size.nx3 + 2*NGHOST;
     ncc3 = block_size.nx3/2 + 2*NGHOST;
+    ndim = 3;
   } else {
     ks = ke = 0;
     ncells3 = 1;
@@ -118,6 +125,62 @@ MeshBlock::MeshBlock(int igid, int ilid, LogicalLocation iloc, RegionSize input_
     pcoord = new GRUser(this, pin, false);
   }
 
+
+//=================================================================
+//set the total number of frequency x angles
+
+  nfre_ang = 0;
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    int nfreq = pin->GetOrAddInteger("radiation","n_frequency",1);
+    int nmu = pin->GetInteger("radiation","nmu");
+    int nzeta = pin->GetOrAddInteger("radiation","nzeta",0);
+    // total number of azimuthal angles covering 0 to pi
+    int npsi = pin->GetOrAddInteger("radiation","npsi",0);
+    int angle_flag = pin->GetOrAddInteger("radiation","angle_flag",0);
+    int n_ang=1; // number of angles per octant and number of octant
+    int noct=2;
+    // calculate total number of angles based on dimensions
+    if (angle_flag == 1) {
+      if (ndim == 1) {
+        noct = 2;
+        n_ang = nzeta;
+      } else if (ndim == 2) {
+        if (npsi <= 1) {
+          n_ang = nzeta;
+        } else if (nzeta == 0) {
+          n_ang = npsi/2;
+        } else {
+          n_ang = nzeta*npsi;
+        }
+        noct = 4;
+      } else if (ndim == 3) {
+        n_ang = nzeta*npsi/2;
+        noct = 8;
+      }
+
+    } else {
+      if (ndim == 1) {
+        n_ang = nmu;
+        noct = 2;
+      } else if (ndim == 2) {
+        noct = 4;
+        if (angle_flag == 0) {
+          n_ang = nmu * (nmu + 1)/2;
+        } else if (angle_flag == 10) {
+          n_ang = nmu;
+        }
+      } else if (ndim == 3) {
+        noct = 8;
+        if (angle_flag == 0) {
+          n_ang = nmu * (nmu + 1)/2;
+        } else if (angle_flag == 10) {
+          n_ang = nmu * nmu/2;
+        }
+      }
+    }
+    nfre_ang = n_ang * noct * nfreq;
+  }
+  //========================================================
   // Reconstruction: constructor may implicitly depend on Coordinates, and PPM variable
   // floors depend on EOS, but EOS isn't needed in Reconstruction constructor-> this is ok
   precon = new Reconstruction(this, pin);
@@ -175,6 +238,19 @@ MeshBlock::MeshBlock(int igid, int ilid, LogicalLocation iloc, RegionSize input_
 
   peos = new EquationOfState(this, pin);
 
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+       //radiation constructor needs the parameter nfre_ang
+    pnrrad = new NRRadiation(this, pin);
+  }
+  if (NR_RADIATION_ENABLED) {
+    pbval->AdvanceCounterPhysID(RadBoundaryVariable::max_phys_id);
+  }
+
+  if (CR_ENABLED) {
+    pcr = new CosmicRay(this, pin);
+    pbval->AdvanceCounterPhysID(CellCenteredBoundaryVariable::max_phys_id);
+  }
+
   // OrbitalAdvection: constructor depends on Coordinates, Hydro, Field, PassiveScalars.
   porb = new OrbitalAdvection(this, pin);
 
@@ -201,11 +277,13 @@ MeshBlock::MeshBlock(int igid, int ilid, Mesh *pm, ParameterInput *pin,
 
   ncells1 = block_size.nx1 + 2*NGHOST;
   ncc1 = block_size.nx1/2 + 2*NGHOST;
+  int ndim = 1;
   if (pmy_mesh->f2) {
     js = NGHOST;
     je = js + block_size.nx2 - 1;
     ncells2 = block_size.nx2 + 2*NGHOST;
     ncc2 = block_size.nx2/2 + 2*NGHOST;
+    ndim = 2;
   } else {
     js = je = 0;
     ncells2 = 1;
@@ -217,6 +295,7 @@ MeshBlock::MeshBlock(int igid, int ilid, Mesh *pm, ParameterInput *pin,
     ke = ks + block_size.nx3 - 1;
     ncells3 = block_size.nx3 + 2*NGHOST;
     ncc3 = block_size.nx3/2 + 2*NGHOST;
+    ndim = 3;
   } else {
     ks = ke = 0;
     ncells3 = 1;
@@ -255,6 +334,65 @@ MeshBlock::MeshBlock(int igid, int ilid, Mesh *pm, ParameterInput *pin,
     pcoord = new GRUser(this, pin, false);
   }
 
+
+  //======================================================================
+  // radiation constructor needs to be done before reconstruction
+  //======================================================================
+
+  nfre_ang = 0;
+
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    int nfreq = pin->GetOrAddInteger("radiation","n_frequency",1);
+    int nmu = pin->GetInteger("radiation","nmu");
+    int nzeta = pin->GetOrAddInteger("radiation","nzeta",0);
+    // total number of azimuthal angles covering 0 to pi
+    int npsi = pin->GetOrAddInteger("radiation","npsi",0);
+    int angle_flag = pin->GetOrAddInteger("radiation","angle_flag",0);
+    int n_ang=1; // number of angles per octant and number of octant
+    int noct=2;
+
+    // calculate total number of angles based on dimensions
+    if (angle_flag == 1) {
+      if (ndim == 1) {
+        noct = 2;
+        n_ang = nzeta;
+      } else if (ndim == 2) {
+        if (npsi <= 1) {
+          n_ang = nzeta;
+        } else if (nzeta == 0) {
+          n_ang = npsi/2;
+        } else {
+          n_ang = nzeta*npsi;
+        }
+        noct = 4;
+      } else if (ndim == 3) {
+        n_ang = nzeta*npsi/2;
+        noct = 8;
+      }
+
+    } else {
+      if (ndim == 1) {
+        n_ang = nmu;
+        noct = 2;
+      } else if (ndim == 2) {
+        noct = 4;
+        if (angle_flag == 0) {
+          n_ang = nmu * (nmu + 1)/2;
+        } else if (angle_flag == 10) {
+          n_ang = nmu;
+        }
+      } else if (ndim == 3) {
+        noct = 8;
+        if (angle_flag == 0) {
+          n_ang = nmu * (nmu + 1)/2;
+        } else if (angle_flag == 10) {
+          n_ang = nmu * nmu/2;
+        }
+      }
+    }
+    nfre_ang = n_ang * noct * nfreq;
+  }
+
   // Reconstruction (constructor may implicitly depend on Coordinates)
   precon = new Reconstruction(this, pin);
 
@@ -292,6 +430,21 @@ MeshBlock::MeshBlock(int igid, int ilid, Mesh *pm, ParameterInput *pin,
 
   peos = new EquationOfState(this, pin);
 
+
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+       //radiation constructor needs the parameter nfre_ang
+    pnrrad = new NRRadiation(this, pin);
+  }
+  if (NR_RADIATION_ENABLED) {
+    pbval->AdvanceCounterPhysID(RadBoundaryVariable::max_phys_id);
+  }
+
+  if (CR_ENABLED) {
+    pcr = new CosmicRay(this, pin);
+    pbval->AdvanceCounterPhysID(CellCenteredBoundaryVariable::max_phys_id);
+  }
+
+
   // OrbitalAdvection: constructor depends on Coordinates, Hydro, Field, PassiveScalars.
   porb = new OrbitalAdvection(this, pin);
 
@@ -318,12 +471,60 @@ MeshBlock::MeshBlock(int igid, int ilid, Mesh *pm, ParameterInput *pin,
     os += pfield->b.x3f.GetSizeInBytes();
   }
 
+
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    if (pnrrad->restart_from_gray) {
+      std::memcpy(pnrrad->ir_gray.data(), &(mbdata[os]),
+                  pnrrad->ir_gray.GetSizeInBytes());
+      AthenaArray<Real> fre_ratio;
+
+      fre_ratio.NewAthenaArray(pnrrad->nfreq);
+      // Real invcrat = 1.0/pnrrad->crat;
+      for (int k=ks; k<=ke; ++k) {
+        for (int j=js; j<=je; ++j) {
+          for (int i=is; i<=ie; ++i) {
+            Real Er = 0.0;
+            for (int n=0; n<pnrrad->nang; ++n) {
+              Er += pnrrad->ir_gray(k,j,i,n) * pnrrad->wmu(n);
+            }
+            Real tr = std::pow(Er,0.25);
+
+            // now split to different frequency bins
+            for (int ifr=0; ifr<pnrrad->nfreq; ++ifr) {
+              fre_ratio(ifr) = pnrrad->IntPlanckFunc(pnrrad->nu_grid(ifr)/tr,
+                                      pnrrad->nu_grid(ifr+1)/tr);
+            }
+
+            for (int ifr=0; ifr<pnrrad->nfreq; ++ifr) {
+              for (int n=0; n<pnrrad->nang; ++n) {
+                pnrrad->ir(k,j,i,ifr*pnrrad->nang+n) = pnrrad->ir_gray(k,j,i,n)
+                                                       * fre_ratio(ifr);
+              }
+            }
+          }
+        }
+      }
+      fre_ratio.DeleteAthenaArray();
+      os += pnrrad->ir_gray.GetSizeInBytes();
+    } else {
+      std::memcpy(pnrrad->ir.data(), &(mbdata[os]), pnrrad->ir.GetSizeInBytes());
+      os += pnrrad->ir.GetSizeInBytes();
+    }
+    //    std::memcpy(prad->ir1.data(), &(mbdata[os]), prad->ir1.GetSizeInBytes());
+    // copy the data
+    pnrrad->ir1 = pnrrad->ir;
+  }
+
+  if (CR_ENABLED) {
+    std::memcpy(pcr->u_cr.data(), &(mbdata[os]), pcr->u_cr.GetSizeInBytes());
+    std::memcpy(pcr->u_cr1.data(), &(mbdata[os]), pcr->u_cr1.GetSizeInBytes());
+    os += pcr->u_cr.GetSizeInBytes();
+  }
   // (conserved variable) Passive scalars:
   if (NSCALARS > 0) {
     std::memcpy(pscalars->s.data(), &(mbdata[os]), pscalars->s.GetSizeInBytes());
     os += pscalars->s.GetSizeInBytes();
   }
-
   // load user MeshBlock data
   for (int n=0; n<nint_user_meshblock_data_; n++) {
     std::memcpy(iuser_meshblock_data[n].data(), &(mbdata[os]),
@@ -335,7 +536,6 @@ MeshBlock::MeshBlock(int igid, int ilid, Mesh *pm, ParameterInput *pin,
                 ruser_meshblock_data[n].GetSizeInBytes());
     os += ruser_meshblock_data[n].GetSizeInBytes();
   }
-
   return;
 }
 
@@ -355,6 +555,9 @@ MeshBlock::~MeshBlock() {
   if (SELF_GRAVITY_ENABLED == 2) delete pmg;
   if (NSCALARS > 0) delete pscalars;
 
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) delete pnrrad;
+  if (CR_ENABLED) delete pcr;
+
   // BoundaryValues should be destructed AFTER all BoundaryVariable objects are destroyed
   delete pbval;
   // delete user output variables array
@@ -453,6 +656,11 @@ std::size_t MeshBlock::GetBlockSizeInBytes() {
   if (NSCALARS > 0)
     size += pscalars->s.GetSizeInBytes();
 
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+    size += pnrrad->ir.GetSizeInBytes();
+  if (CR_ENABLED)
+    size += pcr->u_cr.GetSizeInBytes();
+
   // calculate user MeshBlock data size
   for (int n=0; n<nint_user_meshblock_data_; n++)
     size += iuser_meshblock_data[n].GetSizeInBytes();
@@ -462,6 +670,42 @@ std::size_t MeshBlock::GetBlockSizeInBytes() {
   return size;
 }
 
+
+std::size_t MeshBlock::GetBlockSizeInBytesGray() {
+  std::size_t size;
+  // NEW_OUTPUT_TYPES:
+  size = phydro->u.GetSizeInBytes();
+  if (GENERAL_RELATIVITY) {
+    size += phydro->w.GetSizeInBytes();
+    size += phydro->w1.GetSizeInBytes();
+  }
+  if (MAGNETIC_FIELDS_ENABLED)
+    size += (pfield->b.x1f.GetSizeInBytes() + pfield->b.x2f.GetSizeInBytes()
+             + pfield->b.x3f.GetSizeInBytes());
+
+  if (NSCALARS > 0)
+    size += pscalars->s.GetSizeInBytes();
+
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    if (pnrrad->restart_from_gray > 0)
+      size += pnrrad->ir_gray.GetSizeInBytes();
+    else
+      size += pnrrad->ir.GetSizeInBytes();
+  }
+  if (CR_ENABLED)
+    size += pcr->u_cr.GetSizeInBytes();
+
+
+  // calculate user MeshBlock data size
+  for (int n=0; n<nint_user_meshblock_data_; n++)
+    size += iuser_meshblock_data[n].GetSizeInBytes();
+  for (int n=0; n<nreal_user_meshblock_data_; n++)
+    size += ruser_meshblock_data[n].GetSizeInBytes();
+
+  return size;
+}
+
+
 //----------------------------------------------------------------------------------------
 //! \fn void MeshBlock::SetCostForLoadBalancing(double cost)
 //! \brief stop time measurement and accumulate it in the MeshBlock cost
diff --git a/src/mesh/weighted_ave.cpp b/src/mesh/weighted_ave.cpp
index edfff233b5..49dd2a803f 100644
--- a/src/mesh/weighted_ave.cpp
+++ b/src/mesh/weighted_ave.cpp
@@ -10,6 +10,8 @@
 
 // C++ headers
 #include <algorithm>  // std::binary_search
+#include <sstream>    // sstream
+#include <stdexcept>  // runtime_error
 #include <vector>     // std::vector
 
 // Athena++ headers
@@ -229,6 +231,132 @@ void MeshBlock::WeightedAve(AthenaArray<Real> &u_out, AthenaArray<Real> &u_in1,
 }
 
 
+
+//override function for arrays with different order
+void MeshBlock::WeightedAve(AthenaArray<Real> &u_out, AthenaArray<Real> &u_in1,
+                            AthenaArray<Real> &u_in2, const Real wght[3], int flag) {
+  // consider every possible simplified form of weighted sum operator:
+  // U = a*U + b*U1 + c*U2
+
+  // assuming all 3x arrays are of the same size (or at least u_out is equal or larger
+  // than each input array) in each array dimension, and full range is desired:
+  // nx4*(3D real MeshBlock cells)
+
+  if (flag == 1) {
+    const int nu = u_out.GetDim1() - 1;
+    // u_in2 may be an unallocated AthenaArray if using a 2S time integrator
+    if (wght[0] == 1.0) {
+      if (wght[2] != 0.0) {
+        for (int k=ks; k<=ke; ++k) {
+          for (int j=js; j<=je; ++j) {
+            for (int i=is; i<=ie; ++i) {
+#pragma omp simd
+              for (int n=0; n<=nu; ++n) {
+                u_out(k,j,i,n) += wght[1]*u_in1(k,j,i,n) + wght[2]*u_in2(k,j,i,n);
+              }
+            }
+          }
+        }
+      } else { // do not dereference u_in2
+        if (wght[1] != 0.0) {
+          for (int k=ks; k<=ke; ++k) {
+            for (int j=js; j<=je; ++j) {
+              for (int i=is; i<=ie; ++i) {
+#pragma omp simd
+                for (int n=0; n<=nu; ++n) {
+                  u_out(k,j,i,n) += wght[1]*u_in1(k,j,i,n);
+                }
+              }
+            }
+          }
+        }
+      }
+    } else if (wght[0] == 0.0) {
+      if (wght[2] != 0.0) {
+        for (int k=ks; k<=ke; ++k) {
+          for (int j=js; j<=je; ++j) {
+            for (int i=is; i<=ie; ++i) {
+#pragma omp simd
+              for (int n=0; n<=nu; ++n) {
+                u_out(k,j,i,n) = wght[1]*u_in1(k,j,i,n) + wght[2]*u_in2(k,j,i,n);
+              }
+            }
+          }
+        }
+      } else if (wght[1] == 1.0) {
+        // just deep copy
+        for (int k=ks; k<=ke; ++k) {
+          for (int j=js; j<=je; ++j) {
+            for (int i=is; i<=ie; ++i) {
+#pragma omp simd
+              for (int n=0; n<=nu; ++n) {
+                u_out(k,j,i,n) = u_in1(k,j,i,n);
+              }
+            }
+          }
+        }
+      } else {
+        for (int k=ks; k<=ke; ++k) {
+          for (int j=js; j<=je; ++j) {
+            for (int i=is; i<=ie; ++i) {
+#pragma omp simd
+              for (int n=0; n<=nu; ++n) {
+                u_out(k,j,i,n) = wght[1]*u_in1(k,j,i,n);
+              }
+            }
+          }
+        }
+      }
+    } else {
+      if (wght[2] != 0.0) {
+        for (int k=ks; k<=ke; ++k) {
+          for (int j=js; j<=je; ++j) {
+            for (int i=is; i<=ie; ++i) {
+#pragma omp simd
+              for (int n=0; n<=nu; ++n) {
+                u_out(k,j,i,n) = wght[0]*u_out(k,j,i,n) + wght[1]*u_in1(k,j,i,n)
+                                 + wght[2]*u_in2(k,j,i,n);
+              }
+            }
+          }
+        }
+      } else { // do not dereference u_in2
+        if (wght[1] != 0.0) {
+          for (int k=ks; k<=ke; ++k) {
+            for (int j=js; j<=je; ++j) {
+              for (int i=is; i<=ie; ++i) {
+#pragma omp simd
+                for (int n=0; n<=nu; ++n) {
+                  u_out(k,j,i,n) = wght[0]*u_out(k,j,i,n) + wght[1]*u_in1(k,j,i,n);
+                }
+              }
+            }
+          }
+        } else { // do not dereference u_in1
+          for (int k=ks; k<=ke; ++k) {
+            for (int j=js; j<=je; ++j) {
+              for (int i=is; i<=ie; ++i) {
+#pragma omp simd
+                for (int n=0; n<=nu; ++n) {
+                  u_out(k,j,i,n) *= wght[0];
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  } else {
+      std::stringstream msg;
+      msg << "### FATAL ERROR in MeshBlock::WeightedAve" << std::endl
+          << "flag=" << flag << " not supported!."
+          << std::endl;
+      ATHENA_ERROR(msg);
+  }
+  return;
+}
+
+
 //----------------------------------------------------------------------------------------
 //! \fn void MeshBlock::WeightedAve(FaceField &b_out, FaceField &b_in1,
 //!                                 FaceField &b_in2, FaceField &b_in3,
diff --git a/src/nr_radiation/angulargrid.cpp b/src/nr_radiation/angulargrid.cpp
new file mode 100644
index 0000000000..782a184e91
--- /dev/null
+++ b/src/nr_radiation/angulargrid.cpp
@@ -0,0 +1,565 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file angulargrid.cpp
+//  \brief implementation of angular grid in class Radiation
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <sstream>
+#include <stdexcept>
+
+// Athena++ headers
+#include "../coordinates/coordinates.hpp"
+#include "../mesh/mesh.hpp"
+#include "../utils/utils.hpp"
+#include "./radiation.hpp"
+
+//--------------------------------------------------------------------------------------
+// \!fn void Radiation::AngularGrid(int angle_flag, int nmu)
+// \brief function to create the angular grid
+
+void NRRadiation::AngularGrid(int angle_flag, int nmu) {
+  std::stringstream msg;
+  MeshBlock *pmb=pmy_block;
+  // allocate some temporaray arrays
+  AthenaArray<Real> mu2tmp, mutmp, wtmp2, wtmp;
+  AthenaArray<Real> pmat, pinv, wpf;
+  AthenaArray<int> plab, pl;
+
+  int n_ang = nang/noct;
+
+  mu2tmp.NewAthenaArray(nmu);
+  mutmp.NewAthenaArray(n_ang,3);
+  wtmp2.NewAthenaArray(nmu-1);
+  wtmp.NewAthenaArray(nmu);
+
+  pmat.NewAthenaArray(nmu,nmu);
+  pinv.NewAthenaArray(nmu-1,nmu-1);
+  plab.NewAthenaArray(n_ang);
+  pl.NewAthenaArray(nmu,3);
+  wpf.NewAthenaArray(nmu-1);
+
+
+  // initialize coordinate direction
+  int axisx=0, axisy=1, axisz=2;
+  pmy_block->pcoord->AxisDirection(&axisx, &axisy, &axisz);
+
+  // check the dimension of the problem
+
+  int n1z = pmb->ncells1, n2z = pmb->ncells2, n3z = pmb->ncells3;
+
+  int ndim = 1;
+  if (n2z > 1) ndim = 2;
+  if (n3z > 1) ndim = 3;
+
+  if (angle_flag == 0) {
+    if (ndim > 1) {
+      Real deltamu = 2.0 / (2 * nmu - 1);
+      mu2tmp(0) = 1.0 / (3.0 * (2 * nmu - 1));
+      for (int i=1; i<nmu; i++) {
+        mu2tmp(i) = mu2tmp(i-1) + deltamu;
+      }
+
+      Real w2 = 4.0 * mu2tmp(0);
+      Real wsum2 = std::sqrt(w2);
+      wtmp2(0) = wsum2;
+
+      for (int i=1; i<nmu-2; i++) {
+        w2 += deltamu;
+        wtmp2(i) = std::sqrt(w2);
+        wsum2 += wtmp2(i);
+      }
+
+      if (nmu > 2)
+        wtmp2(nmu-2) = 2.0*(nmu-1)/3.0 - wsum2;
+
+      wtmp(0) = wtmp2(0);
+      Real wsum = wtmp(0);
+      for (int i=1; i<nmu-1; ++i) {
+        wtmp(i) = wtmp2(i) - wtmp2(i-1);
+        wsum += wtmp(i);
+      }
+      wtmp(nmu-1) = 1.0 - wsum;
+
+      int np = 0;
+      int iang = 0;
+
+      // initialize to be zero
+      for (int i=0; i<nmu; ++i)
+        for (int j=0; j<nmu; ++j)
+          pmat(i,j) = 0.0;
+
+      for (int i=0; i<nmu; i++) {
+        for (int j=0; j<nmu; j++) {
+          for (int k=0; k<nmu; k++) {
+            if (i + j + k == nmu - 1) {
+              // assign cosines to temporary array grid
+              mutmp(iang,0) = std::sqrt(mu2tmp(j));
+              mutmp(iang,1) = std::sqrt(mu2tmp(k));
+              mutmp(iang,2) = std::sqrt(mu2tmp(i));
+
+              int ip=Permutation(i,j,k,np,pl);
+              if (ip == -1) {
+                pl(np,0) = i;
+                pl(np,1) = j;
+                pl(np,2) = k;
+                pmat(i,np) += 1.0;
+                plab(iang) = np;
+                np++;
+              } else {
+                pmat(i,ip) += 1.0;
+                plab(iang) = ip;
+              }
+
+              iang++;
+            }
+          }
+        }
+      }
+      if (nmu > 1) {
+        //  Invert matrix of Permutations families */
+        InverseMatrix(nmu-1, pmat,pinv);
+        // Solve for and assign weights for each Permutation family
+        MatrixMult(nmu-1,nmu-1,pinv,wtmp,wpf);
+
+        for (int l=0; l<noct; ++l) {
+          for (int i=0; i<n_ang; ++i) {
+              wmu(l*n_ang+i) = wpf(plab(i));
+          } // end nang
+        } // end noct
+      } else {
+        for (int l=0; l<noct; ++l) {
+            wmu(l) = 1.0;
+        }
+      }
+    }
+
+    if (ndim == 1) {
+      AthenaArray<Real> mutmp1d, wtmp1d;
+      mutmp1d.NewAthenaArray(2*nmu);
+      wtmp1d.NewAthenaArray(2*nmu);
+
+      Gauleg(2*nmu, -1.0, 1.0, mutmp1d, wtmp1d);
+      for (int i=0; i<2*nmu; ++i) {
+        wmu(i) = 0.5 * wtmp1d(i);
+      }
+      for (int n1=0; n1<n1z; ++n1) {
+        for (int i=0; i<2*nmu; ++i) {
+          mu(0,0,0,n1,i)=mutmp1d(i);
+        }
+      }
+      mutmp1d.DeleteAthenaArray();
+      wtmp1d.DeleteAthenaArray();
+    } else if (ndim == 2) {
+      // for spherical coordinate system, it should be r-theta-phi
+      for (int n2=0; n2<n2z; ++n2) {
+        for (int n1=0; n1<n1z; ++n1) {
+          for (int j=0; j<2; ++j) {
+            for (int k=0; k<2; ++k) {
+              int l=2*j+k;
+              for (int i=0; i<n_ang; ++i) {
+                int mi = l*n_ang + i;
+                if (k == 0) {
+                  mu(axisx,0,n2,n1,mi) =  mutmp(i,0);
+                } else {
+                  mu(axisx,0,n2,n1,mi) = -mutmp(i,0);
+                }
+                if (j == 0) {
+                  mu(axisy,0,n2,n1,mi) =  mutmp(i,1);
+                } else {
+                  mu(axisy,0,n2,n1,mi) = -mutmp(i,1);
+                }
+              }
+            }
+          }
+        }
+      }
+      // for the angular weight
+      for (int i=0; i<noct * n_ang; ++i) {
+          wmu(i) *= 0.25;
+      }
+    } else if (ndim == 3) {
+      for (int n3=0; n3<n3z; ++n3) {
+        for (int n2=0; n2<n2z; ++n2) {
+          for (int n1=0; n1<n1z; ++n1) {
+            for (int j=0; j<2; ++j) {
+              for (int k=0; k<2; ++k) {
+                for (int l=0; l<2; ++l) {
+                  int m=4*j+2*k+l;
+
+                  for (int i=0; i<n_ang; ++i) {
+                    int mi = m*n_ang + i;
+                    if (l == 0) {
+                      mu(axisx,n3,n2,n1,mi) =  mutmp(i,0);
+                    } else {
+                      mu(axisx,n3,n2,n1,mi) = -mutmp(i,0);
+                    }
+                    if (k == 0) {
+                      mu(axisy,n3,n2,n1,mi) =  mutmp(i,1);
+                    } else {
+                      mu(axisy,n3,n2,n1,mi) = -mutmp(i,1);
+                    }
+                    if (j == 0) {
+                      mu(axisz,n3,n2,n1,mi) =  mutmp(i,2);
+                    } else {
+                      mu(axisz,n3,n2,n1,mi) = -mutmp(i,2);
+                    }
+                  } // end i
+                } // end l
+              } // end k
+            } // end j
+          } // end n1
+        } // end n2
+      } // end n3
+
+      for (int i=0; i<noct * n_ang; ++i) {
+        wmu(i) *= 0.125;
+      }
+    } // end nDIM = 3
+  } else if (angle_flag == 10) {
+    if ((ndim == 1) || (ndim == 3)) {
+      msg <<"[Warning]: ang_quad = 10 should be used" <<
+      "only for 2D problems. \n" << std::endl;
+      throw std::runtime_error(msg.str().c_str());
+    }
+
+    Real delmu=1.0/std::sqrt((Real)3.0);
+    Real sintheta = std::sqrt((Real)2.0)/std::sqrt((Real)3.0);
+    Real phi = 0.5 * PI / (Real) (2*nmu);
+
+    for (int i=0; i<nmu; ++i) {
+      mutmp(i,0) = sintheta * cos(phi*(Real)(2*i+1));
+      mutmp(i,1) = sintheta * sin(phi*(Real)(2*i+1));
+      mutmp(i,2) = delmu;
+    }
+    for (int n2=0; n2<n2z; ++n2) {
+      for (int n1=0; n1<n1z; ++n1) {
+        for (int j=0; j<2; ++j) {
+          for (int k=0; k<2; ++k) {
+            int l=2*j+k;
+
+            for (int i=0; i<n_ang; ++i) {
+              int mi = l*n_ang + i;
+              if (k == 0) {
+                mu(axisx,0,n2,n1,mi) =  mutmp(i,0);
+              } else {
+                mu(axisx,0,n2,n1,mi) = -mutmp(i,0);
+              }
+
+              if (j == 0) {
+                mu(axisy,0,n2,n1,mi) =  mutmp(i,1);
+              } else {
+                mu(axisy,0,n2,n1,mi) = -mutmp(i,1);
+              }
+            }
+          }
+        }
+      }
+    }
+    for (int i=0; i<noct*n_ang; ++i) {
+      wmu(i) = 0.25/((Real)nmu);
+    }
+  } else {
+    msg << "### FATAL ERROR in function [InitialAngle]" << std::endl
+    << "Type of angular discretization unknow: "<< angle_flag << "\n ";
+    throw std::runtime_error(msg.str().c_str());
+  }
+
+  // Now change the angle cosines to different coordinate systems
+  pmy_block->pcoord->ConvertAngle(pmy_block,nang, mu);
+
+  // free the temporary arrays
+  mu2tmp.DeleteAthenaArray();
+  mutmp.DeleteAthenaArray();
+  wtmp2.DeleteAthenaArray();
+  wtmp.DeleteAthenaArray();
+  pmat.DeleteAthenaArray();
+  pinv.DeleteAthenaArray();
+  plab.DeleteAthenaArray();
+  pl.DeleteAthenaArray();
+  wpf.DeleteAthenaArray();
+  return;
+}
+
+// overwrite function to create angles that are defined with respect
+// to local coordinate system
+
+// This is the so-called spherical polar tetrad,
+// which is ideal for spherical polar coordinate
+
+// nzeta is number of zeta angles in one octant
+// npsi is the number of psi angles between 0 and pi
+void NRRadiation::AngularGrid(int angle_flag, int nzeta, int npsi) {
+  MeshBlock *pmb=pmy_block;
+  if (angle_flag == 1) {
+    // initialize coordinate direction
+    int axisx=0, axisy=1, axisz=2;
+    pmy_block->pcoord->AxisDirection(&axisx, &axisy, &axisz);
+
+    // in spherical polar coordinate
+    //  *axisx = 1;
+    //  *axisy = 2;
+    //  *axisz = 0;
+    // check the dimension of the problem
+
+    int n1z = pmb->ncells1, n2z = pmb->ncells2, n3z = pmb->ncells3;
+
+    int ndim = 1;
+    if (n2z > 1) ndim = 2;
+    if (n3z > 1) ndim = 3;
+
+    // separate ghost zones and active zones
+    // so that they can be compatible with different angular scheme
+    if (nzeta > 0) {
+      coszeta_v.NewAthenaArray(2*nzeta);
+      zeta_v_full.NewAthenaArray(2*nzeta+2*NGHOST);
+      zeta_f_full.NewAthenaArray(2*nzeta+1+2*NGHOST);
+      dzeta_v.NewAthenaArray(2*nzeta+2*NGHOST);
+      dzeta_f.NewAthenaArray(2*nzeta+1+2*NGHOST);
+      coszeta_f.NewAthenaArray(2*nzeta+1);
+      len_zeta.NewAthenaArray(2*nzeta); // This id Delta (cos\theta)
+
+      int zs = 0; // ze = 2*nzeta - 1;
+
+      Real dcoszeta = 1.0/nzeta;
+      coszeta_f(zs) = 1.0;
+      for (int i=1; i<nzeta; ++i) {
+        coszeta_f(i+zs) = 1.0 - i *dcoszeta;
+      }
+      coszeta_f(nzeta+zs) = 0.0;
+      for (int i=nzeta+1; i<2*nzeta+1; ++i)
+        coszeta_f(i+zs) = -coszeta_f(2*nzeta-i+zs);
+
+      for (int i=0; i<nzeta; ++i) {
+        coszeta_v(i+zs) = 0.5*(coszeta_f(i+zs)+coszeta_f(i+zs+1));
+      }
+    // re-normalize
+      Real normalization = 2*nzeta/std::sqrt(4*nzeta*nzeta-1);
+
+      for (int i=0; i<nzeta; ++i) {
+        coszeta_v(i+zs) *= normalization;
+      }
+      for (int i=nzeta; i<2*nzeta; ++i) {
+        coszeta_v(i+zs) = -coszeta_v(2*nzeta-i-1+zs);
+      }
+
+      for (int i=0; i<nzeta*2; ++i) {
+        len_zeta(i) = coszeta_f(i) - coszeta_f(i+1);
+      }
+
+
+      // the following arrays include ghost zones
+      // this is used for reconstruction
+      for (int i=0; i<2*nzeta; ++i) {
+        zeta_v_full(i+NGHOST) = acos(coszeta_v(i));
+      }
+      for (int i=0; i<2*nzeta+1; ++i) {
+        zeta_f_full(i+NGHOST) = acos(coszeta_f(i));
+      }
+      for (int i=1; i<=NGHOST; ++i) {
+        Real delta = zeta_v_full(i+NGHOST) - zeta_v_full(NGHOST+i-1);
+        zeta_v_full(NGHOST-i) = zeta_v_full(NGHOST-i+1) - delta;
+      }
+      for (int i=1; i<=NGHOST; ++i) {
+        Real delta = zeta_v_full(2*nzeta+NGHOST-i) - zeta_v_full(2*nzeta+NGHOST-i-1);
+        zeta_v_full(2*nzeta+NGHOST+i-1) = zeta_v_full(2*nzeta+NGHOST+i-2) + delta;
+      }
+
+      for (int i=0; i<2*nzeta+2*NGHOST-1; ++i) {
+        dzeta_v(i) = zeta_v_full(i+1) - zeta_v_full(i);
+      }
+      for (int i=0; i<2*nzeta+1; ++i) {
+        zeta_f_full(i+NGHOST) = acos(coszeta_f(i));
+      }
+
+      for (int i=1; i<=NGHOST; ++i) {
+        Real delta = zeta_f_full(i+NGHOST) - zeta_f_full(NGHOST+i-1);
+        zeta_f_full(NGHOST-i) = zeta_f_full(NGHOST-i+1) - delta;
+      }
+
+      for (int i=1; i<=NGHOST; ++i) {
+        Real delta = zeta_f_full(2*nzeta+NGHOST-i+1) - zeta_f_full(2*nzeta+NGHOST-i);
+        zeta_f_full(2*nzeta+NGHOST+i) = zeta_f_full(2*nzeta+NGHOST+i-1) + delta;
+      }
+
+      for (int i=0; i<2*nzeta+2*NGHOST; ++i) {
+        dzeta_f(i) = zeta_f_full(i+1)-zeta_f_full(i);
+      }
+
+    // end if nzeta > 0
+    } else {
+      coszeta_v.NewAthenaArray(1);
+      zeta_v_full.NewAthenaArray(1);
+      zeta_f_full.NewAthenaArray(1);
+      dzeta_v.NewAthenaArray(1);
+      dzeta_f.NewAthenaArray(1);
+      coszeta_f.NewAthenaArray(1);
+      len_zeta.NewAthenaArray(1); // This id Delta (cos\theta)
+
+      coszeta_f(0) = 1.0;
+      coszeta_v(0) = 1.0;
+
+      len_zeta(0) = 1.0;
+
+      zeta_v_full(0) = 0.0;
+      zeta_f_full(0) = 0.0;
+
+      dzeta_v(0) = 1.0;
+      dzeta_f(0) = 1.0;
+    }
+
+
+    // construct psi angles
+    // for 2D problem in spherical polar, npsi = 1
+    if (npsi > 0) {
+      psi_v.NewAthenaArray(2*npsi);
+      psi_f.NewAthenaArray(2*npsi+1);
+      sin_psi_f.NewAthenaArray(2*npsi+1);
+      len_psi.NewAthenaArray(2*npsi);
+      psi_v_full.NewAthenaArray(2*npsi+2*NGHOST);
+      psi_f_full.NewAthenaArray(2*npsi+1+2*NGHOST);
+      dpsi_v.NewAthenaArray(2*npsi+2*NGHOST);
+      dpsi_f.NewAthenaArray(2*npsi+2*NGHOST+1);
+
+      Real dpsi=PI/npsi;
+
+      for (int i=0; i<npsi; ++i) {
+        psi_v(i) = (i+0.5)*dpsi;
+      }
+      for (int i=npsi; i<2*npsi; ++i)
+        psi_v(i)=2.0*PI-psi_v(2*npsi-i-1);
+
+      psi_f(0) = 0.0;
+      for (int i=1; i<2*npsi; ++i)
+        psi_f(i)=2*psi_v(i-1) - psi_f(i-1);
+      psi_f(2*npsi) = 2*PI;
+
+      for (int i=0; i<2*npsi+1; ++i)
+        sin_psi_f(i) = sin(psi_f(i));
+
+      for (int i=0; i<2*npsi; ++i)
+        len_psi(i) = psi_f(i+1) - psi_f(i);
+
+      //  now the arrays with ghost zones
+
+      for (int i=0; i<2*npsi; ++i) {
+        psi_v_full(i+NGHOST) = psi_v(i);
+      }
+      for (int i=0; i<2*npsi+1; ++i) {
+        psi_f_full(i+NGHOST) = psi_f(i);
+      }
+      for (int i=1; i<=NGHOST; ++i) {
+        psi_v_full(NGHOST-i) = psi_v(2*npsi-i)-2*PI;
+      }
+      for (int i=1; i<=NGHOST; ++i) {
+        psi_v_full(2*npsi+NGHOST+i-1) = psi_v(i-1) + 2*PI;
+      }
+
+      for (int i=0; i<2*npsi+2*NGHOST-1; ++i) {
+        dpsi_v(i) = psi_v_full(i+1) - psi_v_full(i);
+      }
+
+      for (int i=0; i<2*npsi+1; ++i) {
+        psi_f_full(i+NGHOST) = psi_f(i);
+      }
+
+      for (int i=1; i<=NGHOST; ++i) {
+        psi_f_full(NGHOST-i) = psi_f(2*npsi-i)-2*PI;
+      }
+
+      for (int i=1; i<=NGHOST; ++i) {
+        psi_f_full(2*npsi+NGHOST+i) = psi_f(i) + 2*PI;
+      }
+
+      for (int i=0; i<2*npsi+2*NGHOST; ++i) {
+        dpsi_f(i) = psi_f_full(i+1)-psi_f_full(i);
+      }
+    } // end if npsi > 0
+
+    // for 1D problem
+    if (ndim == 1) {
+      for (int i=0; i<n1z; ++i) {
+         for (int n=0; n<2*nzeta; ++n) {
+            //x,k,j,i,n
+            mu(0,0,0,i,n) = coszeta_v(n);
+         }
+      }
+    } else if (ndim == 2) {
+      for (int j=0; j<n2z; ++j) {
+        for (int i=0; i<n1z; ++i) {
+          if (npsi == 1) {
+            for (int n=0; n<2*nzeta; ++n) {
+              for (int m=0; m<2*npsi; ++m) {
+                int ang_num = n*(2*npsi)+m;
+                Real sinzeta_v = std::sqrt(1.0 - coszeta_v(n)
+                                      * coszeta_v(n));
+                mu(axisz,0,j,i,ang_num) = coszeta_v(n);
+                if (m==0)
+                  mu(axisx,0,j,i,ang_num) = sinzeta_v;
+                else
+                  mu(axisx,0,j,i,ang_num) = -sinzeta_v;
+              }
+            }
+          } else {// the case in x -y plane
+            if (std::strcmp(COORDINATE_SYSTEM, "spherical_polar") == 0) {
+              // in spherical polar, 2D, we still need 3D angular grid
+              for (int n=0; n<2*nzeta; ++n) {
+                for (int m=0; m<2*npsi; ++m) {
+                  int ang_num = n*(2*npsi)+m;
+                  Real sinzeta_v = std::sqrt(1.0 - coszeta_v(n)
+                                      * coszeta_v(n));
+                  mu(axisx,0,j,i,ang_num) = sinzeta_v * cos(psi_v(m));
+                  mu(axisy,0,j,i,ang_num) = sinzeta_v * sin(psi_v(m));
+                  mu(axisz,0,j,i,ang_num) = coszeta_v(n);
+                }
+              }
+            } else {
+              for (int m=0; m<2*npsi; ++m) {
+                mu(0,0,j,i,m) = cos(psi_v(m));
+                mu(1,0,j,i,m) = sin(psi_v(m));
+              }
+            }
+          }
+        } // end i
+      } // end j
+
+    } else {
+      // 3D case
+      for (int k=0; k<n3z; ++k) {
+        for (int j=0; j<n2z; ++j) {
+          for (int i=0; i<n1z; ++i) {
+            for (int n=0; n<2*nzeta; ++n) {
+              for (int m=0; m<2*npsi; ++m) {
+                int ang_num = n*(2*npsi)+m;
+                Real sinzeta_v = std::sqrt(1.0 - coszeta_v(n)
+                                      * coszeta_v(n));
+                mu(axisx,k,j,i,ang_num) = sinzeta_v * cos(psi_v(m));
+                mu(axisy,k,j,i,ang_num) = sinzeta_v * sin(psi_v(m));
+                mu(axisz,k,j,i,ang_num) = coszeta_v(n);
+              }
+            }
+          }
+        }
+      }
+    }
+    // equal weight for all cases
+    for (int n=0; n<nang; ++n) {
+      wmu(n) = 1.0/nang;
+    }
+  } // end ang_flag==1
+}
diff --git a/src/nr_radiation/frequencygrid.cpp b/src/nr_radiation/frequencygrid.cpp
new file mode 100644
index 0000000000..6377e761b3
--- /dev/null
+++ b/src/nr_radiation/frequencygrid.cpp
@@ -0,0 +1,455 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file frequencygrid.cpp
+//  \brief implementation of frequency grid in class Radiation
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <cstdio>  // fopen and fwrite
+#include <sstream>  // msg
+
+// Athena++ headers
+#include "../utils/utils.hpp"
+#include "./radiation.hpp"
+
+//--------------------------------------------------------------------------------------
+// \!fn void FrequencyGrid()
+
+// \brief function to create the frequency grid
+// specific intensities are still defined as frequency integrated over each
+
+void NRRadiation::FrequencyGrid() {
+  Real h_planck = 6.6260755e-27; // Planck constant
+  Real k_b = 1.380649e-16;   // Boltzman constant
+
+  // convert frequency to unit of kT_unit/h
+  if (nu_min < 0.0) {
+    nu_min = -nu_min;
+  } else {
+    nu_min = nu_min * h_planck/(k_b * tunit);
+  }
+  if (nu_max < 0.0) {
+    nu_max = -nu_max;
+  } else {
+    nu_max = nu_max * h_planck/(k_b * tunit);
+  }
+
+  if ((nu_max <= nu_min) && (nfreq > 2)) {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in Radiation Class" << std::endl
+        << "frequency_max needs to be larger than frequency_min!";
+    throw std::runtime_error(msg.str().c_str());
+  }
+
+  if (log_fre_ == 1) {
+    if (fre_ratio > 1) {
+      if (nfreq > 2) {
+        nu_max = nu_min * std::pow(fre_ratio,nfreq-2);
+      } else {
+        nfreq = std::log10(nu_max/nu_min)/std::log10(fre_ratio)+2;
+      }
+    } else {
+      if (nfreq > 2) {
+        fre_ratio = std::log10(nu_max/nu_min)/(nfreq - 2);
+        fre_ratio = std::pow(10.0,fre_ratio);
+      }
+    }
+  }
+
+  if (nfreq > 1) {
+    nu_grid.NewAthenaArray(nfreq);
+    nu_grid(0) = 0.0;
+    nu_grid(1) = nu_min;
+
+    if (nfreq > 2) {
+      if (log_fre_ == 1) {
+        for(int n=2; n<nfreq; ++n)
+          nu_grid(n) = nu_grid(n-1) * fre_ratio;
+      } else {
+        Real d_nu = (nu_max - nu_min)/(nfreq-2);
+
+        for(int n=2; n<nfreq; ++n)
+          nu_grid(n) = nu_grid(n-1) + d_nu;
+      }
+    }// end nfreq > 2
+
+    nu_cen.NewAthenaArray(nfreq);
+
+    for(int n=0; n<nfreq-1; ++n)
+      nu_cen(n) = 0.5*(nu_grid(n)+nu_grid(n+1));
+
+    delta_nu.NewAthenaArray(nfreq);
+
+    for(int n=0; n<nfreq-1; ++n) {
+      delta_nu(n) = nu_grid(n+1)-nu_grid(n);
+    }
+  }
+  emission_spec.NewAthenaArray(nfreq);
+
+  // initialize with default emission spectrum assuming tgas=1
+  if (nfreq == 1) {
+    emission_spec(0) = 1.0;
+  } else {
+    for (int ifr=0; ifr<nfreq-1; ++ifr) {
+      emission_spec(ifr) =  IntPlanckFunc(nu_grid(ifr), nu_grid(ifr+1));
+    }
+      emission_spec(nfreq-1) = 1.0 - FitIntPlanckFunc(nu_grid(nfreq-1));
+  }
+}
+
+// return the integral (15/pi^4)\int_0^{nu/T} x^3 dx/(exp(x)-1)
+// frequency is scaled with kT_0/h
+// using fitting formula to return \int_0^nu_min and \int_0^nu_max
+// The fitting formula is based on asymptotic expression of the integral,
+// as well as polynomial fitting to the numerical values calculated by
+// directly integrating the function in mathematica
+Real NRRadiation::FitIntPlanckFunc(Real nu_t) {
+  // the integral at nu_t=1.5 is 0.6154949828394710
+  Real integral = 0.0;
+  Real nu_2 = nu_t * nu_t;
+  Real nu_3 = nu_t * nu_2;
+  Real nu_7 = nu_2 * nu_2 * nu_3;
+  if (nu_t < 1.9434) {
+    integral = 0.051329911273422 * nu_3 -0.019248716727533 * nu_t * nu_3
+               + 0.002566495563671 * nu_2 * nu_3
+               -3.055351861513195*1.e-5*nu_7;
+  } else if (nu_t < 5.23754) {
+    Real logfre = std::log10(nu_t);
+    integral = -0.6874*logfre*logfre*logfre
+               -0.5698*logfre*logfre+2.671*logfre-1.476;
+    integral = std::pow(10.0,integral);
+  } else if (nu_t < 50) {
+    integral = 1.0-15*ONE_PI_FOUR_POWER*std::exp(-nu_t) *
+               (6.0+6.0*nu_t+3.0*nu_t*nu_t+nu_t*nu_t*nu_t);
+  } else {
+    integral = 1.0;
+  }
+  return integral;
+}
+
+
+Real NRRadiation::IntPlanckFunc(Real nu_min, Real nu_max) {
+  return (FitIntPlanckFunc(nu_max) - FitIntPlanckFunc(nu_min));
+}
+
+// In the last frequency bin, [nu, infty]
+// we assume the spectrum is blackbody with effective temeprature Tr
+// so that intensity=T_r^4 (15/pi^4) int_{nu/T_r}^{infty} x^3dx/(exp(x)-1)
+// This is rearranged to
+// intensity/nu^4=A=(1/y)^4(15/pi^4)\int_y^{infty} x^3dx/(exp(x)-1)
+// we use a fitting formula to get y
+Real NRRadiation::EffectiveBlackBody(Real intensity, Real nu) {
+  Real ir = std::max(intensity,static_cast<Real>(TINY_NUMBER));
+  Real a_nu = ir/(nu*nu*nu*nu); // I/nu^4
+  Real nu_tr = 1.0;
+  if (a_nu > 0.184077200146896) {
+    // solve the fourth order polynomial (1/y)^4-(5/pi^4)(1/y)-a_nu=0
+    // -(pi^4/5) (1/y)^4 + (1/y) + a_nu(pi^4/5)==0
+    Real coef4=-PI_FOUR_POWER*0.2;
+    Real coef = a_nu*0.2*PI_FOUR_POWER;
+    int flag=FouthPolyRoot(coef4, coef, nu_tr);
+    if (flag == -1)
+      nu_tr = std::pow((1.0/a_nu),0.25);
+    else
+      nu_tr = 1.0/nu_tr;
+  } else {
+    Real loganu = -std::log(a_nu);
+    if (loganu < 40.0) {
+      nu_tr = -0.000525 * loganu * loganu * loganu + 0.03138 * loganu * loganu
+              + 0.3223 * loganu + 0.8278;
+    } else {
+      nu_tr = 30.3278;
+    }
+
+    if (a_nu < 1.e-10) {
+      // improve the accuracy with iteration
+      Real yini = nu_tr;
+      int count = 0;
+      Real residual=1.0;
+      while((count < 6) && (residual > 1.e-6)) {
+        Real ratio = yini*yini*yini*yini/(6.0+6.0*yini+3.0*yini*yini+yini*yini*yini);
+        nu_tr = -std::log((PI_FOUR_POWER*a_nu/15.0)*ratio);
+        residual = std::abs((nu_tr-yini)/nu_tr);
+        count++;
+        yini=nu_tr;
+      }
+    }
+  }
+  return nu_tr;
+}
+
+// In the last frequency bin, [nu, infty]
+// we assum, n(nu)=1/(exp(nu/T)-1), the input value
+// n_nu2 = \int_{nu_f}^{infty} n\nu^2 d\nu
+// we fit the formula n_nu2/nu^3=A=(1/y)^3\int_y^{infty} x^2/(exp(x)-1) dx
+
+Real NRRadiation::EffectiveBlackBodyNNu2(Real n_nu2, Real nu) {
+  Real fit_a = n_nu2/(nu*nu*nu);
+  Real log_fit_a=std::log(fit_a);
+  Real nu_tr = 1.0;
+  if (fit_a < 0.001) {
+    nu_tr = 0.001177*log_fit_a*log_fit_a-0.8812*log_fit_a-0.7428;
+  } else if (fit_a < 5.0) {
+    Real exp_nu = (log_fit_a+13.28)/9.551;
+    nu_tr=8.647*std::exp(-exp_nu*exp_nu);
+  } else {
+    nu_tr=std::pow(2.404113806319301/fit_a,1.0/3.0);
+  }
+  return nu_tr;
+}
+
+// return the integral (15/pi^4)\int_{\nu/T}^{\infty} \nu J_nu d\nu
+// =(15/pi^4)\int_{\nu/T}^{\infty} x^4/(exp(x)-1) dx
+// the input is nu_t=nu_f/T
+Real NRRadiation::IntegrateBBNuJ(Real nu_t) {
+  Real nu_sq = nu_t*nu_t;
+  Real nu_four = nu_sq * nu_sq;
+  Real nu_three = nu_t * nu_sq;
+  Real nu_five = nu_t * nu_four;
+  Real nu_six = nu_sq * nu_four;
+  Real exp_nu = std::exp(-nu_t);
+  Real integral = 0.0;
+  if (nu_t < 2.596) {
+    integral = 3.832229496128511
+               - 3.75 * ONE_PI_FOUR_POWER * nu_four
+               + 1.5 * ONE_PI_FOUR_POWER * nu_five
+               - (5.0/24.0) * ONE_PI_FOUR_POWER * nu_six;
+  } else {
+    integral = 15.0*ONE_PI_FOUR_POWER*exp_nu*(24.0
+               +24.0*nu_t+12.0*nu_sq+4.0*nu_three+nu_four);
+  }
+  return integral;
+}
+
+// return the integral (15/pi^4)\int_{\nu/T}^{\infty} (J_nu/nu)^2 d\nu
+// =(15/pi^4)\int_{\nu/T}^{\infty} x^4/(exp(x)-1)^2 dx
+// the input is nu_t=nu_f/T
+Real NRRadiation::IntegrateBBJONuSq(Real nu_t) {
+  Real nu_sq = nu_t*nu_t;
+  Real nu_four = nu_sq * nu_sq;
+  Real nu_three = nu_t * nu_sq;
+  Real exp_nu = std::exp(-2.0*nu_t);
+  Real integral = 0.0;
+  if (nu_t < 10.0) {
+    Real top = 0.01872 * nu_sq - 0.2732 * nu_t + 0.9735;
+    Real bottom = nu_sq - 1.854 * nu_t + 5.828;
+    integral = top/bottom;
+  } else {
+    integral = 3.75*ONE_PI_FOUR_POWER*exp_nu*(3.0
+               +6.0*nu_t+6.0*nu_sq+4.0*nu_three+2.0*nu_four);
+  }
+  return integral;
+}
+
+//For a given value J=(15/pi^4)T_r^4\int x^3/(exp(x)-1) dx,
+// get the integral \int (J/\nu)^2 dx=
+// (15/pi^4)^2 T_r^5\int x^4/(exp(x)-1)^2 dx
+// The input is J and nu_f
+
+Real NRRadiation::BBJToJONuSq(Real &bb_j, Real &nu_f) {
+  Real j_nu4 = bb_j/(nu_f*nu_f*nu_f*nu_f);
+  Real log_j_nu4 = std::log10(j_nu4);
+  Real log_jonu2 = 0.0;
+  if (log_j_nu4 < -2.71777) {
+    log_jonu2 = (1.987*log_j_nu4*log_j_nu4-2.123*log_j_nu4+4.532)/
+       (log_j_nu4-1.733);
+  } else if ((log_j_nu4 >= -2.71777) && (log_j_nu4 < 2.66367)) {
+    log_jonu2 = 0.006977*log_j_nu4*log_j_nu4*log_j_nu4
+         -0.03641*log_j_nu4*log_j_nu4+1.314*log_j_nu4-1.632;
+  } else {
+    log_jonu2 = 1.25*log_j_nu4-1.588;
+  }
+
+  Real jonu2 = std::pow(10.0, log_jonu2);
+  jonu2 = jonu2 * nu_f*nu_f*nu_f*nu_f*nu_f;
+  return jonu2;
+}
+
+// For a given value J=(15/pi^4)T_r^4\int x^3/(exp(x)-1) dx,
+// return the value n(nu_f)=1/(exp(x)-1)
+// The input is J and nu_f
+
+Real NRRadiation::BBJtoNnu(Real &bb_j, Real &nu_f) {
+  Real j_nu4 = bb_j/(nu_f*nu_f*nu_f*nu_f);
+  Real log_j_nu4 = std::log10(j_nu4)+3.0;
+  Real log_nnu = 0.0;
+
+  if (log_j_nu4 < 0.0) {
+    log_nnu = -0.003321*log_j_nu4*log_j_nu4+0.8649*log_j_nu4-1.869;
+  } else if (log_j_nu4 >= 0.0 && log_j_nu4 < 7.07128) {
+    log_nnu = 0.004199*log_j_nu4*log_j_nu4*log_j_nu4
+            -0.07701*log_j_nu4*log_j_nu4+0.7398*log_j_nu4-1.869;
+  } else {
+    log_nnu = -0.002325*log_j_nu4*log_j_nu4+0.2955*log_j_nu4-0.977;
+  }
+
+  Real nnu = std::pow(10.0, log_nnu);
+  return nnu;
+}
+
+// For a given value J=(15/pi^4)T_r^4\int x^3/(exp(x)-1) dx,
+// return the value Jnu=(15/pi^4)T_r^5\int x^4/(exp(x)-1) dx
+// The input is J and nu_f
+
+Real NRRadiation::BBJtoJnu(Real &bb_j, Real &nu_f) {
+  Real j_nu4 = bb_j/(nu_f*nu_f*nu_f*nu_f);
+  Real log_j_nu4 = std::log10(j_nu4);
+  Real log_jnu = 0.0;
+  if (log_j_nu4 < -3.1936) {
+    log_jnu = 6.363e-5 * log_j_nu4*log_j_nu4*log_j_nu4
+              +0.002858*log_j_nu4*log_j_nu4
+              +1.043*log_j_nu4+0.2264;
+  } else if ((log_j_nu4 >= -3.1936) && (log_j_nu4 < 2.6701)) {
+    log_jnu = -0.003183*log_j_nu4*log_j_nu4*log_j_nu4
+              +0.0158*log_j_nu4*log_j_nu4
+              +1.225*log_j_nu4+0.5983;
+  } else {
+    log_jnu = 1.25*log_j_nu4+0.5836;
+  }
+
+  Real jnu = std::pow(10.0, log_jnu);
+  jnu *= (nu_f*nu_f*nu_f*nu_f*nu_f);
+
+  return jnu;
+}
+
+Real NRRadiation::DBBjDNNu2(Real &bb_j, Real &nu_f) {
+  Real j_nu4 = bb_j/(nu_f*nu_f*nu_f*nu_f);
+  Real log_j_nu4 = std::log10(j_nu4);
+  Real log_ratio=0.0;
+  if (log_j_nu4 < -3.06556) {
+    log_ratio=(-0.8125*log_j_nu4*log_j_nu4-1.888*log_j_nu4-2.331)/
+              (log_j_nu4*log_j_nu4+2.091*log_j_nu4+3.25);
+  } else if (log_j_nu4 >= -3.06556 && log_j_nu4 < 2.87328) {
+    log_ratio=(0.2166*log_j_nu4*log_j_nu4*log_j_nu4
+              +1.491*log_j_nu4*log_j_nu4+4.095*log_j_nu4-5.699)/
+              (log_j_nu4*log_j_nu4+6.111*log_j_nu4+24.98);
+  } else {
+    log_ratio=0.2499*log_j_nu4-0.2551;
+  }
+
+  Real ratio = std::pow(10.0, log_ratio);
+  ratio *= nu_f;
+  return ratio;
+}
+
+// return the integral \int_{nu/T}^{\infty} n \nu^2 d\nu
+// = \int_{nu/T}^{\infty} x^2/(exp(x)-1) dx
+// the input is nu_t=nu_f/T_r
+Real NRRadiation::IntegrateBBNNu2(Real nu_t) {
+  Real nu_sq = nu_t*nu_t;
+  Real nu_three = nu_t * nu_sq;
+  Real nu_four = nu_sq * nu_sq;
+  Real nu_six = nu_three * nu_three;
+  Real nu_eight = nu_four * nu_four;
+  Real exp_nu = std::exp(-nu_t);
+  Real integral = 0.0;
+  if (nu_t < 3.297) {
+    integral = 2.404113806319301 - 0.5 * nu_sq + (1.0/6.0) * nu_three
+             - (1.0/48.0) * nu_four + (1.0/4320.0) * nu_six
+             - (1.0/241920.0) * nu_eight;
+  } else {
+    integral = exp_nu * (2.0 + 2.0 * nu_t + nu_sq);
+  }
+  return integral;
+}
+
+// We assume blackbody spectrum in the frequency range nu_f to infty
+// For a given value of J=(15/\pi^4)\int_{\nu_f}^infty nu^3/(exp(nu/Tr)-1) dnu
+// and frequency nu_f, we return the integral
+// nnu2=\int_{nu_f}^{infty} nu^2/(exp(nu/Tr)-1) d\nu
+// We use piecewise linear relationship between J and nnu2
+Real NRRadiation::ConvertBBJNNu2(Real &bb_j, Real &nu_f) {
+  Real jnu4 = bb_j/(nu_f*nu_f*nu_f*nu_f);
+  Real log_jnu4=std::log10(jnu4);
+  Real log_nnu2=1.0;
+  if (log_jnu4 < -3.99793) {
+    log_nnu2 = 0.9971*log_jnu4+0.7464;
+  } else if (log_jnu4 >= -3.99793 && log_jnu4 < 2.73043) {
+    Real jsq = log_jnu4*log_jnu4;
+    log_nnu2 = 0.0003484*jsq*jsq+0.002327*jsq*log_jnu4-0.02137*jsq
+               +0.8037*log_jnu4+0.3249;
+  } else {
+    log_nnu2 = 0.7506*log_jnu4+0.3773;
+  }
+  Real nnu2 = std::pow(10.0,log_nnu2)*nu_f*nu_f*nu_f;
+  return nnu2;
+}
+
+// inverse conversion from nnu2 to J.
+// The conversion from J to nnu2, and then from nnu2 to J
+// Should be exact
+Real NRRadiation::InverseConvertBBJNNu2(Real &nnu2, Real &nu_f) {
+  Real nnu2_nu3=nnu2/(nu_f*nu_f*nu_f);
+  Real log_nnu2=std::log(nnu2_nu3);
+  Real log_jnu4=1.0;
+  if (log_nnu2 > 0.746) {
+    log_jnu4=(log_nnu2-0.746)/0.7644;
+  } else {
+    log_jnu4=(log_nnu2-0.746)/0.9846;
+  }
+  Real bbj = std::exp(log_jnu4)*nu_f*nu_f*nu_f*nu_f;
+
+  return bbj;
+}
+
+// convert J to nnu2 assuming Wien profile spectrum
+
+void NRRadiation::ConvertBBJWien(Real &bb_j, Real &nu_f, Real &tgas,
+                              Real &nuj, Real &jonusq) {
+  Real nu_t = nu_f/tgas;
+
+  // return \int \nu Jd\nu
+  Real j_coef = 6.0+6.0*nu_t+3.0*nu_t*nu_t+nu_t*nu_t*nu_t;
+  Real nj_coef = 24.0 + 24.0*nu_t + 12.0*nu_t*nu_t
+               + 4.0*nu_t*nu_t*nu_t + nu_t*nu_t*nu_t*nu_t;
+  nuj = bb_j*tgas*nj_coef/j_coef;
+
+
+  // return \int (J/\nu)^2 d\nu
+  Real jonu2_coef = 0.25*(3.0+6.0*nu_t+6.0*nu_t*nu_t
+                   +4.0*nu_t*nu_t*nu_t+2.0*nu_t*nu_t*nu_t*nu_t);
+  Real bbj_ratio = bb_j/(tgas*tgas*j_coef);
+  jonusq = bbj_ratio*bbj_ratio*tgas*jonu2_coef;
+  return;
+}
+
+void NRRadiation::ConvertBBJWien2(Real &bb_j, Real &nu_f, Real &tgas,
+                                  Real &nnu2, Real &n_nuf) {
+  Real nu_t = nu_f/tgas;
+  // return \int n\nu^2d\nu
+  Real n_coef = 2.0+2.0*nu_t+nu_t*nu_t;
+  Real j_coef = 6.0+6.0*nu_t+3.0*nu_t*nu_t+nu_t*nu_t*nu_t;
+  nnu2 = bb_j*PI_FOUR_POWER*n_coef/(15.0*tgas*j_coef);
+  // return n(nu_f) = exp(-nu_f/T)/\lambda
+  Real tgas_four = tgas*tgas*tgas*tgas;
+  n_nuf = bb_j*PI_FOUR_POWER/(15.0*tgas_four*j_coef);
+  return;
+}
+
+// convert nnu2 to J assuming Wien profile spectrum
+
+Real NRRadiation::InverseConvertBBJNNu2Wien(Real &nnu2, Real &nu_f, Real &tgas) {
+  Real nu_t = nu_f/tgas;
+  Real n_coef = 2.0+2.0*nu_t+nu_t*nu_t;
+  Real j_coef = 6.0+6.0*nu_t+3.0*nu_t*nu_t+nu_t*nu_t*nu_t;
+
+  Real bb_j = nnu2*tgas*15.0*ONE_PI_FOUR_POWER*j_coef/n_coef;
+
+  return bb_j;
+}
diff --git a/src/nr_radiation/get_moments.cpp b/src/nr_radiation/get_moments.cpp
new file mode 100644
index 0000000000..4bdfa7d4db
--- /dev/null
+++ b/src/nr_radiation/get_moments.cpp
@@ -0,0 +1,278 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file get_moments.cpp
+//  \brief calculate the moments of the radiation field
+//======================================================================================
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+
+#include "../defs.hpp"
+#include "../hydro/hydro.hpp"
+#include "../mesh/mesh.hpp"
+#include "./integrators/rad_integrators.hpp"
+#include "./radiation.hpp"
+
+//--------------------------------------------------------------------------------------
+// \!fn void CalculateMoment()
+// \brief function to create the radiation moments
+// calculate the frequency integrated moments of the radiation field
+// including the ghost zones
+void NRRadiation::CalculateMoment(AthenaArray<Real> &ir_in) {
+  Real er, frx, fry, frz, prxx, pryy, przz, prxy, prxz, pryz;
+  int n1z = pmy_block->block_size.nx1 + 2*(NGHOST);
+  int n2z = pmy_block->block_size.nx2;
+  int n3z = pmy_block->block_size.nx3;
+  if (n2z > 1) n2z += (2*(NGHOST));
+  if (n3z > 1) n3z += (2*(NGHOST));
+  Real *weight = &(wmu(0));
+
+  // reset the moment arrays to be zero
+  // There are 13 3D arrays
+  for (int n=0; n<13; ++n)
+    for (int k=0; k<n3z; ++k)
+      for (int j=0; j<n2z; ++j) {
+          Real *i_mom = &(rad_mom(n,k,j,0));
+        for (int i=0; i<n1z; ++i) {
+          i_mom[i] = 0.0;
+        }
+      }
+
+  for (int k=0; k<n3z; ++k) {
+    for (int j=0; j<n2z; ++j) {
+      for (int i=0; i<n1z; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          er=0.0; frx=0.0; fry=0.0; frz=0.0;
+          prxx=0.0; pryy=0.0; przz=0.0; prxy=0.0;
+          prxz=0.0; pryz=0.0;
+          Real *intensity = &(ir_in(k,j,i,ifr*nang));
+          Real *cosx = &(mu(0,k,j,i,0));
+          Real *cosy = &(mu(1,k,j,i,0));
+          Real *cosz = &(mu(2,k,j,i,0));
+          for (int n=0; n<nang; ++n) {
+            Real irweight = weight[n] * intensity[n];
+            er   += irweight;
+            frx  += irweight * cosx[n];
+            fry  += irweight * cosy[n];
+            frz  += irweight * cosz[n];
+            prxx += irweight * cosx[n] * cosx[n];
+            pryy += irweight * cosy[n] * cosy[n];
+            przz += irweight * cosz[n] * cosz[n];
+            prxy += irweight * cosx[n] * cosy[n];
+            prxz += irweight * cosx[n] * cosz[n];
+            pryz += irweight * cosy[n] * cosz[n];
+          }
+          // assign the moments for each frequency group
+          if (nfreq > 1) {
+            rad_mom_nu(ifr*13+IER,k,j,i) = er;
+            rad_mom_nu(ifr*13+IFR1,k,j,i) = frx;
+            rad_mom_nu(ifr*13+IFR2,k,j,i) = fry;
+            rad_mom_nu(ifr*13+IFR3,k,j,i) = frz;
+            rad_mom_nu(ifr*13+IPR11,k,j,i) = prxx;
+            rad_mom_nu(ifr*13+IPR22,k,j,i) = pryy;
+            rad_mom_nu(ifr*13+IPR33,k,j,i) = przz;
+            rad_mom_nu(ifr*13+IPR12,k,j,i) = prxy;
+            rad_mom_nu(ifr*13+IPR13,k,j,i) = prxz;
+            rad_mom_nu(ifr*13+IPR23,k,j,i) = pryz;
+            rad_mom_nu(ifr*13+IPR21,k,j,i) = prxy;
+            rad_mom_nu(ifr*13+IPR31,k,j,i) = prxz;
+            rad_mom_nu(ifr*13+IPR32,k,j,i) = pryz;
+          }
+
+          // assign the moments
+          rad_mom(IER,k,j,i) += er;
+          rad_mom(IFR1,k,j,i) += frx;
+          rad_mom(IFR2,k,j,i) += fry;
+          rad_mom(IFR3,k,j,i) += frz;
+          rad_mom(IPR11,k,j,i) += prxx;
+          rad_mom(IPR22,k,j,i) += pryy;
+          rad_mom(IPR33,k,j,i) += przz;
+          rad_mom(IPR12,k,j,i) += prxy;
+          rad_mom(IPR13,k,j,i) += prxz;
+          rad_mom(IPR23,k,j,i) += pryz;
+          rad_mom(IPR21,k,j,i) += prxy;
+          rad_mom(IPR31,k,j,i) += prxz;
+          rad_mom(IPR32,k,j,i) += pryz;
+        }
+      }
+    }
+  }
+}
+
+
+//--------------------------------------------------------------------------------------
+// \!fn void CalculateComMoment()
+
+// \brief Calculate the radiation moments in the co-moving frame
+// Also load specific intensity for dump
+
+void NRRadiation::CalculateComMoment() {
+  Hydro *phydro=pmy_block->phydro;
+  Real invcrat = 1.0/crat;
+  Real er, frx, fry, frz;
+  int n1z = pmy_block->block_size.nx1 + 2*(NGHOST);
+  int n2z = pmy_block->block_size.nx2;
+  int n3z = pmy_block->block_size.nx3;
+  if (n2z > 1) n2z += (2*(NGHOST));
+  if (n3z > 1) n3z += (2*(NGHOST));
+
+  AthenaArray<Real> &i_mom = rad_mom_cm;
+  Real *weight = &(wmu(0));
+  Real *ir_output;
+  // Get the temporary array
+  AthenaArray<Real> &wmu_cm = pradintegrator->wmu_cm_;
+  AthenaArray<Real> &tran_coef = pradintegrator->tran_coef_;
+  AthenaArray<Real> &ir_cm = pradintegrator->ir_cm_;
+  AthenaArray<Real> &cm_to_lab = pradintegrator->cm_to_lab_;
+  AthenaArray<Real> &cosx_cm = cosx_cm_;
+  AthenaArray<Real> &cosy_cm = cosy_cm_;
+  AthenaArray<Real> &cosz_cm = cosz_cm_;
+
+  AthenaArray<Real> split_ratio;
+  AthenaArray<int> map_start, map_end;
+
+  // reset the moment arrays to be zero
+  // There are 4 3D arrays
+  for (int n=0; n<4; ++n) {
+    for (int k=0; k<n3z; ++k) {
+      for (int j=0; j<n2z; ++j) {
+//#pragma omp simd
+        for (int i=0; i<n1z; ++i) {
+          i_mom(n,k,j,i) = 0.0;
+        }
+      }
+    }
+  }
+
+  for (int k=0; k<n3z; ++k) {
+    for (int j=0; j<n2z; ++j) {
+      for (int i=0; i<n1z; ++i) {
+        Real *cosx = &(mu(0,k,j,i,0));
+        Real *cosy = &(mu(1,k,j,i,0));
+        Real *cosz = &(mu(2,k,j,i,0));
+
+        Real vx = phydro->u(IM1,k,j,i)/phydro->u(IDN,k,j,i);
+        Real vy = phydro->u(IM2,k,j,i)/phydro->u(IDN,k,j,i);
+        Real vz = phydro->u(IM3,k,j,i)/phydro->u(IDN,k,j,i);
+        Real vsq = vx * vx + vy * vy + vz * vz;
+
+        Real ratio = std::sqrt(vsq) * invcrat;
+         // Limit the velocity to be smaller than the speed of light
+        if (ratio > vmax) {
+          Real factor = vmax/ratio;
+          vx *= factor;
+          vy *= factor;
+          vz *= factor;
+
+          vsq *= (factor*factor);
+        }
+
+            // square of Lorentz factor
+        Real lorzsq = 1.0/(1.0 - vsq * invcrat * invcrat);
+        Real lorz = std::sqrt(lorzsq);
+
+        // first, get co-moving frame ir_cm
+        Real numsum = 0.0;
+        for (int n=0; n<nang; ++n) {
+          Real vdotn = vx * cosx[n] + vy * cosy[n] + vz * cosz[n];
+          Real vnc = 1.0 - vdotn * invcrat;
+          tran_coef(n) = std::sqrt(lorzsq) * vnc;
+          wmu_cm(n) = weight[n]/(lorzsq * vnc * vnc);
+          numsum += wmu_cm(n);
+          cm_to_lab(n) = tran_coef(n)*tran_coef(n)*tran_coef(n)*tran_coef(n);
+          Real angcoef = lorz * invcrat * (1.0
+                         - lorz * vdotn * invcrat/(1.0 + lorz));
+          Real incoef = 1.0/(lorz * vnc);
+          cosx_cm(n) = (cosx[n] - vx * angcoef) * incoef;
+          cosy_cm(n) = (cosy[n] - vy * angcoef) * incoef;
+          cosz_cm(n) = (cosz[n] - vz * angcoef) * incoef;
+        }
+        numsum = 1.0/numsum;
+//#pragma omp simd
+        for (int n=0; n<nang; ++n) {
+           wmu_cm(n) *= numsum;
+        }
+
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for (int n=0; n<nang; ++n) {
+            ir_cm(n+ifr*nang) = ir(k,j,i,ifr*nang+n) * cm_to_lab(n);
+          }
+        }
+        // in the case of multi-groups
+        if (nfreq > 1) {
+          // shift intensity from shifted frequency bins
+
+          // map frequency grid
+          for (int n=0; n<nang; ++n) {
+            for (int ifr=0; ifr<nfreq; ++ifr) {
+              pradintegrator->ir_ori_(ifr) = ir_cm(ifr*nang+n);
+            }
+            split_ratio.InitWithShallowSlice(pradintegrator->split_ratio_,
+                                                                    3,n,1);
+            map_start.InitWithShallowSlice(pradintegrator->map_bin_start_,
+                                                                    2,n,1);
+            map_end.InitWithShallowSlice(pradintegrator->map_bin_end_,
+                                                                    2,n,1);
+
+
+            pradintegrator->MapLabToCmFrequency(tran_coef(n), split_ratio,
+                      map_start, map_end,
+                      pradintegrator->ir_ori_, pradintegrator->ir_done_);
+
+            for (int ifr=0; ifr<nfreq; ++ifr)
+              ir_cm(ifr*nang+n) = pradintegrator->ir_done_(ifr);
+          }
+        }
+
+        Real *cm_weight = &(wmu_cm(0));
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          ir_output = &(ir_cm(ifr*nang));
+          er=0.0; frx=0.0; fry=0.0; frz=0.0;
+          for (int n=0; n<nang; ++n) {
+            Real ir_weight = ir_output[n] * cm_weight[n];
+            er   += ir_weight;
+            frx  += ir_weight * cosx_cm(n);
+            fry  += ir_weight * cosy_cm(n);
+            frz  += ir_weight * cosz_cm(n);
+          }
+
+          // assign the moments for each frequency group
+          if (nfreq > 1) {
+            rad_mom_cm_nu(ifr*4+IER,k,j,i) = er;
+            rad_mom_cm_nu(ifr*4+IFR1,k,j,i) = frx;
+            rad_mom_cm_nu(ifr*4+IFR2,k,j,i) = fry;
+            rad_mom_cm_nu(ifr*4+IFR3,k,j,i) = frz;
+          }
+          i_mom(IER,k,j,i) += er;
+          i_mom(IFR1,k,j,i) += frx;
+          i_mom(IFR2,k,j,i) += fry;
+          i_mom(IFR3,k,j,i) += frz;
+        }
+
+        // prepare the opacity array for output
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          output_sigma(3*ifr+OPAS,k,j,i) = sigma_s(k,j,i,ifr);
+          output_sigma(3*ifr+OPAA,k,j,i) = sigma_a(k,j,i,ifr);
+          output_sigma(3*ifr+OPAP,k,j,i) = sigma_p(k,j,i,ifr);
+        }
+      }
+    }
+  }
+  return;
+}
diff --git a/src/nr_radiation/implicit/implicit_angular_flux.cpp b/src/nr_radiation/implicit/implicit_angular_flux.cpp
new file mode 100644
index 0000000000..91b63f1f9f
--- /dev/null
+++ b/src/nr_radiation/implicit/implicit_angular_flux.cpp
@@ -0,0 +1,211 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file implicit_angular_flux.cpp
+//  \brief implementation of angular flux implicitly
+//======================================================================================
+
+
+// C headers
+
+// C++ headers
+#include <sstream>    // stringstream
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../parameter_input.hpp"
+#include "../../reconstruct/reconstruction.hpp"
+#include "../radiation.hpp"
+
+// class header
+#include "../integrators/rad_integrators.hpp"
+
+// OpenMP header
+#ifdef OPENMP_PARALLEL
+#include <omp.h>
+#endif
+
+// calculate advective flux for the implicit scheme
+// only work for 1D spherical polar
+// The equation to solve is
+// \partial /\partial t - c/r \partial(\sin^2 zeta I)/\sin \zeta\partial \zeta
+// the output is stored in angflx
+
+// calculate the coefficient for angular flux
+void RadIntegrator::ImplicitAngularFluxesCoef(const Real wght) {
+  NRRadiation *prad=pmy_rad;
+  MeshBlock *pmb=prad->pmy_block;
+  Coordinates *pco=pmb->pcoord;
+  std::stringstream msg;
+
+  int &nzeta = prad->nzeta;
+  //int &npsi = prad->npsi;
+  AthenaArray<Real> &area_zeta = zeta_area_, &ang_vol = ang_vol_;
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+  for (int k=ks; k<=ke; ++k)
+    for (int j=js; j<=je; ++j)
+      for (int i=is; i<=ie; ++i) {
+        if (prad->npsi == 0) {
+        // first, related all angle to 2*nzeta
+          pco->GetGeometryZeta(prad,k,j,i,g_zeta_);
+        // The relation is I_m = coef *I_m+1 + const
+
+        // now, all the other angles
+        //  Ir_new - ir_old + coef0 * Ir_l + coef1 ir_ new = 0.0;
+          for (int n=0; n<2*nzeta; ++n) {
+            Real coef0 = wght * g_zeta_(n) * prad->reduced_c *
+                       area_zeta(n)/ang_vol(n);
+            Real coef1 = -wght * g_zeta_(n+1) * prad->reduced_c *
+                       area_zeta(n+1)/ang_vol(n);
+          // the equation is
+          // ir_new - ir_old + coef0 * ir_new + coef1 * ir_new1 = 0
+            if (n == 2*nzeta-1) {
+              imp_ang_coef_r_(k,j,i,n) = 0.0;
+              imp_ang_coef_(k,j,i,n) = coef0 + coef1;
+            } else {
+              imp_ang_coef_r_(k,j,i,n) = coef1;
+              imp_ang_coef_(k,j,i,n) = coef0;
+            }
+          }
+        } else {
+          // first, starting from the zeta angle 2*nzeta-1
+          // zeta area is only 0 at nzeta=0, not zero at nzeta
+          pco->GetGeometryZeta(prad,k,j,i,g_zeta_);
+
+          // now go from 2*nzeta-2 to 0
+          for (int n=0; n<2*nzeta; ++n) {
+            Real zeta_coef0 =  wght * g_zeta_(n) * prad->reduced_c;
+            Real zeta_coef1 = -wght * g_zeta_(n+1) * prad->reduced_c;
+            ImplicitPsiFluxCoef(k,j,i, n, wght, zeta_coef1, zeta_coef0);
+          }
+        }
+      }
+}
+
+
+void RadIntegrator::ImplicitPsiFluxCoef(int k, int j, int i, int n_zeta, Real wght,
+                                                   Real zeta_coef1, Real zeta_coef) {
+  NRRadiation *prad=pmy_rad;
+  Coordinates *pco=prad->pmy_block->pcoord;
+  AthenaArray<Real> &area_psi = psi_area_, &ang_vol = ang_vol_, &zeta_area = zeta_area_;
+  int &npsi = prad->npsi;
+  // the equation to solve
+  //(1+zeta_coef0+zeta_coef1) I + Div F_psi = 0
+  pco->GetGeometryPsi(prad,k,j,i,n_zeta,g_psi_);
+  // g_psi_ =sin zeta * cot \theta sin\psi/r
+  // g_psi_(0) is always 0
+
+  for (int m=0; m<2*npsi; ++m) { // all take the left state
+    int ang_num = n_zeta*2*npsi+m;
+
+    Real coef0 = wght * g_psi_(m) * prad->reduced_c *
+            area_psi(n_zeta,m)/ang_vol(ang_num);
+    Real coef1 = -wght * g_psi_(m+1) * prad->reduced_c *
+            area_psi(n_zeta,m+1)/ang_vol(ang_num);
+    Real z_coef1 = zeta_coef1 * zeta_area(m,n_zeta+1)/ang_vol(ang_num);
+    Real z_coef = zeta_coef * zeta_area(m,n_zeta)/ang_vol(ang_num);
+    Real coef_c = 0.0;
+    if ((g_psi_(m) < 0) || (g_psi_(m+1) < 0)) {
+      imp_ang_psi_l_(k,j,i,ang_num) = coef0;
+      imp_ang_psi_r_(k,j,i,ang_num) = 0.0;
+      coef_c = coef1;
+    } else if ((g_psi_(m) > 0) || (g_psi_(m+1) > 0)) {
+      imp_ang_psi_l_(k,j,i,ang_num) = 0.0;
+      imp_ang_psi_r_(k,j,i,ang_num) = coef1;
+      coef_c = coef0;
+    }
+    if (n_zeta == 2*prad->nzeta-1) {
+      imp_ang_coef_r_(k,j,i,ang_num) = 0.0;
+      imp_ang_coef_(k,j,i,ang_num) = coef_c + z_coef + z_coef1;
+    } else {
+      imp_ang_coef_r_(k,j,i,ang_num) = z_coef1;
+      imp_ang_coef_(k,j,i,ang_num) = coef_c + z_coef;
+    }
+  }
+}
+
+
+void RadIntegrator::ImplicitAngularFluxes(const int k, const int j, const int i,
+                                          AthenaArray<Real> &ir_ini) {
+  NRRadiation *prad=pmy_rad;
+  std::stringstream msg;
+
+  int &nzeta = prad->nzeta;
+  // int &npsi = prad->npsi;
+  int &nang = prad->nang;
+  int &nfreq = prad->nfreq;
+
+  // AthenaArray<Real> &area_zeta = zeta_area_, &ang_vol = ang_vol_;
+
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    if (prad->npsi == 0) {
+      // the angle 2*nzeta-1 does not contribute to ang_flx_
+      Real *p_angflx = &(ang_flx_(k,j,i,ifr*nang));
+      Real *coef_r = &(imp_ang_coef_r_(k,j,i,0));
+      Real *p_ir = &(ir_ini(k,j,i,ifr*nang));
+      for (int n=0; n<2*nzeta-1; ++n) {
+        p_angflx[n] = -coef_r[n] * p_ir[n+1];
+      }
+    } else {
+      // now go from 2*nzeta-2 to 0
+      for (int n=0; n<2*nzeta; ++n) {
+        ImplicitPsiFlux(k,j,i, ifr, n, ir_ini);
+      }
+    }
+  }
+}
+
+
+void RadIntegrator::ImplicitPsiFlux(const int k, const int j, const int i,
+                      int ifr, int n_zeta, AthenaArray<Real> &ir_ini) {
+  NRRadiation *prad=pmy_rad;
+  int &npsi = prad->npsi;
+  // m=0
+  int ang_num = n_zeta*2*npsi;
+  int &nang = prad->nang;
+
+  Real *psi_l = &(imp_ang_psi_l_(k,j,i,ang_num));
+  Real *psi_r = &(imp_ang_psi_r_(k,j,i,ang_num));
+  Real *p_ir = &(ir_ini(k,j,i,ifr*nang+ang_num));
+  Real *p_angflx = &(ang_flx_(k,j,i,ifr*nang+ang_num));
+
+  if (n_zeta == 2*prad->nzeta-1) {
+    // m=0
+    p_angflx[0] = -(psi_l[0] * p_ir[2*npsi-1] + psi_r[0] * p_ir[1]);
+    for (int m=1; m<2*npsi-1; ++m) { // all take the left state
+      p_angflx[m] = -(psi_l[m] * p_ir[m-1] + psi_r[m] * p_ir[m+1]);
+    }
+    //m=2*npsi-1
+    p_angflx[2*npsi-1] = -(psi_l[2*npsi-1] * p_ir[2*npsi-2] + psi_r[2*npsi-1] * p_ir[0]);
+  } else {
+    Real *p_ir_zetar = &(ir_ini(k,j,i,ifr*nang+(n_zeta+1)*2*npsi));
+    Real *zeta_r = &(imp_ang_coef_r_(k,j,i,ang_num));
+
+    p_angflx[0] = -(psi_l[0] * p_ir[2*npsi-1] + psi_r[0] * p_ir[1])
+                  -zeta_r[0] * p_ir_zetar[0];
+    for (int m=1; m<2*npsi-1; ++m) { // all take the left state
+      p_angflx[m] = -(psi_l[m] * p_ir[m-1] + psi_r[m] * p_ir[m+1])
+                    -zeta_r[m] * p_ir_zetar[m];
+    }
+    //m=2*npsi-1
+    p_angflx[2*npsi-1] = -(psi_l[2*npsi-1] * p_ir[2*npsi-2] + psi_r[2*npsi-1] * p_ir[0])
+                         -zeta_r[2*npsi-1] * p_ir_zetar[2*npsi-1];
+  }
+}
diff --git a/src/nr_radiation/implicit/implicit_first_order.cpp b/src/nr_radiation/implicit/implicit_first_order.cpp
new file mode 100644
index 0000000000..a2b5732831
--- /dev/null
+++ b/src/nr_radiation/implicit/implicit_first_order.cpp
@@ -0,0 +1,414 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file implicit_first_order.cpp
+//  \brief implementation of implicit transport with first order reconstruction
+//======================================================================================
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../parameter_input.hpp"
+#include "../../reconstruct/reconstruction.hpp"
+#include "../radiation.hpp"
+
+// class header
+#include "../integrators/rad_integrators.hpp"
+
+// OpenMP header
+#ifdef OPENMP_PARALLEL
+#include <omp.h>
+#endif
+
+
+// the flux is
+// Smax F(U_L)/(Smax-Smin) - Smin F(U_R) /(Smax-Smin)
+// + Smax Smin * (U_R - U_L)/(Smax - Smin)
+// Gauss-Seidel iteration, use upper triangle from last iteration
+// lower triangle is from next iteration
+
+void RadIntegrator::FirstOrderFluxDivergenceCoef(const Real wght) {
+  NRRadiation *prad=pmy_rad;
+  MeshBlock *pmb=prad->pmy_block;
+  Coordinates *pco= pmb->pcoord;
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+  int nang = prad->nang;
+  int nfreq = prad->nfreq;
+
+  AthenaArray<Real> &x1area = x1face_area_, &x2area = x2face_area_,
+                 &x2area_p1 = x2face_area_p1_, &x3area = x3face_area_,
+                 &x3area_p1 = x3face_area_p1_, &vol = cell_volume_;
+
+  // AthenaArray<Real> &x1flux=prad->flux[X1DIR];
+  // AthenaArray<Real> &x2flux=prad->flux[X2DIR];
+  // AthenaArray<Real> &x3flux=prad->flux[X3DIR];
+
+  // AthenaArray<Real> &area_zeta = zeta_area_, &area_psi = psi_area_,
+  //                &ang_vol = ang_vol_;
+  // int &nzeta = prad->nzeta, &npsi = prad->npsi;
+
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      pco->CenterWidth1(k,j,is-1,ie+1,dxw1_);
+      for (int i=is; i<=ie+1; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          // use the signal speed in each frequency group
+          Real sigmal = prad->sigma_a(k,j,i-1,ifr) + prad->sigma_s(k,j,i-1,ifr);
+          Real sigmar = prad->sigma_a(k,j,i,ifr) + prad->sigma_s(k,j,i,ifr);
+          Real taul = dxw1_(i-1) * sigmal;
+          Real taur = dxw1_(i) * sigmar;
+
+          Real f_l = 1.0;
+          Real f_r = 1.0;
+          taul *= taufact(k,j,i-1);
+          taur *= taufact(k,j,i);
+          // the choice of taufactor = sum of L and R cell optical depths is made after
+          // many trial experiments, particularly to minimize artifacts when there is a
+          // sharp opacity jump. other choices such as arithmetic and harmonic mean may
+          // work well in some applications
+          GetTaufactor(taul+taur,f_r,1);
+          GetTaufactor(taul+taur,f_l,-1);
+
+          Real *s1n = &(sfac1_x_(i,ifr*nang));
+          Real *s2n = &(sfac2_x_(i,ifr*nang));
+          Real *velxn = &(velx_(k,j,i,ifr*nang));
+          Real adv = adv_vel(0,k,j,i);
+          SignalSpeed(adv, f_l, f_r, velxn, s1n, s2n);
+        }
+      }
+      // calculate x1-flux divergence
+      pco->Face1Area(k,j,is,ie+1,x1area);
+      for (int i=is; i<=ie; ++i) {
+        Real areal = x1area(i);
+        Real arear = x1area(i+1);
+        Real *smax_l = &(sfac1_x_(i,0));
+        Real *smax_r = &(sfac1_x_(i+1,0));
+        Real *smin_l = &(sfac2_x_(i,0));
+        Real *smin_r = &(sfac2_x_(i+1,0));
+        Real *vel_ln = &(velx_(k,j,i,0));
+        Real *vel_rn = &(velx_(k,j,i+1,0));
+
+        Real *coef1n = &(const_coef_(k,j,i,0));
+        Real *coef1ln= &(const_coef1_l_(k,j,i,0));
+        Real *coef1rn= &(const_coef1_r_(k,j,i,0));
+        Real *exp_coefn=&(exp_coef_(k,j,i,0));
+
+        Real advl = 0.0;
+        Real advr = 0.0;
+        if (adv_flag_ > 0) {
+          advl = adv_vel(0,k,j,i);
+          advr = adv_vel(0,k,j,i+1);
+        }
+        Real *sm_diff1 = &(sm_diff1_(0));
+        Real *sm_diff2 = &(sm_diff2_(0));
+        for (int n=0; n<prad->n_fre_ang; ++n) {
+          Real diff = smax_l[n] - smin_l[n];
+          if (std::abs(diff) < TINY_NUMBER)
+            sm_diff1[n] = 0.0;
+          else
+            sm_diff1[n] = 1.0/diff;
+        }
+        for (int n=0; n<prad->n_fre_ang; ++n) {
+          Real diff = smax_r[n] - smin_r[n];
+          if (std::abs(diff) < TINY_NUMBER)
+            sm_diff2[n] = 0.0;
+          else
+            sm_diff2[n] = 1.0/diff;
+        }
+
+        SplitVelocity(vel_ln, vel_rn,advl, advr,smax_l,smin_l,smax_r,smin_r);
+
+        for (int n=0; n<prad->n_fre_ang; ++n) {
+          Real vl = vel_ln[n] - advl;
+          coef1n[n] = -areal * vel_im_l_(n) * sm_diff1[n];
+          coef1ln[n] = -areal * smax_l[n] * (vl - smin_l[n]) * sm_diff1[n];
+          exp_coefn[n] = -areal * vel_ex_l_(n) * sm_diff1[n];
+
+          Real vr = vel_rn[n] - advr;
+          coef1n[n] += arear * vel_im_r_(n) * sm_diff2[n];
+          coef1rn[n] = arear * smin_r[n] * (smax_r[n] - vr) * sm_diff2[n];
+          exp_coefn[n] += arear * vel_ex_r_(n) * sm_diff2[n];
+        }
+      }
+    }
+  }
+  // calculate x2-flux
+  if (pmb->block_size.nx2 > 1) {
+    for (int k=ks; k<=ke; ++k) {
+      // first, calculate speed
+      for (int j=js; j<=je+1; ++j) {
+        pco->CenterWidth2(k,j-1,is,ie,dxw1_);
+        pco->CenterWidth2(k,j,is,ie,dxw2_);
+        for (int i=is; i<=ie; ++i) {
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            Real sigmal = prad->sigma_a(k,j-1,i,ifr) + prad->sigma_s(k,j-1,i,ifr);
+            Real sigmar = prad->sigma_a(k,j,i,ifr) + prad->sigma_s(k,j,i,ifr);
+            Real taul = dxw1_(i) * sigmal;
+            Real taur = dxw2_(i) * sigmar;
+
+            Real f_l = 1.0;
+            Real f_r = 1.0;
+            taul *= taufact(k,j-1,i);
+            taur *= taufact(k,j,i);
+            GetTaufactor(taul+taur,f_r,1);
+            GetTaufactor(taul+taur,f_l,-1);
+
+            Real *s1n = &(sfac1_y_(j,i,ifr*nang));
+            Real *s2n = &(sfac2_y_(j,i,ifr*nang));
+            Real *velyn = &(vely_(k,j,i,ifr*nang));
+            Real adv = adv_vel(1,k,j,i);
+            SignalSpeed(adv, f_l, f_r, velyn, s1n, s2n);
+          }
+        }
+      }
+      for (int j=js; j<=je; ++j) {
+        pmb->pcoord->Face2Area(k,j  ,is,ie,x2area   );
+        pmb->pcoord->Face2Area(k,j+1,is,ie,x2area_p1);
+
+        for (int i=is; i<=ie; ++i) {
+          Real *vel_ln = &(vely_(k,j,i,0));
+          Real *vel_rn = &(vely_(k,j+1,i,0));
+          Real *coef2n = &(const_coef_(k,j,i,0));
+          Real *coef2ln= &(const_coef2_l_(k,j,i,0));
+          Real *coef2rn= &(const_coef2_r_(k,j,i,0));
+          Real *exp_coefn=&(exp_coef_(k,j,i,0));
+
+          Real *smax_l = &(sfac1_y_(j,i,0));
+          Real *smax_r = &(sfac1_y_(j+1,i,0));
+          Real *smin_l = &(sfac2_y_(j,i,0));
+          Real *smin_r = &(sfac2_y_(j+1,i,0));
+
+          Real areal = x2area(i);
+          Real arear = x2area_p1(i);
+
+          Real advl = 0.0;
+          Real advr = 0.0;
+          if (adv_flag_ > 0) {
+            advl = adv_vel(1,k,j,i);
+            advr = adv_vel(1,k,j+1,i);
+          }
+          Real *sm_diff1 = &(sm_diff1_(0));
+          Real *sm_diff2 = &(sm_diff2_(0));
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real diff = smax_l[n] - smin_l[n];
+            if (std::abs(diff) < TINY_NUMBER)
+              sm_diff1[n] = 0.0;
+            else
+              sm_diff1[n] = 1.0/diff;
+          }
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real diff = smax_r[n] - smin_r[n];
+            if (std::abs(diff) < TINY_NUMBER)
+              sm_diff2[n] = 0.0;
+            else
+              sm_diff2[n] = 1.0/diff;
+          }
+
+          SplitVelocity(vel_ln, vel_rn,advl, advr,smax_l,smin_l,smax_r,smin_r);
+
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real vl = vel_ln[n] - advl;
+            coef2n[n] += -areal * vel_im_l_(n) * sm_diff1[n];
+            coef2ln[n] = -areal * smax_l[n] * (vl - smin_l[n]) * sm_diff1[n];
+            exp_coefn[n] += -areal * vel_ex_l_(n) * sm_diff1[n];
+
+            Real vr = vel_rn[n] - advr;
+            coef2n[n] += arear * vel_im_r_(n) * sm_diff2[n];
+            coef2rn[n] = arear * smin_r[n] * (smax_r[n] - vr) * sm_diff2[n];
+            exp_coefn[n] += arear * vel_ex_r_(n) * sm_diff2[n];
+          }
+        }
+      }
+    }
+  }
+
+  // calculate x3-flux divergence
+  if (pmb->block_size.nx3 > 1) {
+    for (int k=ks; k<=ke+1; ++k) {
+      for (int j=js; j<=je; ++j) {
+        pco->CenterWidth3(k-1,j,is,ie,dxw1_);
+        pco->CenterWidth3(k,j,is,ie,dxw2_);
+        for (int i=is; i<=ie; ++i) {
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            Real sigmal = prad->sigma_a(k-1,j,i,ifr) + prad->sigma_s(k-1,j,i,ifr);
+            Real sigmar = prad->sigma_a(k,j,i,ifr) + prad->sigma_s(k,j,i,ifr);
+            Real taul = dxw1_(i) * sigmal;
+            Real taur = dxw2_(i) * sigmar;
+
+            Real f_l = 1.0;
+            Real f_r = 1.0;
+            taul *= taufact(k-1,j,i);
+            taur *= taufact(k,j,i);
+            GetTaufactor(taul+taur,f_r,1);
+            GetTaufactor(taul+taur,f_l,-1);
+
+            Real *s1n = &(sfac1_z_(k,j,i,ifr*nang));
+            Real *s2n = &(sfac2_z_(k,j,i,ifr*nang));
+            Real *velzn = &(velz_(k,j,i,ifr*nang));
+            Real adv = adv_vel(2,k,j,i);
+            SignalSpeed(adv, f_l, f_r, velzn, s1n, s2n);
+          }
+        }
+      }
+    }
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        pmb->pcoord->Face3Area(k  ,j,is,ie,x3area   );
+        pmb->pcoord->Face3Area(k+1,j,is,ie,x3area_p1);
+        for (int i=is; i<=ie; ++i) {
+          Real *smax_l = &(sfac1_z_(k,j,i,0));
+          Real *smax_r = &(sfac1_z_(k+1,j,i,0));
+          Real *smin_l = &(sfac2_z_(k,j,i,0));
+          Real *smin_r = &(sfac2_z_(k+1,j,i,0));
+          Real *vel_ln = &(velz_(k,j,i,0));
+          Real *vel_rn = &(velz_(k+1,j,i,0));
+          Real *coef3n = &(const_coef_(k,j,i,0));
+          Real *coef3ln= &(const_coef3_l_(k,j,i,0));
+          Real *coef3rn= &(const_coef3_r_(k,j,i,0));
+          Real *exp_coefn=&(exp_coef_(k,j,i,0));
+          Real areal = x3area(i);
+          Real arear = x3area_p1(i);
+          Real advl = 0.0;
+          Real advr = 0.0;
+          if (adv_flag_ > 0) {
+            advl = adv_vel(2,k,j,i);
+            advr = adv_vel(2,k+1,j,i);
+          }
+          Real *sm_diff1 = &(sm_diff1_(0));
+          Real *sm_diff2 = &(sm_diff2_(0));
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real diff = smax_l[n] - smin_l[n];
+            if (std::abs(diff) < TINY_NUMBER)
+              sm_diff1[n] = 0.0;
+            else
+              sm_diff1[n] = 1.0/diff;
+          }
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real diff = smax_r[n] - smin_r[n];
+            if (std::abs(diff) < TINY_NUMBER)
+              sm_diff2[n] = 0.0;
+            else
+              sm_diff2[n] = 1.0/diff;
+          }
+
+          SplitVelocity(vel_ln, vel_rn,advl, advr,smax_l,smin_l,smax_r,smin_r);
+
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real vl = vel_ln[n] - advl;
+            coef3n[n] += -areal * vel_im_l_(n) * sm_diff1[n];
+            coef3ln[n] = -areal * smax_l[n] * (vl - smin_l[n]) * sm_diff1[n];
+            exp_coefn[n] += -areal * vel_ex_l_(n) * sm_diff1[n];
+
+            Real vr = vel_rn[n] - advr;
+            coef3n[n] += arear * vel_im_r_(n) * sm_diff2[n];
+            coef3rn[n] = arear * smin_r[n] * (smax_r[n] - vr) * sm_diff2[n];
+            exp_coefn[n] += arear * vel_ex_r_(n) * sm_diff2[n];
+          }
+        }
+      }
+    }
+  }
+
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      pmb->pcoord->CellVolume(k,j,is,ie,vol);
+      for (int i=is; i<=ie; ++i) {
+        Real dtvol = wght/vol(i);
+        Real *coef1n = &(const_coef_(k,j,i,0));
+        Real *exp_coefn=&(exp_coef_(k,j,i,0));
+        Real *coef1ln=&(const_coef1_l_(k,j,i,0));
+        Real *coef1rn=&(const_coef1_r_(k,j,i,0));
+        for (int n=0; n<prad->n_fre_ang; ++n) {
+          coef1n[n] *= dtvol;
+          exp_coefn[n] *= dtvol;
+          coef1ln[n] *= dtvol;
+          coef1rn[n] *= dtvol;
+        }
+        if (pmb->block_size.nx2 > 1) {
+          Real *coef2ln = &(const_coef2_l_(k,j,i,0));
+          Real *coef2rn = &(const_coef2_r_(k,j,i,0));
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            coef2ln[n] *= dtvol;
+            coef2rn[n] *= dtvol;
+          }
+        }
+        if (pmb->block_size.nx3 > 1) {
+          Real *coef3ln = &(const_coef3_l_(k,j,i,0));
+          Real *coef3rn = &(const_coef3_r_(k,j,i,0));
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            coef3ln[n] *= dtvol;
+            coef3rn[n] *= dtvol;
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+void RadIntegrator::FirstOrderFluxDivergence(const int k, const int j, const int i,
+                                                          AthenaArray<Real> &ir_ini) {
+  NRRadiation *prad=pmy_rad;
+  MeshBlock *pmb=prad->pmy_block;
+
+  Real *divn = &(divflx_(k,j,i,0));
+  Real *irn = &(ir_ini(k,j,i,0));
+  Real *irln = &(ir_ini(k,j,i-1,0));
+  Real *irrn = &(ir_ini(k,j,i+1,0));
+  Real *exp_coefn = &(exp_coef_(k,j,i,0));
+  Real *coef1ln = &(const_coef1_l_(k,j,i,0));
+  Real *coef1rn = &(const_coef1_r_(k,j,i,0));
+
+  for (int n=0; n<prad->n_fre_ang; ++n) {
+    divn[n] = -(coef1ln[n] * irln[n] + coef1rn[n] * irrn[n]
+              + exp_coefn[n] * irn[n]);
+  }
+  if (adv_flag_ > 0) {
+    Real *advn = &(adv_flx_(k,j,i,0));
+    for (int n=0; n<prad->n_fre_ang; ++n) {
+      divn[n] += advn[n];
+    }
+  }
+  if (pmb->block_size.nx2 > 1) {
+    Real *irjln = &(ir_ini(k,j-1,i,0));
+    Real *irjrn = &(ir_ini(k,j+1,i,0));
+    Real *coef2ln = &(const_coef2_l_(k,j,i,0));
+    Real *coef2rn = &(const_coef2_r_(k,j,i,0));
+    for (int n=0; n<prad->n_fre_ang; ++n) {
+      Real temp = -(coef2ln[n] * irjln[n] + coef2rn[n] * irjrn[n]);
+      divn[n] += temp;
+    }
+  }
+
+  if (pmb->block_size.nx3 > 1) {
+    Real *irkln = &(ir_ini(k-1,j,i,0));
+    Real *irkrn = &(ir_ini(k+1,j,i,0));
+    Real *coef3ln = &(const_coef3_l_(k,j,i,0));
+    Real *coef3rn = &(const_coef3_r_(k,j,i,0));
+    for (int n=0; n<prad->n_fre_ang; ++n) {
+      Real temp = -(coef3ln[n] * irkln[n] + coef3rn[n] * irkrn[n]);
+      divn[n] += temp;
+    }
+  }
+  return;
+}
diff --git a/src/nr_radiation/implicit/rad_iteration.cpp b/src/nr_radiation/implicit/rad_iteration.cpp
new file mode 100644
index 0000000000..9d190b0dbd
--- /dev/null
+++ b/src/nr_radiation/implicit/rad_iteration.cpp
@@ -0,0 +1,229 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file rad_iteration.cpp
+//  \brief iterations to solve the transport equation implicitly
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <sstream>    // stringstream
+
+// Athena++ headers
+#include "../../defs.hpp"
+#include "../../field/field.hpp"
+#include "../../globals.hpp"
+#include "../../hydro/hydro.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../task_list/task_list.hpp"
+#include "../integrators/rad_integrators.hpp"
+#include "../radiation.hpp"
+#include "radiation_implicit.hpp"
+
+//--------------------------------------------------------------------------------------
+// \!fn void Iteration()
+// \brief function to perform iterations
+void IMRadiation::Iteration(
+    Mesh *pm, TimeIntegratorTaskList *ptlist, int stage) {
+  // perform Jacobi iteration including both source and flux terms
+  // The iteration step is: calculate flux, calculate source term,
+  // update specific intensity, compute error
+  MeshBlock *pmb = pm->my_blocks(0);
+  std::stringstream msg;
+
+  const Real wght = ptlist->stage_wghts[stage-1].beta*pm->dt;
+
+  if (stage <= ptlist->nstages) {
+    // go through all the mesh blocks
+    bool iteration = true;
+    int niter = 0;
+
+    // store the initial value before the iteration
+    // this is always needed for the RHS
+
+    // first save initial state
+    for(int nb=0; nb<pm->nblocal; ++nb) {
+      pmb = pm->my_blocks(nb);
+      NRRadiation *prad = pmb->pnrrad;
+      Hydro *ph = pmb->phydro;
+      Field *pf = pmb->pfield;
+      AthenaArray<Real> &ir_ini = prad->ir1;
+
+      // prepare t_gas and vel
+      if (stage == 1) {
+        ir_ini = prad->ir;
+      }
+      // use the current stage velocity for advection
+      prad->pradintegrator->GetTgasVel(pmb,wght,ph->u,ph->w,pf->bcc,ir_ini);
+
+      // Calculate advection flux due to flow velocity explicitly
+      // advection velocity uses the partially updated velocity and ir from half
+      // time step
+      if (prad->pradintegrator->adv_flag_ > 0) {
+        if (stage == 1) {
+          prad->pradintegrator->CalculateFluxes(prad->ir, 1);
+        } else {
+          prad->pradintegrator->CalculateFluxes(prad->ir,
+                                                prad->pradintegrator->rad_xorder);
+        }
+        // store the flux divergence due to advection
+        prad->pradintegrator->FluxDivergence(wght);
+      }
+
+      // prepare the coefficients
+      prad->pradintegrator->FirstOrderFluxDivergenceCoef(wght);
+
+      // prepare coefficients for angular fluxes
+      if (prad->angle_flag == 1) {
+        prad->pradintegrator->ImplicitAngularFluxesCoef(wght);
+      }
+
+      // always use initial guess from last step to keep the balance
+      //      if ((stage == 2) && (prad->pradintegrator->split_compton_ > 0))
+      //        prad->ir = ir_ini;
+
+      // ir_old always store the value from last iteration
+      // ir1 store the value at the beginning of the step
+      prad->ir_old = prad->ir;
+    }
+
+    // call the function to set SRJ parameters
+    if (srj_p > 0) {
+      SetSRJParameters(this);
+    }
+
+    srj_level = 0;
+    srj_cnt = 0;
+    if (srj_p > 0)
+      omega = srj_w[0];
+
+    while(iteration) {
+      // initialize the pointer
+      pmb = pm->my_blocks(0);
+      sum_full_ = 0.0;
+      sum_diff_ = 0.0;
+
+      // using TaskList to handle
+      // operations during each iteration
+      if (rb_or_not > 0)
+        rb_or_not = 1;
+      // red cells
+      pimraditlist->DoTaskListOneStage(wght);
+
+      if (rb_or_not > 0) {
+        rb_or_not = 2;
+        // black cells
+        pimraditlist->DoTaskListOneStage(wght);
+      }
+
+
+      for(int nb=0; nb<pm->nblocal; ++nb) {
+        pmb = pm->my_blocks(nb);
+        NRRadiation *prad = pmb->pnrrad;
+        // copy the solution over
+        prad->ir_old = prad->ir;
+        sum_full_ += prad->sum_full;
+        sum_diff_ += prad->sum_diff;
+      }
+
+      // MPI sum across all the cores
+#ifdef MPI_PARALLEL
+      Real global_sum = 0.0;
+      Real global_diff = 0.0;
+      MPI_Allreduce(&sum_full_, &global_sum, 1, MPI_ATHENA_REAL, MPI_SUM, MPI_COMM_WORLD);
+      MPI_Allreduce(&sum_diff_, &global_diff, 1, MPI_ATHENA_REAL,
+                    MPI_SUM, MPI_COMM_WORLD);
+
+      sum_full_ = global_sum;
+      sum_diff_ = global_diff;
+#endif
+
+      niter++;
+      Real tot_res = sum_diff_/sum_full_;
+
+      if ((niter > nlimit_) || tot_res < error_limit_)
+        iteration = false;
+
+      if (srj_p > 0) {
+        srj_cnt++;
+        if (srj_cnt >= srj_q[srj_level]) {
+          srj_level++;
+          if (srj_level == srj_p)
+            srj_level = 0;
+          srj_cnt = 0;
+          omega = srj_w[srj_level];
+        }
+      }
+    }
+
+    if (Globals::my_rank == 0) {
+      int output_info = 0;
+      if (pm->ncycle_out != 0) {
+        if (pm->ncycle%pm->ncycle_out == 0)
+          output_info = 1;
+      }
+      if ( output_info > 0)
+        std::cout << "Iteration stops at niter: " << niter
+                  << " relative error: " << sum_diff_/sum_full_ << std::endl;
+    }
+
+    // now calculate the rad source terms
+    for(int nb=0; nb<pm->nblocal; ++nb) {
+      pmb = pm->my_blocks(nb);
+      NRRadiation *prad = pmb->pnrrad;
+      if (prad->set_source_flag > 0)
+        prad->pradintegrator->GetHydroSourceTerms(pmb, prad->ir1, prad->ir);
+    }
+
+    if ((pm->my_blocks(0)->pnrrad->nfreq > 1) &&
+        (pm->my_blocks(0)->pnrrad->pradintegrator->compton_flag_ > 0) &&
+        pm->my_blocks(0)->pnrrad->pradintegrator->split_compton_ > 0)
+      pimradcomptlist->DoTaskListOneStage(wght);
+
+    // After iteration,
+    // add radiation source term to hydro
+    // update hydro boundary
+    // update opacity
+    pimradhylist->DoTaskListOneStage(wght);
+  }
+}
+
+
+void IMRadiation::CheckResidual(MeshBlock *pmb,
+                                AthenaArray<Real> &ir_old, AthenaArray<Real> &ir_new) {
+  NRRadiation *prad = pmb->pnrrad;
+  int &nang =prad->nang;
+  int &nfreq=prad->nfreq;
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+  prad->sum_diff = 0.0;
+  prad->sum_full = 0.0;
+  for(int k=ks; k<=ke; ++k) {
+    for(int j=js; j<=je; ++j) {
+      for(int i=is; i<=ie; ++i) {
+        for(int ifr=0; ifr<nfreq; ++ifr) {
+          Real *iro = &(ir_old(k,j,i,ifr*nang));
+          Real *irn = &(ir_new(k,j,i,ifr*nang));
+          for(int n=0; n<nang; ++n) {
+            prad->sum_diff += std::abs(iro[n] - irn[n]);
+            prad->sum_full += std::abs(irn[n]);
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/src/nr_radiation/implicit/radiation_implicit.cpp b/src/nr_radiation/implicit/radiation_implicit.cpp
new file mode 100644
index 0000000000..3e03cdc675
--- /dev/null
+++ b/src/nr_radiation/implicit/radiation_implicit.cpp
@@ -0,0 +1,81 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file radiation_implicit.cpp
+//  \brief implementation of implicit radiation class
+//======================================================================================
+
+
+// C headers
+
+// C++ headers
+#include <algorithm>
+#include <cstdio>  // fopen and fwrite
+#include <iostream>  // cout
+#include <sstream>  // msg
+#include <stdexcept> // runtime erro
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../globals.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../parameter_input.hpp"
+#include "../integrators/rad_integrators.hpp"
+#include "../radiation.hpp"
+#include "radiation_implicit.hpp"
+
+
+// set the default parameters for SRJ
+inline void DefaultSRJ(IMRadiation *pimrad) {
+  // One example:
+  // SRJ_P = 7
+  // SRJ_w1 = 2.0, SRJ_w2 = 1.8,SRJ_w3 = 1.6,SRJ_w4 = 1.4,SRJ_w5 = 1.2,
+  // SRJ_w6 = 1.0,SRJ_w7 = 0.6
+  // SRJ_Q1 = 1,SRJ_Q2 = 1,SRJ_Q3 = 1,SRJ_Q4 = 1,SRJ_Q5 = 1,SRJ_Q6 = 1,SRJ_Q7 = 1
+
+  for (int i=0; i<9; ++i) {
+    pimrad->srj_q[i] = 0;
+    pimrad->srj_w[i] = 1.0;
+  }
+  return;
+}
+
+IMRadiation::IMRadiation(Mesh *pm, ParameterInput *pin) {
+  // read in the parameters
+  // maximum number of iterations
+  nlimit_ = pin->GetOrAddInteger("radiation","nlimit",100);
+  error_limit_ =  pin->GetOrAddReal("radiation","error_limit",1.e-6);
+  cfl_rad = pin->GetOrAddReal("radiation","cfl_rad",1.0);
+  ite_scheme = pin->GetOrAddInteger("radiation","iteration",1);
+  rb_or_not = pin->GetOrAddInteger("radiation","red_or_black",0);
+
+  omega = pin->GetOrAddReal("radiation","omega",1.0);
+  srj_p = std::min(9, pin->GetOrAddInteger("radiation","SRJ_P",0));
+  srj_level = 0;
+  srj_cnt = 0;
+
+  pimraditlist = new IMRadITTaskList(pm);
+  pimradhylist = new IMRadHydroTaskList(pm);
+  pimradcomptlist = new IMRadComptTaskList(pm);
+
+  // set default parameters
+  SetSRJParameters = DefaultSRJ;
+}
+
+
+void IMRadiation::EnrollSRJFunction(SRJFunc MySRJFunction) {
+  SetSRJParameters = MySRJFunction;
+}
diff --git a/src/nr_radiation/implicit/radiation_implicit.hpp b/src/nr_radiation/implicit/radiation_implicit.hpp
new file mode 100644
index 0000000000..d34875a5a0
--- /dev/null
+++ b/src/nr_radiation/implicit/radiation_implicit.hpp
@@ -0,0 +1,76 @@
+#ifndef NR_RADIATION_IMPLICIT_RADIATION_IMPLICIT_HPP_
+#define NR_RADIATION_IMPLICIT_RADIATION_IMPLICIT_HPP_
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+// See LICENSE file for full public license information.
+//======================================================================================
+//! \file radiation_implicit.hpp
+//  \brief implicit radiation class definitions
+//======================================================================================
+
+
+
+// C++ headers
+#include <cstdint>     // int64_t
+#include <functional>  // reference_wrapper
+#include <string>
+#include <vector>
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../task_list/im_rad_task_list.hpp"
+
+class Mesh;
+class ParameterInput;
+class NRRadiation;
+class RadIntegrator;
+class TimeIntegratorTaskList;
+
+class IMRadiation {
+  friend class NRRadiation;
+  friend class RadIntegrator;
+ public:
+  IMRadiation(Mesh *pm, ParameterInput *pin);
+  //  ~Radiation();
+
+  void Iteration(Mesh *pm,
+             TimeIntegratorTaskList *ptlist, int stage);
+  void CheckResidual(MeshBlock *pmb,
+        AthenaArray<Real> &ir_old, AthenaArray<Real> &ir_new);
+
+  Real cfl_rad; // the additional CFL number.
+                // Small cfl_rad is good for convergence,
+                // but with smaller time step
+
+  int ite_scheme;
+  int rb_or_not;
+
+  IMRadITTaskList *pimraditlist;
+  IMRadHydroTaskList *pimradhylist;
+  IMRadComptTaskList *pimradcomptlist;
+
+  // scheduled relaxation jacobi method
+  //   // X.Yang, R.Mittal, JCP 274 (2014) 695-708.
+  Real omega;
+  int srj_p; // <= 9; 0 means off.
+  int srj_q[9]; // repetitions
+  Real srj_w[9]; // omegas
+
+  // internal data
+  int srj_level, srj_cnt; // internal counts
+
+  // The function pointer for srj parameters
+  SRJFunc SetSRJParameters;
+  void EnrollSRJFunction(SRJFunc MySRJFunction);
+
+
+ private:
+  Real sum_diff_;
+  Real sum_full_;
+  int nlimit_;       // threadhold for the number of iterations
+  Real error_limit_;
+};
+
+#endif // NR_RADIATION_IMPLICIT_RADIATION_IMPLICIT_HPP_
diff --git a/src/nr_radiation/integrators/frame_transform.cpp b/src/nr_radiation/integrators/frame_transform.cpp
new file mode 100644
index 0000000000..ae9e08b88a
--- /dev/null
+++ b/src/nr_radiation/integrators/frame_transform.cpp
@@ -0,0 +1,156 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file frame_transform.cpp
+//  \brief  Perform frame transformation
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <sstream>  // msg
+#include <stdexcept>  // runtime_error
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../utils/utils.hpp"
+#include "../radiation.hpp"
+
+// this class header
+#include "./rad_integrators.hpp"
+
+//--------------------------------------------------------------------------------------
+//! \fn RadIntegrator::LabToCom(const Real vx, const Real vy, const Real vz,
+//                          AthenaArray<Real> &ir, AthenaArray<Real> &ir_cm)
+//  \brief Transform specific intensity from lab frame to co-moving frame
+// with flow velocity vx, vy, vz
+
+
+// this is only used for one frequency group
+void RadIntegrator::LabToCom(const Real vx, const Real vy, const Real vz,
+                          Real *mux, Real *muy, Real *muz,
+                          Real *ir_lab, AthenaArray<Real> &ir_cm) {
+  //Real& prat = pmy_rad->prat;
+  Real invcrat = 1.0/pmy_rad->crat;
+  int& nang=pmy_rad->nang;
+  int& nfreq=pmy_rad->nfreq;
+
+  // square of Lorentz factor
+  Real lorzsq = 1.0/(1.0 - (vx * vx + vy * vy + vz * vz) * invcrat * invcrat);
+
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    for (int n=0; n<nang; n++) {
+      Real vnc = vx * mux[n] + vy * muy[n] + vz * muz[n];
+      vnc = 1.0 - vnc * invcrat;
+      Real coef = vnc * vnc * vnc * vnc * lorzsq * lorzsq;
+      ir_cm(ifr*nang+n) = ir_lab[ifr*nang+n] * coef;
+    }
+  }
+  return;
+}
+
+//--------------------------------------------------------------------------------------
+//! \fn RadIntegrator::ComToLab(const Real vx, const Real vy, const Real vz,
+//                          AthenaArray<Real> &ir, AthenaArray<Real> &ir_cm)
+//  \brief Transform specific intensity from co-moving frame to lab frame
+// with flow velocity vx, vy, vz
+
+void RadIntegrator::ComToLab(const Real vx, const Real vy, const Real vz,
+                          Real *mux, Real *muy, Real *muz,
+                          AthenaArray<Real> &ir_cm, Real *ir_lab) {
+  Real invcrat = 1.0/pmy_rad->crat;
+  int& nang=pmy_rad->nang;
+  int& nfreq=pmy_rad->nfreq;
+
+  // square of Lorentz factor
+  Real lorzsq = 1.0/(1.0 - (vx * vx + vy * vy + vz * vz) * invcrat * invcrat);
+
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    for (int n=0; n<nang; n++) {
+      Real vnc = vx * mux[n] + vy * muy[n] + vz * muz[n];
+
+      vnc = 1.0 - vnc * invcrat;
+      Real coef = vnc * vnc * vnc * vnc * lorzsq * lorzsq;
+
+      ir_lab[ifr*nang+n] = ir_cm(ifr*nang+n) / coef;
+    }
+  }
+
+
+  return;
+}
+
+// transform co-moving frame intensity to the lab frame for a given spectrum in
+// co-moving frame
+void RadIntegrator::ComToLabMultiGroup(const Real vx, const Real vy, const Real vz,
+                          Real *mux, Real *muy, Real *muz,
+                          AthenaArray<Real> &ir_cm, Real *ir_lab) {
+  Real invcrat = 1.0/pmy_rad->crat;
+  int& nang=pmy_rad->nang;
+  int& nfreq=pmy_rad->nfreq;
+  AthenaArray<Real> split_ratio;
+  AthenaArray<int> map_start, map_end;
+
+  // square of Lorentz factor
+  Real lorz = std::sqrt(1.0/(1.0 - (vx * vx + vy * vy + vz * vz) * invcrat * invcrat));
+  // first, get the lorentz transformation factor
+  // now calculate the actual transformation factor
+
+  for (int n=0; n<nang; ++n) {
+    Real vnc = vx * mux[n] + vy * muy[n] + vz * muz[n];
+    vnc = 1.0 - vnc * invcrat;
+    Real cm_nu = vnc * lorz;
+
+    for (int ifr=0; ifr<nfreq; ++ifr)
+      ir_ori_(ifr) = ir_cm(ifr*nang+n);
+
+    split_ratio.InitWithShallowSlice(split_ratio_,3,n,1);
+    map_start.InitWithShallowSlice(map_bin_start_,2,n,1);
+    map_end.InitWithShallowSlice(map_bin_end_,2,n,1);
+
+    MapCmToLabFrequency(cm_nu,split_ratio,map_start,map_end,ir_ori_,ir_done_);
+
+    for (int ifr=0; ifr<nfreq; ++ifr) {
+      ir_lab[ifr*nang+n] = ir_done_(ifr)/(cm_nu*cm_nu*cm_nu*cm_nu);
+    }
+  }
+  return;
+}
+
+//--------------------------------------------------------------------------------------
+//! \fn RadIntegrator::ComAngle(const Real vx, const Real vy, const Real vz,
+//                  Real mux, Real muy, Real muz, Real mux0, Real muy0, Real muz0)
+//  \brief Transform angles from lab frame to co-moving frame
+
+
+void RadIntegrator::ComAngle(const Real vx, const Real vy, const Real vz,
+                  Real mux, Real muy, Real muz, Real *mux0, Real *muy0, Real *muz0) {
+  Real invcrat = 1.0/pmy_rad->crat;
+  // square of Lorentz factor
+  Real lorz = 1.0/(1.0 - (vx * vx + vy * vy + vz * vz) * invcrat * invcrat);
+  lorz = std::sqrt(lorz);
+  Real vdotn = vx * mux + vy * muy + vz * muz;
+
+  Real angcoef = lorz * invcrat * (1.0 - lorz * vdotn * invcrat/(1.0 + lorz));
+  Real incoef = 1.0/(lorz*(1.0-vdotn * invcrat));
+
+  (*mux0) = (mux - vx * angcoef) * incoef;
+  (*muy0) = (muy - vy * angcoef) * incoef;
+  (*muz0) = (muz - vz * angcoef) * incoef;
+  return;
+}
diff --git a/src/nr_radiation/integrators/multi_group.cpp b/src/nr_radiation/integrators/multi_group.cpp
new file mode 100644
index 0000000000..5b75911a47
--- /dev/null
+++ b/src/nr_radiation/integrators/multi_group.cpp
@@ -0,0 +1,506 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file multi_group.cpp
+//  \brief  Add multi_group  source terms
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <sstream>  // msg
+#include <stdexcept>  // runtime_error
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../utils/utils.hpp"
+#include "../radiation.hpp"
+
+// this class header
+#include "./rad_integrators.hpp"
+
+
+//--------------------------------------------------------------------------------------
+//The function calculate the amount of shift for each I we need
+// The default frequency grid is nu_grid with nfreq frequency bins
+// This covers from 0 to infty
+// the original frequency grid is nu_grid(0,...,nfreq-2)
+// the shifted frequency grid is cm_nu(0,...,nfreq-2)
+// for each angle, the shift is the same \gamma (1-nv/c)
+// The procedure is: Interpolate the intensity to the frequency grid nu_grid from cm_nu
+// solve the source terms and update co-moving frame intensity
+// interpolate the intensity back to the frequency grid cm_nu from nu_grid for each angle
+// Then transform the intensity back to the lab-frame
+
+//Giving the frequency integrated intensity for each group in the co-moving frame,
+// get the monochromatic intensity at the face and center of the frequency grid
+//monochromatic intensity is always 0 at nu=0, the values at frequency grid faces should
+// satisfy 0.5*(ir_face(f)+ir_face(f+1))*delta_nu = ir_f
+// as we assume piecewise linear spectrum shape, and continuous intensity across frequency
+void RadIntegrator::MapLabToCmFrequency(Real &tran_coef,
+                   AthenaArray<Real> &split_ratio,
+                   AthenaArray<int> &map_start, AthenaArray<int> &map_end,
+                   AthenaArray<Real> &ir_cm, AthenaArray<Real> &ir_shift) {
+  int &nfreq=pmy_rad->nfreq;
+
+  // prepare the frequency bin width
+  for (int ifr=0; ifr<nfreq-1; ++ifr) {
+    delta_nu_n_(ifr) = pmy_rad->delta_nu(ifr) * tran_coef;
+  }
+
+  GetCmMCIntensity(ir_cm, delta_nu_n_, ir_face_);
+  // calculate the shift ratio
+  ForwardSplitting(tran_coef, ir_cm, ir_face_, split_ratio, map_start, map_end);
+  MapIrcmFrequency(split_ratio, map_start, map_end, ir_cm,ir_shift);
+  return;
+}
+
+void RadIntegrator::GetCmMCIntensity(AthenaArray<Real> &ir_cm,
+                          AthenaArray<Real> &delta_nu_n, AthenaArray<Real> &ir_face) {
+  int &nfreq = pmy_rad->nfreq;
+  if (nfreq > 1) {
+    ir_face(0) = 0.0;
+    for (int ifr=1; ifr<nfreq; ++ifr) {
+      ir_face(ifr) = std::max(2.0*ir_cm(ifr-1)/delta_nu_n(ifr-1) -
+                                             ir_face(ifr-1), 0.0);
+    }
+  }
+}
+
+// general function to split any array in the frequency bin [\Gamma nu_f]
+// to the frequency bin [nu_f]
+// ir_last_bin is used to determine the shift in the last frequency bin
+// assuming BlackBody spectrum in the last frequency bin
+// In other bins, we assume piecewise constant, split each bin according
+// to frequency overlap
+void RadIntegrator::ForwardSplitting(
+    Real &tran_coef, AthenaArray<Real> &ir_cm, AthenaArray<Real> &ir_face,
+    AthenaArray<Real> &split_ratio, AthenaArray<int> &map_start,
+    AthenaArray<int> &map_end) {
+  int &nfreq = pmy_rad->nfreq;
+  Real *nu_lab = &(pmy_rad->nu_grid(0));
+  // check to make sure nfreq > 2
+  if (nfreq < 2) {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in function [ForwardSplitting]"
+        << std::endl << "nfreq '" << nfreq <<
+          "' is smaller than 2! ";
+    ATHENA_ERROR(msg);
+  }
+  // map intensity to the desired bin
+  // This is a generic function to shift any array
+  if (tran_coef >= 1) {
+    for (int ifr=0; ifr<nfreq-1; ++ifr) {
+      Real nu_l = nu_lab[ifr] * tran_coef;
+      Real nu_r = nu_lab[ifr+1] * tran_coef;
+      int l_bd = ifr;
+      int r_bd = ifr;
+
+      while((nu_l > nu_lab[l_bd+1]) && (l_bd < nfreq-1))   l_bd++;
+      r_bd = l_bd; // r_bd always > l_bd
+      while((nu_r > nu_lab[r_bd+1]) && (r_bd < nfreq-1))   r_bd++;
+
+      if (r_bd-l_bd+1 > nmax_map_) {
+        std::stringstream msg;
+        msg << "### FATAL ERROR in function [ForwardSplitting]"
+          << std::endl << "Frequency shift '" << r_bd-l_bd+1 <<
+          "' larger than maximum allowed " << nmax_map_;
+        ATHENA_ERROR(msg);
+      }
+      map_start(ifr) = l_bd;
+      map_end(ifr) = r_bd;
+      SplitFrequencyBinLinear(l_bd, r_bd, nu_lab, nu_l, nu_r, ir_face(ifr),
+                                    ir_face(ifr+1), &(split_ratio(ifr,0)));
+    }
+    // the last frequency bin
+    map_start(nfreq-1) = nfreq-1;
+    map_end(nfreq-1) = nfreq-1;
+    split_ratio(nfreq-1,0) = 1.0;
+  } else {
+    for (int ifr=0; ifr<nfreq-1; ++ifr) {
+      Real nu_l = nu_lab[ifr] * tran_coef;
+      Real nu_r = nu_lab[ifr+1] * tran_coef;
+      int l_bd = ifr;
+      int r_bd = ifr;
+
+      while((nu_r < nu_lab[r_bd]) && (r_bd > 0))   r_bd--;
+      l_bd = r_bd; // r_bd always > l_bd
+      while((nu_l < nu_lab[l_bd]) && (l_bd > 0))   l_bd--;
+
+      if (r_bd-l_bd+1 > nmax_map_) {
+        std::stringstream msg;
+        msg << "### FATAL ERROR in function [ForwardSplitting]"
+            << std::endl << "Frequency shift '" << r_bd-l_bd+1 <<
+          "' larger than maximum allowed " << nmax_map_;
+        ATHENA_ERROR(msg);
+      }
+      map_start(ifr) = l_bd;
+      map_end(ifr) = r_bd;
+      SplitFrequencyBinLinear(l_bd, r_bd, nu_lab, nu_l, nu_r, ir_face(ifr),
+                                    ir_face(ifr+1), &(split_ratio(ifr,0)));
+    }
+    Real nu_l = nu_lab[nfreq-1] * tran_coef;
+    int r_bd = nfreq-1;
+    int l_bd = nfreq-2;// it will always be <= current bin
+    while((nu_l < nu_lab[l_bd]) && (l_bd > 0))   l_bd--;
+        // This frequency bin now maps to l_bd to r_bd
+    map_start(nfreq-1) = l_bd;
+    map_end(nfreq-1) = r_bd;
+      // nu_l/kt
+    Real nu_tr = pmy_rad->EffectiveBlackBody(ir_cm(nfreq-1), nu_l);
+
+      // FitIntPlanckFunc is integral _0 to nu_tr
+      // the integral we need is 1 - ori_norm
+    Real ori_norm = pmy_rad->FitIntPlanckFunc(nu_tr);
+    Real div_ori = 0.0;
+    if (1.0 - ori_norm > TINY_NUMBER)
+      div_ori = 1.0/(1.0 - ori_norm);
+
+
+      // the first bin
+      // the effective temperature 1/T = nu_tr/nu_l
+    Real ratio = pmy_rad->FitIntPlanckFunc(nu_tr*nu_lab[l_bd+1]/nu_l);
+
+      // the difference is (1 - ori_norm) - (1 - ratio)
+    split_ratio(nfreq-1,0) = (ratio - ori_norm) * div_ori;
+    Real sum = split_ratio(nfreq-1,0);
+
+    for (int m=l_bd+1; m<r_bd; ++m) {
+      Real ratio_r = pmy_rad->FitIntPlanckFunc(nu_tr*nu_lab[m+1]/nu_l);
+      Real ratio_l = pmy_rad->FitIntPlanckFunc(nu_tr*nu_lab[m]/nu_l);
+      split_ratio(nfreq-1,m-l_bd) = (ratio_r - ratio_l) * div_ori;
+      sum += split_ratio(nfreq-1,m-l_bd);
+    }
+
+    split_ratio(nfreq-1,r_bd-l_bd) = 1.0 - sum;
+    // in the case that the blackbody tail is too small, we assume
+    // power law spectrum shape to determine the ratio
+    // Ir=ir_face(\nu/nu_0)^alpha
+    // integral is -ir_face*nu_0/(alpha+1) and alpha < -1
+    // ratio is (nu_2^alpha+1- nu_1^alpha+1)/nu_0^alpha+1
+  }
+  return;
+}
+
+
+// interpolate co-moving frame specific intensity over frequency grid
+// every bin is
+void RadIntegrator::MapIrcmFrequency( AthenaArray<Real> &split_ratio,
+      AthenaArray<int> &map_start, AthenaArray<int> &map_end,
+      AthenaArray<Real> &input_array, AthenaArray<Real> &shift_array) {
+  int &nfreq = pmy_rad->nfreq;
+
+  // initialize ir_shift to be 0
+  shift_array.ZeroClear();
+
+  // check to make sure nfreq > 2
+  if (nfreq < 2) {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in function [MapIrcmFrequency]"
+        << std::endl << "nfreq '" << nfreq <<
+          "' is smaller than 2! ";
+    ATHENA_ERROR(msg);
+  }
+
+  // map intensity to the desired bin
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    // map shifted intensity to the nu_grid
+    int &fre_start=map_start(ifr);
+    int &fre_end = map_end(ifr);
+    for (int m=fre_start; m<=fre_end; ++m) {
+      shift_array(m) += input_array(ifr) * split_ratio(ifr,m-fre_start);
+    }
+  }
+  return;
+}
+
+// constrcutre the matrix when map between different frequency grids
+// Matrix x intensity in shifted grid = intensity in default grid
+// get the matrix inversion if possible
+bool RadIntegrator::FreMapMatrix(AthenaArray<Real> &split_ratio,
+          Real &tran_coef, AthenaArray<int> &map_bin_start,
+          AthenaArray<int> &map_bin_end, AthenaArray<int> &map_count,
+                                      AthenaArray<Real> &map_matrix) {
+  int &nfreq = pmy_rad->nfreq;
+
+  map_matrix.ZeroClear();
+  map_count.ZeroClear();
+
+  bool invertible = true;
+  // first, check the diagonal elements are non-zero
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    if (split_ratio(ifr,ifr-map_bin_start(ifr)) < TINY_NUMBER) {
+      invertible = false;
+      return invertible;
+    }
+  }
+
+  // now construct the frequency map matrix for each angle
+  if (tran_coef >= 1) {
+    // fre_map_matrix is lower triangle
+    // make sure fre_map_matrix(ifr,0) is always the diagonal
+    for (int ifr=0; ifr<nfreq; ++ifr) {
+      int &fre_start=map_bin_start(ifr);
+      int &fre_end = map_bin_end(ifr);
+      // m is always larger than ifr
+      // lower triangle, when store the matrix, we always start from diagonal
+      for (int m=fre_start; m<=fre_end; ++m) {
+        map_matrix(m,m-ifr) = split_ratio(ifr,m-fre_start);
+        map_count(m)++;
+      }
+    }
+  } else {
+    // fre_map_matrix is upper triangle
+    // make sure fre_map_matrix(ifr,0) is always the diagonal
+    for (int ifr=0; ifr<nfreq; ++ifr) {
+      int &fre_start=map_bin_start(ifr);
+      int &fre_end = map_bin_end(ifr);
+      // m is always smaller than ifr
+      // upper triangle, when store the matrix, we always start from diagonal
+      for (int m=fre_start; m<=fre_end; ++m) {
+        map_matrix(m,ifr-m) = split_ratio(ifr,m-fre_start);
+        map_count(m)++;
+      }
+    }
+  }
+  return invertible;
+}
+
+// We have the equation map_matrix * shift_array = input_array
+// need to calculate inverse_map_matrix * input_array
+
+// if this method cause negative intensity, we will give up
+bool RadIntegrator::InverseMapFrequency(
+    Real &tran_coef,
+    AthenaArray<int> &map_count, AthenaArray<Real> &map_matrix,
+    AthenaArray<Real> &input_array, AthenaArray<Real> &shift_array) {
+  int &nfreq = pmy_rad->nfreq;
+  // clear zero
+  shift_array.ZeroClear();
+  // Now invert the matrix split_ratio
+  // we need to do this for each angle, all frequency bins
+  if (tran_coef >= 1) {
+    // map_matrix is a lower triangle matrix
+    // first, ifr=0
+    shift_array(0) = input_array(0)/map_matrix(0,0);
+    for (int ifr=1; ifr<nfreq; ++ifr) {
+      shift_array(ifr) = input_array(ifr);
+      for (int m=1; m<map_count(ifr); ++m) {
+        shift_array(ifr) -= map_matrix(ifr,m) * shift_array(ifr-m);
+      }
+      shift_array(ifr) /= map_matrix(ifr,0);
+      if (shift_array(ifr) < 0.0)
+        return false;
+    }
+  } else {
+    // map_matrix is a upper triangle,
+    // we need to start from ifr=nfreq-1
+
+    // when flag for fail, set intensity to 0
+    shift_array(nfreq-1) = input_array(nfreq-1)/map_matrix(nfreq-1,0);
+
+    for (int ifr=nfreq-2; ifr>=0; --ifr) {
+      shift_array(ifr) = input_array(ifr);
+      for (int m=1; m<map_count(ifr); ++m) {
+        shift_array(ifr) -= map_matrix(ifr,m) * shift_array(ifr+m);
+      }
+      shift_array(ifr) /= map_matrix(ifr,0);
+      if (shift_array(ifr) < 0.0)
+        return false;
+    }
+  }
+  return true;
+}
+
+// fit a linear line between nu_l and nu_r for ir_l and ir_r
+void RadIntegrator::SplitFrequencyBinLinear(
+    int &l_bd, int &r_bd, Real *nu_lab, Real &nu_l, Real &nu_r,
+    Real &ir_l, Real &ir_r, Real *split_ratio) {
+  Real ir_sum = ir_l + ir_r;
+  if ((r_bd == l_bd) || (ir_sum < TINY_NUMBER)) {
+    if (r_bd == l_bd) {
+      split_ratio[0] = 1.0;
+    } else {
+    // determine the ratio based on the frequency grid only
+      Real delta_nu = 1.0/(nu_r - nu_l);
+      Real sum = 0.0;
+      split_ratio[0] = (nu_lab[l_bd+1]-nu_l)*delta_nu;
+      sum=split_ratio[0];
+      for (int m=l_bd+1; m<r_bd; ++m) {
+        split_ratio[m-l_bd] = (nu_lab[m+1]-nu_lab[m])*delta_nu;
+        sum += split_ratio[m-l_bd];
+      }
+      split_ratio[r_bd-l_bd] = 1.0 - sum;
+    }
+  } else {
+    Real sum = 0.0;
+    Real delta_nu = 1.0/(nu_r - nu_l);
+    Real nu_ratio = (nu_lab[l_bd+1] - nu_l)*delta_nu;
+    Real ir_l_bd1 = ir_l + (ir_r-ir_l)*nu_ratio;
+    // between nu_l and nu_lab[l_bd+1]
+    split_ratio[0] = ((ir_l_bd1+ir_l)/ir_sum)*nu_ratio;
+    sum = split_ratio[0];
+    for (int m=l_bd+1; m< r_bd; ++m) {
+      nu_ratio = (nu_lab[m+1] - nu_lab[m])*delta_nu;
+      Real nu_ratio2 = (nu_lab[m+1] - nu_l) * delta_nu;
+      Real nu_ratio1 = (nu_lab[m] - nu_l) * delta_nu;
+      Real ir_l_m =  ir_l + (ir_r-ir_l)*nu_ratio1;
+      Real ir_l_m1 =  ir_l + (ir_r-ir_l)*nu_ratio2;
+      split_ratio[m-l_bd] = ((ir_l_m+ir_l_m1)/ir_sum)*nu_ratio;
+      sum += split_ratio[m-l_bd];
+    }
+    // make sure the sum is always 1
+    split_ratio[r_bd-l_bd] = 1.0 - sum;
+  }
+}
+
+// Ir_cm and ir_shift are both co-moving frame intensities
+// Ir_shift is defined in the same frequency grid for all angles
+// Ir_cm is defined in the corresponding frequency grid as transoformed
+// from lab frame
+void RadIntegrator::MapCmToLabFrequency(Real &tran_coef,
+                    AthenaArray<Real> &split_ratio,
+                    AthenaArray<int> &map_start, AthenaArray<int> &map_end,
+                    AthenaArray<Real> &ir_shift, AthenaArray<Real> &ir_cm) {
+  //Real *nu_fixed = &(pmy_rad->nu_grid(0));
+  AthenaArray<Real> &delta_nu = pmy_rad->delta_nu;
+  // now call the function to get value at frequency face
+  GetCmMCIntensity(ir_shift, delta_nu, ir_face_);
+
+  BackwardSplitting(tran_coef, ir_shift, ir_face_, split_ratio,
+                                     map_start,map_end);
+  MapIrcmFrequency(split_ratio, map_start, map_end, ir_shift, ir_cm);
+  return;
+}
+
+// general function to split any array in the frequency bin [nu_f]
+// to the frequency bin [Gamma nu_f]
+// ir_last_bin is used to determine the shift in the last frequency bin
+// assuming BlackBody spectrum in the last frequency bin
+// In other bins, we assume piecewise constant, split each bin according
+// to frequency overlap
+void RadIntegrator::BackwardSplitting(Real &tran_coef,
+                      AthenaArray<Real> &ir_cm, AthenaArray<Real> &ir_face,
+                      AthenaArray<Real> &split_ratio,
+                      AthenaArray<int> &map_start,AthenaArray<int> &map_end) {
+  int &nfreq = pmy_rad->nfreq;
+  Real *nu_lab = &(pmy_rad->nu_grid(0));
+  Real *nu_shift = &(nu_shift_(0));
+  // check to make sure nfreq > 2
+  if (nfreq < 2) {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in function [ForwardSplitting]"
+        << std::endl << "nfreq '" << nfreq <<
+          "' is smaller than 2! ";
+    ATHENA_ERROR(msg);
+  }
+
+  for (int ifr=0; ifr<nfreq; ++ifr)
+    nu_shift_(ifr) = nu_lab[ifr] * tran_coef;
+
+  // map intensity to the desired bin
+  // This is a generic function to shift any array
+  if (tran_coef < 1) {
+    for (int ifr=0; ifr<nfreq-1; ++ifr) {
+      Real nu_l = nu_lab[ifr];
+      Real nu_r = nu_lab[ifr+1];
+      int l_bd = ifr;
+      int r_bd = ifr;
+
+      while((nu_l > nu_shift[l_bd+1]) && (l_bd < nfreq-1))   l_bd++;
+      r_bd = l_bd; // r_bd always > l_bd
+      while((nu_r > nu_shift[r_bd+1]) && (r_bd < nfreq-1))   r_bd++;
+
+      if (r_bd-l_bd+1 > nmax_map_) {
+        std::stringstream msg;
+        msg << "### FATAL ERROR in function [ForwardSplitting]"
+          << std::endl << "Frequency shift '" << r_bd-l_bd+1 <<
+          "' larger than maximum allowed " << nmax_map_;
+        ATHENA_ERROR(msg);
+      }
+      map_start(ifr) = l_bd;
+      map_end(ifr) = r_bd;
+      SplitFrequencyBinLinear(l_bd, r_bd, nu_shift, nu_l, nu_r, ir_face(ifr),
+                                    ir_face(ifr+1), &(split_ratio(ifr,0)));
+    }
+    // the last frequency bin
+    map_start(nfreq-1) = nfreq-1;
+    map_end(nfreq-1) = nfreq-1;
+    split_ratio(nfreq-1,0) = 1.0;
+  } else {
+    for (int ifr=0; ifr<nfreq-1; ++ifr) {
+      Real nu_l = nu_lab[ifr];
+      Real nu_r = nu_lab[ifr+1];
+      int l_bd = ifr;
+      int r_bd = ifr;
+
+      while((nu_r < nu_shift[r_bd]) && (r_bd > 0))   r_bd--;
+      l_bd = r_bd; // r_bd always > l_bd
+      while((nu_l < nu_shift[l_bd]) && (l_bd > 0))   l_bd--;
+
+      if (r_bd-l_bd+1 > nmax_map_) {
+        std::stringstream msg;
+        msg << "### FATAL ERROR in function [ForwardSplitting]"
+            << std::endl << "Frequency shift '" << r_bd-l_bd+1 <<
+          "' larger than maximum allowed " << nmax_map_;
+        ATHENA_ERROR(msg);
+      }
+      map_start(ifr) = l_bd;
+      map_end(ifr) = r_bd;
+
+      SplitFrequencyBinLinear(l_bd, r_bd, nu_shift, nu_l, nu_r, ir_face(ifr),
+                                    ir_face(ifr+1), &(split_ratio(ifr,0)));
+    }
+
+    //-------------------------------------
+
+    Real nu_l = nu_lab[nfreq-1];
+    int r_bd = nfreq-1;
+    int l_bd = nfreq-2;// it will always be <= current bin
+    while((nu_l < nu_shift[l_bd]) && (l_bd > 0))   l_bd--;
+    // This frequency bin now maps to l_bd to r_bd
+    map_start(nfreq-1) = l_bd;
+    map_end(nfreq-1) = r_bd;
+    // nu_l/kt
+    Real nu_tr = pmy_rad->EffectiveBlackBody(ir_cm(nfreq-1), nu_l);
+
+    // FitIntPlanckFunc is integral _0 to nu_tr
+    // the integral we need is 1 - ori_norm
+    Real ori_norm = pmy_rad->FitIntPlanckFunc(nu_tr);
+    Real div_ori = 0.0;
+    if (1.0 - ori_norm > TINY_NUMBER) {
+      div_ori = 1.0/(1.0 - ori_norm);
+    }
+    // the first bin
+    // the effective temperature 1/T = nu_tr/nu_l
+    Real ratio = pmy_rad->FitIntPlanckFunc(nu_tr*nu_shift[l_bd+1]/nu_l);
+
+      // the difference is (1 - ori_norm) - (1 - ratio)
+    split_ratio(nfreq-1,0) = (ratio - ori_norm) * div_ori;
+    Real sum = split_ratio(nfreq-1,0);
+
+    for (int m=l_bd+1; m<r_bd; ++m) {
+      Real ratio_r = pmy_rad->FitIntPlanckFunc(nu_tr*nu_shift[m+1]/nu_l);
+      Real ratio_l = pmy_rad->FitIntPlanckFunc(nu_tr*nu_shift[m]/nu_l);
+      split_ratio(nfreq-1,m-l_bd) = (ratio_r - ratio_l) * div_ori;
+      sum += split_ratio(nfreq-1,m-l_bd);
+    }
+    split_ratio(nfreq-1,r_bd-l_bd) = 1.0 - sum;
+  }
+  return;
+}
diff --git a/src/nr_radiation/integrators/rad_integrators.cpp b/src/nr_radiation/integrators/rad_integrators.cpp
new file mode 100644
index 0000000000..12cbb9bd25
--- /dev/null
+++ b/src/nr_radiation/integrators/rad_integrators.cpp
@@ -0,0 +1,923 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file rad_integrators.cpp
+//  \brief implementation of radiation integrators
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <algorithm>
+#include <sstream>
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../eos/eos.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../parameter_input.hpp"
+#include "../implicit/radiation_implicit.hpp"
+#include "../radiation.hpp"
+#include "rad_integrators.hpp"
+
+static constexpr Real tau_asymptotic_lim=1.0e-3;
+
+RadIntegrator::RadIntegrator(NRRadiation *prad, ParameterInput *pin) {
+  pmy_rad = prad;
+
+  MeshBlock *pmb = prad->pmy_block;
+  Coordinates *pco=pmb->pcoord;
+
+  int nang=prad->nang;
+  int nfreq=prad->nfreq;
+  rad_xorder = pin->GetOrAddInteger("time","rad_xorder",2);
+  if (rad_xorder == 3) {
+    if (NGHOST < 3) {
+      std::stringstream msg;
+      msg << "### FATAL ERROR in radiation reconstruction constructor" << std::endl
+          << "rad_xorder=" << rad_xorder <<
+          " (PPM) reconstruction selected, but nghost=" << NGHOST << std::endl
+          << "Reconfigure with --nghost=3  " <<std::endl;
+      ATHENA_ERROR(msg);
+    }
+  }
+
+
+  // factor to separate the diffusion and advection part
+  Real taucell = pin->GetOrAddReal("radiation","taucell",5);
+  tau_flag_ = pin->GetOrAddInteger("radiation","tau_scheme",1);
+  compton_flag_=pin->GetOrAddInteger("radiation","Compton",0);
+  compton_t_=pin->GetOrAddInteger("radiation","Compton_t",0);
+  split_compton_=pin->GetOrAddInteger("radiation","Split_compton",0);
+  if ((!IM_RADIATION_ENABLED) || (compton_flag_ == 0))
+    split_compton_ = 0;
+
+  adv_flag_=pin->GetOrAddInteger("radiation","Advection",1);
+  flux_correct_flag_ = pin->GetOrAddInteger("radiation","CorrectFlux",0);
+  imp_ang_flx_ = pin->GetOrAddInteger("radiation","implicit_ang_flux",0);
+
+  doppler_flag_ = pin->GetOrAddInteger("radiation","doppler_flag",1);
+
+  // multi-group iteration
+  // iterations < iterative_tgas_
+  iteration_tgas_  = pin->GetOrAddInteger("radiation","iterative_tgas",5);
+  iteration_compton_  = pin->GetOrAddInteger("radiation","iterative_compton",5);
+  tgas_error_ =   pin->GetOrAddReal("radiation","gas_error",1.e-6);
+  compton_error_ =   pin->GetOrAddReal("radiation","gas_error",1.e-6);
+  // maximum number of bins each frequency bin will map to, default is nfreq/2
+  nmax_map_ = pin->GetOrAddInteger("radiation","max_map_bin",0);
+
+  int ncells1 = pmb->ncells1, ncells2 = pmb->ncells2,
+      ncells3 = pmb->ncells3;
+
+
+
+  x1face_area_.NewAthenaArray(ncells1+1);
+  if (ncells2 > 1) {
+    x2face_area_.NewAthenaArray(ncells1);
+    x2face_area_p1_.NewAthenaArray(ncells1);
+  }
+  if (ncells3 > 1) {
+    x3face_area_.NewAthenaArray(ncells1);
+    x3face_area_p1_.NewAthenaArray(ncells1);
+  }
+  cell_volume_.NewAthenaArray(ncells1);
+
+
+  cwidth2_.NewAthenaArray(ncells1);
+  cwidth3_.NewAthenaArray(ncells1);
+
+  dflx_.NewAthenaArray(ncells1,prad->n_fre_ang);
+
+  // arrays for spatial recontruction
+  il_.NewAthenaArray(ncells1,prad->n_fre_ang);
+  ilb_.NewAthenaArray(ncells1,prad->n_fre_ang);
+
+  ir_.NewAthenaArray(ncells1,prad->n_fre_ang);
+
+  sfac1_x_.NewAthenaArray(ncells1,prad->n_fre_ang);
+  sfac2_x_.NewAthenaArray(ncells1,prad->n_fre_ang);
+  if (ncells2 > 1) {
+    sfac1_y_.NewAthenaArray(ncells2,ncells1,prad->n_fre_ang);
+    sfac2_y_.NewAthenaArray(ncells2,ncells1,prad->n_fre_ang);
+  }
+  if (ncells3 > 1) {
+    sfac1_z_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    sfac2_z_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+  }
+
+
+
+  sm_diff1_.NewAthenaArray(prad->n_fre_ang);
+  sm_diff2_.NewAthenaArray(prad->n_fre_ang);
+  vel_ex_l_.NewAthenaArray(prad->n_fre_ang);
+  vel_ex_r_.NewAthenaArray(prad->n_fre_ang);
+  vel_im_l_.NewAthenaArray(prad->n_fre_ang);
+  vel_im_r_.NewAthenaArray(prad->n_fre_ang);
+
+  adv_vel.NewAthenaArray(3,ncells3,ncells2,ncells1);
+
+
+  if (IM_RADIATION_ENABLED) {
+    limiter_.NewAthenaArray(ncells1,prad->n_fre_ang);
+
+    if (ncells2 > 1) {
+      limiterj_.NewAthenaArray(ncells2,ncells1,prad->n_fre_ang);
+    }
+    if (ncells3 > 1) {
+      limiterk_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    }
+    dql_.NewAthenaArray(prad->n_fre_ang);
+    dqr_.NewAthenaArray(prad->n_fre_ang);
+
+    const_coef_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    exp_coef_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    const_coef1_l_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    const_coef1_r_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    if (ncells2 > 1) {
+      const_coef2_l_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+      const_coef2_r_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    }
+    if (ncells3 > 1) {
+      const_coef3_l_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+      const_coef3_r_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    }
+    divflx_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+
+    left_coef1_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    left_coef2_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    left_coef3_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+    adv_flx_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+  }
+
+  implicit_coef_.NewAthenaArray(prad->n_fre_ang);
+
+  tgas_.NewAthenaArray(ncells3,ncells2,ncells1);
+  tgas_new_.NewAthenaArray(ncells3,ncells2,ncells1);
+  vel_source_.NewAthenaArray(ncells3,ncells2,ncells1,3);
+  taufact.NewAthenaArray(ncells3,ncells2,ncells1);
+
+
+  rad_source.NewAthenaArray(4,ncells3,ncells2,ncells1);
+  delta_source.NewAthenaArray(4,nfreq);
+
+
+  vel_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+  velx_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+  vely_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+  velz_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+
+  vncsigma_.NewAthenaArray(nang);
+  vncsigma2_.NewAthenaArray(prad->n_fre_ang);
+  wmu_cm_.NewAthenaArray(nang);
+  tran_coef_.NewAthenaArray(nang);
+  cm_to_lab_.NewAthenaArray(nang);
+  ir_cm_.NewAthenaArray(prad->n_fre_ang);
+  dxw1_.NewAthenaArray(ncells1);
+  dxw2_.NewAthenaArray(ncells1);
+
+  //----------------------------------------------------
+  // array for multi-group
+  if (nfreq > 1) {
+    // put the angle index first when map in frequency space
+
+    ir_buff_.NewAthenaArray(prad->n_fre_ang);
+
+    if (nmax_map_ == 0)  nmax_map_ = nfreq/2+1;
+    split_ratio_.NewAthenaArray(nang,nfreq,nmax_map_);
+    // this needs to be done for each angle, all frequency groups
+    fre_map_matrix_.NewAthenaArray(nfreq,nmax_map_);
+    delta_nu_n_.NewAthenaArray(nfreq);
+    map_bin_start_.NewAthenaArray(nang,nfreq);
+    map_bin_end_.NewAthenaArray(nang,nfreq);
+    map_count_.NewAthenaArray(nfreq);
+    nu_shift_.NewAthenaArray(nfreq);
+    ir_face_.NewAthenaArray(nfreq);
+    ir_ori_.NewAthenaArray(nfreq);
+    ir_done_.NewAthenaArray(nfreq);
+
+    com_b_face_coef_.NewAthenaArray(nfreq);
+    com_d_face_coef_.NewAthenaArray(nfreq);
+    com_b_coef_l_.NewAthenaArray(nfreq);
+    com_b_coef_r_.NewAthenaArray(nfreq);
+    com_d_coef_l_.NewAthenaArray(nfreq);
+    com_d_coef_r_.NewAthenaArray(nfreq);
+
+    nf_rhs_.NewAthenaArray(nfreq);
+    nf_n0_.NewAthenaArray(nfreq);
+    new_j_nu_.NewAthenaArray(nfreq);
+  }
+
+  sum_nu3_.NewAthenaArray(nfreq);
+  sum_nu2_.NewAthenaArray(nfreq);
+  sum_nu1_.NewAthenaArray(nfreq);
+  eq_sol_.NewAthenaArray(nfreq);
+
+  //----------------------------------------------------
+  // array for multi-group
+
+  // set the default taufact
+  for (int k=0; k<ncells3; ++k)
+    for (int j=0; j<ncells2; ++j)
+      for (int i=0; i<ncells1; ++i) {
+        taufact(k,j,i) = taucell;
+      }
+
+  // The angular grid coefficients
+  ang_flx_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+  imp_ang_coef_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+  imp_ang_coef_r_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+  imp_ang_psi_r_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+  imp_ang_psi_l_.NewAthenaArray(ncells3,ncells2,ncells1,prad->n_fre_ang);
+
+  if (prad->angle_flag == 1) {
+    int &nzeta = prad->nzeta;
+    int &npsi = prad->npsi;
+    if (nzeta > 0) {
+      g_zeta_.NewAthenaArray(2*nzeta+1);
+      q_zeta_.NewAthenaArray(2*nzeta+2*NGHOST);
+      ql_zeta_.NewAthenaArray(2*nzeta+2*NGHOST);
+      qr_zeta_.NewAthenaArray(2*nzeta+2*NGHOST);
+
+
+      if (npsi > 0) {
+        zeta_flux_.NewAthenaArray(ncells3,ncells2,ncells1,nfreq,(2*nzeta+1)*2*npsi);
+        zeta_area_.NewAthenaArray(2*npsi,2*nzeta+1);
+      } else {
+        zeta_flux_.NewAthenaArray(ncells3,ncells2,ncells1,nfreq,2*nzeta+1);
+        zeta_area_.NewAthenaArray(2*nzeta+1);
+      }
+      pco->ZetaArea(prad, zeta_area_);
+    }
+
+    if (npsi > 0) {
+      g_psi_.NewAthenaArray(2*npsi+1);
+      q_psi_.NewAthenaArray(2*npsi+2*NGHOST);
+      ql_psi_.NewAthenaArray(2*npsi+2*NGHOST);
+      qr_psi_.NewAthenaArray(2*npsi+2*NGHOST);
+      if (nzeta > 0) {
+        psi_flux_.NewAthenaArray(ncells3,ncells2,ncells1,nfreq,2*nzeta*(2*npsi+1));
+        psi_area_.NewAthenaArray(2*nzeta,2*npsi+1);
+      } else {
+        psi_flux_.NewAthenaArray(ncells3,ncells2,ncells1,nfreq,2*npsi+1);
+        psi_area_.NewAthenaArray(2*npsi+1);
+      }
+      pco->PsiArea(prad, psi_area_);
+    }
+
+    dflx_ang_.NewAthenaArray(nang);
+    ang_vol_.NewAthenaArray(nang);
+    pco->AngularVol(prad, ang_vol_);
+  }
+
+  // calculate the advection velocity at the cell faces
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+  int il, iu, jl, ju, kl, ku;
+  jl = js, ju=je, kl=ks, ku=ke;
+
+  if (ncells2 > 1) {
+    if (ncells3 == 1) {
+      jl=js-1, ju=je+1, kl=ks, ku=ke;
+    } else {
+      jl=js-1, ju=je+1, kl=ks-1, ku=ke+1;
+    }
+  }
+
+  // calculate velx_
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      // get the velocity at the interface
+      for (int i=is-1; i<=ie+1; ++i) {
+        Real dxl = pco->x1f(i)-pco->x1v(i-1);
+        Real dxr = pco->x1v(i) - pco->x1f(i);
+        Real factl = dxr/(dxl+dxr);
+        Real factr = dxl/(dxl+dxr);
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          Real *cosx = &(prad->mu(0,k,j,i-1,0));
+          Real *cosx1 = &(prad->mu(0,k,j,i,0));
+          Real *veln = &(velx_(k,j,i,ifr*nang));
+          for (int n=0; n<nang; ++n) {
+            // linear intepolation between x1v(i-1), x1f(i), x1v(i)
+            veln[n] = prad->reduced_c *
+                      (factl * cosx[n] + factr * cosx1[n]);
+          }
+        }
+      }
+    }
+  }
+
+  // calculate vely_
+  if (ncells2 > 1) {
+    il = is-1, iu = ie+1, kl = ks, ku = ke;
+    if (ncells3 >  1) // 2D
+      kl = ks-1, ku = ke+1;
+
+    for (int k=kl; k<=ku; ++k) {
+      for (int j=js; j<=je+1; ++j) {
+        // get the velocity at the interface
+        for (int i=il; i<=iu; ++i) {
+          Real dxl = pco->x2f(j)-pco->x2v(j-1);
+          Real dxr = pco->x2v(j) - pco->x2f(j);
+          Real factl = dxr/(dxl+dxr);
+          Real factr = dxl/(dxl+dxr);
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            Real *cosy = &(prad->mu(1,k,j-1,i,0));
+            Real *cosy1 = &(prad->mu(1,k,j,i,0));
+            Real *veln = &(vely_(k,j,i,ifr*nang));
+            for (int n=0; n<nang; ++n) {
+              // linear intepolation between x2v(j-1), x2f(j), x2v(j)
+              veln[n] = prad->reduced_c *
+                        (factl * cosy[n] + factr * cosy1[n]);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // calculate vely_
+  if (ncells3 > 1) {
+    il =is-1, iu=ie+1, jl=js-1, ju=je+1;
+
+    for (int k=ks; k<=ke+1; ++k) {
+      for (int j=jl; j<=ju; ++j) {
+        // get the velocity at the interface
+        for (int i=il; i<=iu; ++i) {
+          Real dxl = pco->x3f(k) - pco->x3v(k-1);
+          Real dxr = pco->x3v(k) - pco->x3f(k);
+          Real factl = dxr/(dxl+dxr);
+          Real factr = dxl/(dxl+dxr);
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            Real *cosz = &(prad->mu(2,k-1,j,i,0));
+            Real *cosz1 = &(prad->mu(2,k,j,i,0));
+            Real *veln = &(velz_(k,j,i,ifr*nang));
+            for (int n=0; n<nang; ++n) {
+              // linear intepolation between x2v(j-1), x2f(j), x2v(j)
+              veln[n] = prad->reduced_c *
+                        (factl * cosz[n] + factr * cosz1[n]);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+// destructor
+
+RadIntegrator::~RadIntegrator() {
+  x1face_area_.DeleteAthenaArray();
+  if (pmy_rad->pmy_block->ncells2 > 1) {
+    x2face_area_.DeleteAthenaArray();
+    x2face_area_p1_.DeleteAthenaArray();
+  }
+  if (pmy_rad->pmy_block->ncells3 > 1) {
+    x3face_area_.DeleteAthenaArray();
+    x3face_area_p1_.DeleteAthenaArray();
+  }
+  cell_volume_.DeleteAthenaArray();
+
+  cwidth2_.DeleteAthenaArray();
+  cwidth3_.DeleteAthenaArray();
+
+  dflx_.DeleteAthenaArray();
+
+
+  il_.DeleteAthenaArray();
+  ilb_.DeleteAthenaArray();
+
+  ir_.DeleteAthenaArray();
+
+  vel_.DeleteAthenaArray();
+  velx_.DeleteAthenaArray();
+  vely_.DeleteAthenaArray();
+  velz_.DeleteAthenaArray();
+
+
+  vncsigma_.DeleteAthenaArray();
+  vncsigma2_.DeleteAthenaArray();
+  wmu_cm_.DeleteAthenaArray();
+  tran_coef_.DeleteAthenaArray();
+  cm_to_lab_.DeleteAthenaArray();
+  ir_cm_.DeleteAthenaArray();
+
+  sfac1_x_.DeleteAthenaArray();
+  sfac2_x_.DeleteAthenaArray();
+  if (pmy_rad->pmy_block->ncells2 > 1) {
+    sfac1_y_.DeleteAthenaArray();
+    sfac2_y_.DeleteAthenaArray();
+  }
+  if (pmy_rad->pmy_block->ncells3 > 1) {
+    sfac1_z_.DeleteAthenaArray();
+    sfac2_z_.DeleteAthenaArray();
+  }
+
+  sm_diff1_.DeleteAthenaArray();
+  sm_diff2_.DeleteAthenaArray();
+  vel_ex_l_.DeleteAthenaArray();
+  vel_ex_r_.DeleteAthenaArray();
+  vel_im_l_.DeleteAthenaArray();
+  vel_im_r_.DeleteAthenaArray();
+
+  adv_vel.DeleteAthenaArray();
+
+  if (IM_RADIATION_ENABLED) {
+    const_coef_.DeleteAthenaArray();
+    exp_coef_.DeleteAthenaArray();
+    const_coef1_l_.DeleteAthenaArray();
+    const_coef1_r_.DeleteAthenaArray();
+    limiter_.DeleteAthenaArray();
+
+    if (pmy_rad->pmy_block->ncells2 > 1) {
+      const_coef2_l_.DeleteAthenaArray();
+      const_coef2_r_.DeleteAthenaArray();
+      limiterj_.DeleteAthenaArray();
+    }
+    if (pmy_rad->pmy_block->ncells3 > 1) {
+      const_coef3_l_.DeleteAthenaArray();
+      const_coef3_r_.DeleteAthenaArray();
+      limiterk_.DeleteAthenaArray();
+    }
+    divflx_.DeleteAthenaArray();
+    dql_.DeleteAthenaArray();
+    dqr_.DeleteAthenaArray();
+
+    left_coef1_.DeleteAthenaArray();
+    left_coef2_.DeleteAthenaArray();
+    left_coef3_.DeleteAthenaArray();
+
+
+    ang_flx_.DeleteAthenaArray();
+    imp_ang_coef_.DeleteAthenaArray();
+    imp_ang_coef_r_.DeleteAthenaArray();
+    imp_ang_psi_l_.DeleteAthenaArray();
+    imp_ang_psi_r_.DeleteAthenaArray();
+    adv_flx_.DeleteAthenaArray();
+  }
+  implicit_coef_.DeleteAthenaArray();
+
+
+  taufact.DeleteAthenaArray();
+  tgas_.DeleteAthenaArray();
+  tgas_new_.DeleteAthenaArray();
+  vel_source_.DeleteAthenaArray();
+  dxw1_.DeleteAthenaArray();
+  dxw2_.DeleteAthenaArray();
+  rad_source.DeleteAthenaArray();
+  delta_source.DeleteAthenaArray();
+
+
+  if (pmy_rad->angle_flag == 1) {
+    int &nzeta = pmy_rad->nzeta;
+    int &npsi = pmy_rad->npsi;
+    if (nzeta > 0) {
+      g_zeta_.DeleteAthenaArray();
+      q_zeta_.DeleteAthenaArray();
+      ql_zeta_.DeleteAthenaArray();
+      qr_zeta_.DeleteAthenaArray();
+      zeta_flux_.DeleteAthenaArray();
+      zeta_area_.DeleteAthenaArray();
+    }
+    if (npsi > 0) {
+      g_psi_.DeleteAthenaArray();
+      q_psi_.DeleteAthenaArray();
+      ql_psi_.DeleteAthenaArray();
+      qr_psi_.DeleteAthenaArray();
+      psi_flux_.DeleteAthenaArray();
+      psi_area_.DeleteAthenaArray();
+    }
+    dflx_ang_.DeleteAthenaArray();
+    ang_vol_.DeleteAthenaArray();
+  }
+
+  // multi-group array
+
+  if (pmy_rad->nfreq > 1) {
+    split_ratio_.DeleteAthenaArray();
+    fre_map_matrix_.DeleteAthenaArray();
+    delta_nu_n_.DeleteAthenaArray();
+    ir_buff_.DeleteAthenaArray();
+    ir_face_.DeleteAthenaArray();
+    map_bin_start_.DeleteAthenaArray();
+    map_bin_end_.DeleteAthenaArray();
+    map_count_.DeleteAthenaArray();
+    nu_shift_.DeleteAthenaArray();
+    ir_ori_.DeleteAthenaArray();
+    ir_done_.DeleteAthenaArray();
+
+    com_b_face_coef_.DeleteAthenaArray();
+    com_d_face_coef_.DeleteAthenaArray();
+    com_b_coef_l_.DeleteAthenaArray();
+    com_b_coef_r_.DeleteAthenaArray();
+    com_d_coef_l_.DeleteAthenaArray();
+    com_d_coef_r_.DeleteAthenaArray();
+
+    nf_rhs_.DeleteAthenaArray();
+    nf_n0_.DeleteAthenaArray();
+    new_j_nu_.DeleteAthenaArray();
+  }
+  sum_nu3_.DeleteAthenaArray();
+  sum_nu2_.DeleteAthenaArray();
+  sum_nu1_.DeleteAthenaArray();
+  eq_sol_.DeleteAthenaArray();
+}
+
+
+void RadIntegrator::GetTgasVel(MeshBlock *pmb, const Real dt,
+                               AthenaArray<Real> &u, AthenaArray<Real> &w,
+                               AthenaArray<Real> &bcc, AthenaArray<Real> &ir) {
+  Real gm1 = pmb->peos->GetGamma() - 1.0;
+
+  Real rho_floor = pmb->peos->GetDensityFloor();
+
+  NRRadiation *prad=pmb->pnrrad;
+  Coordinates *pco=pmb->pcoord;
+
+  Real& prat = prad->prat;
+  Real invcrat = 1.0/prad->crat;
+
+  int &nang =prad->nang;
+  int &nfreq=prad->nfreq;
+
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+  for (int k=0; k<pmb->ncells3; ++k) {
+    for (int j=0; j<pmb->ncells2; ++j) {
+      for (int i=0; i<pmb->ncells1; ++i) {
+        // for implicit update, using the quantities from the partially
+        // updated u, not from w
+        Real rho = u(IDN,k,j,i);
+        rho = std::max(rho,rho_floor);
+        Real vx = u(IM1,k,j,i)/rho;
+        Real vy = u(IM2,k,j,i)/rho;
+        Real vz = u(IM3,k,j,i)/rho;
+        Real pb = 0.0;
+        if (MAGNETIC_FIELDS_ENABLED)
+          pb = 0.5*(SQR(bcc(IB1,k,j,i))+SQR(bcc(IB2,k,j,i))
+                    +SQR(bcc(IB3,k,j,i)));
+
+        Real vsq = vx * vx + vy * vy + vz * vz;
+        Real tgas = u(IEN,k,j,i) - pb - 0.5*rho*vsq;
+        tgas = gm1*tgas/rho;
+        tgas = std::max(tgas,pmb->pnrrad->t_floor_(k,j,i));
+        tgas = std::min(tgas,pmb->pnrrad->t_ceiling_(k,j,i));
+        tgas_(k,j,i) = tgas;
+        // Do not use the velocity directly in strongly radiation pressure
+        // dominated regime
+        // use the predicted velocity based on moment equatio
+
+        // calculate radiation energy density
+        Real er = 0.0;
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          Real *irn = &(ir(k,j,i,ifr*nang));
+          Real *weight = &(prad->wmu(0));
+          Real er_freq = 0.0;
+#pragma omp simd reduction(+:er_freq)
+          for (int n=0; n<nang; ++n) {
+            er_freq += weight[n] * irn[n];
+          }
+          er += er_freq;
+        }
+
+        // now the velocity term,
+        // using velocity from current stage
+        vx = w(IVX,k,j,i);
+        vy = w(IVY,k,j,i);
+        vz = w(IVZ,k,j,i);
+        vsq = vx * vx + vy * vy + vz * vz;
+        if (prat * er * invcrat * invcrat > rho) {
+          PredictVel(ir,k,j,i, 0.5 * dt, rho, &vx, &vy, &vz);
+          vsq = vx * vx + vy * vy + vz * vz;
+        }
+
+        Real ratio = std::sqrt(vsq) * invcrat;
+        // Limit the velocity to be smaller than the speed of light
+        if (ratio > prad->vmax) {
+          Real factor = prad->vmax/ratio;
+          vx *= factor;
+          vy *= factor;
+          vz *= factor;
+        }
+        vel_source_(k,j,i,0) = vx;
+        vel_source_(k,j,i,1) = vy;
+        vel_source_(k,j,i,2) = vz;
+      }
+    }
+  }
+  // Now get interface velocity
+  // vx
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      pco->CenterWidth1(k,j,is-1,ie+1,dxw1_);
+      for (int i=is; i<=ie+1; ++i) {
+        Real tau = 0.0;
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          Real sigmal = prad->sigma_a(k,j,i-1,ifr) + prad->sigma_s(k,j,i-1,ifr);
+          sigmal *= taufact(k,j,i-1);
+          Real sigmar = prad->sigma_a(k,j,i,ifr) + prad->sigma_s(k,j,i,ifr);
+          sigmar *= taufact(k,j,i);
+          tau += (dxw1_(i-1) * sigmal + dxw1_(i) * sigmar);
+        }
+
+        Real factor = 0.0;
+        GetTaufactorAdv(tau,factor);
+        Real vl = vel_source_(k,j,i-1,0);
+        Real vr = vel_source_(k,j,i,0);
+        adv_vel(0,k,j,i) = factor*(vl + (pco->x1f(i) - pco->x1v(i-1)) *
+                                   (vr - vl)/(pco->x1v(i) - pco->x1v(i-1)));
+      }
+    }
+  }
+  if (je > js) {
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je+1; ++j) {
+        Real ratio = (pco->x2f(j) - pco->x2v(j-1))/
+                     (pco->x2v(j) - pco->x2v(j-1));
+        pco->CenterWidth2(k,j-1,is,ie,dxw1_);
+        pco->CenterWidth2(k,j,is,ie,dxw2_);
+        for (int i=is; i<=ie; ++i) {
+          Real tau = 0.0;
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            Real sigmal = prad->sigma_a(k,j-1,i,ifr) + prad->sigma_s(k,j-1,i,ifr);
+            sigmal *= taufact(k,j-1,i);
+            Real sigmar = prad->sigma_a(k,j,i,ifr) + prad->sigma_s(k,j,i,ifr);
+            sigmar *= taufact(k,j,i);
+            tau += (dxw1_(i) * sigmal + dxw2_(i) * sigmar);
+          }
+
+          Real factor = 0.0;
+          GetTaufactorAdv(tau,factor);
+
+          Real vl = vel_source_(k,j-1,i,1);
+          Real vr = vel_source_(k,j,i,1);
+          adv_vel(1,k,j,i) = factor*(vl +  ratio * (vr - vl));
+        }
+      }
+    }
+  }
+
+  if (ke > ks) {
+    for (int k=ks; k<=ke+1; ++k) {
+      Real ratio = (pco->x3f(k) - pco->x3v(k-1))/
+                   (pco->x3v(k) - pco->x3v(k-1));
+      for (int j=js; j<=je; ++j) {
+        pco->CenterWidth3(k-1,j,is,ie,dxw1_);
+        pco->CenterWidth3(k,j,is,ie,dxw2_);
+        for (int i=is; i<=ie; ++i) {
+          Real tau = 0.0;
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            Real sigmal = prad->sigma_a(k-1,j,i,ifr) + prad->sigma_s(k-1,j,i,ifr);
+            sigmal *= taufact(k-1,j,i);
+            Real sigmar = prad->sigma_a(k,j,i,ifr) + prad->sigma_s(k,j,i,ifr);
+            sigmar *= taufact(k,j,i);
+            tau += (dxw1_(i) * sigmal + dxw2_(i) * sigmar);
+          }
+
+          Real factor = 0.0;
+          GetTaufactorAdv(tau,factor);
+
+          Real vl = vel_source_(k-1,j,i,2);
+          Real vr = vel_source_(k,j,i,2);
+          adv_vel(2,k,j,i) = factor * (vl +  ratio * (vr - vl));
+        }
+      }
+    }
+  }
+}
+
+
+// f_l is the factor in the opposite direction of vel
+// f_r is the factor in the same direction of vel
+void RadIntegrator::SignalSpeed(const Real adv, const Real f_l,
+                                const Real f_r, Real *vel, Real *smax, Real *smin) {
+  for (int n=0; n<pmy_rad->nang; ++n) {
+    if (vel[n] > 0.0) {
+      smax[n] = f_r * vel[n];
+      smin[n] = -f_l * vel[n];
+    } else {
+      smax[n] = -f_l * vel[n];
+      smin[n] = f_r * vel[n];
+    }
+  }
+  if (adv_flag_ == 0) {
+    for (int n=0; n<pmy_rad->nang; ++n) {
+      smax[n] += std::abs(adv);
+      smin[n] -= std::abs(adv);
+    }
+  }
+}
+
+void RadIntegrator::SplitVelocity(Real *vel_l, Real *vel_r, const Real advl,
+                                  const Real advr, Real *smax_l, Real *smin_l,
+                                  Real *smax_r, Real *smin_r) {
+  int tot_ang = pmy_rad->n_fre_ang;
+  int iteration = pmy_rad->pmy_block->pmy_mesh->pimrad->ite_scheme;
+  // the left side
+  if (iteration == 0) {
+    for (int n=0; n<tot_ang; ++n) {
+      Real vl = vel_l[n] - advl;
+      vel_ex_l_(n) = 0.0;
+      vel_im_l_(n) = smin_l[n] * (smax_l[n] - vl);
+    }
+    for (int n=0; n<tot_ang; ++n) {
+      Real vr = vel_r[n] - advr;
+      vel_ex_r_(n) = 0.0;
+      vel_im_r_(n) = smax_r[n] * (vr - smin_r[n]);
+    }
+  } else if (iteration == 1) {
+    for (int n=0; n<tot_ang; ++n) {
+      Real vl = vel_l[n] - advl;
+      if (vl > 0.0) {
+        vel_ex_l_(n) = -smin_l[n] * vl;
+        vel_im_l_(n) = smin_l[n] * smax_l[n];
+      } else {
+        vel_ex_l_(n) = 0.0;
+        vel_im_l_(n) = smin_l[n] * (smax_l[n] - vl);
+      }
+    }// end n
+
+    for (int n=0; n<tot_ang; ++n) {
+      Real vr = vel_r[n] - advr;
+      if (vr > 0.0) {
+        vel_ex_r_(n) = 0.0;
+        vel_im_r_(n) = smax_r[n] * (vr - smin_r[n]);
+      } else {
+        vel_ex_r_(n) = smax_r[n] * vr;
+        vel_im_r_(n) = -smax_r[n] * smin_r[n];
+      }
+    }
+  } else {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in function [SplitVelocity]"
+        << std::endl << "ite_scheme '" << iteration << "' not allowed!";
+    ATHENA_ERROR(msg);
+  }
+}
+
+void RadIntegrator::GetTaufactor(const Real tau, Real &factor1, int dir) {
+  std::stringstream msg;
+
+  if (dir > 0) {
+    if (tau_flag_ == 1) {
+      Real tausq = tau * tau;
+      factor1 = 1.0 - 0.5 * tausq;
+      if (tausq > tau_asymptotic_lim) {
+        factor1  = (1.0 - std::exp(-tausq))/tausq;
+      }
+      factor1 = std::sqrt(factor1);
+    } else if (tau_flag_ == 2) {
+      Real tausq = tau;
+      if (tausq > 1)
+        factor1 = 1.0/tausq;
+      else
+        factor1 = 1.0;
+
+    } else {
+      msg << "### FATAL ERROR in function [GetTaufactor]"
+          << std::endl << "tau_flag_ '" << tau_flag_ << "' not allowed!";
+      ATHENA_ERROR(msg);
+    }
+  } else {
+    if (tau_flag_ == 1) {
+      Real tausq = tau * tau;
+      factor1 = tausq;
+      if (tausq > tau_asymptotic_lim) {
+        factor1  = (1.0 - std::exp(-tausq*tausq))/tausq;
+      }
+      factor1 = std::sqrt(factor1);
+    } else if (tau_flag_ == 2) {
+      Real tausq = tau;
+      if (tausq > 1)
+        factor1 = 1.0/tausq;
+      else
+        factor1 = tausq;
+    } else {
+      msg << "### FATAL ERROR in function [GetTaufactor]"
+          << std::endl << "tau_flag_ '" << tau_flag_ << "' not allowed!";
+      ATHENA_ERROR(msg);
+    }
+  }
+}
+
+
+// The tau factor for advection velocity
+void RadIntegrator::GetTaufactorAdv(const Real tau, Real &factor) {
+  Real tausq = tau * tau;
+  factor = tausq - 0.5 * tausq * tausq;
+  if (tausq > tau_asymptotic_lim)
+    factor = (1.0 - std::exp(-tausq));
+  return;
+}
+
+
+void RadIntegrator::PredictVel(AthenaArray<Real> &ir, int k, int j, int i,
+                               Real dt, Real rho, Real *vx, Real *vy, Real *vz) {
+  NRRadiation *prad = pmy_rad;
+
+  Real &prat = prad->prat;
+  Real invcrat = 1.0/prad->crat;
+  Real ct = dt * prad->reduced_c;
+  int& nang =prad->nang;
+  int& nfreq=prad->nfreq;
+  // first, calculate the moments
+  Real er =0.0, fr1=0.0, fr2=0.0, fr3=0.0,
+      pr11=0.0,pr12=0.0,pr13=0.0,pr22=0.0,
+      pr23=0.0,pr33=0.0;
+  Real *weight = &(prad->wmu(0));
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    Real er_f = 0.0,fr1_f=0.0,fr2_f=0.0,fr3_f=0.0,
+        pr11_f=0.0,pr12_f=0.0,pr13_f=0.0,pr22_f=0.0,
+        pr23_f=0.0,pr33_f=0.0;
+
+    Real *irn = &(ir(k,j,i,ifr*nang));
+    Real *cosx = &(prad->mu(0,k,j,i,0));
+    Real *cosy = &(prad->mu(1,k,j,i,0));
+    Real *cosz = &(prad->mu(2,k,j,i,0));
+    for (int n=0; n<nang; ++n) {
+      Real irweight = weight[n] * irn[n];
+      er_f   += irweight;
+      fr1_f  += irweight * cosx[n];
+      fr2_f  += irweight * cosy[n];
+      fr3_f  += irweight * cosz[n];
+      pr11_f += irweight * cosx[n] * cosx[n];
+      pr12_f += irweight * cosx[n] * cosy[n];
+      pr13_f += irweight * cosx[n] * cosz[n];
+      pr22_f += irweight * cosy[n] * cosy[n];
+      pr23_f += irweight * cosy[n] * cosz[n];
+      pr33_f += irweight * cosz[n] * cosz[n];
+    }
+
+
+    er   += er_f;
+    fr1  += fr1_f;
+    fr2  += fr2_f;
+    fr3  += fr3_f;
+    pr11 += pr11_f;
+    pr12 += pr12_f;
+    pr13 += pr13_f;
+    pr22 += pr22_f;
+    pr23 += pr23_f;
+    pr33 += pr33_f;
+  }
+  // calculate the frequency integrated opacity
+  Real grey_sigma_s = 0.0;
+  Real grey_sigma_a = 0.0;
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    grey_sigma_s += prad->sigma_s(k,j,i,ifr);
+    grey_sigma_a += prad->sigma_a(k,j,i,ifr);
+  }
+  Real dtcsigma = ct * (grey_sigma_s + grey_sigma_a);
+
+  Real vx0 = (*vx);
+  Real vy0 = (*vy);
+  Real vz0 = (*vz);
+
+  Real m0x = prat * fr1 * invcrat + rho * vx0;
+  Real m0y = prat * fr2 * invcrat + rho * vy0;
+  Real m0z = prat * fr3 * invcrat + rho * vz0;
+
+
+  Real vx11 = rho * (1.0 + dtcsigma) + prat * dtcsigma * (er + pr11)
+              * invcrat * invcrat;
+  Real vy11 = rho * (1.0 + dtcsigma) + prat * dtcsigma * (er + pr22)
+              * invcrat * invcrat;
+  Real vz11 = rho * (1.0 + dtcsigma) + prat * dtcsigma * (er + pr33)
+              * invcrat * invcrat;
+  Real vx12 = dtcsigma * prat * pr12 * invcrat * invcrat;
+  Real vx13 = dtcsigma * prat * pr13 * invcrat * invcrat;
+  Real vy12 = dtcsigma * prat * pr23 * invcrat * invcrat;
+  Real rhs1 = rho * vx0 + dtcsigma * m0x;
+  Real rhs2 = rho * vy0 + dtcsigma * m0y;
+  Real rhs3 = rho * vz0 + dtcsigma * m0z;
+
+  Real factor = vx11 * vy11 * vz11 - vy11 * vx13 * vx13 + 2.0 * vx12 * vx13 * vy12
+                - vx11 * vy12 * vy12 - vx12 * vx12 * vz11;
+  factor = 1.0/factor;
+
+  (*vx) = factor*(rhs3*(vx12*vy12 - vx13*vy11) + rhs2*(
+      vy12*vx13 - vx12*vz11) + rhs1*(vy11*vz11 - vy12*vy12));
+
+  (*vy) = factor*(rhs3*(vx12*vx13 - vx11*vy12) + rhs2*(
+      vx11*vz11 - vx13*vx13) + rhs1*(vx13*vy12 - vx12*vz11));
+
+  (*vz) = factor*(rhs3*(vx11*vy11 - vx12*vx12) + rhs2*(
+      vx12*vx13 - vx11*vy12) + rhs1*(vx12*vy12 - vx13*vy11));
+}
diff --git a/src/nr_radiation/integrators/rad_integrators.hpp b/src/nr_radiation/integrators/rad_integrators.hpp
new file mode 100644
index 0000000000..41d51d32aa
--- /dev/null
+++ b/src/nr_radiation/integrators/rad_integrators.hpp
@@ -0,0 +1,245 @@
+#ifndef NR_RADIATION_INTEGRATORS_RAD_INTEGRATORS_HPP_
+#define NR_RADIATION_INTEGRATORS_RAD_INTEGRATORS_HPP_
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+// See LICENSE file for full public license information.
+//======================================================================================
+//! \file rad_integrators.hpp
+//  \brief definitions for Radiation class
+//======================================================================================
+
+// Athena++ classes headers
+#include "../../athena_arrays.hpp"
+#include "../../task_list/task_list.hpp"
+#include "../radiation.hpp"
+
+class MeshBlock;
+class ParameterInput;
+class NRRadiation;
+
+//! \class RadIntegrator
+//  \brief integrate algorithm for radiative transfer
+class RadIntegrator {
+  friend class NRRadiation;
+  friend class IMRadiation;
+ public:
+  RadIntegrator(NRRadiation *prad, ParameterInput *pin);
+  ~RadIntegrator();
+
+  NRRadiation *pmy_rad;
+
+
+  void FluxDivergence(const Real wght, AthenaArray<Real> &ir_in,
+                                       AthenaArray<Real> &ir_out);
+  void FluxDivergence(const Real wght);
+
+
+  void FirstOrderFluxDivergenceCoef(const Real wght);
+  void FirstOrderFluxDivergence(const int k, const int j, const int i,
+                                 AthenaArray<Real> &ir);
+
+
+  void ImplicitAngularFluxesCoef(const Real wght);
+  void ImplicitAngularFluxes(const int k, const int j, const int i,
+                             AthenaArray<Real> &ir);
+
+  void ImplicitPsiFluxCoef(int k, int j, int i, int n_zeta, Real wght,
+            Real zeta_coefr, Real zeta_coefl);
+  void ImplicitPsiFlux(int k, int j, int i, int ifr, int n_zeta, AthenaArray<Real> &ir);
+
+  void CalculateFluxes(AthenaArray<Real> &w,
+                       AthenaArray<Real> &ir, const int order);
+  void CalculateFluxes(AthenaArray<Real> &ir, const int order);
+
+  void CalSourceTerms(MeshBlock *pmb, const Real dt, const int k, const int j,
+                      const int i, AthenaArray<Real> &u,
+                      AthenaArray<Real> &ir_ini, AthenaArray<Real> &ir);
+
+  void AddSourceTerms(MeshBlock *pmb, AthenaArray<Real> &u);
+
+  void GetHydroSourceTerms(MeshBlock *pmb,
+                       AthenaArray<Real> &ir_ini, AthenaArray<Real> &ir);
+
+
+  Real AbsorptionScattering(AthenaArray<Real> &wmu_cm,
+          AthenaArray<Real> &tran_coef, Real *sigma_a, Real *sigma_p,
+          Real *sigma_ae, Real *sigma_s, Real dt, Real lorz, Real rho, Real &tgas,
+          AthenaArray<Real> &implicit_coef_, AthenaArray<Real> &ir_cm);
+
+  void GetTgasVel(MeshBlock *pmb, const Real dt,
+    AthenaArray<Real> &u, AthenaArray<Real> &w,
+    AthenaArray<Real> &bcc, AthenaArray<Real> &ir);
+
+
+  void Compton(AthenaArray<Real> &wmu_cm,
+          AthenaArray<Real> &tran_coef, Real *sigma_s,
+          Real dt, Real lorz, Real rho, Real &tgas, AthenaArray<Real> &ir_cm);
+
+
+
+  void LabToCom(const Real vx, const Real vy, const Real vz,
+                          Real *mux, Real *muy, Real *muz,
+                          Real *ir_lab, AthenaArray<Real> &ir_cm);
+
+  void ComToLab(const Real vx, const Real vy, const Real vz,
+                          Real *mux, Real *muy, Real *muz,
+                          AthenaArray<Real> &ir_cm, Real *ir_lab);
+
+  void ComAngle(const Real vx, const Real vy, const Real vz,
+          Real mux, Real muy, Real muz, Real *mux0, Real *muy0, Real *muz0);
+
+  //====================================
+  // multi-group functions
+
+  void MapLabToCmFrequency(Real &tran_coef, AthenaArray<Real> &split_ratio,
+                   AthenaArray<int> &map_start, AthenaArray<int> &map_end,
+                   AthenaArray<Real> &ir_cm, AthenaArray<Real> &ir_shift);
+  void MapCmToLabFrequency(Real &tran_coef, AthenaArray<Real> &split_ratio,
+                   AthenaArray<int> &map_start, AthenaArray<int> &map_end,
+                      AthenaArray<Real> &ir_shift, AthenaArray<Real> &ir_cm);
+
+  void GetCmMCIntensity(AthenaArray<Real> &ir_cm, AthenaArray<Real> &delta_nu_n,
+                                                    AthenaArray<Real> &ir_face);
+
+  void ForwardSplitting(Real &tran_coef,
+                      AthenaArray<Real> &ir_cm, AthenaArray<Real> &ir_face,
+                      AthenaArray<Real> &split_ratio,
+                      AthenaArray<int> &map_start,AthenaArray<int> &map_end);
+
+  void BackwardSplitting(Real &tran_coef,
+                      AthenaArray<Real> &ir_cm, AthenaArray<Real> &ir_face,
+                      AthenaArray<Real> &split_ratio,
+                      AthenaArray<int> &map_start,AthenaArray<int> &map_end);
+
+
+  void MapIrcmFrequency(AthenaArray<Real> &split_ratio,
+                          AthenaArray<int> &map_start, AthenaArray<int> &map_end,
+                  AthenaArray<Real> &input_array, AthenaArray<Real> &shift_array);
+
+  bool FreMapMatrix(AthenaArray<Real> &split_ratio, Real &tran_coef,
+    AthenaArray<int> &map_bin_start, AthenaArray<int> &map_bin_end,
+    AthenaArray<int> &map_count, AthenaArray<Real> &map_matrix);
+
+
+  bool InverseMapFrequency(Real &tran_coef,
+         AthenaArray<int> &map_count, AthenaArray<Real> &map_matrix,
+         AthenaArray<Real> &input_array, AthenaArray<Real> &shift_array);
+
+
+  void SplitFrequencyBinLinear(int &l_bd, int &r_bd,
+                  Real *nu_lab, Real &nu_l, Real &nu_r, Real &ir_l,
+                                      Real &ir_r, Real *split_ratio);
+
+
+
+  void ComToLabMultiGroup(const Real vx, const Real vy, const Real vz,
+                          Real *mux, Real *muy, Real *muz,
+                          AthenaArray<Real> &ir_cm, Real *ir_lab);
+
+  Real MultiGroupAbsScat(AthenaArray<Real> &wmu_cm,
+          AthenaArray<Real> &tran_coef, Real *sigma_a, Real *sigma_p,
+          Real *sigma_ae, Real *sigma_s, Real dt, Real lorz, Real rho, Real &tgas,
+          AthenaArray<Real> &implicit_coef, AthenaArray<Real> &ir_cm);
+
+  void AddMultiGroupCompt(MeshBlock *pmb, const Real dt,
+        AthenaArray<Real> &u, AthenaArray<Real> &ir);
+
+  // multigroup compton scattering function
+  void MultiGroupCompton(AthenaArray<Real> &wmu_cm,
+          AthenaArray<Real> &tran_coef, Real dt, Real lorz, Real rho,
+          Real &tgas_ini, Real &tgas, AthenaArray<Real> &ir_cm);
+
+  Real QuasiEqSol(Real &tgas, Real &tot_n);
+  Real ComptCorrection(Real &tgas);
+
+  void GetTaufactor(const Real tau, Real &factor1, int dir);
+  void GetTaufactorAdv(const Real tau, Real &factor);
+
+  void PredictVel(AthenaArray<Real> &ir, int k, int j, int i, Real dt, Real rho,
+                  Real *vx, Real *vy, Real *vz);
+
+  void SignalSpeed(const Real adv, const Real factor1,
+                 const Real factor2, Real *vel, Real *smax, Real *smin);
+
+  void SplitVelocity(Real *vel_l, Real *vel_r, const Real advl,
+            const Real advr, Real *smax_l, Real *smin_l, Real *smax_r, Real *smin_r);
+
+  int rad_xorder;
+  AthenaArray<Real> adv_vel; // the advectioin velocity that we separate
+  AthenaArray<Real> taufact;
+  AthenaArray<Real> rad_source; // store the radiation source terms
+  AthenaArray<Real> delta_source; // calculate delta E, delta Fr per frequency
+                                  // then sum the different across fre
+
+ private:
+  AthenaArray<Real> vel_, velx_,vely_,velz_;
+  AthenaArray<Real> il_, ilb_, ir_;// for recontruction
+                          // temporary array to store the flux, velocity
+  AthenaArray<Real> vncsigma_, vncsigma2_, wmu_cm_, tran_coef_, ir_cm_;
+  AthenaArray<Real> cm_to_lab_;
+  AthenaArray<Real> g_zeta_, q_zeta_, ql_zeta_, qr_zeta_, zeta_flux_, zeta_area_;
+  AthenaArray<Real> g_psi_, q_psi_, ql_psi_, qr_psi_, psi_flux_, psi_area_;
+  AthenaArray<Real> dflx_ang_, ang_vol_;
+  AthenaArray<Real> tgas_, vel_source_, tgas_new_; // array to store gas temperature,
+                                        // velocity for source term
+  // these are temporary arrays for multi-group source terms
+  AthenaArray<Real> sum_nu3_, sum_nu2_, sum_nu1_;
+  AthenaArray<Real> eq_sol_;
+  // The coefficients to solve the Kompaneets equation
+  AthenaArray<Real> com_b_face_coef_, com_d_face_coef_;
+  AthenaArray<Real> com_b_coef_l_, com_b_coef_r_;
+  AthenaArray<Real> com_d_coef_l_, com_d_coef_r_;
+  AthenaArray<Real> nf_rhs_, nf_n0_, new_j_nu_;
+
+  int tau_flag_;
+  // flag to add simple Compton scattering
+  // compton_t_ flag to estimate gas temperature or not for Compton scattering
+  int compton_flag_, compton_t_, split_compton_;
+  int adv_flag_; // flag used to indicate whether separate
+                 // advection flux from diffustion flux or not.
+  int imp_ang_flx_;
+
+  int doppler_flag_;
+
+  int flux_correct_flag_; // flag to do second order flux crrection or not.
+  AthenaArray<Real> x1face_area_, x2face_area_, x3face_area_;
+  AthenaArray<Real> x2face_area_p1_, x3face_area_p1_;
+  AthenaArray<Real> cell_volume_, dflx_, cwidth2_, cwidth3_;
+  AthenaArray<Real> adv_flx_;
+
+  AthenaArray<Real> const_coef_, exp_coef_;
+  AthenaArray<Real> const_coef1_l_, const_coef1_r_;
+  AthenaArray<Real> const_coef2_l_, const_coef2_r_;
+  AthenaArray<Real> const_coef3_l_, const_coef3_r_;
+  AthenaArray<Real> divflx_, implicit_coef_, ang_flx_, imp_ang_coef_;
+  AthenaArray<Real> imp_ang_coef_r_;
+  AthenaArray<Real> imp_ang_psi_l_, imp_ang_psi_r_;
+  AthenaArray<Real> left_coef1_, left_coef2_, left_coef3_;
+  AthenaArray<Real> limiter_, limiterj_, limiterk_, dql_, dqr_;
+  AthenaArray<Real> sfac1_x_, sfac2_x_;
+  AthenaArray<Real> sfac1_y_, sfac2_y_;
+  AthenaArray<Real> sfac1_z_, sfac2_z_;
+  AthenaArray<Real> sm_diff1_, sm_diff2_;
+  AthenaArray<Real> vel_ex_l_, vel_im_l_, vel_ex_r_, vel_im_r_;
+  AthenaArray<Real> dxw1_, dxw2_;
+
+  //----------------------------------------------------
+  // array for multi-group
+  // This is the coefficient in front I when calculate frequency flux
+
+  // This is the actual flux in frequency space
+  // shift amount from the frequency boundary
+  AthenaArray<Real> split_ratio_, fre_map_matrix_;
+  AthenaArray<Real> delta_nu_n_; // width of frequency bins for each angle
+  AthenaArray<Real> ir_buff_;
+  AthenaArray<Real> ir_face_, ir_ori_, ir_done_;
+  AthenaArray<int> map_bin_start_, map_bin_end_;
+  AthenaArray<int> map_count_;
+  AthenaArray<Real> nu_shift_;
+  int iteration_tgas_, iteration_compton_;
+  Real tgas_error_, compton_error_;
+  int nmax_map_; //maximum number of frequency bins that each bin will map to
+};
+
+#endif // NR_RADIATION_INTEGRATORS_RAD_INTEGRATORS_HPP_
diff --git a/src/nr_radiation/integrators/rad_source.cpp b/src/nr_radiation/integrators/rad_source.cpp
new file mode 100644
index 0000000000..a4e5475fda
--- /dev/null
+++ b/src/nr_radiation/integrators/rad_source.cpp
@@ -0,0 +1,581 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file rad_source.cpp
+//  \brief Add radiation source terms to both radiation and gas
+//======================================================================================
+
+
+// C headers
+
+// C++ headers
+#include <algorithm>
+#include <string>
+#include <vector>
+
+//Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp" //
+#include "../../eos/eos.hpp"
+#include "../../field/field.hpp"
+#include "../../hydro/hydro.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../parameter_input.hpp"
+#include "../implicit/radiation_implicit.hpp"
+#include "../radiation.hpp"
+
+// class header
+#include "./rad_integrators.hpp"
+
+// MPI/OpenMP header
+#ifdef MPI_PARALLEL
+#include <mpi.h>
+#endif
+
+
+// OpenMP header
+#ifdef OPENMP_PARALLEL
+#include <omp.h>
+#endif
+
+
+void RadIntegrator::CalSourceTerms(MeshBlock *pmb, const Real dt,
+        const int k, const int j, const int i, AthenaArray<Real> &u,
+        AthenaArray<Real> &ir_ini, AthenaArray<Real> &ir) {
+  NRRadiation *prad=pmb->pnrrad;
+  Real invcrat = 1.0/prad->crat;
+
+  Real *sigma_at, *sigma_s, *sigma_p, *sigma_pe;
+  Real *lab_ir;
+
+  int &nang =prad->nang;
+  int &nfreq=prad->nfreq;
+
+  // Get the temporary arrays
+  AthenaArray<Real> &wmu_cm = wmu_cm_;
+  AthenaArray<Real> &tran_coef = tran_coef_;
+  AthenaArray<Real> &ir_cm = ir_cm_;
+  AthenaArray<Real> &cm_to_lab = cm_to_lab_;
+
+  AthenaArray<Real> split_ratio;
+  AthenaArray<int> map_start, map_end;
+
+  // for implicit update, using the quantities from the partially
+  // updated u, not from w
+  Real rho = u(IDN,k,j,i);
+  Real vx = vel_source_(k,j,i,0);
+  Real vy = vel_source_(k,j,i,1);
+  Real vz = vel_source_(k,j,i,2);
+  Real vsq = vx*vx + vy*vy + vz*vz;
+
+  Real lorzsq = 1.0/(1.0 - vsq  * invcrat * invcrat);
+  Real lorz = std::sqrt(lorzsq);
+
+  sigma_at = &(prad->sigma_a(k,j,i,0));
+  sigma_s = &(prad->sigma_s(k,j,i,0));
+  sigma_p = &(prad->sigma_p(k,j,i,0));
+  sigma_pe =&(prad->sigma_pe(k,j,i,0));
+
+  // Prepare the transformation coefficients
+  Real numsum = 0.0;
+
+  for (int n=0; n<nang; ++n) {
+    Real vdotn = vx * prad->mu(0,k,j,i,n) + vy * prad->mu(1,k,j,i,n)
+                + vz * prad->mu(2,k,j,i,n);
+    Real vnc = 1.0 - vdotn * invcrat;
+    tran_coef(n) = lorz * vnc;
+    wmu_cm(n) = prad->wmu(n)/(tran_coef(n) * tran_coef(n));
+    numsum += wmu_cm(n);
+    cm_to_lab(n) = tran_coef(n)*tran_coef(n)*tran_coef(n)*tran_coef(n);
+  }
+           // Normalize weight in co-moving frame to make sure the sum is one
+  numsum = 1.0/numsum;
+
+  for (int n=0; n<nang; ++n) {
+    wmu_cm(n) *= numsum;
+  }
+
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    lab_ir=&(ir_ini(k,j,i,ifr*nang));
+    for (int n=0; n<nang; ++n)
+      ir_cm(n+ifr*nang) = lab_ir[n];
+    if (IM_RADIATION_ENABLED) {
+      // add the explicit flux divergence term
+      Real *divflux = &(divflx_(k,j,i,ifr*nang));
+      for (int n=0; n<nang; ++n) {
+        ir_cm(n+ifr*nang) += divflux[n];
+      }
+      // add angular flux
+      if (prad->angle_flag == 1) {
+        Real *p_angflx  = &(ang_flx_(k,j,i,ifr*nang));
+        for (int n=0; n<nang; ++n) {
+          ir_cm(n+ifr*nang) += p_angflx[n];
+        }
+      }
+    }
+    for (int n=0; n<nang; ++n) {
+      ir_cm(n+ifr*nang) *= cm_to_lab(n);
+    }
+    // apply floor to ir_cm
+    for (int n=0; n<nang; ++n) {
+      ir_cm(n+ifr*nang) = std::max(ir_cm(n+ifr*nang),static_cast<Real>(TINY_NUMBER));
+    }
+
+    for (int n=0; n<nang; n++) {
+      implicit_coef_(n+ifr*nang) = 1.0;
+      if (IM_RADIATION_ENABLED) {
+        implicit_coef_(n+ifr*nang) += const_coef_(k,j,i,n+ifr*nang);
+        if (prad->angle_flag == 1) {
+          implicit_coef_(n+ifr*nang) += imp_ang_coef_(k,j,i,n);
+        }
+      }
+    }
+  }
+  if (nfreq == 1) {
+  // Add absorption and scattering opacity source
+    tgas_new_(k,j,i) = AbsorptionScattering(wmu_cm,tran_coef, sigma_at, sigma_p,
+                                            sigma_pe, sigma_s, dt, lorz, rho,
+                                            tgas_(k,j,i),
+                                            implicit_coef_, ir_cm);
+    // Add compton scattering
+    if (compton_flag_ > 0) {
+      Compton(wmu_cm,tran_coef, sigma_s, dt, lorz, rho, tgas_new_(k,j,i), ir_cm);
+    }
+  } else {
+    // map frequency grid
+    if (doppler_flag_ > 0) {
+      for (int n=0; n<nang; ++n) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          ir_ori_(ifr) = ir_cm(ifr*nang+n);
+        }
+        split_ratio.InitWithShallowSlice(split_ratio_,3,n,1);
+        map_start.InitWithShallowSlice(map_bin_start_,2,n,1);
+        map_end.InitWithShallowSlice(map_bin_end_,2,n,1);
+
+        MapLabToCmFrequency(tran_coef(n), split_ratio, map_start, map_end,
+                                                     ir_ori_, ir_done_);
+
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          ir_cm(ifr*nang+n) = ir_done_(ifr);
+        }
+      }
+    }
+    // calculate the source term
+    tgas_new_(k,j,i) = MultiGroupAbsScat(wmu_cm,tran_coef, sigma_at, sigma_p,
+                               sigma_pe, sigma_s, dt, lorz, rho, tgas_(k,j,i),
+                                                     implicit_coef_,ir_cm);
+    // Add compton scattering
+    // Compton scattering for implicit scheme is added separately
+    if ((compton_flag_ > 0) && (split_compton_ == 0)) {
+      Real t_ini = tgas_new_(k,j,i);
+      Real t_old;
+      int count=0;
+      Real relative_error = 1;
+      while((count < iteration_compton_) && (relative_error > compton_error_)) {
+        ir_buff_ = ir_cm;
+        t_old = tgas_new_(k,j,i);
+        MultiGroupCompton(wmu_cm,tran_coef,dt,lorz,rho,t_ini,tgas_new_(k,j,i),
+                                                                    ir_buff_);
+        count++;
+        relative_error = std::abs(t_old-tgas_new_(k,j,i))/tgas_new_(k,j,i);
+      }
+      ir_cm = ir_buff_;
+      // inverseshift
+    }
+    // map frequency grid
+    if (doppler_flag_ > 0) {
+      for (int n=0; n<nang; ++n) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          ir_ori_(ifr) = ir_cm(ifr*nang+n);
+        }
+        split_ratio.InitWithShallowSlice(split_ratio_,3,n,1);
+        map_start.InitWithShallowSlice(map_bin_start_,2,n,1);
+        map_end.InitWithShallowSlice(map_bin_end_,2,n,1);
+
+        bool invertible = FreMapMatrix(split_ratio, tran_coef(n), map_start,
+                                        map_end, map_count_, fre_map_matrix_);
+        if (invertible) {
+          bool success = InverseMapFrequency(tran_coef(n), map_count_,
+                                          fre_map_matrix_, ir_ori_, ir_done_);
+
+          if (!success)
+            MapCmToLabFrequency(tran_coef(n),split_ratio, map_start, map_end,
+                                                        ir_ori_,ir_done_);
+
+        } else {
+          MapCmToLabFrequency(tran_coef(n),split_ratio, map_start, map_end,
+                                                        ir_ori_,ir_done_);
+        }
+
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          ir_cm(ifr*nang+n) = ir_done_(ifr);
+        }
+      }
+    }
+  }
+
+  //update specific intensity in the lab frame
+  // do not modify ir_ini
+  Real omega = 1.0;
+  if (IM_RADIATION_ENABLED) {
+    omega = pmb->pmy_mesh->pimrad->omega;
+  }
+  Real omega_1 = 1.0 - omega;
+
+  if (std::abs(omega_1) < TINY_NUMBER) {
+    for (int ifr=0; ifr<nfreq; ++ifr) {
+      lab_ir = &(ir(k,j,i,nang*ifr));
+      for (int n=0; n<nang; ++n) {
+        lab_ir[n] = std::max(ir_cm(n+ifr*nang)/cm_to_lab(n),
+                             static_cast<Real>(TINY_NUMBER));
+      }
+    }
+  } else {
+    for (int ifr=0; ifr<nfreq; ++ifr) {
+      lab_ir = &(ir(k,j,i,nang*ifr));
+      for (int n=0; n<nang; ++n) {
+        Real ir_old = lab_ir[n];
+        Real ir_new = ir_cm(n+ifr*nang)/cm_to_lab(n);
+        ir_new = omega_1 * ir_old + omega * ir_new;
+        lab_ir[n] = std::max(ir_new, static_cast<Real>(TINY_NUMBER));
+      }
+    }
+    tgas_new_(k,j,i) = omega_1*tgas_(k,j,i) + omega*tgas_new_(k,j,i);
+  }
+}
+
+
+
+void RadIntegrator::AddMultiGroupCompt(MeshBlock *pmb, const Real dt,
+        AthenaArray<Real> &u, AthenaArray<Real> &ir) {
+  // need to transform lab frame ir to co-moving frame
+  NRRadiation *prad=pmb->pnrrad;
+
+  Real& prat = prad->prat;
+  Real invcrat = 1.0/prad->crat;
+  Real invredfactor = prad->crat/prad->reduced_c;
+
+  AthenaArray<Real> split_ratio;
+  AthenaArray<int> map_start, map_end;
+
+  Real *lab_ir;
+
+  int &nang =prad->nang;
+  int &nfreq=prad->nfreq;
+
+  // only apply for multi-grou case
+  if ((nfreq > 1) && (compton_flag_ > 0) && (split_compton_ > 0)) {
+  // Get the temporary arrays
+    AthenaArray<Real> &wmu_cm = wmu_cm_;
+    AthenaArray<Real> &tran_coef = tran_coef_;
+    AthenaArray<Real> &ir_cm = ir_cm_;
+    AthenaArray<Real> &cm_to_lab = cm_to_lab_;
+
+
+    int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+    int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          Real rho = u(IDN,k,j,i);
+          Real vx = vel_source_(k,j,i,0);
+          Real vy = vel_source_(k,j,i,1);
+          Real vz = vel_source_(k,j,i,2);
+          Real vsq = vx*vx + vy*vy + vz*vz;
+
+          Real lorzsq = 1.0/(1.0 - vsq  * invcrat * invcrat);
+          Real lorz = std::sqrt(lorzsq);
+
+          // Prepare the transformation coefficients
+          Real numsum = 0.0;
+
+          for (int n=0; n<nang; ++n) {
+             Real vdotn = vx * prad->mu(0,k,j,i,n) + vy * prad->mu(1,k,j,i,n)
+                        + vz * prad->mu(2,k,j,i,n);
+             Real vnc = 1.0 - vdotn * invcrat;
+             tran_coef(n) = lorz * vnc;
+             wmu_cm(n) = prad->wmu(n)/(tran_coef(n) * tran_coef(n));
+             numsum += wmu_cm(n);
+             cm_to_lab(n) = tran_coef(n)*tran_coef(n)*tran_coef(n)*tran_coef(n);
+          }
+           // Normalize weight in co-moving frame to make sure the sum is one
+          numsum = 1.0/numsum;
+#pragma omp simd
+          for (int n=0; n<nang; ++n) {
+            wmu_cm(n) *= numsum;
+          }
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            lab_ir=&(ir(k,j,i,ifr*nang));
+            for (int n=0; n<nang; ++n)
+              ir_cm(n+ifr*nang) = std::max(lab_ir[n] * cm_to_lab(n),
+                                           static_cast<Real>(TINY_NUMBER));
+            // store the moments
+            Real er_fr = 0.0;
+            for (int n=0; n<nang; ++n) {
+              Real ir_weight = lab_ir[n]*prad->wmu(n);
+              er_fr += ir_weight;
+            }
+            delta_source(0,ifr) = er_fr;
+          }
+          // map frequency grid
+          for (int n=0; n<nang; ++n) {
+            for (int ifr=0; ifr<nfreq; ++ifr)
+              ir_ori_(ifr) = ir_cm(ifr*nang+n);
+
+            split_ratio.InitWithShallowSlice(split_ratio_,3,n,1);
+            map_start.InitWithShallowSlice(map_bin_start_,2,n,1);
+            map_end.InitWithShallowSlice(map_bin_end_,2,n,1);
+
+            MapLabToCmFrequency(tran_coef(n), split_ratio, map_start, map_end,
+                                                            ir_ori_, ir_done_);
+
+
+            for (int ifr=0; ifr<nfreq; ++ifr) {
+              ir_cm(ifr*nang+n) = ir_done_(ifr);
+            }
+          }
+
+          // Add compton scattering
+          Real t_ini = tgas_new_(k,j,i);
+          Real t_old;
+          int count=0;
+          Real relative_error = 1;
+          while((count < iteration_compton_) && (relative_error > compton_error_)) {
+            ir_buff_ = ir_cm;
+            t_old = tgas_new_(k,j,i);
+            MultiGroupCompton(wmu_cm,tran_coef,dt,lorz,rho,t_ini,tgas_new_(k,j,i),
+                                                                        ir_buff_);
+            count++;
+            relative_error = std::abs(t_old-tgas_new_(k,j,i))/tgas_new_(k,j,i);
+          }
+          ir_cm = ir_buff_;
+
+          // map frequency grid
+          for (int n=0; n<nang; ++n) {
+            for (int ifr=0; ifr<nfreq; ++ifr) {
+              ir_ori_(ifr) = ir_cm(ifr*nang+n);
+            }
+
+            split_ratio.InitWithShallowSlice(split_ratio_,3,n,1);
+            map_start.InitWithShallowSlice(map_bin_start_,2,n,1);
+            map_end.InitWithShallowSlice(map_bin_end_,2,n,1);
+
+            bool invertible = FreMapMatrix(split_ratio, tran_coef(n), map_start,
+                                            map_end, map_count_, fre_map_matrix_);
+            if (invertible) {
+              bool success = InverseMapFrequency(tran_coef(n), map_count_,
+                                          fre_map_matrix_, ir_ori_, ir_done_);
+              if (!success) {
+                  MapCmToLabFrequency(tran_coef(n),split_ratio, map_start, map_end,
+                                                        ir_ori_,ir_done_);
+              }
+            } else {
+              MapCmToLabFrequency(tran_coef(n),split_ratio, map_start, map_end,
+                                                             ir_ori_,ir_done_);
+            }
+
+            for (int ifr=0; ifr<nfreq; ++ifr) {
+              ir_cm(ifr*nang+n) = ir_done_(ifr);
+            }
+          }
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            lab_ir = &(ir(k,j,i,nang*ifr));
+            for (int n=0; n<nang; ++n) {
+              lab_ir[n] = std::max(ir_cm(n+ifr*nang)/cm_to_lab(n),
+                                   static_cast<Real>(TINY_NUMBER));
+            }
+          }
+
+          if (prad->set_source_flag > 0) {
+            for (int ifr=0; ifr<nfreq; ++ifr) {
+              Real *p_ir =  &(ir(k,j,i,ifr*nang));
+              Real er_fr = 0.0;
+              for (int n=0; n<nang; ++n) {
+                Real ir_weight = p_ir[n] * prad->wmu(n);
+                er_fr  += ir_weight;
+              }
+              rad_source(0,k,j,i) += (-prat*(er_fr-delta_source(0,ifr))*invredfactor);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+// ir_ini and ir only differ by the source term for explicit scheme
+// for implicit scheme, ir also includes the flux divergence term
+
+void RadIntegrator::GetHydroSourceTerms(MeshBlock *pmb,
+                       AthenaArray<Real> &ir_ini, AthenaArray<Real> &ir) {
+  NRRadiation *prad=pmb->pnrrad;
+  Real& prat = prad->prat;
+  Real invcrat = 1.0/prad->crat;
+  Real invredc = 1.0/prad->reduced_c;
+  Real invredfactor = invredc/invcrat;
+
+  int &nang =prad->nang;
+  int &nfreq=prad->nfreq;
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      for (int i=is; i<=ie; ++i) {
+        // first, calculate Er and Fr in lab frame before the step
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          Real *p_ir0 =  &(ir_ini(k,j,i,ifr*nang));
+          Real er_fr = 0.0;
+          Real frx_fr = 0.0;
+          Real fry_fr = 0.0;
+          Real frz_fr = 0.0;
+          if (IM_RADIATION_ENABLED) {
+            for (int n=0; n<nang; ++n) {
+              Real ir_weight = p_ir0[n];
+              ir_weight += divflx_(k,j,i,ifr*nang+n);
+              if (prad->angle_flag == 1) {
+                ir_weight += ang_flx_(k,j,i,ifr*nang+n);
+                ir_weight -= ((imp_ang_coef_(k,j,i,n))
+                           * ir(k,j,i,ifr*nang+n));
+              }
+              ir_weight -= (const_coef_(k,j,i,ifr*nang+n)
+                           * ir(k,j,i,ifr*nang+n));
+              ir_weight *= prad->wmu(n);
+              er_fr  += ir_weight;
+              frx_fr += ir_weight * prad->mu(0,k,j,i,n);
+              fry_fr += ir_weight * prad->mu(1,k,j,i,n);
+              frz_fr += ir_weight * prad->mu(2,k,j,i,n);
+            }
+          } else { // end IM_Radiation
+#pragma omp simd reduction (+:er_fr,frx_fr,fry_fr,frz_fr)
+            for (int n=0; n<nang; ++n) {
+              Real ir_weight = p_ir0[n];
+              ir_weight *= prad->wmu(n);
+              er_fr  += ir_weight;
+              frx_fr += ir_weight * prad->mu(0,k,j,i,n);
+              fry_fr += ir_weight * prad->mu(1,k,j,i,n);
+              frz_fr += ir_weight * prad->mu(2,k,j,i,n);
+            }
+          }
+          delta_source(0,ifr) = er_fr;
+          delta_source(1,ifr) = frx_fr;
+          delta_source(2,ifr) = fry_fr;
+          delta_source(3,ifr) = frz_fr;
+        }
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          Real *p_ir =  &(ir(k,j,i,ifr*nang));
+          Real er_fr = 0.0;
+          Real frx_fr = 0.0;
+          Real fry_fr = 0.0;
+          Real frz_fr = 0.0;
+#pragma omp simd reduction (+:er_fr,frx_fr,fry_fr,frz_fr)
+          for (int n=0; n<nang; ++n) {
+            Real ir_weight = p_ir[n] * prad->wmu(n);
+            er_fr  += ir_weight;
+            frx_fr += ir_weight * prad->mu(0,k,j,i,n);
+            fry_fr += ir_weight * prad->mu(1,k,j,i,n);
+            frz_fr += ir_weight * prad->mu(2,k,j,i,n);
+          }
+          delta_source(0,ifr) = er_fr  - delta_source(0,ifr);
+          delta_source(1,ifr) = frx_fr - delta_source(1,ifr);
+          delta_source(2,ifr) = fry_fr - delta_source(2,ifr);
+          delta_source(3,ifr) = frz_fr - delta_source(3,ifr);
+        }
+
+        Real delta_er = 0.0, delta_frx=0.0, delta_fry = 0.0, delta_frz = 0.0;
+
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          delta_er  += delta_source(0,ifr);
+          delta_frx += delta_source(1,ifr);
+          delta_fry += delta_source(2,ifr);
+          delta_frz += delta_source(3,ifr);
+        }
+
+        // Now apply the radiation source terms to gas with energy and
+        // momentum conservation
+        rad_source(0,k,j,i) = (-prat*delta_er  * invredfactor);
+        rad_source(1,k,j,i) = (-prat*delta_frx * invredc);
+        rad_source(2,k,j,i) = (-prat*delta_fry * invredc);
+        rad_source(3,k,j,i) = (-prat*delta_frz * invredc);
+      }
+    }
+  }
+}
+
+
+void RadIntegrator::AddSourceTerms(MeshBlock *pmb, AthenaArray<Real> &u) {
+  NRRadiation *prad=pmb->pnrrad;
+  Real invcrat = 1.0/prad->crat;
+  Field *pfield = pmb->pfield;
+
+  Real gm1 = pmb->peos->GetGamma() - 1.0;
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      for (int i=is; i<=ie; ++i) {
+        u(IM1,k,j,i) += rad_source(1,k,j,i);
+        u(IM2,k,j,i) += rad_source(2,k,j,i);
+        u(IM3,k,j,i) += rad_source(3,k,j,i);
+
+        //limit the velocity by speed of light
+        Real vx = u(IM1,k,j,i)/u(IDN,k,j,i);
+        Real vy = u(IM2,k,j,i)/u(IDN,k,j,i);
+        Real vz = u(IM3,k,j,i)/u(IDN,k,j,i);
+        Real vsq = vx*vx+vy*vy+vz*vz;
+        Real vel = std::sqrt(vsq);
+        Real ratio = vel * invcrat;
+        if (ratio > prad->vmax) {
+          Real factor = prad->vmax/ratio;
+          u(IM1,k,j,i) *= factor;
+          u(IM2,k,j,i) *= factor;
+          u(IM3,k,j,i) *= factor;
+        }
+
+        Real ekin = 0.5 *(SQR(u(IM1,k,j,i))+SQR(u(IM2,k,j,i))
+                  +SQR(u(IM3,k,j,i)))/u(IDN,k,j,i);
+        Real pb = 0.0;
+        if (MAGNETIC_FIELDS_ENABLED) {
+          pb = 0.5*(SQR(pfield->bcc(IB1,k,j,i))+SQR(pfield->bcc(IB2,k,j,i))
+               +SQR(pfield->bcc(IB3,k,j,i)));
+        }
+
+        if (prad->set_source_flag == 2) {
+          Real eint = tgas_new_(k,j,i) * u(IDN,k,j,i)/gm1;
+          u(IEN,k,j,i) = eint + pb + ekin;
+        } else {
+          Real e_source = rad_source(0,k,j,i);
+          // first check that gas internal energy will not become negative
+          Real eint = u(IEN,k,j,i) + e_source - ekin - pb;
+          Real tgas = eint * gm1/u(IDN,k,j,i);
+          if (eint < 0.0) {
+            eint = tgas_new_(k,j,i) * u(IDN,k,j,i)/gm1;
+            u(IEN,k,j,i) = eint + pb + ekin;
+          } else if (tgas > prad->t_ceiling_(k,j,i)) {
+            eint = prad->t_ceiling_(k,j,i) * u(IDN,k,j,i)/gm1;
+            u(IEN,k,j,i) = ekin + pb + eint;
+          } else {
+            u(IEN,k,j,i) += e_source;
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/src/nr_radiation/integrators/rad_transport.cpp b/src/nr_radiation/integrators/rad_transport.cpp
new file mode 100644
index 0000000000..71b6ab94c2
--- /dev/null
+++ b/src/nr_radiation/integrators/rad_transport.cpp
@@ -0,0 +1,840 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file rad_transport.cpp
+//  \brief implementation of radiation integrators
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <sstream>    // stringstream
+
+// Athena++ headers
+#include "../../athena.hpp"
+#include "../../athena_arrays.hpp"
+#include "../../coordinates/coordinates.hpp"
+#include "../../mesh/mesh.hpp"
+#include "../../parameter_input.hpp"
+#include "../../reconstruct/reconstruction.hpp"
+#include "../radiation.hpp"
+
+// class header
+#include "rad_integrators.hpp"
+
+// OpenMP header
+#ifdef OPENMP_PARALLEL
+#include <omp.h>
+#endif
+
+// calculate the transport flux as
+// (v- vel)I + vel*I
+void RadIntegrator::CalculateFluxes(AthenaArray<Real> &w,
+                                    AthenaArray<Real> &ir, const int order) {
+  NRRadiation *prad=pmy_rad;
+  MeshBlock *pmb=prad->pmy_block;
+  Coordinates *pco=pmb->pcoord;
+
+  int nang=prad->nang;
+  int nfreq=prad->nfreq;
+  // Real invcrat=1.0/pmy_rad->crat;
+
+  // int ncells1 = pmb->ncells1, ncells2 = pmb->ncells2,
+  // ncells3 = pmb->ncells3;
+  // Real tau_fact;
+
+  AthenaArray<Real> &x1flux=prad->flux[X1DIR];
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+  // int il, iu, jl, ju, kl, ku;
+  // jl = js, ju=je, kl=ks, ku=ke;
+
+  // if (ncells2 > 1) {
+  //   if (ncells3 == 1) {
+  //     jl=js-1, ju=je+1, kl=ks, ku=ke;
+  //   } else {
+  //     jl=js-1, ju=je+1, kl=ks-1, ku=ke+1;
+  //   }
+  // }
+
+  //-----------------------------------------------------------------------------
+  // i-direction
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      pco->CenterWidth1(k,j,is-1,ie+1,dxw1_);
+      for (int i=is; i<=ie+1; ++i) {
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          // use the signal speed in each frequency group
+          Real sigmal = prad->sigma_a(k,j,i-1,ifr) + prad->sigma_s(k,j,i-1,ifr);
+          Real sigmar = prad->sigma_a(k,j,i,ifr) + prad->sigma_s(k,j,i,ifr);
+          Real taul = dxw1_(i-1) * sigmal;
+          Real taur = dxw1_(i) * sigmar;
+
+          Real f_l = 1.0;
+          Real f_r = 1.0;
+          taul *= taufact(k,j,i-1);
+          taur *= taufact(k,j,i);
+          GetTaufactor(taul+taur,f_r,1);
+          GetTaufactor(taul+taur,f_l,-1);
+
+          Real *s1n = &(sfac1_x_(i,ifr*nang));
+          Real *s2n = &(sfac2_x_(i,ifr*nang));
+          Real *velxn = &(velx_(k,j,i,ifr*nang));
+          Real adv = adv_vel(0,k,j,i);
+          SignalSpeed(adv, f_l, f_r, velxn, s1n, s2n);
+        }
+      }
+      if (order == 1) {
+        pmb->precon->DonorCellX1(k, j, is-1, ie+1, ir, -1, il_, ir_);
+      } else if (order == 2) {
+        pmb->precon->PiecewiseLinearX1(k, j, is-1, ie+1, ir, -1, il_, ir_);
+      } else {
+        pmb->precon->PiecewiseParabolicX1(k, j, is-1, ie+1, ir, -1, il_, ir_);
+      }
+
+      // calculate flux with HLLE type flux
+      //(smax F(u_L) - Smin F(U_R))/(smax-smin)+smax*smin(u(R)-u_L)/(smax-smin)
+      for (int i=is; i<=ie+1; ++i) {
+        Real *vel = &(velx_(k,j,i,0));
+        Real *smax = &(sfac1_x_(i,0));
+        Real *smin = &(sfac2_x_(i,0));
+        Real *irln = &(il_(i,0));
+        Real *irrn = &(ir_(i,0));
+        Real adv = 0.0;
+        if (adv_flag_ > 0) {
+          adv = adv_vel(0,k,j,i);
+        }
+        Real *sm_diff = &(sm_diff1_(0));
+        for (int n=0; n<prad->n_fre_ang; ++n) {
+          Real diff = smax[n] - smin[n];
+          if (std::abs(diff) < TINY_NUMBER)
+            sm_diff[n] = 0.0;
+          else
+            sm_diff[n] = 1.0/diff;
+        }
+        for (int n=0; n<prad->n_fre_ang; ++n) {
+          Real vl = vel[n] - adv;
+          x1flux(k,j,i,n) = smax[n] * (vl - smin[n]) * irln[n] * sm_diff[n]
+                            + smin[n] * (smax[n] - vl) * irrn[n] * sm_diff[n];
+        }
+        if (adv_flag_ > 0) {
+          Real &advv = adv_vel(0,k,j,i);
+          if (advv >0) {
+            for (int n=0; n<prad->n_fre_ang; ++n) {
+              x1flux(k,j,i,n) += advv * irln[n];
+            }
+          } else {
+            for (int n=0; n<prad->n_fre_ang; ++n) {
+              x1flux(k,j,i,n) += advv * irrn[n];
+            }
+          }
+        }
+      }
+    }
+  }
+
+
+  // j-direction
+  if (pmb->pmy_mesh->f2) {
+    AthenaArray<Real> &x2flux=prad->flux[X2DIR];
+
+    for (int k=ks; k<=ke; ++k) {
+      // first, calculate speed
+      for (int j=js; j<=je+1; ++j) {
+        pco->CenterWidth2(k,j-1,is,ie,dxw1_);
+        pco->CenterWidth2(k,j,is,ie,dxw2_);
+        for (int i=is; i<=ie; ++i) {
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            Real sigmal = prad->sigma_a(k,j-1,i,ifr) + prad->sigma_s(k,j-1,i,ifr);
+            Real sigmar = prad->sigma_a(k,j,i,ifr) + prad->sigma_s(k,j,i,ifr);
+            Real taul = dxw1_(i) * sigmal;
+            Real taur = dxw2_(i) * sigmar;
+
+            Real f_l = 1.0;
+            Real f_r = 1.0;
+            taul *= taufact(k,j-1,i);
+            taur *= taufact(k,j,i);
+            GetTaufactor(taul+taur,f_r,1);
+            GetTaufactor(taul+taur,f_l,-1);
+
+            Real *s1n = &(sfac1_y_(j,i,ifr*nang));
+            Real *s2n = &(sfac2_y_(j,i,ifr*nang));
+            Real *velyn = &(vely_(k,j,i,ifr*nang));
+            Real adv = adv_vel(1,k,j,i);
+            SignalSpeed(adv, f_l, f_r, velyn, s1n, s2n);
+          }
+        }
+      }
+
+      // reconstruction
+      if (order == 1) {
+        pmb->precon->DonorCellX2(k, js-1, is, ie, ir, -1, il_, ir_);
+      } else if (order == 2) {
+        pmb->precon->PiecewiseLinearX2(k, js-1, is, ie, ir, -1, il_, ir_);
+      } else {
+        pmb->precon->PiecewiseParabolicX2(k, js-1, is, ie, ir, -1, il_, ir_);
+      }
+
+
+      for (int j=js; j<=je+1; ++j) {
+        if (order == 1) {
+          pmb->precon->DonorCellX2(k, j, is, ie, ir, -1, ilb_, ir_);
+        } else if (order == 2) {
+          pmb->precon->PiecewiseLinearX2(k, j, is, ie, ir, -1, ilb_, ir_);
+        } else {
+          pmb->precon->PiecewiseParabolicX2(k, j, is, ie, ir, -1, ilb_, ir_);
+        }
+
+        for (int i=is; i<=ie; ++i) {
+          Real *vel = &(vely_(k,j,i,0));
+          Real *smax = &(sfac1_y_(j,i,0));
+          Real *smin = &(sfac2_y_(j,i,0));
+          Real *irln = &(il_(i,0));
+          Real *irrn = &(ir_(i,0));
+          Real adv = 0.0;
+          if (adv_flag_ > 0) {
+            adv = adv_vel(1,k,j,i);
+          }
+          Real *sm_diff = &(sm_diff1_(0));
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real diff = smax[n] - smin[n];
+            if (std::abs(diff) < TINY_NUMBER)
+              sm_diff[n] = 0.0;
+            else
+              sm_diff[n] = 1.0/diff;
+          }
+
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real vl = vel[n] - adv;
+            x2flux(k,j,i,n) = smax[n] * (vl - smin[n]) * irln[n] * sm_diff[n]
+                              + smin[n] * (smax[n] - vl) * irrn[n] * sm_diff[n];
+            // x2flux(k,j,i,n) = (smax[n] * vl * irln[n] - smin[n] * vl * irrn[n]
+            // + smax[n] * smin[n] * (irrn[n] - irln[n]))/(smax[n] - smin[n]);
+          }
+          if (adv_flag_ > 0) {
+            Real &advv = adv_vel(1,k,j,i);
+            if (advv >0) {
+              for (int n=0; n<prad->n_fre_ang; ++n) {
+                x2flux(k,j,i,n) += advv * irln[n];
+              }
+            } else {
+              for (int n=0; n<prad->n_fre_ang; ++n) {
+                x2flux(k,j,i,n) += advv * irrn[n];
+              }
+            }
+          }
+        }
+        il_.SwapAthenaArray(ilb_);
+      }
+    }
+  }
+
+  if (pmb->pmy_mesh->f3) {
+    AthenaArray<Real> &x3flux=prad->flux[X3DIR];
+    // First, calculate the transport velocity
+
+    for (int k=ks; k<=ke+1; ++k) {
+      for (int j=js; j<=je; ++j) {
+        pco->CenterWidth3(k-1,j,is,ie,dxw1_);
+        pco->CenterWidth3(k,j,is,ie,dxw2_);
+        for (int i=is; i<=ie; ++i) {
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            Real sigmal = prad->sigma_a(k-1,j,i,ifr) + prad->sigma_s(k-1,j,i,ifr);
+            Real sigmar = prad->sigma_a(k,j,i,ifr) + prad->sigma_s(k,j,i,ifr);
+            Real taul = dxw1_(i) * sigmal;
+            Real taur = dxw2_(i) * sigmar;
+
+            Real f_l = 1.0;
+            Real f_r = 1.0;
+            taul *= taufact(k-1,j,i);
+            taur *= taufact(k,j,i);
+            GetTaufactor(taul+taur,f_r,1);
+            GetTaufactor(taul+taur,f_l,-1);
+
+            Real *s1n = &(sfac1_z_(k,j,i,ifr*nang));
+            Real *s2n = &(sfac2_z_(k,j,i,ifr*nang));
+            Real *velzn = &(velz_(k,j,i,ifr*nang));
+            Real adv = adv_vel(2,k,j,i);
+            SignalSpeed(adv, f_l, f_r, velzn, s1n, s2n);
+          }
+        }
+      }
+    }
+
+    for (int j=js; j<=je; ++j) { // this loop ordering is intentional
+      // reconstruct the first row
+
+      if (order == 1) {
+        pmb->precon->DonorCellX3(ks-1, j, is, ie, ir, -1, il_, ir_);
+      } else if (order == 2) {
+        pmb->precon->PiecewiseLinearX3(ks-1, j, is, ie, ir, -1, il_, ir_);
+      } else {
+        pmb->precon->PiecewiseParabolicX3(ks-1, j, is, ie, ir, -1, il_, ir_);
+      }
+
+
+      for (int k=ks; k<=ke+1; ++k) {
+        // reconstruct L/R states at k
+
+        if (order == 1) {
+          pmb->precon->DonorCellX3(k, j, is, ie, ir, -1, ilb_, ir_);
+        } else if (order == 2) {
+          pmb->precon->PiecewiseLinearX3(k, j, is, ie, ir, -1, ilb_, ir_);
+        } else {
+          pmb->precon->PiecewiseParabolicX3(k, j, is, ie, ir, -1, ilb_, ir_);
+        }
+
+        for (int i=is; i<=ie; ++i) {
+          Real *vel = &(velz_(k,j,i,0));
+          Real *smax = &(sfac1_z_(k,j,i,0));
+          Real *smin = &(sfac2_z_(k,j,i,0));
+          Real *irln = &(il_(i,0));
+          Real *irrn = &(ir_(i,0));
+          Real adv = 0.0;
+          if (adv_flag_ > 0) {
+            adv = adv_vel(2,k,j,i);
+          }
+          Real *sm_diff = &(sm_diff1_(0));
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real diff = smax[n] - smin[n];
+            if (std::abs(diff) < TINY_NUMBER)
+              sm_diff[n] = 0.0;
+            else
+              sm_diff[n] = 1.0/diff;
+          }
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            Real vl = vel[n] - adv;
+            x3flux(k,j,i,n) = smax[n] * (vl - smin[n]) * irln[n] * sm_diff[n]
+                              + smin[n] * (smax[n] - vl) * irrn[n] * sm_diff[n];
+            // x3flux(k,j,i,n) = (smax[n] * vl * irln[n] - smin[n] * vl * irrn[n]
+            // + smax[n] * smin[n] * (irrn[n] - irln[n]))/(smax[n] - smin[n]);
+          }// end n
+          if (adv_flag_ > 0) {
+            Real &advv = adv_vel(2,k,j,i);
+            if (advv >0) {
+              for (int n=0; n<prad->n_fre_ang; ++n) {
+                x3flux(k,j,i,n) += advv * irln[n];
+              }
+            } else {
+              for (int n=0; n<prad->n_fre_ang; ++n) {
+                x3flux(k,j,i,n) += advv * irrn[n];
+              }
+            }
+          }
+        }
+        // swap the arrays for the next step
+        il_.SwapAthenaArray(ilb_);
+      }
+    }
+  }
+
+  // calculate flux along angular direction
+  if ((prad->angle_flag == 1) && (prad->nzeta > 0) && (imp_ang_flx_ == 0)) {
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          // going through all frequency groups
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            int &nzeta = prad->nzeta;
+            int &npsi = prad->npsi;
+            int psi_limit=2*npsi;
+            if (npsi == 0) psi_limit=1;
+
+            for (int m=0; m<psi_limit; ++m) {
+              // TODO(KGF): forced to comment out 4x pragmas here and 2x in
+              // ../get_moments.cpp to stop link-time segfault with icpx 2023.0.0
+              // and other problems on 2022.2.0
+//#pragma omp simd
+              for (int n=0; n<nzeta*2; ++n) {
+                int ang_num = ifr*nang + n*psi_limit+m;
+                q_zeta_(n+NGHOST) = ir(k,j,i,ang_num);
+              }
+              // Add ghost zones
+//#pragma omp simd
+              for (int n=1; n<=NGHOST; ++n) {
+                q_zeta_(NGHOST-n) = q_zeta_(NGHOST+n-1);
+              }
+//#pragma omp simd
+              for (int n=1; n<=NGHOST; ++n) {
+                q_zeta_(2*nzeta+NGHOST+n-1) = q_zeta_(2*nzeta+NGHOST-n);
+              }
+              int zl = NGHOST-1;
+              int zu = 2*nzeta+NGHOST;
+              if (order == 1) {
+                pmb->precon->DonorCellZeta(prad,zl,zu,q_zeta_,
+                                           ql_zeta_,qr_zeta_);
+              } else {
+                pmb->precon->PiecewiseLinearZeta(prad,zl,zu,q_zeta_,
+                                                 ql_zeta_,qr_zeta_);
+              }
+
+              // zeta flux
+              pco->GetGeometryZeta(prad,k,j,i,g_zeta_);
+              for (int n=0; n<nzeta*2+1; ++n) {
+                int ang_num = n*psi_limit+m;
+                Real g_coef = g_zeta_(n);
+                if (g_coef > 0)
+                  zeta_flux_(k,j,i,ifr,ang_num) =  -prad->reduced_c * qr_zeta_(n+NGHOST)
+                                                   * g_zeta_(n);
+                else if (g_coef < 0)
+                  zeta_flux_(k,j,i,ifr,ang_num) =  -prad->reduced_c * ql_zeta_(n+NGHOST)
+                                                   * g_zeta_(n);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // Now calculate psi flux
+  if ((prad->angle_flag == 1) && (prad->npsi > 0) && (imp_ang_flx_ == 0)) {
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            int &nzeta = prad->nzeta;
+            int &npsi = prad->npsi;
+            int zeta_limit=2*nzeta;
+            if (nzeta == 0) zeta_limit=1;
+
+            for (int n=0; n<zeta_limit; ++n) {
+              Real *qpsi = &(q_psi_(NGHOST));
+              Real *irm = &(ir(k,j,i,ifr*nang+n*2*npsi));
+              for (int m=0; m<npsi*2; ++m) {
+                qpsi[m] = irm[m];
+              }
+              // Add ghost zones
+              // phi is periodic
+              // the first one qpsi[NGHOST]
+              // The last one is qpsi[NGHOST+2*npsi-1]
+              for (int m=1; m<=NGHOST; ++m) {
+                q_psi_(NGHOST-m) = q_psi_(2*npsi+NGHOST-m);
+              }
+              for (int m=1; m<=NGHOST; ++m) {
+                q_psi_(2*npsi+NGHOST+m-1) = q_psi_(NGHOST+m-1);
+              }
+              int pl = NGHOST-1;
+              int pu = 2*npsi+NGHOST;
+              if (order == 1) {
+                pmb->precon->DonorCellPsi(prad,pl,pu,q_psi_,
+                                          ql_psi_,qr_psi_);
+              } else {
+                pmb->precon->PiecewiseLinearPsi(prad,pl,pu,q_psi_,
+                                                ql_psi_,qr_psi_);
+              }
+
+              // psi flux
+              if (nzeta > 0)
+                pco->GetGeometryPsi(prad,k,j,i,n,g_psi_);
+              else
+                pco->GetGeometryPsi(prad,k,j,i,g_psi_);
+              for (int m=0; m<npsi*2+1; ++m) {
+                int ang_num = n*2*npsi+m;
+                Real g_coef = g_psi_(m);
+                if (g_coef > 0)
+                  psi_flux_(k,j,i,ifr,ang_num) =  -prad->reduced_c * qr_psi_(m+NGHOST)
+                                                  * g_coef;
+                else if (g_coef < 0)
+                  psi_flux_(k,j,i,ifr,ang_num) =  -prad->reduced_c * ql_psi_(m+NGHOST)
+                                                  * g_coef;
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+
+// calculate advective flux for the implicit scheme
+void RadIntegrator::CalculateFluxes(AthenaArray<Real> &ir, const int order) {
+  NRRadiation *prad=pmy_rad;
+  MeshBlock *pmb=prad->pmy_block;
+
+  // int nang=prad->nang;
+  // int nfreq=prad->nfreq;
+
+  // int ncells1 = pmb->ncells1;
+  int ncells2 = pmb->ncells2, ncells3 = pmb->ncells3;
+
+  AthenaArray<Real> &x1flux=prad->flux[X1DIR];
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+  int il, iu, jl, ju, kl, ku;
+  jl = js, ju=je, kl=ks, ku=ke;
+
+  if (ncells2 > 1)  {
+    if (ncells3 == 1) {
+      jl=js-1, ju=je+1, kl=ks, ku=ke;
+    } else {
+      jl=js-1, ju=je+1, kl=ks-1, ku=ke+1;
+    }
+  }
+  //--------------------------------------------------------------------------------------
+  // i-direction
+
+  // First, do reconstruction of the specific intensities
+  for (int k=kl; k<=ku; ++k) {
+    for (int j=jl; j<=ju; ++j) {
+      // reconstruct L/R states
+      if (order == 1) {
+        pmb->precon->DonorCellX1(k, j, is-1, ie+1, ir, -1, il_, ir_);
+      } else if (order == 2) {
+        pmb->precon->PiecewiseLinearX1(k, j, is-1, ie+1, ir, -1, il_, ir_);
+      } else {
+        pmb->precon->PiecewiseParabolicX1(k, j, is-1, ie+1, ir, -1, il_, ir_);
+      }
+
+      // calculate flux with velocity times the interface state
+      for (int i=is; i<=ie+1; ++i) {
+        Real &vel = adv_vel(0,k,j,i);
+        if (vel >0) {
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            x1flux(k,j,i,n) = vel * il_(i,n);
+          }
+        } else {
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            x1flux(k,j,i,n) = vel * ir_(i,n);
+          }
+        }
+      }
+    }
+  }
+
+  // j-direction
+  if (pmb->pmy_mesh->f2) {
+    AthenaArray<Real> &x2flux=prad->flux[X2DIR];
+    il = is-1, iu = ie+1, kl = ks, ku = ke;
+    if (ncells3 ==  1) // 2D
+      kl = ks, ku = ke;
+    else // 3D
+      kl = ks-1, ku = ke+1;
+
+    for (int k=kl; k<=ku; ++k) {
+      // reconstruct the first row
+      if (order == 1) {
+        pmb->precon->DonorCellX2(k, js-1, il, iu, ir, -1, il_, ir_);
+      } else if (order == 2) {
+        pmb->precon->PiecewiseLinearX2(k, js-1, il, iu, ir, -1, il_, ir_);
+      } else {
+        pmb->precon->PiecewiseParabolicX2(k, js-1, il, iu, ir, -1, il_, ir_);
+      }
+
+      for (int j=js; j<=je+1; ++j) {
+        if (order == 1) {
+          pmb->precon->DonorCellX2(k, j, il, iu, ir, -1, ilb_, ir_);
+        } else if (order == 2) {
+          pmb->precon->PiecewiseLinearX2(k, j, il, iu, ir, -1, ilb_, ir_);
+        } else {
+          pmb->precon->PiecewiseParabolicX2(k, j, il, iu, ir, -1, ilb_, ir_);
+        }
+
+        // calculate flux with velocity times the interface state
+        for (int i=il; i<=iu; ++i) {
+          Real &vel = adv_vel(1,k,j,i);
+          if (vel > 0) {
+            for (int n=0; n<prad->n_fre_ang; ++n) {
+              x2flux(k,j,i,n) = vel * il_(i,n);
+            }
+          } else {
+            for (int n=0; n<prad->n_fre_ang; ++n) {
+              x2flux(k,j,i,n) = vel * ir_(i,n);
+            }
+          }
+        }
+        //swap the array for the next step
+        il_.SwapAthenaArray(ilb_);
+      }
+    }
+  }
+
+  if (pmb->pmy_mesh->f3) {
+    AthenaArray<Real> &x3flux=prad->flux[X3DIR];
+
+    il =is-1, iu=ie+1, jl=js-1, ju=je+1;
+
+    // First, calculate the transport velocity
+    for (int j=jl; j<=ju; ++j) { // this loop ordering is intentional
+      // reconstruct the first row
+      if (order == 1) {
+        pmb->precon->DonorCellX3(ks-1, j, il, iu, ir, -1, il_, ir_);
+      } else if (order == 2) {
+        pmb->precon->PiecewiseLinearX3(ks-1, j, il, iu, ir, -1, il_, ir_);
+      } else {
+        pmb->precon->PiecewiseParabolicX3(ks-1, j, il, iu, ir, -1, il_, ir_);
+      }
+
+      for (int k=ks; k<=ke+1; ++k) {
+        // reconstruct L/R states at k
+        if (order == 1) {
+          pmb->precon->DonorCellX3(k, j, il, iu, ir, -1, ilb_, ir_);
+        } else if (order == 2) {
+          pmb->precon->PiecewiseLinearX3(k, j, il, iu, ir, -1, ilb_, ir_);
+        } else {
+          pmb->precon->PiecewiseParabolicX3(k, j, il, iu, ir, -1, ilb_, ir_);
+        }
+
+        // calculate flux with velocity times the interface state
+        for (int i=il; i<=iu; ++i) {
+          Real &vel = adv_vel(2,k,j,i);
+          if (vel > 0) {
+            for (int n=0; n<prad->n_fre_ang; ++n) {
+              x3flux(k,j,i,n) = vel * il_(i,n);
+            }
+          } else {
+            for (int n=0; n<prad->n_fre_ang; ++n) {
+              x3flux(k,j,i,n) = vel * ir_(i,n);
+            }
+          }
+        }
+
+        // swap the arrays for the next step
+        il_.SwapAthenaArray(ilb_);
+      }
+    }
+  }
+}
+
+
+// add flux divergence due to advection for the implicit scheme
+void RadIntegrator::FluxDivergence(const Real wght) {
+  NRRadiation *prad=pmy_rad;
+  MeshBlock *pmb=prad->pmy_block;
+  Coordinates *pco= pmb->pcoord;
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+  AthenaArray<Real> &x1area = x1face_area_, &x2area = x2face_area_,
+                 &x2area_p1 = x2face_area_p1_, &x3area = x3face_area_,
+                 &x3area_p1 = x3face_area_p1_, &vol = cell_volume_;
+
+  AthenaArray<Real> &x1flux=prad->flux[X1DIR];
+  AthenaArray<Real> &x2flux=prad->flux[X2DIR];
+  AthenaArray<Real> &x3flux=prad->flux[X3DIR];
+
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      pco->CellVolume(k,j,is,ie,vol);
+      pco->Face1Area(k,j,is,ie+1,x1area);
+      for (int i=is; i<=ie; ++i) {
+        Real *divn = &(adv_flx_(k,j,i,0));
+        Real *flxr = &(x1flux(k,j,i+1,0));
+        Real *flxl = &(x1flux(k,j,i,0));
+        Real areal = x1area(i);
+        Real arear = x1area(i+1);
+        for (int n=0; n<prad->n_fre_ang; ++n) {
+          divn[n] = -wght * (arear * flxr[n] - areal * flxl[n])/vol(i);
+        }
+      }
+    }
+  }
+  if (pmb->block_size.nx2 > 1) {
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        pco->CellVolume(k,j,is,ie,vol);
+        pco->Face2Area(k,j  ,is,ie,x2area   );
+        pco->Face2Area(k,j+1,is,ie,x2area_p1);
+        for (int i=is; i<=ie; ++i) {
+          Real *divn = &(adv_flx_(k,j,i,0));
+          Real *flxr = &(x2flux(k,j+1,i,0));
+          Real *flxl = &(x2flux(k,j,i,0));
+          Real areal = x2area(i);
+          Real arear = x2area_p1(i);
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            divn[n] += -wght * (arear * flxr[n] - areal * flxl[n])/vol(i);
+          }
+        }
+      }
+    }
+  }
+
+  if (pmb->block_size.nx3 > 1) {
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        pco->CellVolume(k,j,is,ie,vol);
+        pco->Face3Area(k,j  ,is,ie,x3area   );
+        pco->Face3Area(k+1,j,is,ie,x3area_p1);
+        for (int i=is; i<=ie; ++i) {
+          Real *divn = &(adv_flx_(k,j,i,0));
+          Real *flxr = &(x3flux(k+1,j,i,0));
+          Real *flxl = &(x3flux(k,j,i,0));
+          Real areal = x3area(i);
+          Real arear = x3area_p1(i);
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            divn[n] += -wght * (arear * flxr[n] - areal * flxl[n])/vol(i);
+          }
+        }
+      }
+    }
+  }
+}
+
+
+void RadIntegrator::FluxDivergence(const Real wght, AthenaArray<Real> &ir_in,
+                                   AthenaArray<Real> &ir_out) {
+  NRRadiation *prad=pmy_rad;
+  MeshBlock *pmb=prad->pmy_block;
+  int nfreq=prad->nfreq;
+  int nang=prad->nang;
+
+  AthenaArray<Real> &x1flux=prad->flux[X1DIR];
+  AthenaArray<Real> &x2flux=prad->flux[X2DIR];
+  AthenaArray<Real> &x3flux=prad->flux[X3DIR];
+
+  int is = pmb->is; int js = pmb->js; int ks = pmb->ks;
+  int ie = pmb->ie; int je = pmb->je; int ke = pmb->ke;
+
+  AthenaArray<Real> &x1area = x1face_area_, &x2area = x2face_area_,
+                 &x2area_p1 = x2face_area_p1_, &x3area = x3face_area_,
+                 &x3area_p1 = x3face_area_p1_, &vol = cell_volume_, &dflx = dflx_;
+
+  AthenaArray<Real> &area_zeta = zeta_area_, &area_psi = psi_area_,
+                      &ang_vol = ang_vol_, &dflx_ang = dflx_ang_;
+  int &nzeta = prad->nzeta, &npsi = prad->npsi;
+
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      // calculate x1-flux divergence
+      pmb->pcoord->Face1Area(k,j,is,ie+1,x1area);
+      for (int i=is; i<=ie; ++i) {
+        Real *flxr = &(x1flux(k,j,i+1,0));
+        Real *flxl = &(x1flux(k,j,i,0));
+        Real *flxn = &(dflx(i,0));
+        for (int n=0; n<prad->n_fre_ang; ++n) {
+          flxn[n] = (x1area(i+1) *flxr[n] - x1area(i)*flxl[n]);
+        }
+      }
+
+      // calculate x2-flux
+      if (pmb->block_size.nx2 > 1) {
+        pmb->pcoord->Face2Area(k,j  ,is,ie,x2area   );
+        pmb->pcoord->Face2Area(k,j+1,is,ie,x2area_p1);
+        for (int i=is; i<=ie; ++i) {
+          Real *flxr = &(x2flux(k,j+1,i,0));
+          Real *flxl = &(x2flux(k,j,i,0));
+          Real *flxn = &(dflx(i,0));
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            flxn[n] += (x2area_p1(i)*flxr[n] - x2area(i)*flxl[n]);
+          }
+        }
+      }
+
+      // calculate x3-flux divergence
+      if (pmb->block_size.nx3 > 1) {
+        pmb->pcoord->Face3Area(k  ,j,is,ie,x3area   );
+        pmb->pcoord->Face3Area(k+1,j,is,ie,x3area_p1);
+        for (int i=is; i<=ie; ++i) {
+          Real *flxr = &(x3flux(k+1,j,i,0));
+          Real *flxl = &(x3flux(k,j,i,0));
+          Real *flxn = &(dflx(i,0));
+          for (int n=0; n<prad->n_fre_ang; ++n) {
+            flxn[n] += (x3area_p1(i)*flxr[n] - x3area(i)*flxl[n]);
+          }
+        }
+      }
+      // update variable with flux divergence
+      pmb->pcoord->CellVolume(k,j,is,ie,vol);
+      for (int i=is; i<=ie; ++i) {
+        Real *irin = &(ir_in(k,j,i,0));
+        Real *iro = &(ir_out(k,j,i,0));
+        Real *flxn = &(dflx(i,0));
+        for (int n=0; n<prad->n_fre_ang; ++n) {
+          iro[n] = std::max(irin[n]-wght*flxn[n]/vol(i), static_cast<Real>(TINY_NUMBER));
+        }
+      }
+
+      // add angular flux
+      if ((prad->angle_flag == 1) && (imp_ang_flx_ == 0)) {
+        for (int i=is; i<=ie; ++i) {
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            for (int n=0; n<prad->nang; ++n)
+              dflx_ang(n) = 0.0;
+            if (nzeta * npsi > 0) {
+              for (int m=0; m<2*npsi; ++m) {
+//#pragma omp simd
+                for (int n=0; n<2*nzeta; ++n) {
+                  int ang_num = n*2*npsi + m;
+                  int ang_num1 = (n+1)*2*npsi+m;
+                  dflx_ang(ang_num) += (area_zeta(m,n+1) * zeta_flux_(k,j,i,ifr,ang_num1)
+                                        - area_zeta(m,n) * zeta_flux_(k,j,i,ifr,ang_num));
+                }
+              }
+              // now psi flux
+              for (int n=0; n<2*nzeta; ++n) {
+                Real *flxn = &(dflx_ang(n*2*npsi));
+                Real *areapsi = &(area_psi(n,0));
+                Real *psiflx = &(psi_flux_(k,j,i,ifr,n*2*npsi));
+                for (int m=0; m<2*npsi; ++m) {
+                  flxn[m] += (areapsi[m+1] * psiflx[m+1]
+                              - areapsi[m] * psiflx[m]);
+                }
+              }
+            } else if (nzeta >0) {
+              Real *flxn = &(dflx_ang(0));
+              Real *areazeta = &(area_zeta(0));
+              Real *zetaflx = &(zeta_flux_(k,j,i,ifr,0));
+              for (int n=0; n<2*nzeta; ++n) {
+                flxn[n] += (areazeta[n+1] * zetaflx[n+1]
+                            - areazeta[n] * zetaflx[n]);
+              }
+            } else if (npsi > 0) {
+              Real *flxn = &(dflx_ang(0));
+              Real *areapsi = &(area_psi(0));
+              Real *psiflx = &(psi_flux_(k,j,i,ifr,0));
+              for (int m=0; m<2*npsi; ++m) {
+                flxn[m] += (areapsi[m+1] * psiflx[m+1]
+                            - areapsi[m] * psiflx[m]);
+              }
+            }
+            // apply the flux divergence back
+            // We need ir_out, not ir_in, as ir_out is already partially updated
+            Real *iro = &(ir_out(k,j,i,ifr*nang));
+            Real *flxn = &(dflx_ang(0));
+            Real *angv = &(ang_vol(0));
+            for (int n=0; n<prad->nang; ++n) {
+              iro[n] = std::max(iro[n]-wght*flxn[n]/angv[n],
+                                static_cast<Real>(TINY_NUMBER));
+            }
+          }
+        }
+      }
+    }
+  }
+  if (prad->angle_flag == 1 && (imp_ang_flx_ == 1)) {
+    ImplicitAngularFluxesCoef(wght);
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          ImplicitAngularFluxes(k,j,i,ir_out);
+
+          for (int ifr=0; ifr<nfreq; ++ifr) {
+            Real *p_angflx  = &(ang_flx_(k,j,i,ifr*nang));
+            Real *iro = &(ir_out(k,j,i,ifr*nang));
+            Real *ang_coef = &(imp_ang_coef_(k,j,i,0));
+            for (int n=0; n<nang; ++n) {
+              iro[n] += p_angflx[n];
+              iro[n] /= (1.0 + ang_coef[n]);
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/src/nr_radiation/integrators/srcterms/absorption_scattering.cpp b/src/nr_radiation/integrators/srcterms/absorption_scattering.cpp
new file mode 100644
index 0000000000..05535e43a1
--- /dev/null
+++ b/src/nr_radiation/integrators/srcterms/absorption_scattering.cpp
@@ -0,0 +1,148 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file absorption_scattering.cpp
+//  \brief implementation of absorption-scattering source terms
+//======================================================================================
+
+
+// C headers
+
+// C++ headers
+#include <cmath>
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../coordinates/coordinates.hpp"
+#include "../../../eos/eos.hpp"
+#include "../../../hydro/hydro.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../utils/utils.hpp"
+#include "../../radiation.hpp"
+
+// this class header
+#include "../rad_integrators.hpp"
+
+//--------------------------------------------------------------------------------------
+//! \fn RadIntegrator::AbsorptionScattering()
+//  \brief
+
+// wmu_cm is the weight in the co-moving frame
+// wmu_cm=wmu * 1/(1-vdotn/Crat)^2 / Lorz^2
+// tran_coef is (1-vdotn/Crat)*Lorz
+// rho is gas density
+// tgas is gas temperature
+// This function updates normal absorption plus scattering opacity together
+
+Real RadIntegrator::AbsorptionScattering(
+    AthenaArray<Real> &wmu_cm, AthenaArray<Real> &tran_coef, Real *sigma_a,
+    Real *sigma_p, Real *sigma_pe, Real *sigma_s, Real dt, Real lorz,
+    Real rho, Real &tgas, AthenaArray<Real> &implicit_coef, AthenaArray<Real> &ir_cm) {
+  Real& prat = pmy_rad->prat;
+  Real ct = dt * pmy_rad->crat;
+  Real redfactor=pmy_rad->reduced_c/pmy_rad->crat;
+  int& nang=pmy_rad->nang;
+  int& nfreq=pmy_rad->nfreq;
+  Real gamma = pmy_rad->pmy_block->peos->GetGamma();
+
+  // Temporary array
+  //AthenaArray<Real> &vncsigma = vncsigma_;
+  AthenaArray<Real> &vncsigma2 = vncsigma2_;
+
+  int badcell=0;
+
+
+  Real coef[2];
+  for (int ci=0; ci<2; ++ci)
+    coef[ci] = 0.0;
+
+  Real tgasnew = tgas;
+
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    Real suma1=0.0, suma2=0.0, suma3=0.0;
+    Real jr_cm=0.0;
+
+    Real dtcsigmar = ct * sigma_a[ifr]; //Rosseland mean absorption
+    Real dtcsigmas = ct * sigma_s[ifr]; // scattering
+    Real dtcsigmap = ct * sigma_p[ifr];// planck mean absorption
+    Real dtcsigmae = ct * sigma_pe[ifr]; // Energy (planck) mean absorption
+
+    Real rdtcsigmar = redfactor * dtcsigmar;
+    Real rdtcsigmas = redfactor * dtcsigmas;
+    Real rdtcsigmap = redfactor * dtcsigmap;
+    Real rdtcsigmae = redfactor * dtcsigmae;
+
+
+    Real *ircm = &(ir_cm(nang*ifr));
+    Real *vn2 = &(vncsigma2(0));
+    Real *tcoef = &(tran_coef(0));
+    Real *wmu = &(wmu_cm(0));
+    Real *imcoef = &(implicit_coef(0));
+    for (int n=0; n<nang; n++) {
+      Real vn = 1.0/(imcoef[n] + (rdtcsigmar + rdtcsigmas) * tcoef[n]);
+      vn2[n] = tcoef[n] * vn;
+      Real ir_weight = ircm[n] * wmu[n];
+
+      suma1 += (wmu[n] * vn2[n]);
+      suma2 += (ir_weight * vn);
+    }
+    suma3 = suma1 * (rdtcsigmas + rdtcsigmar - rdtcsigmae);
+    suma1 *= (rdtcsigmap);
+
+    // Now solve the equation
+    // rho dT/gamma-1=-\gamma Prat c(sigma T^4 - sigma(a1 T^4 + a2)/(1-a3))
+
+
+    // No need to do this if already in thermal equilibrium
+    coef[1] = prat * (dtcsigmap - dtcsigmae * suma1/(1.0-suma3))
+              * (gamma - 1.0)/rho;
+    coef[0] = -tgas - dtcsigmae * prat * suma2 * (gamma - 1.0)/(rho*(1.0-suma3));
+
+    if (std::abs(coef[1]) > TINY_NUMBER) {
+      int flag = FouthPolyRoot(coef[1], coef[0], tgasnew);
+      if (flag == -1 || tgasnew != tgasnew) {
+        badcell = 1;
+        tgasnew = tgas;
+      }
+    } else {
+      tgasnew = -coef[0];
+    }
+    // even if tr=told, there can be change for intensity, making them isotropic
+    if (!badcell) {
+      Real emission = tgasnew * tgasnew * tgasnew * tgasnew;
+
+      // get update jr_cm
+      jr_cm = (suma1 * emission + suma2)/(1.0-suma3);
+
+      // Update the co-moving frame specific intensity
+      Real *irn = &(ir_cm(nang*ifr));
+      Real *vn2 = &(vncsigma2(0));
+      Real *imcoef = &(implicit_coef(nang*ifr));
+      Real *tcoef = &(tran_coef(0));
+      for (int n=0; n<nang; n++) {
+        irn[n] +=  ((rdtcsigmas + rdtcsigmar - rdtcsigmae) * jr_cm + rdtcsigmap * emission
+                    - ((imcoef[n]-1.0)/tcoef[n] + rdtcsigmas
+                       + rdtcsigmar) * irn[n]) * vn2[n];
+        irn[n] = std::max(irn[n],static_cast<Real>(TINY_NUMBER));
+      }
+    }
+  }
+
+  // Update gas temperature
+  // Do not update gas temperature
+  //  tgas = tgasnew;
+  return tgasnew;
+}
diff --git a/src/nr_radiation/integrators/srcterms/compton.cpp b/src/nr_radiation/integrators/srcterms/compton.cpp
new file mode 100644
index 0000000000..6d47551b5b
--- /dev/null
+++ b/src/nr_radiation/integrators/srcterms/compton.cpp
@@ -0,0 +1,449 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file compton.cpp
+//  \brief  Add compton source terms
+//======================================================================================
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../coordinates/coordinates.hpp"
+#include "../../../eos/eos.hpp"
+#include "../../../hydro/hydro.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../utils/utils.hpp"
+#include "../../radiation.hpp"
+
+// this class header
+#include "../rad_integrators.hpp"
+
+//--------------------------------------------------------------------------------------
+//! \fn RadIntegrator::Compton()
+//  \brief
+
+// wmu_cm is the weight in the co-moving frame
+// wmu_cm=wmu * 1/(1-vdotn/Crat)^2 / Lorz^2
+// tran_coef is (1-vdotn/Crat)*Lorz
+// rho is gas density
+// tgas is gas temperature
+// This function only update the absorption term in each cell
+
+void RadIntegrator::Compton(AthenaArray<Real> &wmu_cm,
+                            AthenaArray<Real> &tran_coef, Real *sigma_s,
+                            Real dt, Real lorz, Real rho, Real &tgas,
+                            AthenaArray<Real> &ir_cm) {
+  Real& prat = pmy_rad->prat;
+  Real ct = dt * pmy_rad->crat;
+  Real redfactor=pmy_rad->reduced_c/pmy_rad->crat;
+
+  int& nang=pmy_rad->nang;
+  int& nfreq=pmy_rad->nfreq;
+  Real gamma = pmy_rad->pmy_block->peos->GetGamma();
+  Real telectron = 1.0/pmy_rad->telectron;
+
+  // Polynomial coefficients for Compton
+  Real coef[2];
+  for (int i=0; i<2; ++i)
+    coef[i] = 0.0;
+
+  Real tgasnew = tgas;
+
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    Real dtcsigma = ct * sigma_s[ifr];
+    Real rdtcsigma = redfactor * dtcsigma;
+
+    // Add Simple Compton scattering using the partically updated jr and ir
+    if (rdtcsigma > TINY_NUMBER) {
+     // Calculate the sum \int gamma (1-vdotn/c) dw_0 4 dt csigma_s /T_e
+      Real suma1 = 0.0, suma2 = 0.0, jr_cm=0.0, source=0.0;
+      Real *irn = &(ir_cm(nang*ifr));
+      Real *wmun = &(wmu_cm(0));
+      Real *tcoef = &(tran_coef(0));
+      for (int n=0; n<nang; n++) {
+         jr_cm += irn[n] * wmun[n];
+         suma1 += tcoef[n] * wmun[n] * 4.0 * rdtcsigma * telectron;
+      }
+      suma2 = 4.0 * lorz * prat * dtcsigma*(gamma-1.0)*telectron/rho;
+
+      Real tr = std::sqrt(std::sqrt(jr_cm));
+      Real trnew;
+
+      if (std::abs(tr - tgas) > 1.e-12) {
+        coef[1] = (1.0 + suma2* jr_cm)/(suma1 * jr_cm);
+        coef[0] = -(1.0+suma2*jr_cm)/suma1-tgas;
+        int flag = FouthPolyRoot(coef[1], coef[0], trnew);
+        if (flag == -1) {
+          trnew = tr;
+          tgasnew = tgas;
+          source = 0.0;
+        } else {
+          Real jrnew = trnew * trnew * trnew * trnew;
+          tgasnew = (jrnew - jr_cm)/(suma1*jr_cm) + trnew;
+          source = rdtcsigma * 4.0 * jr_cm * telectron * (tgasnew - trnew);
+        }
+      }
+
+      // Update the co-moving frame specific intensity
+      for (int n=0; n<nang; n++) {
+        irn[n] += source * tcoef[n];
+      }
+    }
+  }
+
+  // Update gas temperature
+  tgas = tgasnew;
+  return;
+}
+
+
+void RadIntegrator::MultiGroupCompton(
+    AthenaArray<Real> &wmu_cm, AthenaArray<Real> &tran_coef, Real dt, Real lorz, Real rho,
+    Real &tgas_ini, Real &tgas, AthenaArray<Real> &ir_cm) {
+  int bd_cell_flag = 0;
+
+  Real& prat = pmy_rad->prat;
+  Real ct = dt * pmy_rad->crat;
+  Real redfactor=pmy_rad->reduced_c/pmy_rad->crat;
+
+  int& nang=pmy_rad->nang;
+  int& nfreq=pmy_rad->nfreq;
+  Real gamma = pmy_rad->pmy_block->peos->GetGamma();
+  Real one_telectron = 1.0/pmy_rad->telectron;
+  //correction factor for high order terms of energy change per scattering
+  // f_theta is 1 when kT/m_ec^2 -->0
+  Real f_theta=ComptCorrection(tgas);
+
+  //Compton scattering coefficient always use the frequency
+  // independent kappa_es
+  // rho * kappa_es=sigma_es
+
+  //--------------------------------------------------------------------
+  // Estimate gas temperature based on integrated Kompaneets Equation
+
+  Real compt_coef = f_theta*lorz*ct*redfactor*rho*pmy_rad->kappa_es
+                    *one_telectron;
+  Real coef_a1=4*compt_coef;
+  Real coef_a2=(gamma -1.0)*prat*coef_a1/(redfactor*rho);
+
+  // now calculate integral of J
+  Real *j_nu = &(sum_nu1_(0));
+  Real *wmun = &(wmu_cm(0));
+  Real *nu_grid = &(pmy_rad->nu_grid(0));
+  Real *nu_cen = &(pmy_rad->nu_cen(0));
+  Real *delta_nu = &(pmy_rad->delta_nu(0));
+
+  //--------------------------------------------------------------------
+  // do not include the correction factor when estimate T
+  Real tgas_new = tgas;
+
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    j_nu[ifr] = 0.0;
+    Real *irn = &(ir_cm(nang*ifr));
+    for (int n=0; n<nang; ++n) {
+      j_nu[ifr] += wmun[n] * irn[n];
+    }
+  }
+  Real sum_jnu = 0.0;
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    sum_jnu += j_nu[ifr];
+  }
+  if (compton_t_ > 0) {
+    // calculate integral \int nu J_nu d\nu
+    Real sum_nu_jnu = 0.0;
+    for (int ifr=0; ifr<nfreq-1; ++ifr)
+      sum_nu_jnu += nu_cen[ifr] * j_nu[ifr];
+
+  //The last frequency bin is special
+  // determine the effective blackbody temperature
+  //from frequency nu_grid[0], nu_grid[nfreq-1]
+  // This is nu/Tr for the last bin
+
+//  Real nu_tr = pmy_rad->EffectiveBlackBody(j_nu[nfreq-1], nu_grid[nfreq-1]);
+//  Real eff_tr = nu_grid[nfreq-1]/nu_tr;
+
+    Real nu_jnu_last_bin = 0.0;
+    Real jnu_sq_last_bin = 0.0;
+
+
+    pmy_rad->ConvertBBJWien(j_nu[nfreq-1], nu_grid[nfreq-1], tgas,
+                         nu_jnu_last_bin, jnu_sq_last_bin);
+
+  // convert from j_nu to \int nu J_nu d\nu
+//  Real nu_jnu_last_bin = pmy_rad->BBJtoJnu(j_nu[nfreq-1],nu_grid[nfreq-1]);
+//  Real jnu_sq_last_bin = pmy_rad->BBJToJONuSq(j_nu[nfreq-1],nu_grid[nfreq-1]);
+
+    sum_nu_jnu += nu_jnu_last_bin;
+
+  // calculate the integral \int (J_nu/nu)^2 d\nu
+    Real sum_jonu_sq = 0.0;
+    for (int ifr=0; ifr<nfreq-1; ++ifr) {
+      sum_jonu_sq += j_nu[ifr] * j_nu[ifr]/(nu_cen[ifr] * nu_cen[ifr]
+                     * delta_nu[ifr]);
+    }
+  // the last bin
+    sum_jonu_sq += jnu_sq_last_bin;
+
+  // construct the coefficients
+    Real coef_r4 = sum_jnu + coef_a2 * sum_jnu * sum_jnu;
+    Real coef_r5 = 0.25*coef_a1*sum_nu_jnu+coef_a1*PI_FOUR_POWER*sum_jonu_sq/60.0;
+    Real coef_rhs = sum_jnu + coef_a1 * sum_jnu * tgas
+                  + coef_a2 * sum_jnu * sum_jnu;
+
+    Real r_ini = 1.0;
+    Real r_four_sol = coef_rhs/(coef_r4 + coef_r5 * r_ini);
+    r_four_sol = std::pow(r_four_sol,0.25);
+    int count=0;
+    while((count < 10) && (std::abs(r_ini-r_four_sol)/r_four_sol > 1.e-6)) {
+      r_ini = r_four_sol;
+      r_four_sol = coef_rhs/(coef_r4 + coef_r5 * r_ini);
+      r_four_sol = std::pow(r_four_sol,0.25);
+      count++;
+    }
+
+     tgas_new = tgas + ((gamma-1.0)*prat/(redfactor*rho)) * sum_jnu *
+                 (1.0 - r_four_sol * r_four_sol * r_four_sol * r_four_sol);
+  }
+  //--------------------------------------------------------------------
+  Real n_nusq_last_bin = 0.0;
+  Real nf_last = 0.0;
+// update
+  pmy_rad->ConvertBBJWien2(j_nu[nfreq-1], nu_grid[nfreq-1], tgas_new,
+                          n_nusq_last_bin,  nf_last);
+
+
+  //-------------------------------------------------------------------------
+  // now solve the Kompaneets equation
+
+  // frequency centers are: [0] [1] [2]....| [f-2] | [f-1]
+  // frequency faces are: [0] [1] [2]... [f-2]   [f-1]
+  // do not calculate flux for the last face
+
+  // calculate the photon number density
+  // n_nu = (pi^4/15) J_nu/nu^3
+  Real *n_nu = &(sum_nu2_(0));
+  for (int ifr=0; ifr<nfreq-1; ++ifr)
+    n_nu[ifr] = (PI_FOUR_POWER/15.0) * j_nu[ifr]
+               /(nu_cen[ifr]*nu_cen[ifr]*nu_cen[ifr]*delta_nu[ifr]);
+
+  // get the conserved total number density n \nu^2 d\nu
+  // if we assume blackbody spectrum, \int n \nu^2 d\nu
+  // = T^3 \int x^2/(exp(x)-1) dx
+//  Real n_nusq_last_bin = pmy_rad->ConvertBBJNNu2(j_nu[nfreq-1],nu_grid[nfreq-1]);
+
+  Real sum_n_nusq = n_nusq_last_bin;
+  for (int ifr=0; ifr<nfreq-1; ++ifr)
+    sum_n_nusq += n_nu[ifr] * nu_cen[ifr] * nu_cen[ifr] * delta_nu[ifr];
+
+  // determine the quasi-equilibrium solution
+  Real eq_sol_c  = QuasiEqSol(tgas_new, sum_n_nusq);
+
+  // now calculate the quasi-equilibrium solution at the frequency center
+  // equalibrium sol: n_nu = 1/(c exp(nu/T)-1);
+
+  Real *eq_sol = &(eq_sol_(0));
+  for (int ifr=0; ifr<nfreq-1; ++ifr) {
+    eq_sol[ifr] = 1.0/(eq_sol_c * std::exp(nu_cen[ifr]/tgas_new) - 1);
+  }
+
+  // now determine the delta_coefficient at the frequency interface
+  Real *delta_coef = &(sum_nu3_(0));
+  // use the weight based on frequency grid
+  delta_coef[0] = 0.5;
+  // for nearly 0 photon case
+  // assuming linear slope
+  if (eq_sol_c > 1.e16) {
+    for (int ifr=1; ifr<nfreq-1; ++ifr) {
+      delta_coef[ifr] = (nu_cen[ifr]-nu_grid[ifr])/(nu_cen[ifr]-nu_cen[ifr-1]);
+    }
+  } else {
+    for (int ifr=1; ifr<nfreq-1; ++ifr) {
+      Real dndnu = (eq_sol[ifr]-eq_sol[ifr-1])
+                 /(nu_cen[ifr]-nu_cen[ifr-1]);
+      Real eq_n_coef = 1-4.0*tgas_new*dndnu;
+      Real eq_n_face = 0.5*(std::sqrt(eq_n_coef) - 1.0);
+      delta_coef[ifr] = std::max(Real(0.0),(eq_n_face - eq_sol[ifr])/(
+          eq_sol[ifr-1]-eq_sol[ifr]));
+    }
+  }
+  delta_coef[nfreq-1] = 0.5;
+
+  //-------------------------------------------------------------------------
+  // construct the matrix coefficients
+  Real *com_b_face_coef = &(com_b_face_coef_(0));
+  Real *com_d_face_coef = &(com_d_face_coef_(0));
+  Real *com_b_coef_l = &(com_b_coef_l_(0));
+  Real *com_b_coef_r = &(com_b_coef_r_(0));
+  Real *com_d_coef_l = &(com_d_coef_l_(0));
+  Real *com_d_coef_r = &(com_d_coef_r_(0));
+
+  // the conservative equation to solve is
+  // dn_center/dt * \nu_center^2 d\nu
+  // = (Flux_rface - Flux_lface)
+
+  com_b_face_coef[0] = 0.0;
+  com_d_face_coef[0] = 0.0;
+  // the face coefficient: comp_t_coef * nu_f^4 * (T/((d\nu))+(1+n)(1-\delta))
+  for (int ifr=1; ifr<nfreq-1; ++ifr) {
+    Real a_coef = compt_coef*nu_grid[ifr]*nu_grid[ifr]*nu_grid[ifr]*nu_grid[ifr];
+    Real n_face = (1-delta_coef[ifr]) * n_nu[ifr]+delta_coef[ifr]*n_nu[ifr-1];
+    Real tdnu_face=tgas_new/(nu_cen[ifr]-nu_cen[ifr-1]);
+    com_b_face_coef[ifr] = a_coef*(tdnu_face+(1.0+n_face)*(1.0-delta_coef[ifr]));
+    com_d_face_coef[ifr] = a_coef*((1+n_face)*delta_coef[ifr]-tdnu_face);
+  }
+
+  // for the first bin, the left boundary nu_grid[0]=0
+  // com_a_coef_l[0] is 0
+  // split the left and right hand side coefficients
+  for (int ifr=0; ifr<nfreq-2; ++ifr) {
+    Real bottom = 1.0/(nu_cen[ifr]*nu_cen[ifr]*delta_nu[ifr]);
+    com_b_coef_r[ifr]=com_b_face_coef[ifr+1]*bottom;
+    com_d_coef_r[ifr]=com_d_face_coef[ifr+1]*bottom;
+  }
+
+  for (int ifr=0; ifr<nfreq-1; ++ifr) {
+    Real bottom = 1.0/(nu_cen[ifr]*nu_cen[ifr]*delta_nu[ifr]);
+    com_b_coef_l[ifr]=com_b_face_coef[ifr]*bottom;
+    com_d_coef_l[ifr]=com_d_face_coef[ifr]*bottom;
+  }
+
+  // also calculate left hand side coefficients for nfreq-2
+
+  // now we invert the matrix
+  // -B_r * n_f+1 + (1 - D_r + B_l) * n_f + D_l * n_f-1 = n_f^old
+  // now solve the matrix from the left boundary
+  // relate the right variable with the left variable
+  // using Gauss elimination
+
+  // each variable at nf is related to nf+1 as
+  // n_f =  nf_n0 * n_{f+1} + nf_rhs
+
+  Real *nf_rhs = &(nf_rhs_(0));
+  Real *nf_n0 = &(nf_n0_(0));
+
+  nf_rhs[0] = n_nu[0]/(1.0-com_d_coef_r[0]+com_b_coef_l[0]);
+  nf_n0[0] = com_b_coef_r[0]/(1.0-com_d_coef_r[0]+com_b_coef_l[0]);
+
+  for (int ifr=1; ifr<nfreq-2; ++ifr) {
+    //eliminate n_{f-1}, get a relation between n_f and n_{f+1}
+    // n_{f-1} = nf_n0[f-1] * n_{f} + nf_rhs[f-1]
+    Real ef = (1.0-com_d_coef_r[ifr]+com_b_coef_l[ifr]);
+    Real new_ef = 1.0 + com_d_coef_l[ifr] * nf_n0[ifr-1]/ef;
+    nf_rhs[ifr] = n_nu[ifr]/ef-com_d_coef_l[ifr] * nf_rhs[ifr-1]/ef;
+    if (std::abs(new_ef) < TINY_NUMBER) {
+      bd_cell_flag = 1;
+      break;
+    }
+    nf_rhs[ifr] /= new_ef;
+    nf_n0[ifr] = com_b_coef_r[ifr]/(ef*new_ef);
+  }
+
+  // for the flux at the last face, we calculate the flux explicitly
+  //  Real nf_last=1.0/(\lambda exp(nu_tr)-1.0);
+  Real a_coef_last = compt_coef*nu_grid[nfreq-1]*nu_grid[nfreq-1]
+                     *nu_grid[nfreq-1]*nu_grid[nfreq-1]
+                     /(nu_cen[nfreq-2]*nu_cen[nfreq-2]*delta_nu[nfreq-2]);
+
+  Real flux_last_explicit = 0.0;
+  Real flux_last_im_coef = 0.0;
+
+  if (nf_last < n_nu[nfreq-2]) {
+    flux_last_explicit=a_coef_last*(nf_last*(1.0+nf_last)
+                                    +tgas_new*nf_last*2.0/delta_nu[nfreq-2]);
+    flux_last_im_coef=a_coef_last*tgas_new*2.0/delta_nu[nfreq-2];
+  }
+
+  // We use the bin nfreq-2 to get the solution
+  Real n0_coef = (1.0+com_b_coef_l[nfreq-2]+flux_last_im_coef)
+                 +com_d_coef_l[nfreq-2]*nf_n0[nfreq-3];
+  Real rhs_coef = n_nu[nfreq-2]+flux_last_explicit
+                  -com_d_coef_l[nfreq-2]*nf_rhs[nfreq-3];
+
+  if (std::abs(n0_coef) < TINY_NUMBER) {
+    bd_cell_flag = 1;
+    return;
+  }
+
+  // update new solution at nfreq-2
+  n_nu[nfreq-2] = rhs_coef/n0_coef;
+  for (int ifr=nfreq-3; ifr>=0; --ifr) {
+    n_nu[ifr] = nf_n0[ifr] * n_nu[ifr+1] + nf_rhs[ifr];
+  }
+
+  Real flux_last = flux_last_explicit - flux_last_im_coef * n_nu[nfreq-2];
+  // update n_nusq in the last bin
+  n_nusq_last_bin -= flux_last *
+                     (nu_cen[nfreq-2]*nu_cen[nfreq-2]*delta_nu[nfreq-2]);
+
+  n_nusq_last_bin = std::max(n_nusq_last_bin,static_cast<Real>(TINY_NUMBER));
+
+  //-------------------------------------------------------------------------
+  // now go from update n_nu to new_j_nu
+  if (bd_cell_flag == 0) {
+    Real *new_j_nu = &(new_j_nu_(0));
+    for (int ifr=0; ifr<nfreq-1; ++ifr) {
+      new_j_nu[ifr] = 15.0*ONE_PI_FOUR_POWER*n_nu[ifr]*nu_cen[ifr]
+                      *nu_cen[ifr]*nu_cen[ifr]*delta_nu[ifr];
+    }
+    new_j_nu[nfreq-1] = pmy_rad->InverseConvertBBJNNu2Wien(n_nusq_last_bin,
+                                                           nu_grid[nfreq-1], tgas_new);
+    // now calculate the total j
+    Real sum_new_jnu = 0.0;
+    sum_new_jnu = new_j_nu[nfreq-1];
+    for (int ifr=0; ifr<nfreq-1; ++ifr)
+      sum_new_jnu += new_j_nu[ifr];
+
+    // now apply energy conservation
+
+    // Update intensity in each frequency bin
+    // rho * tgas/(gamma-1) + (prat/redfactor)* sum_jnu =
+    // rho * tgas_new/(gamma-1) + (prat/redfactor) * sum_new_jnu
+
+    // now update tgas_new via energy conservation
+    Real tgas_test = (prat/redfactor)*(sum_jnu-sum_new_jnu)
+                     *((gamma-1.0)/rho) + tgas_ini;
+    if (tgas_test < TINY_NUMBER)
+      tgas_test = tgas_new;
+    tgas = tgas_test;
+    // now update intensity isotropically
+    Real *tcoef = &(tran_coef(0));
+    for (int ifr=0; ifr<nfreq; ++ifr) {
+      Real *irn = &(ir_cm(nang*ifr));
+      for (int n=0; n<nang; n++) {
+        irn[n] += tcoef[n]* (new_j_nu[ifr] - j_nu[ifr]);
+        irn[n] = std::max(irn[n],static_cast<Real>(TINY_NUMBER));
+      }
+    }
+  }
+}
+
+Real RadIntegrator::QuasiEqSol(Real &tgas, Real &tot_n) {
+  Real eq_sol_c = 1.0;
+  Real n_t3 = tot_n/(tgas*tgas*tgas);
+  if ((n_t3 < 2.3739) && (n_t3 > 0.6932)) {
+    eq_sol_c = 1.948 * std::pow(n_t3, -1.016) + 0.1907;
+  } else if (n_t3 <= 0.6932) {
+    eq_sol_c = (1.0 + std::sqrt(1.0 + 0.25 * n_t3))/n_t3;
+  }
+  return eq_sol_c;
+}
+
+Real RadIntegrator::ComptCorrection(Real &tgas) {
+  // this is kT/m_ec^2
+  //Real theta=tgas/pmy_rad->telectron;
+  //  Real f_theta=(1.0+3.683*theta+4.0*theta*theta)/(1.0+theta);
+  Real f_theta = 1.0;
+  return f_theta;
+}
diff --git a/src/nr_radiation/integrators/srcterms/multigroup_abs_sca.cpp b/src/nr_radiation/integrators/srcterms/multigroup_abs_sca.cpp
new file mode 100644
index 0000000000..2947b13774
--- /dev/null
+++ b/src/nr_radiation/integrators/srcterms/multigroup_abs_sca.cpp
@@ -0,0 +1,191 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file multigroup_abs_sca.cpp
+//  \brief  Add multi-group absorption scattering
+//======================================================================================
+
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../../../athena.hpp"
+#include "../../../athena_arrays.hpp"
+#include "../../../coordinates/coordinates.hpp"
+#include "../../../eos/eos.hpp"
+#include "../../../hydro/hydro.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../../utils/utils.hpp"
+#include "../../radiation.hpp"
+// this class header
+#include "../rad_integrators.hpp"
+
+//--------------------------------------------------------------------------------------
+//! \fn RadIntegrator::MultiGroupAbsScat()
+//  \brief
+
+// wmu_cm is the weight in the co-moving frame
+// wmu_cm=wmu * 1/(1-vdotn/Crat)^2 / Lorz^2
+// tran_coef is (1-vdotn/Crat)*Lorz
+// rho is gas density
+// tgas is gas temperature
+// This function updates normal absorption plus scattering opacity together
+
+// ir_cm here is the frequency shifted array
+// emission in each frequency bin is assumed to b T^4 emission_spec[ifr]
+
+Real RadIntegrator::MultiGroupAbsScat(
+    AthenaArray<Real> &wmu_cm, AthenaArray<Real> &tran_coef, Real *sigma_a, Real *sigma_p,
+    Real *sigma_pe, Real *sigma_s, Real dt, Real lorz, Real rho, Real &tgas,
+    AthenaArray<Real> &implicit_coef, AthenaArray<Real> &ir_cm) {
+  Real& prat = pmy_rad->prat;
+  Real ct = dt * pmy_rad->crat;
+  Real redfactor=pmy_rad->reduced_c/pmy_rad->crat;
+  int& nang=pmy_rad->nang;
+  int& nfreq=pmy_rad->nfreq;
+  Real gamma = pmy_rad->pmy_block->peos->GetGamma();
+
+  // Temporary array
+  AthenaArray<Real> &vncsigma2 = vncsigma2_;
+
+  bool badcell=false;
+  Real coef[2];
+
+  Real *suma1 = &(sum_nu1_(0));
+  Real *suma2 = &(sum_nu2_(0));
+  Real *suma3 = &(sum_nu3_(0));
+
+  Real tgasnew = tgas;
+  Real tgas_last = tgas;
+
+  int count_iteration = 0;
+
+  // first, calculate the coefficients that are independnet of emission_spec
+  coef[0] = 0.0;
+  for (int ifr=0; ifr<nfreq; ++ifr) {
+    suma1[ifr]=0.0;
+    suma2[ifr]=0.0;
+    suma3[ifr]=0.0;
+
+    Real dtcsigmar = ct * sigma_a[ifr];
+    Real dtcsigmas = ct * sigma_s[ifr];
+    Real dtcsigmae = ct * sigma_pe[ifr];
+    Real dtcsigmap = ct * sigma_p[ifr];
+
+    Real rdtcsigmar = redfactor * dtcsigmar;
+    Real rdtcsigmas = redfactor * dtcsigmas;
+    Real rdtcsigmae = redfactor * dtcsigmae;
+    Real rdtcsigmap = redfactor * dtcsigmap;
+
+    Real *ircm = &(ir_cm(ifr*nang));
+    Real *vn2 = &(vncsigma2(ifr*nang));
+    Real *tcoef = &(tran_coef(0));
+    Real *wmu = &(wmu_cm(0));
+    Real *imcoef = &(implicit_coef(ifr*nang));
+    for (int n=0; n<nang; n++) {
+      Real vn = 1.0/(imcoef[n] + (rdtcsigmar + rdtcsigmas) * tcoef[n]);
+      vn2[n] = tcoef[n] * vn;
+      Real ir_weight = ircm[n] * wmu[n];
+      suma1[ifr] += (wmu[n] * vn2[n]);
+      suma2[ifr] += (ir_weight * vn);
+    }
+    suma3[ifr] = suma1[ifr] * (rdtcsigmas + rdtcsigmar - rdtcsigmae);
+    suma1[ifr] *= (rdtcsigmap);
+
+    coef[0] += - dtcsigmae * prat * suma2[ifr]
+               * (gamma - 1.0)/(rho*(1.0-suma3[ifr]));
+  }
+  coef[0] += -tgas;
+
+  Real tgas_diff = 1.0;
+
+  while ((count_iteration <= iteration_tgas_) && (tgas_diff > tgas_error_)) {
+    coef[1] = 0.0;
+
+    // update emission spectrum
+    pmy_rad->UserEmissionSpec(pmy_rad,tgasnew);
+
+    for (int ifr=0; ifr<nfreq; ++ifr) {
+      // Real dtcsigmar = ct * sigma_a[ifr];
+      Real dtcsigmae = ct * sigma_pe[ifr];
+      //      Real dtcsigmas = ct * sigma_s[ifr];
+      // Real rdtcsigmar = redfactor * dtcsigmar;
+      // Real rdtcsigmae = redfactor * dtcsigmae;
+      //      Real rdtcsigmas = redfactor * dtcsigmas;
+      Real dtcsigmap = ct * sigma_p[ifr];
+      // Real rdtcsigmap = redfactor * dtcsigmap;
+
+      // No need to do this if already in thermal equilibrium
+      coef[1] += prat * (dtcsigmap - dtcsigmae * suma1[ifr]/(1.0-suma3[ifr]))
+                 * pmy_rad->emission_spec(ifr) * (gamma - 1.0)/rho;
+    }
+
+    // The polynomial is
+    // coef[1] * x^4 + x + coef[0] == 0
+
+    if (std::abs(coef[1]) > TINY_NUMBER) {
+      int flag = FouthPolyRoot(coef[1], coef[0], tgasnew);
+      if (flag == -1 || tgasnew != tgasnew) {
+        badcell = true;
+        tgasnew = tgas;
+      }
+    } else {
+      tgasnew = -coef[0];
+    }
+
+    if (badcell) break;
+    tgas_diff = std::abs(tgas_last - tgasnew)/tgas_last;
+    count_iteration++;
+    tgas_last = tgasnew;
+  }
+  // even if tr=told, there can be change for intensity, making them isotropic
+  if (!badcell) {
+    Real emission = tgasnew * tgasnew * tgasnew * tgasnew;
+    for (int ifr=0; ifr<nfreq; ++ifr) {
+      Real dtcsigmar = ct * sigma_a[ifr];
+      Real dtcsigmae = ct * sigma_pe[ifr];
+      Real dtcsigmas = ct * sigma_s[ifr];
+      Real dtcsigmap = ct * sigma_p[ifr];
+
+      Real rdtcsigmar = redfactor * dtcsigmar;
+      Real rdtcsigmae = redfactor * dtcsigmae;
+      Real rdtcsigmas = redfactor * dtcsigmas;
+      Real rdtcsigmap = redfactor * dtcsigmap;
+
+      Real emi_nu = emission * pmy_rad->emission_spec(ifr);
+
+      // get update jr_cm
+      Real jr_cm = (suma1[ifr] * emi_nu + suma2[ifr])/(1.0-suma3[ifr]);
+
+      // Update the co-moving frame specific intensity
+      Real *irn = &(ir_cm(ifr*nang));
+      Real *vn2 = &(vncsigma2(ifr*nang));
+      Real *imcoef = &(implicit_coef(nang*ifr));
+      Real *tcoef = &(tran_coef(0));
+      for (int n=0; n<nang; n++) {
+        irn[n] +=  ((rdtcsigmas + rdtcsigmar - rdtcsigmae) * jr_cm + rdtcsigmap * emi_nu
+                    - ((imcoef[n]-1.0)/tcoef[n] + rdtcsigmas
+                       + rdtcsigmar) * irn[n]) * vn2[n];
+        irn[n] = std::max(irn[n], static_cast<Real>(TINY_NUMBER));
+      }
+    }
+  }
+  // Update gas temperature
+  // Do not update gas temperature
+  //  tgas = tgasnew;
+  return tgasnew;
+}
diff --git a/src/nr_radiation/radiation.cpp b/src/nr_radiation/radiation.cpp
new file mode 100644
index 0000000000..9c89b2b23a
--- /dev/null
+++ b/src/nr_radiation/radiation.cpp
@@ -0,0 +1,423 @@
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+//
+// This program is free software: you can redistribute and/or modify it under the terms
+// of the GNU General Public License (GPL) as published by the Free Software Foundation,
+// either version 3 of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+//
+// You should have received a copy of GNU GPL in the file LICENSE included in the code
+// distribution.  If not see <http://www.gnu.org/licenses/>.
+//======================================================================================
+//! \file radiation.cpp
+//  \brief implementation of functions in class Radiation
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <cstdio>  // fopen and fwrite
+#include <iostream>  // cout
+#include <sstream>  // msg
+#include <stdexcept> // runtime erro
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
+#include "../coordinates/coordinates.hpp"
+#include "../globals.hpp"
+#include "../mesh/mesh.hpp"
+#include "../parameter_input.hpp"
+#include "implicit/radiation_implicit.hpp"
+#include "integrators/rad_integrators.hpp"
+#include "radiation.hpp"
+// constructor, initializes data structures and parameters
+
+// The default opacity function.
+// Do nothing. Keep the opacity as the initial value
+inline void DefaultFrequency(NRRadiation *prad) {
+  return;
+}
+
+// The default opacity function.
+// Do nothing. Keep the opacity as the initial value
+inline void DefaultEmission(NRRadiation *prad, Real tgas) {
+  int &nfreq = prad->nfreq;
+  if (nfreq > 1) {
+    for(int ifr=0; ifr<nfreq-1; ++ifr) {
+      prad->emission_spec(ifr) =
+          prad->IntPlanckFunc(prad->nu_grid(ifr)/tgas, prad->nu_grid(ifr+1)/tgas);
+    }
+    prad->emission_spec(nfreq-1) = 1.0 - prad->FitIntPlanckFunc(
+        prad->nu_grid(nfreq-1)/tgas);
+  }
+  return;
+}
+
+
+// The default opacity function.
+// Do nothing. Keep the opacity as the initial value
+inline void DefaultOpacity(MeshBlock *pmb, AthenaArray<Real> &prim) {
+  return;
+}
+
+
+NRRadiation::NRRadiation(MeshBlock *pmb, ParameterInput *pin):
+    pmy_block(pmb), ir(pmb->ncells3,pmb->ncells2,pmb->ncells1,pmb->nfre_ang),
+    ir1(pmb->ncells3,pmb->ncells2,pmb->ncells1,pmb->nfre_ang),
+    // constructor overload resolution of non-aggregate class type AthenaArray<Real>
+    flux{ {pmb->ncells3, pmb->ncells2, pmb->ncells1+1, pmb->nfre_ang},
+          {pmb->ncells3, pmb->ncells2+1, pmb->ncells1, pmb->nfre_ang,
+           (pmb->pmy_mesh->f2 ? AthenaArray<Real>::DataStatus::allocated :
+            AthenaArray<Real>::DataStatus::empty)},
+          {pmb->ncells3+1, pmb->ncells2, pmb->ncells1, pmb->nfre_ang,
+           (pmb->pmy_mesh->f3 ? AthenaArray<Real>::DataStatus::allocated :
+            AthenaArray<Real>::DataStatus::empty)}
+    },
+    coarse_ir_(pmb->ncc3, pmb->ncc2, pmb->ncc1,pmb->nfre_ang,
+               (pmb->pmy_mesh->multilevel ? AthenaArray<Real>::DataStatus::allocated :
+                AthenaArray<Real>::DataStatus::empty)),
+    rad_bvar(pmb, &ir, &coarse_ir_, flux) {
+  // universal constants we need
+  // https://physics.info/constants/
+  // arad = 4 * sigma/c
+  Real arad = 7.565733250033928e-15;
+  Real c_speed = 2.99792458e10;  // speed of light
+
+  // read in the parameters
+  int nmu = pin->GetInteger("radiation","nmu");
+  // total number of polar angles covering 0 to pi/2
+  nzeta = pin->GetOrAddInteger("radiation","nzeta",0);
+  // total number of azimuthal angles covering 0 to pi
+  npsi = pin->GetOrAddInteger("radiation","npsi",0);
+  angle_flag = pin->GetOrAddInteger("radiation","angle_flag",0);
+
+  rotate_theta=pin->GetOrAddInteger("radiation","rotate_theta",0);
+  rotate_phi=pin->GetOrAddInteger("radiation","rotate_phi",0);
+
+  vmax = pin->GetOrAddReal("radiation","vmax",0.9);
+  user_unit_ = pin->GetOrAddInteger("radiation","unit",0);
+  tunit = pin->GetOrAddReal("radiation","T_unit",1.e7);
+  rhounit = pin->GetOrAddReal("radiation","density_unit",1.0);
+  lunit = pin->GetOrAddReal("radiation","length_unit",1.0);
+  mol_weight = pin->GetOrAddReal("radiation","molecular_weight",0.6);
+  kappa_es = pin->GetOrAddReal("radiation","electron_scattering",0.34);
+
+  kappa_es = kappa_es * rhounit * lunit; // dimensionless electron scattering
+
+  if (user_unit_ == 0) {
+    prat = pin->GetReal("radiation","prat");
+    crat = pin->GetReal("radiation","crat");
+  } else if (user_unit_ == 1) {
+    // calculate prat and crat based on user provided unit
+    Real r_ideal = 8.314462618e7/mol_weight;
+    prat = arad * tunit * tunit * tunit/(rhounit * r_ideal);
+    Real cs_iso = std::sqrt(r_ideal * tunit);
+    crat = c_speed/cs_iso;
+  }
+
+  // equivalent temperature for electron
+  telectron = 5.94065e9;
+  telectron /= tunit;
+
+  reduced_c  = crat * pin->GetOrAddReal("radiation","reduced_factor",1.0);
+
+  Real taucell = pin->GetOrAddReal("radiation","taucell",5);
+
+  //Mesh *pm = pmb->pmy_mesh;
+  //ir_output=pin->GetOrAddInteger("radiation","ir_output",0);
+
+  set_source_flag = pin->GetOrAddInteger("radiation","source_flag",1);
+
+  // number of cells for three dimensions
+  int nc1 = pmb->ncells1, nc2 = pmb->ncells2, nc3 = pmb->ncells3;
+  // calculate noct based on dimension
+  int ndim = 1;
+  if (nc2 > 1) ndim = 2;
+  if (nc3 > 1) ndim = 3;
+
+  int n_ang=0; // number of angles per octant and number of octant
+  // total calculate total number of angles based on dimensions
+  if (angle_flag == 1) {
+    if (ndim == 1) {
+      noct = 2;
+      n_ang = nzeta;
+      if (npsi > 0) {
+        std::stringstream msg;
+        msg << "### FATAL ERROR in radiation class" << std::endl
+            << "1D problem cannot have npsi > 0"   << std::endl;
+        ATHENA_ERROR(msg);
+      }
+    } else if (ndim == 2) {
+      if (npsi <= 1) {
+        n_ang = nzeta;
+      } else if (nzeta == 0) {
+        n_ang = npsi/2;
+      } else {
+        n_ang = nzeta*npsi;
+      }
+      if (std::strcmp(COORDINATE_SYSTEM, "spherical_polar") == 0) {
+        noct = 8;
+        n_ang = nzeta*npsi/2;
+      } else {
+        noct = 4;
+      }
+    } else if (ndim == 3) {
+      n_ang = nzeta*npsi/2;
+      noct = 8;
+    }
+  } else {
+    if (ndim == 1) {
+      n_ang = nmu;
+      noct = 2;
+    } else if (ndim == 2) {
+      noct = 4;
+      if (angle_flag == 0) {
+        n_ang = nmu * (nmu + 1)/2;
+      } else if (angle_flag == 10) {
+        n_ang = nmu;
+      }
+    } else if (ndim == 3) {
+      noct = 8;
+      if (angle_flag == 0) {
+        n_ang = nmu * (nmu + 1)/2;
+      } else if (angle_flag == 10) {
+        n_ang = nmu * nmu/2;
+      }
+    }
+  }
+
+  nang = n_ang * noct;
+
+
+  // frequency grid
+  //frequency grid covers -infty to infty, default nfreq=1, means gray
+  // integrated over all frequency
+
+  // the number of frequeny bins
+  nfreq  = pin->GetOrAddInteger("radiation","n_frequency",1);
+  // when the emission spectrum depends on tgas, we perform
+
+  // minimum frequency
+  nu_min = pin->GetOrAddReal("radiation","frequency_min",0);
+
+  // maximum frequency
+  nu_max = pin->GetOrAddReal("radiation","frequency_max",1);
+
+  // log frequency space ratio
+  fre_ratio= pin->GetOrAddReal("radiation","frequency_ratio",1.0);
+
+  log_fre_ = pin->GetOrAddInteger("radiation","log_frequency_spacing",1);
+
+  restart_from_gray  = pin->GetOrAddInteger("radiation","res_gray",0);
+  // when the emission spectrum depends on tgas, we perform
+
+  // setup the frequency grid
+  FrequencyGrid();
+
+
+  UserFrequency = DefaultFrequency;
+  UserEmissionSpec = DefaultEmission;
+
+
+  n_fre_ang = nang * nfreq;
+  //co-moving frame frequency grid depends on angels
+
+  // do not add radiation to vars_cc, which needs to be done in different order for
+  // restriction/prolongation in AMR
+  //  pmb->RegisterMeshBlockData(ir);
+
+  // If user-requested time integrator is type 3S*, allocate additional memory registers
+  std::string integrator = pin->GetOrAddString("time", "integrator", "vl2");
+  if (integrator == "ssprk5_4" || STS_ENABLED) {
+    // future extension may add "int nregister" to Hydro class
+    ir2.NewAthenaArray(nc3, nc2, nc1, n_fre_ang);
+  }
+
+
+  ir_old.NewAthenaArray(nc3,nc2,nc1,n_fre_ang);
+
+  if (restart_from_gray) {
+    ir_gray.NewAthenaArray(nc3,nc2,nc1,nang);
+  }
+
+  // Do not add to cell-centered refinement, as
+  // radiation variables need to be done in different order
+  //  if (pm->multilevel) {
+  //    refinement_idx = pmy_block->pmr->AddToRefinement(&ir, &coarse_ir_);
+  //  }
+
+  rad_mom.NewAthenaArray(13,nc3,nc2,nc1);
+  rad_mom_cm.NewAthenaArray(4,nc3,nc2,nc1);
+  // dump the moments in each frequency groups
+  if (nfreq > 1) {
+    // moments in different frequency bins
+    rad_mom_nu.NewAthenaArray(13*nfreq,nc3,nc2,nc1);
+    rad_mom_cm_nu.NewAthenaArray(4*nfreq,nc3,nc2,nc1);
+  }
+
+  // the equation is
+  // (sigma_s+sigma_a)(J-I)  // Rosseland mean
+  // + sigma_p * a_rT^4 - sigma_pe * J // Planck mean
+
+  sigma_s.NewAthenaArray(nc3,nc2,nc1,nfreq);
+  sigma_a.NewAthenaArray(nc3,nc2,nc1,nfreq);
+  sigma_p.NewAthenaArray(nc3,nc2,nc1,nfreq);
+  sigma_pe.NewAthenaArray(nc3,nc2,nc1,nfreq);
+
+  t_floor_.NewAthenaArray(nc3,nc2,nc1);
+  t_ceiling_.NewAthenaArray(nc3,nc2,nc1);
+
+  output_sigma.NewAthenaArray(3*nfreq,nc3,nc2,nc1);
+
+
+  mu.NewAthenaArray(3,nc3,nc2,nc1,nang);
+  wmu.NewAthenaArray(nang);
+
+
+
+
+  if (angle_flag == 1) {
+    AngularGrid(angle_flag, nzeta, npsi);
+    if (nc2 > 1) {
+      cot_theta.NewAthenaArray(nc2);
+      for(int i=0; i<nc2; ++i)
+        cot_theta(i) = cos(pmb->pcoord->x2v(i))/sin(pmb->pcoord->x2v(i));
+    }
+  } else {
+    AngularGrid(angle_flag, nmu);
+  }
+
+  // set a default opacity function
+  UpdateOpacity = DefaultOpacity;
+
+
+  pradintegrator = new RadIntegrator(this, pin);
+
+  rad_bvar.bvar_index = pmb->pbval->bvars.size();
+  pmb->pbval->bvars.push_back(&rad_bvar);
+  // enroll radiation boundary value object
+  if (NR_RADIATION_ENABLED) {
+    pmb->pbval->bvars_main_int.push_back(&rad_bvar);
+  }
+
+
+  //------------------------------------------
+  // temporary arrays for multi-group moments
+  cosx_cm_.NewAthenaArray(nang);
+  cosy_cm_.NewAthenaArray(nang);
+  cosz_cm_.NewAthenaArray(nang);
+
+
+
+
+  //------------------------------------------
+
+  // set the default t_floor and t_ceiling
+  for(int k=0; k<nc3; ++k)
+    for(int j=0; j<nc2; ++j)
+      for(int i=0; i<nc1; ++i) {
+        t_floor_(k,j,i) = TINY_NUMBER;
+        t_ceiling_(k,j,i) = HUGE_NUMBER;
+      }
+
+  // dump the angular grid and radiation parameters in a file
+  if (Globals::my_rank ==0) {
+    FILE *pfile;
+    std::stringstream msg;
+    if ((pfile = fopen("Rad_angles.txt","w")) == NULL) {
+      msg << "### FATAL ERROR in Radiation Class" << std::endl
+          << "Output file Rad_angles.txt could not be opened";
+      throw std::runtime_error(msg.str().c_str());
+    }
+    // damp the angular grid in one cell
+
+    fprintf(pfile,"prat          %4.2e \n",prat);
+    fprintf(pfile,"crat          %4.2e \n",crat);
+    fprintf(pfile,"reduced_c     %4.2e \n",reduced_c);
+    fprintf(pfile,"Vmax          %4.2e \n",vmax);
+    fprintf(pfile,"Tunit         %4.2e \n",tunit);
+    fprintf(pfile,"Compt         %d  \n",pradintegrator->compton_flag_);
+    fprintf(pfile,"rotate_theta  %d  \n",rotate_theta);
+    fprintf(pfile,"rotate_phi    %d  \n",rotate_phi);
+    fprintf(pfile,"adv_flag:     %d  \n",pradintegrator->adv_flag_);
+    fprintf(pfile,"nzeta:        %d  \n",nzeta);
+    fprintf(pfile,"npsi:         %d  \n",npsi);
+    fprintf(pfile,"taucell:      %e  \n",taucell);
+    fprintf(pfile,"source_flag:  %d  \n",set_source_flag);
+    fprintf(pfile,"nfreq      :  %d  \n",nfreq);
+    fprintf(pfile,"fre_ratio:    %e  \n",fre_ratio);
+    fprintf(pfile,"nu_min:       %e  \n",nu_min);
+    fprintf(pfile,"nu_max:       %e  \n",nu_max);
+    if (IM_RADIATION_ENABLED) {
+      fprintf(pfile,"iteration:    %d  \n",pmb->pmy_mesh->pimrad->ite_scheme);
+      fprintf(pfile,"red_or_black: %d  \n",pmb->pmy_mesh->pimrad->rb_or_not);
+      fprintf(pfile,"err_limit:    %e  \n",pmb->pmy_mesh->pimrad->error_limit_);
+      fprintf(pfile,"n_limit:      %d  \n",pmb->pmy_mesh->pimrad->nlimit_);
+      fprintf(pfile,"tau_scheme    %d  \n",pradintegrator->tau_flag_);
+    }
+
+    for(int n=0; n<nang; ++n) {
+      fprintf(pfile,"%2d   %e   %e   %e    %e\n",n,mu(0,0,0,0,n),mu(1,0,0,0,n),
+              mu(2,0,0,0,n), wmu(n));
+    }
+    if (nfreq > 1) {
+      fprintf(pfile,"fre   spec\n");
+      for(int ifr=0; ifr<nfreq; ++ifr) {
+        fprintf(pfile,"%e   %e\n",nu_grid(ifr),emission_spec(ifr));
+      }
+    }
+    fclose(pfile);
+  }
+}
+
+// destructor
+// destructor not used
+NRRadiation::~NRRadiation() {
+  ir_old.DeleteAthenaArray();
+  rad_mom.DeleteAthenaArray();
+  rad_mom_cm.DeleteAthenaArray();
+
+  if (restart_from_gray) {
+    ir_gray.DeleteAthenaArray();
+  }
+
+  sigma_s.DeleteAthenaArray();
+  sigma_a.DeleteAthenaArray();
+  sigma_p.DeleteAthenaArray();
+  sigma_pe.DeleteAthenaArray();
+  t_floor_.DeleteAthenaArray();
+  t_ceiling_.DeleteAthenaArray();
+  output_sigma.DeleteAthenaArray();
+
+  mu.DeleteAthenaArray();
+  wmu.DeleteAthenaArray();
+  cosx_cm_.DeleteAthenaArray();
+  cosy_cm_.DeleteAthenaArray();
+  cosz_cm_.DeleteAthenaArray();
+
+  if (nfreq  > 1) {
+    rad_mom_nu.DeleteAthenaArray();
+    rad_mom_cm_nu.DeleteAthenaArray();
+  }
+  delete pradintegrator;
+}
+
+
+void NRRadiation::EnrollOpacityFunction(OpacityFunc MyOpacityFunction) {
+  UpdateOpacity = MyOpacityFunction;
+}
+
+
+void NRRadiation::EnrollFrequencyFunction(FrequencyFunc MyFrequencyFunction) {
+  UserFrequency = MyFrequencyFunction;
+}
+
+
+void NRRadiation::EnrollEmissionFunction(EmissionFunc MyEmissionSpec) {
+  UserEmissionSpec = MyEmissionSpec;
+}
diff --git a/src/nr_radiation/radiation.hpp b/src/nr_radiation/radiation.hpp
new file mode 100644
index 0000000000..8b8fbc422a
--- /dev/null
+++ b/src/nr_radiation/radiation.hpp
@@ -0,0 +1,171 @@
+#ifndef NR_RADIATION_RADIATION_HPP_
+#define NR_RADIATION_RADIATION_HPP_
+//======================================================================================
+// Athena++ astrophysical MHD code
+// Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+// See LICENSE file for full public license information.
+//======================================================================================
+//! \file radiation.hpp
+//  \brief definitions for Radiation class
+//======================================================================================
+
+// C headers
+
+// C++ headers
+#include <cstdint>     // int64_t
+#include <functional>  // reference_wrapper
+#include <string>
+#include <vector>
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
+#include "../bvals/cc/nr_radiation/bvals_rad.hpp"
+#include "../parameter_input.hpp"
+
+class MeshBlock;
+class ParameterInput;
+class RadIntegrator;
+
+//! \class Radiation
+//  \brief radiation data and functions
+
+// prototype for user-defined opacity function for radiative transfer
+
+// Array indices for radiation moments
+enum {IER=0, IFR1=1, IFR2=2, IFR3=3, IPR11=4, IPR22=5, IPR33=6, IPR12=7,
+      IPR13=8, IPR23=9, IPR21=10, IPR31=11, IPR32=12};
+
+enum {OPAS=0, OPAA=1, OPAP=2}; // scattering, absorption, Planck, opacity
+
+class NRRadiation {
+  friend class RadIntegrator;
+  friend class Mesh;
+ public:
+  NRRadiation(MeshBlock *pmb, ParameterInput *pin);
+  ~NRRadiation();
+
+  MeshBlock* pmy_block;    // ptr to MeshBlock containing this Fluid
+  AthenaArray<Real> ir, ir1, ir2, ir_old; // radiation specific intensity
+  AthenaArray<Real> ir_gray;
+  AthenaArray<Real> rad_mom; // frequency integrated radiation moments
+  AthenaArray<Real> rad_mom_cm; // co-moving frame Er, Frx, Fry, Frz
+  AthenaArray<Real> rad_mom_nu, rad_mom_cm_nu; // multi_group radiation moments
+  AthenaArray<Real> sigma_s, sigma_a; //   opacity
+                  //scattering and fluxes weighted Rosseland mean
+  AthenaArray<Real> sigma_p, sigma_pe;
+                  // Planck mean and radiation energy weighted mean
+  AthenaArray<Real> output_sigma; // frequency integrated opacity
+  AthenaArray<Real> mu, wmu; // angles and weight
+
+  AthenaArray<Real> flux[3]; // store transport flux, also need for refinement
+
+  AthenaArray<Real> coarse_ir_;
+  int refinement_idx{-1};
+
+  Real prat, crat; // prat=aT^4/P_0, crat=c/c_s
+  Real vmax;
+  Real reduced_c; // reduced speed of light
+  Real tunit, telectron; // gas temperature cgs unit,
+                         // effective electron scattering temperature
+  Real rhounit; // density unit
+  Real mol_weight; // mean molecular weight
+  Real lunit;  // length unit
+
+  Real sum_diff;
+  Real sum_full; // store
+
+  int nang, noct, n_fre_ang; // n_fre_ang=nang*nfreq
+  int angle_flag;
+  // variables related to the angular space transport
+  int nzeta, npsi;
+  AthenaArray<Real> coszeta_v, zeta_v_full, zeta_f_full, dzeta_v, dzeta_f,
+                    coszeta_f, len_zeta;
+  AthenaArray<Real> psi_v, psi_f, len_psi, psi_v_full, psi_f_full,
+                    dpsi_v, dpsi_f, sin_psi_f, cot_theta;
+
+  // The frequency grid
+  Real nu_min, nu_max;
+  // mininum and maximum frequencies, and number of frequency bins
+  int nfreq; // number of frequency bins
+  int restart_from_gray; //
+  Real fre_ratio; // ratio between neighboring frequency bins
+  // frequency grid, center of each frequency bin
+  AthenaArray<Real> nu_grid, nu_cen, delta_nu;
+  // gas emission term in each frequency bin relative to a_rT^4
+  AthenaArray<Real> emission_spec;
+  FrequencyFunc UserFrequency; // user defined frequency grid
+  void EnrollFrequencyFunction(FrequencyFunc MyFrequencyFunction);
+  EmissionFunc UserEmissionSpec;
+  void EnrollEmissionFunction(EmissionFunc MyEmissionSpec);
+
+  //  int ir_output; // the number of specific intensity to dump
+  //  AthenaArray<int> ir_index; // the array
+  //  AthenaArray<Real> dump_ir;
+
+  RadBoundaryVariable rad_bvar;
+
+  RadIntegrator *pradintegrator;
+
+  // The function pointer for the opacity
+  OpacityFunc UpdateOpacity;
+
+  Real kappa_es; // the frequency independent electron scattering opacity
+
+  int rotate_theta; // flag to rotate the boundary
+  int rotate_phi;
+  int set_source_flag; // flag to add radiation source term or not
+
+  // Functions
+  // Function in problem generators to update opacity
+  void EnrollOpacityFunction(OpacityFunc MyOpacityFunction);
+
+  void CalculateMoment(AthenaArray<Real> &ir_in);
+  void CalculateComMoment();
+
+  void AngularGrid(int angle_flag, int nmu);
+  void AngularGrid(int angle_flag, int nzeta, int npsi);
+
+  void FrequencyGrid();
+
+  Real FitIntPlanckFunc(Real nu_t);
+
+  Real IntPlanckFunc(Real nu_min, Real nu_max);
+  Real EffectiveBlackBody(Real intensity, Real nu);
+  Real EffectiveBlackBodyNNu2(Real n_nu2, Real nu);
+  Real IntegrateBBNuJ(Real nu_t); // integral of
+  Real IntegrateBBJONuSq(Real nu_t); //\integral of (j/\nu)^2d\nu
+  Real IntegrateBBNNu2(Real nu_t); // ingral of n\nu^2d\nu
+  Real ConvertBBJNNu2(Real &bb_j, Real &nu_f);
+  // Convert from n\nu^2 dnu to J
+  Real InverseConvertBBJNNu2(Real &nnu2, Real &nu_f);
+  // Convert J to \int (J/nu)^2
+  Real BBJToJONuSq(Real &bb_j, Real &nu_f);
+  // Convert J to n(nu_f)
+  Real BBJtoNnu(Real &bb_j, Real &nu_f);
+  // Convert J to \int J\nu
+  Real BBJtoJnu(Real &bb_j, Real &nu_f);
+  Real DBBjDNNu2(Real &bb_j, Real &nu_f);
+
+  // convert j to different quantities assuming Wien spectrum
+  void ConvertBBJWien(Real &bb_j, Real &nu_f, Real &tgas,
+                                 Real &nuj, Real &jonusq);
+  void ConvertBBJWien2(Real &bb_j, Real &nu_f, Real &tgas,
+                                 Real &nnu2, Real &n_nuf);
+  Real InverseConvertBBJNNu2Wien(Real &nnu2, Real &nu_f, Real &tgas);
+
+
+  AthenaArray<Real> t_floor_, t_ceiling_; // temperature floor
+
+ private:
+  int user_unit_;
+  // temporary arrays for co-moving moments
+  AthenaArray<Real> cosx_cm_, cosy_cm_, cosz_cm_;
+
+  friend class BoundaryValues;
+
+  // to use log frequency spaceing or not
+  int log_fre_;
+};
+
+#endif // NR_RADIATION_RADIATION_HPP_
diff --git a/src/orbital_advection/orbital_advection.hpp b/src/orbital_advection/orbital_advection.hpp
index 81e88d7486..90f404d1c5 100644
--- a/src/orbital_advection/orbital_advection.hpp
+++ b/src/orbital_advection/orbital_advection.hpp
@@ -60,9 +60,17 @@ class OrbitalAdvection{
   void RemapFluxPlm(AthenaArray<Real> &pflux_, const AthenaArray<Real> &pbuf_,
                     const Real eps_, const int osgn_, const int k, const int j,
                     const int il, const int iu, const int shift_ = 0);
+  void RemapFluxPlm(AthenaArray<Real> &pflux_, const AthenaArray<Real> &pbuf_,
+                    const Real eps_, const int osgn_, const int k, const int j,
+                    const int il, const int iu, const int nl, const int nu,
+                    const int shift_ = 0);
   void RemapFluxPpm(AthenaArray<Real> &pflux_, AthenaArray<Real> &pbuf_,
                     const Real eps_, const int osgn_, const int k, const int j,
                     const int il, const int iu, const int shift_ = 0);
+  void RemapFluxPpm(AthenaArray<Real> &pflux_, AthenaArray<Real> &pbuf_,
+                    const Real eps_, const int osgn_, const int k, const int j,
+                    const int il, const int iu, const int nl, const int nu,
+                    const int shift_ = 0);
 
   // accessor to orbital velocity
   OrbitalVelocityFunc OrbitalVelocity, OrbitalVelocityDerivative[2];
diff --git a/src/orbital_advection/orbital_remapping.cpp b/src/orbital_advection/orbital_remapping.cpp
index 74a0abd769..acfa205fd7 100644
--- a/src/orbital_advection/orbital_remapping.cpp
+++ b/src/orbital_advection/orbital_remapping.cpp
@@ -25,8 +25,7 @@
 #include "../mesh/mesh.hpp"
 
 // this class header
-#include "orbital_advection.hpp"
-
+#include "./orbital_advection.hpp"
 
 //----------------------------------------------------------------------------------------
 //! \fn void OrbitalAdvection::RemapFluxPlm(AthenaArray<Real> &pflux_,
@@ -55,6 +54,26 @@ void OrbitalAdvection::RemapFluxPlm(AthenaArray<Real> &pflux_,
   return;
 }
 
+void OrbitalAdvection::RemapFluxPlm(AthenaArray<Real> &pflux_,
+                                    const AthenaArray<Real> &pbuf_,
+                                    const Real eps_, const int osgn_, const int k,
+                                    const int j, const int il, const int iu,
+                                    const int nl, const int nu, const int shift_) {
+  const Real odi     = osgn_-0.5;
+  const Real coeff   = odi*(1.0-2.0*odi*eps_);
+  for (int i = il; i <= iu; i++) {
+    for (int n=nl; n<=nu; ++n) {
+      Real dul  = pbuf_(k,j,i+shift_,n)-pbuf_(k,j,i+shift_-1,n);
+      Real dur  = pbuf_(k,j,i+shift_+1,n)-pbuf_(k,j,i+shift_,n);
+      Real du2  = dul*dur;
+      Real dum  = 2.0*du2/(dul+dur);
+      if (du2 <= 0.0) dum = 0.0;
+      pflux_(i,n) = eps_*(pbuf_(k,j,i+shift_,n)+coeff*dum);
+    }
+  }
+  return;
+}
+
 
 //----------------------------------------------------------------------------------------
 //! \fn void OrbitalAdvection::RemapFluxPpm(AthenaArray<Real> &pflux_,
@@ -87,7 +106,7 @@ void OrbitalAdvection::RemapFluxPpm(AthenaArray<Real> &pflux_,
 
 //--- Step 1. -------------------------------------------------------------------------
 #pragma omp simd
-  for(int i = il; i <= iu; i++) {
+  for (int i = il; i <= iu; i++) {
     Real qa = q   (i) -q_m1(i);
     Real qb = q_p1(i) -q   (i);
     dd_m1(i) = 0.5*(qa+q_m1(i)-q_m2(i));
@@ -99,14 +118,14 @@ void OrbitalAdvection::RemapFluxPpm(AthenaArray<Real> &pflux_,
   }
 //--- Step 2. -----------------------------------------------------------------------
 #pragma omp simd simdlen(SIMD_WIDTH)
-  for(int i = il; i <= iu; i++) {
+  for (int i = il; i <= iu; i++) {
     d2qc_m1(i) = q_m2(i)+q   (i)-2.0*q_m1(i);
     d2qc   (i) = q_m1(i)+q_p1(i)-2.0*q   (i);
     d2qc_p1(i) = q   (i)+q_p2(i)-2.0*q_p1(i);
   }
   // i-1/2
 #pragma omp simd simdlen(SIMD_WIDTH)
-  for(int i = il; i <= iu; i++) {
+  for (int i = il; i <= iu; i++) {
     Real qa_tmp = dph(i) - q_m1(i); // (CD eq 84a)
     Real qb_tmp = q  (i) - dph (i);     // (CD eq 84b)
     Real qa = 3.0*(q_m1(i) + q(i) - 2.0*dph(i));  // (CD eq 85b)
@@ -225,3 +244,13 @@ void OrbitalAdvection::RemapFluxPpm(AthenaArray<Real> &pflux_,
   }
   return;
 }
+
+void OrbitalAdvection::RemapFluxPpm(AthenaArray<Real> &pflux_,
+                                    AthenaArray<Real> &pbuf_,
+                                    const Real eps_, const int osgn_,
+                                    const int k, const int j, const int il,
+                                    const int iu, const int nl, const int nu,
+                                    const int shift_) {
+  // will implement later
+  return;
+}
diff --git a/src/outputs/athena_hdf5.cpp b/src/outputs/athena_hdf5.cpp
index 6ccbeeca2b..db6ff2a698 100644
--- a/src/outputs/athena_hdf5.cpp
+++ b/src/outputs/athena_hdf5.cpp
@@ -27,6 +27,7 @@
 #include "../gravity/gravity.hpp"
 #include "../hydro/hydro.hpp"
 #include "../mesh/mesh.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../parameter_input.hpp"
 #include "outputs.hpp"
 
@@ -136,6 +137,13 @@ void ATHDF5Output::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
       if (pmb->pgrav->output_defect)
         num_variables[n_dataset] += 1;
     }
+    if(NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+      num_variables[n_dataset] += 20 *pmb->pnrrad->nfreq;
+
+    if(CR_ENABLED)
+      num_variables[n_dataset] += 13;
+
+
     // Passive scalars:
     if (NSCALARS > 0)
       num_variables[n_dataset] += NSCALARS;
@@ -186,12 +194,39 @@ void ATHDF5Output::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
         std::string vname = pod->name + sn;
         std::strncpy(variable_names[n_variable++], vname.c_str(), max_name_length+1);
       }
+    } else if(pod->type=="TENSORS") {
+      for(int i=1; i<=3; i++) {
+        char sn[3];
+        std::snprintf(sn,sizeof(sn),"%d%d", i,i);
+        std::string vname = pod->name + sn;
+        std::strncpy(variable_names[n_variable++], vname.c_str(), max_name_length+1);
+      }
+      for (int i=2; i<=3; i++) {
+        char sn[3];
+        std::snprintf(sn,sizeof(sn),"%d%d", 1,i);
+        std::string vname = pod->name + sn;
+        std::strncpy(variable_names[n_variable++], vname.c_str(), max_name_length+1);
+      }
+      for(int i=3; i>=1; i-=2) {
+        char sn[3];
+        std::snprintf(sn,sizeof(sn),"%d%d", 2,i);
+        std::string vname = pod->name + sn;
+        std::strncpy(variable_names[n_variable++], vname.c_str(), max_name_length+1);
+      }
+      for(int i=1; i<=2; i++) {
+        char sn[3];
+        std::snprintf(sn,sizeof(sn),"%d%d", 3,i);
+        std::string vname = pod->name + sn;
+        std::strncpy(variable_names[n_variable++], vname.c_str(), max_name_length+1);
+      }
     } else {
       std::strncpy(variable_names[n_variable++], pod->name.c_str(), max_name_length+1);
     }
     pod = pod->pnext;
   }
 
+
+
   // Make sure C-strings are null-terminated
   for (int n = 0; n < num_datasets; ++n)
     dataset_names[n][max_name_length] = '\0';
@@ -382,6 +417,8 @@ void ATHDF5Output::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
           }
           int nv=1;
           if (pod->type == "VECTORS") nv=3;
+          else if(pod->type == "TENSORS") nv=9;
+
           for (int v=0; v < nv; v++, ndv++) {
             int index = 0;
             for (int k = out_ks; k <= out_ke; k++) {
@@ -400,6 +437,7 @@ void ATHDF5Output::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
         while (pod != nullptr) {
           int nv=1;
           if (pod->type == "VECTORS") nv=3;
+          else if(pod->type == "TENSORS") nv=9;
           for (int v=0; v < nv; v++, ndv++) {
             int index = 0;
             for (int k = out_ks; k <= out_ke; k++) {
diff --git a/src/outputs/history.cpp b/src/outputs/history.cpp
index 5295b8d485..e472236683 100644
--- a/src/outputs/history.cpp
+++ b/src/outputs/history.cpp
@@ -4,8 +4,8 @@
 // Licensed under the 3-clause BSD License, see LICENSE file for details
 //========================================================================================
 //! \file history.cpp
-//! \brief writes history output data, volume-averaged quantities that are output
-//!        frequently in time to trace their history.
+//  \brief writes history output data, volume-averaged quantities that are output
+//         frequently in time to trace their history.
 
 // C headers
 
@@ -24,11 +24,13 @@
 #include "../athena.hpp"
 #include "../athena_arrays.hpp"
 #include "../coordinates/coordinates.hpp"
+#include "../cr/cr.hpp"
 #include "../field/field.hpp"
 #include "../globals.hpp"
 #include "../gravity/gravity.hpp"
 #include "../hydro/hydro.hpp"
 #include "../mesh/mesh.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../orbital_advection/orbital_advection.hpp"
 #include "../scalars/scalars.hpp"
 #include "outputs.hpp"
@@ -36,7 +38,9 @@
 // NEW_OUTPUT_TYPES:
 
 // "3" for 1-KE, 2-KE, 3-KE additional columns (come before tot-E)
-#define NHISTORY_VARS ((NHYDRO) + (SELF_GRAVITY_ENABLED > 0) + (NFIELD) + 3 + (NSCALARS))
+// 14 radiation variables, 4 cosmic ray variables
+#define NHISTORY_VARS ((NHYDRO) + (SELF_GRAVITY_ENABLED > 0) + (NFIELD) + 3 + (NSCALARS) \
+                      +(NRAD) + (NCR))
 
 //----------------------------------------------------------------------------------------
 //! \fn void HistoryOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag)
@@ -74,6 +78,9 @@ void HistoryOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
     PassiveScalars *psclr = pmb->pscalars;
     Gravity *pgrav = pmb->pgrav;
     OrbitalAdvection *porb = pmb->porb;
+    NRRadiation *prad = pmb->pnrrad;
+    CosmicRay *pcr = pmb->pcr;
+
 
     // Sum history variables over cells. Note ghost cells are never included in sums
     if(porb->orbital_advection_defined
@@ -125,6 +132,50 @@ void HistoryOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
               constexpr int prev_out = NHYDRO + 3 + (SELF_GRAVITY_ENABLED > 0) + NFIELD;
               hst_data[prev_out + n] += vol(i)*s;
             }
+            if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+              if (prad->nfreq == 1) {
+                constexpr int prev_out = NHYDRO + 3 + (SELF_GRAVITY_ENABLED > 0)
+                                       + NFIELD + NSCALARS;
+                hst_data[prev_out + 0] += vol(i)*prad->rad_mom(IER,k,j,i);
+                hst_data[prev_out + 1] += vol(i)*prad->rad_mom(IFR1,k,j,i);
+                hst_data[prev_out + 2] += vol(i)*prad->rad_mom(IFR2,k,j,i);
+                hst_data[prev_out + 3] += vol(i)*prad->rad_mom(IFR3,k,j,i);
+                hst_data[prev_out + 4] += vol(i)*prad->rad_mom_cm(IER,k,j,i);
+                hst_data[prev_out + 5] += vol(i)*prad->rad_mom_cm(IFR1,k,j,i);
+                hst_data[prev_out + 6] += vol(i)*prad->rad_mom_cm(IFR2,k,j,i);
+                hst_data[prev_out + 7] += vol(i)*prad->rad_mom_cm(IFR3,k,j,i);
+                hst_data[prev_out + 8] += vol(i)*prad->rad_mom(IPR11,k,j,i);
+                hst_data[prev_out + 9] += vol(i)*prad->rad_mom(IPR12,k,j,i);
+                hst_data[prev_out + 10] += vol(i)*prad->rad_mom(IPR13,k,j,i);
+                hst_data[prev_out + 11] += vol(i)*prad->rad_mom(IPR22,k,j,i);
+                hst_data[prev_out + 12] += vol(i)*prad->rad_mom(IPR23,k,j,i);
+                hst_data[prev_out + 13] += vol(i)*prad->rad_mom(IPR33,k,j,i);
+              } else {
+                if (4*prad->nfreq > NRAD) {
+                  std::stringstream msg;
+                  msg << "### FATAL ERROR in function [OutputType::HistoryFile]"
+                      << std::endl
+                      << "Incrase NRAD '" << NRAD << "' to 4x number of frequency groups";
+                  ATHENA_ERROR(msg);
+                }
+                constexpr int prev_out = NHYDRO + 3 + (SELF_GRAVITY_ENABLED > 0)
+                                       + NFIELD + NSCALARS;
+                for (int ifr=0; ifr<prad->nfreq; ++ifr) {
+                  hst_data[prev_out + 4*ifr] += vol(i)*prad->rad_mom_nu(ifr*13,k,j,i);
+                  hst_data[prev_out + 4*ifr+1] += vol(i)*prad->rad_mom_nu(ifr*13+1,k,j,i);
+                  hst_data[prev_out + 4*ifr+2] += vol(i)*prad->rad_mom_nu(ifr*13+2,k,j,i);
+                  hst_data[prev_out + 4*ifr+3] += vol(i)*prad->rad_mom_nu(ifr*13+3,k,j,i);
+                }
+              }
+            }
+            if (CR_ENABLED) {
+              constexpr int prev_out = NHYDRO + 3 + (SELF_GRAVITY_ENABLED > 0) + NFIELD +
+                                  NSCALARS + NRAD;
+              hst_data[prev_out + 0] += vol(i)*pcr->u_cr(IER,k,j,i);
+              hst_data[prev_out + 1] += vol(i)*pcr->u_cr(IFR1,k,j,i);
+              hst_data[prev_out + 2] += vol(i)*pcr->u_cr(IFR2,k,j,i);
+              hst_data[prev_out + 3] += vol(i)*pcr->u_cr(IFR3,k,j,i);
+            }
           }
         }
       }
@@ -175,6 +226,50 @@ void HistoryOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
               constexpr int prev_out = NHYDRO + 3 + (SELF_GRAVITY_ENABLED > 0) + NFIELD;
               hst_data[prev_out + n] += vol(i)*s;
             }
+            if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+              if (prad->nfreq == 1) {
+                constexpr int prev_out = NHYDRO + 3 + (SELF_GRAVITY_ENABLED > 0)
+                                       + NFIELD + NSCALARS;
+                hst_data[prev_out + 0] += vol(i)*prad->rad_mom(IER,k,j,i);
+                hst_data[prev_out + 1] += vol(i)*prad->rad_mom(IFR1,k,j,i);
+                hst_data[prev_out + 2] += vol(i)*prad->rad_mom(IFR2,k,j,i);
+                hst_data[prev_out + 3] += vol(i)*prad->rad_mom(IFR3,k,j,i);
+                hst_data[prev_out + 4] += vol(i)*prad->rad_mom_cm(IER,k,j,i);
+                hst_data[prev_out + 5] += vol(i)*prad->rad_mom_cm(IFR1,k,j,i);
+                hst_data[prev_out + 6] += vol(i)*prad->rad_mom_cm(IFR2,k,j,i);
+                hst_data[prev_out + 7] += vol(i)*prad->rad_mom_cm(IFR3,k,j,i);
+                hst_data[prev_out + 8] += vol(i)*prad->rad_mom(IPR11,k,j,i);
+                hst_data[prev_out + 9] += vol(i)*prad->rad_mom(IPR12,k,j,i);
+                hst_data[prev_out + 10] += vol(i)*prad->rad_mom(IPR13,k,j,i);
+                hst_data[prev_out + 11] += vol(i)*prad->rad_mom(IPR22,k,j,i);
+                hst_data[prev_out + 12] += vol(i)*prad->rad_mom(IPR23,k,j,i);
+                hst_data[prev_out + 13] += vol(i)*prad->rad_mom(IPR33,k,j,i);
+              } else {
+                if (4*prad->nfreq > NRAD) {
+                  std::stringstream msg;
+                  msg << "### FATAL ERROR in function [OutputType::HistoryFile]"
+                      << std::endl
+                      << "Incrase NRAD '" << NRAD << "' to 4x number of frequency groups";
+                  ATHENA_ERROR(msg);
+                }
+                constexpr int prev_out = NHYDRO + 3 + (SELF_GRAVITY_ENABLED > 0)
+                                       + NFIELD + NSCALARS;
+                for (int ifr=0; ifr<prad->nfreq; ++ifr) {
+                  hst_data[prev_out + 4*ifr] += vol(i)*prad->rad_mom_nu(ifr*13,k,j,i);
+                  hst_data[prev_out + 4*ifr+1] += vol(i)*prad->rad_mom_nu(ifr*13+1,k,j,i);
+                  hst_data[prev_out + 4*ifr+2] += vol(i)*prad->rad_mom_nu(ifr*13+2,k,j,i);
+                  hst_data[prev_out + 4*ifr+3] += vol(i)*prad->rad_mom_nu(ifr*13+3,k,j,i);
+                }
+              }
+            }
+            if (CR_ENABLED) {
+              constexpr int prev_out = NHYDRO + 3 + (SELF_GRAVITY_ENABLED > 0) + NFIELD +
+                                  NSCALARS + NRAD;
+              hst_data[prev_out + 0] += vol(i)*pcr->u_cr(IER,k,j,i);
+              hst_data[prev_out + 1] += vol(i)*pcr->u_cr(IFR1,k,j,i);
+              hst_data[prev_out + 2] += vol(i)*pcr->u_cr(IFR2,k,j,i);
+              hst_data[prev_out + 3] += vol(i)*pcr->u_cr(IFR3,k,j,i);
+            }
           }
         }
       }
@@ -190,10 +285,12 @@ void HistoryOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
             hst_data[NHISTORY_VARS+n] += usr_val;
             break;
           case UserHistoryOperation::max:
-            hst_data[NHISTORY_VARS+n] = std::max(usr_val, hst_data[NHISTORY_VARS+n]);
+            hst_data[NHISTORY_VARS+n] = std::max(
+                usr_val, hst_data[NHISTORY_VARS+n]);
             break;
           case UserHistoryOperation::min:
-            hst_data[NHISTORY_VARS+n] = std::min(usr_val, hst_data[NHISTORY_VARS+n]);
+            hst_data[NHISTORY_VARS+n] = std::min(
+                usr_val, hst_data[NHISTORY_VARS+n]);
             break;
         }
       }
@@ -203,10 +300,12 @@ void HistoryOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
 #ifdef MPI_PARALLEL
   // sum built-in/predefined hst_data[] over all ranks
   if (Globals::my_rank == 0) {
-    MPI_Reduce(MPI_IN_PLACE, hst_data.get(), NHISTORY_VARS, MPI_ATHENA_REAL, MPI_SUM, 0,
+    MPI_Reduce(MPI_IN_PLACE, hst_data.get(),
+               NHISTORY_VARS, MPI_ATHENA_REAL, MPI_SUM, 0,
                MPI_COMM_WORLD);
   } else {
-    MPI_Reduce(hst_data.get(), hst_data.get(), NHISTORY_VARS, MPI_ATHENA_REAL, MPI_SUM,
+    MPI_Reduce(hst_data.get(), hst_data.get(),
+               NHISTORY_VARS, MPI_ATHENA_REAL, MPI_SUM,
                0, MPI_COMM_WORLD);
   }
   // apply separate chosen operations to each user-defined history output
@@ -275,6 +374,37 @@ void HistoryOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
       for (int n=0; n<NSCALARS; n++) {
         std::fprintf(pfile,"[%d]=%d-scalar    ", iout++, n);
       }
+      if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+        if (pm->my_blocks(0)->pnrrad->nfreq == 1) {
+          std::fprintf(pfile,"[%d]=Er    ", iout++);
+          std::fprintf(pfile,"[%d]=Fr1    ", iout++);
+          std::fprintf(pfile,"[%d]=Fr2    ", iout++);
+          std::fprintf(pfile,"[%d]=Fr3    ", iout++);
+          std::fprintf(pfile,"[%d]=Er0    ", iout++);
+          std::fprintf(pfile,"[%d]=Fr10    ", iout++);
+          std::fprintf(pfile,"[%d]=Fr20    ", iout++);
+          std::fprintf(pfile,"[%d]=Fr30    ", iout++);
+          std::fprintf(pfile,"[%d]=Pr11    ", iout++);
+          std::fprintf(pfile,"[%d]=Pr12    ", iout++);
+          std::fprintf(pfile,"[%d]=Pr13    ", iout++);
+          std::fprintf(pfile,"[%d]=Pr22    ", iout++);
+          std::fprintf(pfile,"[%d]=Pr23    ", iout++);
+          std::fprintf(pfile,"[%d]=Pr33    ", iout++);
+        } else {
+          for (int ifr=0; ifr<pm->my_blocks(0)->pnrrad->nfreq; ++ifr) {
+            std::fprintf(pfile,"[%d]=Er   ", iout++);
+            std::fprintf(pfile,"[%d]=Fr1   ", iout++);
+            std::fprintf(pfile,"[%d]=Fr2   ", iout++);
+            std::fprintf(pfile,"[%d]=Fr3   ", iout++);
+          }
+        }
+      }
+      if (CR_ENABLED) {
+        std::fprintf(pfile,"[%d]=Ec    ", iout++);
+        std::fprintf(pfile,"[%d]=Fc1    ", iout++);
+        std::fprintf(pfile,"[%d]=Fc2    ", iout++);
+        std::fprintf(pfile,"[%d]=Fc3    ", iout++);
+      }
       for (int n=0; n<pm->nuser_history_output_; n++)
         std::fprintf(pfile,"[%d]=%-7s ", iout++,
                      pm->user_history_output_names_[n].c_str());
diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp
index b147e470c1..5ec4278060 100644
--- a/src/outputs/outputs.cpp
+++ b/src/outputs/outputs.cpp
@@ -90,10 +90,12 @@
 #include "../athena.hpp"
 #include "../athena_arrays.hpp"
 #include "../coordinates/coordinates.hpp"
+#include "../cr/cr.hpp"
 #include "../field/field.hpp"
 #include "../gravity/gravity.hpp"
 #include "../hydro/hydro.hpp"
 #include "../mesh/mesh.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../orbital_advection/orbital_advection.hpp"
 #include "../parameter_input.hpp"
 #include "../scalars/scalars.hpp"
@@ -235,7 +237,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin) {
         if (std::strcmp(COORDINATE_SYSTEM, "cylindrical") == 0 ||
             std::strcmp(COORDINATE_SYSTEM, "spherical_polar") == 0)
           op.cartesian_vector = pin->GetOrAddBoolean(op.block_name, "cartesian_vector",
-                                                   false);
+                                                     false);
         else
           op.cartesian_vector = false;
 
@@ -346,6 +348,8 @@ Outputs::~Outputs() {
 void OutputType::LoadOutputData(MeshBlock *pmb) {
   Hydro *phyd = pmb->phydro;
   Field *pfld = pmb->pfield;
+  NRRadiation *prad=pmb->pnrrad;
+  CosmicRay *pcr=pmb->pcr;
   PassiveScalars *psclr = pmb->pscalars;
   Gravity *pgrav = pmb->pgrav;
   OrbitalAdvection *porb = pmb->porb;
@@ -383,8 +387,8 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
       pod = new OutputData;
       pod->type = "SCALARS";
       pod->name = "Etot";
-      if(porb->orbital_advection_defined
-         && !output_params.orbital_system_output) {
+      if (porb->orbital_advection_defined
+          && !output_params.orbital_system_output) {
         porb->ConvertOrbitalSystem(phyd->w, phyd->u, OrbitalTransform::cons);
         pod->data.InitWithShallowSlice(porb->u_orb, 4, IEN, 1);
       } else {
@@ -412,8 +416,8 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
     pod = new OutputData;
     pod->type = "VECTORS";
     pod->name = "mom";
-    if(porb->orbital_advection_defined
-       && !output_params.orbital_system_output) {
+    if (porb->orbital_advection_defined
+        && !output_params.orbital_system_output) {
       porb->ConvertOrbitalSystem(phyd->w, phyd->u, OrbitalTransform::cons);
       pod->data.InitWithShallowSlice(porb->u_orb, 4, IM1, 3);
     } else {
@@ -423,8 +427,8 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
     num_vars_ += 3;
     if (output_params.cartesian_vector) {
       AthenaArray<Real> src;
-      if(porb->orbital_advection_defined
-         && !output_params.orbital_system_output) {
+      if (porb->orbital_advection_defined
+          && !output_params.orbital_system_output) {
         src.InitWithShallowSlice(porb->u_orb, 4, IM1, 3);
       } else {
         src.InitWithShallowSlice(phyd->u, 4, IM1, 3);
@@ -453,9 +457,9 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
     pod = new OutputData;
     pod->type = "SCALARS";
     pod->name = "mom2";
-    if(porb->orbital_advection_defined
-       && !output_params.orbital_system_output
-       && porb->orbital_direction == 1) {
+    if (porb->orbital_advection_defined
+        && !output_params.orbital_system_output
+        && porb->orbital_direction == 1) {
       porb->ConvertOrbitalSystem(phyd->w, phyd->u, OrbitalTransform::cons);
       pod->data.InitWithShallowSlice(porb->u_orb, 4, IM2, 1);
     } else {
@@ -468,9 +472,9 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
     pod = new OutputData;
     pod->type = "SCALARS";
     pod->name = "mom3";
-    if(porb->orbital_advection_defined
-       && !output_params.orbital_system_output
-       && porb->orbital_direction == 2) {
+    if (porb->orbital_advection_defined
+        && !output_params.orbital_system_output
+        && porb->orbital_direction == 2) {
       porb->ConvertOrbitalSystem(phyd->w, phyd->u, OrbitalTransform::cons);
       pod->data.InitWithShallowSlice(porb->u_orb, 4, IM3, 1);
     } else {
@@ -486,8 +490,8 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
     pod = new OutputData;
     pod->type = "VECTORS";
     pod->name = "vel";
-    if(porb->orbital_advection_defined
-       && !output_params.orbital_system_output) {
+    if (porb->orbital_advection_defined
+        && !output_params.orbital_system_output) {
       porb->ConvertOrbitalSystem(phyd->w, phyd->u, OrbitalTransform::prim);
       pod->data.InitWithShallowSlice(porb->w_orb, 4, IVX, 3);
     } else {
@@ -497,8 +501,8 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
     num_vars_ += 3;
     if (output_params.cartesian_vector) {
       AthenaArray<Real> src;
-      if(porb->orbital_advection_defined
-         && !output_params.orbital_system_output) {
+      if (porb->orbital_advection_defined
+          && !output_params.orbital_system_output) {
         src.InitWithShallowSlice(porb->w_orb, 4, IVX, 3);
       } else {
         src.InitWithShallowSlice(phyd->w, 4, IVX, 3);
@@ -529,9 +533,9 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
     pod = new OutputData;
     pod->type = "SCALARS";
     pod->name = "vel2";
-    if(porb->orbital_advection_defined
-       && !output_params.orbital_system_output
-       && porb->orbital_direction == 1) {
+    if (porb->orbital_advection_defined
+        && !output_params.orbital_system_output
+        && porb->orbital_direction == 1) {
       porb->ConvertOrbitalSystem(phyd->w, phyd->u, OrbitalTransform::prim);
       pod->data.InitWithShallowSlice(porb->w_orb, 4, IVY, 1);
     } else {
@@ -545,9 +549,9 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
     pod = new OutputData;
     pod->type = "SCALARS";
     pod->name = "vel3";
-    if(porb->orbital_advection_defined
-       && !output_params.orbital_system_output
-       && porb->orbital_direction == 2) {
+    if (porb->orbital_advection_defined
+        && !output_params.orbital_system_output
+        && porb->orbital_direction == 2) {
       porb->ConvertOrbitalSystem(phyd->w, phyd->u, OrbitalTransform::prim);
       pod->data.InitWithShallowSlice(porb->w_orb, 4, IVZ, 1);
     } else {
@@ -604,6 +608,447 @@ void OutputType::LoadOutputData(MeshBlock *pmb) {
       }
     }
   }
+
+
+  // The following radiation/cosmic ray/thermal conduction are all
+  // cell centered variable
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    if (prad->nfreq == 1) {
+      // (lab-frame) radiation energy density
+      if (ContainVariable(output_params.variable, "Er") ||
+          ContainVariable(output_params.variable, "prim") ||
+          ContainVariable(output_params.variable, "cons")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = "Er";
+        pod->data.InitWithShallowSlice(prad->rad_mom,4,IER,1);
+        AppendOutputDataNode(pod);
+        num_vars_++;
+      }
+
+      // comoving frame fram radiation flux vector
+      if (ContainVariable(output_params.variable, "Fr") ||
+          ContainVariable(output_params.variable, "prim") ||
+          ContainVariable(output_params.variable, "cons")) {
+        pod = new OutputData;
+        pod->type = "VECTORS";
+        pod->name = "Fr";
+        pod->data.InitWithShallowSlice(prad->rad_mom,4,IFR1,3);
+        AppendOutputDataNode(pod);
+        num_vars_+=3;
+        if (output_params.cartesian_vector) {
+          AthenaArray<Real> src;
+          src.InitWithShallowSlice(prad->rad_mom,4,IFR1,3);
+          pod = new OutputData;
+          pod->type = "VECTORS";
+          pod->name = "Fr_xyz";
+          pod->data.NewAthenaArray(3,prad->rad_mom.GetDim3(),prad->rad_mom.GetDim2(),
+                                   prad->rad_mom.GetDim1());
+          CalculateCartesianVector(src, pod->data, pmb->pcoord);
+          AppendOutputDataNode(pod);
+          num_vars_+=3;
+        }
+      }
+
+
+      // each component of radiation flux
+      if (ContainVariable(output_params.variable, "Frx") ||
+          ContainVariable(output_params.variable, "Fr1")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = "Fr1";
+        pod->data.InitWithShallowSlice(prad->rad_mom,4,IFR1,1);
+        AppendOutputDataNode(pod);
+        num_vars_++;
+      }
+
+      if (ContainVariable(output_params.variable, "Fry") ||
+          ContainVariable(output_params.variable, "Fr2")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = "Fr2";
+        pod->data.InitWithShallowSlice(prad->rad_mom,4,IFR2,1);
+        AppendOutputDataNode(pod);
+        num_vars_++;
+      }
+
+      if (ContainVariable(output_params.variable, "Frz") ||
+          ContainVariable(output_params.variable, "Fr3")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = "Fr3";
+        pod->data.InitWithShallowSlice(prad->rad_mom,4,IFR3,1);
+        AppendOutputDataNode(pod);
+        num_vars_++;
+      }
+
+      // lab frame radiation pressure
+      if (ContainVariable(output_params.variable, "Pr") ||
+          ContainVariable(output_params.variable, "prim") ||
+          ContainVariable(output_params.variable, "cons")) {
+        pod = new OutputData;
+        pod->type = "TENSORS";
+        pod->name = "Pr";
+        pod->data.InitWithShallowSlice(prad->rad_mom,4,IPR11,9);
+        AppendOutputDataNode(pod);
+        num_vars_ += 9;
+      }
+
+
+
+      // (comoving-frame) radiation energy density
+      if (ContainVariable(output_params.variable, "Er0") ||
+          ContainVariable(output_params.variable, "prim") ||
+          ContainVariable(output_params.variable, "cons")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = "Er0";
+        pod->data.InitWithShallowSlice(prad->rad_mom_cm,4,IER,1);
+        AppendOutputDataNode(pod);
+        num_vars_++;
+      }
+
+
+
+      // comoving frame fram radiation flux vector
+      if (ContainVariable(output_params.variable, "Fr0") ||
+          ContainVariable(output_params.variable, "prim") ||
+          ContainVariable(output_params.variable, "cons")) {
+        pod = new OutputData;
+        pod->type = "VECTORS";
+        pod->name = "Fr0";
+        pod->data.InitWithShallowSlice(prad->rad_mom_cm,4,IFR1,3);
+        AppendOutputDataNode(pod);
+        num_vars_+=3;
+        if (output_params.cartesian_vector) {
+          AthenaArray<Real> src;
+          src.InitWithShallowSlice(prad->rad_mom_cm,4,IFR1,3);
+          pod = new OutputData;
+          pod->type = "VECTORS";
+          pod->name = "Fr0_xyz";
+          pod->data.NewAthenaArray(3, prad->rad_mom_cm.GetDim3(),
+                                   prad->rad_mom_cm.GetDim2(),
+                                   prad->rad_mom_cm.GetDim1());
+          CalculateCartesianVector(src, pod->data, pmb->pcoord);
+          AppendOutputDataNode(pod);
+          num_vars_+=3;
+        }
+      }
+
+      if (ContainVariable(output_params.variable, "Fr0x") ||
+          ContainVariable(output_params.variable, "Fr01")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = "Fr01";
+        pod->data.InitWithShallowSlice(prad->rad_mom_cm,4,IFR1,1);
+        AppendOutputDataNode(pod);
+        num_vars_++;
+      }
+
+      if (ContainVariable(output_params.variable, "Fr0y") ||
+          ContainVariable(output_params.variable, "Fr02")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = "Fr02";
+        pod->data.InitWithShallowSlice(prad->rad_mom_cm,4,IFR2,1);
+        AppendOutputDataNode(pod);
+        num_vars_++;
+      }
+
+      if (ContainVariable(output_params.variable, "Fr0z") ||
+          ContainVariable(output_params.variable, "Fr03")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = "Fr03";
+        pod->data.InitWithShallowSlice(prad->rad_mom_cm,4,IFR3,1);
+        AppendOutputDataNode(pod);
+        num_vars_++;
+      }
+    } else {
+      //--------/--------/--------/--------/--------/--------/--------
+      for(int ifr=0; ifr<prad->nfreq; ++ifr) {
+        std::string er_ifr = "Er_" + std::to_string(ifr);
+        std::string fr_ifr = "Fr_" + std::to_string(ifr)+"_";
+        std::string frxyz_ifr = "Frxyz_" + std::to_string(ifr)+"_";
+        std::string frx_ifr = "Frx_" + std::to_string(ifr)+"_";
+        std::string fry_ifr = "Fry_" + std::to_string(ifr)+"_";
+        std::string frz_ifr = "Frz_" + std::to_string(ifr)+"_";
+        std::string pr_ifr = "Pr_" + std::to_string(ifr)+"_";
+        std::string er0_ifr = "Er0_" + std::to_string(ifr);
+        std::string fr0_ifr = "Fr0_" + std::to_string(ifr)+"_";
+        std::string fr0xyz_ifr = "Fr0xyz_" + std::to_string(ifr)+"_";
+        std::string fr0x_ifr = "Fr0x_" + std::to_string(ifr)+"_";
+        std::string fr0y_ifr = "Fr0y_" + std::to_string(ifr)+"_";
+        std::string fr0z_ifr = "Fr0z_" + std::to_string(ifr)+"_";
+
+
+        if (ContainVariable(output_params.variable, er_ifr) ||
+            ContainVariable(output_params.variable, "prim") ||
+            ContainVariable(output_params.variable, "cons")) {
+          pod = new OutputData;
+          pod->type = "SCALARS";
+          pod->name = er_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_nu,4,ifr*13+IER,1);
+          AppendOutputDataNode(pod);
+          num_vars_++;
+        }
+
+        // comoving frame fram radiation flux vector
+        if (ContainVariable(output_params.variable, fr_ifr) ||
+            ContainVariable(output_params.variable, "prim") ||
+            ContainVariable(output_params.variable, "cons")) {
+          pod = new OutputData;
+          pod->type = "VECTORS";
+          pod->name = fr_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_nu,4,ifr*13+IFR1,3);
+          AppendOutputDataNode(pod);
+          num_vars_+=3;
+          if (output_params.cartesian_vector) {
+            AthenaArray<Real> src;
+            src.InitWithShallowSlice(prad->rad_mom_nu,4,ifr*13+IFR1,3);
+            pod = new OutputData;
+            pod->type = "VECTORS";
+            pod->name = frxyz_ifr;
+            pod->data.NewAthenaArray(3, prad->rad_mom_nu.GetDim3(),
+                                     prad->rad_mom_nu.GetDim2(),
+                                     prad->rad_mom_nu.GetDim1());
+            CalculateCartesianVector(src, pod->data, pmb->pcoord);
+            AppendOutputDataNode(pod);
+            num_vars_+=3;
+          }
+        }
+
+
+        if (ContainVariable(output_params.variable, frx_ifr)) {
+          pod = new OutputData;
+          pod->type = "SCALARS";
+          pod->name = frx_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_nu,4,ifr*13+IFR1,1);
+          AppendOutputDataNode(pod);
+          num_vars_++;
+        }
+
+        if (ContainVariable(output_params.variable, fry_ifr)) {
+          pod = new OutputData;
+          pod->type = "SCALARS";
+          pod->name = fry_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_nu,4,ifr*13+IFR2,1);
+          AppendOutputDataNode(pod);
+          num_vars_++;
+        }
+
+        if (ContainVariable(output_params.variable, frz_ifr)) {
+          pod = new OutputData;
+          pod->type = "SCALARS";
+          pod->name = frz_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_nu,4,ifr*13+IFR3,1);
+          AppendOutputDataNode(pod);
+          num_vars_++;
+        }
+
+        if (ContainVariable(output_params.variable, pr_ifr) ||
+            ContainVariable(output_params.variable, "prim") ||
+            ContainVariable(output_params.variable, "cons")) {
+          pod = new OutputData;
+          pod->type = "TENSORS";
+          pod->name = pr_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_nu,4,ifr*13+IPR11,9);
+          AppendOutputDataNode(pod);
+          num_vars_+=9;
+        }
+
+        if (ContainVariable(output_params.variable, er0_ifr) ||
+            ContainVariable(output_params.variable, "prim") ||
+            ContainVariable(output_params.variable, "cons")) {
+          pod = new OutputData;
+          pod->type = "SCALARS";
+          pod->name = er0_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_cm_nu,4,ifr*4+IER,1);
+          AppendOutputDataNode(pod);
+          num_vars_++;
+        }
+
+        if (ContainVariable(output_params.variable, fr0_ifr) ||
+            ContainVariable(output_params.variable, "prim") ||
+            ContainVariable(output_params.variable, "cons")) {
+          pod = new OutputData;
+          pod->type = "VECTORS";
+          pod->name = fr0_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_cm_nu,4,ifr*4+IFR1,3);
+          AppendOutputDataNode(pod);
+          num_vars_+=3;
+          if (output_params.cartesian_vector) {
+            AthenaArray<Real> src;
+            src.InitWithShallowSlice(prad->rad_mom_cm_nu,4,ifr*4+IFR1,3);
+            pod = new OutputData;
+            pod->type = "VECTORS";
+            pod->name = fr0xyz_ifr;
+            pod->data.NewAthenaArray(3, prad->rad_mom_cm_nu.GetDim3(),
+                                     prad->rad_mom_cm_nu.GetDim2(),
+                                     prad->rad_mom_cm_nu.GetDim1());
+            CalculateCartesianVector(src, pod->data, pmb->pcoord);
+            AppendOutputDataNode(pod);
+            num_vars_+=3;
+          }
+        }
+
+
+        if (ContainVariable(output_params.variable, fr0x_ifr)) {
+          pod = new OutputData;
+          pod->type = "SCALARS";
+          pod->name = fr0x_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_cm_nu,4,ifr*4+IFR1,1);
+          AppendOutputDataNode(pod);
+          num_vars_++;
+        }
+
+        if (ContainVariable(output_params.variable, fr0y_ifr)) {
+          pod = new OutputData;
+          pod->type = "SCALARS";
+          pod->name = fr0y_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_cm_nu,4,ifr*4+IFR2,1);
+          AppendOutputDataNode(pod);
+          num_vars_++;
+        }
+
+        if (ContainVariable(output_params.variable, fr0z_ifr)) {
+          pod = new OutputData;
+          pod->type = "SCALARS";
+          pod->name = fr0z_ifr;
+          pod->data.InitWithShallowSlice(prad->rad_mom_cm_nu,4,ifr*4+IFR3,1);
+          AppendOutputDataNode(pod);
+          num_vars_++;
+        }
+      }
+    }
+    //-------------------------------------------------------
+    for(int ifr=0; ifr<prad->nfreq; ++ifr) {
+      std::string sigmaa_ifr = "Sigma_a_" + std::to_string(ifr);
+      std::string sigmas_ifr = "Sigma_s_" + std::to_string(ifr);
+      std::string sigmap_ifr = "Sigma_p_" + std::to_string(ifr);
+      if (ContainVariable(output_params.variable, sigmas_ifr) ||
+          ContainVariable(output_params.variable, "prim") ||
+          ContainVariable(output_params.variable, "cons")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = sigmas_ifr;
+        pod->data.InitWithShallowSlice(prad->output_sigma,4,3*ifr+OPAS,1);
+        AppendOutputDataNode(pod);
+        num_vars_ += 1;
+      }
+
+      if (ContainVariable(output_params.variable, sigmaa_ifr) ||
+          ContainVariable(output_params.variable, "prim") ||
+          ContainVariable(output_params.variable, "cons")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = sigmaa_ifr;
+        pod->data.InitWithShallowSlice(prad->output_sigma,4,3*ifr+OPAA,1);
+        AppendOutputDataNode(pod);
+        num_vars_ += 1;
+      }
+
+      if (ContainVariable(output_params.variable, sigmap_ifr) ||
+          ContainVariable(output_params.variable, "prim") ||
+          ContainVariable(output_params.variable, "cons")) {
+        pod = new OutputData;
+        pod->type = "SCALARS";
+        pod->name = sigmap_ifr;
+        pod->data.InitWithShallowSlice(prad->output_sigma,4,3*ifr+OPAP,1);
+        AppendOutputDataNode(pod);
+        num_vars_ += 1;
+      }
+    }// end ifr loop
+  }// End (RADIATION_ENABLED)
+
+
+  if (CR_ENABLED) {
+    if (ContainVariable(output_params.variable, "Ec") ||
+        ContainVariable(output_params.variable, "prim") ||
+        ContainVariable(output_params.variable, "cons")) {
+      pod = new OutputData;
+      pod->type = "SCALARS";
+      pod->name = "Ec";
+      pod->data.InitWithShallowSlice(pcr->u_cr,4,CRE,1);
+      AppendOutputDataNode(pod);
+      num_vars_++;
+    }
+
+    // comoving frame fram radiation flux vector
+    if (ContainVariable(output_params.variable, "Fc") ||
+        ContainVariable(output_params.variable, "prim") ||
+        ContainVariable(output_params.variable, "cons")) {
+      pod = new OutputData;
+      pod->type = "VECTORS";
+      pod->name = "Fc";
+      pod->data.InitWithShallowSlice(pcr->u_cr,4,CRF1,3);
+      AppendOutputDataNode(pod);
+      num_vars_+=3;
+      if (output_params.cartesian_vector) {
+        AthenaArray<Real> src;
+        src.InitWithShallowSlice(pcr->u_cr,4,CRF1,3);
+        pod = new OutputData;
+        pod->type = "VECTORS";
+        pod->name = "Fr_xyz";
+        pod->data.NewAthenaArray(3,pcr->u_cr.GetDim3(),pcr->u_cr.GetDim2(),
+                                 pcr->u_cr.GetDim1());
+        CalculateCartesianVector(src, pod->data, pmb->pcoord);
+        AppendOutputDataNode(pod);
+        num_vars_+=3;
+      }
+    }
+
+    if (ContainVariable(output_params.variable, "Sigma_diff") ||
+        ContainVariable(output_params.variable, "prim") ||
+        ContainVariable(output_params.variable, "cons")) {
+      pod = new OutputData;
+      pod->type = "VECTORS";
+      pod->name = "Sigma_diff";
+      pod->data.InitWithShallowSlice(pcr->sigma_diff,4,0,3);
+      AppendOutputDataNode(pod);
+      num_vars_+=3;
+    }
+
+    if (ContainVariable(output_params.variable, "Sigma_adv") ||
+        ContainVariable(output_params.variable, "prim") ||
+        ContainVariable(output_params.variable, "cons")) {
+      pod = new OutputData;
+      pod->type = "VECTORS";
+      pod->name = "Sigma_adv";
+      pod->data.InitWithShallowSlice(pcr->sigma_adv,4,0,3);
+      AppendOutputDataNode(pod);
+      num_vars_+=3;
+    }
+
+    // The streaming velocity
+    if (ContainVariable(output_params.variable, "Vc") ||
+        ContainVariable(output_params.variable, "prim") ||
+        ContainVariable(output_params.variable, "cons")) {
+      pod = new OutputData;
+      pod->type = "VECTORS";
+      pod->name = "Vc";
+      pod->data.InitWithShallowSlice(pcr->v_adv,4,0,3);
+      AppendOutputDataNode(pod);
+      num_vars_+=3;
+      if (output_params.cartesian_vector) {
+        AthenaArray<Real> src;
+        src.InitWithShallowSlice(pcr->v_adv,4,0,3);
+        pod = new OutputData;
+        pod->type = "VECTORS";
+        pod->name = "Vc_xyz";
+        pod->data.NewAthenaArray(3,pcr->v_adv.GetDim3(),
+                                 pcr->v_adv.GetDim2(),
+                                 pcr->v_adv.GetDim1());
+        CalculateCartesianVector(src, pod->data, pmb->pcoord);
+        AppendOutputDataNode(pod);
+        num_vars_+=3;
+      }
+    }
+  }// end Cosmic Rays
+
+
+
+
   // note, the Bcc variables are stored in a separate HDF5 dataset from the above Output
   // nodes, and it must come after those nodes in the linked list
   if (MAGNETIC_FIELDS_ENABLED) {
@@ -787,14 +1232,27 @@ void Outputs::MakeOutputs(Mesh *pm, ParameterInput *pin, bool wtflag) {
   // wtflag = only true for making final outputs due to signal or wall-time/cycle/time
   // limit. Used by restart file output to change suffix to .final
   bool first=true;
+  MeshBlock *pmb;
+  bool rad_mom=true;
   OutputType* ptype = pfirst_type_;
   while (ptype != nullptr) {
     if (((pm->time == pm->start_time) // output initial conditions, unless next_time set
          && (ptype->output_params.next_time <= pm->start_time ))
-      || (ptype->output_params.dt > 0.0 && pm->time >= ptype->output_params.next_time)
-      || (ptype->output_params.dcycle > 0 && pm->ncycle%ptype->output_params.dcycle == 0)
-      || (pm->time >= pm->tlim)
-      || (wtflag && ptype->output_params.file_type == "rst")) {
+        || (ptype->output_params.dt > 0.0 && pm->time >= ptype->output_params.next_time)
+        || (ptype->output_params.dcycle > 0
+            && pm->ncycle%ptype->output_params.dcycle == 0)
+        || (pm->time >= pm->tlim)
+        || (wtflag && ptype->output_params.file_type == "rst")) {
+      if (rad_mom && (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)) {
+        for(int b=0; b<pm->nblocal; ++b) {
+          pmb = pm->my_blocks(b);
+          // Calculate Com-moving moments and grey opacity for dump
+          pmb->pnrrad->CalculateMoment(pmb->pnrrad->ir);
+          pmb->pnrrad->CalculateComMoment();
+        }
+        rad_mom = false;
+      }
+
       if (first && ptype->output_params.file_type != "hst") {
         pm->ApplyUserWorkBeforeOutput(pin);
         first = false;
@@ -953,7 +1411,6 @@ bool OutputType::SliceOutputData(MeshBlock *pmb, int dim) {
 void OutputType::SumOutputData(MeshBlock* pmb, int dim) {
   // For each node in OutputData doubly linked list, sum arrays containing output data
   OutputData *pdata = pfirst_data_;
-
   while (pdata != nullptr) {
     OutputData *pnew = new OutputData;
     pnew->type = pdata->type;
@@ -1088,7 +1545,7 @@ bool OutputType::ContainVariable(const std::string &haystack, const std::string
   if (haystack.find(needle + ',') == 0)
     return true;
   if (haystack.find(',' + needle) != std::string::npos
-    && haystack.find(',' + needle) == haystack.length() - needle.length() - 1)
+      && haystack.find(',' + needle) == haystack.length() - needle.length() - 1)
     return true;
   return false;
 }
diff --git a/src/outputs/restart.cpp b/src/outputs/restart.cpp
index 72d88d19a3..302f50635d 100644
--- a/src/outputs/restart.cpp
+++ b/src/outputs/restart.cpp
@@ -21,13 +21,15 @@
 // Athena++ headers
 #include "../athena.hpp"
 #include "../athena_arrays.hpp"
+#include "../cr/cr.hpp"
 #include "../field/field.hpp"
 #include "../globals.hpp"
 #include "../hydro/hydro.hpp"
 #include "../mesh/mesh.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../parameter_input.hpp"
 #include "../scalars/scalars.hpp"
-#include "outputs.hpp"
+#include "./outputs.hpp"
 
 
 //----------------------------------------------------------------------------------------
@@ -177,6 +179,16 @@ void RestartOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool force_wr
       pdata += pmb->pfield->b.x3f.GetSizeInBytes();
     }
 
+    if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+      std::memcpy(pdata,pmb->pnrrad->ir.data(),pmb->pnrrad->ir.GetSizeInBytes());
+      pdata += pmb->pnrrad->ir.GetSizeInBytes();
+    }
+
+    if (CR_ENABLED) {
+      std::memcpy(pdata,pmb->pcr->u_cr.data(),pmb->pcr->u_cr.GetSizeInBytes());
+      pdata += pmb->pcr->u_cr.GetSizeInBytes();
+    }
+
     // (conserved variable) Passive scalars:
     if (NSCALARS > 0) {
       AthenaArray<Real> &s = pmb->pscalars->s;
diff --git a/src/outputs/vtk.cpp b/src/outputs/vtk.cpp
index edc45ad10b..7e8fce0329 100644
--- a/src/outputs/vtk.cpp
+++ b/src/outputs/vtk.cpp
@@ -132,7 +132,10 @@ void VTKOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, bool flag) {
     float *data;
     int ndata = std::max(ncoord1, ncoord2);
     ndata = std::max(ndata, ncoord3);
-    data = new float[3*ndata];
+    int array_size = 3*ndata;
+    if(NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+      array_size = 9 * ndata;
+    data = new float[array_size];
 
     // Specify the type of data, dimensions, and coordinates.  If N>1, then write N+1
     // cell faces as binary floats.  If N=1, then write 1 cell center position.
diff --git a/src/pgen/beam.cpp b/src/pgen/beam.cpp
new file mode 100644
index 0000000000..19a6cd1d33
--- /dev/null
+++ b/src/pgen/beam.cpp
@@ -0,0 +1,201 @@
+//======================================================================================
+/* Athena++ astrophysical MHD code
+ * Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+ *
+ * This program is free software: you can redistribute and/or modify it under the terms
+ * of the GNU General Public License (GPL) as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of GNU GPL in the file LICENSE included in the code
+ * distribution.  If not see <http://www.gnu.org/licenses/>.
+ *====================================================================================*/
+
+// C headers
+
+// C++ headers
+#include <algorithm>  // min
+#include <cmath>      // sqrt
+#include <fstream>
+#include <iostream>   // endl
+#include <sstream>    // stringstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
+#include "../bvals/bvals.hpp"
+#include "../coordinates/coordinates.hpp"
+#include "../eos/eos.hpp"
+#include "../field/field.hpp"
+#include "../hydro/hydro.hpp"
+#include "../hydro/srcterms/hydro_srcterms.hpp"
+#include "../mesh/mesh.hpp"
+#include "../nr_radiation/radiation.hpp"
+#include "../parameter_input.hpp"
+
+// File scope variables
+static int ang;
+static int octnum;
+
+int RefinementCondition(MeshBlock *pmb);
+
+//======================================================================================
+/*! \file beam.cpp
+ *  \brief Beam test for the radiative transfer module
+ *
+ *====================================================================================*/
+
+void TwoBeams(MeshBlock *pmb, Coordinates *pco, NRRadiation *prad,
+              const AthenaArray<Real> &w, FaceField &b,
+              AthenaArray<Real> &ir, Real time, Real dt,
+              int is, int ie, int js, int je, int ks, int ke, int ngh);
+
+void TwoBeamHydro(
+    MeshBlock *pmb, Coordinates *pco, AthenaArray<Real> &prim, FaceField &b,
+    Real time, Real dt, int is, int ie, int js, int je, int ks, int ke, int ngh);
+
+void Mesh::InitUserMeshData(ParameterInput *pin) {
+  if (adaptive)
+    EnrollUserRefinementCondition(RefinementCondition);
+
+  EnrollUserBoundaryFunction(BoundaryFace::inner_x2, TwoBeamHydro);
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+    EnrollUserRadBoundaryFunction(BoundaryFace::inner_x2, TwoBeams);
+}
+
+void MeshBlock::InitUserMeshBlockData(ParameterInput *pin) {
+  ang = pin->GetOrAddInteger("problem","ang",0);
+  octnum = pin->GetOrAddInteger("problem","octnum",0);
+  return;
+}
+
+
+//======================================================================================
+//! \fn void MeshBlock::ProblemGenerator(ParameterInput *pin)
+//  \brief beam test
+//======================================================================================
+void MeshBlock::ProblemGenerator(ParameterInput *pin) {
+  Real gamma = peos->GetGamma();
+
+  // Initialize hydro variable
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      for (int i=is; i<=ie; ++i) {
+        phydro->u(IDN,k,j,i) = 1.0;
+        phydro->u(IM1,k,j,i) = 0.0;
+        phydro->u(IM2,k,j,i) = 0.0;
+        phydro->u(IM3,k,j,i) = 0.0;
+        if (NON_BAROTROPIC_EOS) {
+          phydro->u(IEN,k,j,i) = 1.0/(gamma-1.0);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM1,k,j,i))/phydro->u(IDN,k,j,i);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM2,k,j,i))/phydro->u(IDN,k,j,i);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM3,k,j,i))/phydro->u(IDN,k,j,i);
+        }
+      }
+    }
+  }
+
+  // Now initialize opacity and specific intensity
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    int nfreq = pnrrad->nfreq;
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          for (int ifr=0; ifr < nfreq; ++ifr) {
+            pnrrad->sigma_s(k,j,i,ifr) = 0.0;
+            pnrrad->sigma_a(k,j,i,ifr) = 0.0;
+            pnrrad->sigma_pe(k,j,i,ifr) = 0.0;
+            pnrrad->sigma_p(k,j,i,ifr) = 0.0;
+          }
+          for (int n=0; n<pnrrad->n_fre_ang; ++n) {
+              pnrrad->ir(k,j,i,n) = 0.0;
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+void TwoBeams(MeshBlock *pmb, Coordinates *pco, NRRadiation *prad,
+              const AthenaArray<Real> &w, FaceField &b,
+              AthenaArray<Real> &ir,
+              Real time, Real dt,
+              int is, int ie, int js, int je, int ks, int ke, int ngh) {
+  int nang=prad->nang;
+  int noct=prad->noct;
+  int nfreq=prad->nfreq;
+  int ang_oct=nang/noct;
+
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=1; j<=ngh; ++j) {
+      for (int i=is; i<=ie; ++i) {
+        Real const &x1 = pco->x1v(i);
+        Real const &x2 = pco->x2v(js-j);
+        for (int ifr=0; ifr<nfreq; ++ifr) {
+          for (int l=0; l<noct; ++l) {
+            for (int n=0; n<ang_oct; ++n) {
+              int n_ang=l*ang_oct + n;
+
+              Real slope1=-prad->mu(1,k,js-j,i,0)/prad->mu(0,k,js-j,i,0);
+              Real slope2=prad->mu(1,k,js-j,i,0)/prad->mu(0,k,js-j,i,0);
+              Real dis1=std::abs(slope1*(x1-0.1)+(x2+2.0));
+              Real dis2=std::abs(slope2*(x1+0.1)+(x2+2.0));
+              if (ifr == 0) {
+                if (((l==0)&&(n==0)&&(dis1<pco->dx1v(i))) ||
+                    ((l==1)&&(n==0)&&(dis2<pco->dx1v(i)))) {
+                  ir(k,js-j,i,n_ang+ifr*nang) = 10.0;
+                } else {
+                  ir(k,js-j,i,n_ang+ifr*nang) = 0.0;
+                }
+              } else {
+                if (((l==0)&&(n==1)&&(dis1<pco->dx1v(i))) ||
+                    ((l==1)&&(n==1)&&(dis2<pco->dx1v(i)))) {
+                  ir(k,js-j,i,n_ang+ifr*nang) = 10.0;
+                } else {
+                  ir(k,js-j,i,n_ang+ifr*nang) = 0.0;
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+
+void TwoBeamHydro(
+    MeshBlock *pmb, Coordinates *pco, AthenaArray<Real> &prim, FaceField &b,
+    Real time, Real dt, int is, int ie, int js, int je, int ks, int ke, int ngh) {
+  for (int n=0; n<NHYDRO; ++n) {
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=1; j<=ngh; ++j) {
+#pragma omp simd
+        for (int i=is; i<=ie; ++i) {
+          prim(n,k,js-j,i) = prim(n,k,js,i);
+        }
+      }
+    }
+  }
+}
+
+// refinement condition: density curvature
+int RefinementCondition(MeshBlock *pmb) {
+  Coordinates *pco = pmb->pcoord;
+  Real xmin = pco->x1f(pmb->is);
+  Real xmax = pco->x1f(pmb->ie+1);
+  Real ymin = pco->x2f(pmb->js);
+  Real ymax = pco->x2f(pmb->je+1);
+  Real zmin = pco->x3f(pmb->ks);
+  Real zmax = pco->x3f(pmb->ke+1);
+  // refine : delta rho > 0.9*amp
+  if (ymin > -1 && ymax < 1) return 1;
+  return 0;
+}
diff --git a/src/pgen/cr_diffusion.cpp b/src/pgen/cr_diffusion.cpp
new file mode 100644
index 0000000000..3cc3972b3d
--- /dev/null
+++ b/src/pgen/cr_diffusion.cpp
@@ -0,0 +1,429 @@
+//======================================================================================
+/* Athena++ astrophysical MHD code
+ * Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+ *
+ * This program is free software: you can redistribute and/or modify it under the terms
+ * of the GNU General Public License (GPL) as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of GNU GPL in the file LICENSE included in the code
+ * distribution.  If not see <http://www.gnu.org/licenses/>.
+ *====================================================================================*/
+
+// C++ headers
+#include <algorithm>  // min
+#include <cmath>      // sqrt
+#include <fstream>
+#include <iostream>   // endl
+#include <sstream>    // stringstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
+#include "../bvals/bvals.hpp"
+#include "../coordinates/coordinates.hpp"
+#include "../cr/cr.hpp"
+#include "../cr/integrators/cr_integrators.hpp"
+#include "../eos/eos.hpp"
+#include "../field/field.hpp"
+#include "../globals.hpp"
+#include "../hydro/hydro.hpp"
+#include "../hydro/srcterms/hydro_srcterms.hpp"
+#include "../mesh/mesh.hpp"
+#include "../parameter_input.hpp"
+
+
+//======================================================================================
+//! \fn void MeshBlock::ProblemGenerator(ParameterInput *pin)
+//  \brief cosmic ray diffusion test
+//======================================================================================
+
+static Real sigma = 1.e3;
+static Real vx = 0.0;
+static Real vy = 0.0;
+static Real vz = 0.0;
+static int direction = 0;
+
+void Diffusion(MeshBlock *pmb, AthenaArray<Real> &u_cr,
+        AthenaArray<Real> &prim, AthenaArray<Real> &bcc);
+
+void Mesh::UserWorkAfterLoop(ParameterInput *pin) {
+  // calculate the anaysis solution and compare
+  //find the time, going through all the mesh block
+  MeshBlock *pmb = my_blocks(0);
+  Real vmax = pmb->pcr->vmax;
+  Real sum_error=0.0;
+  int num_cell=0;
+  Real diff_coef=vmax/(3.0*sigma);
+  for(int nb=0; nb<nblocal; ++nb) {
+    pmb=my_blocks(nb);
+    int ks=pmb->ks; int ke=pmb->ke; int js=pmb->js; int je=pmb->je;
+    int is=pmb->is; int ie=pmb->ie;
+    for(int k=ks; k<=ke; ++k) {
+      for(int j=js; j<=je; ++j) {
+        for(int i=is; i<=ie; ++i) {
+          Real x1 = pmb->pcoord->x1v(i);
+          Real x2 = pmb->pcoord->x2v(j);
+          Real x3 = pmb->pcoord->x3v(k);
+
+          Real dist_sq=(x1-vx*time)*(x1-vx*time);
+          if (direction == 1)
+            dist_sq=(x2-vy*time)*(x2-vy*time);
+          else if (direction == 2)
+            dist_sq=(x3-vz*time)*(x3-vz*time);
+          Real sol = std::exp(-40.0*dist_sq/(4*diff_coef*time*40.0
+                                             + 1.0))/std::sqrt(4*diff_coef*time*40+1);
+          sum_error += std::abs(pmb->pcr->u_cr(CRE,k,j,i)-sol);
+          num_cell++;
+        }
+      }
+    }
+  }
+
+#ifdef MPI_PARALLEL
+  MPI_Allreduce(&sum_error, &sum_error, 1, MPI_DOUBLE,MPI_SUM, MPI_COMM_WORLD);
+  MPI_Allreduce(&num_cell, &num_cell, 1, MPI_INT,MPI_SUM, MPI_COMM_WORLD);
+
+#endif
+  if (Globals::my_rank == 0) {
+    sum_error /= num_cell;
+
+    std::string fname;
+    fname.assign("diffusion_error.dat");
+    std::stringstream msg;
+    FILE *pfile;
+
+    if ((pfile = fopen(fname.c_str(),"r")) != NULL) {
+      if ((pfile = freopen(fname.c_str(),"a",pfile)) == NULL) {
+        msg << "### FATAL ERROR in function [Mesh::UserWorkAfterLoop]"
+            << std::endl << "Error output file could not be opened" <<std::endl;
+        throw std::runtime_error(msg.str().c_str());
+      }
+
+    // The file does not exist -- open the file in write mode and add headers
+    } else {
+      if ((pfile = fopen(fname.c_str(),"w")) == NULL) {
+        msg << "### FATAL ERROR in function [Mesh::UserWorkAfterLoop]"
+            << std::endl << "Error output file could not be opened" <<std::endl;
+        throw std::runtime_error(msg.str().c_str());
+      }
+      fprintf(pfile,"# Nx1  Nx2  Nx3  vx  vy  sigma  time  error");
+      fprintf(pfile,"\n");
+    }
+
+    // write errors
+    fprintf(pfile,"%d  %d  %d",mesh_size.nx1,mesh_size.nx2,mesh_size.nx3);
+    fprintf(pfile,"  %e  %e  %e  %e  %e  %e\n",vx, vy, vz, sigma,time,sum_error);
+    fclose(pfile);
+  }
+}
+
+void MeshBlock::InitUserMeshBlockData(ParameterInput *pin) {
+  if (CR_ENABLED) {
+    pcr->EnrollOpacityFunction(Diffusion);
+  }
+}
+
+void MeshBlock::ProblemGenerator(ParameterInput *pin) {
+  // read in the mean velocity, diffusion coefficient
+  direction = pin->GetOrAddReal("problem","direction",0);
+  if (direction == 0)
+    vx = pin->GetOrAddReal("problem","v0",0);
+  else if (direction == 1)
+    vy = pin->GetOrAddReal("problem","v0",0);
+  else if (direction == 2)
+    vz = pin->GetOrAddReal("problem","v0",0);
+
+  sigma = pin->GetOrAddReal("problem","sigma",1.e3);
+
+  Real gamma = peos->GetGamma();
+  // Initialize hydro variable
+  for(int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      for (int i=is; i<=ie; ++i) {
+        Real x1 = pcoord->x1v(i);
+        Real x2 = pcoord->x2v(j);
+        Real x3 = pcoord->x3v(k);
+
+        Real dist_sq=x1*x1;
+        if (direction ==1) {
+          dist_sq=x2*x2;
+        } else if (direction == 2) {
+          dist_sq=x3*x3;
+        }
+
+        phydro->u(IDN,k,j,i) = 1.0;
+        phydro->u(IM1,k,j,i) = vx;
+        phydro->u(IM2,k,j,i) = vy;
+        phydro->u(IM3,k,j,i) = vz;
+        if (NON_BAROTROPIC_EOS) {
+          phydro->u(IEN,k,j,i) = 0.5*(vx*vx+vy*vy+vz*vz)+1.0/(gamma-1.0);
+        }
+
+        if (CR_ENABLED) {
+            pcr->u_cr(CRE,k,j,i) = std::exp(-40.0*(dist_sq));
+            if (direction == 0) {
+              pcr->u_cr(CRF1,k,j,i) = vx*4.0*std::exp(-40.0*dist_sq)/(3.0*pcr->vmax)
+                                   +80*x1*std::exp(-40.0*dist_sq)/sigma;
+              pcr->u_cr(CRF2,k,j,i) = 0.0;
+              pcr->u_cr(CRF3,k,j,i) = 0.0;
+            } else if (direction == 1) {
+              pcr->u_cr(CRF2,k,j,i) = vy*4.0*std::exp(-40.0*dist_sq)/(3.0*pcr->vmax)
+                                   +80*x2*std::exp(-40.0*dist_sq)/sigma;
+              pcr->u_cr(CRF1,k,j,i) = 0.0;
+              pcr->u_cr(CRF3,k,j,i) = 0.0;
+            } else if (direction == 2) {
+              pcr->u_cr(CRF3,k,j,i) = vz*4.0*std::exp(-40.0*dist_sq)/(3.0*pcr->vmax)
+                                   +80*x3*std::exp(-40.0*dist_sq)/sigma;
+              pcr->u_cr(CRF1,k,j,i) = 0.0;
+              pcr->u_cr(CRF2,k,j,i) = 0.0;
+            }
+        }
+      }
+    }
+  }
+  //Need to set opactiy sigma in the ghost zones
+  if (CR_ENABLED) {
+  // Default values are 1/3
+    int nz1 = block_size.nx1 + 2*(NGHOST);
+    int nz2 = block_size.nx2;
+    if (nz2 > 1) nz2 += 2*(NGHOST);
+    int nz3 = block_size.nx3;
+    if (nz3 > 1) nz3 += 2*(NGHOST);
+    for(int k=0; k<nz3; ++k) {
+      for(int j=0; j<nz2; ++j) {
+        for(int i=0; i<nz1; ++i) {
+          pcr->sigma_diff(0,k,j,i) = sigma;
+          pcr->sigma_diff(1,k,j,i) = sigma;
+          pcr->sigma_diff(2,k,j,i) = sigma;
+        }
+      }
+    }
+  }
+  // Add horizontal magnetic field lines, to show streaming and diffusion
+  // along magnetic field ines
+  if (MAGNETIC_FIELDS_ENABLED) {
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie+1; ++i) {
+          pfield->b.x1f(k,j,i) = 1.0;
+        }
+      }
+    }
+    if (block_size.nx2 > 1) {
+      for (int k=ks; k<=ke; ++k) {
+        for (int j=js; j<=je+1; ++j) {
+          for (int i=is; i<=ie; ++i) {
+            pfield->b.x2f(k,j,i) = 1.0;
+          }
+        }
+      }
+    }
+    if (block_size.nx3 > 1) {
+      for (int k=ks; k<=ke+1; ++k) {
+        for (int j=js; j<=je; ++j) {
+          for (int i=is; i<=ie; ++i) {
+            pfield->b.x3f(k,j,i) = 0.0;
+          }
+        }
+      }
+    }
+
+    // set cell centerd magnetic field
+    // Add magnetic energy density to the total energy
+    pfield->CalculateCellCenteredField(pfield->b,pfield->bcc,pcoord,is,ie,js,je,ks,ke);
+
+    for(int k=ks; k<=ke; ++k) {
+      for(int j=js; j<=je; ++j) {
+        for(int i=is; i<=ie; ++i) {
+          phydro->u(IEN,k,j,i) +=
+            0.5*(SQR((pfield->bcc(IB1,k,j,i)))
+               + SQR((pfield->bcc(IB2,k,j,i)))
+               + SQR((pfield->bcc(IB3,k,j,i))));
+        }
+      }
+    }
+  }
+  return;
+}
+
+void Diffusion(MeshBlock *pmb, AthenaArray<Real> &u_cr,
+               AthenaArray<Real> &prim, AthenaArray<Real> &bcc) {
+  // set the default opacity to be a large value in the default hydro case
+  CosmicRay *pcr=pmb->pcr;
+  int kl=pmb->ks, ku=pmb->ke;
+  int jl=pmb->js, ju=pmb->je;
+  int il=pmb->is-1, iu=pmb->ie+1;
+  if (pmb->block_size.nx2 > 1) {
+    jl -= 1;
+    ju += 1;
+  }
+  if (pmb->block_size.nx3 > 1) {
+    kl -= 1;
+    ku += 1;
+  }
+
+  for(int k=kl; k<=ku; ++k) {
+    for(int j=jl; j<=ju; ++j) {
+#pragma omp simd
+      for(int i=il; i<=iu; ++i) {
+        pcr->sigma_diff(0,k,j,i) = sigma;
+        pcr->sigma_diff(1,k,j,i) = sigma;
+        pcr->sigma_diff(2,k,j,i) = sigma;
+      }
+    }
+  }
+
+  Real invlim=1.0/pcr->vmax;
+
+  // The information stored in the array
+  // b_angle is
+  // b_angle[0]=sin_theta_b
+  // b_angle[1]=cos_theta_b
+  // b_angle[2]=sin_phi_b
+  // b_angle[3]=cos_phi_b
+
+
+
+
+  if (MAGNETIC_FIELDS_ENABLED) {
+    //First, calculate B_dot_grad_Pc
+    for(int k=kl; k<=ku; ++k) {
+      for(int j=jl; j<=ju; ++j) {
+        // x component
+        pmb->pcoord->CenterWidth1(k,j,il-1,iu+1,pcr->cwidth);
+        for(int i=il; i<=iu; ++i) {
+          Real distance = 0.5*(pcr->cwidth(i-1) + pcr->cwidth(i+1))
+                          + pcr->cwidth(i);
+          Real dprdx=(u_cr(CRE,k,j,i+1) - u_cr(CRE,k,j,i-1))/3.0;
+          dprdx /= distance;
+          pcr->b_grad_pc(k,j,i) = bcc(IB1,k,j,i) * dprdx;
+        }
+        // y component
+        pmb->pcoord->CenterWidth2(k,j-1,il,iu,pcr->cwidth1);
+        pmb->pcoord->CenterWidth2(k,j,il,iu,pcr->cwidth);
+        pmb->pcoord->CenterWidth2(k,j+1,il,iu,pcr->cwidth2);
+        for(int i=il; i<=iu; ++i) {
+          Real distance = 0.5*(pcr->cwidth1(i) + pcr->cwidth2(i))
+                         + pcr->cwidth(i);
+          Real dprdy=(u_cr(CRE,k,j+1,i) - u_cr(CRE,k,j-1,i))/3.0;
+          dprdy /= distance;
+          pcr->b_grad_pc(k,j,i) += bcc(IB2,k,j,i) * dprdy;
+        }
+        // z component
+        pmb->pcoord->CenterWidth3(k-1,j,il,iu,pcr->cwidth1);
+        pmb->pcoord->CenterWidth3(k,j,il,iu,pcr->cwidth);
+        pmb->pcoord->CenterWidth3(k+1,j,il,iu,pcr->cwidth2);
+
+        for(int i=il; i<=iu; ++i) {
+          Real distance = 0.5*(pcr->cwidth1(i) + pcr->cwidth2(i))
+                          + pcr->cwidth(i);
+          Real dprdz=(u_cr(CRE,k+1,j,i) - u_cr(CRE,k-1,j,i))/3.0;
+          dprdz /= distance;
+          pcr->b_grad_pc(k,j,i) += bcc(IB3,k,j,i) * dprdz;
+
+          // now only get the sign
+          //  if (pcr->b_grad_pc(k,j,i) > TINY_NUMBER) pcr->b_grad_pc(k,j,i) = 1.0;
+          //  else if (-pcr->b_grad_pc(k,j,i) > TINY_NUMBER) pcr->b_grad_pc(k,j,i)
+          //    = -1.0;
+          //  else pcr->b_grad_pc(k,j,i) = 0.0;
+        }
+
+      // now calculate the streaming velocity
+      // streaming velocity is calculated with respect to the current coordinate
+      //  system
+      // diffusion coefficient is calculated with respect to B direction
+        for(int i=il; i<=iu; ++i) {
+          Real pb= bcc(IB1,k,j,i)*bcc(IB1,k,j,i)
+                  +bcc(IB2,k,j,i)*bcc(IB2,k,j,i)
+                  +bcc(IB3,k,j,i)*bcc(IB3,k,j,i);
+          Real inv_sqrt_rho = 1.0/std::sqrt(prim(IDN,k,j,i));
+          Real va1 = bcc(IB1,k,j,i)*inv_sqrt_rho;
+          Real va2 = bcc(IB2,k,j,i)*inv_sqrt_rho;
+          Real va3 = bcc(IB3,k,j,i)*inv_sqrt_rho;
+
+          Real va = std::sqrt(pb/prim(IDN,k,j,i));
+
+          Real dpc_sign = 0.0;
+          if (pcr->b_grad_pc(k,j,i) > TINY_NUMBER) dpc_sign = 1.0;
+          else if (-pcr->b_grad_pc(k,j,i) > TINY_NUMBER) dpc_sign = -1.0;
+
+          pcr->v_adv(0,k,j,i) = -va1 * dpc_sign;
+          pcr->v_adv(1,k,j,i) = -va2 * dpc_sign;
+          pcr->v_adv(2,k,j,i) = -va3 * dpc_sign;
+
+          // now the diffusion coefficient
+
+          if (va < TINY_NUMBER) {
+            pcr->sigma_adv(0,k,j,i) = pcr->max_opacity;
+          } else {
+            pcr->sigma_adv(0,k,j,i) = std::abs(pcr->b_grad_pc(k,j,i))
+                          /(std::sqrt(pb)* va * (1.0 + 1.0/3.0)
+                                    * invlim * u_cr(CRE,k,j,i));
+          }
+
+          pcr->sigma_adv(1,k,j,i) = pcr->max_opacity;
+          pcr->sigma_adv(2,k,j,i) = pcr->max_opacity;
+
+          // Now calculate the angles of B
+          Real bxby = std::sqrt(bcc(IB1,k,j,i)*bcc(IB1,k,j,i) +
+                           bcc(IB2,k,j,i)*bcc(IB2,k,j,i));
+          Real btot = std::sqrt(pb);
+          if (btot > TINY_NUMBER) {
+            pcr->b_angle(0,k,j,i) = bxby/btot;
+            pcr->b_angle(1,k,j,i) = bcc(IB3,k,j,i)/btot;
+          } else {
+            pcr->b_angle(0,k,j,i) = 1.0;
+            pcr->b_angle(1,k,j,i) = 0.0;
+          }
+          if (bxby > TINY_NUMBER) {
+            pcr->b_angle(2,k,j,i) = bcc(IB2,k,j,i)/bxby;
+            pcr->b_angle(3,k,j,i) = bcc(IB1,k,j,i)/bxby;
+          } else {
+            pcr->b_angle(2,k,j,i) = 0.0;
+            pcr->b_angle(3,k,j,i) = 1.0;
+          }
+        }
+      }
+    }
+  } else {
+  for(int k=kl; k<=ku; ++k) {
+    for(int j=jl; j<=ju; ++j) {
+  // x component
+      pmb->pcoord->CenterWidth1(k,j,il-1,iu+1,pcr->cwidth);
+      for(int i=il; i<=iu; ++i) {
+         Real distance = 0.5*(pcr->cwidth(i-1) + pcr->cwidth(i+1))
+                        + pcr->cwidth(i);
+         Real grad_pr=(u_cr(CRE,k,j,i+1) - u_cr(CRE,k,j,i-1))/3.0;
+         grad_pr /= distance;
+         Real va = 0.0;
+         if (va < TINY_NUMBER) {
+           pcr->sigma_adv(0,k,j,i) = pcr->max_opacity;
+           pcr->v_adv(0,k,j,i) = 0.0;
+         } else {
+           Real sigma2 = std::abs(grad_pr)/(va * (1.0 + 1.0/3.0)
+                             * invlim * u_cr(CRE,k,j,i));
+           if (std::abs(grad_pr) < TINY_NUMBER) {
+             pcr->sigma_adv(0,k,j,i) = 0.0;
+             pcr->v_adv(0,k,j,i) = 0.0;
+           } else {
+             pcr->sigma_adv(0,k,j,i) = sigma2;
+             pcr->v_adv(0,k,j,i) = -va * grad_pr/std::abs(grad_pr);
+           }
+        }
+        pcr->sigma_adv(1,k,j,i) = pcr->max_opacity;
+        pcr->sigma_adv(2,k,j,i) = pcr->max_opacity;
+
+        pcr->v_adv(1,k,j,i) = 0.0;
+        pcr->v_adv(2,k,j,i) = 0.0;
+      }
+    }
+  }
+  }
+}
diff --git a/src/pgen/rad_linearwave.cpp b/src/pgen/rad_linearwave.cpp
new file mode 100644
index 0000000000..0fe9241226
--- /dev/null
+++ b/src/pgen/rad_linearwave.cpp
@@ -0,0 +1,477 @@
+//======================================================================================
+/* Athena++ astrophysical MHD code
+ * Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+ *
+ * This program is free software: you can redistribute and/or modify it under the terms
+ * of the GNU General Public License (GPL) as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of GNU GPL in the file LICENSE included in the code
+ * distribution.  If not see <http://www.gnu.org/licenses/>.
+ *====================================================================================*/
+
+// C headers
+
+// C++ headers
+#include <algorithm>  // min
+#include <cmath>      // sqrt
+#include <fstream>
+#include <iostream>   // endl
+#include <sstream>    // stringstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
+#include "../bvals/bvals.hpp"
+#include "../coordinates/coordinates.hpp"
+#include "../eos/eos.hpp"
+#include "../field/field.hpp"
+#include "../globals.hpp"
+#include "../hydro/hydro.hpp"
+#include "../hydro/srcterms/hydro_srcterms.hpp"
+#include "../mesh/mesh.hpp"
+#include "../nr_radiation/integrators/rad_integrators.hpp"
+#include "../nr_radiation/radiation.hpp"
+#include "../parameter_input.hpp"
+
+static Real amp = 1.e-6;
+
+
+// The real and imaginary parts of the linear waves
+// used as initial condition as well as test solution to compare with
+static Real v_r, v_i, p_r, p_i, er_r, er_i, fr_r, fr_i;
+static Real omegareal, omegaimg;
+
+// AMR refinement condition
+int RefinementCondition(MeshBlock *pmb);
+
+//======================================================================================
+/*! \file rad_linearwave.cpp
+ *  \brief Linear wave test for the radiative transfer module
+ *
+ *====================================================================================*/
+
+
+//======================================================================================
+//! \fn void MeshBlock::ProblemGenerator(ParameterInput *pin)
+//  \brief beam test
+//======================================================================================
+
+void Mesh::UserWorkAfterLoop(ParameterInput *pin) {
+  if (!pin->GetOrAddBoolean("problem","compute_error",false)) return;
+
+  int totnum=5;
+  // Initialize errors to zero
+  Real * l1_err = new Real[totnum];
+  Real * max_err = new Real[totnum];
+  for (int i=0; i<totnum; ++i) {
+    l1_err[i]=0.0;
+    max_err[i]=0.0;
+  }
+  Real knum = 2.0 * PI;
+
+  for (int nb=0; nb<nblocal; ++nb) {
+    MeshBlock *pmb = my_blocks(nb);
+    if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+      pmb->pnrrad->CalculateMoment(pmb->pnrrad->ir);
+    }
+    //  Compute errors
+    for (int k=pmb->ks; k<=pmb->ke; k++) {
+      for (int j=pmb->js; j<=pmb->je; j++) {
+        for (int i=pmb->is; i<=pmb->ie; i++) {
+        // get the initial solution
+        Real const &x1 = pmb->pcoord->x1v(i);
+        Real theta = knum * x1;
+        Real delrho = amp * cos(theta);
+        Real delv = amp* (v_r * cos(theta) + v_i * sin(theta));
+        Real delp = amp * (p_r * cos(theta) + p_i * sin(theta));
+        Real der = amp * (er_r * cos(theta) + er_i * sin(theta));
+        Real dfr = amp * (fr_r * cos(theta) + fr_i * sin(theta));
+
+        Real diffrho = (pmb->phydro->u(IDN,k,j,i) - 1.0)
+                       - delrho * std::exp(-omegaimg*time);
+        Real diffvel= pmb->phydro->w(IVX,k,j,i)
+                      - delv*std::exp(-omegaimg*time);
+        Real diffpre = (pmb->phydro->w(IEN,k,j,i) - 1.0)
+                       - delp *std::exp(-omegaimg*time);
+        Real differ= (pmb->pnrrad->rad_mom(IER,k,j,i) - 1.0)
+                     - der*std::exp(-omegaimg*time);
+        Real difffr= pmb->pnrrad->rad_mom(IFR1,k,j,i)
+                     -  dfr *std::exp(-omegaimg*time);
+
+        l1_err[0] += std::abs(diffrho);
+        max_err[0] = std::max(std::abs(diffrho), max_err[0]);
+
+        l1_err[1] += std::abs(diffvel);
+        max_err[1] = std::max(std::abs(diffvel), max_err[1]);
+
+        l1_err[2] += std::abs(diffpre);
+        max_err[2] = std::max(std::abs(diffpre), max_err[2]);
+
+        l1_err[3] += std::abs(differ);
+        max_err[3] = std::max(std::abs(differ), max_err[3]);
+
+        l1_err[4] += std::abs(difffr);
+        max_err[4] = std::max(std::abs(difffr), max_err[4]);
+        }
+      }
+    }
+  }
+  // normalize errors by number of cells
+  for (int i=0; i<totnum; ++i) {
+    l1_err[i] = l1_err[i]/ static_cast<Real>(GetTotalCells());
+  }
+  Real rms_err = 0.0, max_max_over_l1=0.0;
+
+#ifdef MPI_PARALLEL
+  if (Globals::my_rank == 0) {
+    MPI_Reduce(MPI_IN_PLACE,&l1_err,totnum,MPI_ATHENA_REAL,MPI_SUM,0,
+               MPI_COMM_WORLD);
+    MPI_Reduce(MPI_IN_PLACE,&max_err,totnum,MPI_ATHENA_REAL,MPI_MAX,0,
+               MPI_COMM_WORLD);
+  } else {
+    MPI_Reduce(&l1_err,&l1_err,totnum,MPI_ATHENA_REAL,MPI_SUM,0,
+               MPI_COMM_WORLD);
+    MPI_Reduce(&max_err,&max_err,totnum,MPI_ATHENA_REAL,MPI_MAX,0,
+               MPI_COMM_WORLD);
+  }
+#endif
+
+  // only the root process outputs the data
+  if (Globals::my_rank == 0) {
+    // compute rms error
+    for (int i=0; i<totnum; ++i) {
+       rms_err += SQR(l1_err[i]);
+       max_max_over_l1 = std::max(max_max_over_l1, (max_err[i]/l1_err[i]));
+    }
+    rms_err = std::sqrt(rms_err);
+
+    // open output file and write out errors
+    std::string fname;
+    fname.assign("linearwave-errors.dat");
+    std::stringstream msg;
+    FILE *pfile;
+
+    // The file exists -- reopen the file in append mode
+    if((pfile = fopen(fname.c_str(),"r")) != NULL) {
+      if((pfile = freopen(fname.c_str(),"a",pfile)) == NULL) {
+        msg << "### FATAL ERROR in function [Mesh::UserWorkAfterLoop]"
+            << std::endl << "Error output file could not be opened" <<std::endl;
+        throw std::runtime_error(msg.str().c_str());
+      }
+
+    // The file does not exist -- open the file in write mode and add headers
+    } else {
+      if((pfile = fopen(fname.c_str(),"w")) == NULL) {
+        msg << "### FATAL ERROR in function [Mesh::UserWorkAfterLoop]"
+            << std::endl << "Error output file could not be opened" <<std::endl;
+        throw std::runtime_error(msg.str().c_str());
+      }
+      fprintf(pfile,"# Nx1  Nx2  Nx3  Ncycle  ");
+      fprintf(pfile,"RMS-L1-Error  d_L1  M1_L1  P_L1  Er_L1  Fr_L1");
+      fprintf(pfile,"  Largest-Max/L1  d_max  M1_max  P_max  Er_max  Fr_max");
+      fprintf(pfile,"\n");
+    }
+
+    // write errors
+    fprintf(pfile,"%d  %d",mesh_size.nx1,mesh_size.nx2);
+    fprintf(pfile,"  %d  %d",mesh_size.nx3,ncycle);
+    fprintf(pfile,"  %e  %e  %e  %e",rms_err,l1_err[0],l1_err[1],l1_err[2]);
+    fprintf(pfile,"  %e  %e",l1_err[3],l1_err[4]);
+    fprintf(pfile,"  %e  %e  ",max_max_over_l1,max_err[0]);
+    fprintf(pfile,"%e  %e  %e  %e",max_err[1],max_err[2],max_err[3],max_err[4]);
+    fprintf(pfile,"\n");
+    fclose(pfile);
+  }
+  delete[] l1_err;
+  delete[] max_err;
+}
+
+
+
+void MeshBlock::ProblemGenerator(ParameterInput *pin) {
+  int regime = pin->GetOrAddInteger("problem","regime",8);
+  Real sigma0;
+  switch(regime) {
+    case 1:
+      v_r= 1.2909945401100994;
+      v_i= 0.00008594287531051318;
+      p_r= 1.666666822510812;
+      p_i= 0.00021916408244003488;
+      er_r= 0.000023938771551742892;
+      er_i= 0.002595343282507656;
+      fr_r= 5.803552452185792e-6;
+      fr_i= 0.004579152314659846;
+      omegareal= 8.111557926068842;
+      omegaimg=  0.0005399950114077836;
+      sigma0 = 0.01;
+      break;
+    case 2:
+      v_r= 1.2910035706152658;
+      v_i= 0.0008586435251690642;
+      p_r= 1.666682219042423;
+      p_i= 0.002189719996892813;
+      er_r= 0.0023915640535869353;
+      er_i= 0.025910075592072698;
+      fr_r= 0.0005794967774284472;
+      fr_i= 0.045749450689271914;
+      omegareal= 8.111614666406222;
+      omegaimg=  0.00539501638144715;
+      sigma0 = 0.1;
+      break;
+    case 3:
+      v_r= 1.2917695143401924;
+      v_i= 0.00787279209054369;
+      p_r= 1.6679409990335867;
+      p_i= 0.0201456313053114;
+      er_r= 0.2181601920018504;
+      er_i= 0.22077999320680775;
+      fr_r= 0.050320573067502435;
+      fr_i= 0.4191946132602506;
+      omegareal= 8.116427232764806;
+      omegaimg=  0.04946621158978377;
+      sigma0 = 1.0;
+      break;
+    case 4:
+      v_r= 1.2920340482874326;
+      v_i= 0.009218020632790785;
+      p_r= 1.6616893859774136;
+      p_i= 0.02432338104787091;
+      er_r= 2.35100088818539;
+      er_i= 0.03457134839094067;
+      fr_r= 0.20390000836732208;
+      fr_i= 0.47734523049957645;
+      omegareal= 8.11808934857535;
+      omegaimg=  0.057918531801229335;
+      sigma0 = 10.0;
+      break;
+    case 5:
+      v_r= 1.2909668031826855;
+      v_i= 0.001045499271992685;
+      p_r= 1.6580540395720993;
+      p_i= 0.002767087292105325;
+      er_r= 2.6288579088325177;
+      er_i= 0.0005865742021802234;
+      fr_r= 0.1725541379069982;
+      fr_i= 0.05379877178913972;
+      omegareal= 8.11138364981405;
+      omegaimg=  0.006569065664451391;
+      sigma0 = 100.0;
+      break;
+    case 6:
+      v_r= 1.290810912416655;
+      v_i= 0.008591414402392937;
+      p_r= 1.6661135402797116;
+      p_i= 0.0219056348896828;
+      er_r= -0.000051245227843874086;
+      er_i= 0.0025938927144686375;
+      fr_r= -0.0001441706051503446;
+      fr_i= 0.004575470538400113;
+      omegareal= 8.110404159243402;
+      omegaimg=  0.05398144874100638;
+      sigma0 = 0.01;
+      break;
+    case 7:
+      v_r= 1.2728792327581575;
+      v_i= 0.08294778800855061;
+      p_r= 1.612846028223176;
+      p_i= 0.2082461771006987;
+      er_r= -0.0047877737151852395;
+      er_i= 0.024520349202385106;
+      fr_r= -0.013679926554110659;
+      fr_i= 0.04217262048775258;
+      omegareal= 7.997736093080079;
+      omegaimg=  0.5211763228783722;
+      sigma0 = 0.1;
+      break;
+    case 8:
+      v_r= 1.0183496112257058;
+      v_i= 0.1121215711780068;
+      p_r= 1.0220380692314723;
+      p_i= 0.18993018794365163;
+      er_r= -0.026018127896336885;
+      er_i= 0.12095401964915764;
+      fr_r= -0.10566859341556321;
+      fr_i= 0.030196412832965945;
+      omegareal= 6.398479314825398;
+      omegaimg=  0.7044806086435424;
+      sigma0 = 1.0;
+      break;
+    case 9:
+      v_r= 1.2092908080104792;
+      v_i= 0.25516371383473957;
+      p_r= 1.1686533807826482;
+      p_i= 0.28223448062039036;
+      er_r= 0.6544229491473885;
+      er_i= 1.0653905194884314;
+      fr_r= -0.04918496753828805;
+      fr_i= 0.17766949176109614;
+      omegareal= 7.598198236998773;
+      omegaimg=  1.6032408976918122;
+      sigma0 = 10.0;
+      break;
+    case 10:
+      v_r= 1.3993585014150265;
+      v_i= 0.03450304866280985;
+      p_r= 1.4211576732442506;
+      p_i= 0.04441546638644912;
+      er_r= 1.6841247487314883;
+      er_i= 0.17440949185811563;
+      fr_r= 0.18330452762195915;
+      fr_i= 0.038269237092134034;
+      omegareal= 8.792428775567739;
+      omegaimg=  0.21678904841106914;
+      sigma0 = 100.0;
+      break;
+    case 11:
+      v_r= 1.271935073054169;
+      v_i= 0.08266820679991217;
+      p_r= 1.6110935636801302;
+      p_i= 0.20757539251245596;
+      er_r= -0.0006798060846963105;
+      er_i= 0.0023794592885675314;
+      fr_r= -0.0014238164476813107;
+      fr_i= 0.004188456336444625;
+      omegareal= 7.991803762700347;
+      omegaimg=  0.5194196623360917;
+      sigma0 = 0.01;
+      break;
+    case 12:
+      v_r= 1.014892795882639;
+      v_i= 0.08976614261743401;
+      p_r= 1.0234931616949725;
+      p_i= 0.15900654619222715;
+      er_r= -0.003705180537531039;
+      er_i= 0.007369404109744249;
+      fr_r= -0.010447571816544906;
+      fr_i= 0.0022692464424416106;
+      omegareal= 6.376759503452208;
+      omegaimg=  0.5640173083760487;
+      sigma0 = 0.1;
+      break;
+    case 13:
+      v_r= 0.9977393625226653;
+      v_i= 0.1324385410980396;
+      p_r= 0.9963589910920785;
+      p_i= 0.03350714894270625;
+      er_r= -0.009348064816826119;
+      er_i= 0.07057816396845434;
+      fr_r= -0.011965869992714585;
+      fr_i=  0.006087909287739512;
+      omegareal= 6.268981302997137;
+      omegaimg= 0.8321358955315021;
+      sigma0 = 1.0;
+      break;
+    case 14:
+      v_r= 2.1806327929670344;
+      v_i= 0.6503652227748439;
+      p_r= 1.2398074098916425;
+      p_i= 0.24885283255343593;
+      er_r= 0.9567316753829218;
+      er_i= 0.9851131720168348;
+      fr_r= 0.12816968396550954;
+      fr_i= 0.2810151431629262;
+      omegareal= 13.701319925124455;
+      omegaimg=  4.086365212039477;
+      sigma0 = 10.0;
+      break;
+    case 15:
+      v_r= 2.2904573528686267;
+      v_i= 0.0604384248330597;
+      p_r= 1.344412710622572;
+      p_i= 0.025960902814908177;
+      er_r= 1.3776131567434966;
+      er_i= 0.10314648118800326;
+      fr_r= 0.3038178476543796;
+      fr_i= 0.032551125836846495;
+      omegareal= 14.391367986265605;
+      omegaimg=  0.37974582290015857;
+      sigma0 = 100.0;
+      break;
+    default:
+    std::stringstream msg;
+    msg << "### FATAL ERROR in ProblemGenerator" << std::endl
+        << "Invalid Linear wave mode "<< regime << " is specified" << std::endl;
+    throw std::runtime_error(msg.str().c_str());
+  }
+
+  Real gamma = peos->GetGamma();
+  Real knum = 2.0 * PI;
+  Real rho0 = 1.0, p0 = 1.0;
+  //Real e0 = p0/(gamma-1.0);
+
+  // Initialize hydro variable
+  for (int k=0; k<ncells3; ++k) {
+    for (int j=0; j<ncells2; ++j) {
+      for (int i=0; i<ncells1; ++i) {
+        Real const &x1 = pcoord->x1v(i);
+        // Real const &x2 = pcoord->x2v(j);
+        // Real const &x3 = pcoord->x3v(k);
+        Real theta = knum * x1;
+        Real delv = amp* (v_r * cos(theta) + v_i * sin(theta));
+        Real delp = amp * (p_r * cos(theta) + p_i * sin(theta));
+
+        phydro->u(IDN,k,j,i) = rho0 + amp * cos(theta);
+        phydro->u(IM1,k,j,i) = delv;
+        phydro->u(IM2,k,j,i) = 0.0;
+        phydro->u(IM3,k,j,i) = 0.0;
+        if (NON_BAROTROPIC_EOS) {
+          phydro->u(IEN,k,j,i) = (p0 + delp)/(gamma-1.0);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM1,k,j,i))/phydro->u(IDN,k,j,i);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM2,k,j,i))/phydro->u(IDN,k,j,i);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM3,k,j,i))/phydro->u(IDN,k,j,i);
+        }
+        if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+          Real der = amp * (er_r * cos(theta) + er_i * sin(theta));
+          Real dfr = amp * (fr_r * cos(theta) + fr_i * sin(theta));
+
+          Real jr = (1.0+der);
+          Real hr = dfr;
+          for (int ifr=0; ifr<pnrrad->nfreq; ++ifr) {
+            for (int n=0; n<pnrrad->nang; ++n) {
+               Real const &weight = pnrrad->wmu(n);
+               Real const &miux = pnrrad->mu(0,k,j,i,n);
+               pnrrad->ir(k,j,i,ifr*pnrrad->nang+n) = (jr/(4.0*weight)
+                                                       + hr/(4.0*weight*miux));
+            }
+            pnrrad->sigma_s(k,j,i,ifr) = 0.0;
+            pnrrad->sigma_a(k,j,i,ifr) = sigma0;
+            pnrrad->sigma_pe(k,j,i,ifr) = sigma0;
+            pnrrad->sigma_p(k,j,i,ifr) = sigma0;
+          }
+        }
+      }
+    }
+  }
+  return;
+}
+
+void Mesh::InitUserMeshData(ParameterInput *pin) {
+  if (adaptive) {
+    EnrollUserRefinementCondition(RefinementCondition);
+  }
+}
+
+
+// refinement condition: density curvature
+int RefinementCondition(MeshBlock *pmb) {
+  AthenaArray<Real> const &w = pmb->phydro->w;
+  Real dmax = 0.0, dmin = 2.0;  // max and min densities
+  for (int k=pmb->ks; k<=pmb->ke; k++) {
+    for (int j=pmb->js; j<=pmb->je; j++) {
+      for (int i=pmb->is; i<=pmb->ie; i++) {
+        if (w(IDN,k,j,i) > dmax) dmax = w(IDN,k,j,i);
+        if (w(IDN,k,j,i) < dmin) dmin = w(IDN,k,j,i);
+      }
+    }
+  }
+  // refine : delta rho > 0.9*amp
+  if (dmax-1.0 > 0.9*amp) return 1;
+  return -1;
+}
diff --git a/src/pgen/thermal_multigroup.cpp b/src/pgen/thermal_multigroup.cpp
new file mode 100644
index 0000000000..0e28d851f9
--- /dev/null
+++ b/src/pgen/thermal_multigroup.cpp
@@ -0,0 +1,234 @@
+//======================================================================================
+/* Athena++ astrophysical MHD code
+ * Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+ *
+ * This program is free software: you can redistribute and/or modify it under the terms
+ * of the GNU General Public License (GPL) as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of GNU GPL in the file LICENSE included in the code
+ * distribution.  If not see <http://www.gnu.org/licenses/>.
+ *====================================================================================*/
+
+// C headers
+
+// C++ headers
+#include <algorithm>  // min
+#include <cmath>      // sqrt
+#include <fstream>
+#include <iostream>   // endl
+#include <sstream>    // stringstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
+#include "../bvals/bvals.hpp"
+#include "../coordinates/coordinates.hpp"
+#include "../eos/eos.hpp"
+#include "../field/field.hpp"
+#include "../hydro/hydro.hpp"
+#include "../hydro/srcterms/hydro_srcterms.hpp"
+#include "../mesh/mesh.hpp"
+#include "../nr_radiation/integrators/rad_integrators.hpp"
+#include "../nr_radiation/radiation.hpp"
+#include "../parameter_input.hpp"
+
+//======================================================================================
+//! \fn void MeshBlock::ProblemGenerator(ParameterInput *pin)
+//  \brief
+//======================================================================================
+void MeshBlock::ProblemGenerator(ParameterInput *pin) {
+  Real tgas, er1, er2, er3;
+  Real sigma1,sigma2,sigma3;
+  Real kappa_es, bd_flag;
+
+  er1 = pin->GetOrAddReal("problem","er_1",10.0);
+  er2 = pin->GetOrAddReal("problem","er_2",20.0);
+  er3 = pin->GetOrAddReal("problem","er_3",30.0);
+  tgas = pin->GetOrAddReal("problem","tgas",1.0);
+  sigma1 = pin->GetOrAddReal("problem","sigma_1",100.0);
+  sigma2 = pin->GetOrAddReal("problem","sigma_2",200.0);
+  sigma3 = pin->GetOrAddReal("problem","sigma_3",300.0);
+  kappa_es = pin->GetOrAddReal("problem","kappa_es",0.0);
+  bd_flag = pin->GetOrAddInteger("problem","black_body",0.0);
+
+  Real gamma = peos->GetGamma();
+  Real tr_ini = std::pow(er1,0.25);
+
+  // Initialize hydro variable
+  for (int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      for (int i=is; i<=ie; ++i) {
+        phydro->u(IDN,k,j,i) = 1.0;
+        phydro->u(IM1,k,j,i) = 0.0;
+        phydro->u(IM2,k,j,i) = 0.0;
+        phydro->u(IM3,k,j,i) = 0.0;
+        if (NON_BAROTROPIC_EOS) {
+          phydro->u(IEN,k,j,i) = tgas/(gamma-1.0);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM1,k,j,i))/phydro->u(IDN,k,j,i);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM2,k,j,i))/phydro->u(IDN,k,j,i);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM3,k,j,i))/phydro->u(IDN,k,j,i);
+        }
+      }
+    }
+  }
+
+  // Now initialize opacity and specific intensity
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    int nfreq = pnrrad->nfreq;
+    int nang = pnrrad->nang;
+    AthenaArray<Real> ir_cm;
+    ir_cm.NewAthenaArray(pnrrad->n_fre_ang);
+
+    pnrrad->kappa_es=kappa_es;
+
+    Real *ir_lab;
+
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          for (int ifr=0; ifr<pnrrad->nfreq; ++ifr) {
+            ir_lab = &(pnrrad->ir(k,j,i,ifr*nang));
+            if (bd_flag == 0) {
+              Real er_ini=1.0;
+              if (ifr == 0) {
+                er_ini = er1;
+              } else if (ifr == 1) {
+                er_ini = er2;
+              } else {
+                er_ini = er3;
+              }
+              for (int n=0; n<pnrrad->nang; n++) {
+                ir_lab[n] = er_ini;
+              }
+            } else {
+              Real emission = er1;
+              // Initialize with blackbody spectrum
+              if (ifr == nfreq-1) {
+                emission *= (1.0-pnrrad->FitIntPlanckFunc(pnrrad->nu_grid(ifr)/tr_ini));
+              } else {
+                emission *= pnrrad->IntPlanckFunc(pnrrad->nu_grid(ifr)/tr_ini,
+                                                  pnrrad->nu_grid(ifr+1)/tr_ini);
+              }
+              for (int n=0; n<pnrrad->nang; n++) {
+                ir_lab[n] = emission;
+              }
+            }
+          }
+        }
+      }
+    }
+
+    for (int k=0; k<ncells3; ++k) {
+      for (int j=0; j<ncells2; ++j) {
+        for (int i=0; i<ncells1; ++i) {
+          for (int ifr=0; ifr < nfreq; ++ifr) {
+            if (ifr == 0) {
+              pnrrad->sigma_s(k,j,i,ifr) = kappa_es;
+              pnrrad->sigma_a(k,j,i,ifr) = sigma1;
+              pnrrad->sigma_pe(k,j,i,ifr) = sigma1;
+              pnrrad->sigma_p(k,j,i,ifr) = sigma1;
+            } else if (ifr == 1) {
+              pnrrad->sigma_s(k,j,i,ifr) = kappa_es;
+              pnrrad->sigma_a(k,j,i,ifr) = sigma2;
+              pnrrad->sigma_pe(k,j,i,ifr) = sigma2;
+              pnrrad->sigma_p(k,j,i,ifr) = sigma2;
+            } else {
+              pnrrad->sigma_s(k,j,i,ifr) = kappa_es;
+              pnrrad->sigma_a(k,j,i,ifr) = sigma3;
+              pnrrad->sigma_pe(k,j,i,ifr) = sigma3;
+              pnrrad->sigma_p(k,j,i,ifr) = sigma3;
+            }
+          }
+        }
+      }
+    }
+    ir_cm.DeleteAthenaArray();
+  }
+  return;
+}
+
+
+// calculate the sum of radiation energy density across the mesh for each frequency groups
+void Mesh::UserWorkAfterLoop(ParameterInput *pin) {
+  MeshBlock *pmb = my_blocks(0);
+  int  totnum = pmb->pnrrad->nfreq+1;
+  AthenaArray<Real> sum_var;
+  sum_var.NewAthenaArray(totnum);
+
+  for (int nb=0; nb<nblocal; ++nb) {
+    pmb = my_blocks(nb);
+    if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED)
+      pmb->pnrrad->CalculateMoment(pmb->pnrrad->ir);
+    //  Compute the sum
+    for (int k=pmb->ks; k<=pmb->ke; k++) {
+      for (int j=pmb->js; j<=pmb->je; j++) {
+        for (int i=pmb->is; i<=pmb->ie; i++) {
+          // get the initial solution
+          Real dx = pmb->pcoord->x1f(i+1) - pmb->pcoord->x1f(i);
+
+          sum_var(0) += dx*pmb->phydro->w(IPR,k,j,i)/pmb->phydro->w(IDN,k,j,i);
+
+          for (int ifr=0; ifr<pmb->pnrrad->nfreq; ++ifr) {
+            sum_var(ifr+1) += pmb->pnrrad->rad_mom_nu(ifr*13+IER,k,j,i) * dx;
+          }
+        }
+      }
+    }
+  }
+
+#ifdef MPI_PARALLEL
+  if (Globals::my_rank == 0) {
+    MPI_Reduce(MPI_IN_PLACE,&sum_var(0),totnum,MPI_ATHENA_REAL,MPI_SUM,0,
+               MPI_COMM_WORLD);
+  } else {
+    MPI_Reduce(&sum_var(0),&sum_var(0),totnum,MPI_ATHENA_REAL,MPI_SUM,0,
+               MPI_COMM_WORLD);
+  }
+#endif
+
+  // only the root process outputs the data
+  if (Globals::my_rank == 0) {
+    // divide by the mesh size
+    for (int i=0; i<totnum; ++i) {
+      sum_var(i) /= (mesh_size.x1max - mesh_size.x1min);
+    }
+    // open output file and write out errors
+    std::string fname;
+    fname.assign("Averaged_quantity.dat");
+    std::stringstream msg;
+    FILE *pfile;
+
+    // The file exists -- reopen the file in append mode
+    if ((pfile = fopen(fname.c_str(),"r")) != NULL) {
+      if ((pfile = freopen(fname.c_str(),"a",pfile)) == NULL) {
+        msg << "### FATAL ERROR in function [Mesh::UserWorkAfterLoop]"
+            << std::endl << "Error output file could not be opened" <<std::endl;
+        throw std::runtime_error(msg.str().c_str());
+      }
+
+      // The file does not exist -- open the file in write mode and add headers
+    } else {
+      if ((pfile = fopen(fname.c_str(),"w")) == NULL) {
+        msg << "### FATAL ERROR in function [Mesh::UserWorkAfterLoop]"
+            << std::endl << "Error output file could not be opened" <<std::endl;
+        throw std::runtime_error(msg.str().c_str());
+      }
+      fprintf(pfile,"# T  Er  ");
+      fprintf(pfile,"\n");
+    }
+
+    // write errors
+    for (int n=0; n<totnum; ++n)
+      fprintf(pfile,"%e  ",sum_var(n));
+    fprintf(pfile,"\n");
+    fclose(pfile);
+  }
+  sum_var.DeleteAthenaArray();
+}
diff --git a/src/pgen/thermal_relaxation.cpp b/src/pgen/thermal_relaxation.cpp
new file mode 100644
index 0000000000..706391aeac
--- /dev/null
+++ b/src/pgen/thermal_relaxation.cpp
@@ -0,0 +1,104 @@
+//======================================================================================
+/* Athena++ astrophysical MHD code
+ * Copyright (C) 2014 James M. Stone  <jmstone@princeton.edu>
+ *
+ * This program is free software: you can redistribute and/or modify it under the terms
+ * of the GNU General Public License (GPL) as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of GNU GPL in the file LICENSE included in the code
+ * distribution.  If not see <http://www.gnu.org/licenses/>.
+ *====================================================================================*/
+
+// C headers
+
+// C++ headers
+#include <algorithm>  // min
+#include <cmath>      // sqrt
+#include <fstream>
+#include <iostream>   // endl
+#include <sstream>    // stringstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
+#include "../bvals/bvals.hpp"
+#include "../coordinates/coordinates.hpp"
+#include "../eos/eos.hpp"
+#include "../field/field.hpp"
+#include "../hydro/hydro.hpp"
+#include "../hydro/srcterms/hydro_srcterms.hpp"
+#include "../mesh/mesh.hpp"
+#include "../nr_radiation/integrators/rad_integrators.hpp"
+#include "../nr_radiation/radiation.hpp"
+#include "../parameter_input.hpp"
+
+//======================================================================================
+//! \fn void MeshBlock::ProblemGenerator(ParameterInput *pin)
+//  \brief
+//======================================================================================
+void MeshBlock::ProblemGenerator(ParameterInput *pin) {
+  Real tgas, er, sigma;
+
+  er = pin->GetOrAddReal("problem","er",10.0);
+  tgas = pin->GetOrAddReal("problem","tgas",1.0);
+  sigma = pin->GetOrAddReal("problem","sigma",100.0);
+  Real gamma = peos->GetGamma();
+
+  // Initialize hydro variable
+  for(int k=ks; k<=ke; ++k) {
+    for (int j=js; j<=je; ++j) {
+      for (int i=is; i<=ie; ++i) {
+        phydro->u(IDN,k,j,i) = 1.0;
+        phydro->u(IM1,k,j,i) = 0.0;
+        phydro->u(IM2,k,j,i) = 0.0;
+        phydro->u(IM3,k,j,i) = 0.0;
+        if (NON_BAROTROPIC_EOS) {
+          phydro->u(IEN,k,j,i) = tgas/(gamma-1.0);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM1,k,j,i))/phydro->u(IDN,k,j,i);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM2,k,j,i))/phydro->u(IDN,k,j,i);
+          phydro->u(IEN,k,j,i) += 0.5*SQR(phydro->u(IM3,k,j,i))/phydro->u(IDN,k,j,i);
+        }
+      }
+    }
+  }
+  // Now initialize opacity and specific intensity
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    int nfreq = pnrrad->nfreq;
+    //int nang = pnrrad->nang;
+    AthenaArray<Real> ir_cm;
+    ir_cm.NewAthenaArray(pnrrad->n_fre_ang);
+    Real *ir_lab;
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          ir_lab = &(pnrrad->ir(k,j,i,0));
+          for (int n=0; n<pnrrad->n_fre_ang; n++) {
+             ir_lab[n] = er;
+          }
+        }
+      }
+    }
+
+    for (int k=0; k<ncells3; ++k) {
+      for (int j=0; j<ncells2; ++j) {
+        for (int i=0; i<ncells1; ++i) {
+          for (int ifr=0; ifr < nfreq; ++ifr) {
+            pnrrad->sigma_s(k,j,i,ifr) = 0.0;
+            pnrrad->sigma_a(k,j,i,ifr) = sigma;
+            pnrrad->sigma_pe(k,j,i,ifr) = sigma;
+            pnrrad->sigma_p(k,j,i,ifr) = sigma;
+          }
+        }
+      }
+    }
+    ir_cm.DeleteAthenaArray();
+  }
+  return;
+}
diff --git a/src/reconstruct/dc_simple.cpp b/src/reconstruct/dc_simple.cpp
index b7e77156e7..2cfb35d750 100644
--- a/src/reconstruct/dc_simple.cpp
+++ b/src/reconstruct/dc_simple.cpp
@@ -39,6 +39,25 @@ void Reconstruction::DonorCellX1(const int k, const int j, const int il, const i
   return;
 }
 
+// function for arrays with different order
+void Reconstruction::DonorCellX1(const int k, const int j, const int il, const int iu,
+                                 AthenaArray<Real> &q, const int array_order,
+                                 AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+  if (array_order < 0) {
+    const int nu = q.GetDim1() - 1;
+    // compute L/R states for each variable
+    for (int i=il; i<=iu; ++i) {
+      Real *qn = &(q(k,j,i,0));
+      Real *qln = &(ql(i+1,0));
+      Real *qrn = &(qr(i,0));
+      for (int n=0; n<=nu; ++n) {
+        qln[n] =  qrn[n] = qn[n];
+      }
+    }
+  }
+  return;
+}
+
 //----------------------------------------------------------------------------------------
 //! \fn Reconstruction::DonorCellX2(const int k, const int j,
 //!                              const int il, const int iu,
@@ -60,6 +79,25 @@ void Reconstruction::DonorCellX2(const int k, const int j, const int il, const i
   return;
 }
 
+// function for arrys with different order
+void Reconstruction::DonorCellX2(const int k, const int j, const int il, const int iu,
+                                 AthenaArray<Real> &q, const int array_order,
+                                 AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+  if (array_order < 0) {
+    const int nu = q.GetDim1() - 1;
+    // compute L/R states for each variable
+    for (int i=il; i<=iu; ++i) {
+      Real *qln = &(ql(i,0));
+      Real *qrn = &(qr(i,0));
+      Real *qn = &(q(k,j,i,0));
+      for (int n=0; n<=nu; ++n) {
+      qln[n] = qrn[n] = qn[n];
+    }
+  }
+  }
+  return;
+}
+
 //----------------------------------------------------------------------------------------
 //! \fn Reconstruction::DonorCellX3(const int k, const int j,
 //!                              const int il, const int iu,
@@ -80,3 +118,52 @@ void Reconstruction::DonorCellX3(const int k, const int j, const int il, const i
   }
   return;
 }
+
+
+
+void Reconstruction::DonorCellX3(const int k, const int j, const int il, const int iu,
+                                 AthenaArray<Real> &q, const int array_order,
+                                 AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+  const int nu = q.GetDim1() - 1;
+  // compute L/R states for each variable
+  for (int i=il; i<=iu; ++i) {
+    Real *qln = &(ql(i,0));
+    Real *qrn = &(qr(i,0));
+    Real *qn = &(q(k,j,i,0));
+    for (int n=0; n<=nu; ++n) {
+      qln[n] = qrn[n] = qn[n];
+    }
+  }
+  return;
+}
+
+void Reconstruction::DonorCellZeta(
+    NRRadiation *prad, const int zs, const int ze,
+    AthenaArray<Real> &q,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+
+    Real *qln = &(ql(zs+1));
+    Real *qrn = &(qr(zs));
+    Real *qn = &(q(zs));
+    for (int n=0; n<=ze-zs; ++n) {
+      qln[n] =  qrn[n] = qn[n];
+    }
+
+  return;
+}
+
+
+void Reconstruction::DonorCellPsi(
+    NRRadiation *prad, const int ps, const int pe,
+    AthenaArray<Real> &q,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+
+    Real *qln = &(ql(ps+1));
+    Real *qrn = &(qr(ps));
+    Real *qn = &(q(ps));
+    for (int m=0; m<=pe-ps; ++m) {
+      // renamed dw* -> dq* from plm.cpp
+      qln[m] =  qrn[m] = qn[m];
+    }
+  return;
+}
diff --git a/src/reconstruct/plm_simple.cpp b/src/reconstruct/plm_simple.cpp
index 8dd7fd1859..ede0b3907b 100644
--- a/src/reconstruct/plm_simple.cpp
+++ b/src/reconstruct/plm_simple.cpp
@@ -22,6 +22,7 @@
 #include "../athena_arrays.hpp"
 #include "../coordinates/coordinates.hpp"
 #include "../eos/eos.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "reconstruction.hpp"
 
 //----------------------------------------------------------------------------------------
@@ -104,6 +105,96 @@ void Reconstruction::PiecewiseLinearX1(
   return;
 }
 
+
+
+void Reconstruction::PiecewiseLinearX1(
+    const int k, const int j, const int il, const int iu,
+    AthenaArray<Real> &q, const int array_order,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+  Coordinates *pco = pmy_block_->pcoord;
+  if (array_order < 0) {
+    // set work arrays to shallow copies of scratch arrays
+    AthenaArray<Real> &qc = scr1_in2_, &dql = scr2_in2_, &dqr = scr3_in2_,
+                     &dqm = scr4_in2_;
+    const int nu = q.GetDim1() - 1;
+
+    // compute L/R slopes for each variable
+    for (int i=il; i<=iu; ++i) {
+      Real *dqln = &(dql(i,0));
+      Real *dqrn = &(dqr(i,0));
+      Real *qn = &(q(k,j,i,0));
+      Real *q1n = &(q(k,j,i+1,0));
+      Real *q2n = &(q(k,j,i-1,0));
+      Real *qcn = &(qc(i,0));
+      for (int n=0; n<=nu; ++n) {
+        dqln[n] = (qn[n] - q2n[n]);
+        dqrn[n] = (q1n[n] - qn[n]);
+        qcn[n] = qn[n];
+        // renamed dw* -> dq* from plm.cpp
+      }
+    }
+
+    // Apply simplified van Leer (VL) limiter expression for a Cartesian-like coordinate
+    // with uniform mesh spacing
+    if (uniform[X1DIR] && !curvilinear[X1DIR]) {
+      for (int i=il; i<=iu; ++i) {
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int n=0; n<=nu; ++n) {
+          Real dq2 = dql(i,n)*dqr(i,n);
+          dqm(i,n) = 2.0*dq2/(dql(i,n) + dqr(i,n));
+          if (dq2 <= 0.0) dqm(i,n) = 0.0;
+        }
+      }
+      // coordinate with nonuniform mesh spacing or for any curvilinear coordinate spacing
+    } else {
+      for (int i=il; i<=iu; ++i) {
+        // variables independent n
+        // cf, cb -> 2 (uniform Cartesian mesh / original VL value) w/ vanishing curvature
+        // (may not exactly hold for nonuniform meshes, but converges w/ smooth
+        // nonuniformity)
+        Real cf = pco->dx1v(i  )/(pco->x1f(i+1) - pco->x1v(i)); // (Mignone eq 33)
+        Real cb = pco->dx1v(i-1)/(pco->x1v(i  ) - pco->x1f(i));
+        Real dxF = pco->dx1f(i)/pco->dx1v(i);
+        Real dxB = pco->dx1f(i)/pco->dx1v(i-1);
+        Real *dqrn = &(dqr(i,0));
+        Real *dqln = &(dql(i,0));
+        Real *dqmn = &(dqm(i,0));
+        for (int n=0; n<=nu; ++n) {
+          Real dqF =  dqrn[n]*dxF;
+          Real dqB =  dqln[n]*dxB;
+          Real dq2 = dqF*dqB;
+          // (modified) VL limiter (Mignone eq 37)
+          // (dQ^F term from eq 31 pulled into eq 37, then multiply by (dQ^F/dQ^F)^2)
+          dqmn[n] = (dq2*(cf*dqB + cb*dqF)/
+                     (SQR(dqB) + SQR(dqF) + dq2*(cf + cb - 2.0)));
+          if (dq2 <= 0.0) dqmn[n] = 0.0; // ---> no concern for divide-by-0 in above line
+          // Real v = dqB/dqF;
+          // monotoniced central (MC) limiter (Mignone eq 38)
+          // (std::min calls should avoid issue if divide-by-zero causes v=Inf)
+          //dqm(n,i) = dqF*std::max(0.0, std::min(0.5*(1.0 + v), std::min(cf, cb*v)));
+        }
+      }
+    }
+
+    // compute ql_(i+1/2) and qr_(i-1/2) using limited slopes
+    for (int i=il; i<=iu; ++i) {
+      Real ratio_l=(pco->x1f(i+1) - pco->x1v(i))/pco->dx1f(i);
+      Real ratio_r=(pco->x1v(i  ) - pco->x1f(i))/pco->dx1f(i);
+      Real *qln = &(ql(i+1,0));
+      Real *qrn = &(qr(i,0));
+      Real *qcn = &(qc(i,0));
+      Real *dqmn = &(dqm(i,0));
+      for (int n=0; n<=nu; ++n) {
+        // Mignone equation 30
+        qln[n] = qcn[n] + ratio_l*dqmn[n];
+        qrn[n] = qcn[n] - ratio_r*dqmn[n];
+      }
+    }
+  }
+  return;
+}
+
+
 //----------------------------------------------------------------------------------------
 //! \fn Reconstruction::PiecewiseLinearX2(const int k, const int j,
 //!                              const int il, const int iu,
@@ -183,6 +274,96 @@ void Reconstruction::PiecewiseLinearX2(
   }
 }
 
+
+void Reconstruction::PiecewiseLinearX2(
+    const int k, const int j, const int il, const int iu,
+    AthenaArray<Real> &q, const int array_order,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+  Coordinates *pco = pmy_block_->pcoord;
+  if (array_order < 0) {
+    // set work arrays to shallow copies of scratch arrays
+    AthenaArray<Real> &qc = scr1_in2_, &dql = scr2_in2_,
+                     &dqr = scr3_in2_, &dqm = scr4_in2_;
+    const int nu = q.GetDim1() - 1;
+
+    // compute L/R slopes for each variable
+    for (int i=il; i<=iu; ++i) {
+      Real *dqln = &(dql(i,0));
+      Real *dqrn = &(dqr(i,0));
+      Real *qcn = &(qc(i,0));
+      Real *qn  = &(q(k,j  ,i,0));
+      Real *q1n = &(q(k,j+1,i,0));
+      Real *q2n = &(q(k,j-1,i,0));
+      for (int n=0; n<=nu; ++n) {
+        // renamed dw* -> dq* from plm.cpp
+        dqln[n] = (qn[n] - q2n[n]);
+        dqrn[n] = (q1n[n] - qn[n]);
+        qcn[n] = qn[n];
+      }
+    }
+
+    // Apply simplified van Leer (VL) limiter expression for a Cartesian-like coordinate
+    // with uniform mesh spacing
+    if (uniform[X2DIR] && !curvilinear[X2DIR]) {
+      for (int i=il; i<=iu; ++i) {
+        Real *dqln = &(dql(i,0));
+        Real *dqrn = &(dqr(i,0));
+        Real *dqmn = &(dqm(i,0));
+        for (int n=0; n<=nu; ++n) {
+          Real dq2 = dqln[n]*dqrn[n];
+          dqmn[n] = 2.0*dq2/(dqln[n] + dqrn[n]);
+          if (dq2 <= 0.0) dqmn[n] = 0.0;
+        }
+      }
+
+      // Apply general VL limiter expression w/ the Mignone correction for a
+      // Cartesian-like coordinate sys with nonuniform mesh spacing or for
+      // any curvilinear coordinate spacing
+    } else {
+      Real cf = pco->dx2v(j  )/(pco->x2f(j+1) - pco->x2v(j));
+      Real cb = pco->dx2v(j-1)/(pco->x2v(j  ) - pco->x2f(j));
+      Real dxF = pco->dx2f(j)/pco->dx2v(j); // dimensionless, not technically a dx qty
+      Real dxB = pco->dx2f(j)/pco->dx2v(j-1);
+
+      for (int i=il; i<=iu; ++i) {
+        Real *dqrn = &(dqr(i,0));
+        Real *dqln = &(dql(i,0));
+        Real *dqmn = &(dqm(i,0));
+        for (int n=0; n<=nu; ++n) {
+          Real dqF =  dqrn[n]*dxF;
+          Real dqB =  dqln[n]*dxB;
+          Real dq2 = dqF*dqB;
+          // (modified) VL limiter (Mignone eq 37)
+          dqmn[n] = (dq2*(cf*dqB + cb*dqF)/
+                     (SQR(dqB) + SQR(dqF) + dq2*(cf + cb - 2.0)));
+          if (dq2 <= 0.0) dqmn[n] = 0.0; // ---> no concern for divide-by-0 in above line
+
+          // Real v = dqB/dqF;
+          // // monotoniced central (MC) limiter (Mignone eq 38)
+          // // (std::min calls should avoid issue if divide-by-zero causes v=Inf)
+          // dqm(n,i) = dqF*std::max(0.0, std::min(0.5*(1.0 + v), std::min(cf, cb*v)));
+        }
+      }
+    }
+
+    // compute ql_(j+1/2) and qr_(j-1/2) using limited slopes
+    // dimensionless, not technically a "dx" quantity
+    Real dxp = (pco->x2f(j+1) - pco->x2v(j))/pco->dx2f(j);
+    Real dxm = (pco->x2v(j  ) - pco->x2f(j))/pco->dx2f(j);
+    for (int i=il; i<=iu; ++i) {
+      Real *qln = &(ql(i,0));
+      Real *qrn = &(qr(i,0));
+      Real *dqmn = &(dqm(i,0));
+      Real *qcn = &(qc(i,0));
+      for (int n=0; n<=nu; ++n) {
+        qln[n] = qcn[n] + dxp*dqmn[n];
+        qrn[n] = qcn[n] - dxm*dqmn[n];
+      }
+    }
+  }// End array_order
+}
+
+
 //----------------------------------------------------------------------------------------
 //! \fn Reconstruction::PiecewiseLinearX3(const int k, const int j,
 //!                              const int il, const int iu,
@@ -255,3 +436,180 @@ void Reconstruction::PiecewiseLinearX3(
   }
   return;
 }
+
+
+
+void Reconstruction::PiecewiseLinearX3(
+    const int k, const int j, const int il, const int iu,
+    AthenaArray<Real> &q, const int array_order,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+  Coordinates *pco = pmy_block_->pcoord;
+  if (array_order < 0) {
+    // set work arrays to shallow copies of scratch arrays
+    AthenaArray<Real> &qc = scr1_in2_, &dql = scr2_in2_, &dqr = scr3_in2_,
+                     &dqm = scr4_in2_;
+    const int nu = q.GetDim1() - 1;
+
+    // compute L/R slopes for each variable
+    for (int i=il; i<=iu; ++i) {
+      Real *dqln = &(dql(i,0));
+      Real *dqrn = &(dqr(i,0));
+      Real *qcn = &(qc(i,0));
+      Real *qn  = &(q(k,j  ,i,0));
+      Real *q1n = &(q(k+1,j,i,0));
+      Real *q2n = &(q(k-1,j,i,0));
+      for (int n=0; n<=nu; ++n) {
+        // renamed dw* -> dq* from plm.cpp
+        dqln[n] = (qn[n] - q2n[n]);
+        dqrn[n] = (q1n[n] - qn[n]);
+        qcn[n] = qn[n];
+      }
+    }
+
+    // Apply simplified van Leer (VL) limiter expression for a Cartesian-like coordinate
+    // with uniform mesh spacing
+    if (uniform[X3DIR]) {
+      for (int i=il; i<=iu; ++i) {
+        Real *dqln = &(dql(i,0));
+        Real *dqrn = &(dqr(i,0));
+        Real *dqmn = &(dqm(i,0));
+        for (int n=0; n<=nu; ++n) {
+          Real dq2 = dqln[n]*dqrn[n];
+          dqmn[n] = 2.0*dq2/(dqln[n] + dqrn[n]);
+          if (dq2 <= 0.0) dqmn[n] = 0.0;
+        }
+      }
+
+      // Apply original VL limiter's general expression for a Cartesian-like
+      // coordinate system with nonuniform mesh spacing
+    } else {
+      Real dxF = pco->dx3f(k)/pco->dx3v(k);
+      Real dxB = pco->dx3f(k)/pco->dx3v(k-1);
+
+      for (int i=il; i<=iu; ++i) {
+        Real *dqmn = &(dqm(i,0));
+        Real *dqrn = &(dqr(i,0));
+        Real *dqln = &(dql(i,0));
+        for (int n=0; n<=nu; ++n) {
+          Real dqF =  dqrn[n]*dxF;
+          Real dqB =  dqln[n]*dxB;
+          Real dq2 = dqF*dqB;
+          // original VL limiter (Mignone eq 36)
+          dqmn[n] = 2.0*dq2/(dqF + dqB);
+          // dq2 > 0 ---> dqF, dqB are nonzero and have the same sign ----> no risk for
+          // (dqF + dqB) = 0 cancellation causing a divide-by-0 in the above line
+          if (dq2 <= 0.0) dqmn[n] = 0.0;
+        }
+      }
+    }
+
+    // compute ql_(k+1/2) and qr_(k-1/2) using limited slopes
+    Real dxp = (pco->x3f(k+1) - pco->x3v(k))/pco->dx3f(k);
+    Real dxm = (pco->x3v(k  ) - pco->x3f(k))/pco->dx3f(k);
+
+    for (int i=il; i<=iu; ++i) {
+      Real *qln = &(ql(i,0));
+      Real *qrn = &(qr(i,0));
+      Real *qcn = &(qc(i,0));
+      Real *dqmn = &(dqm(i,0));
+      for (int n=0; n<=nu; ++n) {
+        qln[n] = qcn[n] + dxp*dqmn[n];
+        qrn[n] = qcn[n] - dxm*dqmn[n];
+      }
+    }
+  }
+  return;
+}
+
+// reconstruction in the angular space
+// q, ql, and qr are 2D array in zeta, psi
+// zs, ze are starting/end index for zeta
+// ps, pe are starting/end index for psi
+// q is always in the order (zeta, psi)
+// q, ql and qr can change
+void Reconstruction::PiecewiseLinearZeta(
+    NRRadiation *prad, const int zs, const int ze,
+    AthenaArray<Real> &q,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+
+  // set work arrays to shallow copies of scratch arrays
+  AthenaArray<Real> &qc = scr1_nn_, &dql = scr2_nn_, &dqr = scr3_nn_,
+                   &dqm = scr4_nn_;
+#pragma omp simd
+  for (int n=zs; n<=ze; ++n) {
+    // renamed dw* -> dq* from plm.cpp
+    dql(n) = (q(n) - q(n-1));
+    dqr(n) = (q(n+1) - q(n));
+    qc(n) = q(n);
+  }
+  for (int n=zs; n<=ze; ++n) {
+    Real cf = prad->dzeta_v(n)/(prad->zeta_f_full(n+1)
+                                - prad->zeta_v_full(n)); // (Mignone eq 33)
+    Real cb = prad->dzeta_v(n-1)/(prad->zeta_v_full(n) - prad->zeta_f_full(n));
+    Real dqF =  dqr(n)*prad->dzeta_f(n)/prad->dzeta_v(n);
+    Real dqB =  dql(n)*prad->dzeta_f(n)/prad->dzeta_v(n-1);
+    Real dq2 = dqF*dqB;
+    // (modified) VL limiter (Mignone eq 37)
+    // (dQ^F term from eq 31 pulled into eq 37, then multiply by (dQ^F/dQ^F)^2)
+    dqm(n) = (dq2*(cf*dqB + cb*dqF)/
+              (SQR(dqB) + SQR(dqF) + dq2*(cf + cb - 2.0)));
+    if (dq2 <= 0.0) dqm(n) = 0.0; // ---> no concern for divide-by-0 in above line
+    // Real v = dqB/dqF;
+    // monotoniced central (MC) limiter (Mignone eq 38)
+    // (std::min calls should avoid issue if divide-by-zero causes v=Inf)
+    //dqm(n,i) = dqF*std::max(0.0, std::min(0.5*(1.0 + v), std::min(cf, cb*v)));
+  }
+  for (int n=zs; n<=ze; ++n) {
+    Real ratio_l=(prad->zeta_f_full(n+1) - prad->zeta_v_full(n))/prad->dzeta_f(n);
+    Real ratio_r=(prad->zeta_v_full(n) - prad->zeta_f_full(n))/prad->dzeta_f(n);
+    // Mignone equation 30
+    ql(n+1) = qc(n) + ratio_l*dqm(n);
+    qr(n  ) = qc(n) - ratio_r*dqm(n);
+  }
+  return;
+}
+
+void Reconstruction::PiecewiseLinearPsi(
+    NRRadiation *prad, const int ps, const int pe,
+    AthenaArray<Real> &q,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+  // set work arrays to shallow copies of scratch arrays
+  AthenaArray<Real> &qc = scr1_nn_, &dql = scr2_nn_, &dqr = scr3_nn_,
+                   &dqm = scr4_nn_;
+#pragma omp simd
+  for (int m=ps; m<=pe; ++m) {
+    // renamed dw* -> dq* from plm.cpp
+    dql(m) = (q(m) - q(m-1));
+    dqr(m) = (q(m+1) - q(m));
+    qc(m) = q(m);
+  }
+
+#pragma omp simd
+  for (int m=ps; m<=pe; ++m) {
+    Real cf = prad->dpsi_v(m)/(prad->psi_f_full(m+1)
+                               - prad->psi_v_full(m)); // (Mignone eq 33)
+    Real cb = prad->dpsi_v(m-1)/(prad->psi_v_full(m) - prad->psi_f_full(m));
+    Real dqF =  dqr(m)*prad->dpsi_f(m)/prad->dpsi_v(m);
+    Real dqB =  dql(m)*prad->dpsi_f(m)/prad->dpsi_v(m-1);
+    Real dq2 = dqF*dqB;
+    // (modified) VL limiter (Mignone eq 37)
+    // (dQ^F term from eq 31 pulled into eq 37, then multiply by (dQ^F/dQ^F)^2)
+    dqm(m) = (dq2*(cf*dqB + cb*dqF)/
+              (SQR(dqB) + SQR(dqF) + dq2*(cf + cb - 2.0)));
+    if (dq2 <= 0.0) dqm(m) = 0.0; // ---> no concern for divide-by-0 in above line
+    // Real v = dqB/dqF;
+    // monotoniced central (MC) limiter (Mignone eq 38)
+    // (std::min calls should avoid issue if divide-by-zero causes v=Inf)
+    //dqm(n,i) = dqF*std::max(0.0, std::min(0.5*(1.0 + v), std::min(cf, cb*v)));
+  }
+
+#pragma omp simd
+  for (int m=ps; m<=pe; ++m) {
+    Real ratio_l=(prad->psi_f_full(m+1) - prad->psi_v_full(m))/prad->dpsi_f(m);
+    Real ratio_r=(prad->psi_v_full(m) - prad->psi_f_full(m))/prad->dpsi_f(m);
+    // Mignone equation 30
+    ql(m+1) = qc(m) + ratio_l*dqm(m);
+    qr(m  ) = qc(m) - ratio_r*dqm(m);
+  }
+  return;
+}
diff --git a/src/reconstruct/ppm_simple.cpp b/src/reconstruct/ppm_simple.cpp
index d4941b785c..3d2bcc680d 100644
--- a/src/reconstruct/ppm_simple.cpp
+++ b/src/reconstruct/ppm_simple.cpp
@@ -296,6 +296,261 @@ void Reconstruction::PiecewiseParabolicX1(
   return;
 }
 
+
+void Reconstruction::PiecewiseParabolicX1(
+    const int k, const int j, const int il, const int iu,
+    const AthenaArray<Real> &q, const int array_order,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+
+  if (array_order < 0) {
+    const int nu = q.GetDim1() - 1;
+    // CS08 constant used in second derivative limiter, >1 , independent of h
+    const Real C2 = 1.25;
+    // TODO(felker): renumber scratch array references; not using 2x from ppm.cpp
+    // bx (MHD) and wc (characteristic projection)
+
+    // set work arrays used for primitive/characterstic cell-averages to scratch
+    AthenaArray<Real> &q_im2 = scr2_in_, &q_im1 = scr3_in_,
+                    &q_i = scr4_in_, &q_ip1 = scr5_in_, &q_ip2 = scr6_in_,
+                    &qr_imh = scr7_in_, &ql_iph = scr8_in_;
+
+    // set work PPM work arrays to shallow copies of scratch arrays:
+    AthenaArray<Real> &dd = scr02_i_, &dd_im1 = scr03_i_, &dd_ip1 = scr04_i_,
+                   &dph = scr05_i_, &dph_ip1 = scr06_i_;
+
+    AthenaArray<Real> &d2qc_im1 = scr07_i_, &d2qc = scr08_i_, &d2qc_ip1 = scr09_i_,
+                        &d2qf = scr10_i_;
+
+    AthenaArray<Real> &qplus = scr11_i_, &qminus = scr12_i_, &dqf_plus = scr13_i_,
+                &dqf_minus = scr14_i_;
+
+    // cache the x1-sliced primitive states for eigensystem calculation
+    for (int i=il; i<=iu; ++i) {
+#pragma omp simd
+      for (int n=0; n<=nu; ++n) {
+        q_i  (n,i) = q(k,j,i  ,n);
+        q_im2(n,i) = q(k,j,i-2,n);
+        q_im1(n,i) = q(k,j,i-1,n);
+        q_ip1(n,i) = q(k,j,i+1,n);
+        q_ip2(n,i) = q(k,j,i+2,n);
+      }
+    }
+
+  //--- Step 1. --------------------------------------------------------------------------
+  // Reconstruct interface averages <a>_{i-1/2} and <a>_{i+1/2}
+    for (int n=0; n<=nu; ++n) {
+    // Compute average slope in i-1, i, i+1 zones
+#pragma omp simd simdlen(SIMD_WIDTH)
+      for (int i=il; i<=iu; ++i) {
+      // nonuniform or uniform Cartesian-like coord reconstruction from volume averages:
+        if (!curvilinear[X1DIR]) {
+          Real qa = (q_i(n,i) - q_im1(n,i));
+          Real qb = (q_ip1(n,i) - q_i(n,i));
+          dd_im1(i) = c1i(i-1)*qa + c2i(i-1)*(q_im1(n,i) - q_im2(n,i));
+          dd    (i) = c1i(i  )*qb + c2i(i  )*qa;
+          dd_ip1(i) = c1i(i+1)*(q_ip2(n,i) - q_ip1(n,i)) + c2i(i+1)*qb;
+
+        // Approximate interface average at i-1/2 and i+1/2 using PPM (CW eq 1.6)
+        // KGF: group the biased stencil quantities to preserve FP symmetry
+          dph(i) = (c3i(i)*q_im1(n,i) + c4i(i)*q_i(n,i)) +
+                 (c5i(i)*dd_im1(i) + c6i(i)*dd(i));
+          dph_ip1(i) = (c3i(i+1)*q_i(n,i) + c4i(i+1)*q_ip1(n,i)) +
+                     (c5i(i+1)*dd(i) + c6i(i+1)*dd_ip1(i) );
+        } else { // radial coordinate
+          dph(i) = c1i(i)*q_im2(n,i) + c2i(i)*q_im1(n,i) + c3i(i)*q_i(n,i)
+                 + c4i(i)*q_ip1(n,i);
+          dph_ip1(i) = c1i(i+1)*q_im1(n,i) + c2i(i+1)*q_i(n,i) + c3i(i+1)*q_ip1(n,i)
+                     + c4i(i+1)*q_ip2(n,i);
+        }
+      }
+
+    //--- Step 2a. -----------------------------------------------------------------------
+    // Uniform Cartesian-like coordinate: limit interpolated interface states (CD 4.3.1)
+      if (uniform[X1DIR] && !curvilinear[X1DIR]) {
+      // approximate second derivative at interfaces for smooth extrema preservation
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu+1; ++i) {
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          d2qc_im1(i) = q_im2(n,i) + q_i  (n,i) - 2.0*q_im1(n,i);
+          d2qc    (i) = q_im1(n,i) + q_ip1(n,i) - 2.0*q_i  (n,i); // (CD eq 85a) (no 1/2)
+          d2qc_ip1(i) = q_i  (n,i) + q_ip2(n,i) - 2.0*q_ip1(n,i);
+        }
+
+      // i-1/2
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=(iu+1); ++i) {
+          Real qa_tmp = dph(i) - q_im1(n,i); // (CD eq 84a)
+          Real qb_tmp = q_i(n,i) - dph(i);     // (CD eq 84b)
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          Real qa = 3.0*(q_im1(n,i) + q_i(n,i)  - 2.0*dph(i));  // (CD eq 85b)
+          Real qb = d2qc_im1(i);    // (CD eq 85a) (no 1/2)
+          Real qc = d2qc(i);   // (CD eq 85c) (no 1/2)
+          Real qd = 0.0;
+          if (SIGN(qa) == SIGN(qb) && SIGN(qa) == SIGN(qc)) {
+            qd = SIGN(qa)* std::min(C2*std::abs(qb),
+                                  std::min(C2*std::abs(qc), std::abs(qa)));
+          }
+          Real dph_tmp = 0.5*(q_im1(n,i) + q_i(n,i)) - qd/6.0;
+          if (qa_tmp*qb_tmp < 0.0) { // Local extrema detected at i-1/2 face
+            dph(i) = dph_tmp;
+          }
+        }
+
+      // i+1/2
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=(iu+1); ++i) {
+          Real qa_tmp = dph_ip1(i) - q_i(n,i);       // (CD eq 84a)
+          Real qb_tmp = q_ip1(n,i) - dph_ip1(i);   // (CD eq 84b)
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          Real qa = 3.0*(q_i(n,i) + q_ip1(n,i) - 2.0*dph_ip1(i));  // (CD eq 85b)
+          Real qb = d2qc(i);            // (CD eq 85a) (no 1/2)
+          Real qc = d2qc_ip1(i);   // (CD eq 85c) (no 1/2)
+          Real qd = 0.0;
+          if (SIGN(qa) == SIGN(qb) && SIGN(qa) == SIGN(qc)) {
+            qd = SIGN(qa)* std::min(C2*std::abs(qb),
+                                  std::min(C2*std::abs(qc), std::abs(qa)));
+          }
+          Real dphip1_tmp = 0.5*(q_i(n,i) + q_ip1(n,i)) - qd/6.0;
+          if (qa_tmp*qb_tmp < 0.0) { // Local extrema detected at i+1/2 face
+            dph_ip1(i) = dphip1_tmp;
+          }
+        }
+
+#pragma omp simd
+        for (int i=il; i<=iu; ++i) {
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          d2qf(i) = 6.0*(dph(i) + dph_ip1(i) - 2.0*q_i(n,i)); // a6 coefficient * -2
+        }
+
+      //--- Step 2b. ---------------------------------------------------------------------
+      // Nonuniform and/or curvilinear coordinate: apply strict monotonicity constraints
+      // (Mignone eq 45)
+      } else {
+#pragma omp simd
+        for (int i=il; i<=iu; ++i) {
+          dph    (i) = std::min(dph    (i), std::max(q_i(n,i),q_im1(n,i)));
+          dph_ip1(i) = std::min(dph_ip1(i), std::max(q_i(n,i),q_ip1(n,i)));
+
+          dph    (i) = std::max(dph    (i), std::min(q_i(n,i),q_im1(n,i)));
+          dph_ip1(i) = std::max(dph_ip1(i), std::min(q_i(n,i),q_ip1(n,i)));
+        }
+      }
+
+    // Cache Riemann states for both non-/uniform limiters
+#pragma omp simd
+      for (int i=il; i<=iu; ++i) {
+        qminus(i) = dph(i  );
+        qplus(i) =  dph_ip1(i );
+      }
+
+    //--- Step 3. ------------------------------------------------------------------------
+    // Compute cell-centered difference stencils (MC section 2.4.1)
+#pragma omp simd
+      for (int i=il; i<=iu; ++i) {
+        dqf_minus(i) = q_i(n,i) - qminus(i); // (CS eq 25) = -dQ^- in Mignone's notation
+        dqf_plus(i)  = qplus(i) - q_i(n,i);
+      }
+
+    //--- Step 4a. -----------------------------------------------------------------------
+    // For uniform Cartesian-like coordinate: apply CS limiters to parabolic interpolant
+      if (uniform[X1DIR] && !curvilinear[X1DIR]) {
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu; ++i) {
+          Real qa_tmp = dqf_minus(i)*dqf_plus(i);
+          Real qb_tmp = (q_ip1(n,i) - q_i(n,i))*(q_i(n,i) - q_im1(n,i));
+
+          Real qa = d2qc_im1(i);
+          Real qb = d2qc(i);
+          Real qc = d2qc_ip1(i);
+          Real qd = d2qf(i);
+          Real qe = 0.0;
+          if (SIGN(qa) == SIGN(qb) && SIGN(qa) == SIGN(qc) && SIGN(qa) == SIGN(qd)) {
+          // Extrema is smooth
+            qe = SIGN(qd)* std::min(std::min(C2*std::abs(qa), C2*std::abs(qb)),
+                                  std::min(C2*std::abs(qc),
+                                           std::abs(qd))); // (CS eq 22)
+          }
+
+        // Check if 2nd derivative is close to roundoff error
+          qa = std::max(std::abs(q_im1(n,i)), std::abs(q_im2(n,i)));
+          qb = std::max(std::max(std::abs(q_i(n,i)), std::abs(q_ip1(n,i))),
+                      std::abs(q_ip2(n,i)));
+
+          Real rho = 0.0;
+          if (std::abs(qd) > (1.0e-12)*std::max(qa,qb)) {
+          // Limiter is not sensitive to roundoff. Use limited ratio (MC eq 27)
+            rho = qe/qd;
+          }
+
+          Real tmp_m = q_i(n,i) - rho*dqf_minus(i);
+          Real tmp_p = q_i(n,i) + rho*dqf_plus(i);
+          Real tmp2_m = q_i(n,i) - 2.0*dqf_plus(i);
+          Real tmp2_p = q_i(n,i) + 2.0*dqf_minus(i);
+
+        // Check for local extrema
+          if ((qa_tmp <= 0.0 || qb_tmp <= 0.0)) {
+          // Check if relative change in limited 2nd deriv is > roundoff
+            if (rho <= (1.0 - (1.0e-12))) {
+            // Limit smooth extrema
+              qminus(i) = tmp_m;// (CS eq 23)
+              qplus(i) = tmp_p;
+            }
+          // No extrema detected
+          } else {
+          // Overshoot i-1/2,R / i,(-) state
+            if (std::abs(dqf_minus(i)) >= 2.0*std::abs(dqf_plus(i))) {
+              qminus(i) = tmp2_m;
+            }
+          // Overshoot i+1/2,L / i,(+) state
+            if (std::abs(dqf_plus(i)) >= 2.0*std::abs(dqf_minus(i))) {
+              qplus(i) = tmp2_p;
+            }
+          }
+        }
+
+      //--- Step 4b. ---------------------------------------------------------------------
+      // Nonuniform and/or curvilinear coordinate: apply Mignone limiters to parabolic
+      // interpolant. Note, the Mignone limiter does not check for cell-averaged extrema:
+      } else {
+        for (int i=il; i<=iu; ++i) {
+          Real qa = dqf_minus(i)*dqf_plus(i);
+          if (qa <= 0.0) { // Local extrema detected
+            qminus(i) = q_i(n,i);
+            qplus(i) = q_i(n,i);
+          } else { // No extrema detected
+          // Overshoot i-1/2,R / i,(-) state
+            if (std::abs(dqf_minus(i)) >= hplus_ratio_i(i)*std::abs(dqf_plus(i))) {
+              qminus(i) = q_i(n,i) - hplus_ratio_i(i)*dqf_plus(i);
+            }
+          // Overshoot i+1/2,L / i,(+) state
+            if (std::abs(dqf_plus(i)) >= hminus_ratio_i(i)*std::abs(dqf_minus(i))) {
+              qplus(i) = q_i(n,i) + hminus_ratio_i(i)*dqf_minus(i);
+            }
+          }
+        }
+      }
+    //--- Step 5. ------------------------------------------------------------------------
+    // Convert limited cell-centered values to interface-centered L/R Riemann states
+    // both L/R values defined over [il,iu]
+#pragma omp simd
+      for (int i=il; i<=iu; ++i) {
+        ql_iph(n,i ) = qplus(i);
+        qr_imh(n,i ) = qminus(i);
+      }
+    }
+
+    // compute ql_(i+1/2) and qr_(i-1/2)
+    for (int i=il; i<=iu; ++i) {
+#pragma omp simd
+      for (int n=0; n<=nu; ++n) {
+        ql(i+1,n) = ql_iph(n,i);
+        qr(i  ,n) = qr_imh(n,i);
+      }
+    }
+  }
+  return;
+}
+
 //-------------------------------------------------------------------------------------
 //! \fn Reconstruction::PiecewiseParabolicX2(const int k, const int j,
 //!                              const int il, const int iu,
@@ -553,6 +808,259 @@ void Reconstruction::PiecewiseParabolicX2(
   return;
 }
 
+
+void Reconstruction::PiecewiseParabolicX2(
+    const int k, const int j, const int il, const int iu,
+    const AthenaArray<Real> &q, const int array_order,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+  if (array_order < 0) {
+    const int nu = q.GetDim1() - 1;
+  // CS08 constant used in second derivative limiter, >1 , independent of h
+    const Real C2 = 1.25;
+
+  // set work arrays used for primitive/characterstic cell-averages to scratch
+    AthenaArray<Real> &q_jm2 = scr2_in_, &q_jm1 = scr3_in_,
+                    &q_j = scr4_in_, &q_jp1 = scr5_in_, &q_jp2 = scr6_in_,
+                    &qr_jmh = scr7_in_, &ql_jph = scr8_in_;
+
+  // set work PPM work arrays to shallow copies of scratch arrays:
+    AthenaArray<Real> &dd = scr02_i_, &dd_jm1 = scr03_i_, &dd_jp1 = scr04_i_,
+                   &dph = scr05_i_, &dph_jp1 = scr06_i_;
+
+    AthenaArray<Real> &d2qc_jm1 = scr07_i_, &d2qc = scr08_i_, &d2qc_jp1 = scr09_i_,
+                        &d2qf = scr10_i_;
+
+    AthenaArray<Real> &qplus = scr11_i_, &qminus = scr12_i_, &dqf_plus = scr13_i_,
+                &dqf_minus = scr14_i_;
+
+  // cache the x1-sliced primitive states for eigensystem calculation
+    for (int i=il; i<=iu; ++i) {
+#pragma omp simd
+      for (int n=0; n<=nu; ++n) {
+        q_j  (n,i) = q(k,j  ,i,n);
+        q_jm2(n,i) = q(k,j-2,i,n);
+        q_jm1(n,i) = q(k,j-1,i,n);
+        q_jp1(n,i) = q(k,j+1,i,n);
+        q_jp2(n,i) = q(k,j+2,i,n);
+      }
+    }
+
+  //--- Step 1. ------------------------------------------------------------------------
+  // Reconstruct interface averages <a>_{j-1/2} and <a>_{j+1/2}
+    for (int n=0; n<=nu; ++n) {
+    // Compute average slope in j-1, j, j+1 zones
+#pragma omp simd simdlen(SIMD_WIDTH)
+      for (int i=il; i<=iu; ++i) {
+      // nonuniform or uniform Cartesian-like coord reconstruction from volume averages:
+        if (!curvilinear[X2DIR]) {
+          Real qa = (q_j(n,i) - q_jm1(n,i));
+          Real qb = (q_jp1(n,i) - q_j(n,i));
+          dd_jm1(i) = c1j(j-1)*qa + c2j(j-1)*(q_jm1(n,i) - q_jm2(n,i));
+          dd    (i) = c1j(j  )*qb + c2j(j  )*qa;
+          dd_jp1(i) = c1j(j+1)*(q_jp2(n,i) - q_jp1(n,i)) + c2j(j+1)*qb;
+
+        // Approximate interface average at j-1/2 and j+1/2 using PPM (CW eq 1.6)
+        // KGF: group the biased stencil quantities to preserve FP symmetry
+          dph(i) = (c3j(j)*q_jm1(n,i) + c4j(j)*q_j(n,i)) +
+                   (c5j(j)*dd_jm1(i) + c6j(j)*dd(i));
+          dph_jp1(i) = (c3j(j+1)*q_j(n,i) + c4j(j+1)*q_jp1(n,i)) +
+                     (c5j(j+1)*dd(i) + c6j(j+1)*dd_jp1(i));
+        } else { // spherical-polar coordinates meridional x2 direction:
+        // TODO(felker): implement weight calculation in reconstruction.cpp
+          dph(i) = c1j(j)*q_jm2(n,i) + c2j(j)*q_jm1(n,i) + c3j(j)*q_j(n,i)
+                   + c4j(j)*q_jp1(n,i);
+          dph_jp1(i) = c1j(j+1)*q_jm1(n,i) + c2j(j+1)*q_j(n,i) + c3j(j+1)*q_jp1(n,i)
+                     + c4j(j+1)*q_jp2(n,i);
+        }
+      }
+
+    //--- Step 2a. ---------------------------------------------------------------------
+    // Uniform Cartesian-like coordinate: limit interpolated interface states (CD 4.3.1)
+      if (uniform[X2DIR] && !curvilinear[X2DIR]) {
+      // approximate second derivative at interfaces for smooth extrema preservation
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu; ++i) {
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          d2qc_jm1(i) = q_jm2(n,i) + q_j  (n,i) - 2.0*q_jm1(n,i);
+          d2qc    (i) = q_jm1(n,i) + q_jp1(n,i) - 2.0*q_j  (n,i); //(CD eq 85a) (no 1/2)
+          d2qc_jp1(i) = q_j  (n,i) + q_jp2(n,i) - 2.0*q_jp1(n,i);
+        }
+
+      // j-1/2
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu; ++i) {
+          Real qa_tmp = dph(i) - q_jm1(n,i); // (CD eq 84a)
+          Real qb_tmp = q_j(n,i) - dph(i);     // (CD eq 84b)
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          Real qa = 3.0*(q_jm1(n,i) + q_j(n,i) - 2.0*dph(i));  // (CD eq 85b)
+          Real qb = d2qc_jm1(i);    // (CD eq 85a) (no 1/2)
+          Real qc = d2qc(i);   // (CD eq 85c) (no 1/2)
+          Real qd = 0.0;
+          if (SIGN(qa) == SIGN(qb) && SIGN(qa) == SIGN(qc)) {
+            qd = SIGN(qa)* std::min(C2*std::abs(qb),
+                                  std::min(C2*std::abs(qc), std::abs(qa)));
+          }
+          Real dph_tmp = 0.5*(q_jm1(n,i) + q_j(n,i)) - qd/6.0;
+          if (qa_tmp*qb_tmp < 0.0) { // Local extrema detected at j-1/2 face
+            dph(i) = dph_tmp;
+          }
+        }
+      // j+1/2
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu; ++i) {
+          Real qa_tmp = dph_jp1(i) - q_j(n,i);       // (CD eq 84a)
+          Real qb_tmp = q_jp1(n,i) - dph_jp1(i);   // (CD eq 84b)
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          Real qa = 3.0*(q_j(n,i) + q_jp1(n,i)  - 2.0*dph_jp1(i));  // (CD eq 85b)
+          Real qb = d2qc(i);            // (CD eq 85a) (no 1/2)
+          Real qc = d2qc_jp1(i);   // (CD eq 85c) (no 1/2)
+          Real qd = 0.0;
+          if (SIGN(qa) == SIGN(qb) && SIGN(qa) == SIGN(qc)) {
+            qd = SIGN(qa)* std::min(C2*std::abs(qb),
+                                  std::min(C2*std::abs(qc), std::abs(qa)));
+          }
+          Real dphjp1_tmp = 0.5*(q_j(n,i) + q_jp1(n,i)) - qd/6.0;
+          if (qa_tmp*qb_tmp < 0.0) { // Local extrema detected at j+1/2 face
+            dph_jp1(i) = dphjp1_tmp;
+          }
+        }
+
+#pragma omp simd
+        for (int i=il; i<=iu; ++i) {
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          d2qf(i) = 6.0*(dph(i) + dph_jp1(i) - 2.0*q_j(n,i)); // a6 coefficient * -2
+        }
+
+      //--- Step 2b. -------------------------------------------------------------------
+      // Nonuniform and/or curvilinear coordinate: apply strict monotonicity constraints
+      // (Mignone eq 45)
+      } else {
+#pragma omp simd
+        for (int i=il; i<=iu; ++i) {
+          dph    (i) = std::min(dph    (i), std::max(q_j(n,i),q_jm1(n,i)));
+          dph_jp1(i) = std::min(dph_jp1(i), std::max(q_j(n,i),q_jp1(n,i)));
+
+          dph    (i) = std::max(dph    (i), std::min(q_j(n,i),q_jm1(n,i)));
+          dph_jp1(i) = std::max(dph_jp1(i), std::min(q_j(n,i),q_jp1(n,i)));
+        }
+      }
+
+    // Cache Riemann states for both non-/uniform limiters
+#pragma omp simd
+      for (int i=il; i<=iu; ++i) {
+        qminus(i) = dph(i  );
+        qplus(i) =  dph_jp1(i );
+      }
+
+    //--- Step 3. ----------------------------------------------------------------------
+    // Compute cell-centered difference stencils (MC section 2.4.1)
+#pragma omp simd
+      for (int i=il; i<=iu; ++i) {
+        dqf_minus(i) = q_j(n,i) - qminus(i); // (CS eq 25)
+        dqf_plus(i)  = qplus(i) - q_j(n,i);
+      }
+
+    //--- Step 4a. ---------------------------------------------------------------------
+    // For uniform Cartesian-like coordinate: apply CS limiters to parabolic interpolant
+     if (uniform[X2DIR] && !curvilinear[X2DIR]) {
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu; ++i) {
+          Real qa_tmp = dqf_minus(i)*dqf_plus(i);
+          Real qb_tmp = (q_jp1(n,i) - q_j(n,i))*(q_j(n,i) - q_jm1(n,i));
+
+          Real qa = d2qc_jm1(i);
+          Real qb = d2qc(i);
+          Real qc = d2qc_jp1(i);
+          Real qd = d2qf(i);
+          Real qe = 0.0;
+          if (SIGN(qa) == SIGN(qb) && SIGN(qa) == SIGN(qc) && SIGN(qa) == SIGN(qd)) {
+            // Extrema is smooth
+            qe = SIGN(qd)* std::min(std::min(C2*std::abs(qa), C2*std::abs(qb)),
+                                  std::min(C2*std::abs(qc), std::abs(qd)));// (CS eq 22)
+          }
+
+        // Check if 2nd derivative is close to roundoff error
+          qa = std::max(std::abs(q_jm1(n,i)), std::abs(q_jm2(n,i)));
+          qb = std::max(std::max(std::abs(q_j(n,i)),
+                               std::abs(q_jp1(n,i))), std::abs(q_jp2(n,i)));
+
+          Real rho = 0.0;
+          if (std::abs(qd) > (1.0e-12)*std::max(qa, qb)) {
+          // Limiter is not sensitive to roundoff. Use limited ratio (MC eq 27)
+            rho = qe/qd;
+          }
+
+          Real tmp_m = q_j(n,i) - rho*dqf_minus(i);
+          Real tmp_p = q_j(n,i) + rho*dqf_plus(i);
+          Real tmp2_m = q_j(n,i) - 2.0*dqf_plus(i);
+          Real tmp2_p = q_j(n,i) + 2.0*dqf_minus(i);
+
+        // Check if relative change in limited 2nd deriv is > roundoff
+        // Check for local extrema
+          if ((qa_tmp <= 0.0 || qb_tmp <= 0.0)) {
+            if (rho <= (1.0 - (1.0e-12))) {
+            // Limit smooth extrema
+              qminus(i) = tmp_m; // (CS eq 23)
+              qplus(i) = tmp_p;
+            }
+          // No extrema detected
+          } else {
+           // Overshoot j-1/2,R / j,(-) state
+            if (std::abs(dqf_minus(i)) >= 2.0*std::abs(dqf_plus(i))) {
+              qminus(i) = tmp2_m;
+            }
+          // Overshoot j+1/2,L / j,(+) state
+            if (std::abs(dqf_plus(i)) >= 2.0*std::abs(dqf_minus(i))) {
+              qplus(i) = tmp2_p;
+            }
+          }
+        }
+
+      //--- Step 4b. -------------------------------------------------------------------
+      // Nonuniform and/or curvilinear coordinate: apply Mignone limiters to parabolic
+      // interpolant. Note, the Mignone limiter does not check for cell-averaged extrema:
+      } else {
+        for (int i=il; i<=iu; ++i) {
+          Real qa = dqf_minus(i)*dqf_plus(i);
+          if (qa <= 0.0) { // Local extrema detected
+            qminus(i) = q_j(n,i);
+            qplus(i) = q_j(n,i);
+          } else { // No extrema detected
+          // Overshoot j-1/2,R / j,(-) state
+            if (std::abs(dqf_minus(i)) >= hplus_ratio_j(j)*std::abs(dqf_plus(i))) {
+              qminus(i) = q_j(n,i) - hplus_ratio_j(j)*dqf_plus(i);
+            }
+          // Overshoot j+1/2,L / j,(+) state
+            if (std::abs(dqf_plus(i)) >= hminus_ratio_j(j)*std::abs(dqf_minus(i))) {
+              qplus(i) = q_j(n,i) + hminus_ratio_j(j)*dqf_minus(i);
+            }
+          }
+        }
+      }
+    //--- Step 5. ----------------------------------------------------------------------
+    // Convert limited cell-centered values to interface-centered L/R Riemann states
+    // both L/R values defined over [il,iu]
+#pragma omp simd
+      for (int i=il; i<=iu; ++i) {
+        ql_jph(n,i ) = qplus(i);
+        qr_jmh(n,i ) = qminus(i);
+      }
+    } // end char PPM loop over =nu
+
+
+  // compute ql_(j+1/2) and qr_(j-1/2)
+    for (int i=il; i<=iu; ++i) {
+#pragma omp simd
+      for (int n=0; n<=nu; ++n) {
+        ql(i,n) = ql_jph(n,i);
+        qr(i,n) = qr_jmh(n,i);
+      }
+    }
+  }
+  return;
+}
+
+
 //----------------------------------------------------------------------------------------
 //! \fn Reconstruction::PiecewiseParabolicX3(const int k, const int j,
 //!                              const int il, const int iu,
@@ -801,3 +1309,248 @@ void Reconstruction::PiecewiseParabolicX3(
   }
   return;
 }
+
+
+
+void Reconstruction::PiecewiseParabolicX3(
+    const int k, const int j, const int il, const int iu,
+    const AthenaArray<Real> &q, const int array_order,
+    AthenaArray<Real> &ql, AthenaArray<Real> &qr) {
+
+  if (array_order < 0) {
+    const int nu = q.GetDim1() - 1;
+    // CS08 constant used in second derivative limiter, >1 , independent of h
+    const Real C2 = 1.25;
+    // set work arrays used for primitive/characterstic cell-averages to scratch
+    AthenaArray<Real> &q_km2 = scr2_in_, &q_km1 = scr3_in_,
+                    &q_k = scr4_in_, &q_kp1 = scr5_in_, &q_kp2 = scr6_in_,
+                    &qr_kmh = scr7_in_, &ql_kph = scr8_in_;
+
+    // set work PPM work arrays to shallow copies of scratch arrays:
+    AthenaArray<Real> &dd = scr02_i_, &dd_km1 = scr03_i_, &dd_kp1 = scr04_i_,
+                   &dph = scr05_i_, &dph_kp1 = scr06_i_;
+
+    AthenaArray<Real> &d2qc_km1 = scr07_i_, &d2qc = scr08_i_,
+                    &d2qc_kp1 = scr09_i_, &d2qf = scr10_i_;
+
+    AthenaArray<Real> &qplus = scr11_i_, &qminus = scr12_i_, &dqf_plus = scr13_i_,
+                &dqf_minus = scr14_i_;
+
+  // cache the x1-sliced primitive states for eigensystem calculation
+    for (int i=il; i<=iu; ++i) {
+//#pragma omp simd
+      for (int n=0; n<=nu; ++n) {
+        q_k  (n,i) = q(k  ,j,i,n);
+        q_km2(n,i) = q(k-2,j,i,n);
+        q_km1(n,i) = q(k-1,j,i,n);
+        q_kp1(n,i) = q(k+1,j,i,n);
+        q_kp2(n,i) = q(k+2,j,i,n);
+      }
+    }
+
+  //--- Step 1. -------------------------------------------------------------------------
+  // Reconstruct interface averages <a>_{k-1/2} and <a>_{k+1/2}
+    for (int n=0; n<=nu; ++n) {
+    // Compute average slope in k-1, k, k+1 zones
+#pragma omp simd simdlen(SIMD_WIDTH)
+      for (int i=il; i<=iu; ++i) {
+      // nonuniform or uniform Cartesian-like coord reconstruction from volume averages:
+        Real qa = (q_k(n,i) - q_km1(n,i));
+        Real qb = (q_kp1(n,i) - q_k(n,i));
+        dd_km1(i) = c1k(k-1)*qa + c2k(k-1)*(q_km1(n,i) - q_km2(n,i));
+        dd    (i) = c1k(k  )*qb + c2k(k  )*qa;
+        dd_kp1(i) = c1k(k+1)*(q_kp2(n,i) - q_kp1(n,i)) + c2k(k+1)*qb;
+
+      // Approximate interface average at k-1/2 and k+1/2 using PPM (CW eq 1.6)
+      // KGF: group the biased stencil quantities to preserve FP symmetry
+        dph(i) = (c3k(k)*q_km1(n,i) + c4k(k)*q_k(n,i)) +
+               (c5k(k)*dd_km1(i) + c6k(k)*dd(i));
+        dph_kp1(i) = (c3k(k+1)*q_k(n,i) + c4k(k+1)*q_kp1(n,i)) +
+                   (c5k(k+1)*dd(i) + c6k(k+1)*dd_kp1(i));
+      }
+
+    //--- Step 2a. -----------------------------------------------------------------------
+    // Uniform Cartesian-like coordinate: limit interpolated interface states (CD 4.3.1)
+      if (uniform[X3DIR]) {
+      // approximate second derivative at interfaces for smooth extrema preservation
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu; ++i) {
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          d2qc_km1(i) = q_km2(n,i) + q_k  (n,i) - 2.0*q_km1(n,i);
+          d2qc    (i) = q_km1(n,i) + q_kp1(n,i) - 2.0*q_k  (n,i); // (CD eq 85a) (no 1/2)
+          d2qc_kp1(i) = q_k  (n,i) + q_kp2(n,i) - 2.0*q_kp1(n,i);
+        }
+
+      // k-1/2
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu; ++i) {
+          Real qa_tmp = dph(i) - q_km1(n,i); // (CD eq 84a)
+          Real qb_tmp = q_k(n,i) - dph(i);     // (CD eq 84b)
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          Real qa = 3.0*(q_km1(n,i) + q_k(n,i) - 2.0*dph(i));  // (CD eq 85b)
+          Real qb = d2qc_km1(i);    // (CD eq 85a) (no 1/2)
+          Real qc = d2qc(i);   // (CD eq 85c) (no 1/2)
+          Real qd = 0.0;
+          if (SIGN(qa) == SIGN(qb) && SIGN(qa) == SIGN(qc)) {
+            qd = SIGN(qa)* std::min(C2*std::abs(qb),
+                                  std::min(C2*std::abs(qc), std::abs(qa)));
+          }
+          Real dph_tmp = 0.5*(q_km1(n,i) + q_k(n,i)) - qd/6.0;
+          if (qa_tmp*qb_tmp < 0.0) {  // Local extrema detected at k-1/2 face
+            dph(i) = dph_tmp;
+          }
+        }
+      // k+1/2
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu; ++i) {
+          Real qa_tmp = dph_kp1(i) - q_k(n,i);       // (CD eq 84a)
+          Real qb_tmp = q_kp1(n,i) - dph_kp1(i);   // (CD eq 84b)
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          Real qa = 3.0*(q_k(n,i) + q_kp1(n,i) - 2.0*dph_kp1(i));  // (CD eq 85b)
+          Real qb = d2qc(i);            // (CD eq 85a) (no 1/2)
+          Real qc = d2qc_kp1(i);   // (CD eq 85c) (no 1/2)
+          Real qd = 0.0;
+          if (SIGN(qa) == SIGN(qb) && SIGN(qa) == SIGN(qc)) {
+            qd = SIGN(qa)* std::min(C2*std::abs(qb),
+                                  std::min(C2*std::abs(qc), std::abs(qa)));
+          }
+          Real dphkp1_tmp = 0.5*(q_k(n,i) + q_kp1(n,i)) - qd/6.0;
+          if (qa_tmp*qb_tmp < 0.0) { // Local extrema detected at k+1/2 face
+            dph_kp1(i) = dphkp1_tmp;
+          }
+        }
+
+#pragma omp simd
+        for (int i=il; i<=iu; ++i) {
+        // KGF: add the off-centered quantities first to preserve FP symmetry
+          d2qf(i) = 6.0*(dph(i) + dph_kp1(i) - 2.0*q_k(n,i)); // a6 coefficient * -2
+        }
+      //--- Step 2b. ---------------------------------------------------------------------
+      // Nonuniform coordinate spacing: apply strict monotonicity constraints
+      // (Mignone eq 45)
+      } else {
+#pragma omp simd
+        for (int i=il; i<=iu; ++i) {
+          dph    (i) = std::min(dph    (i), std::max(q_k(n,i),q_km1(n,i)));
+          dph_kp1(i) = std::min(dph_kp1(i), std::max(q_k(n,i),q_kp1(n,i)));
+
+          dph    (i) = std::max(dph    (i), std::min(q_k(n,i),q_km1(n,i)));
+          dph_kp1(i) = std::max(dph_kp1(i), std::min(q_k(n,i),q_kp1(n,i)));
+        }
+      }
+
+    // Cache Riemann states for both non-/uniform limiters
+#pragma omp simd
+      for (int i=il; i<=iu; ++i) {
+        qminus(i) = dph(i  );
+        qplus(i) =  dph_kp1(i );
+      }
+
+    //--- Step 3. --------------------------------------------------------------------
+    // Compute cell-centered difference stencils (MC section 2.4.1)
+#pragma omp simd
+      for (int i=il; i<=iu; ++i) {
+        dqf_minus(i) = q_k(n,i) - qminus(i); // (CS eq 25)
+        dqf_plus(i)  = qplus(i) - q_k(n,i);
+      }
+
+    //--- Step 4a. -----------------------------------------------------------------------
+    // For uniform Cartesian-like coordinate: apply CS limiters to parabolic interpolant
+      if (uniform[X3DIR]) {
+#pragma omp simd simdlen(SIMD_WIDTH)
+        for (int i=il; i<=iu; ++i) {
+          Real qa_tmp = dqf_minus(i)*dqf_plus(i);
+          Real qb_tmp = (q_kp1(n,i) - q_k(n,i))*(q_k(n,i) - q_km1(n,i));
+
+        // Check if extrema is smooth
+          Real qa = d2qc_km1(i);
+          Real qb = d2qc(i);
+          Real qc = d2qc_kp1(i);
+          Real qd = d2qf(i);
+          Real qe = 0.0;
+          if (SIGN(qa) == SIGN(qb) && SIGN(qa) == SIGN(qc) && SIGN(qa) == SIGN(qd)) {
+          // Extrema is smooth
+            qe = SIGN(qd)* std::min(std::min(C2*std::abs(qa),C2*std::abs(qb)),
+                                  std::min(C2*std::abs(qc),std::abs(qd))); // (CS eq 22)
+          }
+
+        // Check if 2nd derivative is close to roundoff error
+          qa = std::max(std::abs(q_km1(n,i)),std::abs(q_km2(n,i)));
+          qb = std::max(std::max(std::abs(q_k(n,i)),
+                               std::abs(q_kp1(n,i))), std::abs(q_kp2(n,i)));
+
+          Real rho = 0.0;
+          if (std::abs(qd) > (1.0e-12)*std::max(qa,qb)) {
+          // Limiter is not sensitive to roundoff. Use limited ratio (MC eq 27)
+            rho = qe/qd;
+          }
+
+          Real tmp_m = q_k(n,i) - rho*dqf_minus(i);
+          Real tmp_p = q_k(n,i) + rho*dqf_plus(i);
+          Real tmp2_m = q_k(n,i) - 2.0*dqf_plus(i);
+          Real tmp2_p = q_k(n,i) + 2.0*dqf_minus(i);
+
+        // Check for local extrema
+          if (qa_tmp <= 0.0 || qb_tmp <= 0.0 ) {
+          // Check if relative change in limited 2nd deriv is > roundoff
+            if (rho <= (1.0 - (1.0e-12))) {
+            // Limit smooth extrema
+              qminus(i) = tmp_m; // (CS eq 23)
+              qplus(i) = tmp_p;
+            }
+
+          // No extrema detected
+          } else {
+          // Overshoot k-1/2,R / k,(-) state
+            if (std::abs(dqf_minus(i)) >= 2.0*std::abs(dqf_plus(i))) {
+              qminus(i) = tmp2_m;
+            }
+          // Overshoot k+1/2,L / k,(+) state
+            if (std::abs(dqf_plus(i)) >= 2.0*std::abs(dqf_minus(i))) {
+              qplus(i) = tmp2_p;
+            }
+          }
+        }
+      //--- Step 4b. ---------------------------------------------------------------------
+      // Nonuniform coordinate spacing: apply Mignone limiters to parabolic interpolant
+      // Note, the Mignone limiter does not check for cell-averaged extrema:
+      } else {
+        for (int i=il; i<=iu; ++i) {
+          Real qa = dqf_minus(i)*dqf_plus(i);
+          if (qa <= 0.0) { // Local extrema detected
+            qminus(i) = q_k(n,i);
+            qplus(i) = q_k(n,i);
+          } else { // No extrema detected
+          // TODO(felker): could delete hplus_ratio_k() arrays for curvilinear PPMx3
+          // Overshoot k-1/2,R / k,(-) state
+            if (std::abs(dqf_minus(i)) >= hplus_ratio_k(k)*std::abs(dqf_plus(i))) {
+              qminus(i) = q_k(n,i) - hplus_ratio_k(k)*dqf_plus(i);
+            }
+          // Overshoot k+1/2,L / k,(+) state
+            if (std::abs(dqf_plus(i)) >= hminus_ratio_k(k)*std::abs(dqf_minus(i))) {
+              qplus(i) = q_k(n,i) + hminus_ratio_k(k)*dqf_minus(i);
+            }
+          }
+        }
+      }
+    //--- Step 5. --------------------------------------------------------------------
+    // Convert limited cell-centered values to interface-centered L/R Riemann states
+    // both L/R values defined over [il,iu]
+#pragma omp simd
+      for (int i=il; i<=iu; ++i) {
+        ql_kph(n,i ) = qplus(i);
+        qr_kmh(n,i ) = qminus(i);
+      }
+    } // end char PPM loop over =nu
+
+  // compute ql_(k+1/2) and qr_(k-1/2)
+    for (int i=il; i<=iu; ++i) {
+#pragma omp simd
+      for (int n=0; n<=nu; ++n) {
+        ql(i,n) = ql_kph(n,i);
+        qr(i,n) = qr_kmh(n,i);
+      }
+    }
+  }
+  return;
+}
diff --git a/src/reconstruct/reconstruction.cpp b/src/reconstruct/reconstruction.cpp
index 4a861bd3e9..c5bdb9ac3c 100644
--- a/src/reconstruct/reconstruction.cpp
+++ b/src/reconstruct/reconstruction.cpp
@@ -46,6 +46,7 @@ Reconstruction::Reconstruction(MeshBlock *pmb, ParameterInput *pin) :
   // Read and set type of spatial reconstruction
   // --------------------------------
   std::string input_recon = pin->GetOrAddString("time", "xorder", "2");
+  nvar_ = pmb->nfre_ang;
 
   if (input_recon == "1") {
     xorder = 1;
@@ -229,12 +230,43 @@ Reconstruction::Reconstruction(MeshBlock *pmb, ParameterInput *pin) :
   scr01_i_.NewAthenaArray(nc1);
   scr02_i_.NewAthenaArray(nc1);
 
-  scr1_ni_.NewAthenaArray(std::max(NWAVE, NSCALARS), nc1);
-  scr2_ni_.NewAthenaArray(std::max(NWAVE, NSCALARS), nc1);
-  scr3_ni_.NewAthenaArray(std::max(NWAVE, NSCALARS), nc1);
-  scr4_ni_.NewAthenaArray(std::max(NWAVE, NSCALARS), nc1);
+  int nsize = std::max(NWAVE, NSCALARS);
+  if(CR_ENABLED)
+    nsize = std::max(nsize,5);
 
-  if ((xorder == 3) || (xorder == 4)) {
+
+  scr1_ni_.NewAthenaArray(nsize, nc1);
+  scr2_ni_.NewAthenaArray(nsize, nc1);
+  scr3_ni_.NewAthenaArray(nsize, nc1);
+  scr4_ni_.NewAthenaArray(nsize, nc1);
+
+  scr1_in_.NewAthenaArray(nvar_, nc1);
+  scr2_in_.NewAthenaArray(nvar_, nc1);
+  scr3_in_.NewAthenaArray(nvar_, nc1);
+  scr4_in_.NewAthenaArray(nvar_, nc1);
+
+  // temporary array for reconstruction in angular space
+  scr1_nn_.NewAthenaArray(nvar_+2*NGHOST,nvar_+2*NGHOST);
+  scr2_nn_.NewAthenaArray(nvar_+2*NGHOST,nvar_+2*NGHOST);
+  scr3_nn_.NewAthenaArray(nvar_+2*NGHOST,nvar_+2*NGHOST);
+  scr4_nn_.NewAthenaArray(nvar_+2*NGHOST,nvar_+2*NGHOST);
+
+  scr1_in2_.NewAthenaArray(nc1, nvar_);
+  scr2_in2_.NewAthenaArray(nc1, nvar_);
+  scr3_in2_.NewAthenaArray(nc1, nvar_);
+  scr4_in2_.NewAthenaArray(nc1, nvar_);
+
+  int order_flag = xorder;
+  if (NR_RADIATION_ENABLED || IM_RADIATION_ENABLED) {
+    int rad_order = pin->GetOrAddInteger("time", "rad_xorder", 2);
+    order_flag = std::max(rad_order,xorder);
+  }
+  if (CR_ENABLED) {
+    int cr_order=pin->GetOrAddInteger("time", "cr_xorder", 2);
+    order_flag = std::max(cr_order,order_flag);
+  }
+
+  if ((order_flag == 3) || (order_flag == 4)) {
     Coordinates *pco = pmb->pcoord;
     scr03_i_.NewAthenaArray(nc1);
     scr04_i_.NewAthenaArray(nc1);
@@ -249,10 +281,15 @@ Reconstruction::Reconstruction(MeshBlock *pmb, ParameterInput *pin) :
     scr13_i_.NewAthenaArray(nc1);
     scr14_i_.NewAthenaArray(nc1);
 
-    scr5_ni_.NewAthenaArray(std::max(NWAVE, NSCALARS), nc1);
-    scr6_ni_.NewAthenaArray(std::max(NWAVE, NSCALARS), nc1);
-    scr7_ni_.NewAthenaArray(std::max(NWAVE, NSCALARS), nc1);
-    scr8_ni_.NewAthenaArray(std::max(NWAVE, NSCALARS), nc1);
+    scr5_ni_.NewAthenaArray(nsize, nc1);
+    scr6_ni_.NewAthenaArray(nsize, nc1);
+    scr7_ni_.NewAthenaArray(nsize, nc1);
+    scr8_ni_.NewAthenaArray(nsize, nc1);
+
+    scr5_in_.NewAthenaArray(nvar_, nc1);
+    scr6_in_.NewAthenaArray(nvar_, nc1);
+    scr7_in_.NewAthenaArray(nvar_, nc1);
+    scr8_in_.NewAthenaArray(nvar_, nc1);
 
     // Precompute PPM coefficients in x1-direction ---------------------------------------
     c1i.NewAthenaArray(nc1);
diff --git a/src/reconstruct/reconstruction.hpp b/src/reconstruct/reconstruction.hpp
index 410bb8f1ce..0644fe8a9a 100644
--- a/src/reconstruct/reconstruction.hpp
+++ b/src/reconstruct/reconstruction.hpp
@@ -129,6 +129,61 @@ class Reconstruction {
                             const AthenaArray<Real> &q,
                             AthenaArray<Real> &ql, AthenaArray<Real> &qr);
 
+  // overloads for cell centered variables with memory order as [k,j,i,n]
+  // Notice that the default order in Athena++ is [n,k,j,i]
+  void DonorCellX1(const int k, const int j, const int il, const int iu,
+                   AthenaArray<Real> &q, const int array_order,
+                   AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void DonorCellX2(const int k, const int j, const int il, const int iu,
+                   AthenaArray<Real> &q, const int array_order,
+                   AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void DonorCellX3(const int k, const int j, const int il, const int iu,
+                   AthenaArray<Real> &q, const int array_order,
+                   AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void PiecewiseLinearX1(const int k, const int j, const int il, const int iu,
+                         AthenaArray<Real> &q, const int array_order,
+                         AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void PiecewiseLinearX2(const int k, const int j, const int il, const int iu,
+                         AthenaArray<Real> &q, const int array_order,
+                         AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void PiecewiseLinearX3(const int k, const int j, const int il, const int iu,
+                         AthenaArray<Real> &q, const int array_order,
+                         AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void PiecewiseParabolicX1(const int k, const int j, const int il, const int iu,
+                            const AthenaArray<Real> &q, const int array_order,
+                            AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void PiecewiseParabolicX2(const int k, const int j, const int il, const int iu,
+                            const AthenaArray<Real> &q, const int array_order,
+                            AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void PiecewiseParabolicX3(const int k, const int j, const int il, const int iu,
+                            const AthenaArray<Real> &q, const int array_order,
+                            AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void DonorCellZeta(NRRadiation *prad, const int zs, const int ze,
+      AthenaArray<Real> &q,
+      AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void DonorCellPsi(NRRadiation *prad, const int ps, const int pe,
+      AthenaArray<Real> &q,
+      AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+  void PiecewiseLinearZeta(NRRadiation *prad,
+      const int zs, const int ze, AthenaArray<Real> &q,
+      AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
+
+  void PiecewiseLinearPsi(NRRadiation *prad,
+      const int zs, const int ze, AthenaArray<Real> &q,
+      AthenaArray<Real> &ql, AthenaArray<Real> &qr);
+
  private:
   MeshBlock* pmy_block_;  // ptr to MeshBlock containing this Reconstruction
 
@@ -138,5 +193,12 @@ class Reconstruction {
   AthenaArray<Real> scr11_i_, scr12_i_, scr13_i_, scr14_i_;
   AthenaArray<Real> scr1_ni_, scr2_ni_, scr3_ni_, scr4_ni_, scr5_ni_;
   AthenaArray<Real> scr6_ni_, scr7_ni_, scr8_ni_;
+
+  // scratch arrays for arrays with different ordering
+  int nvar_;// the maximum numver of variables for reconstruction
+  AthenaArray<Real> scr1_in_, scr2_in_, scr3_in_, scr4_in_, scr5_in_;
+  AthenaArray<Real> scr1_in2_, scr2_in2_, scr3_in2_, scr4_in2_;
+  AthenaArray<Real> scr1_nn_, scr2_nn_, scr3_nn_, scr4_nn_;
+  AthenaArray<Real> scr6_in_, scr7_in_, scr8_in_;
 };
 #endif // RECONSTRUCT_RECONSTRUCTION_HPP_
diff --git a/src/task_list/im_rad_compt_task_list.cpp b/src/task_list/im_rad_compt_task_list.cpp
new file mode 100644
index 0000000000..d01173cd02
--- /dev/null
+++ b/src/task_list/im_rad_compt_task_list.cpp
@@ -0,0 +1,158 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file im_rad_compt_task_list.cpp
+//! \brief function implementation for implicit radiation with Compton
+
+// C headers
+
+// C++ headers
+#include <iostream>   // endl
+#include <sstream>    // sstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../hydro/hydro.hpp"
+#include "../mesh/mesh.hpp"
+#include "../nr_radiation/implicit/radiation_implicit.hpp"
+#include "../nr_radiation/integrators/rad_integrators.hpp"
+#include "../nr_radiation/radiation.hpp"
+#include "./im_rad_task_list.hpp"
+
+//----------------------------------------------------------------------------------------
+//! IMRadTaskList constructor
+
+IMRadComptTaskList::IMRadComptTaskList(Mesh *pm) {
+  pmy_mesh = pm;
+  // Now assemble list of tasks for each stage of time integrator
+  {using namespace IMRadComptTaskNames; // NOLINT (build/namespace)
+    // compute hydro fluxes, integrate hydro variables
+    AddTask(CAL_COMPT,NONE);
+    AddTask(SEND_RAD_BND,CAL_COMPT);
+    AddTask(RECV_RAD_BND,CAL_COMPT);
+    AddTask(SETB_RAD_BND,(RECV_RAD_BND|SEND_RAD_BND));
+    if (pm->shear_periodic) {
+      AddTask(SEND_RAD_SH,SETB_RAD_BND);
+      AddTask(RECV_RAD_SH,SEND_RAD_SH|RECV_RAD_BND);
+    }
+    TaskID setb = SETB_RAD_BND;
+    if (pm->shear_periodic)
+      setb=(setb|RECV_RAD_SH);
+    if (pm->multilevel) {
+      AddTask(PRLN_RAD_BND,setb);
+      AddTask(RAD_PHYS_BND,PRLN_RAD_BND);
+    } else {
+      AddTask(RAD_PHYS_BND,setb);
+    }
+    AddTask(CLEAR_RAD, RAD_PHYS_BND);
+  }
+}
+
+
+void IMRadComptTaskList::AddTask(const TaskID& id, const TaskID& dep) {
+  task_list_[ntasks].task_id=id;
+  task_list_[ntasks].dependency=dep;
+
+  using namespace IMRadComptTaskNames; // NOLINT (build/namespace)
+  if (id == CLEAR_RAD) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadComptTaskList::ClearRadBoundary);
+  } else if (id == SEND_RAD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadComptTaskList::SendRadBoundary);
+  } else if (id == RECV_RAD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadComptTaskList::ReceiveRadBoundary);
+  } else if (id == SETB_RAD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadComptTaskList::SetRadBoundary);
+  } else if (id == RAD_PHYS_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadComptTaskList::PhysicalBoundary);
+  } else if (id == PRLN_RAD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadComptTaskList::ProlongateBoundary);
+  } else if (id == SEND_RAD_SH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadComptTaskList::SendRadBoundaryShear);
+  } else if (id == RECV_RAD_SH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadComptTaskList::ReceiveRadBoundaryShear);
+  } else if (id == CAL_COMPT) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadComptTaskList::CalComptTerms);
+  } else {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in IMRadTaskList::AddTask" << std::endl
+        << "Invalid Task is specified" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  ntasks++;
+  return;
+}
+
+
+TaskStatus IMRadComptTaskList::CalComptTerms(MeshBlock *pmb) {
+  NRRadiation *prad = pmb->pnrrad;
+  Hydro *ph = pmb->phydro;
+  prad->pradintegrator->AddMultiGroupCompt(pmb, dt, ph->u, prad->ir);
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadComptTaskList::ClearRadBoundary(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.ClearBoundary(BoundaryCommSubset::radiation);
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadComptTaskList::SendRadBoundary(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.SendBoundaryBuffers();
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadComptTaskList::SendRadBoundaryShear(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.SendShearingBoxBoundaryBuffers();
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadComptTaskList::ReceiveRadBoundary(MeshBlock *pmb) {
+  bool ret = pmb->pnrrad->rad_bvar.ReceiveBoundaryBuffers();
+  if (!ret)
+    return TaskStatus::fail;
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadComptTaskList::ReceiveRadBoundaryShear(MeshBlock *pmb) {
+  bool ret = pmb->pnrrad->rad_bvar.ReceiveShearingBoxBoundaryBuffers();
+  if (ret) {
+    pmb->pnrrad->rad_bvar.SetShearingBoxBoundaryBuffers();
+    return TaskStatus::success;
+  } else {
+    return TaskStatus::fail;
+  }
+}
+
+TaskStatus IMRadComptTaskList::SetRadBoundary(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.SetBoundaries();
+  return TaskStatus::success;
+}
+
+void IMRadComptTaskList::StartupTaskList(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.StartReceiving(BoundaryCommSubset::radiation);
+  if (pmy_mesh->shear_periodic) {
+    pmb->pnrrad->rad_bvar.StartReceivingShear(BoundaryCommSubset::radiation);
+  }
+  return;
+}
diff --git a/src/task_list/im_rad_task_list.cpp b/src/task_list/im_rad_task_list.cpp
new file mode 100644
index 0000000000..63a14804cd
--- /dev/null
+++ b/src/task_list/im_rad_task_list.cpp
@@ -0,0 +1,101 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file task_list.cpp
+//! \brief functions for TaskList base class
+
+
+// C headers
+
+// C++ headers
+//#include <vector> // formerly needed for vector of MeshBlock ptrs in DoTaskListOneStage
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../globals.hpp"
+#include "../mesh/mesh.hpp"
+#include "../nr_radiation/radiation.hpp"
+#include "im_rad_task_list.hpp"
+
+#ifdef OPENMP_PARALLEL
+#include <omp.h>
+#endif
+
+//----------------------------------------------------------------------------------------
+//! \fn TaskListStatus TaskList::DoAllAvailableTasks
+//! \brief do all tasks that can be done (are not waiting for a dependency to be
+//! cleared) in this TaskList, return status.
+
+TaskListStatus IMRadTaskList::DoAllAvailableTasks(MeshBlock *pmb, TaskStates &ts) {
+  int skip = 0;
+  TaskStatus ret;
+  if (ts.num_tasks_left == 0) return TaskListStatus::nothing_to_do;
+
+  for (int i=ts.indx_first_task; i<ntasks; i++) {
+    IMRadTask &taski = task_list_[i];
+    if (ts.finished_tasks.IsUnfinished(taski.task_id)) { // task not done
+      // check if dependency clear
+      if (ts.finished_tasks.CheckDependencies(taski.dependency)) {
+        ret = (this->*task_list_[i].TaskFunc)(pmb);
+        if (ret != TaskStatus::fail) { // success
+          ts.num_tasks_left--;
+          ts.finished_tasks.SetFinished(taski.task_id);
+          if (skip == 0) ts.indx_first_task++;
+          if (ts.num_tasks_left == 0) return TaskListStatus::complete;
+          if (ret == TaskStatus::next) continue;
+          return TaskListStatus::running;
+        }
+      }
+      skip++; // increment number of tasks processed
+
+    } else if (skip == 0) { // this task is already done AND it is at the top of the list
+      ts.indx_first_task++;
+    }
+  }
+  // there are still tasks to do but nothing can be done now
+  return TaskListStatus::stuck;
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void TaskList::DoTaskListOneStage(Mesh *pmesh, int stage)
+//! \brief completes all tasks in this list, will not return until all are tasks done
+
+void IMRadTaskList::DoTaskListOneStage(Real wght) {
+  time = pmy_mesh->time + wght;
+  dt = wght;
+  int nmb = pmy_mesh->nblocal;
+  for (int i=0; i<nmb; ++i) {
+    pmy_mesh->my_blocks(i)->tasks.Reset(ntasks);
+    StartupTaskList(pmy_mesh->my_blocks(i));
+  }
+
+  int nmb_left = nmb;
+  // cycle through all MeshBlocks and perform all tasks possible
+  while (nmb_left > 0) {
+    //! \note
+    //! KNOWN ISSUE: Workaround for unknown OpenMP race condition. See #183 on GitHub.
+    for (int i=0; i<nmb; ++i) {
+      if (DoAllAvailableTasks(pmy_mesh->my_blocks(i), pmy_mesh->my_blocks(i)->tasks)
+          == TaskListStatus::complete) {
+        nmb_left--;
+      }
+    }
+  }
+  return;
+}
+
+TaskStatus IMRadTaskList::PhysicalBoundary(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.var_cc = &(pmb->pnrrad->ir);
+  pmb->pbval->ApplyPhysicalBoundaries(time, dt, pmb->pbval->bvars_main_int);
+  return TaskStatus::success;
+}
+
+
+TaskStatus IMRadTaskList::ProlongateBoundary(MeshBlock *pmb) {
+  pmb->pbval->ProlongateBoundaries(time, dt, pmb->pbval->bvars_main_int);
+  return TaskStatus::success;
+}
+
+
diff --git a/src/task_list/im_rad_task_list.hpp b/src/task_list/im_rad_task_list.hpp
new file mode 100644
index 0000000000..669dadf040
--- /dev/null
+++ b/src/task_list/im_rad_task_list.hpp
@@ -0,0 +1,170 @@
+#ifndef TASK_LIST_IM_RAD_TASK_LIST_HPP_
+#define TASK_LIST_IM_RAD_TASK_LIST_HPP_
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file im_rad_task_list.hpp
+//! \brief define im_rad_task_list class
+//! \brief This is used to handle the boundary condition
+
+// C headers
+
+// C++ headers
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "./task_list.hpp"
+
+// forward declarations
+class Mesh;
+class MeshBlock;
+class Radiation;
+class IMRadTaskList;
+class TaskID;
+
+
+//----------------------------------------------------------------------------------------
+//! \struct IMRadTask
+//! \brief data and function pointer for an individual IMRadTask
+
+struct IMRadTask {
+  TaskID task_id;      //!> encodes task using bit positions in IMRadTaskNames
+  TaskID dependency;   //!> encodes dependencies to other tasks using IMRadTaskNames
+  TaskStatus (IMRadTaskList::*TaskFunc)(MeshBlock *); //!> ptr to a task
+};
+
+//----------------------------------------------------------------------------------------
+//! \class IMRadTaskList
+//! \brief data and function definitions for IMRadTaskList class
+
+class IMRadTaskList {
+ public:
+  IMRadTaskList() : ntasks(0), task_list_{} {} // 2x direct + zero initialization
+  virtual ~IMRadTaskList() = default;
+
+  Mesh *pmy_mesh;
+  int ntasks;     //!> number of tasks in this list
+  Real time, dt;
+
+  TaskListStatus DoAllAvailableTasks(MeshBlock *pmb, TaskStates &ts);
+  void DoTaskListOneStage(Real wght);
+  TaskStatus PhysicalBoundary(MeshBlock *pmb);
+  TaskStatus ProlongateBoundary(MeshBlock *pmb);
+
+ protected:
+  IMRadTask task_list_[64*TaskID::kNField_];
+
+ private:
+  virtual void AddTask(const TaskID& id, const TaskID& dep) = 0;
+  virtual void StartupTaskList(MeshBlock *pmb) = 0;
+};
+
+class IMRadITTaskList : public IMRadTaskList {
+ public:
+  explicit IMRadITTaskList(Mesh *pm);
+
+  TaskStatus ClearRadBoundary(MeshBlock *pmb);
+  TaskStatus SendRadBoundary(MeshBlock *pmb);
+  TaskStatus ReceiveRadBoundary(MeshBlock *pmb);
+  TaskStatus SetRadBoundary(MeshBlock *pmb);
+  TaskStatus SendRadBoundaryShear(MeshBlock *pmb);
+  TaskStatus ReceiveRadBoundaryShear(MeshBlock *pmb);
+  TaskStatus CheckResidual(MeshBlock *pmb);
+  TaskStatus AddFluxAndSourceTerms(MeshBlock *pmb);
+
+ private:
+  void StartupTaskList(MeshBlock *pmb) override;
+  void AddTask(const TaskID& id, const TaskID& dep) override;
+};
+
+//----------------------------------------------
+//! IMRadHydroTaskList
+//! Derived Class to handle Hydro boundary update
+class IMRadHydroTaskList : public IMRadTaskList{
+ public:
+  explicit IMRadHydroTaskList(Mesh *pm);
+
+  TaskStatus ClearHydroBoundary(MeshBlock *pmb);
+  TaskStatus SendHydroBoundary(MeshBlock *pmb);
+  TaskStatus ReceiveHydroBoundary(MeshBlock *pmb);
+  TaskStatus SetHydroBoundary(MeshBlock *pmb);
+  TaskStatus SendHydroBoundaryShear(MeshBlock *pmb);
+  TaskStatus ReceiveHydroBoundaryShear(MeshBlock *pmb);
+  TaskStatus UpdateOpacity(MeshBlock *pmb);
+  TaskStatus AddRadSource(MeshBlock *pmb);
+  TaskStatus Primitive(MeshBlock *pmb);
+
+ private:
+  void StartupTaskList(MeshBlock *pmb) override;
+  void AddTask(const TaskID& id, const TaskID& dep) override;
+};
+
+// task list for separate compton scattering
+// need to update boundary condition
+class IMRadComptTaskList : public IMRadTaskList {
+ public:
+  explicit IMRadComptTaskList(Mesh *pm);
+
+  TaskStatus ClearRadBoundary(MeshBlock *pmb);
+  TaskStatus SendRadBoundary(MeshBlock *pmb);
+  TaskStatus ReceiveRadBoundary(MeshBlock *pmb);
+  TaskStatus SetRadBoundary(MeshBlock *pmb);
+  TaskStatus SendRadBoundaryShear(MeshBlock *pmb);
+  TaskStatus ReceiveRadBoundaryShear(MeshBlock *pmb);
+  TaskStatus CalComptTerms(MeshBlock *pmb);
+ private:
+  void StartupTaskList(MeshBlock *pmb) override;
+  void AddTask(const TaskID& id, const TaskID& dep) override;
+};
+
+
+//----------------------------------------------------------------------------------------
+//! 64-bit integers with "1" in different bit positions used to ID each IMRadITT task.
+
+namespace IMRadITTaskNames {
+const TaskID NONE(0);
+const TaskID CLEAR_RAD(1);     // clear radiation boundary
+const TaskID SEND_RAD_BND(2);  // send radiation boundary
+const TaskID RECV_RAD_BND(3); // receive radiation boundary
+const TaskID SETB_RAD_BND(4); // set radiation physical boundary
+const TaskID RAD_PHYS_BND(5);// radiation physical boundary
+const TaskID PRLN_RAD_BND(6); // prolongation
+const TaskID SEND_RAD_SH(7); // send shearing box boundary
+const TaskID RECV_RAD_SH(8); // receive shearing box boundary
+const TaskID CHK_RAD_RES(9); // check residual
+const TaskID FLX_AND_SCR(10);  // calculate the source term and all others together
+
+} // namespace IMRadITTaskNames
+
+namespace IMRadHydroTaskNames {
+const TaskID NONE(0);
+const TaskID CLEAR_HYD(1);    // clear radiation boundary
+const TaskID SEND_HYD_BND(2); // send radiation boundary
+const TaskID RECV_HYD_BND(3); // receive radiation boundary
+const TaskID SETB_HYD_BND(4); // set radiation physical boundary
+const TaskID HYD_PHYS_BND(5); // radiation physical boundary
+const TaskID PRLN_HYD_BND(6); // prolongation
+const TaskID SEND_HYD_SH(7);  // send shearing box boundary
+const TaskID RECV_HYD_SH(8);  // receive shearing box boundary
+const TaskID UPD_OPA(9);      // check residual
+const TaskID ADD_RAD_SCR(10); // add radiation source term
+const TaskID CONS_TO_PRIM(11);// convert conservative to primitive variables
+} // namespace IMRadHydroTaskNames
+
+namespace IMRadComptTaskNames {
+const TaskID NONE(0);
+const TaskID CLEAR_RAD(1);     // clear radiation boundary
+const TaskID SEND_RAD_BND(2);  // send radiation boundary
+const TaskID RECV_RAD_BND(3); // receive radiation boundary
+const TaskID SETB_RAD_BND(4); // set radiation physical boundary
+const TaskID RAD_PHYS_BND(5);// radiation physical boundary
+const TaskID PRLN_RAD_BND(6); // prolongation
+const TaskID SEND_RAD_SH(7); // send shearing box boundary
+const TaskID RECV_RAD_SH(8); // receive shearing box boundary
+const TaskID CAL_COMPT(9); // add flux divergence term
+
+} // namespace IMRadComptTaskNames
+
+#endif // TASK_LIST_IM_RAD_TASK_LIST_HPP_
diff --git a/src/task_list/im_radhydro_task_list.cpp b/src/task_list/im_radhydro_task_list.cpp
new file mode 100644
index 0000000000..028dddbab8
--- /dev/null
+++ b/src/task_list/im_radhydro_task_list.cpp
@@ -0,0 +1,210 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file im_radhydro_task_list.cpp
+//! \brief function implementation for radiation hydro task list
+
+// C headers
+
+// C++ headers
+#include <iostream>   // endl
+#include <sstream>    // sstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../bvals/bvals.hpp"
+#include "../eos/eos.hpp"
+#include "../field/field.hpp"
+#include "../hydro/hydro.hpp"
+#include "../mesh/mesh.hpp"
+#include "../nr_radiation/implicit/radiation_implicit.hpp"
+#include "../nr_radiation/integrators/rad_integrators.hpp"
+#include "../nr_radiation/radiation.hpp"
+#include "../scalars/scalars.hpp"
+#include "./im_rad_task_list.hpp"
+
+//----------------------------------------------------------------------------------------
+//! IMRadTaskList constructor
+
+IMRadHydroTaskList::IMRadHydroTaskList(Mesh *pm) {
+  pmy_mesh = pm;
+  // Now assemble list of tasks for each stage of time integrator
+
+  {using namespace IMRadHydroTaskNames; // NOLINT (build/namespace)
+    // compute hydro fluxes, integrate hydro variables
+    AddTask(ADD_RAD_SCR,NONE);
+    AddTask(SEND_HYD_BND,ADD_RAD_SCR);
+    AddTask(RECV_HYD_BND,NONE);
+    AddTask(SETB_HYD_BND,(RECV_HYD_BND|SEND_HYD_BND));
+    if (pm->shear_periodic) {
+      AddTask(SEND_HYD_SH,SETB_HYD_BND);
+      AddTask(RECV_HYD_SH,SEND_HYD_SH|RECV_HYD_BND);
+    }
+    TaskID setb = SETB_HYD_BND;
+    if (pm->shear_periodic)
+      setb=(setb|RECV_HYD_SH);
+    if (pm->multilevel) {
+      AddTask(PRLN_HYD_BND,setb);
+      AddTask(CONS_TO_PRIM,PRLN_HYD_BND);
+    } else {
+      AddTask(CONS_TO_PRIM,setb);
+    }
+    AddTask(HYD_PHYS_BND,CONS_TO_PRIM);
+    AddTask(CLEAR_HYD, HYD_PHYS_BND);
+    AddTask(UPD_OPA,HYD_PHYS_BND);
+  } // end of using namespace block
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void FFTGravitySolverTaskList::AddTask(const TaskID& id, const TaskID& dep)
+//! \brief Sets id and dependency for "ntask" member of task_list_ array, then iterates
+//! value of ntask.
+
+void IMRadHydroTaskList::AddTask(const TaskID& id, const TaskID& dep) {
+  task_list_[ntasks].task_id=id;
+  task_list_[ntasks].dependency=dep;
+
+  using namespace IMRadHydroTaskNames; // NOLINT (build/namespace)
+  if (id == CLEAR_HYD) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadHydroTaskList::ClearHydroBoundary);
+  } else if (id == SEND_HYD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadHydroTaskList::SendHydroBoundary);
+  } else if (id == RECV_HYD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadHydroTaskList::ReceiveHydroBoundary);
+  } else if (id == SETB_HYD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadHydroTaskList::SetHydroBoundary);
+  } else if (id == HYD_PHYS_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadTaskList::PhysicalBoundary);
+  } else if (id == PRLN_HYD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadTaskList::ProlongateBoundary);
+  } else if (id == SEND_HYD_SH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadHydroTaskList::SendHydroBoundaryShear);
+  } else if (id == RECV_HYD_SH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadHydroTaskList::ReceiveHydroBoundaryShear);
+  } else if (id == UPD_OPA) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadHydroTaskList::UpdateOpacity);
+  } else if (id == ADD_RAD_SCR) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadHydroTaskList::AddRadSource);
+  } else if (id == CONS_TO_PRIM) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadHydroTaskList::Primitive);
+  } else {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in IMRadTaskList::AddTask" << std::endl
+        << "Invalid Task is specified" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  ntasks++;
+  return;
+}
+
+TaskStatus IMRadHydroTaskList::ClearHydroBoundary(MeshBlock *pmb) {
+  pmb->phydro->hbvar.ClearBoundary(BoundaryCommSubset::radhydro);
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadHydroTaskList::SendHydroBoundary(MeshBlock *pmb) {
+  pmb->phydro->hbvar.SwapHydroQuantity(pmb->phydro->u, HydroBoundaryQuantity::cons);
+  pmb->phydro->hbvar.SendBoundaryBuffers();
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadHydroTaskList::SendHydroBoundaryShear(MeshBlock *pmb) {
+  pmb->phydro->hbvar.SendShearingBoxBoundaryBuffers();
+  return TaskStatus::success;
+}
+
+
+TaskStatus IMRadHydroTaskList::ReceiveHydroBoundary(MeshBlock *pmb) {
+  bool ret = pmb->phydro->hbvar.ReceiveBoundaryBuffers();
+  if (!ret)
+    return TaskStatus::fail;
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadHydroTaskList::ReceiveHydroBoundaryShear(MeshBlock *pmb) {
+  bool ret = pmb->phydro->hbvar.ReceiveShearingBoxBoundaryBuffers();
+  if (ret) {
+    pmb->phydro->hbvar.SetShearingBoxBoundaryBuffers();
+    return TaskStatus::success;
+  } else {
+    return TaskStatus::fail;
+  }
+}
+
+
+TaskStatus IMRadHydroTaskList::SetHydroBoundary(MeshBlock *pmb) {
+  pmb->phydro->hbvar.SwapHydroQuantity(pmb->phydro->u, HydroBoundaryQuantity::cons);
+  pmb->phydro->hbvar.SetBoundaries();
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadHydroTaskList::Primitive(MeshBlock *pmb) {
+  Hydro *ph = pmb->phydro;
+  Field *pf = pmb->pfield;
+  PassiveScalars *ps = pmb->pscalars;
+  BoundaryValues *pbval = pmb->pbval;
+  int il = pmb->is, iu = pmb->ie, jl = pmb->js, ju = pmb->je, kl = pmb->ks, ku = pmb->ke;
+  if (pbval->nblevel[1][1][0] != -1) il -= NGHOST;
+  if (pbval->nblevel[1][1][2] != -1) iu += NGHOST;
+  if (pbval->nblevel[1][0][1] != -1) jl -= NGHOST;
+  if (pbval->nblevel[1][2][1] != -1) ju += NGHOST;
+  if (pbval->nblevel[0][1][1] != -1) kl -= NGHOST;
+  if (pbval->nblevel[2][1][1] != -1) ku += NGHOST;
+  pmb->peos->ConservedToPrimitive(ph->u, ph->w, pf->b,
+                                  ph->w1, pf->bcc, pmb->pcoord,
+                                  il, iu, jl, ju, kl, ku);
+  if (NSCALARS > 0) {
+    // r1/r_old for GR is currently unused:
+    pmb->peos->PassiveScalarConservedToPrimitive(ps->s, ph->u, ps->r, ps->r,
+                                                 pmb->pcoord, il, iu, jl, ju, kl, ku);
+  }
+  ph->w.SwapAthenaArray(ph->w1);
+
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadHydroTaskList::UpdateOpacity(MeshBlock *pmb) {
+  pmb->pnrrad->UpdateOpacity(pmb, pmb->phydro->w);
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadHydroTaskList::AddRadSource(MeshBlock *pmb) {
+  if (pmb->pnrrad->set_source_flag > 0) {
+    pmb->pnrrad->pradintegrator->AddSourceTerms(pmb, pmb->phydro->u);
+  }
+  return TaskStatus::success;
+}
+
+void IMRadHydroTaskList::StartupTaskList(MeshBlock *pmb) {
+  pmb->phydro->hbvar.StartReceiving(BoundaryCommSubset::radhydro);
+  if (pmy_mesh->shear_periodic) {
+    pmb->phydro->hbvar.StartReceivingShear(BoundaryCommSubset::radhydro);
+  }
+  return;
+}
diff --git a/src/task_list/im_radit_task_list.cpp b/src/task_list/im_radit_task_list.cpp
new file mode 100644
index 0000000000..1914d9cfdb
--- /dev/null
+++ b/src/task_list/im_radit_task_list.cpp
@@ -0,0 +1,244 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+//! \file im_radit_task_list.cpp
+//! \brief function implementation for iteration in implicit radiation
+
+// C headers
+
+// C++ headers
+#include <iostream>   // endl
+#include <sstream>    // sstream
+#include <stdexcept>  // runtime_error
+#include <string>     // c_str()
+
+// Athena++ headers
+#include "../athena.hpp"
+#include "../hydro/hydro.hpp"
+#include "../mesh/mesh.hpp"
+#include "../nr_radiation/implicit/radiation_implicit.hpp"
+#include "../nr_radiation/integrators/rad_integrators.hpp"
+#include "../nr_radiation/radiation.hpp"
+#include "./im_rad_task_list.hpp"
+
+//----------------------------------------------------------------------------------------
+//! IMRadTaskList constructor
+
+IMRadITTaskList::IMRadITTaskList(Mesh *pm) {
+  pmy_mesh = pm;
+  // Now assemble list of tasks for each stage of time integrator
+  {using namespace IMRadITTaskNames; // NOLINT (build/namespace)
+    // compute hydro fluxes, integrate hydro variables
+    AddTask(FLX_AND_SCR,NONE);
+    AddTask(SEND_RAD_BND,FLX_AND_SCR);
+    AddTask(RECV_RAD_BND,FLX_AND_SCR);
+    AddTask(SETB_RAD_BND,(RECV_RAD_BND|SEND_RAD_BND));
+    if (pm->shear_periodic) {
+      AddTask(SEND_RAD_SH,SETB_RAD_BND);
+      AddTask(RECV_RAD_SH,SEND_RAD_SH|RECV_RAD_BND);
+    }
+    TaskID setb = SETB_RAD_BND;
+    if (pm->shear_periodic)
+      setb=(setb|RECV_RAD_SH);
+    if (pm->multilevel) {
+      AddTask(PRLN_RAD_BND,setb);
+      AddTask(RAD_PHYS_BND,PRLN_RAD_BND);
+    } else {
+      AddTask(RAD_PHYS_BND,setb);
+    }
+    AddTask(CLEAR_RAD, RAD_PHYS_BND);
+    // check residual does not need ghost zones
+    AddTask(CHK_RAD_RES,FLX_AND_SCR);
+  } // end of using namespace block
+}
+
+//----------------------------------------------------------------------------------------
+//! \fn void FFTGravitySolverTaskList::AddTask(const TaskID& id, const TaskID& dep)
+//! \brief Sets id and dependency for "ntask" member of task_list_ array, then iterates
+//! value of ntask.
+
+void IMRadITTaskList::AddTask(const TaskID& id, const TaskID& dep) {
+  task_list_[ntasks].task_id=id;
+  task_list_[ntasks].dependency=dep;
+
+  using namespace IMRadITTaskNames; // NOLINT (build/namespace)
+  if (id == CLEAR_RAD) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::ClearRadBoundary);
+  } else if (id == SEND_RAD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::SendRadBoundary);
+  } else if (id == RECV_RAD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::ReceiveRadBoundary);
+  } else if (id == SETB_RAD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::SetRadBoundary);
+  } else if (id == RAD_PHYS_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::PhysicalBoundary);
+  } else if (id == PRLN_RAD_BND) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::ProlongateBoundary);
+  } else if (id == SEND_RAD_SH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::SendRadBoundaryShear);
+  } else if (id == RECV_RAD_SH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::ReceiveRadBoundaryShear);
+  } else if (id == CHK_RAD_RES) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::CheckResidual);
+  } else if (id == FLX_AND_SCR) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (IMRadTaskList::*)(MeshBlock*)>
+        (&IMRadITTaskList::AddFluxAndSourceTerms);
+  } else {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in IMRadTaskList::AddTask" << std::endl
+        << "Invalid Task is specified" << std::endl;
+    ATHENA_ERROR(msg);
+  }
+  ntasks++;
+  return;
+}
+
+TaskStatus IMRadITTaskList::AddFluxAndSourceTerms(MeshBlock *pmb) {
+  NRRadiation *prad = pmb->pnrrad;
+  Hydro *ph = pmb->phydro;
+  int &rb_or_not = pmy_mesh->pimrad->rb_or_not;
+
+  int is=pmb->is, ie=pmb->ie;
+  int js=pmb->js, je=pmb->je;
+  int ks=pmb->ks, ke=pmb->ke;
+
+  if (rb_or_not == 0) {
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          prad->pradintegrator->FirstOrderFluxDivergence(k, j, i,
+                                                         prad->ir_old);
+
+          // add angular flux
+          if (prad->angle_flag == 1) {
+            prad->pradintegrator->ImplicitAngularFluxes(k,j,i,prad->ir);
+          }// end angular flux
+
+          // add the source terms together
+          prad->pradintegrator->CalSourceTerms(pmb, dt, k,j,i, ph->u,
+                                               prad->ir1, prad->ir);
+        }
+      }
+    }
+  } else if (rb_or_not == 1) {
+    // first the red points
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          if (((i-is)+(j-js)+(k-ks))%2 == 0) {
+            prad->pradintegrator->FirstOrderFluxDivergence(k, j, i,
+                                                           prad->ir);
+
+            // add angular flux
+            if (prad->angle_flag == 1) {
+              prad->pradintegrator->ImplicitAngularFluxes(k,j,i,prad->ir);
+            }
+
+            // add the source terms together
+            prad->pradintegrator->CalSourceTerms(pmb, dt, k,j,i, ph->u,
+                                                 prad->ir1, prad->ir);
+          }
+        }
+      }
+    }
+  } else if (rb_or_not == 2) {
+    // now the black points
+    for (int k=ks; k<=ke; ++k) {
+      for (int j=js; j<=je; ++j) {
+        for (int i=is; i<=ie; ++i) {
+          if (((i-is)+(j-js)+(k-ks))%2 == 1) {
+            prad->pradintegrator->FirstOrderFluxDivergence(k, j, i,
+                                                           prad->ir);
+
+            // add angular flux
+            if (prad->angle_flag == 1) {
+              prad->pradintegrator->ImplicitAngularFluxes(k,j,i,prad->ir);
+            }
+
+            // add the source terms together
+            prad->pradintegrator->CalSourceTerms(pmb, dt, k,j,i, ph->u,
+                                                 prad->ir1, prad->ir);
+          }
+        }
+      }
+    }
+  } else {
+    std::stringstream msg;
+    msg << "### FATAL ERROR in function [Iteration]"
+        << std::endl << "red_or_black '" << rb_or_not << "' not allowed!";
+    ATHENA_ERROR(msg);
+  }
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadITTaskList::ClearRadBoundary(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.ClearBoundary(BoundaryCommSubset::radiation);
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadITTaskList::SendRadBoundary(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.SendBoundaryBuffers();
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadITTaskList::SendRadBoundaryShear(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.SendShearingBoxBoundaryBuffers();
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadITTaskList::ReceiveRadBoundary(MeshBlock *pmb) {
+  bool ret = pmb->pnrrad->rad_bvar.ReceiveBoundaryBuffers();
+  if (!ret) {
+    return TaskStatus::fail;
+  }
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadITTaskList::ReceiveRadBoundaryShear(MeshBlock *pmb) {
+  bool ret = pmb->pnrrad->rad_bvar.ReceiveShearingBoxBoundaryBuffers();
+  if (ret) {
+    pmb->pnrrad->rad_bvar.SetShearingBoxBoundaryBuffers();
+    return TaskStatus::success;
+  } else {
+    return TaskStatus::fail;
+  }
+}
+
+TaskStatus IMRadITTaskList::SetRadBoundary(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.SetBoundaries();
+  return TaskStatus::success;
+}
+
+TaskStatus IMRadITTaskList::CheckResidual(MeshBlock *pmb) {
+  pmy_mesh->pimrad->CheckResidual(pmb, pmb->pnrrad->ir_old,pmb->pnrrad->ir);
+  return TaskStatus::success;
+}
+
+void IMRadITTaskList::StartupTaskList(MeshBlock *pmb) {
+  pmb->pnrrad->rad_bvar.StartReceiving(BoundaryCommSubset::radiation);
+  if (pmy_mesh->shear_periodic) {
+    pmb->pnrrad->rad_bvar.StartReceivingShear(BoundaryCommSubset::radiation);
+  }
+  return;
+}
diff --git a/src/task_list/task_list.hpp b/src/task_list/task_list.hpp
index 4707d0a7d3..230e6d1377 100644
--- a/src/task_list/task_list.hpp
+++ b/src/task_list/task_list.hpp
@@ -59,6 +59,8 @@ class TaskID {  // POD but not aggregate (there is a user-provided ctor)
 
   friend class TaskList;
   friend class MultigridTaskList;
+  friend class IMRadTaskList;
+  friend class IMRadHydroTaskList;
 };
 
 
@@ -120,6 +122,7 @@ class TaskList {
 //! \brief data and function definitions for TimeIntegratorTaskList derived class
 
 class TimeIntegratorTaskList : public TaskList {
+  friend class IMRadiation;
  public:
   TimeIntegratorTaskList(ParameterInput *pin, Mesh *pm);
 
@@ -206,6 +209,37 @@ class TimeIntegratorTaskList : public TaskList {
   TaskStatus ReceiveFieldOrbital(MeshBlock *pmb, int stage);
   TaskStatus CalculateFieldOrbital(MeshBlock *pmb, int stage);
 
+
+// Tasks for radiation transport
+  TaskStatus CalculateRadFlux(MeshBlock *pmb, int stage);
+  TaskStatus IntegrateRad(MeshBlock *pmb, int stage);
+  TaskStatus AddSourceTermsRad(MeshBlock *pmb, int stage);
+  TaskStatus AddSourceTermsIMRad(MeshBlock *pmb, int stage);
+  TaskStatus SendRadFlux(MeshBlock *pmb, int stage);
+  TaskStatus ReceiveAndCorrectRadFlux(MeshBlock *pmb, int stage);
+  TaskStatus SendRad(MeshBlock *pmb, int stage);
+  TaskStatus ReceiveRad(MeshBlock *pmb, int stage);
+  TaskStatus SetBoundariesRad(MeshBlock *pmb, int stage);
+  TaskStatus RadMomOpacity(MeshBlock *pmb, int stage);
+  // shearing box
+  TaskStatus SendRadFluxShear(MeshBlock *pmb, int stage);
+  TaskStatus ReceiveRadFluxShear(MeshBlock *pmb, int stage);
+  TaskStatus SendRadShear(MeshBlock *pmb, int stage);
+  TaskStatus ReceiveRadShear(MeshBlock *pmb, int stage);
+
+
+  // Task functions for cosmic rays
+  TaskStatus CalculateCRTCFlux(MeshBlock *pmb, int stage);
+  TaskStatus IntegrateCRTC(MeshBlock *pmb, int stage);
+  TaskStatus AddSourceTermsCRTC(MeshBlock *pmb, int stage);
+  TaskStatus SendCRTCFlux(MeshBlock *pmb, int stage);
+  TaskStatus ReceiveAndCorrectCRTCFlux(MeshBlock *pmb, int stage);
+  TaskStatus SendCRTC(MeshBlock *pmb, int stage);
+  TaskStatus ReceiveCRTC(MeshBlock *pmb, int stage);
+  TaskStatus SetBoundariesCRTC(MeshBlock *pmb, int stage);
+  TaskStatus CRTCOpacity(MeshBlock *pmb, int stage);
+
+
   bool CheckNextMainStage(int stage) const {return stage_wghts[stage%nstages].main_stage;}
 
  private:
@@ -276,36 +310,45 @@ const TaskID CALC_CHMFLX(5);
 
 const TaskID SEND_HYDFLX(6);
 const TaskID SEND_FLDFLX(7);
-// const TaskID SEND_RADFLX(8);
+const TaskID SEND_RADFLX(8);
+const TaskID SEND_CRTCFLX(9);
 // const TaskID SEND_CHMFLX(9);
 
 const TaskID RECV_HYDFLX(10);
 const TaskID RECV_FLDFLX(11);
-// const TaskID RECV_RADFLX(12);
+const TaskID RECV_RADFLX(12);
+const TaskID RECV_CRTCFLX(13);
+
 // const TaskID RECV_CHMFLX(13);
 
 const TaskID SRC_TERM(14);
-// const TaskID SRCTERM_FLD(15);
-// const TaskID SRCTERM_RAD(16);
-// const TaskID SRCTERM_CHM(17);
+const TaskID SRCTERM_CRTC(15);
+const TaskID SRCTERM_RAD(16);
 
+// const TaskID SRCTERM_CHM(17);
+const TaskID INT_CRTC(17);
 const TaskID INT_HYD(18);
 const TaskID INT_FLD(19);
-// const TaskID INT_RAD(20);
+const TaskID INT_RAD(20);
 // const TaskID INT_CHM(21);
 
+const TaskID SEND_CRTC(21);
 const TaskID SEND_HYD(22);
 const TaskID SEND_FLD(23);
-// const TaskID SEND_RAD(24);
-// const TaskID SEND_CHM(25);
+const TaskID SEND_RAD(24);
 
+// const TaskID SEND_CHM(25);
+const TaskID RECV_CRTC(25);
 const TaskID RECV_HYD(26);
 const TaskID RECV_FLD(27);
+const TaskID RECV_RAD(28);
 // const TaskID RECV_RAD(28);
 // const TaskID RECV_CHM(29);
-
+const TaskID SETB_CRTC(29);
 const TaskID SETB_HYD(30);
 const TaskID SETB_FLD(31);
+const TaskID SETB_RAD(32);
+const TaskID CALC_CRTCFLX(33);
 // const TaskID SETB_RAD(32);
 // const TaskID SETB_CHM(33);
 
@@ -348,5 +391,15 @@ const TaskID SEND_FLDORB(65);
 const TaskID RECV_FLDORB(66);
 const TaskID CALC_FLDORB(67);
 
+const TaskID CRTC_OPACITY(69);
+const TaskID RAD_MOMOPACITY(70);
+
+const TaskID SEND_RADFLXSH(71);
+const TaskID RECV_RADFLXSH(72);
+const TaskID SEND_RADSH(73);
+const TaskID RECV_RADSH(74);
+
+const TaskID SRCTERM_IMRAD(75);
+
 }  // namespace HydroIntegratorTaskNames
 #endif  // TASK_LIST_TASK_LIST_HPP_
diff --git a/src/task_list/time_integrator.cpp b/src/task_list/time_integrator.cpp
index 51bae3bb98..097afa538f 100644
--- a/src/task_list/time_integrator.cpp
+++ b/src/task_list/time_integrator.cpp
@@ -18,6 +18,8 @@
 // Athena++ headers
 #include "../athena.hpp"
 #include "../bvals/bvals.hpp"
+#include "../cr/cr.hpp"
+#include "../cr/integrators/cr_integrators.hpp"
 #include "../eos/eos.hpp"
 #include "../field/field.hpp"
 #include "../field/field_diffusion/field_diffusion.hpp"
@@ -26,6 +28,8 @@
 #include "../hydro/hydro_diffusion/hydro_diffusion.hpp"
 #include "../hydro/srcterms/hydro_srcterms.hpp"
 #include "../mesh/mesh.hpp"
+#include "../nr_radiation/integrators/rad_integrators.hpp"
+#include "../nr_radiation/radiation.hpp"
 #include "../orbital_advection/orbital_advection.hpp"
 #include "../parameter_input.hpp"
 #include "../reconstruct/reconstruction.hpp"
@@ -75,6 +79,9 @@ TimeIntegratorTaskList::TimeIntegratorTaskList(ParameterInput *pin, Mesh *pm) {
   //! - validate Field and Hydro diffusion with RK3, RK4, SSPRK(5,4)
   integrator = pin->GetOrAddString("time", "integrator", "vl2");
 
+  // nr_radiation enabled but not implicit_radiation
+  bool radiation_flag = (NR_RADIATION_ENABLED && (!IM_RADIATION_ENABLED));
+
   // Read a flag for orbital advection
   ORBITAL_ADVECTION = (pm->orbital_advection != 0)? true : false;
 
@@ -929,20 +936,66 @@ TimeIntegratorTaskList::TimeIntegratorTaskList(ParameterInput *pin, Mesh *pm) {
     } else {
       AddTask(INT_HYD, CALC_HYDFLX);
     }
+
+    if (radiation_flag) {
+      AddTask(CALC_RADFLX,NONE);
+      if (pm->multilevel || SHEAR_PERIODIC) { // SMR or AMR or shear periodic
+        AddTask(SEND_RADFLX,CALC_RADFLX);
+        AddTask(RECV_RADFLX,CALC_RADFLX);
+        if (SHEAR_PERIODIC) {
+          AddTask(SEND_RADFLXSH,RECV_RADFLX);
+          AddTask(RECV_RADFLXSH,(SEND_RADFLX|RECV_RADFLX));
+          AddTask(INT_RAD,RECV_RADFLXSH);
+        } else {
+          AddTask(INT_RAD,RECV_RADFLX);
+        }
+      } else {
+        AddTask(INT_RAD, CALC_RADFLX);
+      }
+      AddTask(SRCTERM_RAD,INT_RAD);
+      AddTask(SEND_RAD,SRCTERM_RAD);
+      AddTask(RECV_RAD,NONE);
+      AddTask(SETB_RAD,(RECV_RAD|SRCTERM_RAD));
+    }
+
+    if (CR_ENABLED) {
+      AddTask(CALC_CRTCFLX,NONE);
+      if (pm->multilevel) { // SMR or AMR
+        AddTask(SEND_CRTCFLX,CALC_CRTCFLX);
+        AddTask(RECV_CRTCFLX,CALC_CRTCFLX);
+        AddTask(INT_CRTC,RECV_CRTCFLX);
+      } else {
+        AddTask(INT_CRTC, CALC_CRTCFLX);
+      }
+      AddTask(SRCTERM_CRTC,INT_CRTC);
+      AddTask(SEND_CRTC,SRCTERM_CRTC);
+      AddTask(RECV_CRTC,NONE);
+      AddTask(SETB_CRTC,(RECV_CRTC|SRCTERM_CRTC));
+    }
+
     if (NSCALARS > 0) {
       AddTask(SRC_TERM,(INT_HYD|INT_SCLR));
     } else {
       AddTask(SRC_TERM,INT_HYD);
     }
+
+    // Hydro will also be updated with radiation source term
+    TaskID src_aterm = SRC_TERM;
+    if (radiation_flag)
+      src_aterm = (src_aterm | SRCTERM_RAD);
+
+    if (CR_ENABLED)
+      src_aterm = (src_aterm | SRCTERM_CRTC);
+
     if (ORBITAL_ADVECTION) {
-      AddTask(SEND_HYDORB,SRC_TERM);
+      AddTask(SEND_HYDORB,src_aterm);
       AddTask(RECV_HYDORB,NONE);
       AddTask(CALC_HYDORB,(SEND_HYDORB|RECV_HYDORB));
       AddTask(SEND_HYD,CALC_HYDORB);
       AddTask(RECV_HYD,NONE);
       AddTask(SETB_HYD,(RECV_HYD|CALC_HYDORB));
     } else {
-      AddTask(SEND_HYD,SRC_TERM);
+      AddTask(SEND_HYD,src_aterm);
       AddTask(RECV_HYD,NONE);
       AddTask(SETB_HYD,(RECV_HYD|SRC_TERM));
     }
@@ -950,6 +1003,13 @@ TimeIntegratorTaskList::TimeIntegratorTaskList(ParameterInput *pin, Mesh *pm) {
     if (SHEAR_PERIODIC) {
       AddTask(SEND_HYDSH,SETB_HYD);
       AddTask(RECV_HYDSH,SEND_HYDSH);
+
+      if (radiation_flag) {
+        AddTask(SEND_RADSH,SETB_RAD);
+        // shearing periodic boundary of radiation requires
+        // shearing velocity to be set first
+        AddTask(RECV_RADSH,SEND_RADSH|RECV_HYDSH);
+      }
     }
 
     if (NSCALARS > 0) {
@@ -1016,21 +1076,32 @@ TimeIntegratorTaskList::TimeIntegratorTaskList(ParameterInput *pin, Mesh *pm) {
 
       // prolongate, compute new primitives
       if (pm->multilevel) { // SMR or AMR
+       TaskID setb=(SEND_HYD|SEND_FLD);
         if (SHEAR_PERIODIC) {
           if (NSCALARS > 0) {
-            AddTask(PROLONG,(SEND_HYD|RECV_HYDSH|SEND_FLD|RECV_FLDSH
-                             |SEND_SCLR|RECV_SCLRSH));
+            setb=(setb|RECV_HYDSH|SEND_SCLR|RECV_SCLRSH|RECV_FLDSH);
           } else {
-            AddTask(PROLONG,(SEND_HYD|RECV_HYDSH|SEND_FLD|RECV_FLDSH));
+            setb=(setb|RECV_HYDSH|RECV_FLDSH);
           }
         } else {
           if (NSCALARS > 0) {
-            AddTask(PROLONG,(SEND_HYD|SETB_HYD|SEND_FLD|SETB_FLD
-                             |SEND_SCLR|SETB_SCLR));
+            setb=(setb|SETB_HYD|SETB_FLD|SEND_SCLR|SETB_SCLR);
           } else {
-            AddTask(PROLONG,(SEND_HYD|SETB_HYD|SEND_FLD|SETB_FLD));
+            setb=(setb|SETB_HYD|SETB_FLD);
           }
         }
+
+        if (radiation_flag) {
+          setb=(setb|SEND_RAD|SETB_RAD);
+          if (SHEAR_PERIODIC)
+            setb=(setb|RECV_RADSH);
+        }
+
+        if (CR_ENABLED)
+          setb=(setb|SEND_CRTC|SETB_CRTC);
+
+
+        AddTask(PROLONG,setb);
         AddTask(CONS2PRIM,PROLONG);
       } else {
         if (SHEAR_PERIODIC) {
@@ -1050,19 +1121,31 @@ TimeIntegratorTaskList::TimeIntegratorTaskList(ParameterInput *pin, Mesh *pm) {
     } else {  // HYDRO
       // prolongate, compute new primitives
       if (pm->multilevel) { // SMR or AMR
+        TaskID setb=(SEND_HYD);
         if (SHEAR_PERIODIC) {
           if (NSCALARS > 0) {
-            AddTask(PROLONG,(SEND_HYD|RECV_HYDSH|SEND_SCLR|RECV_SCLRSH));
+            setb=(setb|RECV_HYDSH|SEND_SCLR|RECV_SCLRSH);
           } else {
-            AddTask(PROLONG,(SEND_HYD|RECV_HYDSH));
+            setb=(setb|RECV_HYDSH);
           }
         } else {
           if (NSCALARS > 0) {
-            AddTask(PROLONG,(SEND_HYD|SETB_HYD|SEND_SCLR|SETB_SCLR));
+            setb=(setb|SETB_HYD|SEND_SCLR|SETB_SCLR);
           } else {
-            AddTask(PROLONG,(SEND_HYD|SETB_HYD));
+            setb=(setb|SETB_HYD);
           }
         }
+
+        if (radiation_flag) {
+          setb=(setb|SEND_RAD|SETB_RAD);
+          if (SHEAR_PERIODIC)
+            setb=(setb|RECV_RADSH);
+        }
+
+        if (CR_ENABLED)
+          setb=(setb|SEND_CRTC|SETB_CRTC);
+
+        AddTask(PROLONG,setb);
         AddTask(CONS2PRIM,PROLONG);
       } else {
         if (SHEAR_PERIODIC) {
@@ -1082,9 +1165,32 @@ TimeIntegratorTaskList::TimeIntegratorTaskList(ParameterInput *pin, Mesh *pm) {
     }
 
     // everything else
-    AddTask(PHY_BVAL,CONS2PRIM);
+
+    TaskID before_bval = CONS2PRIM;
+    TaskID before_userwork = PHY_BVAL;
+    if (radiation_flag) {
+      before_bval = (before_bval|SETB_RAD|SEND_RAD);
+      if (SHEAR_PERIODIC)
+        before_bval = (before_bval|RECV_RADSH);
+
+      before_userwork = (before_userwork|RAD_MOMOPACITY);
+    }
+
+    if (CR_ENABLED) {
+      before_bval = (before_bval|SETB_CRTC|SEND_CRTC);
+      before_userwork = (before_userwork|CRTC_OPACITY);
+    }
+
+    AddTask(PHY_BVAL,before_bval);
+
+    if (radiation_flag)
+      AddTask(RAD_MOMOPACITY,PHY_BVAL);
+
+    if (CR_ENABLED)
+      AddTask(CRTC_OPACITY,PHY_BVAL);
+
     if (!STS_ENABLED || pm->sts_integrator == "rkl1") {
-      AddTask(USERWORK,PHY_BVAL);
+      AddTask(USERWORK,before_userwork);
       AddTask(NEW_DT,USERWORK);
       if (pm->adaptive) {
         AddTask(FLAG_AMR,USERWORK);
@@ -1368,6 +1474,116 @@ void TimeIntegratorTaskList::AddTask(const TaskID& id, const TaskID& dep) {
         static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
         (&TimeIntegratorTaskList::CalculateFieldOrbital);
     task_list_[ntasks].lb_time = true;
+  } else if (id == CALC_RADFLX) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::CalculateRadFlux);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == INT_RAD) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::IntegrateRad);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == SEND_RADFLX) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::SendRadFlux);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == RECV_RADFLX) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::ReceiveAndCorrectRadFlux);
+    task_list_[ntasks].lb_time = false;
+  } else if (id == SRCTERM_RAD) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::AddSourceTermsRad);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == SEND_RAD) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::SendRad);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == RECV_RAD) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::ReceiveRad);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == SETB_RAD) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::SetBoundariesRad);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == RAD_MOMOPACITY ) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::RadMomOpacity);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == SEND_RADFLXSH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::SendRadFluxShear);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == RECV_RADFLXSH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::ReceiveRadFluxShear);
+    task_list_[ntasks].lb_time = false;
+  } else if (id == SEND_RADSH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::SendRadShear);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == RECV_RADSH) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::ReceiveRadShear);
+    task_list_[ntasks].lb_time = false;
+  } else if (id == CALC_CRTCFLX) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::CalculateCRTCFlux);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == INT_CRTC) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::IntegrateCRTC);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == SEND_CRTCFLX) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::SendCRTCFlux);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == RECV_CRTCFLX) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::ReceiveAndCorrectCRTCFlux);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == SEND_CRTC) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::SendCRTC);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == RECV_CRTC) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::ReceiveCRTC);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == SRCTERM_CRTC) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::AddSourceTermsCRTC);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == SETB_CRTC) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::SetBoundariesCRTC);
+    task_list_[ntasks].lb_time = true;
+  } else if (id == CRTC_OPACITY) {
+    task_list_[ntasks].TaskFunc=
+        static_cast<TaskStatus (TaskList::*)(MeshBlock*,int)>
+        (&TimeIntegratorTaskList::CRTCOpacity);
+    task_list_[ntasks].lb_time = true;
   } else {
     std::stringstream msg;
     msg << "### FATAL ERROR in AddTask" << std::endl
@@ -1383,6 +1599,7 @@ void TimeIntegratorTaskList::AddTask(const TaskID& id, const TaskID& dep) {
 //! \brief Initialize time abscissae
 
 void TimeIntegratorTaskList::StartupTaskList(MeshBlock *pmb, int stage) {
+  bool radiation_flag = (NR_RADIATION_ENABLED && (!IM_RADIATION_ENABLED));
   if (stage == 1) {
     // Initialize storage registers
     Hydro *ph = pmb->phydro;
@@ -1409,6 +1626,18 @@ void TimeIntegratorTaskList::StartupTaskList(MeshBlock *pmb, int stage) {
       if (integrator == "ssprk5_4")
         ps->s2 = ps->s;
     }
+
+    if (radiation_flag) {
+      pmb->pnrrad->ir1.ZeroClear();
+      if (integrator == "ssprk5_4")
+        pmb->pnrrad->ir2 = pmb->pnrrad->ir;
+    }
+
+    if (CR_ENABLED) {
+      pmb->pcr->u_cr1.ZeroClear();
+      if (integrator == "ssprk5_4")
+        pmb->pcr->u_cr2 = pmb->pcr->u_cr;
+    }
   }
 
   if (SHEAR_PERIODIC) {
@@ -1580,10 +1809,11 @@ TaskStatus TimeIntegratorTaskList::IntegrateHydro(MeshBlock *pmb, int stage) {
       ave_wghts[0] = stage_wghts[stage-1].gamma_1;
       ave_wghts[1] = stage_wghts[stage-1].gamma_2;
       ave_wghts[2] = stage_wghts[stage-1].gamma_3;
-      if (ave_wghts[0] == 0.0 && ave_wghts[1] == 1.0 && ave_wghts[2] == 0.0)
+      if (ave_wghts[0] == 0.0 && ave_wghts[1] == 1.0 && ave_wghts[2] == 0.0) {
         ph->u.SwapAthenaArray(ph->u1);
-      else
+      } else {
         pmb->WeightedAve(ph->u, ph->u1, ph->u2, ph->u0, ph->fl_div, ave_wghts);
+      }
 
       const Real wght = stage_wghts[stage-1].beta*pmb->pmy_mesh->dt;
       ph->AddFluxDivergence(wght, ph->u);
@@ -2156,10 +2386,11 @@ TaskStatus TimeIntegratorTaskList::IntegrateScalars(MeshBlock *pmb, int stage) {
       ave_wghts[0] = stage_wghts[stage-1].gamma_1;
       ave_wghts[1] = stage_wghts[stage-1].gamma_2;
       ave_wghts[2] = stage_wghts[stage-1].gamma_3;
-      if (ave_wghts[0] == 0.0 && ave_wghts[1] == 1.0 && ave_wghts[2] == 0.0)
+      if (ave_wghts[0] == 0.0 && ave_wghts[1] == 1.0 && ave_wghts[2] == 0.0) {
         ps->s.SwapAthenaArray(ps->s1);
-      else
+      } else {
         pmb->WeightedAve(ps->s, ps->s1, ps->s2, ps->s0, ps->s_fl_div, ave_wghts);
+      }
 
       const Real wght = stage_wghts[stage-1].beta*pmb->pmy_mesh->dt;
       ps->AddFluxDivergence(wght, ps->s);
@@ -2405,3 +2636,428 @@ TaskStatus TimeIntegratorTaskList::CalculateFieldOrbital(MeshBlock *pmb, int sta
   }
   return TaskStatus::fail;
 }
+
+
+
+//----------------------------------------------------------------------------------------
+// New steps necessary for radiation/cosmic ray modules
+
+TaskStatus TimeIntegratorTaskList::IntegrateRad(MeshBlock *pmb, int stage) {
+  NRRadiation *prad = pmb->pnrrad;
+
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      Real ave_wghts[3];
+      ave_wghts[0] = 1.0;
+      ave_wghts[1] = stage_wghts[stage-1].delta;
+      ave_wghts[2] = 0.0;
+      pmb->WeightedAve(prad->ir1, prad->ir, prad->ir2, ave_wghts,1);
+
+      ave_wghts[0] = stage_wghts[stage-1].gamma_1;
+      ave_wghts[1] = stage_wghts[stage-1].gamma_2;
+      ave_wghts[2] = stage_wghts[stage-1].gamma_3;
+      if (ave_wghts[0] == 0.0 && ave_wghts[1] == 1.0 && ave_wghts[2] == 0.0) {
+        prad->ir.SwapAthenaArray(prad->ir1);
+      } else {
+        pmb->WeightedAve(prad->ir, prad->ir1, prad->ir2, ave_wghts,1);
+      }
+      const Real wght = stage_wghts[stage-1].beta*pmb->pmy_mesh->dt;
+      // ir is already partially updated
+      prad->pradintegrator->FluxDivergence(wght, prad->ir, prad->ir);
+
+      // no geometric source term for radiation
+
+      // Hardcode an additional flux divergence weighted average for the penultimate
+      // stage of SSPRK(5,4) since it cannot be expressed in a 3S* framework
+      if (stage == 4 && integrator == "ssprk5_4") {
+        // From Gottlieb (2009), u^(n+1) partial calculation
+        ave_wghts[0] = -1.0; // -u^(n) coeff.
+        ave_wghts[1] = 0.0;
+        ave_wghts[2] = 0.0;
+        const Real beta = 0.063692468666290; // F(u^(3)) coeff.
+        const Real wght_ssp = beta*pmb->pmy_mesh->dt;
+        // writing out to u2 register
+        pmb->WeightedAve(prad->ir2, prad->ir1, prad->ir2, ave_wghts,1);
+        prad->pradintegrator->FluxDivergence(wght_ssp, prad->ir2, prad->ir2);
+      }
+    }
+    return TaskStatus::next;
+  }
+  return TaskStatus::fail;
+}
+
+// tasks for radiation transport
+TaskStatus TimeIntegratorTaskList::CalculateRadFlux(MeshBlock *pmb, int stage) {
+  Hydro *phydro = pmb->phydro;
+  NRRadiation *prad = pmb->pnrrad;
+  if (stage_wghts[stage-1].main_stage) {
+    Real dt = (stage_wghts[(stage-1)].beta)*(pmb->pmy_mesh->dt);
+    prad->pradintegrator->GetTgasVel(pmb,dt,phydro->u,phydro->w,
+                                   pmb->pfield->bcc,prad->ir);
+  }
+
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      if ((stage == 1) && (integrator == "vl2")) {
+        prad->pradintegrator->CalculateFluxes(phydro->w,  prad->ir, 1);
+      } else {
+        prad->pradintegrator->CalculateFluxes(phydro->w,  prad->ir,
+                                     prad->pradintegrator->rad_xorder);
+      }
+    }
+    return TaskStatus::next;
+  }
+  return TaskStatus::fail;
+}
+
+
+//----------------------------------------------------------------------------------------
+// Functions to add source terms
+
+TaskStatus TimeIntegratorTaskList::AddSourceTermsRad(MeshBlock *pmb, int stage) {
+  Hydro *ph = pmb->phydro;
+  NRRadiation *prad = pmb->pnrrad;
+
+  int is=pmb->is, ie=pmb->ie;
+  int js=pmb->js, je=pmb->je;
+  int ks=pmb->ks, ke=pmb->ke;
+
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      // Real t_start_stage = pmb->pmy_mesh->time
+      //                      + stage_wghts[(stage-1)].sbeta*pmb->pmy_mesh->dt;
+
+      // Scaled coefficient for RHS update
+      Real dt = (stage_wghts[(stage-1)].beta)*(pmb->pmy_mesh->dt);
+
+      // Evaluate the time-dependent source terms
+      // Both u and ir are partially updated, only w is from the beginning of the step
+      prad->ir_old = prad->ir;
+
+      for(int k=ks; k<=ke; ++k)
+        for(int j=js; j<=je; ++j)
+          for(int i=is; i<=ie; ++i) {
+            prad->pradintegrator->CalSourceTerms(pmb, dt, k, j, i, ph->u,
+                                                       prad->ir, prad->ir);
+          }
+
+      if (prad->set_source_flag > 0) {
+        prad->pradintegrator->GetHydroSourceTerms(pmb, prad->ir_old, prad->ir);
+        prad->pradintegrator->AddSourceTerms(pmb, ph->u);
+      }
+    }
+    return TaskStatus::next;
+  }
+  return TaskStatus::fail;
+}
+
+
+//----------------------------------------------------------------------------------------
+// Functions to communicate fluxes between MeshBlocks for flux correction with AMR
+
+TaskStatus TimeIntegratorTaskList::SendRadFlux(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      pmb->pnrrad->rad_bvar.SendFluxCorrection();
+    }
+    return TaskStatus::success;
+  }
+  return TaskStatus::fail;
+}
+
+
+TaskStatus TimeIntegratorTaskList::ReceiveAndCorrectRadFlux(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      if (pmb->pnrrad->rad_bvar.ReceiveFluxCorrection()) {
+        return TaskStatus::next;
+      } else {
+        return TaskStatus::fail;
+      }
+    } else {
+      return TaskStatus::next;
+    }
+  }
+  return TaskStatus::fail;
+}
+
+
+TaskStatus TimeIntegratorTaskList::SendRad(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    pmb->pnrrad->rad_bvar.SendBoundaryBuffers();
+  } else {
+    return TaskStatus::fail;
+  }
+  return TaskStatus::success;
+}
+
+
+TaskStatus TimeIntegratorTaskList::ReceiveRad(MeshBlock *pmb, int stage) {
+  bool ret;
+  if (stage <= nstages) {
+    ret = pmb->pnrrad->rad_bvar.ReceiveBoundaryBuffers();
+  } else {
+    return TaskStatus::fail;
+  }
+  if (ret) {
+    return TaskStatus::success;
+  } else {
+    return TaskStatus::fail;
+  }
+}
+
+TaskStatus TimeIntegratorTaskList::SetBoundariesRad(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    pmb->pnrrad->rad_bvar.SetBoundaries();
+    return TaskStatus::success;
+  }
+  return TaskStatus::fail;
+}
+
+TaskStatus TimeIntegratorTaskList::RadMomOpacity(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    // pmb->pnrrad->CalculateMoment(pmb->pnrrad->ir);
+    pmb->pnrrad->UpdateOpacity(pmb,pmb->phydro->w);
+    return TaskStatus::success;
+  }
+  return TaskStatus::fail;
+}
+
+
+//----------------------------------------------------------------------------------------
+//! Functions to communicate Field variables between MeshBlocks with shear
+
+TaskStatus TimeIntegratorTaskList::SendRadFluxShear(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      pmb->pnrrad->rad_bvar.SendFluxShearingBoxBoundaryBuffers();
+    }
+    return TaskStatus::success;
+  }
+  return TaskStatus::fail;
+}
+
+
+TaskStatus TimeIntegratorTaskList::ReceiveRadFluxShear(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      if (pmb->pnrrad->rad_bvar.ReceiveFluxShearingBoxBoundaryBuffers()) {
+        pmb->pnrrad->rad_bvar.SetFluxShearingBoxBoundaryBuffers();
+        return TaskStatus::success;
+      } else {
+        return TaskStatus::fail;
+      }
+    } else {
+      return TaskStatus::success;
+    }
+  }
+  return TaskStatus::fail;
+}
+
+
+TaskStatus TimeIntegratorTaskList::SendRadShear(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    pmb->pnrrad->rad_bvar.SendShearingBoxBoundaryBuffers();
+  } else {
+    return TaskStatus::fail;
+  }
+  return TaskStatus::success;
+}
+
+
+TaskStatus TimeIntegratorTaskList::ReceiveRadShear(MeshBlock *pmb, int stage) {
+  bool ret;
+  ret = false;
+  if (stage <= nstages) {
+    ret = pmb->pnrrad->rad_bvar.ReceiveShearingBoxBoundaryBuffers();
+  } else {
+    return TaskStatus::fail;
+  }
+  if (ret) {
+    pmb->pnrrad->rad_bvar.SetShearingBoxBoundaryBuffers();
+    return TaskStatus::success;
+  } else {
+    return TaskStatus::fail;
+  }
+}
+
+//----------------------------------------------------------------------------------
+//function for cosmic ray transport
+
+TaskStatus TimeIntegratorTaskList::CalculateCRTCFlux(MeshBlock *pmb, int stage) {
+  Hydro *phydro = pmb->phydro;
+  CosmicRay *pcr = pmb->pcr;
+  Field *pf = pmb->pfield;
+
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      if ((stage == 1) && (integrator == "vl2")) {
+        if (CR_ENABLED)
+          pcr->pcrintegrator->CalculateFluxes(phydro->w, pf->bcc, pcr->u_cr, 1);
+      } else {
+        if (CR_ENABLED)
+          pcr->pcrintegrator->CalculateFluxes(phydro->w, pf->bcc, pcr->u_cr,
+                                           pcr->pcrintegrator->cr_xorder);
+      }
+    }
+    return TaskStatus::next;
+  }
+  return TaskStatus::fail;
+}
+
+
+
+//----------------------------------------------------------------------------------------
+// Functions to integrate conserved variables
+
+TaskStatus TimeIntegratorTaskList::IntegrateCRTC(MeshBlock *pmb, int stage) {
+  CosmicRay *pcr = pmb->pcr;
+
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      Real ave_wghts[5];
+      ave_wghts[0] = 1.0;
+      ave_wghts[1] = stage_wghts[stage-1].delta;
+      ave_wghts[2] = 0.0;
+      ave_wghts[3] = 0.0;
+      ave_wghts[4] = 0.0;
+      if (CR_ENABLED) {
+        pmb->WeightedAve(pcr->u_cr1, pcr->u_cr, pcr->u_cr2, pcr->u_cr2, pcr->u_cr2,
+                         ave_wghts);
+      }
+      ave_wghts[0] = stage_wghts[stage-1].gamma_1;
+      ave_wghts[1] = stage_wghts[stage-1].gamma_2;
+      ave_wghts[2] = stage_wghts[stage-1].gamma_3;
+      if (ave_wghts[0] == 0.0 && ave_wghts[1] == 1.0 && ave_wghts[2] == 0.0) {
+        if (CR_ENABLED)
+          pcr->u_cr1.SwapAthenaArray(pcr->u_cr);
+      } else {
+        // ave_wghts[3] and ave_wght[4] = 0
+        if (CR_ENABLED)
+          pmb->WeightedAve(pcr->u_cr, pcr->u_cr1, pcr->u_cr2, pcr->u_cr2, pcr->u_cr2,
+                           ave_wghts);
+      }
+      const Real wght = stage_wghts[stage-1].beta*pmb->pmy_mesh->dt;
+      if (CR_ENABLED) {
+        pcr->pcrintegrator->FluxDivergence(wght, pcr->u_cr);
+      }
+    }
+    return TaskStatus::next;
+  }
+  return TaskStatus::fail;
+}
+
+TaskStatus TimeIntegratorTaskList::SendCRTCFlux(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      if (CR_ENABLED)
+        pmb->pcr->cr_bvar.SendFluxCorrection();
+    }
+    return TaskStatus::success;
+  }
+  return TaskStatus::fail;
+}
+
+//----------------------------------------------------------------------------------------
+// Functions to receive fluxes between MeshBlocks
+
+TaskStatus TimeIntegratorTaskList::ReceiveAndCorrectCRTCFlux(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      bool flag_cr = true;
+      if (CR_ENABLED) {
+        flag_cr = pmb->pcr->cr_bvar.ReceiveFluxCorrection();
+      }
+
+      if (flag_cr) {
+        return TaskStatus::next;
+      } else {
+        return TaskStatus::fail;
+      }
+    } else {
+      return TaskStatus::next;
+    }
+  }
+  return TaskStatus::fail;
+}
+
+
+TaskStatus TimeIntegratorTaskList::SendCRTC(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    // Swap Hydro quantity in BoundaryVariable interface back to conserved var formulation
+    // (also needed in SetBoundariesHydro(), since the tasks are independent)
+    if (CR_ENABLED)
+      pmb->pcr->cr_bvar.SendBoundaryBuffers();
+  } else {
+    return TaskStatus::fail;
+  }
+  return TaskStatus::success;
+}
+
+//----------------------------------------------------------------------------------------
+// Functions to receive conserved variables between MeshBlocks
+
+TaskStatus TimeIntegratorTaskList::ReceiveCRTC(MeshBlock *pmb, int stage) {
+  bool ret_cr = true;
+  if (stage <= nstages) {
+    if (CR_ENABLED)
+      ret_cr = pmb->pcr->cr_bvar.ReceiveBoundaryBuffers();
+  } else {
+    return TaskStatus::fail;
+  }
+  if (ret_cr) {
+    return TaskStatus::success;
+  } else {
+    return TaskStatus::fail;
+  }
+}
+
+
+
+
+TaskStatus TimeIntegratorTaskList::SetBoundariesCRTC(MeshBlock *pmb, int stage) {
+  if (stage <= nstages) {
+    if (CR_ENABLED)
+      pmb->pcr->cr_bvar.SetBoundaries();
+    return TaskStatus::success;
+  }
+  return TaskStatus::fail;
+}
+
+
+TaskStatus TimeIntegratorTaskList::CRTCOpacity(MeshBlock *pmb, int stage) {
+  Hydro *ph = pmb->phydro;
+  CosmicRay *pcr = pmb->pcr;
+  Field *pf = pmb->pfield;
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      if (CR_ENABLED)
+        pcr->UpdateOpacity(pmb, pcr->u_cr, ph->w, pf->bcc);
+    }
+    return TaskStatus::success;
+  }
+  return TaskStatus::fail;
+}
+
+
+TaskStatus TimeIntegratorTaskList::AddSourceTermsCRTC(MeshBlock *pmb, int stage) {
+  Hydro *ph = pmb->phydro;
+  CosmicRay *pcr = pmb->pcr;
+  Field *pf = pmb->pfield;
+
+  if (stage <= nstages) {
+    if (stage_wghts[stage-1].main_stage) {
+      // Real t_start_stage = pmb->pmy_mesh->time
+      //                      + stage_wghts[(stage-1)].sbeta*pmb->pmy_mesh->dt;
+      // Scaled coefficient for RHS update
+      Real dt = (stage_wghts[(stage-1)].beta)*(pmb->pmy_mesh->dt);
+
+      // Evaluate the time-dependent source terms
+      // Both u and ir are partially updated, only w is from the beginning of the step
+      if (CR_ENABLED)
+        pcr->pcrintegrator->AddSourceTerms(pmb, dt, ph->u, ph->w, pf->bcc, pcr->u_cr);
+    }
+    return TaskStatus::next;
+  }
+  return TaskStatus::fail;
+}
diff --git a/src/utils/buffer_utils.cpp b/src/utils/buffer_utils.cpp
index 52b55877ac..6a79edc704 100644
--- a/src/utils/buffer_utils.cpp
+++ b/src/utils/buffer_utils.cpp
@@ -26,7 +26,6 @@ template <typename T> void PackData(const AthenaArray<T> &src, T *buf,
   for (int n=sn; n<=en; ++n) {
     for (int k=sk; k<=ek; k++) {
       for (int j=sj; j<=ej; j++) {
-#pragma omp simd
         for (int i=si; i<=ei; i++)
           buf[offset++] = src(n,k,j,i);
       }
@@ -44,7 +43,6 @@ template <typename T> void PackData(const AthenaArray<T> &src, T *buf,
                                     int &offset) {
   for (int k=sk; k<=ek; k++) {
     for (int j=sj; j<=ej; j++) {
-#pragma omp simd
       for (int i=si; i<=ei; i++)
         buf[offset++] = src(k, j, i);
     }
@@ -62,7 +60,6 @@ template <typename T> void UnpackData(const T *buf, AthenaArray<T> &dst,
   for (int n=sn; n<=en; ++n) {
     for (int k=sk; k<=ek; ++k) {
       for (int j=sj; j<=ej; ++j) {
-#pragma omp simd
         for (int i=si; i<=ei; ++i)
           dst(n,k,j,i) = buf[offset++];
       }
@@ -80,7 +77,6 @@ template <typename T> void UnpackData(const T *buf, AthenaArray<T> &dst,
                            int si, int ei, int sj, int ej, int sk, int ek, int &offset) {
   for (int k=sk; k<=ek; ++k) {
     for (int j=sj; j<=ej; ++j) {
-#pragma omp simd
       for (int i=si; i<=ei; ++i)
         dst(k,j,i) = buf[offset++];
     }
diff --git a/src/utils/fourth_polynomial_root.cpp b/src/utils/fourth_polynomial_root.cpp
new file mode 100644
index 0000000000..3c17508771
--- /dev/null
+++ b/src/utils/fourth_polynomial_root.cpp
@@ -0,0 +1,60 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+
+// C headers
+
+// C++ headers
+#include <iostream>
+
+// Athena headers
+#include "../athena.hpp"
+
+//======================================================================================
+//! \file FourthPolyRoot.cpp
+//======================================================================================
+
+// Exact solution for fourth order polynomical with the format
+// coef4 * x^4 + x + tconst == 0
+
+int FouthPolyRoot(const Real coef4, const Real tconst, Real &root) {
+  // First, get the real root of
+  // z^3-4*tconst/coef4 * z - 1/coef4^2==0
+
+  //Real asquar = coef4 * coef4;
+  //Real acubic = coef4 * asquar;
+  Real ccubic = tconst * tconst * tconst;
+  Real delta1 = 0.25 - 64.0 * ccubic * coef4/27.0;
+  if (delta1 < 0.0) {
+    return -1;
+  } else {
+    delta1 = std::sqrt(delta1);
+  }
+  if (delta1 < 0.5) {
+      return -1;
+  }
+  Real zroot = 0.0;
+  if (delta1 > BIG_NUMBER) {
+    // to avoid small number cancellation
+    zroot = std::pow(delta1,-2.0/3.0)/3.0;
+  } else {
+    zroot = (std::pow(0.5 + delta1, 1.0/3.0) -
+             std::pow(-0.5 + delta1, 1.0/3.0));
+  }
+  if ((zroot < 0.0) || (zroot != zroot)) return -1;
+
+  zroot *= std::pow(coef4,-2.0/3.0);
+
+  Real rcoef = std::sqrt(zroot);
+  Real delta2 = -zroot + 2.0/(coef4*rcoef);
+  if (delta2 < 0.0) {
+    return -1;
+  } else {
+    delta2 = std::sqrt(delta2);
+  }
+  root = 0.5 * (delta2 - rcoef);
+  if ((root < 0.0) || (root != root)) return -1;
+  return 0;
+}
diff --git a/src/utils/gauleg.cpp b/src/utils/gauleg.cpp
new file mode 100644
index 0000000000..87e6317f6c
--- /dev/null
+++ b/src/utils/gauleg.cpp
@@ -0,0 +1,53 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+
+// Athena headers
+#include "../athena.hpp"
+
+
+//======================================================================================
+//! \file Gauleg.cpp
+//======================================================================================
+
+
+
+//--------------------------------------------------------------------------------------
+//! \fn  void Gauleg(int n, Real x1, Real x2,  AthenaArray<Real> &x,
+//          AthenaArray<Real> &w);
+// \brief gauss-legendre weight routine
+
+void Gauleg(int n, Real x1, Real x2,  AthenaArray<Real> &x,
+            AthenaArray<Real> &w) {
+  Real eps = 3.0e-14;
+  Real xm, xl, z, z1;
+  Real p1, p2, p3, pp;
+  int m;
+
+  m = (n + 1) / 2;
+  xm = 0.5 * (x2 + x1);
+  xl = 0.5 * (x2 - x1);
+
+  for (int i=1; i<=m; ++i) {
+    z = std::cos(PI * ((Real)i - 0.25) / ((Real)n + 0.5));
+    do {
+      p1=1.0;
+      p2=0.0;
+      for (int j=1; j<=n; ++j) {
+        p3 = p2;
+        p2 = p1;
+        p1 = ((2.0 * (Real)j - 1.0) * z * p2 - ((Real)j - 1.0) * p3) / (Real)j;
+      }
+      pp = (Real)n * (z * p1 - p2) / (z * z - 1.0);
+      z1 = z;
+      z = z1 - p1 / pp;
+    }  while(std::abs(z - z1) > eps);
+    x(i-1) = xm - xl * z;
+    x(n-i) = xm + xl * z;
+    w(i-1) = 2.0 * xl / ((1.0 - z * z) * pp * pp);
+    w(n-i) = w(i-1);
+  }
+  return;
+}
diff --git a/src/utils/matrix_mult.cpp b/src/utils/matrix_mult.cpp
new file mode 100644
index 0000000000..270c2e7b21
--- /dev/null
+++ b/src/utils/matrix_mult.cpp
@@ -0,0 +1,192 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+
+// C++ headers
+#include <sstream>
+#include <stdexcept>
+
+// Athena headers
+#include "../athena.hpp"
+#include "./utils.hpp"
+
+//======================================================================================
+//! \file matrixmult.cpp
+//======================================================================================
+
+//--------------------------------------------------------------------------------------
+//! \fn  void MatrixMult(int m, int n, AthenaArray<Real> &a,
+//    AthenaArray<Real> &b, AthenaArray<Real> &c)
+// \brief perform a(m,n)*b(n)=c(m)
+
+void MatrixMult(int m, int n, AthenaArray<Real> &a,
+                AthenaArray<Real> &b, AthenaArray<Real> &c) {
+  for (int i=0; i<m; ++i) {
+    c(i) = 0.0;
+    for (int j=0; j<n; ++j) {
+      Real& ap = a(i,j);
+      Real& bp = b(j);
+      c(i) += ap * bp;
+    }
+  }
+}
+
+//--------------------------------------------------------------------------------------
+//! \fn  void  InverseMatrix(int n, AthenaArray<Real> &a, AthenaArray<Real> &b)
+// \brief Inverse matrix solver
+// a: input matrix; n: matrix size, b: return matrix
+// Note: the input matrix will be DESTROYED
+
+void InverseMatrix(int n, AthenaArray<Real> &a, AthenaArray<Real> &b) {
+  AthenaArray<int> indx;
+  AthenaArray<Real> col;
+  Real d;
+
+  indx.NewAthenaArray(n);
+  col.NewAthenaArray(n);
+  Ludcmp_nr(n,a,indx,&d);
+
+  for (int j=0; j<n; ++j) {
+    for (int i=0; i<n; ++i) {
+      col(i)=0.0;
+    }
+    col(j) = 1.0;
+    Lubksb_nr(n, a, indx, col);
+    for (int i=0; i<n; i++) {
+      b(i,j) = col(i);
+    }
+  }
+  indx.DeleteAthenaArray();
+  col.DeleteAthenaArray();
+  return;
+}
+
+
+
+//======================================================================================
+//! \file ludcmp.cpp
+//======================================================================================
+
+
+//--------------------------------------------------------------------------------------
+//! \fn  void  Ludcmp_nr(int n, AthenaArray<Real> &a, AthenaArray<int> &indx,
+//          Real *d);
+// \brief LU decomposition
+// Using Crout's method with partial pivoting
+//a is the input matrix, and is returned with LU decomposition readily made,
+// is the matrix size, indx records the history of row permutation,
+// whereas d =1(-1) for even(odd) number of permutations.
+
+void Ludcmp_nr(int n, AthenaArray<Real> &a, AthenaArray<int> &indx,
+               Real *d) {
+  int imax=0;
+  Real big,dum,sum,temp;
+  AthenaArray<Real> rowscale;  // the implicit scaling of each row
+  std::stringstream msg;
+  rowscale.NewAthenaArray(n);
+  *d=1.0;  // No row interchanges yet
+
+  // Loop over rows to get the implicit scaling information
+  for (int i=0; i<n; ++i) {
+    big=0.0;
+    for (int j=0; j<n; ++j)
+      if ((temp=std::abs(a(i,j))) > big) big=temp;
+    if (big == 0.0) {
+      msg << "### [LUdecomp]:Input matrix is singular!"
+      << "### FATAL ERROR in function [LUdecomp_nr]" << std::endl;
+      throw std::runtime_error(msg.str().c_str());
+    }
+    rowscale(i)=1.0/big;  // Save the scaling
+  }
+
+  for (int j=0; j<n; ++j) { // Loop over columns of Crout's method
+    // Calculate the upper block
+    for (int i=0; i<j; ++i) {
+      sum=a(i,j);
+      for (int k=0; k<i; ++k) {
+        Real& aik = a(i,k);
+        Real& akj = a(k,j);
+        sum -= aik * akj;
+      }
+      a(i,j)=sum;
+    }
+    // Calculate the lower block (first step)
+    big=0.0;
+    for (int i=j; i<n; ++i) {
+      sum=a(i,j);
+      for (int k=0; k<j; ++k) {
+        Real& aik = a(i,k);
+        Real& akj = a(k,j);
+        sum -= aik * akj;
+      }
+      a(i,j)=sum;
+      // search for the largest pivot element
+      if ( (dum=rowscale(i)*std::abs(sum)) >= big) {
+        big=dum;
+        imax=i;
+      }
+    }
+    // row interchange
+    if (j != imax) {
+      for (int k=0; k<n; ++k) {
+        dum=a(imax,k);
+        a(imax,k)=a(j,k);
+        a(j,k)=dum;
+      }
+      *d = -(*d);
+      rowscale(imax)=rowscale(j);
+    }
+    indx(j)=imax; // record row interchange history
+    // Calculate the lower block (second step)
+    if (a(j,j) == 0.0) a(j,j)=TINY_NUMBER;
+    dum=1.0/(a(j,j));
+    for (int i=j+1; i<n; ++i)
+      a(i,j) *= dum;
+  }
+  // free the memory
+  rowscale.DeleteAthenaArray();
+}
+
+//======================================================================================
+//! \file lubksb.cpp
+//======================================================================================
+
+
+
+//--------------------------------------------------------------------------------------
+//! \fn  void  Lubksb_nr(int n, AthenaArray<Real> &a, AthenaArray<int> &indx,
+//      AthenaArray<Real> &b)
+// \brief Backward substitution
+// a is the input matrix done with LU decomposition, n is the matrix size
+// indx id the history of row permutation
+// b is the vector on the right (AX=b), and is returned with the solution
+void Lubksb_nr(int n, AthenaArray<Real> &a, AthenaArray<int> &indx,
+               AthenaArray<Real> &b) {
+  int ii=-1,ip;
+  Real sum;
+  // Solve L*y=b
+  for (int i=0; i<n; ++i) {
+    ip = indx(i);
+    sum=b(ip);
+    b(ip)=b(i);
+    if (ii>=0) {
+      for (int j=ii; j<=i-1; ++j) sum -= a(i,j)*b(j);
+    } else if (sum) {
+      ii=i;
+    }
+    b(i)=sum;
+  }
+  // Solve U*x=y
+  for (int i=n-1; i>=0; --i) {
+    sum=b(i);
+    for (int j=i+1; j<n; ++j) {
+      Real& ap = a(i,j);
+      Real& bp = b(j);
+      sum -= ap * bp;
+    }
+    b(i)=sum/a(i,i);
+  }
+}
+
diff --git a/src/utils/permutation.cpp b/src/utils/permutation.cpp
new file mode 100644
index 0000000000..ccedbc89a2
--- /dev/null
+++ b/src/utils/permutation.cpp
@@ -0,0 +1,42 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+
+// Athena headers
+#include "../athena.hpp"
+
+
+//======================================================================================
+//! \file permutation.cpp
+//======================================================================================
+
+//--------------------------------------------------------------------------------------
+//! \fn int Permutation(int i, int j, int k, int np, AthenaArray<int> &pl)
+// \brief permutate the array element
+
+int Permutation(int i, int j, int k, int np, AthenaArray<int> &pl) {
+  int ip = -1;
+  for (int l=0; l<np; l++) {
+    // check each permutation in the table
+    for (int m=0; m<3; m++)
+      if (i == pl(l,m)) {
+        for (int n=0; n<3; n++) {
+          if (n != m) {
+            if (j == pl(l,n)) {
+              for (int o=0; o<3; o++) {
+                if ((o != m) && (o != n)) {
+                  if (k == pl(l,o)) {
+                    ip = l; {
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+  }
+  return ip;
+}
diff --git a/src/utils/rotate_vectors.cpp b/src/utils/rotate_vectors.cpp
new file mode 100644
index 0000000000..230ebe3736
--- /dev/null
+++ b/src/utils/rotate_vectors.cpp
@@ -0,0 +1,71 @@
+//========================================================================================
+// Athena++ astrophysical MHD code
+// Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+
+// Athena headers
+#include "../athena.hpp"
+#include "./utils.hpp"
+
+
+//======================================================================================
+//! \file rotate_vectors.cpp
+//======================================================================================
+
+
+//--------------------------------------------------------------------------------------
+
+void RotateVec(const Real sint, const Real cost,
+               const Real sinp, const Real cosp,
+               Real &v1, Real &v2, Real &v3) {
+  // vel1, vel2, vel3 are input
+  // v1, v2, v3 are output
+  // The two rotation matrix
+  //R_1=
+  //[cos_p  sin_p 0]
+  //[-sin_p cos_p 0]
+  //[0       0    1]
+
+  //R_2=
+  //[sin_t  0 cos_t]
+  //[0      1    0]
+  //[-cos_t 0 sin_t]
+
+  // First apply R1, then apply R2
+  Real newv1 =  cosp * v1 + sinp * v2;
+  v2 = -sinp * v1 + cosp * v2;
+
+  // now apply R2
+  v1 =  sint * newv1 + cost * v3;
+  Real newv3 = -cost * newv1 + sint * v3;
+  v3 = newv3;
+  return;
+}
+
+
+void InvRotateVec(const Real sint, const Real cost,
+                 const Real sinp, const Real cosp,
+                 Real &v1, Real &v2, Real &v3) {
+  // vel1, vel2, vel3 are input
+  // v1, v2, v3 are output
+  // The two rotation matrix
+  //R_1^-1=
+  //[cos_p  -sin_p 0]
+  //[sin_p cos_p 0]
+  //[0       0    1]
+
+  //R_2^-1=
+  //[sin_t  0 -cos_t]
+  //[0      1    0]
+  //[cos_t 0 sin_t]
+
+  // First apply R2^-1, then apply R1^-1
+  Real newv1 = sint * v1 - cost * v3;
+  v3 = cost * v1 + sint * v3;
+
+  // now apply R1^-1
+  v1 = cosp * newv1 - sinp * v2;
+  Real newv2 = sinp * newv1 + cosp * v2;
+  v2 = newv2;
+}
diff --git a/src/utils/show_config.cpp b/src/utils/show_config.cpp
index 9e81df2ab5..f2dae42369 100644
--- a/src/utils/show_config.cpp
+++ b/src/utils/show_config.cpp
@@ -41,6 +41,22 @@ void ShowConfig() {
   } else {
     std::cout<<"  General relativity:         OFF " << std::endl;
   }
+  if (NR_RADIATION_ENABLED) {
+    std::cout<<"  Radiative Transfer:         ON" << std::endl;
+  } else {
+    std::cout<<"  Radiative Transfer:         OFF" << std::endl;
+  }
+  if (IM_RADIATION_ENABLED) {
+    std::cout<<"  Implicit Radiation:         ON" << std::endl;
+  } else {
+    std::cout<<"  Implicit Radiation:         OFF" << std::endl;
+  }
+  if (CR_ENABLED) {
+    std::cout<<"  Cosmic Ray Transport:       ON" << std::endl;
+  } else {
+    std::cout<<"  Cosmic Ray Transport:       OFF" << std::endl;
+  }
+
   // configure.py output: "Frame transformations"
   if (SELF_GRAVITY_ENABLED == 1) {
     std::cout<<"  Self-Gravity:               FFT" << std::endl;
diff --git a/src/utils/utils.hpp b/src/utils/utils.hpp
index ed61124046..a5fca765ce 100644
--- a/src/utils/utils.hpp
+++ b/src/utils/utils.hpp
@@ -13,12 +13,36 @@
 // C++ headers
 #include <csignal>   // sigset_t POSIX C extension
 #include <cstdint>   // std::int64_t
+#include "../athena.hpp"
+#include "../athena_arrays.hpp"
 
 // Athena++ headers
 
 void ChangeRunDir(const char *pdir);
 double ran2(std::int64_t *idum);
 void ShowConfig();
+double MarkTime();
+
+void InverseMatrix(int n, AthenaArray<Real> &a, AthenaArray<Real> &b);
+int FouthPolyRoot(const Real coef4, const Real tconst, Real &root);
+void MatrixMult(int m, int n, AthenaArray<Real> &a,
+                AthenaArray<Real> &b, AthenaArray<Real> &c);
+int Permutation(int i, int j, int k, int np, AthenaArray<int> &pl);
+void Gauleg(int n, Real x1, Real x2,  AthenaArray<Real> &x,
+            AthenaArray<Real> &w);
+void Ludcmp_nr(int n, AthenaArray<Real> &a, AthenaArray<int> &indx,
+               Real *d);
+void Lubksb_nr(int n, AthenaArray<Real> &a, AthenaArray<int> &indx,
+               AthenaArray<Real> &b);
+
+void RotateVec(const Real sint, const Real cost,
+                 const Real sinp, const Real cosp,
+                    Real &v1, Real &v2, Real &v3);
+
+void InvRotateVec(const Real sint, const Real cost,
+                 const Real sinp, const Real cosp,
+                 Real &v1, Real &v2, Real &v3);
+
 
 //----------------------------------------------------------------------------------------
 //! \namespace SignalHandler
diff --git a/tst/ci/jenkins/run_jenkins_stellar.sh b/tst/ci/jenkins/run_jenkins_stellar.sh
index 3a8c2834c6..2e5997342d 100755
--- a/tst/ci/jenkins/run_jenkins_stellar.sh
+++ b/tst/ci/jenkins/run_jenkins_stellar.sh
@@ -81,6 +81,11 @@ time python -u ./run_tests.py mhd --silent  # (mhd/mhd_linwave.py is currenlty t
 time python -u ./run_tests.py shearingbox --silent
 time python -u ./run_tests.py diffusion --silent
 
+time python -u ./run_tests.py cr --silent
+time python -u ./run_tests.py nr_radiation --silent
+time python -u ./run_tests.py implicit_radiation --silent
+time python -u ./run_tests.py multi_group --silent
+
 # High-order solver regression tests w/ GCC
 time python -u ./run_tests.py hydro4 --silent
 
@@ -126,6 +131,11 @@ time python -u ./run_tests.py symmetry --config=--cxx=icpx --silent
 time python -u ./run_tests.py eos --config=--cxx=icpx --silent
 time python -u ./run_tests.py scalars/mignone_radial_1d --config=--cxx=icpx --silent
 
+time python -u ./run_tests.py cr --config=--cxx=icpx --silent
+time python -u ./run_tests.py nr_radiation --config=--cxx=icpx --silent
+time python -u ./run_tests.py implicit_radiation --config=--cxx=icpx --silent
+time python -u ./run_tests.py multi_group --config=--cxx=icpx --silent
+
 # High-order solver regression tests w/ Intel compiler
 time python -u ./run_tests.py hydro4 --config=--cxx=icpx --silent
 
diff --git a/tst/ci/set_warning_cflag.sh b/tst/ci/set_warning_cflag.sh
index 86ba74b35d..06fb324028 100755
--- a/tst/ci/set_warning_cflag.sh
+++ b/tst/ci/set_warning_cflag.sh
@@ -60,6 +60,7 @@ set_warning_cflag () {
 		     # Add even more warnings:
 		     # "-Wconversion"
 		     "-Wshorten-64-to-32"
+		     "-Wno-array-bounds"  # false positives for isothermal problems with wli[4] in hlle.cpp
 		     "-Wno-pass-failed"  # #pragma unroll (in mesh_refinement.cpp in ICX 2021.1.2), #pragma distribute_point (HLLC outside loop only)
 		     "-fp-model=precise"  # Or use -Wno-tautological-constant-compare
 	             # std::isnan() calls in default -fp-model=fast throw errors with -Wall,-Wextra,-Werror
diff --git a/tst/regression/scripts/tests/cr/__init__.py b/tst/regression/scripts/tests/cr/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tst/regression/scripts/tests/cr/cr_diffusion.py b/tst/regression/scripts/tests/cr/cr_diffusion.py
new file mode 100644
index 0000000000..3cc0df9b29
--- /dev/null
+++ b/tst/regression/scripts/tests/cr/cr_diffusion.py
@@ -0,0 +1,109 @@
+# Regression test based on cosmic ray diffusion test problem
+
+# Modules
+import sys
+import scripts.utils.athena as athena
+sys.path.insert(0, '../../vis/python')
+
+# Prepare Athena++
+
+
+def prepare(**kwargs):
+    athena.configure('cr',
+                     prob='cr_diffusion',
+                     coord='cartesian',
+                     flux='hllc')
+    athena.make()
+
+# Run Athena++
+
+
+def run(**kwargs):
+    # case 1: static diffusion along x direction
+    arguments = ['mesh/nx1=256', 'mesh/ix1_bc=outflow', 'mesh/ox1_bc=outflow',
+                 'mesh/nx2=4', 'mesh/ix2_bc=periodic', 'mesh/ox2_bc=periodic',
+                 'meshblock/nx1=32', 'meshblock/nx2=4', 'problem/direction=0',
+                 'problem/v0=0', 'time/ncycle_out=100']
+    athena.run('cosmic_ray/athinput.cr_diffusion', arguments)
+
+    # case 2: dynamic static diffusion along x direction
+    arguments = ['mesh/nx1=256', 'mesh/ix1_bc=outflow', 'mesh/ox1_bc=outflow',
+                 'mesh/nx2=4', 'mesh/ix2_bc=periodic', 'mesh/ox2_bc=periodic',
+                 'meshblock/nx1=32', 'meshblock/nx2=4', 'problem/direction=0',
+                 'problem/v0=1', 'time/ncycle_out=100']
+    athena.run('cosmic_ray/athinput.cr_diffusion', arguments)
+
+    # case 3: static diffusion along y direction
+    arguments = ['mesh/nx1=4', 'mesh/ix1_bc=periodic', 'mesh/ox1_bc=periodic',
+                 'mesh/nx2=256', 'mesh/ix2_bc=outflow', 'mesh/ox2_bc=outflow',
+                 'meshblock/nx1=4', 'meshblock/nx2=32' 'problem/direction=0',
+                 'problem/v0=0', 'time/ncycle_out=100']
+    athena.run('cosmic_ray/athinput.cr_diffusion', arguments)
+
+    # case 4: dynamic static diffusion along y direction
+    arguments = ['mesh/nx1=4', 'mesh/ix1_bc=periodic', 'mesh/ox1_bc=periodic',
+                 'mesh/nx2=256', 'mesh/ix2_bc=outflow', 'mesh/ox2_bc=outflow',
+                 'meshblock/nx1=4', 'meshblock/nx2=32', 'problem/direction=1',
+                 'problem/v0=1', 'time/ncycle_out=100']
+    athena.run('cosmic_ray/athinput.cr_diffusion', arguments)
+
+    # case 5: static diffusion along z direction
+    arguments = ['mesh/nx1=4', 'mesh/ix1_bc=periodic', 'mesh/ox1_bc=periodic',
+                 'mesh/nx2=4', 'mesh/ix2_bc=periodic', 'mesh/ox2_bc=periodic',
+                 'mesh/nx3=256', 'mesh/ix3_bc=outflow', 'mesh/ox3_bc=outflow',
+                 'meshblock/nx1=4', 'meshblock/nx2=4', 'meshblock/nx3=32',
+                 'problem/direction=2', 'problem/v0=0', 'time/ncycle_out=100']
+    athena.run('cosmic_ray/athinput.cr_diffusion', arguments)
+
+    # case 6: dynamic static diffusion along z direction
+    arguments = ['mesh/nx1=4', 'mesh/ix1_bc=periodic', 'mesh/ox1_bc=periodic',
+                 'mesh/nx2=4', 'mesh/ix2_bc=periodic', 'mesh/ox2_bc=periodic',
+                 'mesh/nx3=256', 'mesh/ix3_bc=outflow', 'mesh/ox3_bc=outflow',
+                 'meshblock/nx1=4', 'meshblock/nx2=4', 'meshblock/nx3=32',
+                 'problem/direction=2', 'problem/v0=1', 'time/ncycle_out=100']
+    athena.run('cosmic_ray/athinput.cr_diffusion', arguments)
+
+
+# Analyze outputs
+
+
+def analyze():
+    filename = 'bin/diffusion_error.dat'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+
+    # check absolute error and convergence of all three waves
+    # regime1
+    if data[0][8] > 8e-5:
+        print("error in static diffusion along x: ", data[0][8])
+        return False
+
+        # regime2
+    if data[1][8] > 9e-5:
+        print("error in dynamic diffusion along x: ", data[1][8])
+        return False
+
+    if data[2][8] > 8e-5:
+        print("error in static diffusion along y: ", data[2][8])
+        return False
+
+        # regime2
+    if data[3][8] > 9e-5:
+        print("error in dynamic diffusion along y: ", data[3][8])
+        return False
+
+    if data[4][8] > 8e-5:
+        print("error in static diffusion along z: ", data[4][8])
+        return False
+
+        # regime2
+    if data[5][8] > 9e-5:
+        print("error in dynamic diffusion along z: ", data[5][8])
+        return False
+
+    return True
diff --git a/tst/regression/scripts/tests/implicit_radiation/__init__.py b/tst/regression/scripts/tests/implicit_radiation/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tst/regression/scripts/tests/implicit_radiation/amr_linwave.py b/tst/regression/scripts/tests/implicit_radiation/amr_linwave.py
new file mode 100644
index 0000000000..7c5b215988
--- /dev/null
+++ b/tst/regression/scripts/tests/implicit_radiation/amr_linwave.py
@@ -0,0 +1,51 @@
+# Regression test based on linear wave convergence problem
+# for full radiation hydro equations, using the
+# implicit radiation hydro module and AMR
+
+
+# Modules
+import logging
+import scripts.utils.athena as athena
+import sys
+sys.path.insert(0, '../../vis/python')
+import athena_read  # noqa
+athena_read.check_nan_flag = True
+logger = logging.getLogger('athena' + __name__[7:])  # set logger name based on module
+
+
+# Prepare Athena++
+def prepare(**kwargs):
+    logger.debug('Running test ' + __name__)
+    athena.configure('implicit_radiation',
+                     prob='rad_linearwave',
+                     coord='cartesian',
+                     flux='hllc', **kwargs)
+    athena.make()
+
+
+# Run Athena++
+def run(**kwargs):
+    # L-going fast wave (set by default in input)
+    arguments = ['time/cfl_number=0.3',  # default =0.4, but tolerances measured w/ 0.3
+                 'time/ncycle_out=100'
+                 ]
+    athena.run('radiation/athinput.rad_linearwave_amr', arguments)
+
+
+# Analyze outputs
+def analyze():
+    # read data from error file
+    filename = 'bin/linearwave-errors.dat'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+
+    if data[0][4] > 1.05e-8:
+        print("error in regime 8: ", data[0][4])
+        return False
+
+    return True
diff --git a/tst/regression/scripts/tests/implicit_radiation/rad_linearwave.py b/tst/regression/scripts/tests/implicit_radiation/rad_linearwave.py
new file mode 100644
index 0000000000..73c811daad
--- /dev/null
+++ b/tst/regression/scripts/tests/implicit_radiation/rad_linearwave.py
@@ -0,0 +1,203 @@
+# Regression test based on linear wave convergence problem
+# for full radiation hydro equations, using the
+# implicit radiation hydro module
+
+# Modules
+import sys
+import scripts.utils.athena as athena
+sys.path.insert(0, '../../vis/python')
+
+# Prepare Athena++
+
+
+def prepare(**kwargs):
+    athena.configure('implicit_radiation',
+                     prob='rad_linearwave',
+                     coord='cartesian',
+                     flux='hllc')
+    athena.make()
+
+# Run Athena++
+
+
+def run(**kwargs):
+    # case 1
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=1', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'radiation/error_limit=1.e-15',
+                     'time/tlim=0.7745966144169111', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+#
+#  bashcommand="cp bin/linearwave-errors.dat ~/linearwave-errors_regime1.dat"
+#  os.system(bashcommand)
+
+    # case 2
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=2', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'radiation/error_limit=1.e-15',
+                     'time/tlim=0.7745911961524788', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 3
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=3', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'radiation/error_limit=1.e-15',
+                     'time/tlim=0.7741319089038714', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 4
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=4', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'radiation/error_limit=1.e-15',
+                     'time/tlim=0.7739734114015662', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 5
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=5', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'radiation/error_limit=1.e-15', 'radiation/taucell=15',
+                     'time/tlim=0.7746132569285668', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 6
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=6', 'radiation/prat=1.0', 'radiation/crat=10.0',
+                     'radiation/error_limit=1.e-15',
+                     'time/tlim=0.7747068066908429', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 7
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=7', 'radiation/prat=1.0', 'radiation/crat=10.0',
+                     'radiation/error_limit=1.e-15',
+                     'time/tlim=0.7856204848539599', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 8
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=8', 'radiation/prat=1.0', 'radiation/crat=10.0',
+                     'radiation/error_limit=1.e-15',
+                     'time/tlim=0.9819810298708518', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+#  bashcommand="cp bin/linearwave-errors.dat ~/linearwave-errors.dat"
+#  os.system(bashcommand)
+
+# Analyze outputs
+
+
+def analyze():
+    filename = 'bin/linearwave-errors.dat'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+
+    # check absolute error and convergence of all three waves
+    # regime1
+    if data[3][4] > 8.0e-10:
+        print("error in regime 1: ", data[3][4])
+        return False
+    if data[3][4]/data[2][4] > 0.55:
+        print("not converging for regime 1: ", data[3][4], data[2][4])
+        return False
+
+        # regime2
+    if data[7][4] > 1.2e-9:
+        print("error in regime 2: ", data[7][4])
+        return False
+    if data[7][4]/data[6][4] > 0.55:
+        print("not converging for regime 2: ", data[7][4], data[6][4])
+        return False
+
+    # regime3
+    if data[11][4] > 6.6e-9:
+        print("error in regime 3: ", data[11][4])
+        return False
+    if data[11][4]/data[10][4] > 0.55:
+        print("not converging for regime 3: ", data[11][4], data[10][4])
+        return False
+
+    # regime4
+    if data[15][4] > 8.1e-9:
+        print("error in regime 4: ", data[15][4])
+        return False
+    if data[15][4]/data[14][4] > 0.55:
+        print("not converging for regime 4: ", data[15][4], data[14][4])
+        return False
+
+    # regime5
+    if data[19][4] > 5.0e-9:
+        print("error in regime 5: ", data[19][4])
+        return False
+    if data[19][4]/data[18][4] > 0.55:
+        print("not converging for regime 5: ", data[19][4], data[18][4])
+        return False
+
+    # regime6
+    if data[23][4] > 8.4e-10:
+        print("error in regime 6: ", data[23][4])
+        return False
+    if data[23][4]/data[22][4] > 0.55:
+        print("not converging for regime 6: ", data[23][4], data[22][4])
+        return False
+
+    # regime7
+    if data[27][4] > 2.11e-9:
+        print("error in regime 7: ", data[27][4])
+        return False
+    if data[27][4]/data[26][4] > 0.55:
+        print("not converging for regime 7: ", data[27][4], data[26][4])
+        return False
+
+    # regime8
+    if data[31][4] > 2.1e-9:
+        print("error in regime 8: ", data[31][4])
+        return False
+    if data[31][4]/data[30][4] > 0.55:
+        print("not converging for regime 8: ", data[31][4], data[30][4])
+        return False
+
+    return True
diff --git a/tst/regression/scripts/tests/implicit_radiation/rad_source.py b/tst/regression/scripts/tests/implicit_radiation/rad_source.py
new file mode 100644
index 0000000000..67915422b4
--- /dev/null
+++ b/tst/regression/scripts/tests/implicit_radiation/rad_source.py
@@ -0,0 +1,102 @@
+# Regression test based on thermalize relaxation problem
+# for full radiation hydro equations, using the
+# implicit radiation hydro module
+
+# Modules
+import numpy as np
+import sys
+import scripts.utils.athena as athena
+import os
+sys.path.insert(0, '../../vis/python')
+
+# Prepare Athena++
+
+
+def prepare(**kwargs):
+    athena.configure('implicit_radiation',
+                     prob='thermal_relaxation',
+                     coord='cartesian',
+                     flux='hllc')
+    athena.make()
+
+# Run Athena++
+
+
+def run(**kwargs):
+    # case 1
+    arguments = ['problem/er=10.0', 'problem/tgas=1.0', 'problem/sigma=1.0',
+                 'radiation/prat=0.01', 'radiation/crat=10.0',
+                 'radiation/error_limit=1.e-12',
+                 'time/ncycle_out=100']
+    athena.run('radiation/athinput.thermal_relaxation', arguments)
+    bashcommand = "mv bin/thermal.hst bin/thermal1.hst"
+    os.system(bashcommand)
+    # case 2
+    arguments = ['problem/er=10.0', 'problem/tgas=1.0', 'problem/sigma=100.0',
+                 'radiation/prat=100.0', 'radiation/crat=10.0',
+                 'radiation/error_limit=1.e-12',
+                 'time/ncycle_out=100']
+    athena.run('radiation/athinput.thermal_relaxation', arguments)
+    bashcommand = "mv bin/thermal.hst bin/thermal2.hst"
+    os.system(bashcommand)
+    # case 3
+    arguments = ['problem/er=1.0', 'problem/tgas=10.0', 'problem/sigma=100.0',
+                 'radiation/prat=1.0', 'radiation/crat=10.0',
+                 'radiation/error_limit=1.e-12',
+                 'time/ncycle_out=100']
+    athena.run('radiation/athinput.thermal_relaxation', arguments)
+    bashcommand = "mv bin/thermal.hst bin/thermal3.hst"
+    os.system(bashcommand)
+# Analyze outputs
+
+
+def analyze():
+    filename = 'bin/thermal1.hst'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+    dim = np.shape(data)
+
+    # check absolute error and convergence of all three waves
+    if np.abs(data[dim[0]-1][9]-1.58746) > 1.0e-5 or \
+            + np.abs(data[dim[0]-1][10]-1.25442) > 1.0e-5:
+        print("error in case 1: tgas or Er", data[dim[0]-1][9], data[dim[0]-1][10])
+        return False
+
+    filename = 'bin/thermal2.hst'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+    dim = np.shape(data)
+
+    # check absolute error and convergence of all three waves
+    if np.abs(data[dim[0]-1][9]-2.66664) > 1.0e-3 or \
+            + np.abs(data[dim[0]-1][10]-9.98833) > 1.0e-3:
+        print("error in case 2: tgas or Er", data[dim[0]-1][9], data[dim[0]-1][10])
+        return False
+
+    filename = 'bin/thermal3.hst'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+    dim = np.shape(data)
+
+    # check absolute error and convergence of all three waves
+    if np.abs(data[dim[0]-1][9]-2.85609) > 1.0e-5 or \
+            + np.abs(data[dim[0]-1][10]-13.1439) > 1.0e-4:
+        print("error in case 3: tgas or Er", data[dim[0]-1][9], data[dim[0]-1][10])
+        return False
+
+    return True
diff --git a/tst/regression/scripts/tests/multi_group/rad_source.py b/tst/regression/scripts/tests/multi_group/rad_source.py
new file mode 100644
index 0000000000..677292ef80
--- /dev/null
+++ b/tst/regression/scripts/tests/multi_group/rad_source.py
@@ -0,0 +1,68 @@
+# Regression test based on thermalize relaxation problem
+# for full radiation hydro equations, using the
+# implicit multi-group radiation module
+
+# Modules
+import numpy as np
+import sys
+import scripts.utils.athena as athena
+sys.path.insert(0, '../../vis/python')
+
+# Prepare Athena++
+
+
+def prepare(**kwargs):
+    athena.configure('implicit_radiation',
+                     prob='thermal_multigroup',
+                     coord='cartesian',
+                     flux='hllc')
+    athena.make()
+
+# Run Athena++
+
+
+def run(**kwargs):
+    # case 1
+    arguments = [
+        'problem/er_1=10.0',
+        'problem/er_2=20.0',
+        'problem/er_3=30.0',
+        'problem/tgas=1.0',
+        'problem/sigma_1=100.0',
+        'problem/sigma_2=200.0',
+        'problem/sigma_1=300.0',
+        'radiation/prat=1.0',
+        'radiation/crat=10.0',
+        'radiation/error_limit=1.e-12',
+        'radiation/n_frequency=3',
+        'radiation/frequency_min=-4',
+        'radiation/frequency_max=-8',
+        'time/ncycle_out=100']
+    athena.run('radiation/athinput.thermal_multigroup', arguments)
+
+# Analyze outputs
+
+
+def analyze():
+    filename = 'bin/Averaged_quantity.dat'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+
+    # check absolute error and convergence of all three waves
+    ref_sol = np.array([2.752165, 5.044411, 1.632063e+01, 3.600671e+01])
+    max_err = np.abs(data[0][0]-ref_sol[0])
+    for i in range(1, 4):
+        err = np.abs(data[0][i]-ref_sol[i])
+        if err > max_err:
+            max_err = err
+
+    if max_err > 1.e-5:
+        print("error in case 1: T, Er0, Er1, Er2:", data[0], data[1], data[2], data[3])
+        return False
+
+    return True
diff --git a/tst/regression/scripts/tests/nr_radiation/__init__.py b/tst/regression/scripts/tests/nr_radiation/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tst/regression/scripts/tests/nr_radiation/amr_linwave.py b/tst/regression/scripts/tests/nr_radiation/amr_linwave.py
new file mode 100644
index 0000000000..6050b50588
--- /dev/null
+++ b/tst/regression/scripts/tests/nr_radiation/amr_linwave.py
@@ -0,0 +1,50 @@
+# Regression test based on linear wave convergence problem
+# for full radiation hydro equations, using the
+# explicit radiation hydro module and AMR
+
+# Modules
+import logging
+import scripts.utils.athena as athena
+import sys
+sys.path.insert(0, '../../vis/python')
+import athena_read  # noqa
+athena_read.check_nan_flag = True
+logger = logging.getLogger('athena' + __name__[7:])  # set logger name based on module
+
+
+# Prepare Athena++
+def prepare(**kwargs):
+    logger.debug('Running test ' + __name__)
+    athena.configure('nr_radiation',
+                     prob='rad_linearwave',
+                     coord='cartesian',
+                     flux='hllc', **kwargs)
+    athena.make()
+
+
+# Run Athena++
+def run(**kwargs):
+    # L-going fast wave (set by default in input)
+    arguments = ['time/cfl_number=0.3',  # default =0.4, but tolerances measured w/ 0.3
+                 'time/ncycle_out=100'
+                 ]
+    athena.run('radiation/athinput.rad_linearwave_amr', arguments)
+
+
+# Analyze outputs
+def analyze():
+    # read data from error file
+    filename = 'bin/linearwave-errors.dat'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+
+    if data[0][4] > 1.05e-8:
+        print("error in regime 8: ", data[0][4])
+        return False
+
+    return True
diff --git a/tst/regression/scripts/tests/nr_radiation/rad_linearwave.py b/tst/regression/scripts/tests/nr_radiation/rad_linearwave.py
new file mode 100644
index 0000000000..67e1bd3ba1
--- /dev/null
+++ b/tst/regression/scripts/tests/nr_radiation/rad_linearwave.py
@@ -0,0 +1,197 @@
+# Regression test based on linear wave convergence problem
+# for full radiation hydro equations, using the
+# explicit radiation hydro module
+
+
+# Modules
+import sys
+import scripts.utils.athena as athena
+sys.path.insert(0, '../../vis/python')
+
+# Prepare Athena++
+
+
+def prepare(**kwargs):
+    athena.configure('nr_radiation',
+                     prob='rad_linearwave',
+                     coord='cartesian',
+                     flux='hllc')
+    athena.make()
+
+# Run Athena++
+
+
+def run(**kwargs):
+    # case 1
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=1', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'time/tlim=0.7745966144169111', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+#
+#  bashcommand="cp bin/linearwave-errors.dat ~/linearwave-errors_regime1.dat"
+#  os.system(bashcommand)
+
+    # case 2
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=2', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'time/tlim=0.7745911961524788', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 3
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=3', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'time/tlim=0.7741319089038714', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 4
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=4', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'time/tlim=0.7739734114015662', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 5
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=5', 'radiation/prat=0.01', 'radiation/crat=10.0',
+                     'time/tlim=0.7746132569285668', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 6
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=6', 'radiation/prat=1.0', 'radiation/crat=10.0',
+                     'time/tlim=0.7747068066908429', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 7
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=7', 'radiation/prat=1.0', 'radiation/crat=10.0',
+                     'time/tlim=0.7856204848539599', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+    # case 8
+    for i in (32, 64, 128, 256):
+        arguments = ['problem/regime=8', 'radiation/prat=1.0', 'radiation/crat=10.0',
+                     'time/tlim=0.9819810298708518', 'problem/compute_error=true',
+                     'mesh/nx1=' + repr(i), 'mesh/nx2=8', 'mesh/nx3=1',
+                     'meshblock/nx1=' + repr(i/2),
+                     'meshblock/nx2=8',
+                     'meshblock/nx3=1',
+                     'time/ncycle_out=100']
+        athena.run('radiation/athinput.rad_linearwave', arguments)
+
+#  bashcommand="cp bin/linearwave-errors.dat ~/linearwave-errors.dat"
+#  os.system(bashcommand)
+
+# Analyze outputs
+
+
+def analyze():
+    filename = 'bin/linearwave-errors.dat'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+
+    # check absolute error and convergence of all three waves
+    # regime1
+    if data[3][4] > 1.2e-9:
+        print("error in regime 1: ", data[3][4])
+        return False
+    if data[3][4]/data[2][4] > 0.55:
+        print("not converging for regime 1: ", data[3][4], data[2][4])
+        return False
+
+        # regime2
+    if data[7][4] > 1.2e-9:
+        print("error in regime 2: ", data[7][4])
+        return False
+    if data[7][4]/data[6][4] > 0.55:
+        print("not converging for regime 2: ", data[7][4], data[6][4])
+        return False
+
+    # regime3
+    if data[11][4] > 1.1e-9:
+        print("error in regime 3: ", data[11][4])
+        return False
+    if data[11][4]/data[10][4] > 0.55:
+        print("not converging for regime 3: ", data[11][4], data[10][4])
+        return False
+
+    # regime4
+    if data[15][4] > 1.7e-9:
+        print("error in regime 4: ", data[15][4])
+        return False
+    if data[15][4]/data[14][4] > 0.55:
+        print("not converging for regime 4: ", data[15][4], data[14][4])
+        return False
+
+    # regime5
+    if data[19][4] > 2.2e-9:
+        print("error in regime 5: ", data[19][4])
+        return False
+    if data[19][4]/data[18][4] > 0.55:
+        print("not converging for regime 5: ", data[19][4], data[18][4])
+        return False
+
+    # regime6
+    if data[23][4] > 1.2e-9:
+        print("error in regime 6: ", data[23][4])
+        return False
+    if data[23][4]/data[22][4] > 0.55:
+        print("not converging for regime 6: ", data[23][4], data[22][4])
+        return False
+
+    # regime7
+    if data[27][4] > 8.0e-10:
+        print("error in regime 7: ", data[27][4])
+        return False
+    if data[27][4]/data[26][4] > 0.55:
+        print("not converging for regime 7: ", data[27][4], data[26][4])
+        return False
+
+    # regime8
+    if data[31][4] > 1.3e-9:
+        print("error in regime 8: ", data[31][4])
+        return False
+    if data[31][4]/data[30][4] > 0.55:
+        print("not converging for regime 8: ", data[31][4], data[30][4])
+        return False
+
+    return True
diff --git a/tst/regression/scripts/tests/nr_radiation/rad_source.py b/tst/regression/scripts/tests/nr_radiation/rad_source.py
new file mode 100644
index 0000000000..c987e0d604
--- /dev/null
+++ b/tst/regression/scripts/tests/nr_radiation/rad_source.py
@@ -0,0 +1,96 @@
+# Regression test based on thermalize relaxation problem
+# for full radiation hydro equations, using the
+# explicit radiation hydro module
+
+# Modules
+import numpy as np
+import sys
+import scripts.utils.athena as athena
+import os
+sys.path.insert(0, '../../vis/python')
+
+# Prepare Athena++
+
+
+def prepare(**kwargs):
+    athena.configure('nr_radiation',
+                     prob='thermal_relaxation',
+                     coord='cartesian',
+                     flux='hllc')
+    athena.make()
+
+# Run Athena++
+
+
+def run(**kwargs):
+    # case 1
+    arguments = ['problem/er=10.0', 'problem/tgas=1.0', 'problem/sigma=1.0',
+                 'radiation/prat=0.01', 'radiation/crat=10.0', 'time/ncycle_out=100']
+    athena.run('radiation/athinput.thermal_relaxation', arguments)
+    bashcommand = "mv bin/thermal.hst bin/thermal1.hst"
+    os.system(bashcommand)
+    # case 2
+    arguments = ['problem/er=10.0', 'problem/tgas=1.0', 'problem/sigma=100.0',
+                 'radiation/prat=100.0', 'radiation/crat=10.0', 'time/ncycle_out=100']
+    athena.run('radiation/athinput.thermal_relaxation', arguments)
+    bashcommand = "mv bin/thermal.hst bin/thermal2.hst"
+    os.system(bashcommand)
+    # case 3
+    arguments = ['problem/er=1.0', 'problem/tgas=10.0', 'problem/sigma=100.0',
+                 'radiation/prat=1.0', 'radiation/crat=10.0', 'time/ncycle_out=100']
+    athena.run('radiation/athinput.thermal_relaxation', arguments)
+    bashcommand = "mv bin/thermal.hst bin/thermal3.hst"
+    os.system(bashcommand)
+# Analyze outputs
+
+
+def analyze():
+    filename = 'bin/thermal1.hst'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+    dim = np.shape(data)
+
+    # check absolute error and convergence of all three waves
+    if np.abs(data[dim[0]-1][9]-1.58746) > 1.0e-5 or \
+            + np.abs(data[dim[0]-1][10]-1.25442) > 1.0e-5:
+        print("error in case 1: tgas or Er", data[dim[0]-1][9], data[dim[0]-1][10])
+        return False
+
+    filename = 'bin/thermal2.hst'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+    dim = np.shape(data)
+
+    # check absolute error and convergence of all three waves
+    if np.abs(data[dim[0]-1][9]-2.66664) > 1.0e-5 or \
+            + np.abs(data[dim[0]-1][10]-9.98833) > 1.0e-5:
+        print("error in case 2: tgas or Er", data[dim[0]-1][9], data[dim[0]-1][10])
+        return False
+
+    filename = 'bin/thermal3.hst'
+    data = []
+    with open(filename, 'r') as f:
+        raw_data = f.readlines()
+        for line in raw_data:
+            if line.split()[0][0] == '#':
+                continue
+            data.append([float(val) for val in line.split()])
+    dim = np.shape(data)
+
+    # check absolute error and convergence of all three waves
+    if np.abs(data[dim[0]-1][9]-2.85609) > 1.0e-5 or \
+            + np.abs(data[dim[0]-1][10]-13.1439) > 1.0e-4:
+        print("error in case 3: tgas or Er", data[dim[0]-1][9], data[dim[0]-1][10])
+        return False
+
+    return True
diff --git a/vis/python/plot_mesh.py b/vis/python/plot_mesh.py
index fe20e63bbe..7d51b7d8dc 100755
--- a/vis/python/plot_mesh.py
+++ b/vis/python/plot_mesh.py
@@ -40,7 +40,7 @@ def main(**kwargs):
                 x.append(float(numbers_str[0]))
                 y.append(float(numbers_str[1]))
                 # append zero if 2D
-                if(len(numbers_str) > 2):
+                if (len(numbers_str) > 2):
                     z.append(float(numbers_str[2]))
                 else:
                     z.append(0.0)