spades::SPADES Class Reference

Main SPADES class. More...

#include <SPADES.H>

Inheritance diagram for spades::SPADES:

Collaboration diagram for spades::SPADES:

Public Member Functions
	SPADES ()
	~SPADES () override
void	init_data ()
	Initializes data.
void	evolve ()
	Advance solution to final time.
void	MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
	Make a new level.
void	RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
	Remake an existing level.
void	ClearLevel (int lev) override
	Delete level data.
void	MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
	Make a level from scratch.
void	ErrorEst (int, amrex::TagBoxArray &, amrex::Real, int) override
	Tag cells for refinement.
std::unique_ptr< amrex::MultiFab >	get_field (const std::string &name, const int ngrow)
	Get a field based on a variable name.
void	plot_file_mf ()
	Put together the MultiFab for output.
void	initialize_state ()
	Initialize state.
void	init_particle_containers ()
	Initialize the particle containers.
void	process_messages ()
	Process messages.
void	rollback ()
	Perform rollback.
void	rollback_statistics ()
	Print rollback statistics.
void	update_gvt ()
	Update the global virtual time.
void	update_lbts ()
	Update the Lower Bound on Incoming Time Stamp.

Static Public Member Functions
static amrex::Real	est_time_step ()
	Compute the time step.

Static Public Attributes
static constexpr int	LEV {0}
	Level index.

Private Member Functions
void	read_parameters ()
	Read parameters.
void	compute_dt ()
	Wrapper for EstTimeStep.
void	time_step (const amrex::Real time)
	Advance by the time step.
void	advance (const amrex::Real time, const amrex::Real dt)
	Advance for a single time step.
bool	check_field_existence (const std::string &name)
	Check if a field exists.
void	set_ics ()
	Set the user defined IC functions.
std::string	plot_file_name (const int step) const
	Get plotfile name.
std::string	chk_file_name (const int step) const
	Get checkpoint file name.
amrex::Vector< std::string >	plot_file_var_names () const
	Set plotfile variables names.
void	write_plot_file ()
	Write plotfile to disk.
void	write_checkpoint_file () const
	Write checkpoint file to disk.
void	read_checkpoint_file ()
	Read checkpoint file from disk.
void	write_info_file (const std::string &path) const
	Write job info to disk.
void	write_rng_file (const std::string &path) const
	Write random number generator seed info.
void	read_rng_file (const std::string &path) const
	Read random number generator seed info.
void	init_rng () const
	Initialize the random number generator.
void	write_data_file (const bool is_init) const
	Write simulation information.
void	summary ()
	Compute summary data.

Static Private Member Functions
static void	post_time_step ()
	Perform work after a time step.
static int	get_field_component (const std::string &name, const amrex::Vector< std::string > &varnames)
	Get field component.

Private Attributes
int	m_istep {0}
	Current step.
amrex::Real	m_gvt {constants::LOW_NUM}
	Global virtual time.
amrex::Real	m_t_new {0.0}
	New time.
amrex::Real	m_t_old {constants::LOW_NUM}
	Old time.
amrex::Real	m_dt {constants::LARGE_NUM}
	Time step.
amrex::Real	m_lbts {constants::LOW_NUM}
	Lower bound on incoming time stamp.
amrex::Long	m_ncells {0}
	Cell count.
amrex::Long	m_ntotal_messages {0}
	Total message count.
amrex::Vector< amrex::Long >	m_nmessages
	Message counts.
amrex::Vector< amrex::Long >	m_min_messages
	Min of message counts.
amrex::Vector< amrex::Long >	m_max_messages
	Max of message counts.
amrex::Long	m_nprocessed_messages {0}
	Count of processed messages.
amrex::Long	m_ntotal_entities {0}
	Total entity count.
amrex::Long	m_nentities {0}
	Entity count.
amrex::Vector< int >	m_nrollbacks
	Number of rollbacks.
amrex::Vector< amrex::Real >	m_min_timings
	Min timings for each step.
amrex::Vector< amrex::Real >	m_max_timings
	Max timings for each step.
amrex::Vector< amrex::Real >	m_avg_timings
	Average timings for each step.
int	m_max_step = std::numeric_limits<int>::max()
	Maximum number of steps.
amrex::Real	m_stop_time = std::numeric_limits<amrex::Real>::max()
	Stop time.
amrex::Vector< std::string >	m_spades_varnames
	All variable names for output.
amrex::Vector< std::string >	m_state_varnames
	State variable names for output.
amrex::Vector< std::string >	m_message_counts_varnames
	Message count names for output.
amrex::Vector< std::string >	m_entity_counts_varnames
	Entity count names for output.
amrex::MultiFab	m_state
	Multifabs to store the solution.
amrex::MultiFab	m_plt_mf
	Multifabs for output.
const int	m_state_ngrow = 0
	Number of ghost cells.
std::string	m_restart_chkfile
	Restart file name, restart from this checkpoint if it is not empty.
std::string	m_plot_file {"plt"}
	Plotfile prefix (optional user input)
int	m_plot_int = -1
	Plotfile frequency (optional user input)
std::string	m_chk_file {"chk"}
	Checkpoint prefix (optional user input)
int	m_chk_int = -1
	Checkpoint frequency (optional user input)
int	m_nfiles {constants::TWO_TO_EIGHT}
	Number of plot and checkpoint data files per write.
int	m_file_name_digits {constants::FILE_NAME_DIGITS}
	Digits used in the plotfile and checkpoint file names.
int	m_rng_file_name_digits {constants::RNG_FILE_NAME_DIGITS}
	Digits used in the rng seed file names.
bool	m_write_messages {false}
	Boolean to output messages (optional user input)
bool	m_write_entities {false}
	Boolean to output entities (optional user input)
std::string	m_data_fname {"data.csv"}
	Filename for simulation data.
const int	m_data_precision {6}
	Data precision for data output.
amrex::Real	m_lookahead {1.0}
	Lookahead value (optional user input)
amrex::Real	m_window_size {constants::LARGE_NUM}
	Window size for processing messages (optional user input)
int	m_messages_per_step {1}
	Number of messages to process in each step (optional user input)
int	m_entities_per_lp {1}
	Number of entities per logical process (optional user input)
int	m_messages_per_lp {1}
	Number of messages per logical process (optional user input)
int	m_seed {0}
	Random number generator seed (optional user input)
amrex::Real	m_lambda {1.0}
std::unique_ptr< ic::InitializerBase >	m_ic_op
	Initial condition operator.
std::string	m_ic_type
	Initial condition type (optional user input)
std::unique_ptr< particles::MessageParticleContainer >	m_message_pc
	Message particle container.
std::unique_ptr< particles::EntityParticleContainer >	m_entity_pc
	Entity article container.

Detailed Description

Main SPADES class.

Constructor & Destructor Documentation

SPADES()

spades::SPADES::SPADES

(

)

{
    BL_PROFILE("spades::SPADES::SPADES()");
    read_parameters();
 
    if (max_level > 0) {
        amrex::Abort(
            "spades::SPADES::SPADES(): not supporting multilevel right now");
    }
 
    m_state_varnames.push_back("lvt");
    m_state_varnames.push_back("rollback");
    AMREX_ALWAYS_ASSERT(m_state_varnames.size() == constants::N_STATES);
 
    m_message_counts_varnames.resize(particles::MessageTypes::NTYPES);
    m_message_counts_varnames[particles::MessageTypes::MESSAGE] = "message";
    m_message_counts_varnames[particles::MessageTypes::PROCESSED] =
        "processed_message";
    m_message_counts_varnames[particles::MessageTypes::ANTI] = "anti_message";
    m_message_counts_varnames[particles::MessageTypes::CONJUGATE] =
        "conjugate_message";
    m_message_counts_varnames[particles::MessageTypes::UNDEFINED] =
        "undefined_message";
    for (const auto& mm : m_message_counts_varnames) {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            !mm.empty(), "Message variable name needs to be specified");
    }
    m_nmessages.resize(particles::MessageTypes::NTYPES);
    m_min_messages.resize(particles::MessageTypes::NTYPES);
    m_max_messages.resize(particles::MessageTypes::NTYPES);
 
    m_entity_counts_varnames.resize(particles::EntityTypes::NTYPES);
    m_entity_counts_varnames[particles::EntityTypes::ENTITY] = "entity";
    m_entity_counts_varnames[particles::EntityTypes::BACKUP] = "backup_entity";
    m_entity_counts_varnames[particles::EntityTypes::UNDEFINED] =
        "undefined_entity";
    for (const auto& mm : m_entity_counts_varnames) {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            !mm.empty(), "Entity variable name needs to be specified");
    }
 
    for (const auto& vname : m_state_varnames) {
        m_spades_varnames.push_back(vname);
    }
    for (const auto& vname : m_message_counts_varnames) {
        m_spades_varnames.push_back(vname);
    }
    for (const auto& vname : m_entity_counts_varnames) {
        m_spades_varnames.push_back(vname);
    }
    AMREX_ALWAYS_ASSERT(
        m_spades_varnames.size() ==
        (constants::N_STATES + particles::MessageTypes::NTYPES +
         particles::EntityTypes::NTYPES));
 
    m_nrollbacks.resize(constants::MAX_ROLLBACK_ITERS, 0);
    m_min_timings.resize(2, 0);
    m_max_timings.resize(2, 0);
    m_avg_timings.resize(2, 0);
}

Here is the call graph for this function:

~SPADES()

spades::SPADES::~SPADES ( )

overridedefault

Member Function Documentation

advance()

void spades::SPADES::advance ( const amrex::Real time, const amrex::Real dt )

private

Advance for a single time step.

Parameters

time	[in] time
dt	[in] time step

{
    BL_PROFILE("spades::SPADES::advance()");
 
    m_message_pc->update_undefined();
 
    update_gvt();
    m_message_pc->garbage_collect(m_gvt);
    m_message_pc->sort();
 
    update_lbts();
 
    const auto n_processed_messages_pre =
        m_message_pc->total_count(particles::MessageTypes::PROCESSED);
 
    process_messages();
    m_message_pc->Redistribute();
    m_message_pc->sort();
 
    rollback();
 
    const auto n_processed_messages_post =
        m_message_pc->total_count(particles::MessageTypes::PROCESSED);
 
    m_nprocessed_messages =
        static_cast<int>(n_processed_messages_post - n_processed_messages_pre);
    AMREX_ALWAYS_ASSERT(m_nprocessed_messages >= 0);
 
    m_t_old = m_t_new; // old time is now current time (time)
    m_t_new += dt;     // new time is ahead
}

Here is the call graph for this function:

Here is the caller graph for this function:

check_field_existence()

bool spades::SPADES::check_field_existence ( const std::string & name )

private

Check if a field exists.

Parameters

name	[in] field name

Returns

True if field of name exists

{
    BL_PROFILE("spades::SPADES::check_field_existence()");
    const auto vnames = {
        m_state_varnames, m_message_counts_varnames, m_entity_counts_varnames};
    return std::any_of(vnames.begin(), vnames.end(), [=](const auto& vn) {
        return get_field_component(name, vn) != -1;
    });
}

Here is the caller graph for this function:

chk_file_name()

std::string spades::SPADES::chk_file_name ( const int step ) const

nodiscardprivate

Get checkpoint file name.

Parameters

step	[in] current time step

Returns

chkfile name

{
    return amrex::Concatenate(m_chk_file, step, m_file_name_digits);
}

Here is the caller graph for this function:

ClearLevel()

void spades::SPADES::ClearLevel ( int lev )

override

Delete level data.

Overrides the pure virtual function in AmrCore.

Parameters

lev	[in] level

{
    BL_PROFILE("spades::SPADES::ClearLevel()");
    m_message_pc->clear_state();
    m_entity_pc->clear_state();
    m_state.clear();
    m_plt_mf.clear();
}

compute_dt()

void spades::SPADES::compute_dt ( )

private

Wrapper for EstTimeStep.

{
    BL_PROFILE("spades::SPADES::compute_dt()");
    amrex::Real dt_tmp = est_time_step();
 
    amrex::ParallelDescriptor::ReduceRealMin(dt_tmp);
 
    constexpr amrex::Real change_max = 1.1;
    amrex::Real dt_0 = std::min(dt_tmp, change_max * m_dt);
 
    // Limit dt's by the value of stop_time.
    const amrex::Real eps = 1.e-3 * dt_0;
    if (m_t_new + dt_0 > m_stop_time - eps) {
        dt_0 = m_stop_time - m_t_new;
    }
 
    m_dt = dt_0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

ErrorEst()

void spades::SPADES::ErrorEst ( int , amrex::TagBoxArray & , amrex::Real , int  )

inlineoverride

Tag cells for refinement.

Overrides the pure virtual function in AmrCore

    {}

est_time_step()

amrex::Real spades::SPADES::est_time_step ( )

static

Compute the time step.

Returns

the time step

{
    BL_PROFILE("spades::SPADES::est_time_step()");
    return 1.0;
}

Here is the caller graph for this function:

evolve()

void spades::SPADES::evolve

(

)

Advance solution to final time.

{
    BL_PROFILE("spades::SPADES::evolve()");
 
    amrex::Real cur_time = m_t_new;
    int last_plot_file_step = 0;
    int last_chk_file_step = 0;
 
    for (int step = m_istep; step < m_max_step && cur_time < m_stop_time;
         ++step) {
        compute_dt();
        const amrex::Real start_time = amrex::ParallelDescriptor::second();
 
        amrex::Print() << "\n==============================================="
                          "================================================="
                       << std::endl;
        amrex::Print() << "Starting from step " << step
                       << ", advancing from time " << cur_time << " to time "
                       << cur_time + m_dt << " (dt = " << m_dt << ")"
                       << std::endl;
 
        time_step(cur_time);
 
        post_time_step();
 
        cur_time += m_dt;
 
        // sync up time
        m_t_new = cur_time;
 
        if (m_plot_int > 0 && (step + 1) % m_plot_int == 0) {
            last_plot_file_step = step + 1;
            write_plot_file();
        }
 
        if (m_chk_int > 0 && (step + 1) % m_chk_int == 0) {
            last_chk_file_step = step + 1;
            write_checkpoint_file();
        }
 
        const amrex::Real delta_time =
            amrex::ParallelDescriptor::second() - start_time;
        const auto n_processed_messages = m_nprocessed_messages;
        const amrex::Vector<amrex::Real> timings{
            delta_time,
            static_cast<amrex::Real>(n_processed_messages) / delta_time};
        for (int i = 0; i < timings.size(); i++) {
            m_min_timings[i] = timings[i];
            m_max_timings[i] = timings[i];
            m_avg_timings[i] = timings[i];
        }
 
        amrex::ParallelReduce::Min(
            m_min_timings.data(), static_cast<int>(m_min_timings.size()),
            amrex::ParallelDescriptor::IOProcessorNumber(),
            amrex::ParallelDescriptor::Communicator());
        amrex::ParallelReduce::Max(
            m_max_timings.data(), static_cast<int>(m_max_timings.size()),
            amrex::ParallelDescriptor::IOProcessorNumber(),
            amrex::ParallelDescriptor::Communicator());
        amrex::ParallelReduce::Sum(
            m_avg_timings.data(), static_cast<int>(m_avg_timings.size()),
            amrex::ParallelDescriptor::IOProcessorNumber(),
            amrex::ParallelDescriptor::Communicator());
 
        if (amrex::ParallelDescriptor::IOProcessor()) {
            for (auto& tm : m_avg_timings) {
                tm /= amrex::ParallelDescriptor::NProcs();
            }
        }
 
        write_data_file(false);
        amrex::Print() << "Wallclock time for this step (min, avg, max) [s]: "
                       << m_min_timings[0] << " " << m_avg_timings[0] << " "
                       << m_max_timings[0] << std::endl;
        amrex::Print() << "Current rate (min, avg, max) [msg/s]: "
                       << m_min_timings[1] << " " << m_avg_timings[1] << " "
                       << m_max_timings[1] << std::endl;
 
        if (cur_time >= m_stop_time - constants::TOL * m_dt) {
            break;
        }
    }
    if (m_plot_int > 0 && m_istep > last_plot_file_step) {
        write_plot_file();
    }
    if (m_chk_int > 0 && m_istep > last_chk_file_step) {
        write_checkpoint_file();
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_field()

std::unique_ptr< amrex::MultiFab > spades::SPADES::get_field	(	const std::string &	name,
		const int	ngrow )

Get a field based on a variable name.

Parameters

name	[in] field name
ngrow	[in] number of grow cells

Returns

the requested field

{
    BL_PROFILE("spades::SPADES::get_field()");
 
    if (!check_field_existence(name)) {
        amrex::Abort(
            "spades::SPADES::get_field(): this field was not found: " + name);
    }
 
    const int nc = 1;
    std::unique_ptr<amrex::MultiFab> mf = std::make_unique<amrex::MultiFab>(
        boxArray(LEV), DistributionMap(LEV), nc, ngrow);
 
    const int srccomp_sd = get_field_component(name, m_state_varnames);
    if (srccomp_sd != -1) {
        amrex::MultiFab::Copy(*mf, m_state, srccomp_sd, 0, nc, ngrow);
    }
    const int srccomp_mid =
        get_field_component(name, m_message_counts_varnames);
    if (srccomp_mid != -1) {
        auto const& msg_cnt_arrs = m_message_pc->counts().const_arrays();
        auto const& mf_arrs = mf->arrays();
        amrex::ParallelFor(
            *mf, mf->nGrowVect(), m_message_pc->counts().nComp(),
            [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept {
                mf_arrs[nbx](i, j, k, n) = msg_cnt_arrs[nbx](i, j, k, n);
            });
    }
    const int srccomp_eid = get_field_component(name, m_entity_counts_varnames);
    if (srccomp_eid != -1) {
        auto const& msg_cnt_arrs = m_entity_pc->counts().const_arrays();
        auto const& mf_arrs = mf->arrays();
        amrex::ParallelFor(
            *mf, mf->nGrowVect(), m_entity_pc->counts().nComp(),
            [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept {
                mf_arrs[nbx](i, j, k, n) = msg_cnt_arrs[nbx](i, j, k, n);
            });
    }
    amrex::Gpu::streamSynchronize();
    return mf;
}

Here is the call graph for this function:

get_field_component()

int spades::SPADES::get_field_component ( const std::string & name, const amrex::Vector< std::string > & varnames )

staticprivate

Get field component.

Parameters

name	[in] field name
varnames	[in] vector of the field names

Returns

field component

{
    BL_PROFILE("spades::SPADES::get_field_component()");
    const auto itr = std::find(varnames.begin(), varnames.end(), name);
    if (itr != varnames.cend()) {
        return static_cast<int>(std::distance(varnames.begin(), itr));
    }
    return -1;
}

Here is the caller graph for this function:

init_data()

void spades::SPADES::init_data

(

)

Initializes data.

{
    BL_PROFILE("spades::SPADES::init_data()");
 
    if (m_restart_chkfile.empty()) {
 
        init_rng();
 
        // start simulation from the beginning
        const amrex::Real time = 0.0;
        set_ics();
 
        init_particle_containers();
 
        InitFromScratch(time);
 
        update_gvt();
        update_lbts();
 
        compute_dt();
 
        if (m_chk_int > 0) {
            write_checkpoint_file();
        }
    } else {
        // restart from a checkpoint
        read_checkpoint_file();
    }
 
    if (m_plot_int > 0) {
        write_plot_file();
    }
 
    amrex::Print() << "Grid summary: " << std::endl;
    if (amrex::ParallelDescriptor::IOProcessor()) {
        printGridSummary(amrex::OutStream(), 0, finest_level);
    }
 
    summary();
 
    write_data_file(true);
}

Here is the call graph for this function:

Here is the caller graph for this function:

init_particle_containers()

void spades::SPADES::init_particle_containers

(

)

Initialize the particle containers.

{
    BL_PROFILE("spades::SPADES::init_particle_containers()");
    m_message_pc =
        std::make_unique<particles::MessageParticleContainer>(GetParGDB());
    m_entity_pc =
        std::make_unique<particles::EntityParticleContainer>(GetParGDB());
    m_message_pc->read_parameters();
    m_entity_pc->read_parameters();
}

Here is the caller graph for this function:

init_rng()

void spades::SPADES::init_rng ( ) const

private

Initialize the random number generator.

{
    BL_PROFILE("spades::SPADES::init_rng()");
 
    if (m_seed > 0) {
        amrex::InitRandom(
            m_seed + amrex::ParallelDescriptor::MyProc(),
            amrex::ParallelDescriptor::NProcs(),
            m_seed + amrex::ParallelDescriptor::MyProc());
    } else if (m_seed == 0) {
        auto now = std::chrono::system_clock::now();
        auto now_ns =
            std::chrono::time_point_cast<std::chrono::nanoseconds>(now);
        auto rand_seed = now_ns.time_since_epoch().count();
        amrex::ParallelDescriptor::Bcast(
            &rand_seed, 1, amrex::ParallelDescriptor::IOProcessorNumber());
        amrex::InitRandom(
            rand_seed + amrex::ParallelDescriptor::MyProc(),
            amrex::ParallelDescriptor::NProcs(),
            rand_seed + amrex::ParallelDescriptor::MyProc());
    } else {
        amrex::Abort("RNG seed must be non-negative");
    }
}

Here is the caller graph for this function:

initialize_state()

void spades::SPADES::initialize_state

(

)

Initialize state.

{
    BL_PROFILE("spades::SPADES::initialize_state()");
 
    m_ic_op->initialize(Geom(LEV).data());
 
    m_state.FillBoundary(Geom(LEV).periodicity());
}

Here is the caller graph for this function:

MakeNewLevelFromCoarse()

void spades::SPADES::MakeNewLevelFromCoarse ( int lev, amrex::Real time, const amrex::BoxArray & ba, const amrex::DistributionMapping & dm )

override

Make a new level.

Make a new level using provided BoxArray and DistributionMapping and fill with interpolated coarse level data. Overrides the pure virtual function in AmrCore.

Parameters

lev	[in] level
time	[in] time
ba	[in] box array
dm	[in] distribution map

{
    BL_PROFILE("spades::SPADES::MakeNewLevelFromCoarse()");
    amrex::Abort("spades::SPADES::MakeNewLevelFromCoarse(): not implemented");
}

MakeNewLevelFromScratch()

void spades::SPADES::MakeNewLevelFromScratch ( int lev, amrex::Real time, const amrex::BoxArray & ba, const amrex::DistributionMapping & dm )

override

Make a level from scratch.

Make a new level from scratch using provided BoxArray and DistributionMapping. Only used during initialization. Overrides the pure virtual function in AmrCore

Parameters

lev	[in] level
time	[in] time
ba	[in] box array
dm	[in] distribution map

{
    BL_PROFILE("spades::SPADES::MakeNewLevelFromScratch()");
 
    m_state.define(ba, dm, constants::N_STATES, m_state_ngrow, amrex::MFInfo());
 
    m_t_new = time;
    m_t_old = constants::LOW_NUM;
 
    // Initialize the data
    initialize_state();
 
    // Update message particle container
    m_message_pc->Define(Geom(LEV), dm, ba);
    m_message_pc->initialize_variable_names();
    m_message_pc->initialize_messages(m_lookahead);
    m_message_pc->initialize_state();
    m_message_pc->sort();
 
    // Update entity particle container
    m_entity_pc->Define(Geom(LEV), dm, ba);
    m_entity_pc->initialize_variable_names();
    m_entity_pc->initialize_entities();
    m_entity_pc->initialize_state();
    m_entity_pc->sort();
}

Here is the call graph for this function:

plot_file_mf()

void spades::SPADES::plot_file_mf

(

)

Put together the MultiFab for output.

{
    m_plt_mf.clear();
    m_plt_mf.define(
        boxArray(LEV), DistributionMap(LEV),
        static_cast<int>(plot_file_var_names().size()), 0, amrex::MFInfo());
    m_plt_mf.setVal(0.0);
    int cnt = 0;
    amrex::MultiFab::Copy(m_plt_mf, m_state, 0, cnt, m_state.nComp(), 0);
    cnt += m_state.nComp();
    auto const& plt_mf_arrs = m_plt_mf.arrays();
    auto const& msg_cnt_arrs = m_message_pc->counts().const_arrays();
    amrex::ParallelFor(
        m_plt_mf, m_plt_mf.nGrowVect(), m_message_pc->counts().nComp(),
        [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept {
            plt_mf_arrs[nbx](i, j, k, n + cnt) = msg_cnt_arrs[nbx](i, j, k, n);
        });
    cnt += m_message_pc->counts().nComp();
    auto const& ent_cnt_arrs = m_entity_pc->counts().const_arrays();
    amrex::ParallelFor(
        m_plt_mf, m_plt_mf.nGrowVect(), m_entity_pc->counts().nComp(),
        [=] AMREX_GPU_DEVICE(int nbx, int i, int j, int k, int n) noexcept {
            plt_mf_arrs[nbx](i, j, k, n + cnt) = ent_cnt_arrs[nbx](i, j, k, n);
        });
    amrex::Gpu::streamSynchronize();
}

Here is the call graph for this function:

Here is the caller graph for this function:

plot_file_name()

std::string spades::SPADES::plot_file_name ( const int step ) const

nodiscardprivate

Get plotfile name.

Parameters

step	[in] current time step

Returns

plotfile name

{
    return amrex::Concatenate(m_plot_file, step, m_file_name_digits);
}

Here is the caller graph for this function:

plot_file_var_names()

amrex::Vector< std::string > spades::SPADES::plot_file_var_names ( ) const

nodiscardprivate

Set plotfile variables names.

Returns

vector of variable names

{
    return m_spades_varnames;
}

Here is the caller graph for this function:

post_time_step()

void spades::SPADES::post_time_step ( )

staticprivate

Perform work after a time step.

{
    BL_PROFILE("spades::SPADES::post_time_step()");
}

Here is the caller graph for this function:

process_messages()

void spades::SPADES::process_messages

(

)

Process messages.

{
    BL_PROFILE("spades::SPADES::process_messages()");
    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
        m_state_ngrow == m_message_pc->ngrow(),
        "Particle and state cells must be equal for now");
 
    const auto& plo = Geom(LEV).ProbLoArray();
    const auto& dx = Geom(LEV).CellSizeArray();
    const auto& dom = Geom(LEV).Domain();
    const auto& dlo = dom.smallEnd();
    const auto& dhi = dom.bigEnd();
    const auto lbts = m_lbts;
    const auto lookahead = m_lookahead;
    const auto window_size = m_window_size;
    const auto messages_per_step = m_messages_per_step;
    const auto lambda = m_lambda;
    const auto entities_per_lp = m_entities_per_lp;
 
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (amrex::MFIter mfi = m_message_pc->MakeMFIter(LEV); mfi.isValid();
         ++mfi) {
        const amrex::Box& box = mfi.tilebox();
        const auto index = std::make_pair(mfi.index(), mfi.LocalTileIndex());
        const auto& sarr = m_state.array(mfi);
        const auto& msg_cnt_arr = m_message_pc->counts().const_array(mfi);
        const auto& msg_offsets_arr = m_message_pc->offsets().const_array(mfi);
        const auto& ent_cnt_arr = m_entity_pc->counts().const_array(mfi);
        const auto& ent_offsets_arr = m_entity_pc->offsets().const_array(mfi);
        auto& msg_pti = m_message_pc->GetParticles(LEV)[index];
        const auto msg_parrs = m_message_pc->particle_arrays(msg_pti);
        auto& ent_pti = m_entity_pc->GetParticles(LEV)[index];
        const auto ent_parrs = m_entity_pc->particle_arrays(ent_pti);
 
        amrex::ParallelForRNG(
            box, [=] AMREX_GPU_DEVICE(
                     int i, int j, int AMREX_D_PICK(, , k),
                     amrex::RandomEngine const& engine) noexcept {
                const amrex::IntVect iv(AMREX_D_DECL(i, j, k));
                const auto msg_getter =
                    particles::Get(iv, msg_cnt_arr, msg_offsets_arr, msg_parrs);
                const auto ent_getter =
                    particles::Get(iv, ent_cnt_arr, ent_offsets_arr, ent_parrs);
 
                for (int n = 0;
                     n < amrex::min<int>(
                             msg_cnt_arr(iv, particles::MessageTypes::MESSAGE),
                             messages_per_step);
                     n++) {
 
                    const auto prcv_soa =
                        msg_getter(n, particles::MessageTypes::MESSAGE);
                    const auto ts =
                        msg_parrs.m_rdata[particles::CommonRealData::timestamp]
                                         [prcv_soa];
                    if (ts >= lbts + lookahead + window_size) {
                        break;
                    }
 
                    AMREX_ASSERT(sarr(iv, constants::LVT_IDX) < ts);
 
                    const auto ent =
                        msg_parrs
                            .m_idata[particles::MessageIntData::receiver_entity]
                                    [prcv_soa];
                    const auto pe_soa =
                        ent_getter(ent, particles::EntityTypes::ENTITY);
                    AMREX_ASSERT(
                        dom.atOffset(
                            ent_parrs.m_idata[particles::EntityIntData::owner]
                                             [pe_soa]) == iv);
                    AMREX_ASSERT(
                        ent_parrs.m_rdata[particles::CommonRealData::timestamp]
                                         [pe_soa] < ts);
 
                    // process the event
                    AMREX_ASSERT(
                        dom.atOffset(
                            msg_parrs
                                .m_idata[particles::MessageIntData::receiver_lp]
                                        [prcv_soa]) == iv);
                    msg_parrs.m_rdata[particles::MessageRealData::old_timestamp]
                                     [prcv_soa] =
                        ent_parrs.m_rdata[particles::CommonRealData::timestamp]
                                         [pe_soa];
                    ent_parrs
                        .m_rdata[particles::CommonRealData::timestamp][pe_soa] =
                        ts;
                    msg_parrs
                        .m_idata[particles::CommonIntData::type_id][prcv_soa] =
                        particles::MessageTypes::PROCESSED;
 
                    // Create a new message to send
                    const auto ent_lvt =
                        ent_parrs.m_rdata[particles::CommonRealData::timestamp]
                                         [pe_soa];
                    const amrex::IntVect iv_dest(AMREX_D_DECL(
                        amrex::Random_int(dhi[0] - dlo[0] + 1, engine) + dlo[0],
                        amrex::Random_int(dhi[1] - dlo[1] + 1, engine) + dlo[1],
                        amrex::Random_int(dhi[2] - dlo[2] + 1, engine) +
                            dlo[2]));
                    const amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> pos = {
                        AMREX_D_DECL(
                            plo[0] + (iv_dest[0] + constants::HALF) * dx[0],
                            plo[1] + (iv_dest[1] + constants::HALF) * dx[1],
                            plo[2] + (iv_dest[2] + constants::HALF) * dx[2])};
                    const int rcv_ent = static_cast<int>(
                        amrex::Random_int(entities_per_lp, engine));
                    const amrex::Real next_ts =
                        ent_lvt + random_exponential(lambda, engine) +
                        lookahead;
 
                    const auto psnd_soa =
                        msg_getter(2 * n, particles::MessageTypes::UNDEFINED);
                    particles::CreateMessage()(
                        psnd_soa, msg_parrs, next_ts, pos, iv_dest,
                        static_cast<int>(dom.index(iv)), ent,
                        static_cast<int>(dom.index(iv_dest)), rcv_ent);
                    auto& prcv = msg_parrs.m_aos[prcv_soa];
                    AMREX_ALWAYS_ASSERT(
                        prcv.id() < std::numeric_limits<int>::max());
                    msg_parrs
                        .m_idata[particles::MessageIntData::pair_id][psnd_soa] =
                        static_cast<int>(prcv.id());
                    msg_parrs.m_idata[particles::MessageIntData::pair_cpu]
                                     [psnd_soa] = prcv.cpu();
                    msg_parrs.m_rdata[particles::MessageRealData::creation_time]
                                     [psnd_soa] = ent_lvt;
 
                    // Create the conjugate message
                    const auto pcnj_soa = msg_getter(
                        2 * n + 1, particles::MessageTypes::UNDEFINED);
 
                    // This is weird. The conjugate
                    // position is iv but the receiver_lp is
                    // still updated (we need to know who to
                    // send this to)
                    const amrex::GpuArray<amrex::Real, AMREX_SPACEDIM>
                        conj_pos = {AMREX_D_DECL(
                            plo[0] + (iv[0] + constants::HALF) * dx[0],
                            plo[1] + (iv[1] + constants::HALF) * dx[1],
                            plo[2] + (iv[2] + constants::HALF) * dx[2])};
 
                    particles::CreateMessage()(
                        pcnj_soa, msg_parrs, next_ts, conj_pos, iv,
                        static_cast<int>(dom.index(iv)), ent,
                        static_cast<int>(dom.index(iv_dest)), rcv_ent);
                    msg_parrs
                        .m_idata[particles::MessageIntData::pair_id][pcnj_soa] =
                        static_cast<int>(prcv.id());
                    msg_parrs.m_idata[particles::MessageIntData::pair_cpu]
                                     [pcnj_soa] = prcv.cpu();
                    msg_parrs.m_rdata[particles::MessageRealData::creation_time]
                                     [pcnj_soa] = ent_lvt;
                    msg_parrs
                        .m_idata[particles::CommonIntData::type_id][pcnj_soa] =
                        particles::MessageTypes::CONJUGATE;
                }
 
                // Update LVT for the logical process
                amrex::Real max_ent_lvt = constants::LOW_NUM;
                for (int n = 0;
                     n < ent_cnt_arr(iv, particles::EntityTypes::ENTITY); n++) {
                    const auto pe =
                        ent_getter(n, particles::EntityTypes::ENTITY);
                    const auto ent_lvt =
                        ent_parrs
                            .m_rdata[particles::CommonRealData::timestamp][pe];
                    if (ent_lvt > max_ent_lvt) {
                        max_ent_lvt = ent_lvt;
                    }
                }
                AMREX_ASSERT(sarr(iv, constants::LVT_IDX) <= max_ent_lvt);
                sarr(iv, constants::LVT_IDX) = max_ent_lvt;
            });
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

read_checkpoint_file()

void spades::SPADES::read_checkpoint_file ( )

private

Read checkpoint file from disk.

{
    BL_PROFILE("spades::SPADES::read_checkpoint_file()");
    const auto& varnames = m_state_varnames;
 
    amrex::Print() << "Restarting from checkpoint file " << m_restart_chkfile
                   << std::endl;
 
    // Header
    const std::string file(m_restart_chkfile + "/Header");
 
    amrex::VisMF::IO_Buffer io_buffer(amrex::VisMF::GetIOBufferSize());
 
    amrex::Vector<char> file_char_ptr;
    amrex::ParallelDescriptor::ReadAndBcastFile(file, file_char_ptr);
    std::string file_char_ptr_string(file_char_ptr.dataPtr());
    std::istringstream is(file_char_ptr_string, std::istringstream::in);
 
    std::string line;
    std::string word;
 
    // read in title line
    std::getline(is, line);
 
    // read in finest_level
    is >> finest_level;
    goto_next_line(is);
 
    // read in array of istep
    std::getline(is, line);
    {
        std::istringstream lis(line);
        while (lis >> word) {
            m_istep = std::stoi(word);
        }
    }
 
    // read in array of dt
    std::getline(is, line);
    {
        std::istringstream lis(line);
        while (lis >> word) {
            m_dt = std::stod(word);
        }
    }
 
    // read in array of t_new
    std::getline(is, line);
    {
        std::istringstream lis(line);
        while (lis >> word) {
            m_t_new = std::stod(word);
        }
    }
 
    // read in level 'lev' BoxArray from Header
    amrex::BoxArray ba;
    ba.readFrom(is);
    goto_next_line(is);
 
    // create a distribution mapping
    amrex::DistributionMapping dm{ba, amrex::ParallelDescriptor::NProcs()};
 
    // set BoxArray grids and DistributionMapping dmap in
    // AMReX_AmrMesh.H class
    SetBoxArray(LEV, ba);
    SetDistributionMap(LEV, dm);
 
    // build MultiFabs
    const int ncomp = static_cast<int>(varnames.size());
    AMREX_ALWAYS_ASSERT(ncomp == constants::N_STATES);
    m_state.define(ba, dm, constants::N_STATES, m_state_ngrow, amrex::MFInfo());
 
    // read in the MultiFab data
    amrex::VisMF::Read(
        m_state, amrex::MultiFabFileFullPrefix(
                     LEV, m_restart_chkfile, "Level_", varnames[0]));
    update_gvt();
 
    read_rng_file(m_restart_chkfile);
 
    init_particle_containers();
 
    m_message_pc->initialize_variable_names();
    m_message_pc->Restart(m_restart_chkfile, m_message_pc->identifier());
    m_message_pc->initialize_state();
    m_message_pc->sort();
 
    m_entity_pc->initialize_variable_names();
    m_entity_pc->Restart(m_restart_chkfile, m_entity_pc->identifier());
    m_entity_pc->initialize_state();
    m_entity_pc->sort();
}

Here is the call graph for this function:

Here is the caller graph for this function:

read_parameters()

void spades::SPADES::read_parameters ( )

private

Read parameters.

{
    BL_PROFILE("spades::SPADES::read_parameters()");
 
    {
        amrex::ParmParse pp;
        pp.query("max_step", m_max_step);
        pp.query("stop_time", m_stop_time);
    }
 
    {
        amrex::ParmParse pp("amr");
 
        pp.query("plot_file", m_plot_file);
        pp.query("plot_int", m_plot_int);
        pp.query("chk_file", m_chk_file);
        pp.query("chk_int", m_chk_int);
        pp.query("nfiles", m_nfiles);
        pp.query("restart", m_restart_chkfile);
        pp.query("file_name_digits", m_file_name_digits);
        pp.query("rng_file_name_digits", m_rng_file_name_digits);
        AMREX_ALWAYS_ASSERT(
            m_rng_file_name_digits >=
            std::ceil(std::log10(amrex::ParallelDescriptor::NProcs())));
    }
 
    {
        amrex::ParmParse pp("spades");
        pp.get("ic_type", m_ic_type);
        pp.query("write_messages", m_write_messages);
        pp.query("write_entities", m_write_entities);
        pp.query("seed", m_seed);
        pp.query("lookahead", m_lookahead);
        pp.query("window_size", m_window_size);
        pp.query("messages_per_step", m_messages_per_step);
        pp.query("lambda", m_lambda);
        pp.query("data_fname", m_data_fname);
        pp.query("entities_per_lp", m_entities_per_lp);
        pp.query("messages_per_lp", m_messages_per_lp);
    }
 
    // force periodic bcs
    for (int dir = 0; dir < AMREX_SPACEDIM; dir++) {
        if (!amrex::DefaultGeometry().isPeriodic(dir)) {
            amrex::Abort(
                "spades::SPADES::read_parameters(): geometry needs to be "
                "periodic");
        }
    }
}

Here is the caller graph for this function:

read_rng_file()

void spades::SPADES::read_rng_file ( const std::string & path ) const

private

Read random number generator seed info.

Parameters

path	[in] path for file

{
    BL_PROFILE("spades::SPADES::read_rng_file()");
 
    if (m_seed < 0) {
        const std::string base(path + "/rng");
        const std::string& fname = amrex::Concatenate(
            base, amrex::ParallelDescriptor::MyProc(), m_rng_file_name_digits);
        amrex::Vector<char> file_char_ptr;
        read_file(fname, file_char_ptr, true);
        std::string file_str(file_char_ptr.dataPtr());
        std::istringstream is(file_str, std::istringstream::in);
        amrex::RestoreRandomState(is, 1, 0);
    } else {
        amrex::Warning(
            "Warning: Restarting without using the saved RNG seeds. Run with "
            "spades.seed=-1 to use those");
        AMREX_ASSERT_WITH_MESSAGE(
            m_seed < 0,
            "Use spades.seed=-1 when debugging and using restart files");
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

RemakeLevel()

void spades::SPADES::RemakeLevel ( int lev, amrex::Real time, const amrex::BoxArray & ba, const amrex::DistributionMapping & dm )

override

Remake an existing level.

Remake an existing level using provided BoxArray and DistributionMapping and fill with existing fine and coarse data. Overrides the pure virtual function in AmrCore.

Parameters

lev	[in] level
time	[in] time
ba	[in] box array
dm	[in] distribution map

{
    BL_PROFILE("spades::SPADES::RemakeLevel()");
    amrex::Abort("spades::SPADES::RemakeLevel(): not implemented");
}

rollback()

void spades::SPADES::rollback

(

)

Perform rollback.

{
    BL_PROFILE("spades::SPADES::rollback()");
 
    if (m_window_size <= 0) {
        return;
    }
 
    const auto& plo = Geom(LEV).ProbLoArray();
    const auto& dx = Geom(LEV).CellSizeArray();
    const auto& dom = Geom(LEV).Domain();
 
    amrex::iMultiFab rollback(boxArray(LEV), DistributionMap(LEV), 1, 0);
    rollback.setVal(1.0);
    bool require_rollback = true;
 
    int iter = 0;
    m_state.setVal(0.0, constants::RLB_IDX, 1);
    while (require_rollback && (iter < constants::MAX_ROLLBACK_ITERS)) {
 
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for (amrex::MFIter mfi = m_message_pc->MakeMFIter(LEV); mfi.isValid();
             ++mfi) {
            const amrex::Box& box = mfi.tilebox();
            const auto& sarr = m_state.array(mfi);
            const auto& rarr = rollback.array(mfi);
            const auto& msg_cnt_arr = m_message_pc->counts().const_array(mfi);
            const auto& msg_offsets_arr =
                m_message_pc->offsets().const_array(mfi);
            const auto& ent_cnt_arr = m_entity_pc->counts().const_array(mfi);
            const auto& ent_offsets_arr =
                m_entity_pc->offsets().const_array(mfi);
            const auto index =
                std::make_pair(mfi.index(), mfi.LocalTileIndex());
            auto& msg_pti = m_message_pc->GetParticles(LEV)[index];
            const auto msg_parrs = m_message_pc->particle_arrays(msg_pti);
            auto& ent_pti = m_entity_pc->GetParticles(LEV)[index];
            const auto ent_parrs = m_entity_pc->particle_arrays(ent_pti);
 
            amrex::ParallelFor(
                box, [=] AMREX_GPU_DEVICE(
                         int i, int j, int AMREX_D_PICK(, , k)) noexcept {
                    const amrex::IntVect iv(AMREX_D_DECL(i, j, k));
                    const auto msg_getter = particles::Get(
                        iv, msg_cnt_arr, msg_offsets_arr, msg_parrs);
 
                    const amrex::Real msg_lvt =
                        msg_cnt_arr(iv, particles::MessageTypes::MESSAGE) > 0
                            ? msg_parrs
                                  .m_rdata[particles::CommonRealData::timestamp]
                                          [msg_getter(
                                              0,
                                              particles::MessageTypes::MESSAGE)]
                            : constants::LARGE_NUM;
                    const amrex::Real ant_lvt =
                        msg_cnt_arr(iv, particles::MessageTypes::ANTI) > 0
                            ? msg_parrs
                                  .m_rdata[particles::CommonRealData::timestamp]
                                          [msg_getter(
                                              0, particles::MessageTypes::ANTI)]
                            : constants::LARGE_NUM;
                    const amrex::Real rollback_timestamp =
                        amrex::min<amrex::Real>(msg_lvt, ant_lvt);
 
                    rarr(iv) = 0;
                    if (rollback_timestamp <= sarr(iv, constants::LVT_IDX)) {
                        AMREX_ASSERT(
                            msg_cnt_arr(
                                iv, particles::MessageTypes::PROCESSED) > 0);
                        rarr(iv) = 1;
                        sarr(iv, constants::RLB_IDX) += 1;
 
                        for (int n = 0;
                             n < msg_cnt_arr(
                                     iv, particles::MessageTypes::PROCESSED);
                             n++) {
                            const auto pprd_soa = msg_getter(
                                n, particles::MessageTypes::PROCESSED);
                            if (msg_parrs.m_rdata
                                    [particles::CommonRealData::timestamp]
                                    [pprd_soa] >= rollback_timestamp) {
 
                                auto& pprd = msg_parrs.m_aos[pprd_soa];
                                for (int m = 0;
                                     m <
                                     msg_cnt_arr(
                                         iv,
                                         particles::MessageTypes::CONJUGATE);
                                     m++) {
 
                                    // This is a conjugate message that was
                                    // already treated, expect it to be an anti
                                    // message
                                    if (!msg_getter.check(
                                            m, particles::MessageTypes::
                                                   CONJUGATE)) {
                                        msg_getter.assert_different(
                                            m,
                                            particles::MessageTypes::CONJUGATE,
                                            particles::MessageTypes::ANTI);
                                        continue;
                                    }
 
                                    const auto pcnj_soa = msg_getter(
                                        m, particles::MessageTypes::CONJUGATE);
 
                                    if ((pprd.id() ==
                                         msg_parrs.m_idata
                                             [particles::MessageIntData::
                                                  pair_id][pcnj_soa]) &&
                                        (pprd.cpu() ==
                                         msg_parrs.m_idata
                                             [particles::MessageIntData::
                                                  pair_cpu][pcnj_soa])) {
                                        AMREX_ASSERT(
                                            std::abs(
                                                msg_parrs.m_rdata
                                                    [particles::CommonRealData::
                                                         timestamp][pprd_soa] -
                                                msg_parrs.m_rdata
                                                    [particles::
                                                         MessageRealData::
                                                             creation_time]
                                                    [pcnj_soa]) <
                                            constants::EPS);
                                        msg_parrs.m_idata
                                            [particles::CommonIntData::type_id]
                                            [pcnj_soa] =
                                            particles::MessageTypes::ANTI;
                                        const auto piv_soa = dom.atOffset(
                                            msg_parrs.m_idata
                                                [particles::MessageIntData::
                                                     receiver_lp][pcnj_soa]);
                                        auto& pcnj = msg_parrs.m_aos[pcnj_soa];
                                        AMREX_D_TERM(
                                            pcnj.pos(0) =
                                                plo[0] +
                                                (piv_soa[0] + constants::HALF) *
                                                    dx[0];
                                            , pcnj.pos(1) =
                                                  plo[1] + (piv_soa[1] +
                                                            constants::HALF) *
                                                               dx[1];
                                            , pcnj.pos(2) =
                                                  plo[2] + (piv_soa[2] +
                                                            constants::HALF) *
                                                               dx[2];)
                                        AMREX_D_TERM(
                                            msg_parrs.m_idata
                                                [particles::CommonIntData::i]
                                                [pcnj_soa] = piv_soa[0];
                                            , msg_parrs.m_idata
                                                  [particles::CommonIntData::j]
                                                  [pcnj_soa] = piv_soa[1];
                                            , msg_parrs.m_idata
                                                  [particles::CommonIntData::k]
                                                  [pcnj_soa] = piv_soa[2];)
                                    }
                                }
 
                                msg_parrs
                                    .m_idata[particles::CommonIntData::type_id]
                                            [pprd_soa] =
                                    particles::MessageTypes::MESSAGE;
 
                                // restore the state
                                const auto ent_getter = particles::Get(
                                    iv, ent_cnt_arr, ent_offsets_arr,
                                    ent_parrs);
                                auto pe_soa = ent_getter(
                                    msg_parrs
                                        .m_idata[particles::MessageIntData::
                                                     receiver_entity][pprd_soa],
                                    particles::EntityTypes::ENTITY);
                                ent_parrs.m_rdata[particles::CommonRealData::
                                                      timestamp][pe_soa] =
                                    ent_parrs.m_rdata
                                                [particles::CommonRealData::
                                                     timestamp][pe_soa] >
                                            msg_parrs.m_rdata
                                                [particles::MessageRealData::
                                                     old_timestamp][pprd_soa]
                                        ? msg_parrs.m_rdata
                                              [particles::MessageRealData::
                                                   old_timestamp][pprd_soa]
                                        : ent_parrs.m_rdata
                                              [particles::CommonRealData::
                                                   timestamp][pe_soa];
 
                                amrex::Real max_ent_lvt = constants::LOW_NUM;
                                for (int ne = 0;
                                     ne <
                                     ent_cnt_arr(
                                         iv, particles::EntityTypes::ENTITY);
                                     ne++) {
                                    const auto pe = ent_getter(
                                        ne, particles::EntityTypes::ENTITY);
                                    const auto ent_lvt =
                                        ent_parrs
                                            .m_rdata[particles::CommonRealData::
                                                         timestamp][pe];
                                    if (ent_lvt > max_ent_lvt) {
                                        max_ent_lvt = ent_lvt;
                                    }
                                }
                                sarr(iv, constants::LVT_IDX) =
                                    sarr(iv, constants::LVT_IDX) > max_ent_lvt
                                        ? max_ent_lvt
                                        : sarr(iv, constants::LVT_IDX);
                            }
                        }
                    }
                });
        }
 
        m_message_pc->Redistribute();
        m_message_pc->sort();
        require_rollback = rollback.max(0) > 0;
        iter++;
    }
    if (iter == constants::MAX_ROLLBACK_ITERS) {
        amrex::Abort(
            "Maximum number of rollbacks reached "
            "(constants::MAX_ROLLBACK_ITERS = " +
            std::to_string(constants::MAX_ROLLBACK_ITERS) + ")");
    } else {
        rollback_statistics();
    }
 
    m_message_pc->resolve_pairs();
 
    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
        m_message_pc->counts().sum(particles::MessageTypes::ANTI) == 0,
        "There should be no anti-messages left after rollback");
}

Here is the call graph for this function:

Here is the caller graph for this function:

rollback_statistics()

void spades::SPADES::rollback_statistics

(

)

Print rollback statistics.

{
    BL_PROFILE("spades::SPADES::rollback_statistics()");
 
    auto max_rlb = static_cast<int>(m_state.max(constants::RLB_IDX));
    amrex::ParallelAllReduce::Max<int>(
        max_rlb, amrex::ParallelContext::CommunicatorSub());
 
    amrex::Vector<amrex::Real> nrlbks(max_rlb + 1, 0);
    AMREX_ALWAYS_ASSERT(nrlbks.size() <= m_nrollbacks.size());
    for (int n = 0; n < nrlbks.size(); n++) {
        nrlbks[n] = amrex::ReduceSum(
            m_state, 0,
            [=] AMREX_GPU_HOST_DEVICE(
                const amrex::Box& bx,
                const amrex::Array4<amrex::Real const>& sarr) -> amrex::Real {
                amrex::Real nrlbk = 0;
                amrex::Loop(
                    bx, [=, &nrlbk](
                            int i, int j, int AMREX_D_PICK(, , k)) noexcept {
                        const amrex::IntVect iv(AMREX_D_DECL(i, j, k));
                        nrlbk += (sarr(iv, constants::RLB_IDX) == n) ? 1 : 0;
                    });
                return nrlbk;
            });
    }
    amrex::ParallelDescriptor::ReduceRealSum(
        nrlbks.data(), static_cast<int>(nrlbks.size()),
        amrex::ParallelDescriptor::IOProcessorNumber());
 
    if (amrex::ParallelDescriptor::IOProcessor()) {
        m_nrollbacks.assign(m_nrollbacks.size(), 0);
        for (int n = 0; n < nrlbks.size(); n++) {
            m_nrollbacks[n] = static_cast<int>(nrlbks[n]);
        }
        const auto nta = static_cast<amrex::Long>(
            std::accumulate(nrlbks.begin(), nrlbks.end(), 0.0));
        const auto ntb = static_cast<amrex::Long>(
            std::accumulate(m_nrollbacks.begin(), m_nrollbacks.end(), 0.0));
        AMREX_ALWAYS_ASSERT(nta == boxArray(LEV).numPts());
        AMREX_ALWAYS_ASSERT(nta == ntb);
    }
}

Here is the caller graph for this function:

set_ics()

void spades::SPADES::set_ics ( )

private

Set the user defined IC functions.

{
    BL_PROFILE("spades::SPADES::set_ics()");
    if (m_ic_type == "constant") {
        m_ic_op = std::make_unique<ic::Initializer<ic::Constant>>(
            ic::Constant(ic::Constant()), m_state);
    } else {
        amrex::Abort(
            "spades::SPADES::set_ics(): User must specify a valid initial "
            "condition");
    }
}

Here is the caller graph for this function:

summary()

void spades::SPADES::summary ( )

private

Compute summary data.

{
    BL_PROFILE("spades::SPADES::summary()");
 
    m_ntotal_messages = m_message_pc->TotalNumberOfParticles(LEV != 0);
    for (int typ = 0; typ < particles::MessageTypes::NTYPES; typ++) {
        m_nmessages[typ] = m_message_pc->total_count(typ);
        m_min_messages[typ] = m_message_pc->min_count(typ);
        m_max_messages[typ] = m_message_pc->max_count(typ);
    }
    m_ntotal_entities = m_entity_pc->TotalNumberOfParticles(LEV != 0);
    m_nentities = m_entity_pc->total_count(particles::EntityTypes::ENTITY);
    m_ncells = CountCells(LEV);
    if (Verbose() > 0) {
        amrex::Print() << "Step " << m_istep << " Summary:" << std::endl;
        amrex::Print() << "  " << m_ntotal_messages << " total messages"
                       << std::endl;
        for (int typ = 0; typ < particles::MessageTypes::NTYPES; typ++) {
            amrex::Print() << "  " << m_nmessages[typ] << " "
                           << m_message_counts_varnames[typ]
                           << "s (min: " << m_min_messages[typ]
                           << ", max: " << m_max_messages[typ] << ")"
                           << std::endl;
        }
        amrex::Print() << "  " << m_nprocessed_messages
                       << " messages processed during this step" << std::endl;
        amrex::Print() << "  " << m_ntotal_entities << " total entities"
                       << std::endl;
        amrex::Print() << "  " << m_nentities << " entities" << std::endl;
        amrex::Print() << "  " << m_ncells << " logical processes" << std::endl;
        for (int n = 0; n < m_nrollbacks.size(); n++) {
            const auto nrlbk = m_nrollbacks[n];
            if (nrlbk > 0) {
                amrex::Print() << "  " << nrlbk << " logical processes doing "
                               << n << " rollbacks" << std::endl;
            }
        }
    }
    AMREX_ALWAYS_ASSERT(
        m_nmessages[particles::MessageTypes::MESSAGE] == m_nentities);
}

Here is the caller graph for this function:

time_step()

void spades::SPADES::time_step ( const amrex::Real time )

private

Advance by the time step.

Parameters

time

[in] time

{
    BL_PROFILE("spades::SPADES::time_step()");
 
    amrex::Print() << "Performing step " << m_istep + 1;
    amrex::Print() << " at time = " << m_t_new << ", dt = " << m_dt
                   << ", gvt = " << m_gvt << ", lbts = " << m_lbts << std::endl;
 
    advance(time, m_dt);
 
    ++m_istep;
 
    summary();
}

Here is the call graph for this function:

Here is the caller graph for this function:

update_gvt()

void spades::SPADES::update_gvt

(

)

Update the global virtual time.

{
    BL_PROFILE("spades::SPADES::update_gvt()");
    const amrex::Real gvt = m_message_pc->compute_gvt();
    AMREX_ALWAYS_ASSERT(gvt >= m_gvt);
    AMREX_ALWAYS_ASSERT(gvt >= m_state.min(constants::LVT_IDX, 0));
    m_gvt = gvt;
}

Here is the caller graph for this function:

update_lbts()

void spades::SPADES::update_lbts

(

)

Update the Lower Bound on Incoming Time Stamp.

{
    BL_PROFILE("spades::SPADES::update_lbts()");
 
    amrex::MultiFab lbts(boxArray(LEV), DistributionMap(LEV), 1, 0);
    lbts.setVal(constants::LARGE_NUM);
 
#ifdef AMREX_USE_OMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (amrex::MFIter mfi = m_message_pc->MakeMFIter(LEV); mfi.isValid();
         ++mfi) {
        const amrex::Box& box = mfi.tilebox();
        const auto& msg_cnt_arr = m_message_pc->counts().const_array(mfi);
        const auto& msg_offsets_arr = m_message_pc->offsets().const_array(mfi);
        const auto& lbts_arr = lbts.array(mfi);
        const auto index = std::make_pair(mfi.index(), mfi.LocalTileIndex());
        auto& msg_pti = m_message_pc->GetParticles(LEV)[index];
        const auto msg_parrs = m_message_pc->particle_arrays(msg_pti);
 
        amrex::ParallelFor(
            box, [=] AMREX_GPU_DEVICE(
                     int i, int j, int AMREX_D_PICK(, , k)) noexcept {
                const amrex::IntVect iv(AMREX_D_DECL(i, j, k));
 
                if (msg_cnt_arr(iv, particles::MessageTypes::MESSAGE) > 0) {
                    const auto msg_getter = particles::Get(
                        iv, msg_cnt_arr, msg_offsets_arr, msg_parrs);
                    const auto prcv_soa =
                        msg_getter(0, particles::MessageTypes::MESSAGE);
                    lbts_arr(iv) =
                        msg_parrs.m_rdata[particles::CommonRealData::timestamp]
                                         [prcv_soa];
                }
            });
    }
 
    m_lbts = lbts.min(0, 0);
    AMREX_ALWAYS_ASSERT(m_lbts >= m_gvt);
}

Here is the caller graph for this function:

write_checkpoint_file()

void spades::SPADES::write_checkpoint_file ( ) const

private

Write checkpoint file to disk.

{
    BL_PROFILE("spades::SPADES::write_checkpoint_file()");
    const auto& varnames = m_state_varnames;
 
    // chk00010            write a checkpoint file with this root directory
    // chk00010/Header     this contains information you need to save (e.g.,
    // finest_level, t_new, etc.) and also
    //                     the BoxArrays at each level
    // chk00010/Level_0/
    // chk00010/Level_1/
    // etc.                these subdirectories will hold the MultiFab data
    // at each level of refinement
 
    const std::string& checkpointname = chk_file_name(m_istep);
 
    amrex::Print() << "Writing checkpoint file " << checkpointname
                   << " at time " << m_t_new << std::endl;
 
    const int nlevels = finest_level + 1;
 
    // ---- prebuild a hierarchy of directories
    // ---- dirName is built first.  if dirName exists, it is renamed.  then
    // build
    // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
    // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
    // ---- after all directories are built
    // ---- ParallelDescriptor::IOProcessor() creates the directories
    amrex::PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
 
    // write Header file
    if (amrex::ParallelDescriptor::IOProcessor()) {
 
        const std::string header_file_name(checkpointname + "/Header");
        amrex::VisMF::IO_Buffer io_buffer(amrex::VisMF::IO_Buffer_Size);
        std::ofstream header_file;
        header_file.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
        header_file.open(
            header_file_name.c_str(),
            std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
 
        if (!header_file.good()) {
            amrex::FileOpenFailed(header_file_name);
        }
 
        header_file.precision(constants::HEADER_FILE_PRECISION);
 
        // write out title line
        header_file << "Checkpoint file for SPADES\n";
 
        // write out finest_level
        header_file << finest_level << "\n";
 
        // write out istep
        header_file << m_istep << " ";
        header_file << "\n";
 
        // write out dt
        header_file << m_dt << " ";
        header_file << "\n";
 
        // write out t_new
        header_file << m_t_new << " ";
        header_file << "\n";
 
        // write the BoxArray at each level
 
        boxArray(LEV).writeOn(header_file);
        header_file << '\n';
        header_file.close();
    }
 
    // write the MultiFab data to, e.g., chk00010/Level_0/
 
    amrex::VisMF::Write(
        m_state, amrex::MultiFabFileFullPrefix(
                     LEV, checkpointname, "Level_", varnames[0]));
 
    m_message_pc->Checkpoint(checkpointname, m_message_pc->identifier());
 
    m_entity_pc->Checkpoint(checkpointname, m_entity_pc->identifier());
 
    write_info_file(checkpointname);
 
    write_rng_file(checkpointname);
}

Here is the call graph for this function:

Here is the caller graph for this function:

write_data_file()

void spades::SPADES::write_data_file ( const bool is_init ) const

private

Write simulation information.

Parameters

is_init

[in] boolean indicating if this is the initializing step

{
    BL_PROFILE("spades::SPADES::write_data_file()");
 
    if (amrex::ParallelDescriptor::IOProcessor()) {
 
        amrex::Print() << "Writing simulation data to " << m_data_fname
                       << " at time " << m_t_new << std::endl;
 
        if (is_init) {
            std::ofstream fh(m_data_fname.c_str(), std::ios::out);
            fh.precision(m_data_precision);
            fh << "step,gvt,lbts,lps,total_messages,";
            for (int typ = 0; typ < particles::MessageTypes::NTYPES; typ++) {
                fh << m_message_counts_varnames[typ] + "s,"
                   << "min_" + m_message_counts_varnames[typ] + "s,"
                   << "max_" + m_message_counts_varnames[typ] + "s,";
            }
            fh << "total_entities,"
                  "entities,"
                  "processed_messages_per_step,"
                  "min_time,avg_time,max_time,min_rate,avg_"
                  "rate,"
                  "max_rate";
            for (int i = 0; i < m_nrollbacks.size(); i++) {
                fh << ",rollback_" << i;
            }
            fh << "\n";
            fh.close();
        }
 
        std::ofstream fh(m_data_fname.c_str(), std::ios::app);
        fh.precision(m_data_precision);
        if (!fh.good()) {
            amrex::FileOpenFailed(m_data_fname);
        }
 
        const auto n_processed_messages = m_nprocessed_messages;
        fh << m_t_new << "," << m_gvt << "," << m_lbts << "," << m_ncells << ","
           << m_ntotal_messages << ",";
        for (int typ = 0; typ < particles::MessageTypes::NTYPES; typ++) {
            fh << m_nmessages[typ] << "," << m_min_messages[typ] << ","
               << m_max_messages[typ] << ",";
        }
        fh << n_processed_messages << "," << m_ntotal_entities << ","
           << m_nentities << "," << m_min_timings[0] << "," << m_avg_timings[0]
           << "," << m_max_timings[0] << "," << m_min_timings[1] << ","
           << m_avg_timings[1] << "," << m_max_timings[1];
 
        for (const auto& rlbk : m_nrollbacks) {
            fh << "," << rlbk;
        }
        fh << "\n";
        fh.close();
    }
}

Here is the caller graph for this function:

write_info_file()

void spades::SPADES::write_info_file ( const std::string & path ) const

private

Write job info to disk.

Parameters

path	[in] path for file

{
    BL_PROFILE("spades::SPADES::write_info_file()");
    if (!amrex::ParallelDescriptor::IOProcessor()) {
        return;
    }
 
    const std::string dash_line = "\n" + std::string(78, '-') + "\n";
    const std::string fname(path + "/spades_info");
    std::ofstream fh(fname.c_str(), std::ios::out);
    if (!fh.good()) {
        amrex::FileOpenFailed(fname);
    }
 
    fh << dash_line << "Grid information: " << std::endl;
 
    fh << "  Level: " << LEV << "\n"
       << "    num. boxes = " << boxArray().size() << "\n"
       << "    maximum zones = ";
 
    for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
        fh << Geom(LEV).Domain().length(dir) << " ";
    }
    fh << "\n";
 
    fh << dash_line << "Input file parameters: " << std::endl;
    amrex::ParmParse::dumpTable(fh, true);
    fh.close();
}

Here is the caller graph for this function:

write_plot_file()

void spades::SPADES::write_plot_file ( )

private

Write plotfile to disk.

{
    BL_PROFILE("spades::SPADES::write_plot_file()");
    const std::string& plotfilename = plot_file_name(m_istep);
    plot_file_mf();
    const auto& varnames = plot_file_var_names();
 
    amrex::Print() << "Writing plot file " << plotfilename << " at time "
                   << m_t_new << std::endl;
 
    amrex::VisMF::SetNOutFiles(m_nfiles);
    amrex::WriteSingleLevelPlotfile(
        plotfilename, m_plt_mf, varnames, Geom(LEV), m_t_new, m_istep);
    if (m_write_messages) {
        m_message_pc->write_plot_file(plotfilename);
    }
    if (m_write_entities) {
        m_entity_pc->write_plot_file(plotfilename);
    }
 
    write_info_file(plotfilename);
}

Here is the call graph for this function:

Here is the caller graph for this function:

write_rng_file()

void spades::SPADES::write_rng_file ( const std::string & path ) const

private

Write random number generator seed info.

Parameters

path	[in] path for file

{
    BL_PROFILE("spades::SPADES::write_rng_file()");
 
    const std::string base(path + "/rng");
    const std::string& fname = amrex::Concatenate(
        base, amrex::ParallelDescriptor::MyProc(), m_rng_file_name_digits);
    std::ofstream fh(fname.c_str(), std::ios::out);
    if (!fh.good()) {
        amrex::FileOpenFailed(fname);
    }
    amrex::SaveRandomState(fh);
    fh.close();
}

Here is the caller graph for this function:

Member Data Documentation

LEV

int spades::SPADES::LEV {0}

staticconstexpr

Level index.

{0};

m_avg_timings

amrex::Vector<amrex::Real> spades::SPADES::m_avg_timings

private

Average timings for each step.

m_chk_file

std::string spades::SPADES::m_chk_file {"chk"}

private

Checkpoint prefix (optional user input)

{"chk"};

m_chk_int

int spades::SPADES::m_chk_int = -1

private

Checkpoint frequency (optional user input)

m_data_fname

std::string spades::SPADES::m_data_fname {"data.csv"}

private

Filename for simulation data.

{"data.csv"};

m_data_precision

const int spades::SPADES::m_data_precision {6}

private

Data precision for data output.

{6};

m_dt

amrex::Real spades::SPADES::m_dt {constants::LARGE_NUM}

private

Time step.

{constants::LARGE_NUM};

m_entities_per_lp

int spades::SPADES::m_entities_per_lp {1}

private

Number of entities per logical process (optional user input)

{1};

m_entity_counts_varnames

amrex::Vector<std::string> spades::SPADES::m_entity_counts_varnames

private

Entity count names for output.

m_entity_pc

std::unique_ptr<particles::EntityParticleContainer> spades::SPADES::m_entity_pc

private

Entity article container.

m_file_name_digits

int spades::SPADES::m_file_name_digits {constants::FILE_NAME_DIGITS}

private

Digits used in the plotfile and checkpoint file names.

{constants::FILE_NAME_DIGITS};

m_gvt

amrex::Real spades::SPADES::m_gvt {constants::LOW_NUM}

private

Global virtual time.

{constants::LOW_NUM};

m_ic_op

std::unique_ptr<ic::InitializerBase> spades::SPADES::m_ic_op

private

Initial condition operator.

m_ic_type

std::string spades::SPADES::m_ic_type

private

Initial condition type (optional user input)

m_istep

int spades::SPADES::m_istep {0}

private

Current step.

{0};

m_lambda

amrex::Real spades::SPADES::m_lambda {1.0}

private

Width of exponential distribution (optional user input) smaller values of lambda = larger variance in random values larger values of lambda = smaller variance in random values

{1.0};

m_lbts

amrex::Real spades::SPADES::m_lbts {constants::LOW_NUM}

private

Lower bound on incoming time stamp.

{constants::LOW_NUM};

m_lookahead

amrex::Real spades::SPADES::m_lookahead {1.0}

private

Lookahead value (optional user input)

{1.0};

m_max_messages

amrex::Vector<amrex::Long> spades::SPADES::m_max_messages

private

Max of message counts.

m_max_step

int spades::SPADES::m_max_step = std::numeric_limits<int>::max()

private

Maximum number of steps.

m_max_timings

amrex::Vector<amrex::Real> spades::SPADES::m_max_timings

private

Max timings for each step.

m_message_counts_varnames

amrex::Vector<std::string> spades::SPADES::m_message_counts_varnames

private

Message count names for output.

m_message_pc

std::unique_ptr<particles::MessageParticleContainer> spades::SPADES::m_message_pc

private

Message particle container.

m_messages_per_lp

int spades::SPADES::m_messages_per_lp {1}

private

Number of messages per logical process (optional user input)

{1};

m_messages_per_step

int spades::SPADES::m_messages_per_step {1}

private

Number of messages to process in each step (optional user input)

{1};

m_min_messages

amrex::Vector<amrex::Long> spades::SPADES::m_min_messages

private

Min of message counts.

m_min_timings

amrex::Vector<amrex::Real> spades::SPADES::m_min_timings

private

Min timings for each step.

m_ncells

amrex::Long spades::SPADES::m_ncells {0}

private

Cell count.

{0};

m_nentities

amrex::Long spades::SPADES::m_nentities {0}

private

Entity count.

{0};

m_nfiles

int spades::SPADES::m_nfiles {constants::TWO_TO_EIGHT}

private

Number of plot and checkpoint data files per write.

{constants::TWO_TO_EIGHT};

m_nmessages

amrex::Vector<amrex::Long> spades::SPADES::m_nmessages

private

Message counts.

m_nprocessed_messages

amrex::Long spades::SPADES::m_nprocessed_messages {0}

private

Count of processed messages.

{0};

m_nrollbacks

amrex::Vector<int> spades::SPADES::m_nrollbacks

private

Number of rollbacks.

m_ntotal_entities

amrex::Long spades::SPADES::m_ntotal_entities {0}

private

Total entity count.

{0};

m_ntotal_messages

amrex::Long spades::SPADES::m_ntotal_messages {0}

private

Total message count.

{0};

m_plot_file

std::string spades::SPADES::m_plot_file {"plt"}

private

Plotfile prefix (optional user input)

{"plt"};

m_plot_int

int spades::SPADES::m_plot_int = -1

private

Plotfile frequency (optional user input)

m_plt_mf

amrex::MultiFab spades::SPADES::m_plt_mf

private

Multifabs for output.

m_restart_chkfile

std::string spades::SPADES::m_restart_chkfile

private

Restart file name, restart from this checkpoint if it is not empty.

m_rng_file_name_digits

int spades::SPADES::m_rng_file_name_digits {constants::RNG_FILE_NAME_DIGITS}

private

Digits used in the rng seed file names.

{constants::RNG_FILE_NAME_DIGITS};

m_seed

int spades::SPADES::m_seed {0}

private

Random number generator seed (optional user input)

{0};

m_spades_varnames

amrex::Vector<std::string> spades::SPADES::m_spades_varnames

private

All variable names for output.

m_state

amrex::MultiFab spades::SPADES::m_state

private

Multifabs to store the solution.

m_state_ngrow

const int spades::SPADES::m_state_ngrow = 0

private

Number of ghost cells.

m_state_varnames

amrex::Vector<std::string> spades::SPADES::m_state_varnames

private

State variable names for output.

m_stop_time

amrex::Real spades::SPADES::m_stop_time = std::numeric_limits<amrex::Real>::max()

private

Stop time.

m_t_new

amrex::Real spades::SPADES::m_t_new {0.0}

private

New time.

{0.0};

m_t_old

amrex::Real spades::SPADES::m_t_old {constants::LOW_NUM}

private

Old time.

{constants::LOW_NUM};

m_window_size

amrex::Real spades::SPADES::m_window_size {constants::LARGE_NUM}

private

Window size for processing messages (optional user input)

{constants::LARGE_NUM};

m_write_entities

bool spades::SPADES::m_write_entities {false}

private

Boolean to output entities (optional user input)

{false};

m_write_messages

bool spades::SPADES::m_write_messages {false}

private

Boolean to output messages (optional user input)

{false};

---

The documentation for this class was generated from the following files:

• /home/runner/work/spades/spades/Source/SPADES.H
• /home/runner/work/spades/spades/Source/SPADES.cpp