Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix the (cuda|hip)DeviceReset issue. #557

Merged
merged 2 commits into from
Jun 9, 2020
Merged

Fix the (cuda|hip)DeviceReset issue. #557

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4586679
Fix the (cuda|hip)DeviceReset issue.
tcojean Jun 5, 2020
14205b6
Add comment on when `DeviceReset` is called.
tcojean Jun 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions benchmark/utils/general.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,11 +260,11 @@ const std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
{"cuda",
[] {
return gko::CudaExecutor::create(FLAGS_device_id,
gko::OmpExecutor::create());
gko::OmpExecutor::create(), true);
}},
{"hip", [] {
return gko::HipExecutor::create(FLAGS_device_id,
gko::OmpExecutor::create());
gko::OmpExecutor::create(), true);
}}};


Expand Down
4 changes: 2 additions & 2 deletions core/device_hooks/cuda_hooks.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ version version_info::get_cuda_version() noexcept


std::shared_ptr<CudaExecutor> CudaExecutor::create(
int device_id, std::shared_ptr<Executor> master)
int device_id, std::shared_ptr<Executor> master, bool device_reset)
{
return std::shared_ptr<CudaExecutor>(
new CudaExecutor(device_id, std::move(master)));
new CudaExecutor(device_id, std::move(master), device_reset));
}


Expand Down
4 changes: 2 additions & 2 deletions core/device_hooks/hip_hooks.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ version version_info::get_hip_version() noexcept


std::shared_ptr<HipExecutor> HipExecutor::create(
int device_id, std::shared_ptr<Executor> master)
int device_id, std::shared_ptr<Executor> master, bool device_reset)
{
return std::shared_ptr<HipExecutor>(
new HipExecutor(device_id, std::move(master)));
new HipExecutor(device_id, std::move(master), device_reset));
}


Expand Down
64 changes: 62 additions & 2 deletions core/test/base/executor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,8 @@ TEST(ReferenceExecutor, IsItsOwnMaster)
TEST(CudaExecutor, RunsCorrectOperation)
{
int value = 0;
exec_ptr cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec_ptr cuda =
gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);

cuda->run(ExampleOperation(value));
ASSERT_EQ(2, value);
Expand All @@ -288,7 +289,8 @@ TEST(CudaExecutor, RunsCorrectLambdaOperation)
auto omp_lambda = [&value]() { value = 1; };
auto cuda_lambda = [&value]() { value = 2; };
auto hip_lambda = [&value]() { value = 3; };
exec_ptr cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec_ptr cuda =
gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);

cuda->run(omp_lambda, cuda_lambda, hip_lambda);
ASSERT_EQ(2, value);
Expand All @@ -313,6 +315,35 @@ TEST(CudaExecutor, KnowsItsDeviceId)
}


TEST(CudaExecutor, CanGetDeviceResetBoolean)
{
auto omp = gko::OmpExecutor::create();
auto cuda = gko::CudaExecutor::create(0, omp);

ASSERT_EQ(false, cuda->get_device_reset());
}


TEST(CudaExecutor, CanSetDefaultDeviceResetBoolean)
{
auto omp = gko::OmpExecutor::create();
auto cuda = gko::CudaExecutor::create(0, omp, true);

ASSERT_EQ(true, cuda->get_device_reset());
}


TEST(CudaExecutor, CanSetDeviceResetBoolean)
{
auto omp = gko::OmpExecutor::create();
auto cuda = gko::CudaExecutor::create(0, omp);

cuda->set_device_reset(true);

ASSERT_EQ(true, cuda->get_device_reset());
}


TEST(HipExecutor, RunsCorrectOperation)
{
int value = 0;
Expand Down Expand Up @@ -354,6 +385,35 @@ TEST(HipExecutor, KnowsItsDeviceId)
}


TEST(HipExecutor, CanGetDeviceResetBoolean)
{
auto omp = gko::OmpExecutor::create();
auto hip = gko::HipExecutor::create(0, omp);

ASSERT_EQ(false, hip->get_device_reset());
}


TEST(HipExecutor, CanSetDefaultDeviceResetBoolean)
{
auto omp = gko::OmpExecutor::create();
auto hip = gko::HipExecutor::create(0, omp, true);

ASSERT_EQ(true, hip->get_device_reset());
}


TEST(HipExecutor, CanSetDeviceResetBoolean)
{
auto omp = gko::OmpExecutor::create();
auto hip = gko::HipExecutor::create(0, omp);

hip->set_device_reset(true);

ASSERT_EQ(true, hip->get_device_reset());
}


template <typename T>
struct mock_free : T {
/**
Expand Down
7 changes: 4 additions & 3 deletions cuda/base/executor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,14 @@ namespace gko {


std::shared_ptr<CudaExecutor> CudaExecutor::create(
int device_id, std::shared_ptr<Executor> master)
int device_id, std::shared_ptr<Executor> master, bool device_reset)
{
return std::shared_ptr<CudaExecutor>(
new CudaExecutor(device_id, std::move(master)),
new CudaExecutor(device_id, std::move(master), device_reset),
[device_id](CudaExecutor *exec) {
delete exec;
if (!CudaExecutor::get_num_execs(device_id)) {
if (!CudaExecutor::get_num_execs(device_id) &&
exec->get_device_reset()) {
cuda::device_guard g(device_id);
cudaDeviceReset();
}
Expand Down
2 changes: 1 addition & 1 deletion cuda/test/utils.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ namespace {

// prevent device reset after each test
auto no_reset_exec =
gko::CudaExecutor::create(0, gko::ReferenceExecutor::create());
gko::CudaExecutor::create(0, gko::ReferenceExecutor::create(), true);


} // namespace
Expand Down
4 changes: 2 additions & 2 deletions examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down
2 changes: 1 addition & 1 deletion examples/custom-logger/custom-logger.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
Expand Down
4 changes: 2 additions & 2 deletions examples/custom-matrix-format/custom-matrix-format.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,8 @@ int main(int argc, char *argv[])
const auto omp = gko::OmpExecutor::create();
std::map<std::string, std::shared_ptr<gko::Executor>> exec_map{
{"omp", omp},
{"cuda", gko::CudaExecutor::create(0, omp)},
{"hip", gko::HipExecutor::create(0, omp)},
{"cuda", gko::CudaExecutor::create(0, omp, true)},
{"hip", gko::HipExecutor::create(0, omp, true)},
{"reference", gko::ReferenceExecutor::create()}};

// executor where Ginkgo will perform the computation
Expand Down
4 changes: 2 additions & 2 deletions examples/custom-stopping-criterion/custom-stopping-criterion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,10 +148,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down
4 changes: 2 additions & 2 deletions examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down
4 changes: 2 additions & 2 deletions examples/inverse-iteration/inverse-iteration.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down
4 changes: 2 additions & 2 deletions examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if ((argc == 2 || argc == 3) && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if ((argc == 2 || argc == 3) && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor] [sweeps]"
<< std::endl;
Expand Down
6 changes: 3 additions & 3 deletions examples/iterative-refinement/iterative-refinement.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down Expand Up @@ -93,7 +93,7 @@ int main(int argc, char *argv[])

// copy b again
b->copy_from(host_x.get());
gko::size_type max_iters= 10000u;
gko::size_type max_iters = 10000u;
gko::remove_complex<ValueType> outer_reduction_factor = 1e-12;
auto iter_stop =
gko::stop::Iteration::build().with_max_iters(max_iters).on(exec);
Expand Down
2 changes: 1 addition & 1 deletion examples/minimal-cuda-solver/minimal-cuda-solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
int main()
{
// Instantiate a CUDA executor
auto gpu = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
auto gpu = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
// Read data
auto A = gko::read<gko::matrix::Csr<>>(std::cin, gpu);
auto b = gko::read<gko::matrix::Dense<>>(std::cin, gpu);
Expand Down
4 changes: 2 additions & 2 deletions examples/mixed-precision-ir/mixed-precision-ir.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down
4 changes: 2 additions & 2 deletions examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,8 @@ void solve_system(const std::string &executor_string,
const auto omp = gko::OmpExecutor::create();
std::map<std::string, std::shared_ptr<gko::Executor>> exec_map{
{"omp", omp},
{"cuda", gko::CudaExecutor::create(0, omp)},
{"hip", gko::HipExecutor::create(0, omp)},
{"cuda", gko::CudaExecutor::create(0, omp, true)},
{"hip", gko::HipExecutor::create(0, omp, true)},
{"reference", gko::ReferenceExecutor::create()}};
// executor where Ginkgo will perform the computation
const auto exec = exec_map.at(executor_string); // throws if not valid
Expand Down
4 changes: 2 additions & 2 deletions examples/papi-logging/papi-logging.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,10 +144,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down
2 changes: 1 addition & 1 deletion examples/performance-debugging/performance-debugging.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc > 1 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
print_usage(argv[0]);
}
Expand Down
4 changes: 2 additions & 2 deletions examples/poisson-solver/poisson-solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,8 @@ int main(int argc, char *argv[])
const auto omp = gko::OmpExecutor::create();
std::map<std::string, std::shared_ptr<gko::Executor>> exec_map{
{"omp", omp},
{"cuda", gko::CudaExecutor::create(0, omp)},
{"hip", gko::HipExecutor::create(0, omp)},
{"cuda", gko::CudaExecutor::create(0, omp, true)},
{"hip", gko::HipExecutor::create(0, omp, true)},
{"reference", gko::ReferenceExecutor::create()}};

// executor where Ginkgo will perform the computation
Expand Down
4 changes: 2 additions & 2 deletions examples/preconditioned-solver/preconditioned-solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down
4 changes: 2 additions & 2 deletions examples/simple-solver-logging/simple-solver-logging.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down
4 changes: 2 additions & 2 deletions examples/simple-solver/simple-solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,10 @@ int main(int argc, char *argv[])
exec = gko::OmpExecutor::create();
} else if (argc == 2 && std::string(argv[1]) == "cuda" &&
gko::CudaExecutor::get_num_devices() > 0) {
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
} else if (argc == 2 && std::string(argv[1]) == "hip" &&
gko::HipExecutor::get_num_devices() > 0) {
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
} else {
std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
std::exit(-1);
Expand Down
4 changes: 2 additions & 2 deletions examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,8 @@ void solve_system(const std::string &executor_string,
const auto omp = gko::OmpExecutor::create();
std::map<std::string, std::shared_ptr<gko::Executor>> exec_map{
{"omp", omp},
{"cuda", gko::CudaExecutor::create(0, omp)},
{"hip", gko::HipExecutor::create(0, omp)},
{"cuda", gko::CudaExecutor::create(0, omp, true)},
{"hip", gko::HipExecutor::create(0, omp, true)},
{"reference", gko::ReferenceExecutor::create()}};
// executor where Ginkgo will perform the computation
const auto exec = exec_map.at(executor_string); // throws if not valid
Expand Down
4 changes: 2 additions & 2 deletions examples/twentyseven-pt-stencil-solver/twentyseven-pt-stencil-solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,8 +283,8 @@ void solve_system(const std::string &executor_string,
const auto omp = gko::OmpExecutor::create();
std::map<std::string, std::shared_ptr<gko::Executor>> exec_map{
{"omp", omp},
{"cuda", gko::CudaExecutor::create(0, omp)},
{"hip", gko::HipExecutor::create(0, omp)},
{"cuda", gko::CudaExecutor::create(0, omp, true)},
{"hip", gko::HipExecutor::create(0, omp, true)},
{"reference", gko::ReferenceExecutor::create()}};
// executor where Ginkgo will perform the computation
const auto exec = exec_map.at(executor_string); // throws if not valid
Expand Down
Loading