From c796977b6b9c471ba468e8aaaee251f04b1f856d Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 14 Jan 2022 09:16:26 +0000 Subject: [PATCH] add amp debug --- Amp/.gdbinit | 6 ++ Amp/back_trace_autocast | 28 +++++ Amp/log | 19 ++++ Amp/test.py | 31 +++--- Amp/test_gdb.py | 17 +++ Amp/trace.json | 232 ++++++++++++++++++++-------------------- 6 files changed, 203 insertions(+), 130 deletions(-) create mode 100644 Amp/.gdbinit create mode 100644 Amp/back_trace_autocast create mode 100644 Amp/log create mode 100644 Amp/test_gdb.py diff --git a/Amp/.gdbinit b/Amp/.gdbinit new file mode 100644 index 0000000..e0f0960 --- /dev/null +++ b/Amp/.gdbinit @@ -0,0 +1,6 @@ +set directory /home/colin/develop/pytorch/ + +set breakpoint pending on +set history filename ~/.gdb_history +set history save on +set confirm off \ No newline at end of file diff --git a/Amp/back_trace_autocast b/Amp/back_trace_autocast new file mode 100644 index 0000000..e92fb44 --- /dev/null +++ b/Amp/back_trace_autocast @@ -0,0 +1,28 @@ +#0 at::autocast::(anonymous namespace)::is_autocast_eligible (tensor=..., device_type=c10::DeviceType::CUDA) + at /home/colin/develop/pytorch/aten/src/ATen/autocast_mode.h:24 +#1 0x00007fffb9799261 in at::autocast::is_eligible (arg=..., device_type=c10::DeviceType::CUDA) + at /home/colin/develop/pytorch/aten/src/ATen/autocast_mode.h:136 +#2 0x00007fffb978dcec in at::autocast::cached_cast (to_type=c10::ScalarType::Half, arg=..., + device_type=c10::DeviceType::CUDA) at /home/colin/develop/pytorch/aten/src/ATen/autocast_mode.cpp:112 +#3 0x00007fffb979d5d2 in at::autocast::WrapFunction_<(at::autocast::CastPolicy)0, (c10::DeviceType)1, at::Tensor (at::Tensor const&, at::Tensor const&), &at::mm, at::Tensor, c10::guts::typelist::typelist >::call(at::Tensor const&, at::Tensor const&) (args#0=..., args#1=...) + at /home/colin/develop/pytorch/aten/src/ATen/autocast_mode.cpp:174 +#4 0x00007fffb96a9e2f in c10::impl::detail::WrapFunctionIntoRuntimeFunctor_ >::operator() ( + this=0x555555f60280, args#0=..., args#1=...) + at /home/colin/develop/pytorch/aten/src/ATen/core/boxing/impl/WrapFunctionIntoRuntimeFunctor.h:18 +#5 0x00007fffb96b1463 in c10::impl::wrap_kernel_functor_unboxed_ >, at::Tensor (at::Tensor const&, at::Tensor const&)>::call(c10::OperatorKernel*, c10::DispatchKeySet, at::Tensor const&, at::Tensor const&) (functor=0x555555f60280, args#0=..., args#1=...) + at /home/colin/develop/pytorch/aten/src/ATen/core/boxing/impl/make_boxed_from_unboxed_functor.h:424 +#6 0x00007fffba3aeb3c in c10::callUnboxedKernelFunction ( + unboxed_kernel_func=0x7fffb96b13fa >, at::Tensor (at::Tensor const&, at::Tensor const&)>::call(c10::OperatorKernel*, c10::DispatchKeySet, at::Tensor const&, at::Tensor const&)>, functor=0x555555f60280, dispatchKeySet=...) + at /home/colin/develop/pytorch/aten/src/ATen/core/boxing/KernelFunction_impl.h:57 +#7 0x00007fffba86a05d in c10::KernelFunction::call ( + dispatchKeySet=..., opHandle=..., this=0x555555f50088) + at /home/colin/develop/pytorch/aten/src/ATen/core/boxing/KernelFunction_impl.h:67 +#8 c10::Dispatcher::call(c10::TypedOperatorHandle const&, at::Tensor const&, at::Tensor const&) const (op=..., + this=0x7fffc7690520 ) + at /home/colin/develop/pytorch/aten/src/ATen/core/dispatch/Dispatcher.h:548 +#9 c10::TypedOperatorHandle::call(at::Tensor const&, at::Tensor const&) const (args#1=..., args#0=..., this=) + at /home/colin/develop/pytorch/aten/src/ATen/core/dispatch/Dispatcher.h:414 +#10 at::_ops::mm::call (self=..., mat2=...) at /home/colin/develop/pytorch/build/aten/src/ATen/Operators_3.cpp:3328 +#11 0x00007fffc85c1f2c in at::Tensor::mm (this=0x7fffffffdb00, mat2=...) + at /home/colin/develop/pytorch/build/aten/src/ATen/core/TensorBody.h:2787 +#12 0x00007fffc868d046 in torch::autograd::::operator()(const at::Tensor &, const at::Tensor &) const (__closure=0x7fffffffdaf8, self=..., mat2=...) \ No newline at end of file diff --git a/Amp/log b/Amp/log new file mode 100644 index 0000000..ad57fdc --- /dev/null +++ b/Amp/log @@ -0,0 +1,19 @@ +b autocast_mode.h:23 if cast is support +b Dispatcher.h:548 +b autocast_mode.cpp:174 do cast +p dispatchKeySet.highestPriorityTypeId() $4 = c10::DispatchKey::AutocastCUDA +p kernel unboxed_kernel_func_ = 0x7fffbc8d7792 , at::Tensor, + c10::guts::typelist::typelist >, at::Tensor(c10::DispatchKeySet, const at::Tensor&, const at::Tensor&)> + ::call(c10::OperatorKernel *, c10::DispatchKeySet, const at::Tensor &, const at::Tensor &)>} + + +# /home/colin/develop/pytorch/aten/src/ATen/core/boxing/KernelFunction_impl.h:57 +# break pytorch/build/aten/src/ATen/Operators_1.cpp:3641 +# break Dispatcher.cpp:75 + +# autocast_mode.cpp:193 +# Dispatcher.h:548 diff --git a/Amp/test.py b/Amp/test.py index d9f58f8..30b9ce1 100644 --- a/Amp/test.py +++ b/Amp/test.py @@ -99,28 +99,31 @@ class Net(nn.Module): model = Net() -# model = model.cuda() +model = model.cuda() optimizer = optim.SGD(model.parameters(), lr=0.0005) for batch_idx, (data, target) in enumerate(train_loader): - # data = data.cuda() - # target = target.cuda() + data = data.cuda() + target = target.cuda() optimizer.zero_grad() - with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], with_stack=True, record_shapes=True) as prof: - with record_function("model_inference"): - with autocast(): - output = model(data) - loss = F.nll_loss(output, target) + # with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], with_stack=True, record_shapes=True) as prof: + # with record_function("model_inference"): + # with autocast(): + # output = model(data) + # loss = F.nll_loss(output, target) + # # print(prof.key_averages().table(sort_by="cpu_time_total")) # print(prof.key_averages().table(sort_by="cpu_time_total")) - print(prof.key_averages().table(sort_by="cpu_time_total")) - prof.export_chrome_trace("trace.json") + # prof.export_chrome_trace("trace.json") - exit() - # 反向传播在autocast上下文之外 - loss.backward() - optimizer.step() + with autocast(): + output = model(data) + loss = F.nll_loss(output, target) + + # # 反向传播在autocast上下文之外 + # loss.backward() + # optimizer.step() # from torch.cuda.amp import autocast as autocast diff --git a/Amp/test_gdb.py b/Amp/test_gdb.py new file mode 100644 index 0000000..81d5f02 --- /dev/null +++ b/Amp/test_gdb.py @@ -0,0 +1,17 @@ +import torch +from torch.cuda.amp import autocast as autocast +from torch.profiler import profile, ProfilerActivity, record_function + +data = torch.ones((300, 300)) +data = data.cuda() + +# with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], with_stack=True, record_shapes=True) as prof: +# with record_function("model_inference"): +# with autocast(): +# output = torch.mm(data, data) +# print(prof.key_averages().table(sort_by="cpu_time_total")) + +with autocast(): + output = torch.mm(data, data) + +print(output.cpu()) diff --git a/Amp/trace.json b/Amp/trace.json index 6b53e34..b14af71 100644 --- a/Amp/trace.json +++ b/Amp/trace.json @@ -14,8 +14,8 @@ "traceEvents": [ { "ph": "X", "cat": "cpu_op", - "name": "aten::empty", "pid": 14436, "tid": 14436, - "ts": 1641800282593594, "dur": 7, + "name": "aten::empty", "pid": 1673, "tid": 1673, + "ts": 1641986274562095, "dur": 8, "args": { "External id": 2, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -24,8 +24,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::zero_", "pid": 14436, "tid": 14436, - "ts": 1641800282593614, "dur": 1, + "name": "aten::zero_", "pid": 1673, "tid": 1673, + "ts": 1641986274562123, "dur": 1, "args": { "External id": 3, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -34,8 +34,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::zeros", "pid": 14436, "tid": 14436, - "ts": 1641800282593579, "dur": 38, + "name": "aten::zeros", "pid": 1673, "tid": 1673, + "ts": 1641986274561890, "dur": 238, "args": { "External id": 1, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -44,8 +44,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::empty", "pid": 14436, "tid": 14436, - "ts": 1641800282593648, "dur": 1, + "name": "aten::empty", "pid": 1673, "tid": 1673, + "ts": 1641986274562166, "dur": 1, "args": { "External id": 5, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -54,8 +54,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::empty", "pid": 14436, "tid": 14436, - "ts": 1641800282602596, "dur": 6, + "name": "aten::empty", "pid": 1673, "tid": 1673, + "ts": 1641986274568374, "dur": 8, "args": { "External id": 10, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -64,8 +64,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::as_strided_", "pid": 14436, "tid": 14436, - "ts": 1641800282602862, "dur": 4, + "name": "aten::as_strided_", "pid": 1673, "tid": 1673, + "ts": 1641986274568866, "dur": 5, "args": { "External id": 11, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -74,8 +74,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::mkldnn_convolution", "pid": 14436, "tid": 14436, - "ts": 1641800282593898, "dur": 8986, + "name": "aten::mkldnn_convolution", "pid": 1673, "tid": 1673, + "ts": 1641986274562855, "dur": 6026, "args": { "External id": 9, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -84,8 +84,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::_convolution", "pid": 14436, "tid": 14436, - "ts": 1641800282593879, "dur": 9007, + "name": "aten::_convolution", "pid": 1673, "tid": 1673, + "ts": 1641986274562686, "dur": 6199, "args": { "External id": 8, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -94,8 +94,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::convolution", "pid": 14436, "tid": 14436, - "ts": 1641800282593860, "dur": 9027, + "name": "aten::convolution", "pid": 1673, "tid": 1673, + "ts": 1641986274562654, "dur": 6233, "args": { "External id": 7, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -104,8 +104,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::conv2d", "pid": 14436, "tid": 14436, - "ts": 1641800282593839, "dur": 9051, + "name": "aten::conv2d", "pid": 1673, "tid": 1673, + "ts": 1641986274562628, "dur": 6264, "args": { "External id": 6, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -114,8 +114,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::max_pool2d_with_indices", "pid": 14436, "tid": 14436, - "ts": 1641800282603040, "dur": 428, + "name": "aten::max_pool2d_with_indices", "pid": 1673, "tid": 1673, + "ts": 1641986274569358, "dur": 613, "args": { "External id": 13, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -124,8 +124,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::max_pool2d", "pid": 14436, "tid": 14436, - "ts": 1641800282603015, "dur": 461, + "name": "aten::max_pool2d", "pid": 1673, "tid": 1673, + "ts": 1641986274569259, "dur": 716, "args": { "External id": 12, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -134,8 +134,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::sigmoid", "pid": 14436, "tid": 14436, - "ts": 1641800282603523, "dur": 365, + "name": "aten::sigmoid", "pid": 1673, "tid": 1673, + "ts": 1641986274570156, "dur": 301, "args": { "External id": 14, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -144,8 +144,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::empty", "pid": 14436, "tid": 14436, - "ts": 1641800282604168, "dur": 2, + "name": "aten::empty", "pid": 1673, "tid": 1673, + "ts": 1641986274570899, "dur": 4, "args": { "External id": 19, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -154,8 +154,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::as_strided_", "pid": 14436, "tid": 14436, - "ts": 1641800282604218, "dur": 0, + "name": "aten::as_strided_", "pid": 1673, "tid": 1673, + "ts": 1641986274570972, "dur": 1, "args": { "External id": 20, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -164,8 +164,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::mkldnn_convolution", "pid": 14436, "tid": 14436, - "ts": 1641800282603942, "dur": 280, + "name": "aten::mkldnn_convolution", "pid": 1673, "tid": 1673, + "ts": 1641986274570551, "dur": 429, "args": { "External id": 18, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -174,8 +174,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::_convolution", "pid": 14436, "tid": 14436, - "ts": 1641800282603931, "dur": 293, + "name": "aten::_convolution", "pid": 1673, "tid": 1673, + "ts": 1641986274570531, "dur": 452, "args": { "External id": 17, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -184,8 +184,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::convolution", "pid": 14436, "tid": 14436, - "ts": 1641800282603921, "dur": 308, + "name": "aten::convolution", "pid": 1673, "tid": 1673, + "ts": 1641986274570514, "dur": 476, "args": { "External id": 16, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -194,8 +194,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::conv2d", "pid": 14436, "tid": 14436, - "ts": 1641800282603911, "dur": 319, + "name": "aten::conv2d", "pid": 1673, "tid": 1673, + "ts": 1641986274570497, "dur": 495, "args": { "External id": 15, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -204,8 +204,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::max_pool2d_with_indices", "pid": 14436, "tid": 14436, - "ts": 1641800282604261, "dur": 19, + "name": "aten::max_pool2d_with_indices", "pid": 1673, "tid": 1673, + "ts": 1641986274571045, "dur": 22, "args": { "External id": 22, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -214,8 +214,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::max_pool2d", "pid": 14436, "tid": 14436, - "ts": 1641800282604251, "dur": 31, + "name": "aten::max_pool2d", "pid": 1673, "tid": 1673, + "ts": 1641986274571027, "dur": 42, "args": { "External id": 21, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -224,8 +224,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::sigmoid", "pid": 14436, "tid": 14436, - "ts": 1641800282604295, "dur": 8, + "name": "aten::sigmoid", "pid": 1673, "tid": 1673, + "ts": 1641986274571090, "dur": 16, "args": { "External id": 23, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -234,8 +234,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::empty", "pid": 14436, "tid": 14436, - "ts": 1641800282605085, "dur": 2, + "name": "aten::empty", "pid": 1673, "tid": 1673, + "ts": 1641986274571929, "dur": 4, "args": { "External id": 28, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -244,8 +244,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::as_strided_", "pid": 14436, "tid": 14436, - "ts": 1641800282606358, "dur": 1, + "name": "aten::as_strided_", "pid": 1673, "tid": 1673, + "ts": 1641986274573076, "dur": 1, "args": { "External id": 29, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -254,8 +254,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::mkldnn_convolution", "pid": 14436, "tid": 14436, - "ts": 1641800282604356, "dur": 2008, + "name": "aten::mkldnn_convolution", "pid": 1673, "tid": 1673, + "ts": 1641986274571201, "dur": 1884, "args": { "External id": 27, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -264,8 +264,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::_convolution", "pid": 14436, "tid": 14436, - "ts": 1641800282604344, "dur": 2021, + "name": "aten::_convolution", "pid": 1673, "tid": 1673, + "ts": 1641986274571182, "dur": 1906, "args": { "External id": 26, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -274,8 +274,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::convolution", "pid": 14436, "tid": 14436, - "ts": 1641800282604333, "dur": 2034, + "name": "aten::convolution", "pid": 1673, "tid": 1673, + "ts": 1641986274571163, "dur": 1927, "args": { "External id": 25, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -284,8 +284,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::conv2d", "pid": 14436, "tid": 14436, - "ts": 1641800282604322, "dur": 2046, + "name": "aten::conv2d", "pid": 1673, "tid": 1673, + "ts": 1641986274571146, "dur": 1946, "args": { "External id": 24, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -294,8 +294,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::view", "pid": 14436, "tid": 14436, - "ts": 1641800282606400, "dur": 13, + "name": "aten::view", "pid": 1673, "tid": 1673, + "ts": 1641986274573139, "dur": 17, "args": { "External id": 30, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -304,8 +304,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::_log_softmax", "pid": 14436, "tid": 14436, - "ts": 1641800282606479, "dur": 147, + "name": "aten::_log_softmax", "pid": 1673, "tid": 1673, + "ts": 1641986274573506, "dur": 279, "args": { "External id": 32, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -314,8 +314,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::log_softmax", "pid": 14436, "tid": 14436, - "ts": 1641800282606463, "dur": 164, + "name": "aten::log_softmax", "pid": 1673, "tid": 1673, + "ts": 1641986274573487, "dur": 301, "args": { "External id": 31, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -324,8 +324,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::nll_loss_forward", "pid": 14436, "tid": 14436, - "ts": 1641800282606769, "dur": 15, + "name": "aten::nll_loss_forward", "pid": 1673, "tid": 1673, + "ts": 1641986274574609, "dur": 20, "args": { "External id": 35, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -334,8 +334,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::nll_loss", "pid": 14436, "tid": 14436, - "ts": 1641800282606751, "dur": 34, + "name": "aten::nll_loss", "pid": 1673, "tid": 1673, + "ts": 1641986274574475, "dur": 156, "args": { "External id": 34, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -344,8 +344,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "aten::nll_loss_nd", "pid": 14436, "tid": 14436, - "ts": 1641800282606731, "dur": 63, + "name": "aten::nll_loss_nd", "pid": 1673, "tid": 1673, + "ts": 1641986274574185, "dur": 458, "args": { "External id": 33, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -354,8 +354,8 @@ }, { "ph": "X", "cat": "cpu_op", - "name": "model_inference", "pid": 14436, "tid": 14436, - "ts": 1641800282593642, "dur": 13169, + "name": "model_inference", "pid": 1673, "tid": 1673, + "ts": 1641986274562158, "dur": 12505, "args": { "External id": 4, "Trace name": "PyTorch Profiler", "Trace iteration": 0, @@ -364,8 +364,8 @@ }, { "ph": "X", "cat": "Runtime", - "name": "cudaGetDeviceCount", "pid": 14436, "tid": 14436, - "ts": 1641800282606912, "dur": 328, + "name": "cudaGetDeviceCount", "pid": 1673, "tid": 1673, + "ts": 1641986274574776, "dur": 524, "args": { "cbid": 3, "correlation": 1, "external id": 0, "external ts": 0 @@ -373,8 +373,8 @@ }, { "ph": "X", "cat": "Runtime", - "name": "cudaGetDeviceProperties", "pid": 14436, "tid": 14436, - "ts": 1641800282607278, "dur": 691, + "name": "cudaGetDeviceProperties", "pid": 1673, "tid": 1673, + "ts": 1641986274575567, "dur": 176, "args": { "cbid": 4, "correlation": 4, "external id": 0, "external ts": 0 @@ -382,8 +382,8 @@ }, { "ph": "X", "cat": "Runtime", - "name": "cudaGetDeviceCount", "pid": 14436, "tid": 14436, - "ts": 1641800282607978, "dur": 0, + "name": "cudaGetDeviceCount", "pid": 1673, "tid": 1673, + "ts": 1641986274575757, "dur": 0, "args": { "cbid": 3, "correlation": 6, "external id": 0, "external ts": 0 @@ -391,8 +391,8 @@ }, { "ph": "X", "cat": "Runtime", - "name": "cudaGetDeviceProperties", "pid": 14436, "tid": 14436, - "ts": 1641800282607980, "dur": 117, + "name": "cudaGetDeviceProperties", "pid": 1673, "tid": 1673, + "ts": 1641986274575759, "dur": 157, "args": { "cbid": 4, "correlation": 7, "external id": 0, "external ts": 0 @@ -400,189 +400,189 @@ }, { "ph": "X", "cat": "Runtime", - "name": "cudaDeviceSynchronize", "pid": 14436, "tid": 14436, - "ts": 1641800282608432, "dur": 3091248, + "name": "cudaDeviceSynchronize", "pid": 1673, "tid": 1673, + "ts": 1641986274576391, "dur": 3469523, "args": { "cbid": 165, "correlation": 10, "external id": 0, "external ts": 0 } }, { - "name": "process_name", "ph": "M", "ts": 1641800282593491, "pid": 14436, "tid": 0, + "name": "process_name", "ph": "M", "ts": 1641986274561617, "pid": 1673, "tid": 0, "args": { "name": "python" } }, { - "name": "process_labels", "ph": "M", "ts": 1641800282593491, "pid": 14436, "tid": 0, + "name": "process_labels", "ph": "M", "ts": 1641986274561617, "pid": 1673, "tid": 0, "args": { "labels": "CPU" } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 14436, "tid": 0, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 1673, "tid": 0, "args": { - "sort_index": 14436 + "sort_index": 1673 } }, { - "name": "process_name", "ph": "M", "ts": 1641800282593491, "pid": 0, "tid": 0, + "name": "process_name", "ph": "M", "ts": 1641986274561617, "pid": 0, "tid": 0, "args": { "name": "python" } }, { - "name": "process_labels", "ph": "M", "ts": 1641800282593491, "pid": 0, "tid": 0, + "name": "process_labels", "ph": "M", "ts": 1641986274561617, "pid": 0, "tid": 0, "args": { "labels": "GPU 0" } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 0, "tid": 0, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 0, "tid": 0, "args": { "sort_index": 16777216 } }, { - "name": "process_name", "ph": "M", "ts": 1641800282593491, "pid": 1, "tid": 0, + "name": "process_name", "ph": "M", "ts": 1641986274561617, "pid": 1, "tid": 0, "args": { "name": "python" } }, { - "name": "process_labels", "ph": "M", "ts": 1641800282593491, "pid": 1, "tid": 0, + "name": "process_labels", "ph": "M", "ts": 1641986274561617, "pid": 1, "tid": 0, "args": { "labels": "GPU 1" } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 1, "tid": 0, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 1, "tid": 0, "args": { "sort_index": 16777217 } }, { - "name": "process_name", "ph": "M", "ts": 1641800282593491, "pid": 2, "tid": 0, + "name": "process_name", "ph": "M", "ts": 1641986274561617, "pid": 2, "tid": 0, "args": { "name": "python" } }, { - "name": "process_labels", "ph": "M", "ts": 1641800282593491, "pid": 2, "tid": 0, + "name": "process_labels", "ph": "M", "ts": 1641986274561617, "pid": 2, "tid": 0, "args": { "labels": "GPU 2" } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 2, "tid": 0, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 2, "tid": 0, "args": { "sort_index": 16777218 } }, { - "name": "process_name", "ph": "M", "ts": 1641800282593491, "pid": 3, "tid": 0, + "name": "process_name", "ph": "M", "ts": 1641986274561617, "pid": 3, "tid": 0, "args": { "name": "python" } }, { - "name": "process_labels", "ph": "M", "ts": 1641800282593491, "pid": 3, "tid": 0, + "name": "process_labels", "ph": "M", "ts": 1641986274561617, "pid": 3, "tid": 0, "args": { "labels": "GPU 3" } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 3, "tid": 0, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 3, "tid": 0, "args": { "sort_index": 16777219 } }, { - "name": "process_name", "ph": "M", "ts": 1641800282593491, "pid": 4, "tid": 0, + "name": "process_name", "ph": "M", "ts": 1641986274561617, "pid": 4, "tid": 0, "args": { "name": "python" } }, { - "name": "process_labels", "ph": "M", "ts": 1641800282593491, "pid": 4, "tid": 0, + "name": "process_labels", "ph": "M", "ts": 1641986274561617, "pid": 4, "tid": 0, "args": { "labels": "GPU 4" } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 4, "tid": 0, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 4, "tid": 0, "args": { "sort_index": 16777220 } }, { - "name": "process_name", "ph": "M", "ts": 1641800282593491, "pid": 5, "tid": 0, + "name": "process_name", "ph": "M", "ts": 1641986274561617, "pid": 5, "tid": 0, "args": { "name": "python" } }, { - "name": "process_labels", "ph": "M", "ts": 1641800282593491, "pid": 5, "tid": 0, + "name": "process_labels", "ph": "M", "ts": 1641986274561617, "pid": 5, "tid": 0, "args": { "labels": "GPU 5" } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 5, "tid": 0, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 5, "tid": 0, "args": { "sort_index": 16777221 } }, { - "name": "process_name", "ph": "M", "ts": 1641800282593491, "pid": 6, "tid": 0, + "name": "process_name", "ph": "M", "ts": 1641986274561617, "pid": 6, "tid": 0, "args": { "name": "python" } }, { - "name": "process_labels", "ph": "M", "ts": 1641800282593491, "pid": 6, "tid": 0, + "name": "process_labels", "ph": "M", "ts": 1641986274561617, "pid": 6, "tid": 0, "args": { "labels": "GPU 6" } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 6, "tid": 0, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 6, "tid": 0, "args": { "sort_index": 16777222 } }, { - "name": "process_name", "ph": "M", "ts": 1641800282593491, "pid": 7, "tid": 0, + "name": "process_name", "ph": "M", "ts": 1641986274561617, "pid": 7, "tid": 0, "args": { "name": "python" } }, { - "name": "process_labels", "ph": "M", "ts": 1641800282593491, "pid": 7, "tid": 0, + "name": "process_labels", "ph": "M", "ts": 1641986274561617, "pid": 7, "tid": 0, "args": { "labels": "GPU 7" } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 7, "tid": 0, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 7, "tid": 0, "args": { "sort_index": 16777223 } }, { - "name": "thread_name", "ph": "M", "ts": 1641800282593491, "pid": 14436, "tid": 14436, + "name": "thread_name", "ph": "M", "ts": 1641986274561617, "pid": 1673, "tid": 1673, "args": { - "name": "thread 14436 (python)" + "name": "thread 1673 (python)" } }, { - "name": "thread_sort_index", "ph": "M", "ts": 1641800282593491, "pid": 14436, "tid": 14436, + "name": "thread_sort_index", "ph": "M", "ts": 1641986274561617, "pid": 1673, "tid": 1673, "args": { - "sort_index": 14436 + "sort_index": 1673 } }, { - "ph": "X", "cat": "Trace", "ts": 1641800282593491, "dur": 3106207, + "ph": "X", "cat": "Trace", "ts": 1641986274561617, "dur": 3484317, "pid": "Spans", "tid": "PyTorch Profiler", "name": "PyTorch Profiler (0)", "args": { @@ -590,7 +590,7 @@ } }, { - "name": "process_sort_index", "ph": "M", "ts": 1641800282593491, + "name": "process_sort_index", "ph": "M", "ts": 1641986274561617, "pid": "Spans", "tid": 0, "args": { "sort_index": 536870912 @@ -598,10 +598,10 @@ }, { "name": "Iteration Start: PyTorch Profiler", "ph": "i", "s": "g", - "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1641800282593491 + "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1641986274561617 }, { "name": "Record Window End", "ph": "i", "s": "g", - "pid": "", "tid": "", "ts": 1641800285699878 + "pid": "", "tid": "", "ts": 1641986278046116 } ]} \ No newline at end of file