Created
March 11, 2026 06:35
-
-
Save shunting314/3b77ec9ca473dc4d00f3cb36bb641b40 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| (pytorch) [shunting@devgpu011.ldc3 ~/ws/helion (all-reduce)]$ python test/test_distributed.py -k TestDistributed.test_matmul_reduce_scatter_autotuner_LFBOTreeSearch | |
| INFO: Started process 0 with pid 2143318 | |
| INFO: Started process 1 with pid 2143323 | |
| INFO: Started process 2 with pid 2143330 | |
| INFO: Started process 3 with pid 2143331 | |
| NCCL version 2.28.9+cuda12.9 | |
| [rank2]:[W310 23:33:39.020723478 CUDASymmetricMemory.cu:804] Warning: Pointer not within any SymmetricMemory allocation, is the tensor allocated from SymmetricMemory? (function rendezvous) | |
| [rank2]:[W310 23:33:39.020748826 CUDASymmetricMemory.cu:804] Warning: Pointer not within any SymmetricMemory allocation, is the tensor allocated from SymmetricMemory? (function rendezvous) | |
| [rank1]:[W310 23:33:39.030396315 CUDASymmetricMemory.cu:804] Warning: Pointer not within any SymmetricMemory allocation, is the tensor allocated from SymmetricMemory? (function rendezvous) | |
| [rank1]:[W310 23:33:39.030415704 CUDASymmetricMemory.cu:804] Warning: Pointer not within any SymmetricMemory allocation, is the tensor allocated from SymmetricMemory? (function rendezvous) | |
| [rank3]:[W310 23:33:39.031116638 CUDASymmetricMemory.cu:804] Warning: Pointer not within any SymmetricMemory allocation, is the tensor allocated from SymmetricMemory? (function rendezvous) | |
| [rank3]:[W310 23:33:39.031137159 CUDASymmetricMemory.cu:804] Warning: Pointer not within any SymmetricMemory allocation, is the tensor allocated from SymmetricMemory? (function rendezvous) | |
| [0s] Starting autotuning process, this may take a while... | |
| [0s] Autotune random seed: 3684293166 | |
| [rank0]:[W310 23:33:39.031885915 CUDASymmetricMemory.cu:804] Warning: Pointer not within any SymmetricMemory allocation, is the tensor allocated from SymmetricMemory? (function rendezvous) | |
| [rank0]:[W310 23:33:39.031903001 CUDASymmetricMemory.cu:804] Warning: Pointer not within any SymmetricMemory allocation, is the tensor allocated from SymmetricMemory? (function rendezvous) | |
| [0s] Starting LFBOTreeSearch with initial_population=FROM_RANDOM, copies=2, max_generations=3, similarity_penalty=1.0 | |
| Initial population precompiling 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 6/6 3.2 configs/s | |
| Initial population exploring neighbors 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 6/6 3.5 configs/s | |
| [4s] Adaptive compile timeout: 30s (90% percentile=1.6s, bounds=[30.0s, 60s]) | |
| Verifying initial results 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100/100 - configs/s | |
| [5s] Initial random population of 6, 2 starting points: | |
| error=1 | |
| ok=5 | |
| min=0.4127 | |
| mid=1.7370 | |
| max=8.9956 | |
| best={'block_sizes': [16, 16, 16], | |
| 'indexing': ['pointer', 'pointer', 'pointer', 'pointer', 'pointer', 'pointer', 'pointer', | |
| 'pointer'], | |
| 'l2_groupings': [1], | |
| 'load_eviction_policies': ['', '', '', '', '', ''], | |
| 'loop_orders': [[0, 1]], | |
| 'num_stages': 1, | |
| 'num_warps': 4, | |
| 'pid_type': 'flat', | |
| 'range_flattens': [None, None], | |
| 'range_multi_buffers': [None, None], | |
| 'range_num_stages': [], | |
| 'range_unroll_factors': [0, 0], | |
| 'range_warp_specializes': []} | |
| [5s] Fitting surrogate: 6 points, 6 targets | |
| [5s] Generation 1 starting: 13 neighbors, 2 active search path(s) | |
| Generation 1: precompiling 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 14/14 26.0 configs/s | |
| Generation 1: exploring neighbors 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 14/14 3.4 configs/s | |
| Generation 1: verifying top configs 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100/100 - configs/s | |
| [13s] Generation 1 complete: | |
| ok=15 | |
| min=0.1316 | |
| mid=0.4455 | |
| max=2.0410 | |
| best={'block_sizes': [16, 64, 16], | |
| 'indexing': ['tensor_descriptor', 'pointer', 'pointer', 'pointer', 'tensor_descriptor', 'pointer', | |
| 'pointer', 'tensor_descriptor'], | |
| 'l2_groupings': [2], | |
| 'load_eviction_policies': ['', 'last', '', '', '', 'first'], | |
| 'loop_orders': [[1, 0]], | |
| 'num_stages': 3, | |
| 'num_warps': 8, | |
| 'pid_type': 'flat', | |
| 'range_flattens': [None, None], | |
| 'range_multi_buffers': [None, False], | |
| 'range_num_stages': [], | |
| 'range_unroll_factors': [0, 0], | |
| 'range_warp_specializes': []} | |
| [13s] Fitting surrogate: 21 points, 21 targets | |
| [14s] Generation 2 starting: 13 neighbors, 2 active search path(s) | |
| Generation 2: precompiling 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 13/13 32.1 configs/s | |
| Generation 2: exploring neighbors 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 13/13 4.3 configs/s | |
| Generation 2: verifying top configs 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100/100 - configs/s | |
| [21s] Generation 2 complete: | |
| error=1 | |
| ok=14 | |
| min=0.0619 | |
| mid=0.0976 | |
| max=0.3678 | |
| best={'block_sizes': [64, 64, 16], | |
| 'indexing': ['tensor_descriptor', 'pointer', 'pointer', 'pointer', 'tensor_descriptor', 'pointer', | |
| 'pointer', 'tensor_descriptor'], | |
| 'l2_groupings': [2], | |
| 'load_eviction_policies': ['', 'last', '', 'first', '', 'first'], | |
| 'loop_orders': [[1, 0]], | |
| 'num_stages': 3, | |
| 'num_warps': 2, | |
| 'pid_type': 'flat', | |
| 'range_flattens': [None, None], | |
| 'range_multi_buffers': [None, False], | |
| 'range_num_stages': [], | |
| 'range_unroll_factors': [0, 0], | |
| 'range_warp_specializes': []} | |
| [21s] Fitting surrogate: 36 points, 36 targets | |
| [22s] Generation 3 starting: 32 neighbors, 2 active search path(s) | |
| Generation 3: precompiling 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 34/34 3.5 configs/s | |
| Generation 3: exploring neighbors 100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 34/34 4.3 configs/s | |
| Generation 3: verifying top configs 24% ━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 24/100 - configs/sERROR: Caught exception: | |
| Traceback (most recent call last): | |
| File "/home/shunting/ws/pytorch/torch/testing/_internal/common_distributed.py", line 942, in run_test | |
| getattr(self, test_name)() | |
| File "/home/shunting/ws/pytorch/torch/testing/_internal/common_distributed.py", line 789, in wrapper | |
| fn() | |
| File "/home/shunting/ws/pytorch/torch/testing/_internal/common_utils.py", line 3370, in wrapper | |
| method(*args, **kwargs) | |
| File "/home/shunting/ws/pytorch/torch/testing/_internal/common_utils.py", line 587, in instantiated_test | |
| test(self, **param_kwargs) | |
| File "/home/shunting/ws/pytorch/torch/testing/_internal/common_distributed.py", line 235, in wrapper | |
| return func(*args, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^ | |
| File "/home/shunting/ws/helion/test/test_distributed.py", line 387, in test_matmul_reduce_scatter | |
| self.do_test_matmul_reduce_scatter(kernel, mod.reference_matmul_reduce_scatter) | |
| File "/home/shunting/ws/helion/test/test_distributed.py", line 403, in do_test_matmul_reduce_scatter | |
| result = kernel( | |
| ^^^^^^^ | |
| File "/home/shunting/ws/helion/helion/runtime/kernel.py", line 352, in __call__ | |
| return self.bind(args)(*args) | |
| ^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/home/shunting/ws/helion/helion/runtime/kernel.py", line 826, in __call__ | |
| self.ensure_config_exists(args) | |
| File "/home/shunting/ws/helion/helion/runtime/kernel.py", line 794, in ensure_config_exists | |
| self.autotune(args, force=False) | |
| File "/home/shunting/ws/helion/helion/runtime/kernel.py", line 662, in autotune | |
| config = self.env.backend.autotune(self, args, force=force, **kwargs) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/home/shunting/ws/helion/helion/_compiler/backend.py", line 518, in autotune | |
| ).autotune(skip_cache=force) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/home/shunting/ws/helion/helion/autotuner/base_cache.py", line 249, in autotune | |
| config = self.autotuner.autotune() | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/home/shunting/ws/helion/helion/autotuner/base_search.py", line 1000, in autotune | |
| best = self._autotune() | |
| ^^^^^^^^^^^^^^^^ | |
| File "/home/shunting/ws/helion/helion/autotuner/surrogate_pattern_search.py", line 411, in _autotune | |
| self.rebenchmark_population( | |
| File "/home/shunting/ws/helion/helion/autotuner/base_search.py", line 1422, in rebenchmark_population | |
| self.rebenchmark([p for p in members if self.should_rebenchmark(p)], desc=desc) | |
| File "/home/shunting/ws/helion/helion/autotuner/base_search.py", line 1398, in rebenchmark | |
| new_timings = bench_fn(iterator, repeat=repeat, desc=desc) | |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| File "/home/shunting/ws/helion/helion/autotuner/benchmarking.py", line 132, in interleaved_bench | |
| start_events[j][i].record() | |
| File "/home/shunting/ws/pytorch/torch/cuda/streams.py", line 209, in record | |
| super().record(stream) | |
| torch.AcceleratorError: CUDA error: an illegal instruction was encountered | |
| Search for `cudaErrorIllegalInstruction' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information. | |
| CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. | |
| For debugging consider passing CUDA_LAUNCH_BLOCKING=1 | |
| Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. | |
| To execute this test, run the following from the base repo dir: | |
| python test/test_distributed.py TestDistributed.test_matmul_reduce_scatter_autotuner_LFBOTreeSearch | |
| This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0 | |
| exiting process 2 with exit code: 10 | |
| [rank2]:[W310 23:34:23.265047073 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265068406 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265072832 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265077590 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265082207 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265086303 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265091350 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265095437 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265099453 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265103499 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265106924 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265113164 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265122197 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265130630 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265136499 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265142077 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265148407 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265153565 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265160155 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265166244 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265172624 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265180736 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265185884 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265193255 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265207216 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265214527 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265218944 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265224192 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265230442 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265239135 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265244613 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265253096 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265259315 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265264323 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265268169 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265276161 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265281880 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265289371 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265295070 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265302180 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265306677 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265313077 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265318936 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265324134 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265328120 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265332657 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265336733 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265340479 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265348771 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265352517 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265356252 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265359948 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265363684 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265367389 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265371636 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265375532 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265379458 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265383334 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265387029 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265390755 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265394561 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265398356 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265401992 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265405798 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265409553 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265413650 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265418076 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265421832 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265425648 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265429383 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265433099 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265436845 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265444156 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265447881 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265451617 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265455663 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265459369 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265463085 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265466810 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265470536 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265474282 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265477997 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265482254 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265486119 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265489865 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265493551 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265497226 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265500972 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265504798 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265508594 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265513070 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265516986 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265520792 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265524518 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265528514 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265532370 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265539921 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265544208 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265547943 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265551779 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265555485 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265559230 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265562976 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265567313 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265570988 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265574724 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265578670 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265582346 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265586712 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265590658 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265594384 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265598099 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265601795 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265605861 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265609877 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265613663 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265617329 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265621365 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265625251 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265629637 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265637770 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265641575 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265645231 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265649467 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265653263 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265656929 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265661345 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265665311 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265669007 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265672743 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265676578 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265680695 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265684420 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265688156 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265691842 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265696138 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265699944 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265703620 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265707365 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265711091 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265715137 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265719103 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265722769 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265726394 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265730160 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265737721 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265741978 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265745794 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265749579 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265753325 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265757742 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265761587 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265765343 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265769029 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265772815 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265777011 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265780977 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265784683 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265788478 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265792404 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265796701 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265800557 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265804292 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265808138 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265811774 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265816080 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265820076 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265823822 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265827578 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265831263 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265840527 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265844283 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265848189 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265851895 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265856181 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265859987 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265863672 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265867398 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265871134 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265875340 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265879006 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265882731 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265886407 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265890243 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265894529 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265898415 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265902331 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265906087 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265909913 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265915441 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265919157 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265923113 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265926938 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265930634 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265936172 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265943694 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265947369 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265951676 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265955772 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265959538 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265963364 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265967820 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265971556 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265975242 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265979057 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265983274 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265987160 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265990835 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265994621 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.265998868 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266012448 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266016214 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266020130 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266023966 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266027811 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266031567 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266035884 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266039770 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266043465 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266047141 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266054872 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266058688 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266062504 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266066730 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266070526 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266074412 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266078308 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266081964 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266085920 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266089846 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266093571 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266097397 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266101824 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266105790 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266109806 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266114042 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266117848 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266121574 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266125219 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266129015 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266132761 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266137147 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266142205 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266147814 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266153813 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266164409 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266170668 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266177128 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266181354 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266185230 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266189126 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266193032 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266196758 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266200544 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266204249 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266207975 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266212091 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266216087 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266219883 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266223739 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266227504 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266231210 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266235016 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266238741 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266242467 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266246223 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266250279 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266254035 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266257690 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266261556 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266269248 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266273184 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266276849 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266280615 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266284341 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266288036 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266291762 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266295558 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266299393 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266303049 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266306765 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266310480 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266314196 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266318002 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266322128 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266325814 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266330370 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266334517 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266338363 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266342238 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266346195 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266349940 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266353676 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266357422 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266361157 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266369420 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266373115 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266376961 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266381278 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266385194 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266389060 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266392855 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266396681 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266400457 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266404213 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266407908 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266411734 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266415500 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266419185 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266423161 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266427037 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266430723 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266434559 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266438985 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266442841 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266446707 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266450533 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266454449 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266458234 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266462160 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266470393 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266474169 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266477934 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266481680 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266485436 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266489121 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266493328 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266497444 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266501230 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266505566 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266509482 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266513408 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266517154 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266521691 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266525867 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266529693 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266533499 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266537224 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266541441 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266545637 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266549603 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266553299 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266556974 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266561291 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266565087 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266573189 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266576885 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266580490 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266584256 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266587981 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266592068 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266595923 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266599569 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266603255 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266607000 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266610806 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266614441 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266618508 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266622374 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266626650 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266630696 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266636415 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266643215 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266647291 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266651107 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266654943 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266658619 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266662785 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266666400 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266670256 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266678278 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266682925 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266687803 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266691859 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266695565 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266699310 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266703236 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266706982 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266710978 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266715054 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266718790 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266722866 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266727072 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266730818 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266734634 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266739682 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266745230 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266750868 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266756457 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266762156 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266767574 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266774094 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266779772 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266785581 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266791520 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266800774 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266805041 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266808907 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266812732 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266817029 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266822237 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266826093 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266829989 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266834075 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266837800 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266841516 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266845162 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266849088 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266852833 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266857110 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266860795 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266864581 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266869018 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266872814 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266877310 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266881086 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266885493 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266889479 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266893245 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266896910 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266905353 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266909529 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266913265 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266917411 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266921698 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266925834 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266929770 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266933606 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266938693 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266942099 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266945384 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266949410 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266953215 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266957322 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266961087 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266964823 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266968599 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266972485 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266976311 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266980277 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266984353 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266989030 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266992876 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.266996561 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267007838 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267016401 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267020668 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267025044 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267028900 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267032616 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267036572 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267040438 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267044835 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267048901 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267052686 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267056312 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267060088 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267063753 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267067529 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267071425 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267075441 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267079247 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267082982 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267086668 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267090434 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267094139 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267097905 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267101551 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267105266 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267108892 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267117795 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267121721 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267125657 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267129483 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267133569 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267137485 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267141371 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267145257 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267149123 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267152908 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267156734 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267160530 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267164196 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267167971 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267171767 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267175543 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267179349 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267183245 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267187140 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267190906 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267194712 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267198438 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267202193 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267205929 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267209605 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267218528 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267222865 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267226550 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267230286 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267234182 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267237877 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267242825 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267246641 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267250497 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267254312 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267258098 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267261854 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267265620 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267269395 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267273071 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267276847 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267280572 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267284268 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267288034 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267291799 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267295475 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267299190 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267302946 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267306692 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267314343 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267318059 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267321745 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267325470 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267329356 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267333112 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267337338 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267341815 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267345751 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267350108 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267354034 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267357960 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267362046 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267366162 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267370128 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267374044 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267377910 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267381856 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267385862 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267389588 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267394445 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267398301 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267402107 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267405802 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267413804 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267417580 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267421246 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267425011 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267428677 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267432373 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267436278 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267440194 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267444451 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267448217 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267452012 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267455718 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267459534 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267463650 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267467416 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267470841 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267475748 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267479494 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267483560 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267487386 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267491102 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267494927 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267498713 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267502439 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267510701 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267514377 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267518153 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267521858 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267525624 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267529380 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267533115 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267536921 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267540637 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267544813 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267548579 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267552685 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267556401 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267560367 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267564142 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267567958 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267571674 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267575490 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267579326 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267583482 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267587238 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267591434 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267595200 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267598845 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267607188 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267611274 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267615060 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267618755 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267622541 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267626317 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267630063 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267633848 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267637574 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267641400 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267645125 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267648861 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267652767 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267656633 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267660439 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267664685 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267668531 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267672196 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267676052 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267679938 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267683744 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267687860 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267691726 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267695402 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267703654 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267707430 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267711216 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267715011 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267718777 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267722543 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267726439 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267730144 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267734221 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267737996 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267741812 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267745528 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267749273 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267753029 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267756885 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267760851 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267764707 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267768613 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267772489 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267776254 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267780070 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267783926 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267788162 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267792229 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267796185 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267804107 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267807912 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267811738 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267815594 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267819310 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267823266 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267827021 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267830827 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267834743 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267840622 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267846170 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267851599 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267857137 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267861934 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267865950 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267869886 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267873712 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267877638 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267881494 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267885250 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267889096 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267892881 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267896627 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267900523 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267904669 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267912761 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267916587 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267920443 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267924209 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267927995 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267931780 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267935556 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267939302 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267943869 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267947725 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267951520 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267955296 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267959012 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267962878 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267966633 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267970679 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267974786 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267978481 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267982227 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267985983 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267989748 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267993474 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.267997200 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268013124 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268018913 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268026304 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268030480 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268034616 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268039033 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268043490 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268047636 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268052714 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268057171 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268061497 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268065513 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268069369 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268073135 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268076931 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268080736 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268084472 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268088338 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268092214 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268096050 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268099855 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268103821 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268107637 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268111513 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268115619 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268119535 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268124102 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268132144 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268135930 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268139596 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268143311 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268146987 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268150803 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268154528 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268158434 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268162180 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268165956 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268169591 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268173357 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268177143 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268181089 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268184914 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268188620 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268192306 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268196131 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268199847 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268203653 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268207429 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268211605 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268215281 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268219026 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268222862 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268230454 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268234339 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268238155 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268242011 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268245867 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268249583 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268253368 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268257124 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268260830 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268264816 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268268621 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268272337 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268276083 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268279818 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268283654 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268287500 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268291777 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268295522 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268299358 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268303094 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268307360 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268311236 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268315192 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268319008 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268322754 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268330065 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268333830 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268337586 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268341452 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268345308 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268349124 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268352939 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268356785 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268360661 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268364767 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268368753 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268372509 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268376305 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268380091 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268383866 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268387852 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268391618 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268395354 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268399660 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268403596 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268408424 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268412089 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268415745 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268419490 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268423326 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268431258 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268435084 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268438930 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268442776 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268446531 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268450367 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268454203 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268457959 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268461765 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268465520 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268469386 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268473172 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268476887 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268480613 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268484509 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268488836 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268492511 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268496397 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268500193 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268504129 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268508415 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268512321 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268516027 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268519853 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268523608 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268531050 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268535006 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268538721 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268542557 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268546303 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268550069 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268553974 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268557690 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268561406 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268565101 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268568797 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268572543 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268576509 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268580815 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268584571 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268588156 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268591932 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268595638 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268599834 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268603720 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268607385 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268611251 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268614927 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268618633 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268622288 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268629820 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268634076 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268637742 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268641487 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268645283 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268649189 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268652945 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268656700 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268660376 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268664082 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268667787 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268671503 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268675990 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268679885 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268683581 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268687297 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268691072 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268694788 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268698514 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268702289 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268705965 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268709731 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268713416 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268717042 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268720747 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268728780 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268732495 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268736211 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268739917 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268743672 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268747348 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268751124 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268754999 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268758885 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268762651 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268766347 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268770383 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268774139 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268777864 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268781540 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268785275 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268788991 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268792717 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268796452 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268800078 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268803783 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268807399 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268811065 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268815061 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268818806 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268827009 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268830734 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268834510 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268840419 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268846949 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268852948 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268859768 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268865647 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268871626 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268875933 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268879859 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268883755 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268887551 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268891396 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268895112 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268898908 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268902724 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268906690 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268911978 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268917726 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268923085 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268928463 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268935083 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268941192 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268947361 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268958168 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268964167 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268969956 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268975885 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268980672 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268984598 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268988734 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268992670 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.268996456 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269005810 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269010738 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269014613 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269018489 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269022896 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269027072 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269030908 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269034714 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269038500 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269042406 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269046221 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269050037 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269054043 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269057849 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269061645 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269065731 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269073793 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269077619 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269081425 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269085201 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269089016 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269092812 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269096708 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269102036 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269105962 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269109748 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269113473 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269117379 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269121235 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269125552 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269130069 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269133954 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269137720 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269141706 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269146133 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269150049 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269153895 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269157711 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269161426 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269165783 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269169428 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269176870 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269181256 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269185032 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269188798 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269193164 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269197851 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269201948 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269206615 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269210641 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269214477 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269218322 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269222879 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269226685 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269230531 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269234727 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269238794 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269242679 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269246425 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269250441 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269254367 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269258153 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269262019 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269267257 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269270982 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| [rank2]:[W310 23:34:23.269274588 CUDAEvent.h:57] Warning: CUDA warning: an illegal instruction was encountered (function ~CUDAEvent) | |
| SIGABRT(6), PID: 2143330, Thread 2143330: | |
| frame #0: c10::FatalSignalHandler::stacktraceSignalHandler(bool) + 0x7a (0x7f1806f99bfa in /home/shunting/ws/pytorch/torch/lib/libc10.so) | |
| frame #1: c10::FatalSignalHandler::fatalSignalHandler(int, siginfo_t*) + 0x28a (0x7f1806f9a16a in /home/shunting/ws/pytorch/torch/lib/libc10.so) | |
| frame #2: <unknown function> + 0x3fc30 (0x7f1807c3fc30 in /lib64/libc.so.6) | |
| frame #3: <unknown function> + 0x8d03c (0x7f1807c8d03c in /lib64/libc.so.6) | |
| frame #4: raise + 0x16 (0x7f1807c3fb86 in /lib64/libc.so.6) | |
| frame #5: abort + 0xd3 (0x7f1807c29873 in /lib64/libc.so.6) | |
| frame #6: <unknown function> + 0xb135a (0x7f17cb2b135a in /home/shunting/.conda/envs/pytorch/lib/libstdc++.so.6) | |
| frame #7: <unknown function> + 0xb03b9 (0x7f17cb2b03b9 in /home/shunting/.conda/envs/pytorch/lib/libstdc++.so.6) | |
| frame #8: __gxx_personality_v0 + 0x87 (0x7f17cb2b0ae7 in /home/shunting/.conda/envs/pytorch/lib/libstdc++.so.6) | |
| frame #9: <unknown function> + 0x111e4 (0x7f180738e1e4 in /home/shunting/.conda/envs/pytorch/lib/libgcc_s.so.1) | |
| frame #10: _Unwind_Resume + 0x12e (0x7f180738ec1e in /home/shunting/.conda/envs/pytorch/lib/libgcc_s.so.1) | |
| frame #11: <unknown function> + 0xc274 (0x7f18073b7274 in /home/shunting/ws/pytorch/torch/lib/libc10_cuda.so) | |
| frame #12: c10d::symmetric_memory::AllocationRef::~AllocationRef() + 0xa4 (0x7f17e6662864 in /home/shunting/ws/pytorch/torch/lib/libtorch_cuda.so) | |
| frame #13: c10::intrusive_ptr<c10d::symmetric_memory::AllocationRef, c10::detail::intrusive_target_default_null_type<c10d::symmetric_memory::AllocationRef> >::reset_not_null_(c10d::symmetric_memory::AllocationRef*) + 0x7e (0x7f17e66663ae in /home/shunting/ws/pytorch/torch/lib/libtorch_cuda.so) | |
| frame #14: c10d::symmetric_memory::CUDAPeerAllocInfo::~CUDAPeerAllocInfo() + 0x66 (0x7f17e66666c6 in /home/shunting/ws/pytorch/torch/lib/libtorch_cuda.so) | |
| frame #15: c10d::symmetric_memory::Block::~Block() + 0x39 (0x7f17e6666539 in /home/shunting/ws/pytorch/torch/lib/libtorch_cuda.so) | |
| frame #16: c10d::symmetric_memory::CUDASymmetricMemoryAllocator::free(void*) + 0xad (0x7f17e66636fd in /home/shunting/ws/pytorch/torch/lib/libtorch_cuda.so) | |
| frame #17: <unknown function> + 0x349fd (0x7f1806f219fd in /home/shunting/ws/pytorch/torch/lib/libc10.so) | |
| frame #18: <unknown function> + 0x7ec09 (0x7f1806f6bc09 in /home/shunting/ws/pytorch/torch/lib/libc10.so) | |
| frame #19: c10::TensorImpl::~TensorImpl() + 0x9 (0x7f1806f65ca9 in /home/shunting/ws/pytorch/torch/lib/libc10.so) | |
| frame #20: <unknown function> + 0x90f365 (0x7f180610f365 in /home/shunting/ws/pytorch/torch/lib/libtorch_python.so) | |
| frame #21: <unknown function> + 0x90f401 (0x7f180610f401 in /home/shunting/ws/pytorch/torch/lib/libtorch_python.so) | |
| frame #22: /home/shunting/.conda/envs/pytorch/bin/python() [0x56f71e] | |
| frame #23: /home/shunting/.conda/envs/pytorch/bin/python() [0x514f0f] | |
| frame #24: /home/shunting/.conda/envs/pytorch/bin/python() [0x62a9fb] | |
| frame #25: /home/shunting/.conda/envs/pytorch/bin/python() [0x62a953] | |
| frame #26: /home/shunting/.conda/envs/pytorch/bin/python() [0x5066e7] | |
| frame #27: /home/shunting/.conda/envs/pytorch/bin/python() [0x55292a] | |
| frame #28: /home/shunting/.conda/envs/pytorch/bin/python() [0x551569] | |
| frame #29: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #30: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #31: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #32: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #33: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #34: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #35: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #36: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #37: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #38: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #39: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #40: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #41: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #42: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #43: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #44: /home/shunting/.conda/envs/pytorch/bin/python() [0x551562] | |
| frame #45: /home/shunting/.conda/envs/pytorch/bin/python() [0x568dce] | |
| frame #46: /home/shunting/.conda/envs/pytorch/bin/python() [0x56f71e] | |
| frame #47: /home/shunting/.conda/envs/pytorch/bin/python() [0x50db63] | |
| frame #48: /home/shunting/.conda/envs/pytorch/bin/python() [0x6196f3] | |
| frame #49: _PyEval_EvalFrameDefault + 0x97c (0x52130c in /home/shunting/.conda/envs/pytorch/bin/python) | |
| frame #50: PyEval_EvalCode + 0xae (0x5dc01e in /home/shunting/.conda/envs/pytorch/bin/python) | |
| frame #51: /home/shunting/.conda/envs/pytorch/bin/python() [0x619487] | |
| frame #52: /home/shunting/.conda/envs/pytorch/bin/python() [0x613ff7] | |
| frame #53: PyRun_StringFlags + 0x5f (0x6100bf in /home/shunting/.conda/envs/pytorch/bin/python) | |
| frame #54: PyRun_SimpleStringFlags + 0x3a (0x60fc5a in /home/shunting/.conda/envs/pytorch/bin/python) | |
| frame #55: Py_RunMain + 0x4e1 (0x60d5f1 in /home/shunting/.conda/envs/pytorch/bin/python) | |
| frame #56: Py_BytesMain + 0x39 (0x5c4809 in /home/shunting/.conda/envs/pytorch/bin/python) | |
| frame #57: <unknown function> + 0x2a610 (0x7f1807c2a610 in /lib64/libc.so.6) | |
| frame #58: __libc_start_main + 0x80 (0x7f1807c2a6c0 in /lib64/libc.so.6) | |
| frame #59: /home/shunting/.conda/envs/pytorch/bin/python() [0x5c4635] | |
| SIGABRT(6), PID: 2143330, Thread 2145003: | |
| frame #0: c10::FatalSignalHandler::stacktraceSignalHandler(bool) + 0x7a (0x7f1806f99bfa in /home/shunting/ws/pytorch/torch/lib/libc10.so) | |
| frame #1: <unknown function> + 0x3fc30 (0x7f1807c3fc30 in /lib64/libc.so.6) | |
| frame #2: <unknown function> + 0x8d03c (0x7f1807c8d03c in /lib64/libc.so.6) | |
| frame #3: raise + 0x16 (0x7f1807c3fb86 in /lib64/libc.so.6) | |
| frame #4: <unknown function> + 0x85d5f05 (0x7f169b9d5f05 in /home/shunting/.conda/envs/pytorch/lib/python3.12/site-packages/triton/_C/libtriton.so) | |
| frame #5: <unknown function> + 0x3fc30 (0x7f1807c3fc30 in /lib64/libc.so.6) | |
| frame #6: __poll + 0x4f (0x7f1807d030ff in /lib64/libc.so.6) | |
| frame #7: <unknown function> + 0x31cb37 (0x7f174391cb37 in /lib64/libcuda.so.1) | |
| frame #8: <unknown function> + 0x40bd77 (0x7f1743a0bd77 in /lib64/libcuda.so.1) | |
| frame #9: <unknown function> + 0x308883 (0x7f1743908883 in /lib64/libcuda.so.1) | |
| frame #10: <unknown function> + 0x8b2fa (0x7f1807c8b2fa in /lib64/libc.so.6) | |
| frame #11: <unknown function> + 0x110400 (0x7f1807d10400 in /lib64/libc.so.6) | |
| Generation 3: verifying top configs 24% ━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 24/100 - configs/ssignal lost waiting for stacktrace 2143330:2147297 | |
| Generation 3: verifying top configs 24% ━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 24/100 - configs/s | |
| Generation 3: verifying top configs 24% ━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 24/100 - configs/s | |
| Generation 3: verifying top configs 24% ━━━━━━━━━━━━━━━━━━━━━╺━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 24/100 - configs/s |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment