WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv Global seed set to 84 Downloading (…)okenizer_config.json: 0%| | 0.00/2.32k [00:00 main() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/metaflow/ddp_trainer.py", line 76, in main model.train(**kwargs) File "/metaflow/src/trainer.py", line 224, in train trainer.fit(self.t5_model, self.data_module) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 89, in launch self._call_children_scripts() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 94, in _call_children_scripts self._check_can_spawn_children() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 134, in _check_can_spawn_children raise RuntimeError( RuntimeError: Lightning attempted to launch new distributed processes with `local_rank > 0`. This should not happen. Possible reasons: 1) LOCAL_RANK environment variable was incorrectly modified by the user, 2) `ClusterEnvironment.creates_processes_externally` incorrectly implemented. Traceback (most recent call last): File "/metaflow/ddp_trainer.py", line 86, in main() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/metaflow/ddp_trainer.py", line 76, in main model.train(**kwargs) File "/metaflow/src/trainer.py", line 224, in train trainer.fit(self.t5_model, self.data_module) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 89, in launch self._call_children_scripts() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 94, in _call_children_scripts self._check_can_spawn_children() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 134, in _check_can_spawn_children raise RuntimeError( RuntimeError: Lightning attempted to launch new distributed processes with `local_rank > 0`. This should not happen. Possible reasons: 1) LOCAL_RANK environment variable was incorrectly modified by the user, 2) `ClusterEnvironment.creates_processes_externally` incorrectly implemented. 0it [00:00, ?it/s] 0it [00:00, ?it/s] 2023-07-05 22:21:34 ERROR [__main__] main failed Traceback (most recent call last): File "/metaflow/ddp_trainer.py", line 86, in main() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/metaflow/ddp_trainer.py", line 76, in main model.train(**kwargs) File "/metaflow/src/trainer.py", line 224, in train trainer.fit(self.t5_model, self.data_module) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 89, in launch self._call_children_scripts() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 94, in _call_children_scripts self._check_can_spawn_children() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 134, in _check_can_spawn_children raise RuntimeError( RuntimeError: Lightning attempted to launch new distributed processes with `local_rank > 0`. This should not happen. Possible reasons: 1) LOCAL_RANK environment variable was incorrectly modified by the user, 2) `ClusterEnvironment.creates_processes_externally` incorrectly implemented. Traceback (most recent call last): File "/metaflow/ddp_trainer.py", line 86, in main() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/metaflow/ddp_trainer.py", line 76, in main model.train(**kwargs) File "/metaflow/src/trainer.py", line 224, in train trainer.fit(self.t5_model, self.data_module) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 89, in launch self._call_children_scripts() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 94, in _call_children_scripts self._check_can_spawn_children() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 134, in _check_can_spawn_children raise RuntimeError( RuntimeError: Lightning attempted to launch new distributed processes with `local_rank > 0`. This should not happen. Possible reasons: 1) LOCAL_RANK environment variable was incorrectly modified by the user, 2) `ClusterEnvironment.creates_processes_externally` incorrectly implemented. [rank: 3] Global seed set to 84 [rank: 0] Global seed set to 84 Initializing distributed: GLOBAL_RANK: 8, MEMBER: 9/16 2023-07-05 22:21:35 INFO [torch.distributed.distributed_c10d] Added key: store_based_barrier_key:1 to store for rank: 8 0it [00:05, ?it/s] 0it [00:00, ?it/s] 0it [00:00, ?it/s] 2023-07-05 22:21:36 ERROR [__main__] main failed Traceback (most recent call last): File "/metaflow/ddp_trainer.py", line 86, in main() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/metaflow/ddp_trainer.py", line 76, in main model.train(**kwargs) File "/metaflow/src/trainer.py", line 224, in train trainer.fit(self.t5_model, self.data_module) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 89, in launch self._call_children_scripts() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 94, in _call_children_scripts self._check_can_spawn_children() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 134, in _check_can_spawn_children raise RuntimeError( RuntimeError: Lightning attempted to launch new distributed processes with `local_rank > 0`. This should not happen. Possible reasons: 1) LOCAL_RANK environment variable was incorrectly modified by the user, 2) `ClusterEnvironment.creates_processes_externally` incorrectly implemented. Traceback (most recent call last): File "/metaflow/ddp_trainer.py", line 86, in main() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/metaflow/ddp_trainer.py", line 76, in main model.train(**kwargs) File "/metaflow/src/trainer.py", line 224, in train trainer.fit(self.t5_model, self.data_module) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 89, in launch self._call_children_scripts() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 94, in _call_children_scripts self._check_can_spawn_children() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 134, in _check_can_spawn_children raise RuntimeError( RuntimeError: Lightning attempted to launch new distributed processes with `local_rank > 0`. This should not happen. Possible reasons: 1) LOCAL_RANK environment variable was incorrectly modified by the user, 2) `ClusterEnvironment.creates_processes_externally` incorrectly implemented. 2023-07-05 22:21:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [00:15, ?it/s] 2023-07-05 22:21:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [00:25, ?it/s] 2023-07-05 22:22:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [00:35, ?it/s] 2023-07-05 22:22:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [00:45, ?it/s] 2023-07-05 22:22:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [00:55, ?it/s] 2023-07-05 22:22:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [01:05, ?it/s] 2023-07-05 22:22:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [01:15, ?it/s] 2023-07-05 22:22:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [01:25, ?it/s] 2023-07-05 22:23:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [01:35, ?it/s] 2023-07-05 22:23:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [01:45, ?it/s] 2023-07-05 22:23:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [01:55, ?it/s] 2023-07-05 22:23:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [02:05, ?it/s] 2023-07-05 22:23:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [02:15, ?it/s] 2023-07-05 22:23:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [02:25, ?it/s] 2023-07-05 22:24:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [02:35, ?it/s] 2023-07-05 22:24:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [02:45, ?it/s] 2023-07-05 22:24:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [02:55, ?it/s] 2023-07-05 22:24:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [03:05, ?it/s] 2023-07-05 22:24:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [03:15, ?it/s] 2023-07-05 22:24:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [03:25, ?it/s] 2023-07-05 22:25:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [03:35, ?it/s] 2023-07-05 22:25:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [03:45, ?it/s] 2023-07-05 22:25:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [03:55, ?it/s] 2023-07-05 22:25:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [04:05, ?it/s] 2023-07-05 22:25:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [04:15, ?it/s] 2023-07-05 22:25:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [04:25, ?it/s] 2023-07-05 22:26:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [04:35, ?it/s] 2023-07-05 22:26:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [04:45, ?it/s] 2023-07-05 22:26:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [04:55, ?it/s] 2023-07-05 22:26:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [05:05, ?it/s] 2023-07-05 22:26:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [05:15, ?it/s] 2023-07-05 22:26:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [05:25, ?it/s] 2023-07-05 22:27:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [05:35, ?it/s] 2023-07-05 22:27:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [05:45, ?it/s] 2023-07-05 22:27:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [05:55, ?it/s] 2023-07-05 22:27:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [06:05, ?it/s] 2023-07-05 22:27:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [06:15, ?it/s] 2023-07-05 22:27:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [06:25, ?it/s] 2023-07-05 22:28:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [06:35, ?it/s] 2023-07-05 22:28:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [06:45, ?it/s] 2023-07-05 22:28:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [06:55, ?it/s] 2023-07-05 22:28:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [07:05, ?it/s] 2023-07-05 22:28:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [07:15, ?it/s] 2023-07-05 22:28:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [07:25, ?it/s] 2023-07-05 22:29:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [07:35, ?it/s] 2023-07-05 22:29:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [07:45, ?it/s] 2023-07-05 22:29:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [07:55, ?it/s] 2023-07-05 22:29:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [08:05, ?it/s] 2023-07-05 22:29:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [08:15, ?it/s] 2023-07-05 22:29:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [08:25, ?it/s] 2023-07-05 22:30:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [08:35, ?it/s] 2023-07-05 22:30:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [08:45, ?it/s] 2023-07-05 22:30:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [08:55, ?it/s] 2023-07-05 22:30:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [09:05, ?it/s] 2023-07-05 22:30:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [09:15, ?it/s] 2023-07-05 22:30:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [09:25, ?it/s] 2023-07-05 22:31:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [09:35, ?it/s] 2023-07-05 22:31:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [09:46, ?it/s] 2023-07-05 22:31:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [09:56, ?it/s] 2023-07-05 22:31:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [10:06, ?it/s] 2023-07-05 22:31:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [10:16, ?it/s] 2023-07-05 22:31:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [10:26, ?it/s] 2023-07-05 22:32:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [10:36, ?it/s] 2023-07-05 22:32:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [10:46, ?it/s] 2023-07-05 22:32:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [10:56, ?it/s] 2023-07-05 22:32:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [11:06, ?it/s] 2023-07-05 22:32:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [11:16, ?it/s] 2023-07-05 22:32:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [11:26, ?it/s] 2023-07-05 22:33:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [11:36, ?it/s] 2023-07-05 22:33:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [11:46, ?it/s] 2023-07-05 22:33:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [11:56, ?it/s] 2023-07-05 22:33:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [12:06, ?it/s] 2023-07-05 22:33:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [12:16, ?it/s] 2023-07-05 22:33:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [12:26, ?it/s] 2023-07-05 22:34:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [12:36, ?it/s] 2023-07-05 22:34:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [12:46, ?it/s] 2023-07-05 22:34:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [12:56, ?it/s] 2023-07-05 22:34:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [13:06, ?it/s] 2023-07-05 22:34:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [13:16, ?it/s] 2023-07-05 22:34:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [13:26, ?it/s] 2023-07-05 22:35:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [13:36, ?it/s] 2023-07-05 22:35:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [13:46, ?it/s] 2023-07-05 22:35:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [13:56, ?it/s] 2023-07-05 22:35:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [14:06, ?it/s] 2023-07-05 22:35:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [14:16, ?it/s] 2023-07-05 22:35:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [14:26, ?it/s] 2023-07-05 22:36:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [14:36, ?it/s] 2023-07-05 22:36:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [14:46, ?it/s] 2023-07-05 22:36:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [14:56, ?it/s] 2023-07-05 22:36:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [15:06, ?it/s] 2023-07-05 22:36:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [15:16, ?it/s] 2023-07-05 22:36:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [15:26, ?it/s] 2023-07-05 22:37:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [15:36, ?it/s] 2023-07-05 22:37:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [15:46, ?it/s] 2023-07-05 22:37:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [15:56, ?it/s] 2023-07-05 22:37:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [16:06, ?it/s] 2023-07-05 22:37:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [16:16, ?it/s] 2023-07-05 22:37:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [16:26, ?it/s] 2023-07-05 22:38:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [16:36, ?it/s] 2023-07-05 22:38:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [16:46, ?it/s] 2023-07-05 22:38:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [16:56, ?it/s] 2023-07-05 22:38:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [17:06, ?it/s] 2023-07-05 22:38:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [17:16, ?it/s] 2023-07-05 22:38:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [17:26, ?it/s] 2023-07-05 22:39:05 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [17:36, ?it/s] 2023-07-05 22:39:15 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [17:46, ?it/s] 2023-07-05 22:39:25 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [17:56, ?it/s] 2023-07-05 22:39:35 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [18:06, ?it/s] 2023-07-05 22:39:45 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [18:16, ?it/s] 2023-07-05 22:39:55 INFO [torch.distributed.distributed_c10d] Waiting in store based barrier to initialize process group for rank: 8, key: store_based_barrier_key:1 (world_size=16, worker_count=3, timeout=0:30:00) 0it [18:26, ?it/s] 0it [18:30, ?it/s] 2023-07-05 22:40:00 ERROR [__main__] main failed Traceback (most recent call last): File "/metaflow/ddp_trainer.py", line 86, in main() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/metaflow/ddp_trainer.py", line 76, in main model.train(**kwargs) File "/metaflow/src/trainer.py", line 224, in train trainer.fit(self.t5_model, self.data_module) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 90, in launch return function(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 645, in _fit_impl self._run(model, ckpt_path=self.ckpt_path) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1034, in _run self.strategy.setup_environment() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 153, in setup_environment self.setup_distributed() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 204, in setup_distributed _init_dist_connection(self.cluster_environment, self._process_group_backend, timeout=self._timeout) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/lightning_lite/utilities/distributed.py", line 237, in _init_dist_connection torch.distributed.init_process_group(torch_distributed_backend, rank=global_rank, world_size=world_size, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 627, in init_process_group _store_based_barrier(rank, store, timeout) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 242, in _store_based_barrier worker_count = store.add(store_key, 0) RuntimeError: Connection reset by peer Traceback (most recent call last): File "/metaflow/ddp_trainer.py", line 86, in main() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/metaflow/ddp_trainer.py", line 76, in main model.train(**kwargs) File "/metaflow/src/trainer.py", line 224, in train trainer.fit(self.t5_model, self.data_module) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 90, in launch return function(*args, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 645, in _fit_impl self._run(model, ckpt_path=self.ckpt_path) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1034, in _run self.strategy.setup_environment() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 153, in setup_environment self.setup_distributed() File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 204, in setup_distributed _init_dist_connection(self.cluster_environment, self._process_group_backend, timeout=self._timeout) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/lightning_lite/utilities/distributed.py", line 237, in _init_dist_connection torch.distributed.init_process_group(torch_distributed_backend, rank=global_rank, world_size=world_size, **kwargs) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 627, in init_process_group _store_based_barrier(rank, store, timeout) File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 242, in _store_based_barrier worker_count = store.add(store_key, 0) RuntimeError: Connection reset by peer failed: Internal error Traceback (most recent call last): File "/metaflow/metaflow/cli.py", line 1172, in main start(auto_envvar_prefix="METAFLOW", obj=state) File "/metaflow/metaflow/_vendor/click/core.py", line 829, in __call__ return self.main(args, kwargs) File "/metaflow/metaflow/_vendor/click/core.py", line 782, in main rv = self.invoke(ctx) File "/metaflow/metaflow/_vendor/click/core.py", line 1259, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/metaflow/metaflow/_vendor/click/core.py", line 1066, in invoke return ctx.invoke(self.callback, ctx.params) File "/metaflow/metaflow/_vendor/click/core.py", line 610, in invoke return callback(args, kwargs) File "/metaflow/metaflow/_vendor/click/decorators.py", line 21, in new_func return f(get_current_context(), args, kwargs) File "/metaflow/metaflow/cli.py", line 581, in step task.run_step( File "/metaflow/metaflow/task.py", line 586, in run_step self._exec_step_function(step_func) File "/metaflow/metaflow/task.py", line 60, in _exec_step_function step_function() File "/metaflow/metaflow/plugins/parallel_decorator.py", line 34, in _step_func_with_setup step_func() File "/metaflow/gpu_profile.py", line 148, in func f(s) File "/metaflow/custom_decorators.py", line 55, in wrapper return function(args, kwargs) File "/metaflow/ddp_flow.py", line 157, in train subprocess.run( File "/metaflow/metaflow_T5DDPFlow_linux-64_53554079b7dbe8c9b8991241feb21cc7cda7c922/lib/python3.10/subprocess.py", line 524, in run raise CalledProcessError(retcode, process.args, subprocess.CalledProcessError: Command '['python', 'ddp_trainer.py', '--output-dir', 'outputs', '--source-max-token-length', '256', '--target-max-token-length', '256', '--batch-size', '6', '--max-epochs', '1', '--learning-rate', '3e-4', '--weight-decay', '0.1', '--adam-epsilon', '9e-7', '--warmup-steps', '0', '--gradient-accumulation-steps', '16', '--n-gpu', '-1', '--num-nodes=4', '--early-stopping-patience-epochs', '0', '--precision', 'bf16', '--logger', 'default', '--dataloader-num-workers', '2', '--opt-level', '01', '--max-grad-norm', '0.5', '--seed', '42', '--use-gpu']' returned non-zero exit status 1.