WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv 0%| | 0/50000 [00:00 failed: Internal error Traceback (most recent call last): File "/metaflow/metaflow/cli.py", line 1172, in main start(auto_envvar_prefix="METAFLOW", obj=state) File "/metaflow/metaflow/_vendor/click/core.py", line 829, in __call__ return self.main(args, kwargs) File "/metaflow/metaflow/_vendor/click/core.py", line 782, in main rv = self.invoke(ctx) File "/metaflow/metaflow/_vendor/click/core.py", line 1259, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/metaflow/metaflow/_vendor/click/core.py", line 1066, in invoke return ctx.invoke(self.callback, ctx.params) File "/metaflow/metaflow/_vendor/click/core.py", line 610, in invoke return callback(args, kwargs) File "/metaflow/metaflow/_vendor/click/decorators.py", line 21, in new_func return f(get_current_context(), args, kwargs) File "/metaflow/metaflow/cli.py", line 581, in step task.run_step( File "/metaflow/metaflow/task.py", line 586, in run_step self._exec_step_function(step_func) File "/metaflow/metaflow/task.py", line 60, in _exec_step_function step_function() File "/metaflow/metaflow/plugins/parallel_decorator.py", line 34, in _step_func_with_setup step_func() File "/metaflow/gpu_profile.py", line 148, in func f(s) File "/metaflow/custom_decorators.py", line 32, in wrapper return function(args, kwargs) File "/metaflow/fsdp_flow.py", line 247, in train trainer.fit( File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 38, in _call_and_handle_interrupt return trainer_fn(args, kwargs) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 645, in _fit_impl self._run(model, ckpt_path=self.ckpt_path) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1098, in _run results = self._run_stage() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1177, in _run_stage self._run_train() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1190, in _run_train self._run_sanity_check() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1255, in _run_sanity_check val_loop._reload_evaluation_dataloaders() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 234, in _reload_evaluation_dataloaders self.trainer.reset_val_dataloader() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1635, in reset_val_dataloader self.num_val_batches, self.val_dataloaders = self._data_connector._reset_eval_dataloader( File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 391, in _reset_eval_dataloader len(dataloader) if has_len_all_ranks(dataloader, self.trainer.strategy, module) else float("inf") File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/utilities/data.py", line 108, in has_len_all_ranks total_length = strategy.reduce(torch.tensor(local_length, device=strategy.root_device), reduce_op="sum") File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/strategies/fully_sharded_native.py", line 308, in reduce tensor = _sync_ddp_if_available(tensor, group, reduce_op=reduce_op) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/lightning_lite/utilities/distributed.py", line 106, in _sync_ddp_if_available return _sync_ddp(result, group=group, reduce_op=reduce_op) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/lightning_lite/utilities/distributed.py", line 150, in _sync_ddp torch.distributed.barrier(group=group) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py", line 2791, in barrier work.wait() RuntimeError: [/opt/conda/conda-bld/pytorch_1656352645774/work/third_party/gloo/gloo/transport/tcp/pair.cc:598] Connection closed by peer [10.14.52.12]:26036 AWS Batch error: Essential container in task exited This could be a transient error. Use @retry to retry. Task failed.