WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv 0%| | 0/50000 [00:00 failed: Internal error Traceback (most recent call last): File "/metaflow/metaflow/cli.py", line 1172, in main start(auto_envvar_prefix="METAFLOW", obj=state) File "/metaflow/metaflow/_vendor/click/core.py", line 829, in __call__ return self.main(args, kwargs) File "/metaflow/metaflow/_vendor/click/core.py", line 782, in main rv = self.invoke(ctx) File "/metaflow/metaflow/_vendor/click/core.py", line 1259, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/metaflow/metaflow/_vendor/click/core.py", line 1066, in invoke return ctx.invoke(self.callback, ctx.params) File "/metaflow/metaflow/_vendor/click/core.py", line 610, in invoke return callback(args, kwargs) File "/metaflow/metaflow/_vendor/click/decorators.py", line 21, in new_func return f(get_current_context(), args, kwargs) File "/metaflow/metaflow/cli.py", line 581, in step task.run_step( File "/metaflow/metaflow/task.py", line 586, in run_step self._exec_step_function(step_func) File "/metaflow/metaflow/task.py", line 60, in _exec_step_function step_function() File "/metaflow/metaflow/plugins/parallel_decorator.py", line 34, in _step_func_with_setup step_func() File "/metaflow/gpu_profile.py", line 148, in func f(s) File "/metaflow/custom_decorators.py", line 32, in wrapper return function(args, kwargs) File "/metaflow/fsdp_flow.py", line 247, in train trainer.fit( File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 603, in fit call._call_and_handle_interrupt( File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 38, in _call_and_handle_interrupt return trainer_fn(args, kwargs) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 645, in _fit_impl self._run(model, ckpt_path=self.ckpt_path) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1098, in _run results = self._run_stage() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1177, in _run_stage self._run_train() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1190, in _run_train self._run_sanity_check() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1255, in _run_sanity_check val_loop._reload_evaluation_dataloaders() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 234, in _reload_evaluation_dataloaders self.trainer.reset_val_dataloader() File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1635, in reset_val_dataloader self.num_val_batches, self.val_dataloaders = self._data_connector._reset_eval_dataloader( File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 377, in _reset_eval_dataloader dataloaders = [self._prepare_dataloader(dl, mode=mode) for dl in dataloaders if dl is not None] File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 377, in dataloaders = [self._prepare_dataloader(dl, mode=mode) for dl in dataloaders if dl is not None] File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 283, in _prepare_dataloader sampler = self._resolve_sampler(dataloader, shuffle=shuffle, mode=mode) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 300, in _resolve_sampler sampler = self._get_distributed_sampler( File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 339, in _get_distributed_sampler sampler = cls(dataloader.sampler, kwargs) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/lightning_lite/utilities/distributed.py", line 300, in __init__ super().__init__(_DatasetSamplerWrapper(sampler), args, kwargs) File "/metaflow/metaflow_FSDPFlow_linux-64_0194a13432df777baea5577cb5634714a2a36c5e/lib/python3.10/site-packages/torch/utils/data/distributed.py", line 73, in __init__ raise ValueError( ValueError: Invalid rank 1, rank should be in the interval [0, 0]