13:56Z | md5sum | cut -d ' ' -f 1 | tail -c 9)) \\u0026\\u0026 export PYTHONUNBUFFERED=x MF_PATHSPEC=MinimumFlow/argo-minflow.user.bbrandt.min
imumflow-xz7gb/start/$METAFLOW_TASK_ID MF_DATASTORE=azure MF_ATTEMPT=0 MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stde
rr \\u0026\\u0026 mflog(){ T=$(date -u -Ins|tr , .); echo \\\"[MFLOG|0|${T:0:26}Z|task|$T]$1\\\" \\u003e\\u003e $MFLOG_STDOUT; echo $1;  } \\u
0026\\u0026 mflog 'Setting up task environment.' \\u0026\\u0026 python -m pip install requests -qqq \\u0026\\u0026 python -m pip install azure
-identity azure-storage-blob simple-azure-blob-downloader -qqq \\u0026\\u0026 mkdir metaflow \\u0026\\u0026 cd metaflow \\u0026\\u0026 mkdir .
metaflow \\u0026\\u0026 i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; download-azure-blob --blob-endpoint=${METAFLOW_AZURE_
STORAGE_BLOB_SERVICE_ENDPOINT%/} --container=metaflow-storage-container --blob=tf-full-stack-sysroot/MinimumFlow/data/6f/6fed11fa0d73ea8e34545
2aaaed3fdc92f55252a --output-file=job.tar \\u0026\\u0026 mflog 'Code package downloaded.' \\u0026\\u0026 break; sleep 10; i=$((i+1)); done \\u
0026\\u0026 if [ $i -gt 5 ]; then mflog 'Failed to download code package from metaflow-storage-container/tf-full-stack-sysroot/MinimumFlow/dat
a/6f/6fed11fa0d73ea8e345452aaaed3fdc92f55252a after 6 tries. Exiting...' \\u0026\\u0026 exit 1; fi \\u0026\\u0026 TAR_OPTIONS='--warning=no-ti
mestamp' tar xf job.tar \\u0026\\u0026 mflog 'Task is starting.' \\u0026\\u0026 (echo 'Bootstrapping environment...' \\u0026\\u0026 python -m
metaflow.plugins.conda.batch_bootstrap MinimumFlow metaflow_MinimumFlow_linux-64_c5b6dc01d4f4a08e54fef899876a247da78dfef5 azure \\u0026\\u0026
 echo 'Environment bootstrapped.' \\u0026\\u0026 if ! metaflow_MinimumFlow_linux-64_c5b6dc01d4f4a08e54fef899876a247da78dfef5/bin/python -s min
imum_flow.py dump --max-value-size=0 argo-minflow.user.bbrandt.minimumflow-xz7gb/_parameters/$METAFLOW_TASK_ID-params \\u003e/dev/null 2\\u003
e/dev/null; then metaflow_MinimumFlow_linux-64_c5b6dc01d4f4a08e54fef899876a247da78dfef5/bin/python -s minimum_flow.py --with retry:times=3,min
utes_between_retries=2 --with 'conda:libraries={}' --with 'kubernetes:cpu=1,memory=4096,disk=10240,image=python:3.10,service_account=default,n
amespace=argo,gpu_vendor=nvidia,tolerations=[{\\\"key\\\": \\\"kubernetes.azure.com/scalesetpriority\\\", \\\"value\\\": \\\"spot\\\", \\\"eff
ect\\\": \\\"PreferNoSchedule\\\"}, {\\\"key\\\": \\\"kubernetes.azure.com/scalesetpriority\\\", \\\"value\\\": \\\"spot\\\", \\\"effect\\\":
\\\"NoSchedule\\\"}]' --with 'environment:vars={}' --quiet --metadata=service --environment=conda --datastore=azure --datastore-root=metaflow-
storage-container/tf-full-stack-sysroot --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=argo_workflows_intern
al init --run-id argo-minflow.user.bbrandt.minimumflow-xz7gb --task-id $METAFLOW_TASK_ID-params; fi \\u0026\\u0026 metaflow_MinimumFlow_linux-
64_c5b6dc01d4f4a08e54fef899876a247da78dfef5/bin/python -s minimum_flow.py --with retry:times=3,minutes_between_retries=2 --with 'conda:librari
es={}' --with 'kubernetes:cpu=1,memory=4096,disk=10240,image=python:3.10,service_account=default,namespace=argo,gpu_vendor=nvidia,tolerations=
[{\\\"key\\\": \\\"kubernetes.azure.com/scalesetpriority\\\", \\\"value\\\": \\\"spot\\\", \\\"effect\\\": \\\"PreferNoSchedule\\\"}, {\\\"key
\\\": \\\"kubernetes.azure.com/scalesetpriority\\\", \\\"value\\\": \\\"spot\\\", \\\"effect\\\": \\\"NoSchedule\\\"}]' --with 'environment:va
rs={}' --quiet --metadata=service --environment=conda --datastore=azure --datastore-root=metaflow-storage-container/tf-full-stack-sysroot --ev
ent-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=argo_workflows_internal step start --run-id argo-minflow.user.bbr
andt.minimumflow-xz7gb --task-id $METAFLOW_TASK_ID --retry-count 0 --max-user-code-retries 3 --input-paths argo-minflow.user.bbrandt.minimumfl13:56Z | md5sum | cut -d ' ' -f 1 | tail -c 9)) \\u0026\\u0026 export PYTHONUNBUFFERED=x MF_PATHSPEC=MinimumFlow/argo-minflow.user.bbrandt.min
imumflow-xz7gb/start/$METAFLOW_TASK_ID MF_DATASTORE=azure MF_ATTEMPT=0 MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stde
rr \\u0026\\u0026 mflog(){ T=$(date -u -Ins|tr , .); echo \\\"[MFLOG|0|${T:0:26}Z|task|$T]$1\\\" \\u003e\\u003e $MFLOG_STDOUT; echo $1;  } \\u
0026\\u0026 mflog 'Setting up task environment.' \\u0026\\u0026 python -m pip install requests -qqq \\u0026\\u0026 python -m pip install azure
-identity azure-storage-blob simple-azure-blob-downloader -qqq \\u0026\\u0026 mkdir metaflow \\u0026\\u0026 cd metaflow \\u0026\\u0026 mkdir .
metaflow \\u0026\\u0026 i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; download-azure-blob --blob-endpoint=${METAFLOW_AZURE_
STORAGE_BLOB_SERVICE_ENDPOINT%/} --container=metaflow-storage-container --blob=tf-full-stack-sysroot/MinimumFlow/data/6f/6fed11fa0d73ea8e34545
2aaaed3fdc92f55252a --output-file=job.tar \\u0026\\u0026 mflog 'Code package downloaded.' \\u0026\\u0026 break; sleep 10; i=$((i+1)); done \\u
0026\\u0026 if [ $i -gt 5 ]; then mflog 'Failed to download code package from metaflow-storage-container/tf-full-stack-sysroot/MinimumFlow/dat
a/6f/6fed11fa0d73ea8e345452aaaed3fdc92f55252a after 6 tries. Exiting...' \\u0026\\u0026 exit 1; fi \\u0026\\u0026 TAR_OPTIONS='--warning=no-ti
mestamp' tar xf job.tar \\u0026\\u0026 mflog 'Task is starting.' \\u0026\\u0026 (echo 'Bootstrapping environment...' \\u0026\\u0026 python -m
metaflow.plugins.conda.batch_bootstrap MinimumFlow metaflow_MinimumFlow_linux-64_c5b6dc01d4f4a08e54fef899876a247da78dfef5 azure \\u0026\\u0026
 echo 'Environment bootstrapped.' \\u0026\\u0026 if ! metaflow_MinimumFlow_linux-64_c5b6dc01d4f4a08e54fef899876a247da78dfef5/bin/python -s min
imum_flow.py dump --max-value-size=0 argo-minflow.user.bbrandt.minimumflow-xz7gb/_parameters/$METAFLOW_TASK_ID-params \\u003e/dev/null 2\\u003
e/dev/null; then metaflow_MinimumFlow_linux-64_c5b6dc01d4f4a08e54fef899876a247da78dfef5/bin/python -s minimum_flow.py --with retry:times=3,min
utes_between_retries=2 --with 'conda:libraries={}' --with 'kubernetes:cpu=1,memory=4096,disk=10240,image=python:3.10,service_account=default,n
amespace=argo,gpu_vendor=nvidia,tolerations=[{\\\"key\\\": \\\"kubernetes.azure.com/scalesetpriority\\\", \\\"value\\\": \\\"spot\\\", \\\"eff
ect\\\": \\\"PreferNoSchedule\\\"}, {\\\"key\\\": \\\"kubernetes.azure.com/scalesetpriority\\\", \\\"value\\\": \\\"spot\\\", \\\"effect\\\":
\\\"NoSchedule\\\"}]' --with 'environment:vars={}' --quiet --metadata=service --environment=conda --datastore=azure --datastore-root=metaflow-
storage-container/tf-full-stack-sysroot --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=argo_workflows_intern
al init --run-id argo-minflow.user.bbrandt.minimumflow-xz7gb --task-id $METAFLOW_TASK_ID-params; fi \\u0026\\u0026 metaflow_MinimumFlow_linux-
64_c5b6dc01d4f4a08e54fef899876a247da78dfef5/bin/python -s minimum_flow.py --with retry:times=3,minutes_between_retries=2 --with 'conda:librari
es={}' --with 'kubernetes:cpu=1,memory=4096,disk=10240,image=python:3.10,service_account=default,namespace=argo,gpu_vendor=nvidia,tolerations=
[{\\\"key\\\": \\\"kubernetes.azure.com/scalesetpriority\\\", \\\"value\\\": \\\"spot\\\", \\\"effect\\\": \\\"PreferNoSchedule\\\"}, {\\\"key
\\\": \\\"kubernetes.azure.com/scalesetpriority\\\", \\\"value\\\": \\\"spot\\\", \\\"effect\\\": \\\"NoSchedule\\\"}]' --with 'environment:va
rs={}' --quiet --metadata=service --environment=conda --datastore=azure --datastore-root=metaflow-storage-container/tf-full-stack-sysroot --ev
ent-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=argo_workflows_internal step start --run-id argo-minflow.user.bbr
andt.minimumflow-xz7gb --task-id $METAFLOW_TASK_ID --retry-count 0 --max-user-code-retries 3 --input-paths argo-minflow.user.bbrandt.minimumflow-xz7gb/_parameters/$METAFLOW_TASK_ID-params) 1\\u003e\\u003e \\u003e(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2\\u003e\\u003e \\u003
e(python -m metaflow.mflog.tee task $MFLOG_STDERR \\u003e\\u00262); c=$?; python -m metaflow.mflog.save_logs; exit $c\"],\"envFrom\":[{\"secre
tRef\":{\"name\":\"metaflow-azure-storage-credentials\"}}],\"env\":[{\"name\":\"METAFLOW_CODE_URL\",\"value\":\"metaflow-storage-container/tf-
full-stack-sysroot/MinimumFlow/data/6f/6fed11fa0d73ea8e345452aaaed3fdc92f55252a\"},{\"name\":\"METAFLOW_CODE_SHA\",\"value\":\"6fed11fa0d73ea8
e345452aaaed3fdc92f55252a\"},{\"name\":\"METAFLOW_CODE_DS\",\"value\":\"azure\"},{\"name\":\"METAFLOW_SERVICE_URL\",\"value\":\"http://metadat
a-service.default:8080/\"},{\"name\":\"METAFLOW_SERVICE_HEADERS\",\"value\":\"{}\"},{\"name\":\"METAFLOW_USER\",\"value\":\"argo-workflows\"},
{\"name\":\"METAFLOW_DEFAULT_DATASTORE\",\"value\":\"azure\"},{\"name\":\"METAFLOW_DEFAULT_METADATA\",\"value\":\"service\"},{\"name\":\"METAF
LOW_KUBERNETES_WORKLOAD\",\"value\":\"1\"},{\"name\":\"METAFLOW_RUNTIME_ENVIRONMENT\",\"value\":\"kubernetes\"},{\"name\":\"METAFLOW_OWNER\",\
"value\":\"bbrandt\"},{\"name\":\"METAFLOW_FLOW_NAME\",\"value\":\"MinimumFlow\"},{\"name\":\"METAFLOW_STEP_NAME\",\"value\":\"start\"},{\"nam
e\":\"METAFLOW_RUN_ID\",\"value\":\"argo-minflow.user.bbrandt.minimumflow-xz7gb\"},{\"name\":\"METAFLOW_RETRY_COUNT\",\"value\":\"0\"},{\"name
\":\"METAFLOW_PRODUCTION_TOKEN\",\"value\":\"mfprj-y7u4njbwyjoegagn-0-lera\"},{\"name\":\"ARGO_WORKFLOW_TEMPLATE\",\"value\":\"minflow.user.bb
randt.minimumflow\"},{\"name\":\"ARGO_WORKFLOW_NAME\",\"value\":\"minflow.user.bbrandt.minimumflow-xz7gb\"},{\"name\":\"ARGO_WORKFLOW_NAMESPAC
E\",\"value\":\"argo\"},{\"name\":\"METAFLOW_RUNTIME_NAME\",\"value\":\"argo-workflows\"},{\"name\":\"USER\",\"value\":\"bbrandt\"},{\"name\":
\"METAFLOW_AZURE_STORAGE_BLOB_SERVICE_ENDPOINT\",\"value\":\"redacted\"},{\"name\":\"METAFLOW_DA
TASTORE_SYSROOT_AZURE\",\"value\":\"metaflow-storage-container/tf-full-stack-sysroot\"},{\"name\":\"METAFLOW_CARD_AZUREROOT\",\"value\":\"meta
flow-storage-container/tf-full-stack-sysroot/mf.cards\"},{\"name\":\"METAFLOW_VERSION\",\"value\":\"{\\\"platform\\\": \\\"Linux\\\", \\\"user
name\\\": \\\"bbrandt\\\", \\\"production_token\\\": \\\"mfprj-y7u4njbwyjoegagn-0-lera\\\", \\\"runtime\\\": \\\"dev\\\", \\\"app\\\": null, \
\\"environment_type\\\": \\\"local\\\", \\\"use_r\\\": false, \\\"python_version\\\": \\\"3.10.9 | packaged by conda-forge | (main, Feb  2 202
3, 20:20:04) [GCC 11.3.0]\\\", \\\"python_version_code\\\": \\\"3.10.9\\\", \\\"metaflow_version\\\": \\\"2.8.0\\\", \\\"script\\\": \\\"minim
um_flow.py\\\", \\\"ext_info\\\": [{\\\"_pythonpath_0\\\": {\\\"root_paths\\\": null, \\\"meta_module\\\": null, \\\"files\\\": []}}, {}], \\\
"flow_name\\\": \\\"MinimumFlow\\\"}\"},{\"name\":\"METAFLOW_KUBERNETES_POD_NAMESPACE\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.
namespace\"}}},{\"name\":\"METAFLOW_KUBERNETES_POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"METAFLOW
_KUBERNETES_POD_ID\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.uid\"}}},{\"name\":\"METAFLOW_KUBERNETES_SERVICE_ACCOUNT_NAME\",\"v
alueFrom\":{\"fieldRef\":{\"fieldPath\":\"spec.serviceAccountName\"}}},{\"name\":\"METAFLOW_KUBERNETES_NODE_IP\",\"valueFrom\":{\"fieldRef\":{
\"fieldPath\":\"status.hostIP\"}}}],\"resources\":{\"requests\":{\"cpu\":\"1\",\"ephemeral-storage\":\"10240M\",\"memory\":\"4096M\"}},\"volumeMounts\":[{\"name\":\"out\",\"mountPath\":\"/mnt/out\"}]},\"volumes\":[{\"name\":\"out\",\"emptyDir\":{}}],\"archiveLocation\":{\"archiveLogs
\":true,\"s3\":{\"endpoint\":\"minio:9000\",\"bucket\":\"my-bucket\",\"insecure\":true,\"accessKeySecret\":{\"name\":\"my-minio-cred\",\"key\"
:\"accesskey\"},\"redacted\":{\"name\":\"my-minio-cred\",\"key\":\"redacted\"},\"key\":\"minflow.user.bbrandt.minimumflow-xz7gb/minflo
w.user.bbrandt.minimumflow-xz7gb-start-2964333059\"}},\"activeDeadlineSeconds\":432000,\"retryStrategy\":{\"limit\":3,\"retryPolicy\":\"Always
\",\"backoff\":{\"duration\":\"2m\"}},\"failFast\":true,\"tolerations\":[{\"key\":\"kubernetes.azure.com/scalesetpriority\",\"value\":\"spot\"
,\"effect\":\"PreferNoSchedule\"},{\"key\":\"kubernetes.azure.com/scalesetpriority\",\"value\":\"spot\",\"effect\":\"NoSchedule\"}],\"serviceA
ccountName\":\"default\"}" version="&Version{Version:v3.4.4,BuildDate:2022-11-29T16:49:53Z,GitCommit:3b2626ff900aff2424c086a51af5929fb0b2d7e5,
GitTag:v3.4.4,GitTreeState:clean,GoVersion:go1.18.8,Compiler:gc,Platform:linux/amd64,}"
time="2023-02-28T17:15:29.557Z" level=info msg="Starting deadline monitor"
time="2023-02-28T17:16:05.563Z" level=info msg="Main container completed" error="<nil>"
time="2023-02-28T17:16:05.563Z" level=info msg="No Script output reference in workflow. Capturing script output ignored"
time="2023-02-28T17:16:05.563Z" level=info msg="Saving output parameters"
time="2023-02-28T17:16:05.563Z" level=info msg="Saving path output parameter: task-id"
time="2023-02-28T17:16:05.563Z" level=info msg="Copying /mnt/out/task_id from volume mount"
time="2023-02-28T17:16:05.563Z" level=error msg="executor error: open /mainctrfs/mnt/out/task_id: no such file or directory"
time="2023-02-28T17:16:05.563Z" level=info msg="No output artifacts"
time="2023-02-28T17:16:05.563Z" level=info msg="S3 Save path: /tmp/argo/outputs/logs/main.log, key: minflow.user.bbrandt.minimumflow-xz7gb/min
flow.user.bbrandt.minimumflow-xz7gb-start-2964333059/main.log"
time="2023-02-28T17:16:05.563Z" level=info msg="Creating minio client using static credentials" endpoint="minio:9000"
time="2023-02-28T17:16:05.563Z" level=info msg="Saving file to s3" bucket=my-bucket endpoint="minio:9000" key=minflow.user.bbrandt.minimumflow
-xz7gb/minflow.user.bbrandt.minimumflow-xz7gb-start-2964333059/main.log path=/tmp/argo/outputs/logs/main.log
time="2023-02-28T17:16:05.595Z" level=info msg="Save artifact" artifactName=main-logs duration=31.736704ms error="<nil>" key=minflow.user.bbra
ndt.minimumflow-xz7gb/minflow.user.bbrandt.minimumflow-xz7gb-start-2964333059/main.log
time="2023-02-28T17:16:05.595Z" level=info msg="not deleting local artifact" localArtPath=/tmp/argo/outputs/logs/main.log
time="2023-02-28T17:16:05.595Z" level=info msg="Successfully saved file: /tmp/argo/outputs/logs/main.log"
time="2023-02-28T17:16:05.646Z" level=info msg="Create workflowtaskresults 201"
time="2023-02-28T17:16:05.647Z" level=info msg="Deadline monitor stopped"
time="2023-02-28T17:16:05.647Z" level=info msg="Alloc=7724 TotalAlloc=13017 Sys=19154 NumGC=4 Goroutines=10"
time="2023-02-28T17:16:05.647Z" level=info msg="stopping progress monitor (context done)" error="context canceled"
time="2023-02-28T17:16:05.663Z" level=fatal msg="open /mainctrfs/mnt/out/task_id: no such file or directory"