{ "apiVersion": "argoproj.io/v1alpha1", "kind": "WorkflowTemplate", "metadata": { "name": "workflow-test", "namespace": "default", "labels": { "app.kubernetes.io/name": "metaflow-flow", "app.kubernetes.io/part-of": "metaflow" }, "annotations": { "metaflow/production_token": "workflow-test-0-ypbq", "metaflow/owner": "amann", "metaflow/user": "argo-workflows", "metaflow/flow_name": "workflow-test", "metaflow/deployment_timestamp": "2024-02-28T19:08:35.748941+00:00", "metaflow/parameters": "{\"project_file\": {\"name\": \"project_file\", \"value\": \"\\\"{\\\\\\\"type\\\\\\\": \\\\\\\"uploader-v2\\\\\\\", \\\\\\\"url\\\\\\\": \\\\\\\"s3://artifact-bucket/metaflow_artifacts/workflow-test/339baab520ce77f8def66f326bb9b9fe8423acef\\\\\\\", \\\\\\\"is_text\\\\\\\": true, \\\\\\\"encoding\\\\\\\": \\\\\\\"utf-8\\\\\\\", \\\\\\\"note\\\\\\\": \\\\\\\"Internal representation of IncludeFile(MLProject)\\\\\\\", \\\\\\\"sub-type\\\\\\\": \\\\\\\"uploaded\\\\\\\", \\\\\\\"size\\\\\\\": 857}\\\"\", \"type\": \"FilePath\", \"description\": \"The MLFlow project definition file\", \"is_required\": false}}" } }, "spec": { "parallelism": 100, "workflowMetadata": { "labels": { "app.kubernetes.io/name": "metaflow-run", "app.kubernetes.io/part-of": "metaflow" }, "annotations": { "metaflow/production_token": "workflow-test-0-ypbq", "metaflow/owner": "amann", "metaflow/user": "argo-workflows", "metaflow/flow_name": "workflow-test", "metaflow/deployment_timestamp": "2024-02-28T19:08:35.748941+00:00", "metaflow/parameters": "{\"project_file\": {\"name\": \"project_file\", \"value\": \"\\\"{\\\\\\\"type\\\\\\\": \\\\\\\"uploader-v2\\\\\\\", \\\\\\\"url\\\\\\\": \\\\\\\"s3://artifact-bucket/metaflow_artifacts/workflow-test/339baab520ce77f8def66f326bb9b9fe8423acef\\\\\\\", \\\\\\\"is_text\\\\\\\": true, \\\\\\\"encoding\\\\\\\": \\\\\\\"utf-8\\\\\\\", \\\\\\\"note\\\\\\\": \\\\\\\"Internal representation of IncludeFile(MLProject)\\\\\\\", \\\\\\\"sub-type\\\\\\\": \\\\\\\"uploaded\\\\\\\", \\\\\\\"size\\\\\\\": 857}\\\"\", \"type\": \"FilePath\", \"description\": \"The MLFlow project definition file\", \"is_required\": false}}", "metaflow/run_id": "argo-{{workflow.name}}" } }, "arguments": { "parameters": [ { "name": "project_file", "value": "\"{\\\"type\\\": \\\"uploader-v2\\\", \\\"url\\\": \\\"s3://artifact-bucket/metaflow_artifacts/workflow-test/339baab520ce77f8def66f326bb9b9fe8423acef\\\", \\\"is_text\\\": true, \\\"encoding\\\": \\\"utf-8\\\", \\\"note\\\": \\\"Internal representation of IncludeFile(MLProject)\\\", \\\"sub-type\\\": \\\"uploaded\\\", \\\"size\\\": 857}\"", "description": "The MLFlow project definition file" } ] }, "podMetadata": { "labels": { "app.kubernetes.io/name": "metaflow-task", "app.kubernetes.io/part-of": "metaflow" }, "annotations": { "metaflow/production_token": "workflow-test-0-ypbq", "metaflow/owner": "amann", "metaflow/user": "argo-workflows", "metaflow/flow_name": "workflow-test", "metaflow/deployment_timestamp": "2024-02-28T19:08:35.748941+00:00", "metaflow/parameters": "{\"project_file\": {\"name\": \"project_file\", \"value\": \"\\\"{\\\\\\\"type\\\\\\\": \\\\\\\"uploader-v2\\\\\\\", \\\\\\\"url\\\\\\\": \\\\\\\"s3://artifact-bucket/metaflow_artifacts/workflow-test/339baab520ce77f8def66f326bb9b9fe8423acef\\\\\\\", \\\\\\\"is_text\\\\\\\": true, \\\\\\\"encoding\\\\\\\": \\\\\\\"utf-8\\\\\\\", \\\\\\\"note\\\\\\\": \\\\\\\"Internal representation of IncludeFile(MLProject)\\\\\\\", \\\\\\\"sub-type\\\\\\\": \\\\\\\"uploaded\\\\\\\", \\\\\\\"size\\\\\\\": 857}\\\"\", \"type\": \"FilePath\", \"description\": \"The MLFlow project definition file\", \"is_required\": false}}" } }, "entrypoint": "workflow-test", "hooks": {}, "templates": [ { "name": "workflow-test", "dag": { "failFast": true, "tasks": [ { "name": "start", "template": "start" }, { "name": "train", "dependencies": [ "start" ], "template": "train", "arguments": { "parameters": [ { "name": "input-paths", "value": "argo-{{workflow.name}}/start/{{tasks.start.outputs.parameters.task-id}}" } ] } }, { "name": "end", "dependencies": [ "train" ], "template": "end", "arguments": { "parameters": [ { "name": "input-paths", "value": "argo-{{workflow.name}}/train/{{tasks.train.outputs.parameters.task-id}}" } ] } } ] } }, { "name": "start", "activeDeadlineSeconds": 432000, "serviceAccountName": "argo-workflows-workflow", "inputs": { "parameters": [] }, "outputs": { "parameters": [ { "name": "task-id", "valueFrom": { "path": "/mnt/out/task_id" } } ] }, "failFast": true, "metadata": { "annotations": { "metaflow/step_name": "start", "metaflow/attempt": "0" } }, "volumes": [ { "name": "out", "emptyDir": {} }, { "name": "dhsm", "emptyDir": { "medium": "Memory" } } ], "nodeSelector": {}, "tolerations": null, "container": { "args": null, "command": [ "bash", "-c", "${METAFLOW_INIT_SCRIPT:+eval \"${METAFLOW_INIT_SCRIPT}\"} && mkdir -p $PWD/.logs && export METAFLOW_TASK_ID=(t-$(echo start-{{workflow.creationTimestamp}} | md5sum | cut -d ' ' -f 1 | tail -c 9)) && export PYTHONUNBUFFERED=x MF_PATHSPEC=workflow-test/argo-{{workflow.name}}/start/$METAFLOW_TASK_ID MF_DATASTORE=s3 MF_ATTEMPT=0 MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://artifact-bucket/workflow-test/data/00/00839a0b2be1025b649e875c0a24d33b495d7ae0 job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://artifact-bucket/workflow-test/data/00/00839a0b2be1025b649e875c0a24d33b495d7ae0 after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (if ! python workflow.py dump --max-value-size=0 argo-{{workflow.name}}/_parameters/$METAFLOW_TASK_ID-params >/dev/null 2>/dev/null; then python workflow.py --with kubernetes:cpu=1,memory=4096,disk=10240,image=,service_account=argo-workflows-workflow,namespace=default,gpu_vendor=nvidia,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --with 'environment:vars={}' --quiet --metadata=local --environment=local --datastore=s3 --datastore-root=s3://artifact-bucket --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=argo_workflows_internal:auto-emit-argo-events=1 init --run-id argo-{{workflow.name}} --task-id $METAFLOW_TASK_ID-params --project_file={{workflow.parameters.project_file}}; fi && python workflow.py --with kubernetes:cpu=1,memory=4096,disk=10240,image=,service_account=argo-workflows-workflow,namespace=default,gpu_vendor=nvidia,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --with 'environment:vars={}' --quiet --metadata=local --environment=local --datastore=s3 --datastore-root=s3://artifact-bucket --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=argo_workflows_internal:auto-emit-argo-events=1 step start --run-id argo-{{workflow.name}} --task-id $METAFLOW_TASK_ID --retry-count 0 --max-user-code-retries 0 --input-paths argo-{{workflow.name}}/_parameters/$METAFLOW_TASK_ID-params) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "env": [ { "name": "METAFLOW_CODE_URL", "value": "s3://artifact-bucket/workflow-test/data/00/00839a0b2be1025b649e875c0a24d33b495d7ae0", "valueFrom": null }, { "name": "METAFLOW_CODE_SHA", "value": "00839a0b2be1025b649e875c0a24d33b495d7ae0", "valueFrom": null }, { "name": "METAFLOW_CODE_DS", "value": "s3", "valueFrom": null }, { "name": "METAFLOW_SERVICE_HEADERS", "value": "{}", "valueFrom": null }, { "name": "METAFLOW_USER", "value": "argo-workflows", "valueFrom": null }, { "name": "METAFLOW_DATASTORE_SYSROOT_S3", "value": "s3://artifact-bucket", "valueFrom": null }, { "name": "METAFLOW_DATATOOLS_S3ROOT", "value": "s3://artifact-bucket/metaflow_artifacts", "valueFrom": null }, { "name": "METAFLOW_DEFAULT_DATASTORE", "value": "s3", "valueFrom": null }, { "name": "METAFLOW_DEFAULT_METADATA", "value": "local", "valueFrom": null }, { "name": "METAFLOW_CARD_S3ROOT", "value": "s3://artifact-bucket/mf.cards", "valueFrom": null }, { "name": "METAFLOW_KUBERNETES_WORKLOAD", "value": "1", "valueFrom": null }, { "name": "METAFLOW_KUBERNETES_FETCH_EC2_METADATA", "value": "False", "valueFrom": null }, { "name": "METAFLOW_RUNTIME_ENVIRONMENT", "value": "kubernetes", "valueFrom": null }, { "name": "METAFLOW_OWNER", "value": "amann", "valueFrom": null }, { "name": "METAFLOW_ARGO_EVENTS_EVENT_BUS", "value": "default", "valueFrom": null }, { "name": "METAFLOW_ARGO_EVENTS_WEBHOOK_AUTH", "value": "none", "valueFrom": null }, { "name": "METAFLOW_FLOW_NAME", "value": "workflow-test", "valueFrom": null }, { "name": "METAFLOW_STEP_NAME", "value": "start", "valueFrom": null }, { "name": "METAFLOW_RUN_ID", "value": "argo-{{workflow.name}}", "valueFrom": null }, { "name": "METAFLOW_RETRY_COUNT", "value": "0", "valueFrom": null }, { "name": "METAFLOW_PRODUCTION_TOKEN", "value": "workflow-test-0-ypbq", "valueFrom": null }, { "name": "ARGO_WORKFLOW_TEMPLATE", "value": "workflow-test", "valueFrom": null }, { "name": "ARGO_WORKFLOW_NAME", "value": "{{workflow.name}}", "valueFrom": null }, { "name": "ARGO_WORKFLOW_NAMESPACE", "value": "default", "valueFrom": null }, { "name": "METAFLOW_RUNTIME_NAME", "value": "argo-workflows", "valueFrom": null }, { "name": "USER", "value": "amann", "valueFrom": null }, { "name": "METAFLOW_VERSION", "value": "{\"platform\": \"Darwin\", \"username\": \"amann\", \"production_token\": \"workflow-test-0-ypbq\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.11.4 (main, Jul 25 2023, 17:36:13) [Clang 14.0.3 (clang-1403.0.22.14.1)]\", \"python_version_code\": \"3.11.4\", \"metaflow_version\": \"2.11.2\", \"script\": \"workflow.py\", \"flow_name\": \"workflow-test\"}", "valueFrom": null }, { "name": "METAFLOW_KUBERNETES_POD_NAMESPACE", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "metadata.namespace" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_POD_NAME", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "metadata.name" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_POD_ID", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "metadata.uid" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_SERVICE_ACCOUNT_NAME", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "spec.serviceAccountName" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_NODE_IP", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "status.hostIP" }, "resourceFieldRef": null, "secretKeyRef": null } } ], "envFrom": [], "image": "", "imagePullPolicy": null, "lifecycle": null, "livenessProbe": null, "name": "start", "ports": null, "readinessProbe": null, "resizePolicy": null, "resources": { "claims": null, "limits": {}, "requests": { "cpu": "1", "memory": "4096M", "ephemeral-storage": "10240M" } }, "restartPolicy": null, "securityContext": null, "startupProbe": null, "stdin": null, "stdinOnce": null, "terminationMessagePath": null, "terminationMessagePolicy": null, "tty": null, "volumeDevices": null, "volumeMounts": [ { "mountPath": "/mnt/out", "mountPropagation": null, "name": "out", "readOnly": null, "subPath": null, "subPathExpr": null } ], "workingDir": null } }, { "name": "train", "activeDeadlineSeconds": 432000, "serviceAccountName": "argo-workflows-workflow", "inputs": { "parameters": [ { "name": "input-paths" } ] }, "outputs": { "parameters": [ { "name": "task-id", "valueFrom": { "path": "/mnt/out/task_id" } } ] }, "failFast": true, "metadata": { "annotations": { "metaflow/step_name": "train", "metaflow/attempt": "0" } }, "volumes": [ { "name": "out", "emptyDir": {} }, { "name": "dhsm", "emptyDir": { "sizeLimit": "14000Mi", "medium": "Memory" } }, { "name": "feature-store-dev-s3-claim", "persistentVolumeClaim": { "claimName": "feature-store-dev-s3-claim" } } ], "nodeSelector": {}, "tolerations": null, "container": { "args": null, "command": [ "bash", "-c", "${METAFLOW_INIT_SCRIPT:+eval \"${METAFLOW_INIT_SCRIPT}\"} && mkdir -p $PWD/.logs && export METAFLOW_TASK_ID=(t-$(echo train-{{workflow.creationTimestamp}}-{{inputs.parameters.input-paths}} | md5sum | cut -d ' ' -f 1 | tail -c 9)) && export PYTHONUNBUFFERED=x MF_PATHSPEC=workflow-test/argo-{{workflow.name}}/train/$METAFLOW_TASK_ID MF_DATASTORE=s3 MF_ATTEMPT=0 MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://artifact-bucket/workflow-test/data/00/00839a0b2be1025b649e875c0a24d33b495d7ae0 job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://artifact-bucket/workflow-test/data/00/00839a0b2be1025b649e875c0a24d33b495d7ae0 after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (python workflow.py --with 'environment:vars={}' --quiet --metadata=local --environment=local --datastore=s3 --datastore-root=s3://artifact-bucket --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=argo_workflows_internal:auto-emit-argo-events=1 step train --run-id argo-{{workflow.name}} --task-id $METAFLOW_TASK_ID --retry-count 0 --max-user-code-retries 0 --input-paths {{inputs.parameters.input-paths}}) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "env": [ { "name": "METAFLOW_CODE_URL", "value": "s3://artifact-bucket/workflow-test/data/00/00839a0b2be1025b649e875c0a24d33b495d7ae0", "valueFrom": null }, { "name": "METAFLOW_CODE_SHA", "value": "00839a0b2be1025b649e875c0a24d33b495d7ae0", "valueFrom": null }, { "name": "METAFLOW_CODE_DS", "value": "s3", "valueFrom": null }, { "name": "METAFLOW_SERVICE_HEADERS", "value": "{}", "valueFrom": null }, { "name": "METAFLOW_USER", "value": "argo-workflows", "valueFrom": null }, { "name": "METAFLOW_DATASTORE_SYSROOT_S3", "value": "s3://artifact-bucket", "valueFrom": null }, { "name": "METAFLOW_DATATOOLS_S3ROOT", "value": "s3://artifact-bucket/metaflow_artifacts", "valueFrom": null }, { "name": "METAFLOW_DEFAULT_DATASTORE", "value": "s3", "valueFrom": null }, { "name": "METAFLOW_DEFAULT_METADATA", "value": "local", "valueFrom": null }, { "name": "METAFLOW_CARD_S3ROOT", "value": "s3://artifact-bucket/mf.cards", "valueFrom": null }, { "name": "METAFLOW_KUBERNETES_WORKLOAD", "value": "1", "valueFrom": null }, { "name": "METAFLOW_KUBERNETES_FETCH_EC2_METADATA", "value": "False", "valueFrom": null }, { "name": "METAFLOW_RUNTIME_ENVIRONMENT", "value": "kubernetes", "valueFrom": null }, { "name": "METAFLOW_OWNER", "value": "amann", "valueFrom": null }, { "name": "METAFLOW_ARGO_EVENTS_EVENT_BUS", "value": "default", "valueFrom": null }, { "name": "METAFLOW_ARGO_EVENTS_WEBHOOK_AUTH", "value": "none", "valueFrom": null }, { "name": "METAFLOW_FLOW_NAME", "value": "workflow-test", "valueFrom": null }, { "name": "METAFLOW_STEP_NAME", "value": "train", "valueFrom": null }, { "name": "METAFLOW_RUN_ID", "value": "argo-{{workflow.name}}", "valueFrom": null }, { "name": "METAFLOW_RETRY_COUNT", "value": "0", "valueFrom": null }, { "name": "METAFLOW_PRODUCTION_TOKEN", "value": "workflow-test-0-ypbq", "valueFrom": null }, { "name": "ARGO_WORKFLOW_TEMPLATE", "value": "workflow-test", "valueFrom": null }, { "name": "ARGO_WORKFLOW_NAME", "value": "{{workflow.name}}", "valueFrom": null }, { "name": "ARGO_WORKFLOW_NAMESPACE", "value": "default", "valueFrom": null }, { "name": "METAFLOW_RUNTIME_NAME", "value": "argo-workflows", "valueFrom": null }, { "name": "USER", "value": "amann", "valueFrom": null }, { "name": "METAFLOW_VERSION", "value": "{\"platform\": \"Darwin\", \"username\": \"amann\", \"production_token\": \"workflow-test-0-ypbq\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.11.4 (main, Jul 25 2023, 17:36:13) [Clang 14.0.3 (clang-1403.0.22.14.1)]\", \"python_version_code\": \"3.11.4\", \"metaflow_version\": \"2.11.2\", \"script\": \"workflow.py\", \"flow_name\": \"workflow-test\"}", "valueFrom": null }, { "name": "METAFLOW_KUBERNETES_POD_NAMESPACE", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "metadata.namespace" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_POD_NAME", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "metadata.name" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_POD_ID", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "metadata.uid" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_SERVICE_ACCOUNT_NAME", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "spec.serviceAccountName" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_NODE_IP", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "status.hostIP" }, "resourceFieldRef": null, "secretKeyRef": null } } ], "envFrom": [], "image": "", "imagePullPolicy": null, "lifecycle": null, "livenessProbe": null, "name": "train", "ports": null, "readinessProbe": null, "resizePolicy": null, "resources": { "claims": null, "limits": {}, "requests": { "cpu": "1", "memory": "4096M", "ephemeral-storage": "10240M" } }, "restartPolicy": null, "securityContext": null, "startupProbe": null, "stdin": null, "stdinOnce": null, "terminationMessagePath": null, "terminationMessagePolicy": null, "tty": null, "volumeDevices": null, "volumeMounts": [ { "mountPath": "/mnt/out", "mountPropagation": null, "name": "out", "readOnly": null, "subPath": null, "subPathExpr": null }, { "mountPath": "/dev/shm", "mountPropagation": null, "name": "dhsm", "readOnly": null, "subPath": null, "subPathExpr": null }, { "mountPath": "pbf/dataset_v2", "mountPropagation": null, "name": "feature-store-dev-s3-claim", "readOnly": null, "subPath": null, "subPathExpr": null } ], "workingDir": null } }, { "name": "end", "activeDeadlineSeconds": 432000, "serviceAccountName": "argo-workflows-workflow", "inputs": { "parameters": [ { "name": "input-paths" } ] }, "outputs": { "parameters": [] }, "failFast": true, "metadata": { "annotations": { "metaflow/step_name": "end", "metaflow/attempt": "0" } }, "volumes": [ { "name": "out", "emptyDir": {} }, { "name": "dhsm", "emptyDir": { "medium": "Memory" } } ], "nodeSelector": {}, "tolerations": null, "container": { "args": null, "command": [ "bash", "-c", "${METAFLOW_INIT_SCRIPT:+eval \"${METAFLOW_INIT_SCRIPT}\"} && mkdir -p $PWD/.logs && export METAFLOW_TASK_ID=(t-$(echo end-{{workflow.creationTimestamp}}-{{inputs.parameters.input-paths}} | md5sum | cut -d ' ' -f 1 | tail -c 9)) && export PYTHONUNBUFFERED=x MF_PATHSPEC=workflow-test/argo-{{workflow.name}}/end/$METAFLOW_TASK_ID MF_DATASTORE=s3 MF_ATTEMPT=0 MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://artifact-bucket/workflow-test/data/00/00839a0b2be1025b649e875c0a24d33b495d7ae0 job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://artifact-bucket/workflow-test/data/00/00839a0b2be1025b649e875c0a24d33b495d7ae0 after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (python workflow.py --with kubernetes:cpu=1,memory=4096,disk=10240,image=,service_account=argo-workflows-workflow,namespace=default,gpu_vendor=nvidia,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --with 'environment:vars={}' --quiet --metadata=local --environment=local --datastore=s3 --datastore-root=s3://artifact-bucket --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=argo_workflows_internal:auto-emit-argo-events=1 step end --run-id argo-{{workflow.name}} --task-id $METAFLOW_TASK_ID --retry-count 0 --max-user-code-retries 0 --input-paths {{inputs.parameters.input-paths}}) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "env": [ { "name": "METAFLOW_CODE_URL", "value": "s3://artifact-bucket/workflow-test/data/00/00839a0b2be1025b649e875c0a24d33b495d7ae0", "valueFrom": null }, { "name": "METAFLOW_CODE_SHA", "value": "00839a0b2be1025b649e875c0a24d33b495d7ae0", "valueFrom": null }, { "name": "METAFLOW_CODE_DS", "value": "s3", "valueFrom": null }, { "name": "METAFLOW_SERVICE_HEADERS", "value": "{}", "valueFrom": null }, { "name": "METAFLOW_USER", "value": "argo-workflows", "valueFrom": null }, { "name": "METAFLOW_DATASTORE_SYSROOT_S3", "value": "s3://artifact-bucket", "valueFrom": null }, { "name": "METAFLOW_DATATOOLS_S3ROOT", "value": "s3://artifact-bucket/metaflow_artifacts", "valueFrom": null }, { "name": "METAFLOW_DEFAULT_DATASTORE", "value": "s3", "valueFrom": null }, { "name": "METAFLOW_DEFAULT_METADATA", "value": "local", "valueFrom": null }, { "name": "METAFLOW_CARD_S3ROOT", "value": "s3://artifact-bucket/mf.cards", "valueFrom": null }, { "name": "METAFLOW_KUBERNETES_WORKLOAD", "value": "1", "valueFrom": null }, { "name": "METAFLOW_KUBERNETES_FETCH_EC2_METADATA", "value": "False", "valueFrom": null }, { "name": "METAFLOW_RUNTIME_ENVIRONMENT", "value": "kubernetes", "valueFrom": null }, { "name": "METAFLOW_OWNER", "value": "amann", "valueFrom": null }, { "name": "METAFLOW_ARGO_EVENTS_EVENT_BUS", "value": "default", "valueFrom": null }, { "name": "METAFLOW_ARGO_EVENTS_WEBHOOK_AUTH", "value": "none", "valueFrom": null }, { "name": "METAFLOW_FLOW_NAME", "value": "workflow-test", "valueFrom": null }, { "name": "METAFLOW_STEP_NAME", "value": "end", "valueFrom": null }, { "name": "METAFLOW_RUN_ID", "value": "argo-{{workflow.name}}", "valueFrom": null }, { "name": "METAFLOW_RETRY_COUNT", "value": "0", "valueFrom": null }, { "name": "METAFLOW_PRODUCTION_TOKEN", "value": "workflow-test-0-ypbq", "valueFrom": null }, { "name": "ARGO_WORKFLOW_TEMPLATE", "value": "workflow-test", "valueFrom": null }, { "name": "ARGO_WORKFLOW_NAME", "value": "{{workflow.name}}", "valueFrom": null }, { "name": "ARGO_WORKFLOW_NAMESPACE", "value": "default", "valueFrom": null }, { "name": "METAFLOW_RUNTIME_NAME", "value": "argo-workflows", "valueFrom": null }, { "name": "USER", "value": "amann", "valueFrom": null }, { "name": "METAFLOW_VERSION", "value": "{\"platform\": \"Darwin\", \"username\": \"amann\", \"production_token\": \"workflow-test-0-ypbq\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.11.4 (main, Jul 25 2023, 17:36:13) [Clang 14.0.3 (clang-1403.0.22.14.1)]\", \"python_version_code\": \"3.11.4\", \"metaflow_version\": \"2.11.2\", \"script\": \"workflow.py\", \"flow_name\": \"workflow-test\"}", "valueFrom": null }, { "name": "METAFLOW_KUBERNETES_POD_NAMESPACE", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "metadata.namespace" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_POD_NAME", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "metadata.name" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_POD_ID", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "metadata.uid" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_SERVICE_ACCOUNT_NAME", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "spec.serviceAccountName" }, "resourceFieldRef": null, "secretKeyRef": null } }, { "name": "METAFLOW_KUBERNETES_NODE_IP", "value": null, "valueFrom": { "configMapKeyRef": null, "fieldRef": { "apiVersion": null, "fieldPath": "status.hostIP" }, "resourceFieldRef": null, "secretKeyRef": null } } ], "envFrom": [], "image": "", "imagePullPolicy": null, "lifecycle": null, "livenessProbe": null, "name": "end", "ports": null, "readinessProbe": null, "resizePolicy": null, "resources": { "claims": null, "limits": {}, "requests": { "cpu": "1", "memory": "4096M", "ephemeral-storage": "10240M" } }, "restartPolicy": null, "securityContext": null, "startupProbe": null, "stdin": null, "stdinOnce": null, "terminationMessagePath": null, "terminationMessagePolicy": null, "tty": null, "volumeDevices": null, "volumeMounts": [ { "mountPath": "/mnt/out", "mountPropagation": null, "name": "out", "readOnly": null, "subPath": null, "subPathExpr": null } ], "workingDir": null } } ] } }