{ "StartAt": "start", "States": { "start": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobDefinition": "arn:aws:batch:us-east-1:667858956048:job-definition/metaflow_e68ca648e3679aee14ad8511465bc62382ee94a40de9cf098f82234f:1", "JobName": "SFN-StandardModelBuilder--start--", "JobQueue": "arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow", "Parameters": { "metaflow.user": "SFN", "metaflow.owner": "unmeshph", "metaflow.flow_name": "StandardModelBuilder", "metaflow.step_name": "start", "metaflow.run_id.$": "$$.Execution.Name", "metaflow.version": "2.9.9+netflix-ext(0.2.3)", "step_name": "start", "metaflow.production_token": "mfprj-phsemr7h5nvipkn6-0-ykfb" }, "ContainerOverrides": { "Command": [ "bash", "-c", "true && mkdir -p $PWD/.logs && export PYTHONUNBUFFERED=x MF_PATHSPEC=StandardModelBuilder/sfn-$METAFLOW_RUN_ID/start/$AWS_BATCH_JOB_ID MF_DATASTORE=s3 MF_ATTEMPT=$((AWS_BATCH_JOB_ATTEMPT-1)) MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (if ! python standardmodelbuilder.py dump --max-value-size=0 sfn-${METAFLOW_RUN_ID}/_parameters/${AWS_BATCH_JOB_ID}-params >/dev/null 2>/dev/null; then python -m metaflow.plugins.aws.step_functions.set_batch_environment parameters mzsiutrmlx && . `pwd`/mzsiutrmlx && python standardmodelbuilder.py --with batch:cpu=1,gpu=0,memory=4096,image=667858956048.dkr.ecr.us-east-1.amazonaws.com/python:3.8.14,queue=arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow,iam_role=arn:aws:iam::667858956048:role/metaflow-BatchS3TaskRole-1U80KJ4G0N189,use_tmpfs=False,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --quiet --metadata=service --environment=local --datastore=s3 --datastore-root=s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=step_functions_internal init --run-id sfn-$METAFLOW_RUN_ID --task-id ${AWS_BATCH_JOB_ID}-params; fi && python standardmodelbuilder.py --with batch:cpu=1,gpu=0,memory=4096,image=667858956048.dkr.ecr.us-east-1.amazonaws.com/python:3.8.14,queue=arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow,iam_role=arn:aws:iam::667858956048:role/metaflow-BatchS3TaskRole-1U80KJ4G0N189,use_tmpfs=False,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --quiet --metadata=service --environment=local --datastore=s3 --datastore-root=s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=step_functions_internal step start --run-id sfn-$METAFLOW_RUN_ID --task-id ${AWS_BATCH_JOB_ID} --retry-count $((AWS_BATCH_JOB_ATTEMPT-1)) --max-user-code-retries 0 --input-paths sfn-${METAFLOW_RUN_ID}/_parameters/${AWS_BATCH_JOB_ID}-params) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "ResourceRequirements": [ { "Value": "1", "Type": "VCPU" }, { "Value": "4096", "Type": "MEMORY" } ], "Environment": [ { "Name": "AWS_DEFAULT_REGION", "Value": "us-east-1" }, { "Name": "METAFLOW_CODE_SHA", "Value": "27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_DS", "Value": "s3" }, { "Name": "METAFLOW_USER", "Value": "SFN" }, { "Name": "METAFLOW_SERVICE_URL", "Value": "http://metaflo-NLB-3NPX8MX9Y8S3-b12323d6edc8788c.elb.us-east-1.amazonaws.com/" }, { "Name": "METAFLOW_DATASTORE_SYSROOT_S3", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow" }, { "Name": "METAFLOW_DATATOOLS_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/data" }, { "Name": "METAFLOW_DEFAULT_DATASTORE", "Value": "s3" }, { "Name": "METAFLOW_DEFAULT_METADATA", "Value": "service" }, { "Name": "METAFLOW_CARD_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/mf.cards" }, { "Name": "METAFLOW_RUNTIME_ENVIRONMENT", "Value": "aws-batch" }, { "Name": "DISABLE_PARALLELISM", "Value": "0" }, { "Name": "AWS_ROLE_SESSION_TIMEOUT", "Value": "3600" }, { "Name": "METAFLOW_PARAMETERS", "Value.$": "$.Parameters" }, { "Name": "METAFLOW_DEFAULT_PARAMETERS", "Value": "{\"configs\": \"[]\", \"force_refresh\": false, \"incremental_training_configs\": \"[]\"}" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_FLOW_NAME", "Value": "StandardModelBuilder" }, { "Name": "METAFLOW_STEP_NAME", "Value": "start" }, { "Name": "METAFLOW_RUN_ID", "Value.$": "$$.Execution.Name" }, { "Name": "METAFLOW_PRODUCTION_TOKEN", "Value": "mfprj-phsemr7h5nvipkn6-0-ykfb" }, { "Name": "SFN_STATE_MACHINE", "Value": "model_builder.user.unmeshph.StandardModelBuilder" }, { "Name": "METAFLOW_OWNER", "Value": "unmeshph" }, { "Name": "METAFLOW_RUNTIME_NAME", "Value": "step-functions" }, { "Name": "USER", "Value": "unmeshph" }, { "Name": "METAFLOW_VERSION", "Value": "{\"platform\": \"Linux\", \"username\": \"unmeshph\", \"production_token\": \"mfprj-phsemr7h5nvipkn6-0-ykfb\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]\", \"python_version_code\": \"3.10.12\", \"metaflow_version\": \"2.9.9+netflix-ext(0.2.3)\", \"script\": \"standardmodelbuilder.py\", \"flow_name\": \"StandardModelBuilder\"}" } ] }, "RetryStrategy": { "Attempts": 1 }, "Timeout": { "AttemptDurationSeconds": 64800 } }, "OutputPath": "$.['JobId', 'Parameters', 'Index', 'SplitParentTaskId']", "Next": "7065a0e912a3b6abefe009b3161bf26c861d4d4d0d1118d1a6f004bd" }, "7065a0e912a3b6abefe009b3161bf26c861d4d4d0d1118d1a6f004bd": { "Type": "Parallel", "Next": "join", "Branches": [ { "StartAt": "load_train_data", "States": { "load_train_data": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobDefinition": "arn:aws:batch:us-east-1:667858956048:job-definition/metaflow_e68ca648e3679aee14ad8511465bc62382ee94a40de9cf098f82234f:1", "JobName": "SFN-StandardModelBuilder--load_train_data--", "JobQueue": "arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow", "Parameters": { "metaflow.user": "SFN", "metaflow.owner": "unmeshph", "metaflow.flow_name": "StandardModelBuilder", "metaflow.step_name": "load_train_data", "metaflow.run_id.$": "$$.Execution.Name", "metaflow.version": "2.9.9+netflix-ext(0.2.3)", "step_name": "load_train_data" }, "ContainerOverrides": { "Command": [ "bash", "-c", "true && mkdir -p $PWD/.logs && export PYTHONUNBUFFERED=x MF_PATHSPEC=StandardModelBuilder/sfn-$METAFLOW_RUN_ID/load_train_data/$AWS_BATCH_JOB_ID MF_DATASTORE=s3 MF_ATTEMPT=$((AWS_BATCH_JOB_ATTEMPT-1)) MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (python standardmodelbuilder.py --with batch:cpu=1,gpu=0,memory=4096,image=667858956048.dkr.ecr.us-east-1.amazonaws.com/python:3.8.14,queue=arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow,iam_role=arn:aws:iam::667858956048:role/metaflow-BatchS3TaskRole-1U80KJ4G0N189,use_tmpfs=False,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --quiet --metadata=service --environment=local --datastore=s3 --datastore-root=s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=step_functions_internal step load_train_data --run-id sfn-$METAFLOW_RUN_ID --task-id ${AWS_BATCH_JOB_ID} --retry-count $((AWS_BATCH_JOB_ATTEMPT-1)) --max-user-code-retries 0 --input-paths sfn-${METAFLOW_RUN_ID}/start/${METAFLOW_PARENT_TASK_ID}) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "ResourceRequirements": [ { "Value": "1", "Type": "VCPU" }, { "Value": "4096", "Type": "MEMORY" } ], "Environment": [ { "Name": "AWS_DEFAULT_REGION", "Value": "us-east-1" }, { "Name": "METAFLOW_CODE_SHA", "Value": "27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_DS", "Value": "s3" }, { "Name": "METAFLOW_USER", "Value": "SFN" }, { "Name": "METAFLOW_SERVICE_URL", "Value": "http://metaflo-NLB-3NPX8MX9Y8S3-b12323d6edc8788c.elb.us-east-1.amazonaws.com/" }, { "Name": "METAFLOW_DATASTORE_SYSROOT_S3", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow" }, { "Name": "METAFLOW_DATATOOLS_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/data" }, { "Name": "METAFLOW_DEFAULT_DATASTORE", "Value": "s3" }, { "Name": "METAFLOW_DEFAULT_METADATA", "Value": "service" }, { "Name": "METAFLOW_CARD_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/mf.cards" }, { "Name": "METAFLOW_RUNTIME_ENVIRONMENT", "Value": "aws-batch" }, { "Name": "DISABLE_PARALLELISM", "Value": "0" }, { "Name": "AWS_ROLE_SESSION_TIMEOUT", "Value": "3600" }, { "Name": "METAFLOW_PARENT_TASK_ID", "Value.$": "$.JobId" }, { "Name": "METAFLOW_INPUT_PATHS", "Value": "sfn-${METAFLOW_RUN_ID}/start/${METAFLOW_PARENT_TASK_ID}" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_FLOW_NAME", "Value": "StandardModelBuilder" }, { "Name": "METAFLOW_STEP_NAME", "Value": "load_train_data" }, { "Name": "METAFLOW_RUN_ID", "Value.$": "$$.Execution.Name" }, { "Name": "METAFLOW_PRODUCTION_TOKEN", "Value": "mfprj-phsemr7h5nvipkn6-0-ykfb" }, { "Name": "SFN_STATE_MACHINE", "Value": "model_builder.user.unmeshph.StandardModelBuilder" }, { "Name": "METAFLOW_OWNER", "Value": "unmeshph" }, { "Name": "METAFLOW_RUNTIME_NAME", "Value": "step-functions" }, { "Name": "USER", "Value": "unmeshph" }, { "Name": "METAFLOW_VERSION", "Value": "{\"platform\": \"Linux\", \"username\": \"unmeshph\", \"production_token\": \"mfprj-phsemr7h5nvipkn6-0-ykfb\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]\", \"python_version_code\": \"3.10.12\", \"metaflow_version\": \"2.9.9+netflix-ext(0.2.3)\", \"script\": \"standardmodelbuilder.py\", \"flow_name\": \"StandardModelBuilder\"}" } ] }, "RetryStrategy": { "Attempts": 1 }, "Timeout": { "AttemptDurationSeconds": 64800 } }, "OutputPath": "$.['JobId', 'Parameters', 'Index', 'SplitParentTaskId']", "Next": "train" }, "train": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobDefinition": "arn:aws:batch:us-east-1:667858956048:job-definition/metaflow_e68ca648e3679aee14ad8511465bc62382ee94a40de9cf098f82234f:1", "JobName": "SFN-StandardModelBuilder--train--", "JobQueue": "arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow", "Parameters": { "metaflow.user": "SFN", "metaflow.owner": "unmeshph", "metaflow.flow_name": "StandardModelBuilder", "metaflow.step_name": "train", "metaflow.run_id.$": "$$.Execution.Name", "metaflow.version": "2.9.9+netflix-ext(0.2.3)", "step_name": "train" }, "ContainerOverrides": { "Command": [ "bash", "-c", "true && mkdir -p $PWD/.logs && export PYTHONUNBUFFERED=x MF_PATHSPEC=StandardModelBuilder/sfn-$METAFLOW_RUN_ID/train/$AWS_BATCH_JOB_ID MF_DATASTORE=s3 MF_ATTEMPT=$((AWS_BATCH_JOB_ATTEMPT-1)) MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (python standardmodelbuilder.py --with batch:cpu=1,gpu=0,memory=4096,image=667858956048.dkr.ecr.us-east-1.amazonaws.com/python:3.8.14,queue=arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow,iam_role=arn:aws:iam::667858956048:role/metaflow-BatchS3TaskRole-1U80KJ4G0N189,use_tmpfs=False,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --quiet --metadata=service --environment=local --datastore=s3 --datastore-root=s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=step_functions_internal step train --run-id sfn-$METAFLOW_RUN_ID --task-id ${AWS_BATCH_JOB_ID} --retry-count $((AWS_BATCH_JOB_ATTEMPT-1)) --max-user-code-retries 0 --input-paths sfn-${METAFLOW_RUN_ID}/load_train_data/${METAFLOW_PARENT_TASK_ID}) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "ResourceRequirements": [ { "Value": "1", "Type": "VCPU" }, { "Value": "4096", "Type": "MEMORY" } ], "Environment": [ { "Name": "AWS_DEFAULT_REGION", "Value": "us-east-1" }, { "Name": "METAFLOW_CODE_SHA", "Value": "27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_DS", "Value": "s3" }, { "Name": "METAFLOW_USER", "Value": "SFN" }, { "Name": "METAFLOW_SERVICE_URL", "Value": "http://metaflo-NLB-3NPX8MX9Y8S3-b12323d6edc8788c.elb.us-east-1.amazonaws.com/" }, { "Name": "METAFLOW_DATASTORE_SYSROOT_S3", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow" }, { "Name": "METAFLOW_DATATOOLS_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/data" }, { "Name": "METAFLOW_DEFAULT_DATASTORE", "Value": "s3" }, { "Name": "METAFLOW_DEFAULT_METADATA", "Value": "service" }, { "Name": "METAFLOW_CARD_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/mf.cards" }, { "Name": "METAFLOW_RUNTIME_ENVIRONMENT", "Value": "aws-batch" }, { "Name": "NVIDIA_DRIVER_CAPABILITIES", "Value": "compute,utility" }, { "Name": "CUDA_VISIBLE_DEVICES", "Value": "0" }, { "Name": "DISABLE_PARALLELISM", "Value": "0" }, { "Name": "METAFLOW_PARENT_TASK_ID", "Value.$": "$.JobId" }, { "Name": "METAFLOW_INPUT_PATHS", "Value": "sfn-${METAFLOW_RUN_ID}/load_train_data/${METAFLOW_PARENT_TASK_ID}" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_FLOW_NAME", "Value": "StandardModelBuilder" }, { "Name": "METAFLOW_STEP_NAME", "Value": "train" }, { "Name": "METAFLOW_RUN_ID", "Value.$": "$$.Execution.Name" }, { "Name": "METAFLOW_PRODUCTION_TOKEN", "Value": "mfprj-phsemr7h5nvipkn6-0-ykfb" }, { "Name": "SFN_STATE_MACHINE", "Value": "model_builder.user.unmeshph.StandardModelBuilder" }, { "Name": "METAFLOW_OWNER", "Value": "unmeshph" }, { "Name": "METAFLOW_RUNTIME_NAME", "Value": "step-functions" }, { "Name": "USER", "Value": "unmeshph" }, { "Name": "METAFLOW_VERSION", "Value": "{\"platform\": \"Linux\", \"username\": \"unmeshph\", \"production_token\": \"mfprj-phsemr7h5nvipkn6-0-ykfb\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]\", \"python_version_code\": \"3.10.12\", \"metaflow_version\": \"2.9.9+netflix-ext(0.2.3)\", \"script\": \"standardmodelbuilder.py\", \"flow_name\": \"StandardModelBuilder\"}" } ] }, "RetryStrategy": { "Attempts": 1 }, "Timeout": { "AttemptDurationSeconds": 64800 } }, "OutputPath": "$.['JobId', 'Parameters', 'Index', 'SplitParentTaskId']", "Next": "deploy" }, "deploy": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobDefinition": "arn:aws:batch:us-east-1:667858956048:job-definition/metaflow_e68ca648e3679aee14ad8511465bc62382ee94a40de9cf098f82234f:1", "JobName": "SFN-StandardModelBuilder--deploy--", "JobQueue": "arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow", "Parameters": { "metaflow.user": "SFN", "metaflow.owner": "unmeshph", "metaflow.flow_name": "StandardModelBuilder", "metaflow.step_name": "deploy", "metaflow.run_id.$": "$$.Execution.Name", "metaflow.version": "2.9.9+netflix-ext(0.2.3)", "step_name": "deploy" }, "ContainerOverrides": { "Command": [ "bash", "-c", "true && mkdir -p $PWD/.logs && export PYTHONUNBUFFERED=x MF_PATHSPEC=StandardModelBuilder/sfn-$METAFLOW_RUN_ID/deploy/$AWS_BATCH_JOB_ID MF_DATASTORE=s3 MF_ATTEMPT=$((AWS_BATCH_JOB_ATTEMPT-1)) MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (python standardmodelbuilder.py --with batch:cpu=1,gpu=0,memory=4096,image=667858956048.dkr.ecr.us-east-1.amazonaws.com/python:3.8.14,queue=arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow,iam_role=arn:aws:iam::667858956048:role/metaflow-BatchS3TaskRole-1U80KJ4G0N189,use_tmpfs=False,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --quiet --metadata=service --environment=local --datastore=s3 --datastore-root=s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=step_functions_internal step deploy --run-id sfn-$METAFLOW_RUN_ID --task-id ${AWS_BATCH_JOB_ID} --retry-count $((AWS_BATCH_JOB_ATTEMPT-1)) --max-user-code-retries 4 --input-paths sfn-${METAFLOW_RUN_ID}/train/${METAFLOW_PARENT_TASK_ID}) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "ResourceRequirements": [ { "Value": "1", "Type": "VCPU" }, { "Value": "4096", "Type": "MEMORY" } ], "Environment": [ { "Name": "AWS_DEFAULT_REGION", "Value": "us-east-1" }, { "Name": "METAFLOW_CODE_SHA", "Value": "27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_DS", "Value": "s3" }, { "Name": "METAFLOW_USER", "Value": "SFN" }, { "Name": "METAFLOW_SERVICE_URL", "Value": "http://metaflo-NLB-3NPX8MX9Y8S3-b12323d6edc8788c.elb.us-east-1.amazonaws.com/" }, { "Name": "METAFLOW_DATASTORE_SYSROOT_S3", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow" }, { "Name": "METAFLOW_DATATOOLS_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/data" }, { "Name": "METAFLOW_DEFAULT_DATASTORE", "Value": "s3" }, { "Name": "METAFLOW_DEFAULT_METADATA", "Value": "service" }, { "Name": "METAFLOW_CARD_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/mf.cards" }, { "Name": "METAFLOW_RUNTIME_ENVIRONMENT", "Value": "aws-batch" }, { "Name": "METAFLOW_PARENT_TASK_ID", "Value.$": "$.JobId" }, { "Name": "METAFLOW_INPUT_PATHS", "Value": "sfn-${METAFLOW_RUN_ID}/train/${METAFLOW_PARENT_TASK_ID}" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_FLOW_NAME", "Value": "StandardModelBuilder" }, { "Name": "METAFLOW_STEP_NAME", "Value": "deploy" }, { "Name": "METAFLOW_RUN_ID", "Value.$": "$$.Execution.Name" }, { "Name": "METAFLOW_PRODUCTION_TOKEN", "Value": "mfprj-phsemr7h5nvipkn6-0-ykfb" }, { "Name": "SFN_STATE_MACHINE", "Value": "model_builder.user.unmeshph.StandardModelBuilder" }, { "Name": "METAFLOW_OWNER", "Value": "unmeshph" }, { "Name": "METAFLOW_RUNTIME_NAME", "Value": "step-functions" }, { "Name": "USER", "Value": "unmeshph" }, { "Name": "METAFLOW_VERSION", "Value": "{\"platform\": \"Linux\", \"username\": \"unmeshph\", \"production_token\": \"mfprj-phsemr7h5nvipkn6-0-ykfb\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]\", \"python_version_code\": \"3.10.12\", \"metaflow_version\": \"2.9.9+netflix-ext(0.2.3)\", \"script\": \"standardmodelbuilder.py\", \"flow_name\": \"StandardModelBuilder\"}" } ] }, "RetryStrategy": { "Attempts": 5 }, "Timeout": { "AttemptDurationSeconds": 3600 } }, "OutputPath": "$.['JobId', 'Parameters', 'Index', 'SplitParentTaskId']", "End": true } } }, { "StartAt": "load_test_data", "States": { "load_test_data": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobDefinition": "arn:aws:batch:us-east-1:667858956048:job-definition/metaflow_e68ca648e3679aee14ad8511465bc62382ee94a40de9cf098f82234f:1", "JobName": "SFN-StandardModelBuilder--load_test_data--", "JobQueue": "arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow", "Parameters": { "metaflow.user": "SFN", "metaflow.owner": "unmeshph", "metaflow.flow_name": "StandardModelBuilder", "metaflow.step_name": "load_test_data", "metaflow.run_id.$": "$$.Execution.Name", "metaflow.version": "2.9.9+netflix-ext(0.2.3)", "step_name": "load_test_data" }, "ContainerOverrides": { "Command": [ "bash", "-c", "true && mkdir -p $PWD/.logs && export PYTHONUNBUFFERED=x MF_PATHSPEC=StandardModelBuilder/sfn-$METAFLOW_RUN_ID/load_test_data/$AWS_BATCH_JOB_ID MF_DATASTORE=s3 MF_ATTEMPT=$((AWS_BATCH_JOB_ATTEMPT-1)) MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (python standardmodelbuilder.py --with batch:cpu=1,gpu=0,memory=4096,image=667858956048.dkr.ecr.us-east-1.amazonaws.com/python:3.8.14,queue=arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow,iam_role=arn:aws:iam::667858956048:role/metaflow-BatchS3TaskRole-1U80KJ4G0N189,use_tmpfs=False,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --quiet --metadata=service --environment=local --datastore=s3 --datastore-root=s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=step_functions_internal step load_test_data --run-id sfn-$METAFLOW_RUN_ID --task-id ${AWS_BATCH_JOB_ID} --retry-count $((AWS_BATCH_JOB_ATTEMPT-1)) --max-user-code-retries 0 --input-paths sfn-${METAFLOW_RUN_ID}/start/${METAFLOW_PARENT_TASK_ID}) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "ResourceRequirements": [ { "Value": "1", "Type": "VCPU" }, { "Value": "4096", "Type": "MEMORY" } ], "Environment": [ { "Name": "AWS_DEFAULT_REGION", "Value": "us-east-1" }, { "Name": "METAFLOW_CODE_SHA", "Value": "27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_DS", "Value": "s3" }, { "Name": "METAFLOW_USER", "Value": "SFN" }, { "Name": "METAFLOW_SERVICE_URL", "Value": "http://metaflo-NLB-3NPX8MX9Y8S3-b12323d6edc8788c.elb.us-east-1.amazonaws.com/" }, { "Name": "METAFLOW_DATASTORE_SYSROOT_S3", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow" }, { "Name": "METAFLOW_DATATOOLS_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/data" }, { "Name": "METAFLOW_DEFAULT_DATASTORE", "Value": "s3" }, { "Name": "METAFLOW_DEFAULT_METADATA", "Value": "service" }, { "Name": "METAFLOW_CARD_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/mf.cards" }, { "Name": "METAFLOW_RUNTIME_ENVIRONMENT", "Value": "aws-batch" }, { "Name": "DISABLE_PARALLELISM", "Value": "0" }, { "Name": "AWS_ROLE_SESSION_TIMEOUT", "Value": "3600" }, { "Name": "METAFLOW_PARENT_TASK_ID", "Value.$": "$.JobId" }, { "Name": "METAFLOW_INPUT_PATHS", "Value": "sfn-${METAFLOW_RUN_ID}/start/${METAFLOW_PARENT_TASK_ID}" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_FLOW_NAME", "Value": "StandardModelBuilder" }, { "Name": "METAFLOW_STEP_NAME", "Value": "load_test_data" }, { "Name": "METAFLOW_RUN_ID", "Value.$": "$$.Execution.Name" }, { "Name": "METAFLOW_PRODUCTION_TOKEN", "Value": "mfprj-phsemr7h5nvipkn6-0-ykfb" }, { "Name": "SFN_STATE_MACHINE", "Value": "model_builder.user.unmeshph.StandardModelBuilder" }, { "Name": "METAFLOW_OWNER", "Value": "unmeshph" }, { "Name": "METAFLOW_RUNTIME_NAME", "Value": "step-functions" }, { "Name": "USER", "Value": "unmeshph" }, { "Name": "METAFLOW_VERSION", "Value": "{\"platform\": \"Linux\", \"username\": \"unmeshph\", \"production_token\": \"mfprj-phsemr7h5nvipkn6-0-ykfb\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]\", \"python_version_code\": \"3.10.12\", \"metaflow_version\": \"2.9.9+netflix-ext(0.2.3)\", \"script\": \"standardmodelbuilder.py\", \"flow_name\": \"StandardModelBuilder\"}" } ] }, "RetryStrategy": { "Attempts": 1 }, "Timeout": { "AttemptDurationSeconds": 64800 } }, "OutputPath": "$.['JobId', 'Parameters', 'Index', 'SplitParentTaskId']", "Next": "test" }, "test": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobDefinition": "arn:aws:batch:us-east-1:667858956048:job-definition/metaflow_e68ca648e3679aee14ad8511465bc62382ee94a40de9cf098f82234f:1", "JobName": "SFN-StandardModelBuilder--test--", "JobQueue": "arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow", "Parameters": { "metaflow.user": "SFN", "metaflow.owner": "unmeshph", "metaflow.flow_name": "StandardModelBuilder", "metaflow.step_name": "test", "metaflow.run_id.$": "$$.Execution.Name", "metaflow.version": "2.9.9+netflix-ext(0.2.3)", "step_name": "test" }, "ContainerOverrides": { "Command": [ "bash", "-c", "true && mkdir -p $PWD/.logs && export PYTHONUNBUFFERED=x MF_PATHSPEC=StandardModelBuilder/sfn-$METAFLOW_RUN_ID/test/$AWS_BATCH_JOB_ID MF_DATASTORE=s3 MF_ATTEMPT=$((AWS_BATCH_JOB_ATTEMPT-1)) MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (python standardmodelbuilder.py --with batch:cpu=1,gpu=0,memory=4096,image=667858956048.dkr.ecr.us-east-1.amazonaws.com/python:3.8.14,queue=arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow,iam_role=arn:aws:iam::667858956048:role/metaflow-BatchS3TaskRole-1U80KJ4G0N189,use_tmpfs=False,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --quiet --metadata=service --environment=local --datastore=s3 --datastore-root=s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=step_functions_internal step test --run-id sfn-$METAFLOW_RUN_ID --task-id ${AWS_BATCH_JOB_ID} --retry-count $((AWS_BATCH_JOB_ATTEMPT-1)) --max-user-code-retries 0 --input-paths sfn-${METAFLOW_RUN_ID}/load_test_data/${METAFLOW_PARENT_TASK_ID}) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "ResourceRequirements": [ { "Value": "1", "Type": "VCPU" }, { "Value": "4096", "Type": "MEMORY" } ], "Environment": [ { "Name": "AWS_DEFAULT_REGION", "Value": "us-east-1" }, { "Name": "METAFLOW_CODE_SHA", "Value": "27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_DS", "Value": "s3" }, { "Name": "METAFLOW_USER", "Value": "SFN" }, { "Name": "METAFLOW_SERVICE_URL", "Value": "http://metaflo-NLB-3NPX8MX9Y8S3-b12323d6edc8788c.elb.us-east-1.amazonaws.com/" }, { "Name": "METAFLOW_DATASTORE_SYSROOT_S3", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow" }, { "Name": "METAFLOW_DATATOOLS_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/data" }, { "Name": "METAFLOW_DEFAULT_DATASTORE", "Value": "s3" }, { "Name": "METAFLOW_DEFAULT_METADATA", "Value": "service" }, { "Name": "METAFLOW_CARD_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/mf.cards" }, { "Name": "METAFLOW_RUNTIME_ENVIRONMENT", "Value": "aws-batch" }, { "Name": "NVIDIA_DRIVER_CAPABILITIES", "Value": "compute,utility" }, { "Name": "CUDA_VISIBLE_DEVICES", "Value": "0" }, { "Name": "DISABLE_PARALLELISM", "Value": "0" }, { "Name": "METAFLOW_PARENT_TASK_ID", "Value.$": "$.JobId" }, { "Name": "METAFLOW_INPUT_PATHS", "Value": "sfn-${METAFLOW_RUN_ID}/load_test_data/${METAFLOW_PARENT_TASK_ID}" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_FLOW_NAME", "Value": "StandardModelBuilder" }, { "Name": "METAFLOW_STEP_NAME", "Value": "test" }, { "Name": "METAFLOW_RUN_ID", "Value.$": "$$.Execution.Name" }, { "Name": "METAFLOW_PRODUCTION_TOKEN", "Value": "mfprj-phsemr7h5nvipkn6-0-ykfb" }, { "Name": "SFN_STATE_MACHINE", "Value": "model_builder.user.unmeshph.StandardModelBuilder" }, { "Name": "METAFLOW_OWNER", "Value": "unmeshph" }, { "Name": "METAFLOW_RUNTIME_NAME", "Value": "step-functions" }, { "Name": "USER", "Value": "unmeshph" }, { "Name": "METAFLOW_VERSION", "Value": "{\"platform\": \"Linux\", \"username\": \"unmeshph\", \"production_token\": \"mfprj-phsemr7h5nvipkn6-0-ykfb\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]\", \"python_version_code\": \"3.10.12\", \"metaflow_version\": \"2.9.9+netflix-ext(0.2.3)\", \"script\": \"standardmodelbuilder.py\", \"flow_name\": \"StandardModelBuilder\"}" } ] }, "RetryStrategy": { "Attempts": 1 }, "Timeout": { "AttemptDurationSeconds": 64800 } }, "OutputPath": "$.['JobId', 'Parameters', 'Index', 'SplitParentTaskId']", "End": true } } } ] }, "join": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobDefinition": "arn:aws:batch:us-east-1:667858956048:job-definition/metaflow_e68ca648e3679aee14ad8511465bc62382ee94a40de9cf098f82234f:1", "JobName": "SFN-StandardModelBuilder--join--", "JobQueue": "arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow", "Parameters": { "metaflow.user": "SFN", "metaflow.owner": "unmeshph", "metaflow.flow_name": "StandardModelBuilder", "metaflow.step_name": "join", "metaflow.run_id.$": "$$.Execution.Name", "metaflow.version": "2.9.9+netflix-ext(0.2.3)", "step_name": "join" }, "ContainerOverrides": { "Command": [ "bash", "-c", "true && mkdir -p $PWD/.logs && export PYTHONUNBUFFERED=x MF_PATHSPEC=StandardModelBuilder/sfn-$METAFLOW_RUN_ID/join/$AWS_BATCH_JOB_ID MF_DATASTORE=s3 MF_ATTEMPT=$((AWS_BATCH_JOB_ATTEMPT-1)) MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (python standardmodelbuilder.py --with batch:cpu=1,gpu=0,memory=4096,image=667858956048.dkr.ecr.us-east-1.amazonaws.com/python:3.8.14,queue=arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow,iam_role=arn:aws:iam::667858956048:role/metaflow-BatchS3TaskRole-1U80KJ4G0N189,use_tmpfs=False,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --quiet --metadata=service --environment=local --datastore=s3 --datastore-root=s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=step_functions_internal step join --run-id sfn-$METAFLOW_RUN_ID --task-id ${AWS_BATCH_JOB_ID} --retry-count $((AWS_BATCH_JOB_ATTEMPT-1)) --max-user-code-retries 0 --input-paths sfn-${METAFLOW_RUN_ID}:/${METAFLOW_PARENT_0_STEP}/${METAFLOW_PARENT_0_TASK_ID},/${METAFLOW_PARENT_1_STEP}/${METAFLOW_PARENT_1_TASK_ID}) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "ResourceRequirements": [ { "Value": "1", "Type": "VCPU" }, { "Value": "4096", "Type": "MEMORY" } ], "Environment": [ { "Name": "AWS_DEFAULT_REGION", "Value": "us-east-1" }, { "Name": "METAFLOW_CODE_SHA", "Value": "27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_DS", "Value": "s3" }, { "Name": "METAFLOW_USER", "Value": "SFN" }, { "Name": "METAFLOW_SERVICE_URL", "Value": "http://metaflo-NLB-3NPX8MX9Y8S3-b12323d6edc8788c.elb.us-east-1.amazonaws.com/" }, { "Name": "METAFLOW_DATASTORE_SYSROOT_S3", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow" }, { "Name": "METAFLOW_DATATOOLS_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/data" }, { "Name": "METAFLOW_DEFAULT_DATASTORE", "Value": "s3" }, { "Name": "METAFLOW_DEFAULT_METADATA", "Value": "service" }, { "Name": "METAFLOW_CARD_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/mf.cards" }, { "Name": "METAFLOW_RUNTIME_ENVIRONMENT", "Value": "aws-batch" }, { "Name": "METAFLOW_PARENT_0_TASK_ID", "Value.$": "$.[0].JobId" }, { "Name": "METAFLOW_PARENT_0_STEP", "Value.$": "$.[0].Parameters.step_name" }, { "Name": "METAFLOW_PARENT_1_TASK_ID", "Value.$": "$.[1].JobId" }, { "Name": "METAFLOW_PARENT_1_STEP", "Value.$": "$.[1].Parameters.step_name" }, { "Name": "METAFLOW_INPUT_PATHS", "Value": "sfn-${METAFLOW_RUN_ID}:/${METAFLOW_PARENT_0_STEP}/${METAFLOW_PARENT_0_TASK_ID},/${METAFLOW_PARENT_1_STEP}/${METAFLOW_PARENT_1_TASK_ID}" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_FLOW_NAME", "Value": "StandardModelBuilder" }, { "Name": "METAFLOW_STEP_NAME", "Value": "join" }, { "Name": "METAFLOW_RUN_ID", "Value.$": "$$.Execution.Name" }, { "Name": "METAFLOW_PRODUCTION_TOKEN", "Value": "mfprj-phsemr7h5nvipkn6-0-ykfb" }, { "Name": "SFN_STATE_MACHINE", "Value": "model_builder.user.unmeshph.StandardModelBuilder" }, { "Name": "METAFLOW_OWNER", "Value": "unmeshph" }, { "Name": "METAFLOW_RUNTIME_NAME", "Value": "step-functions" }, { "Name": "USER", "Value": "unmeshph" }, { "Name": "METAFLOW_VERSION", "Value": "{\"platform\": \"Linux\", \"username\": \"unmeshph\", \"production_token\": \"mfprj-phsemr7h5nvipkn6-0-ykfb\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]\", \"python_version_code\": \"3.10.12\", \"metaflow_version\": \"2.9.9+netflix-ext(0.2.3)\", \"script\": \"standardmodelbuilder.py\", \"flow_name\": \"StandardModelBuilder\"}" } ] }, "RetryStrategy": { "Attempts": 1 }, "Timeout": { "AttemptDurationSeconds": 3600 } }, "OutputPath": "$.['JobId', 'Parameters', 'Index', 'SplitParentTaskId']", "Next": "end" }, "end": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobDefinition": "arn:aws:batch:us-east-1:667858956048:job-definition/metaflow_e68ca648e3679aee14ad8511465bc62382ee94a40de9cf098f82234f:1", "JobName": "SFN-StandardModelBuilder--end--", "JobQueue": "arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow", "Parameters": { "metaflow.user": "SFN", "metaflow.owner": "unmeshph", "metaflow.flow_name": "StandardModelBuilder", "metaflow.step_name": "end", "metaflow.run_id.$": "$$.Execution.Name", "metaflow.version": "2.9.9+netflix-ext(0.2.3)", "step_name": "end" }, "ContainerOverrides": { "Command": [ "bash", "-c", "true && mkdir -p $PWD/.logs && export PYTHONUNBUFFERED=x MF_PATHSPEC=StandardModelBuilder/sfn-$METAFLOW_RUN_ID/end/$AWS_BATCH_JOB_ID MF_DATASTORE=s3 MF_ATTEMPT=$((AWS_BATCH_JOB_ATTEMPT-1)) MFLOG_STDOUT=$PWD/.logs/mflog_stdout MFLOG_STDERR=$PWD/.logs/mflog_stderr && mflog(){ T=$(date -u -Ins|tr , .); echo \"[MFLOG|0|${T:0:26}Z|task|$T]$1\" >> $MFLOG_STDOUT; echo $1; } && mflog 'Setting up task environment.' && python -m pip install requests -qqq && python -m pip install awscli boto3 -qqq && mkdir metaflow && cd metaflow && mkdir .metaflow && i=0; while [ $i -le 5 ]; do mflog 'Downloading code package...'; python -m awscli ${METAFLOW_S3_ENDPOINT_URL:+--endpoint-url=\"${METAFLOW_S3_ENDPOINT_URL}\"} s3 cp s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb job.tar >/dev/null && mflog 'Code package downloaded.' && break; sleep 10; i=$((i+1)); done && if [ $i -gt 5 ]; then mflog 'Failed to download code package from s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb after 6 tries. Exiting...' && exit 1; fi && TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar && mflog 'Task is starting.' && (python standardmodelbuilder.py --with batch:cpu=1,gpu=0,memory=4096,image=667858956048.dkr.ecr.us-east-1.amazonaws.com/python:3.8.14,queue=arn:aws:batch:us-east-1:667858956048:job-queue/job-queue-metaflow,iam_role=arn:aws:iam::667858956048:role/metaflow-BatchS3TaskRole-1U80KJ4G0N189,use_tmpfs=False,tmpfs_tempdir=True,tmpfs_path=/metaflow_temp --quiet --metadata=service --environment=local --datastore=s3 --datastore-root=s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow --event-logger=nullSidecarLogger --monitor=nullSidecarMonitor --no-pylint --with=step_functions_internal step end --run-id sfn-$METAFLOW_RUN_ID --task-id ${AWS_BATCH_JOB_ID} --retry-count $((AWS_BATCH_JOB_ATTEMPT-1)) --max-user-code-retries 4 --input-paths sfn-${METAFLOW_RUN_ID}/join/${METAFLOW_PARENT_TASK_ID}) 1>> >(python -m metaflow.mflog.tee task $MFLOG_STDOUT) 2>> >(python -m metaflow.mflog.tee task $MFLOG_STDERR >&2); c=$?; python -m metaflow.mflog.save_logs; exit $c" ], "ResourceRequirements": [ { "Value": "1", "Type": "VCPU" }, { "Value": "4096", "Type": "MEMORY" } ], "Environment": [ { "Name": "AWS_DEFAULT_REGION", "Value": "us-east-1" }, { "Name": "METAFLOW_CODE_SHA", "Value": "27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_CODE_DS", "Value": "s3" }, { "Name": "METAFLOW_USER", "Value": "SFN" }, { "Name": "METAFLOW_SERVICE_URL", "Value": "http://metaflo-NLB-3NPX8MX9Y8S3-b12323d6edc8788c.elb.us-east-1.amazonaws.com/" }, { "Name": "METAFLOW_DATASTORE_SYSROOT_S3", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow" }, { "Name": "METAFLOW_DATATOOLS_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/data" }, { "Name": "METAFLOW_DEFAULT_DATASTORE", "Value": "s3" }, { "Name": "METAFLOW_DEFAULT_METADATA", "Value": "service" }, { "Name": "METAFLOW_CARD_S3ROOT", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/mf.cards" }, { "Name": "METAFLOW_RUNTIME_ENVIRONMENT", "Value": "aws-batch" }, { "Name": "METAFLOW_PARENT_TASK_ID", "Value.$": "$.JobId" }, { "Name": "METAFLOW_INPUT_PATHS", "Value": "sfn-${METAFLOW_RUN_ID}/join/${METAFLOW_PARENT_TASK_ID}" }, { "Name": "METAFLOW_CODE_URL", "Value": "s3://metaflow-metaflows3bucket-1ildf2a016xdx/metaflow/StandardModelBuilder/data/27/27f4243ab7ba28cfdfa323dfe75313aea05dbcdb" }, { "Name": "METAFLOW_FLOW_NAME", "Value": "StandardModelBuilder" }, { "Name": "METAFLOW_STEP_NAME", "Value": "end" }, { "Name": "METAFLOW_RUN_ID", "Value.$": "$$.Execution.Name" }, { "Name": "METAFLOW_PRODUCTION_TOKEN", "Value": "mfprj-phsemr7h5nvipkn6-0-ykfb" }, { "Name": "SFN_STATE_MACHINE", "Value": "model_builder.user.unmeshph.StandardModelBuilder" }, { "Name": "METAFLOW_OWNER", "Value": "unmeshph" }, { "Name": "METAFLOW_RUNTIME_NAME", "Value": "step-functions" }, { "Name": "USER", "Value": "unmeshph" }, { "Name": "METAFLOW_VERSION", "Value": "{\"platform\": \"Linux\", \"username\": \"unmeshph\", \"production_token\": \"mfprj-phsemr7h5nvipkn6-0-ykfb\", \"runtime\": \"dev\", \"app\": null, \"environment_type\": \"local\", \"use_r\": false, \"python_version\": \"3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:40:32) [GCC 12.3.0]\", \"python_version_code\": \"3.10.12\", \"metaflow_version\": \"2.9.9+netflix-ext(0.2.3)\", \"script\": \"standardmodelbuilder.py\", \"flow_name\": \"StandardModelBuilder\"}" } ] }, "RetryStrategy": { "Attempts": 5 }, "Timeout": { "AttemptDurationSeconds": 3600 } }, "OutputPath": "$.['JobId', 'Parameters', 'Index', 'SplitParentTaskId']", "End": true } } }