I ran a WDL. The WDL has a scatter in it.
One of the scatters had the error message : Warning: Creating VM and disk(s) would exceed \"CPUS\" in region us-central1, will try again
I found the message in the operationsID.
I saw that the message was given twice yesterday.
After seeing this, nothing happened with the task, it seemingly "stalled". The VM never booted up and the task never ran.
To get the WDL to run to completion, I aborted the submission, then resubmitted it.
Happily, in the re-submission all the other tasks in the WDL got "call-cache hits". In the re-submission, the task actually started running on the VM. In recent events it says "pulling image", so it got started.
If the operation would have continued to give the "Warning: Creating VM and disk(s) would exceed \"CPUS\" in region us-central1, will try again" message and the subsequently tried again (3, 4, more times, ), I would not have made this forum post, but though it said "will try again", it seemingly did not after trying twice. I saw "trying twice" because the message is given twice in the operation description. Given that it said it would try more but seemingly did not, I suspect that this is a google issue.
Is this a google issue? is it a known google issue?
{
"done": true,
"error": {
"code": 1,
"details": [],
"message": "Operation canceled at 2017-06-30T07:48:02-07:00"
},
"metadata": {
"request": {
"@type": "type.googleapis.com/google.genomics.v1alpha2.RunPipelineRequest",
"ephemeralPipeline": {
"name": "msm_update_wf",
"description": "",
"projectId": "rebc-oct16",
"docker": {
"imageName": "broadinstitute/msmutect@sha256:0a875d79efe074919d668bbb31a30774c0b03afdaa3a0a5df3074dbb4efec56b",
"cmd": "/bin/bash /cromwell_root/exec.sh"
},
"inputParameters": [{
"name": "__extra_config_gcs_path",
"description": "",
"defaultValue": ""
}, {
"name": "msm_chrom_scatter.lociList-0",
"description": "",
"defaultValue": "",
"localCopy": {
"path": "fc-1ba248c5-545d-4ac1-8b1e-b812cc282d7d/loci.list",
"disk": "local-disk"
}
}, {
"name": "msm_chrom_scatter.bamIdx-0",
"description": "",
"defaultValue": "",
"localCopy": {
"path": "fc-4520e968-446a-4f9f-85e2-8fcc901269dc/c38896b0-ee5d-471b-bff4-18a5251e6215/ClipReadsWF/567cc8ae-d74c-414a-9868-314a1ae0fc29/call-GatherBamFiles/ReadsClipped.REBC-YQ-A90G-10A.bai",
"disk": "local-disk"
}
}, {
"name": "msm_chrom_scatter.lociBed-0",
"description": "",
"defaultValue": "",
"localCopy": {
"path": "fc-1ba248c5-545d-4ac1-8b1e-b812cc282d7d/loci.bed",
"disk": "local-disk"
}
}, {
"name": "exec",
"description": "",
"defaultValue": "",
"localCopy": {
"path": "exec.sh",
"disk": "local-disk"
}
}, {
"name": "msm_chrom_scatter.bam-0",
"description": "",
"defaultValue": "",
"localCopy": {
"path": "fc-4520e968-446a-4f9f-85e2-8fcc901269dc/c38896b0-ee5d-471b-bff4-18a5251e6215/ClipReadsWF/567cc8ae-d74c-414a-9868-314a1ae0fc29/call-GatherBamFiles/ReadsClipped.REBC-YQ-A90G-10A.bam",
"disk": "local-disk"
}
}],
"resources": {
"preemptible": false,
"bootDiskSizeGb": 10,
"disks": [{
"name": "local-disk",
"source": "",
"readOnly": false,
"autoDelete": false,
"sizeGb": 163,
"mountPoint": "/cromwell_root",
"type": "PERSISTENT_HDD"
}],
"minimumCpuCores": 1,
"minimumRamGb": 3,
"noAddress": false,
"zones": ["us-central1-b", "us-central1-c", "us-central1-f"]
},
"outputParameters": [{
"name": "msm_chrom_scatter-18-rc.txt",
"description": "",
"defaultValue": "",
"localCopy": {
"path": "msm_chrom_scatter-18-rc.txt",
"disk": "local-disk"
}
}, {
"name": "REBC-REBC-YQ-A90G-Normal-SM-5XVDO.A_19.hist",
"description": "",
"defaultValue": "",
"localCopy": {
"path": "REBC-REBC-YQ-A90G-Normal-SM-5XVDO.A_19.hist",
"disk": "local-disk"
}
}],
"pipelineId": ""
},
"pipelineArgs": {
"serviceAccount": {
"email": "default",
"scopes": ["https://www.googleapis.com/auth/genomics", "https://www.googleapis.com/auth/compute"]
},
"logging": {
"gcsPath": "gs://fc-9c84e685-79f8-4d84-9e52-640943257a9b/1b90ba41-a3e3-48cc-8aad-ab58a19e6a22/msm_update_wf/0fbc7974-3930-482d-a8b8-380efb4ae3b0/call-msm_chrom_scatter/shard-18/msm_chrom_scatter-18.log"
},
"clientId": "",
"outputs": {
"msm_chrom_scatter-18-rc.txt": "gs://fc-9c84e685-79f8-4d84-9e52-640943257a9b/1b90ba41-a3e3-48cc-8aad-ab58a19e6a22/msm_update_wf/0fbc7974-3930-482d-a8b8-380efb4ae3b0/call-msm_chrom_scatter/shard-18/msm_chrom_scatter-18-rc.txt",
"REBC-REBC-YQ-A90G-Normal-SM-5XVDO.A_19.hist": "gs://fc-9c84e685-79f8-4d84-9e52-640943257a9b/1b90ba41-a3e3-48cc-8aad-ab58a19e6a22/msm_update_wf/0fbc7974-3930-482d-a8b8-380efb4ae3b0/call-msm_chrom_scatter/shard-18/REBC-REBC-YQ-A90G-Normal-SM-5XVDO.A_19.hist"
},
"projectId": "rebc-oct16",
"inputs": {
"msm_chrom_scatter.lociBed-0": "gs://fc-1ba248c5-545d-4ac1-8b1e-b812cc282d7d/loci.bed",
"msm_chrom_scatter.lociList-0": "gs://fc-1ba248c5-545d-4ac1-8b1e-b812cc282d7d/loci.list",
"__extra_config_gcs_path": "gs://cromwell-auth-rebc-oct16/0fbc7974-3930-482d-a8b8-380efb4ae3b0_auth.json",
"msm_chrom_scatter.bam-0": "gs://fc-4520e968-446a-4f9f-85e2-8fcc901269dc/c38896b0-ee5d-471b-bff4-18a5251e6215/ClipReadsWF/567cc8ae-d74c-414a-9868-314a1ae0fc29/call-GatherBamFiles/ReadsClipped.REBC-YQ-A90G-10A.bam",
"exec": "gs://fc-9c84e685-79f8-4d84-9e52-640943257a9b/1b90ba41-a3e3-48cc-8aad-ab58a19e6a22/msm_update_wf/0fbc7974-3930-482d-a8b8-380efb4ae3b0/call-msm_chrom_scatter/shard-18/exec.sh",
"msm_chrom_scatter.bamIdx-0": "gs://fc-4520e968-446a-4f9f-85e2-8fcc901269dc/c38896b0-ee5d-471b-bff4-18a5251e6215/ClipReadsWF/567cc8ae-d74c-414a-9868-314a1ae0fc29/call-GatherBamFiles/ReadsClipped.REBC-YQ-A90G-10A.bai"
},
"labels": {
"cromwell-workflow-name": "msm-update-wf",
"wdl-task-name": "msm-chrom-scatter",
"cromwell-workflow-id": "cromwell-0fbc7974-3930-482d-a8b8-380efb4ae3b0"
},
"resources": {
"preemptible": false,
"bootDiskSizeGb": 10,
"disks": [{
"name": "local-disk",
"source": "",
"readOnly": false,
"autoDelete": false,
"sizeGb": 163,
"mountPoint": "",
"type": "PERSISTENT_HDD"
}],
"minimumCpuCores": 1,
"minimumRamGb": 3,
"noAddress": false,
"zones": ["us-central1-b", "us-central1-c", "us-central1-f"]
}
}
},
"clientId": "",
"createTime": "2017-06-29T20:01:27Z",
"projectId": "rebc-oct16",
"endTime": "2017-06-30T14:48:02Z",
"labels": {
"cromwell-workflow-name": "msm-update-wf",
"wdl-task-name": "msm-chrom-scatter",
"cromwell-workflow-id": "cromwell-0fbc7974-3930-482d-a8b8-380efb4ae3b0"
},
"@type": "type.googleapis.com/google.genomics.v1.OperationMetadata",
"events": [{
"description": "Warning: Creating VM and disk(s) would exceed \"CPUS\" in region us-central1, will try again",
"startTime": "2017-06-29T20:08:56.443727575Z"
}, {
"description": "Warning: Creating VM and disk(s) would exceed \"CPUS\" in region us-central1, will try again",
"startTime": "2017-06-29T20:35:58.853234031Z"
}],
"runtimeMetadata": {
"@type": "type.googleapis.com/google.genomics.v1alpha2.RuntimeMetadata",
"computeEngine": {
"instanceName": "ggp-18433542094782475500",
"zone": "us-central1-c",
"machineType": "us-central1-c/n1-standard-1",
"diskNames": ["local-disk-18433542094782475500"]
}
}
},
"name": "operations/EOnfxqvPKxjs2eD965vG6P8BIP3g3tG1AioPcHJvZHVjdGlvblF1ZXVl"
}