Stuck on an issue?

Lightrun Answers was designed to reduce the constant googling that comes with debugging 3rd party libraries. It collects links to all the places you might be looking at while hunting down a tough bug.

And, if you’re still stuck at the end, we’re happy to hop on a call to see how we can help out.

Failed GCS read during output evaluation not retried, fails workflow

See original GitHub issue

[ERROR] [01/27/2017 13:33:05.570] [cromwell-system-akka.dispatchers.engine-dispatcher-30] [akka://cromwell-system/user/cromwell-service/WorkflowManagerActor] WorkflowManagerActor Workflow 0fcd7ba4-f1e0-4e16-9b67-c83ad2378f44 failed (during ExecutingWorkflowState): Could not evaluate no_address.out = read_string(stdout())
java.lang.RuntimeException: Could not evaluate no_address.out = read_string(stdout())
	at wdl4s.Task$$anonfun$11$$anonfun$4.applyOrElse(Task.scala:182)
	at wdl4s.Task$$anonfun$11$$anonfun$4.applyOrElse(Task.scala:181)
	at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36)
	at scala.util.Failure.recoverWith(Try.scala:203)
	at wdl4s.Task$$anonfun$11.apply(Task.scala:181)
	at wdl4s.Task$$anonfun$11.apply(Task.scala:174)
	at scala.collection.TraversableOnce$$anonfun$foldLeft$1.apply(TraversableOnce.scala:157)
	at scala.collection.TraversableOnce$$anonfun$foldLeft$1.apply(TraversableOnce.scala:157)
	at scala.collection.Iterator$class.foreach(Iterator.scala:893)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
	at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at scala.collection.TraversableOnce$class.foldLeft(TraversableOnce.scala:157)
	at scala.collection.AbstractTraversable.foldLeft(Traversable.scala:104)
	at wdl4s.Task.evaluateOutputs(Task.scala:174)
	at cromwell.backend.wdl.OutputEvaluator$.evaluateOutputs(OutputEvaluator.scala:15)
	at cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.postProcess(JesAsyncBackendJobExecutionActor.scala:366)
	at cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.handleExecutionSuccess(JesAsyncBackendJobExecutionActor.scala:391)
	at cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.handleExecutionSuccess(JesAsyncBackendJobExecutionActor.scala:46)
	at cromwell.backend.standard.StandardAsyncExecutionActor$class.handleExecutionResult(StandardAsyncExecutionActor.scala:397)
	at cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.handleExecutionResult(JesAsyncBackendJobExecutionActor.scala:46)
	at cromwell.backend.standard.StandardAsyncExecutionActor$class.handlePollSuccess(StandardAsyncExecutionActor.scala:326)
	at cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.handlePollSuccess(JesAsyncBackendJobExecutionActor.scala:46)
	at cromwell.backend.standard.StandardAsyncExecutionActor$$anonfun$poll$2.apply(StandardAsyncExecutionActor.scala:291)
	at cromwell.backend.standard.StandardAsyncExecutionActor$$anonfun$poll$2.apply(StandardAsyncExecutionActor.scala:290)
	at scala.util.Success$$anonfun$map$1.apply(Try.scala:237)
	at scala.util.Try$.apply(Try.scala:192)
	at scala.util.Success.map(Try.scala:237)
	at scala.concurrent.Future$$anonfun$map$1.apply(Future.scala:237)
	at scala.concurrent.Future$$anonfun$map$1.apply(Future.scala:237)
	at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)
	at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55)
	at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:91)
	at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)
	at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)
	at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
	at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:90)
	at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:39)
	at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:415)
	at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
	at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Caused by: com.google.cloud.storage.StorageException: 503 Service Unavailable
Backend Error
	at com.google.cloud.storage.spi.DefaultStorageRpc.translate(DefaultStorageRpc.java:190)
	at com.google.cloud.storage.spi.DefaultStorageRpc.read(DefaultStorageRpc.java:482)
	at com.google.cloud.storage.BlobReadChannel$1.call(BlobReadChannel.java:127)
	at com.google.cloud.storage.BlobReadChannel$1.call(BlobReadChannel.java:124)
	at com.google.cloud.RetryHelper.doRetry(RetryHelper.java:179)
	at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:244)
	at com.google.cloud.storage.BlobReadChannel.read(BlobReadChannel.java:124)
	at com.google.cloud.storage.contrib.nio.CloudStorageReadChannel.read(CloudStorageReadChannel.java:85)
	at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
	at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:109)
	at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
	at java.nio.file.Files.read(Files.java:3105)
	at java.nio.file.Files.readAllBytes(Files.java:3158)
	at better.files.File.loadBytes(File.scala:163)
	at better.files.File.byteArray(File.scala:166)
	at better.files.File.contentAsString(File.scala:206)
	at cromwell.backend.wdl.ReadLikeFunctions$class.readFile(ReadLikeFunctions.scala:31)
	at cromwell.backend.impl.jes.JesExpressionFunctions.readFile(JesExpressionFunctions.scala:16)
	at wdl4s.expression.WdlStandardLibraryFunctions$class.fileContentsToString(WdlStandardLibraryFunctions.scala:14)
	at cromwell.backend.impl.jes.JesExpressionFunctions.fileContentsToString(JesExpressionFunctions.scala:16)
	at cromwell.backend.wdl.ReadLikeFunctions$$anonfun$readContentsFromSingleFileParameter$1.apply(ReadLikeFunctions.scala:22)
	at cromwell.backend.wdl.ReadLikeFunctions$$anonfun$readContentsFromSingleFileParameter$1.apply(ReadLikeFunctions.scala:21)
	at scala.util.Success$$anonfun$map$1.apply(Try.scala:237)
	at scala.util.Try$.apply(Try.scala:192)
	at scala.util.Success.map(Try.scala:237)
	at cromwell.backend.wdl.ReadLikeFunctions$class.readContentsFromSingleFileParameter(ReadLikeFunctions.scala:21)
	at cromwell.backend.wdl.ReadLikeFunctions$class.read_string(ReadLikeFunctions.scala:62)
	at cromwell.backend.impl.jes.JesExpressionFunctions.read_string(JesExpressionFunctions.scala:16)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:497)
	at wdl4s.expression.WdlFunctions$$anonfun$getFunction$1.apply(WdlFunctions.scala:12)
	at wdl4s.expression.WdlFunctions$$anonfun$getFunction$1.apply(WdlFunctions.scala:12)
	at wdl4s.expression.ValueEvaluator.evaluate(ValueEvaluator.scala:181)
	at wdl4s.WdlExpression$.evaluate(WdlExpression.scala:85)
	at wdl4s.WdlExpression.evaluate(WdlExpression.scala:161)
	at wdl4s.Task$$anonfun$11.apply(Task.scala:180)
	... 38 more
Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 503 Service Unavailable
Backend Error
	at com.google.api.client.googleapis.json.GoogleJsonResponseException.from(GoogleJsonResponseException.java:146)
	at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:113)
	at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:40)
	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest$1.interceptResponse(AbstractGoogleClientRequest.java:321)
	at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1065)
	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:419)
	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:352)
	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeMedia(AbstractGoogleClientRequest.java:380)
	at com.google.api.services.storage.Storage$Objects$Get.executeMedia(Storage.java:4784)
	at com.google.cloud.storage.spi.DefaultStorageRpc.read(DefaultStorageRpc.java:478)
	... 74 more

Offending task:

task no_address {
  Boolean no_address_requested

  command {
    echo foo
  }
  output {
    String out = read_string(stdout())
  }
  runtime {
    docker: "python:latest"
    noAddress: "${no_address_requested}"
  }
}