I am wondering what GATK's method is for retries upon GCS failures and whether GATK is the one that will retry, or if this is something the user of the tool should do. Here is an example of failure to access a file from GCS storage:
ERROR stack trace
com.google.cloud.storage.StorageException: Read timed out
at com.google.cloud.storage.spi.v1.HttpStorageRpc.translate(HttpStorageRpc.java:189)
at com.google.cloud.storage.spi.v1.HttpStorageRpc.read(HttpStorageRpc.java:515)
at com.google.cloud.storage.BlobReadChannel$1.call(BlobReadChannel.java:127)
at com.google.cloud.storage.BlobReadChannel$1.call(BlobReadChannel.java:124)
at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:93)
at com.google.api.gax.retrying.RetryingFutureImpl.executeAttempt(RetryingFutureImpl.java:140)
at com.google.api.gax.retrying.RetryingFutureImpl.access$500(RetryingFutureImpl.java:58)
at com.google.api.gax.retrying.RetryingFutureImpl$AttemptFutureCallback.onFailure(RetryingFutureImpl.java:176)
at com.google.api.core.ApiFutures$1.onFailure(ApiFutures.java:52)
at com.google.common.util.concurrent.Futures$4.run(Futures.java:1160)
at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
at com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:100)
at com.google.common.util.concurrent.ForwardingListenableFuture.addListener(ForwardingListenableFuture.java:47)
at com.google.api.core.ApiFutureToListenableFuture.addListener(ApiFutureToListenableFuture.java:52)
at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1172)
at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1108)
at com.google.api.core.ApiFutures.addCallback(ApiFutures.java:47)
at com.google.api.gax.retrying.RetryingFutureImpl.setAttemptFuture(RetryingFutureImpl.java:106)
at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:101)
at com.google.api.gax.retrying.RetryingFutureImpl.executeAttempt(RetryingFutureImpl.java:140)
at com.google.api.gax.retrying.RetryingFutureImpl.access$500(RetryingFutureImpl.java:58)
at com.google.api.gax.retrying.RetryingFutureImpl$AttemptFutureCallback.onFailure(RetryingFutureImpl.java:176)
at com.google.api.core.ApiFutures$1.onFailure(ApiFutures.java:52)
at com.google.common.util.concurrent.Futures$4.run(Futures.java:1160)
at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
at com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:100)
at com.google.common.util.concurrent.ForwardingListenableFuture.addListener(ForwardingListenableFuture.java:47)
at com.google.api.core.ApiFutureToListenableFuture.addListener(ApiFutureToListenableFuture.java:52)
at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1172)
at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1108)
at com.google.api.core.ApiFutures.addCallback(ApiFutures.java:47)
at com.google.api.gax.retrying.RetryingFutureImpl.setAttemptFuture(RetryingFutureImpl.java:106)
at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:101)
at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:49)
at com.google.cloud.storage.BlobReadChannel.read(BlobReadChannel.java:124)
at com.google.cloud.storage.contrib.nio.CloudStorageReadChannel.read(CloudStorageReadChannel.java:109)
at htsjdk.samtools.seekablestream.SeekablePathStream.read(SeekablePathStream.java:86)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:286)
at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
at htsjdk.samtools.seekablestream.SeekableBufferedStream.read(SeekableBufferedStream.java:104)
at htsjdk.samtools.AbstractBAMFileIndex$IndexStreamBuffer.readFully(AbstractBAMFileIndex.java:660)
at htsjdk.samtools.AbstractBAMFileIndex$IndexStreamBuffer.readInteger(AbstractBAMFileIndex.java:695)
at htsjdk.samtools.AbstractBAMFileIndex.readInteger(AbstractBAMFileIndex.java:442)
at htsjdk.samtools.AbstractBAMFileIndex.query(AbstractBAMFileIndex.java:279)
at htsjdk.samtools.DiskBasedBAMFileIndex.getSpanOverlapping(DiskBasedBAMFileIndex.java:61)
at htsjdk.samtools.BAMFileReader.getFileSpan(BAMFileReader.java:898)
at htsjdk.samtools.BAMFileReader.createIndexIterator(BAMFileReader.java:915)
at htsjdk.samtools.BAMFileReader.query(BAMFileReader.java:575)
at htsjdk.samtools.SamReader$PrimitiveSamReaderToSamReaderAdapter.query(SamReader.java:528)
at htsjdk.samtools.SamReader$PrimitiveSamReaderToSamReaderAdapter.query(SamReader.java:533)
at htsjdk.samtools.SamReader$PrimitiveSamReaderToSamReaderAdapter.queryOverlapping(SamReader.java:395)
at org.broadinstitute.sv.genotyping.ReadPairMapper.getReadPairs(ReadPairMapper.java:170)
at org.broadinstitute.sv.genotyping.ReadPairMapper.getReadPairs(ReadPairMapper.java:105)
at org.broadinstitute.sv.genotyping.GenotypingReadPairModule.getReadPairs(GenotypingReadPairModule.java:210)
at org.broadinstitute.sv.genotyping.GenotypingReadPairModule.genotypeSample(GenotypingReadPairModule.java:103)
at org.broadinstitute.sv.genotyping.GenotypingReadPairModule.genotypeCnp(GenotypingReadPairModule.java:65)
at org.broadinstitute.sv.genotyping.GenotypingReadPairModule.genotypeCnp(GenotypingReadPairModule.java:39)
at org.broadinstitute.sv.genotyping.GenotypingAlgorithm.genotypeCnpInternal(GenotypingAlgorithm.java:149)
at org.broadinstitute.sv.genotyping.GenotypingAlgorithm.genotypeCnp(GenotypingAlgorithm.java:114)
at org.broadinstitute.sv.genotyping.SVGenotyperWalker.processVCFFile(SVGenotyperWalker.java:276)
at org.broadinstitute.sv.genotyping.SVGenotyperWalker.map(SVGenotyperWalker.java:220)
at org.broadinstitute.sv.genotyping.SVGenotyperWalker.map(SVGenotyperWalker.java:57)
at org.broadinstitute.gatk.engine.traversals.TraverseLociNano.traverse(TraverseLociNano.java:106)
at org.broadinstitute.gatk.engine.traversals.TraverseLociNano.traverse(TraverseLociNano.java:48)
at org.broadinstitute.gatk.engine.executive.LinearMicroScheduler.execute(LinearMicroScheduler.java:98)
at org.broadinstitute.gatk.engine.GenomeAnalysisEngine.execute(GenomeAnalysisEngine.java:316)
at org.broadinstitute.gatk.engine.CommandLineExecutable.execute(CommandLineExecutable.java:123)
at org.broadinstitute.sv.main.SVCommandLine.execute(SVCommandLine.java:133)
at org.broadinstitute.gatk.utils.commandline.CommandLineProgram.start(CommandLineProgram.java:256)
at org.broadinstitute.gatk.utils.commandline.CommandLineProgram.start(CommandLineProgram.java:158)
at org.broadinstitute.sv.main.SVCommandLine.main(SVCommandLine.java:87)
at org.broadinstitute.sv.main.SVGenotyper.main(SVGenotyper.java:21)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
at sun.security.ssl.InputRecord.read(InputRecord.java:503)
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:286)
at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:704)
at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:647)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1569)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1474)
at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480)
at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:338)
at com.google.api.client.http.javanet.NetHttpResponse.(NetHttpResponse.java:37)
at com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:94)
at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:972)
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:419)
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:352)
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeMedia(AbstractGoogleClientRequest.java:380)
at com.google.api.services.storage.Storage$Objects$Get.executeMedia(Storage.java:5130)
at com.google.cloud.storage.spi.v1.HttpStorageRpc.read(HttpStorageRpc.java:494)
... 71 more