Exception read DownloadResponse info.eTag on blob download

Which service(blob, file, queue, table) does this issue concern?

blob

Which version of the Azurite was used?

6e64127 (Azurite v3.0.0-preview) Azurite 2.6.5

Which version of the SDK was used?

azure-storage-blob 11.0.0 (azure-sdk-for-java)

What problem was encountered?

The following exception is thrown from azure-storage-blob client on blob download:

java.lang.IllegalArgumentException: The argument must not be null or an empty string. Argument name: info.eTag.
	at com.microsoft.azure.storage.blob.Utility.assertNotNull(Utility.java:77) ~[azure-storage-blob-11.0.0.jar!/:na]
	at com.microsoft.azure.storage.blob.DownloadResponse.<init>(DownloadResponse.java:56) ~[azure-storage-blob-11.0.0.jar!/:na]
	at com.microsoft.azure.storage.blob.BlobURL.lambda$0(BlobURL.java:369) ~[azure-storage-blob-11.0.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:57) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:64) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleResumeNext$ResumeMainSingleObserver.onSuccess(SingleResumeNext.java:65) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:111) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.maybe.MaybeToSingle$ToSingleMaybeSubscriber.onSuccess(MaybeToSingle.java:83) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.maybe.MaybeMap$MapMaybeObserver.onSuccess(MaybeMap.java:89) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.maybe.MaybeJust.subscribeActual(MaybeJust.java:36) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.Maybe.subscribe(Maybe.java:4156) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.maybe.MaybeMap.subscribeActual(MaybeMap.java:40) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.Maybe.subscribe(Maybe.java:4156) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.maybe.MaybeToSingle.subscribeActual(MaybeToSingle.java:46) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.Single.subscribe(Single.java:3394) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:84) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:111) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleJust.subscribeActual(SingleJust.java:30) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.Single.subscribe(Single.java:3394) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:84) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleResumeNext$ResumeMainSingleObserver.onSuccess(SingleResumeNext.java:65) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:111) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleJust.subscribeActual(SingleJust.java:30) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.Single.subscribe(Single.java:3394) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:84) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.observers.ResumeSingleObserver.onSuccess(ResumeSingleObserver.java:46) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleTimeout$TimeoutMainObserver.onSuccess(SingleTimeout.java:133) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleDoOnSuccess$DoOnSuccess.onSuccess(SingleDoOnSuccess.java:59) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:64) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:111) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleJust.subscribeActual(SingleJust.java:30) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.Single.subscribe(Single.java:3394) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:84) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleDoOnSuccess$DoOnSuccess.onSuccess(SingleDoOnSuccess.java:59) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52) ~[rxjava-2.2.0.jar!/:na]
	at io.reactivex.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68) ~[rxjava-2.2.0.jar!/:na]
	at com.microsoft.rest.v2.http.NettyClient$HttpClientInboundHandler.channelRead(NettyClient.java:918) ~[client-runtime-2.1.0.jar!/:na]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[netty-codec-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) ~[netty-codec-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) ~[netty-codec-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[netty-codec-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:546) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:500) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) ~[netty-transport-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884) ~[netty-common-4.1.28.Final.jar!/:4.1.28.Final]
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-common-4.1.28.Final.jar!/:4.1.28.Final]
	at java.base/java.lang.Thread.run(Thread.java:834) ~[na:na]

Steps to reproduce the issue?

1. Create a container
2. Create a blob into the container
3. Run a Java application with the following code:

public void read(String containerName, String blobName) {
    ContainerURL containerURL = serviceURL.createContainerURL(containerName);
    BlobURL blobURL = containerURL.createBlobURL(blobName);
    Throwable throwable = blobURL.download().flatMapCompletable(
        stream -> writeToSystemOut(stream.body(null)))
        .blockingGet();
}

public Completable writeToSystemOut(Flowable<ByteBuffer> content) {
    return Completable.create(
        emitter -> {
            content.subscribe(new BasicFlowableSubscriber(emitter));
        });
}

public class BasicFlowableSubscriber implements FlowableSubscriber<ByteBuffer> {
    private final CompletableEmitter emitter;
    private Subscription subscription;

    public BasicFlowableSubscriber(CompletableEmitter emitter) {
        this.emitter = emitter;
    }

    @Override
    public void onSubscribe(Subscription subscription) {
        this.subscription = subscription;
        subscription.request(1);
    }

    @Override
    public void onNext(ByteBuffer byteBuffer) {
        String value = StandardCharsets.UTF_8.decode(byteBuffer).toString();
        System.out.println(value);
    }

    @Override
    public void onError(Throwable t) {
        subscription.cancel();
        emitter.onError(t);
    }

    @Override
    public void onComplete() {
        emitter.onComplete();
    }
}

Have you found a mitigation/solution?

no

[rickle-msft]

@amolyneaux @marcoMobilab Can I suggest you guys move away from v11, either to v8 or v12? We have not had this issue reported in v8, which is much more mature, so I hope that you won’t hit this issue there, and a quick look at the code in the v12 preview suggests that the headers are case insensitive there.

We are generally suggesting people pause development in v11 because of some stability issues, so I am hoping either v8 or v12 preview will be able to unblock you guys.

[jfgosselin]

Did anyone successfully use v12 with Azurite ?

I can’t use http://127.0.0.1:10000/devstoreaccount1 because it’s stripped by the BlobServiceClientBuilder …

https://github.com/Azure/azure-sdk-for-java/blob/master/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceClientBuilder.java#L76