Stuck on an issue?

Lightrun Answers was designed to reduce the constant googling that comes with debugging 3rd party libraries. It collects links to all the places you might be looking at while hunting down a tough bug.

And, if you’re still stuck at the end, we’re happy to hop on a call to see how we can help out.

ListBlobs from azure-sdk-for-go v49.0.0 fails with "parsing time" error

See original GitHub issue

Which service(blob, file, queue, table) does this issue concern?

Blob

Which version of the Azurite was used?

Latest (v3.9.0)

Where do you get Azurite? (npm, DockerHub, NuGet, Visual Studio Code Extension)

DockerHub (mcr.microsoft.com/azure-storage/azurite:3.9.0)

What’s the Node.js version?

Whatever the mcr.microsoft.com/azure-storage/azurite:3.9.0 docker container is running

What problem was encountered?

Container.ListBlobs from github.com/Azure/azure-sdk-for-go v49.0.0 fails with error parsing time "" as "2006-01-02T15:04:05Z07:00": cannot parse "" as "2006"

Steps to reproduce the issue?

Run the following code:

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/ory/dockertest/v3"
	"github.com/ory/dockertest/v3/docker"
)

// RunAzurite starts an azurite container
func RunAzurite(pool *dockertest.Pool) (*dockertest.Resource, error) {
	opts := dockertest.RunOptions{
		Repository:   "mcr.microsoft.com/azure-storage/azurite",
		Tag:          "3.9.0",
		ExposedPorts: []string{"10000"},
		PortBindings: map[docker.Port][]docker.PortBinding{
			"10000": {{HostIP: "0.0.0.0", HostPort: "10000"}},
		},
	}
	azurite, err := pool.RunWithOptions(&opts)
	if err != nil {
		return nil, err
	}
	if eerr := azurite.Expire(10); eerr != nil {
		return nil, eerr
	}
	pool.MaxWait = 10 * time.Second
	rerr := pool.Retry(func() error {
		client, eerr := storage.NewEmulatorClient()
		if eerr != nil {
			return eerr
		}
		s := client.GetBlobService()
		c := s.GetContainerReference("cont")
		if _, err = c.Exists(); err != nil {
			return err
		}
		return nil
	})
	return azurite, rerr
}

type AzuriteTransport struct {
	fixBug bool
}

func (t AzuriteTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	resp, err := http.DefaultTransport.RoundTrip(req)
	if err != nil {
		return resp, err
	}

	reqURL := req.URL.String()
	log.Printf("Request URL: %s", reqURL)

	bytes, err := httputil.DumpResponse(resp, true)
	if err != nil {
		log.Fatalf("Error dumping HTTP response: %s", err)
	}
	log.Print(string(bytes))

	// Ugly hack: Detect API calls to storage.Container.ListBlobs and delete the
	// empty `<Snapshot/>` node from the XML response because azure-sdk-for-go
	// fails to deserialise an empty string to a valid timestamp
	if t.fixBug && strings.Contains(reqURL, "comp=list") &&
		strings.Contains(reqURL, "restype=container") {
		bodyBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return resp, fmt.Errorf("failed to read response body: %w", err)
		}
		newBody := strings.ReplaceAll(string(bodyBytes), "<Snapshot/>", "")
		resp.Body = ioutil.NopCloser(strings.NewReader(newBody))
		resp.ContentLength = int64(len(newBody))
	}

	bytes, err = httputil.DumpResponse(resp, true)
	if err != nil {
		log.Fatalf("Error dumping HTTP response: %s", err)
	}
	log.Print(string(bytes))

	return resp, err
}

func main() {
	pool, err := dockertest.NewPool("")
	if err != nil {
		log.Fatalf("Failed to create dockertest pool: %s", err)
	}

	azurite, err := RunAzurite(pool)
	defer pool.Purge(azurite)

	if err != nil {
		log.Fatalf("Failed to start Azurite: %s", err)
	}

	client, err := storage.NewEmulatorClient()
	if err != nil {
		log.Fatalf("Failed to create storage client: %s", err)
	}

	blobClient := client.GetBlobService()
	dummyContainer := "foobar"
	containerRef := blobClient.GetContainerReference(dummyContainer)
	blobRef := containerRef.GetBlobReference("test.txt")

	dummyData := "deadbeef"
	err = blobRef.CreateBlockBlobFromReader(strings.NewReader(dummyData), nil)
	if serr, ok := err.(storage.AzureStorageServiceError); ok && serr.Code == "ContainerNotFound" {
		err := containerRef.Create(nil)
		if err != nil {
			log.Fatalf("Failed to create container: %s", err)
		}
		// The container should exist now, so we can try to create the blob again
		err = blobRef.CreateBlockBlobFromReader(strings.NewReader(dummyData), nil)
		if err != nil {
			log.Fatalf("Failed to create blob: %s", err)
		}
	} else if err != nil {
		log.Fatalf("Unexpected error while creating block: %s", err)
	}

	fixBug := false
	origDefaultClientTransport := http.DefaultClient.Transport
	http.DefaultClient.Transport = AzuriteTransport{fixBug: fixBug}
	defer func() {
		http.DefaultClient.Transport = origDefaultClientTransport
	}()

	response, err := containerRef.ListBlobs(storage.ListBlobsParameters{})
	if err != nil {
		log.Fatalf("Failed to list blobs: %s", err)
	}
	if len(response.Blobs) == 0 {
		log.Fatal("Didn't find any blobs")
	}

	log.Print("That's all folks!")
}

Here’s the go.mod file for posterity:

module example.com

go 1.15

require (
	github.com/Azure/azure-sdk-for-go v49.0.0+incompatible
	github.com/Azure/go-autorest/autorest v0.11.13 // indirect
	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
	github.com/dnaeon/go-vcr v1.1.0 // indirect
	github.com/ory/dockertest/v3 v3.6.2
	github.com/satori/go.uuid v1.2.0 // indirect
)

Here’s the output from running the above code:

2020/12/16 00:20:42 Request URL: http://127.0.0.1:10000/devstoreaccount1/foobar?comp=list&restype=container
2020/12/16 00:20:42 HTTP/1.1 200 OK
Transfer-Encoding: chunked
Connection: keep-alive
Content-Type: application/xml
Date: Wed, 16 Dec 2020 00:20:42 GMT
Keep-Alive: timeout=5
Server: Azurite-Blob/3.9.0
X-Ms-Request-Id: 23db7aa5-9465-4983-ba3a-545e39201c00
X-Ms-Version: 2020-02-10

38b
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><EnumerationResults ServiceEndpoint="http://127.0.0.1:10000/devstoreaccount1" ContainerName="foobar"><Prefix/><Marker/><MaxResults>5000</MaxResults><Delimiter/><Blobs><Blob><Name>test.txt</Name><Snapshot/><Properties><Creation-Time>Wed, 16 Dec 2020 00:20:42 GMT</Creation-Time><Last-Modified>Wed, 16 Dec 2020 00:20:42 GMT</Last-Modified><Etag>0x21DA8BBE2791960</Etag><Content-Length>8</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-MD5>T0EkOEfaaTpPNWwEhhFLxg==</Content-MD5><BlobType>BlockBlob</BlobType><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><ServerEncrypted>true</ServerEncrypted><AccessTier>Hot</AccessTier><AccessTierInferred>true</AccessTierInferred><AccessTierChangeTime>Wed, 16 Dec 2020 00:20:42 GMT</AccessTierChangeTime></Properties></Blob></Blobs><NextMarker/></EnumerationResults>
0

2020/12/16 00:20:42 HTTP/1.1 200 OK
Transfer-Encoding: chunked
Connection: keep-alive
Content-Type: application/xml
Date: Wed, 16 Dec 2020 00:20:42 GMT
Keep-Alive: timeout=5
Server: Azurite-Blob/3.9.0
X-Ms-Request-Id: 23db7aa5-9465-4983-ba3a-545e39201c00
X-Ms-Version: 2020-02-10

38b
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><EnumerationResults ServiceEndpoint="http://127.0.0.1:10000/devstoreaccount1" ContainerName="foobar"><Prefix/><Marker/><MaxResults>5000</MaxResults><Delimiter/><Blobs><Blob><Name>test.txt</Name><Snapshot/><Properties><Creation-Time>Wed, 16 Dec 2020 00:20:42 GMT</Creation-Time><Last-Modified>Wed, 16 Dec 2020 00:20:42 GMT</Last-Modified><Etag>0x21DA8BBE2791960</Etag><Content-Length>8</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-MD5>T0EkOEfaaTpPNWwEhhFLxg==</Content-MD5><BlobType>BlockBlob</BlobType><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><ServerEncrypted>true</ServerEncrypted><AccessTier>Hot</AccessTier><AccessTierInferred>true</AccessTierInferred><AccessTierChangeTime>Wed, 16 Dec 2020 00:20:42 GMT</AccessTierChangeTime></Properties></Blob></Blobs><NextMarker/></EnumerationResults>
0

2020/12/16 00:20:42 Failed to list blobs: parsing time "" as "2006-01-02T15:04:05Z07:00": cannot parse "" as "2006"
exit status 1

Have you found a mitigation/solution?

Because Azurite returns an empty <Snapshot/> node in the response XML when Container.ListBlobs is invoked, which azure-sdk-for-go tries to parse into Blob.Snapshot defined here, which is of type time.Time. Calling time.Parse() on an empty string with time.RFC3339 layout returns the above error.

I was able to work around this issue by injecting a custom http.Transport into http.DefaultClient.Transport and manually adjusting the XML payload in the custom transport RoundTrip() method before letting it return the payload to azure-sdk-for-go. If you want to test this out, set fixBug := true in the above code and run it again. You should get the following output:

2020/12/16 00:24:32 Request URL: http://127.0.0.1:10000/devstoreaccount1/foobar?comp=list&restype=container
2020/12/16 00:24:32 HTTP/1.1 200 OK
Transfer-Encoding: chunked
Connection: keep-alive
Content-Type: application/xml
Date: Wed, 16 Dec 2020 00:24:32 GMT
Keep-Alive: timeout=5
Server: Azurite-Blob/3.9.0
X-Ms-Request-Id: c264cd52-1f29-4c5d-a8b2-bf5d4fbe39b7
X-Ms-Version: 2020-02-10

38b
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><EnumerationResults ServiceEndpoint="http://127.0.0.1:10000/devstoreaccount1" ContainerName="foobar"><Prefix/><Marker/><MaxResults>5000</MaxResults><Delimiter/><Blobs><Blob><Name>test.txt</Name><Snapshot/><Properties><Creation-Time>Wed, 16 Dec 2020 00:24:32 GMT</Creation-Time><Last-Modified>Wed, 16 Dec 2020 00:24:32 GMT</Last-Modified><Etag>0x1FFBEE6B98D2200</Etag><Content-Length>8</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-MD5>T0EkOEfaaTpPNWwEhhFLxg==</Content-MD5><BlobType>BlockBlob</BlobType><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><ServerEncrypted>true</ServerEncrypted><AccessTier>Hot</AccessTier><AccessTierInferred>true</AccessTierInferred><AccessTierChangeTime>Wed, 16 Dec 2020 00:24:32 GMT</AccessTierChangeTime></Properties></Blob></Blobs><NextMarker/></EnumerationResults>
0

2020/12/16 00:24:32 HTTP/1.1 200 OK
Transfer-Encoding: chunked
Connection: keep-alive
Content-Type: application/xml
Date: Wed, 16 Dec 2020 00:24:32 GMT
Keep-Alive: timeout=5
Server: Azurite-Blob/3.9.0
X-Ms-Request-Id: c264cd52-1f29-4c5d-a8b2-bf5d4fbe39b7
X-Ms-Version: 2020-02-10

380
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><EnumerationResults ServiceEndpoint="http://127.0.0.1:10000/devstoreaccount1" ContainerName="foobar"><Prefix/><Marker/><MaxResults>5000</MaxResults><Delimiter/><Blobs><Blob><Name>test.txt</Name><Properties><Creation-Time>Wed, 16 Dec 2020 00:24:32 GMT</Creation-Time><Last-Modified>Wed, 16 Dec 2020 00:24:32 GMT</Last-Modified><Etag>0x1FFBEE6B98D2200</Etag><Content-Length>8</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-MD5>T0EkOEfaaTpPNWwEhhFLxg==</Content-MD5><BlobType>BlockBlob</BlobType><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><ServerEncrypted>true</ServerEncrypted><AccessTier>Hot</AccessTier><AccessTierInferred>true</AccessTierInferred><AccessTierChangeTime>Wed, 16 Dec 2020 00:24:32 GMT</AccessTierChangeTime></Properties></Blob></Blobs><NextMarker/></EnumerationResults>
0

2020/12/16 00:24:32 That's all folks!

Issue Analytics

State:
Created 3 years ago
Comments:5 (4 by maintainers)

Top GitHub Comments

1reaction

bluewwcommented, Feb 10, 2021

The fix for removing empty <snapshot/> tag is already released in v3.11.0.

0reactions

mihaitodorcommented, Jan 14, 2021

@zezha-msft Indeed, I’ve seen that one, but, unfortunately, as far as I’m aware, it doesn’t yet come with support for connection strings: https://github.com/Azure/azure-storage-blob-go/issues/211 and https://github.com/Azure/azure-storage-blob-go/issues/244. The suggested workaround seems a bit hackish and I’d like to see a documented way of using connection strings in combination with https://github.com/azure/azure-storage-blob-go.