diff --git a/internal/bucket/azure/blob.go b/internal/bucket/azure/blob.go index 94489add9..ad70acb4e 100644 --- a/internal/bucket/azure/blob.go +++ b/internal/bucket/azure/blob.go @@ -344,6 +344,14 @@ func (c *BlobClient) FGetObject(ctx context.Context, bucketName, objectName, loc return etag, nil } +// quoteETag returns the ETag in the quoted form used by Get Blob responses. +func quoteETag(etag string) string { + if etag == "" || etag[0] == '"' { + return etag + } + return `"` + etag + `"` +} + // VisitObjects iterates over the items in the provided object storage // bucket, calling visit for every item. // If the underlying client or the visit callback returns an error, @@ -371,7 +379,7 @@ func (c *BlobClient) VisitObjects(ctx context.Context, bucketName string, prefix } var etag string if blob.Properties != nil && blob.Properties.ETag != nil { - etag = fmt.Sprintf("%x", *blob.Properties.ETag) + etag = quoteETag(string(*blob.Properties.ETag)) } if err := visit(*blob.Name, etag); err != nil { err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucketName, err) diff --git a/internal/bucket/azure/blob_test.go b/internal/bucket/azure/blob_test.go index 433f0f14c..21ab3d7d5 100644 --- a/internal/bucket/azure/blob_test.go +++ b/internal/bucket/azure/blob_test.go @@ -542,17 +542,94 @@ func TestBlobClient_VisitObjects_Prefix(t *testing.T) { withoutRetries()) g.Expect(err).ToNot(HaveOccurred()) - var visited []string + visited := map[string]string{} err = client.VisitObjects(t.Context(), bucketName, tt.prefix, func(path, etag string) error { - visited = append(visited, path) + visited[path] = etag return nil }) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(visited).To(Equal([]string{tt.prefix + "file.txt"})) + g.Expect(visited).To(Equal(map[string]string{ + tt.prefix + "file.txt": `"0x8D9B2A2A2A2A2A2"`, + })) }) } } +// TestBlobClient_VisitObjects_EtagMatchesFGetObject verifies that the ETag +// returned when listing a blob equals the ETag returned when downloading the +// same blob. Azure returns the ETag unquoted in the List Blobs XML body and +// quoted in the Get Blob response header, while the Bucket reconciler compares +// the two values to decide whether the container changed. A mismatch makes the +// reconciler re-download the whole container on every reconcile. +func TestBlobClient_VisitObjects_EtagMatchesFGetObject(t *testing.T) { + g := NewWithT(t) + + bucketName := "test-bucket" + objectName := "file.txt" + const rawEtag = "0x8D9B2A2A2A2A2A2" + + bucketListener, bucketAddr, _ := testlistener.New(t) + bucketEndpoint := fmt.Sprintf("http://%s", bucketAddr) + bucketHandler := http.NewServeMux() + // List Blobs returns the ETag unquoted in the XML body. + bucketHandler.HandleFunc(fmt.Sprintf("GET /%s", bucketName), func(w http.ResponseWriter, r *http.Request) { + resp := fmt.Sprintf(` + + + + %s + + %s + + + + +`, bucketEndpoint, bucketName, objectName, rawEtag) + _, err := w.Write([]byte(resp)) + g.Expect(err).ToNot(HaveOccurred()) + }) + // Get Blob returns the ETag quoted in the response header. + bucketHandler.HandleFunc(fmt.Sprintf("GET /%s/%s", bucketName, objectName), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", fmt.Sprintf("%q", rawEtag)) + _, err := w.Write([]byte("file contents")) + g.Expect(err).ToNot(HaveOccurred()) + }) + bucketServer := &http.Server{ + Addr: bucketAddr, + Handler: bucketHandler, + } + go bucketServer.Serve(bucketListener) + defer bucketServer.Shutdown(context.Background()) + + bucket := &sourcev1.Bucket{ + Spec: sourcev1.BucketSpec{ + Endpoint: bucketEndpoint, + }, + } + client, err := NewClient(t.Context(), + bucket, + withoutCredentials(), + withoutRetries()) + g.Expect(err).ToNot(HaveOccurred()) + + var listEtag string + err = client.VisitObjects(t.Context(), bucketName, "", func(path, etag string) error { + if path == objectName { + listEtag = etag + } + return nil + }) + g.Expect(err).ToNot(HaveOccurred()) + + localPath := filepath.Join(t.TempDir(), objectName) + downloadEtag, err := client.FGetObject(t.Context(), bucketName, objectName, localPath) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(listEtag).To(Equal(`"` + rawEtag + `"`)) + g.Expect(downloadEtag).To(Equal(`"` + rawEtag + `"`)) + g.Expect(listEtag).To(Equal(downloadEtag)) +} + func TestBlobClient_FGetObject_MissingEtag(t *testing.T) { g := NewWithT(t)