Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions internal/push/parity_golden_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ func runGoPreflight(t *testing.T, c parityCase) string {
Root: dir,
LabelsCSV: filepath.Join(dir, c.CSV),
Images: listImages(t, dir),
Sidecars: listSidecars(t, dir),
}
spec := SpecArgs{
Category: c.Category,
Expand Down Expand Up @@ -179,6 +180,41 @@ func listImages(t *testing.T, dir string) []string {
return out
}

// listSidecars maps each sidecar directory in a case (any subdir that isn't
// images/ — annotations/, masks/, texts/, sequences/) to its files, mirroring
// how the production Discover populates LocalLayout.Sidecars. Without this the
// object_detection / semantic_segmentation pairing previews would see an empty
// sidecar set and reject every case (cli#288).
func listSidecars(t *testing.T, dir string) map[string][]string {
t.Helper()
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
sidecars := make(map[string][]string)
for _, e := range entries {
if !e.IsDir() || e.Name() == "images" {
continue
}
files, err := os.ReadDir(filepath.Join(dir, e.Name()))
if err != nil {
t.Fatal(err)
}
var out []string
for _, f := range files {
if !f.IsDir() {
out = append(out, filepath.Join(dir, e.Name(), f.Name()))
}
}
sort.Strings(out)
sidecars[e.Name()] = out
}
if len(sidecars) == 0 {
return nil
}
return sidecars
}

func mustLoad(t *testing.T, path string, into any) {
t.Helper()
b, err := os.ReadFile(path)
Expand Down
371 changes: 369 additions & 2 deletions internal/push/testdata/parity/cases.json

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/clm-ok/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/clm-ok/texts/a.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
plain pretraining prose with no tabs
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/clm-ok/texts/b.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
summarize this article it is about parity
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/emb-malformed/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
just one field with no tab
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
query text matching passage
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/emb-ok/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/emb-ok/texts/a.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
anchor sentence a close paraphrase
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/emb-ok/texts/b.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
query text matching passage unrelated passage
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,Annotation,Visibility
a.jpg,"not json at all","{""nose"": 1}"
b.jpg,"{""nose"": [6, 7], ""tail"": [18, 22]}","{""nose"": 1, ""tail"": 0}"
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/kp-ok/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,Annotation,Visibility
a.jpg,"{""nose"": [4, 5], ""tail"": [20, 25]}","{""nose"": 1, ""tail"": 1}"
b.jpg,"{""nose"": [6, 7], ""tail"": [18, 22]}","{""nose"": 1, ""tail"": 0}"
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,Annotation,Visibility
a.jpg,"{""nose"": [4, 5], ""tail"": [20, 25]}","{""nose"": 1, ""tail"": 1}"
b.jpg,"{""nose"": [6, 7], ""tail"": [18, 22]}","{""nose"": 1, ""tail"": 0}"
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
an unreferenced sequence
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/mlm-ok/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/mlm-ok/sequences/a.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
the quick brown fox jumps over the lazy dog
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/mlm-ok/sequences/b.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pack my box with five dozen liquor jugs
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
<annotation>
<folder>images</folder>
<filename>a.jpg</filename>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>20</xmin>
<ymin>3</ymin>
<xmax>2</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>b.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/objdet-bad-xml/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>a.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
26 changes: 26 additions & 0 deletions internal/push/testdata/parity/cases/objdet-ok/annotations/a.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>a.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
26 changes: 26 additions & 0 deletions internal/push/testdata/parity/cases/objdet-ok/annotations/b.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>b.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/objdet-ok/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>a.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>b.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>c.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,mask_id
a.jpg,a_mask.png
b.jpg,b.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,mask_id
a.jpg,a_mask.png
b.jpg,b_mask.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/semseg-ok/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,mask_id
a.jpg,a_mask.png
b.jpg,b_mask.png
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,entails
b.txt,entails
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a man is sleeping a person rests
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
water is wet liquid feels wet
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,entails
b.txt,contradicts
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
no tab separator in this file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
the sky is blue the sky is green
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/sentpair-ok/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,entails
b.txt,contradicts
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a man is sleeping a person rests
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
the sky is blue the sky is green
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/seq2seq-ok/labels.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/seq2seq-ok/texts/a.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
hello world hallo welt
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/seq2seq-ok/texts/b.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
good morning guten morgen
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
age,income
34,50000
41,62000
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
age,label
34,2.0
41,2.0
29,2.0
4 changes: 4 additions & 0 deletions internal/push/testdata/parity/cases/tabreg-ok/data.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
age,label
34,1.5
41,2.75
29,0.25
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,pos
b.txt,pos
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
the movie was great
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
loved every minute
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,PERSON O O
b.txt,B-ORG O B-PER
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
John lives here
Loading
Loading