Search results depend on the order of fields

NOTE: I’ve rewritten the entire issue because I’ve found a way to reproduce my issue on a very small dataset.

I’ve noticed that I’m missing search results depending on the order of fields that I provide when creating the index.

In the following example, there are two objects where notation:0 matches the search term WW 8840, and one object where prefLabel:de matches WW 8840. In the first example, only the latter object is returned as a search result even though all fields are supposed to be searched. The second example returns the correct search results just by reordering the fields (putting notation:0 to the end). Note that when specifying notation:0 as the only field to search, it will return the correct results in both cases.

Non-working example (prints 1 and 2 even though the first query should return 3 results):

const FlexSearch = require("flexsearch")

let index = new FlexSearch({
  doc: {
    id: "uri",
    field: [
      "prefLabel:de",
      "notation",
      "editorialNote:de",
    ]
  },
  profile: "score"
})

// Example dataset
let concepts = [
  {"@context":"https://gbv.github.io/jskos/context.json","broader":[{"uri":"http://rvk.uni-regensburg.de/nt/WW%208720%20-%20WW%209239"}],"created":"2012-07-05","editorialNote":{"de":"(Blutgruppen s. XD 3200)"},"http://www.w3.org/2004/02/skos/core#closeMatch":[{"uri":"http://d-nb.info/gnd/4130604-1"},{"uri":"http://d-nb.info/gnd/4022814-9"},{"uri":"http://d-nb.info/gnd/4070945-0"},{"uri":"http://d-nb.info/gnd/4074195-3"}],"identifier":["152145:13422"],"inScheme":[{"uri":"http://uri.gbv.de/terminology/rvk/"}],"modified":"2018-12-14","notation":"WW 8840 - WW 8879","prefLabel":{"de":"Blutkörperchen (Erythrozyt, Leukozyt), Hämoglobin"},"type":["http://www.w3.org/2004/02/skos/core#Concept"],"uri":"http://rvk.uni-regensburg.de/nt/WW%208840%20-%20WW%208879"},
  {"@context":"https://gbv.github.io/jskos/context.json","broader":[{"uri":"http://rvk.uni-regensburg.de/nt/WD%205000%20-%20WD%205970"}],"created":"2012-07-05","editorialNote":{"de":"(Antibiotika s. XI 3500)"},"http://www.w3.org/2004/02/skos/core#closeMatch":[{"uri":"http://d-nb.info/gnd/4155845-5"},{"uri":"http://d-nb.info/gnd/4276935-8"},{"uri":"http://d-nb.info/gnd/4176522-9"},{"uri":"http://d-nb.info/gnd/4175383-5"},{"uri":"http://d-nb.info/gnd/4148701-1"}],"identifier":["148204:"],"inScheme":[{"uri":"http://uri.gbv.de/terminology/rvk/"}],"modified":"2018-12-14","notation":"WD 5380","prefLabel":{"de":"Pyrrolfarbstoffe, Cytochrome, Chromoproteine (Hämoglobin s. WW 8840)"},"type":["http://www.w3.org/2004/02/skos/core#Concept"],"uri":"http://rvk.uni-regensburg.de/nt/WD%205380"},
  {"@context":"https://gbv.github.io/jskos/context.json","broader":[{"uri":"http://rvk.uni-regensburg.de/nt/WW%208840%20-%20WW%208879"}],"created":"2012-07-05","editorialNote":{},"identifier":["152145:13423"],"inScheme":[{"uri":"http://uri.gbv.de/terminology/rvk/"}],"modified":"2018-12-14","notation":"WW 8840","prefLabel":{"de":"Allgemeines"},"type":["http://www.w3.org/2004/02/skos/core#Concept"],"uri":"http://rvk.uni-regensburg.de/nt/WW%208840"}
]

index.add(concepts)

let results
results = index.search("WW 8840")
console.log(results.length) // only matches the second concept (which mentions "WW 8840" in label)

results = index.search("WW 8840", {
  field: "notation"
})
console.log(results.length) // correctly matches two concepts
// with large dataset, also correctly matches the two concepts

Working example (prints 3 and 2 as expected, just by reordering fields):

const FlexSearch = require("flexsearch")

let index = new FlexSearch({
  doc: {
    id: "uri",
    field: [
      "prefLabel:de",
      "editorialNote:de",
      "notation",
    ]
  },
  profile: "score"
})

// Example dataset
let concepts = [
  {"@context":"https://gbv.github.io/jskos/context.json","broader":[{"uri":"http://rvk.uni-regensburg.de/nt/WW%208720%20-%20WW%209239"}],"created":"2012-07-05","editorialNote":{"de":"(Blutgruppen s. XD 3200)"},"http://www.w3.org/2004/02/skos/core#closeMatch":[{"uri":"http://d-nb.info/gnd/4130604-1"},{"uri":"http://d-nb.info/gnd/4022814-9"},{"uri":"http://d-nb.info/gnd/4070945-0"},{"uri":"http://d-nb.info/gnd/4074195-3"}],"identifier":["152145:13422"],"inScheme":[{"uri":"http://uri.gbv.de/terminology/rvk/"}],"modified":"2018-12-14","notation":"WW 8840 - WW 8879","prefLabel":{"de":"Blutkörperchen (Erythrozyt, Leukozyt), Hämoglobin"},"type":["http://www.w3.org/2004/02/skos/core#Concept"],"uri":"http://rvk.uni-regensburg.de/nt/WW%208840%20-%20WW%208879"},
  {"@context":"https://gbv.github.io/jskos/context.json","broader":[{"uri":"http://rvk.uni-regensburg.de/nt/WD%205000%20-%20WD%205970"}],"created":"2012-07-05","editorialNote":{"de":"(Antibiotika s. XI 3500)"},"http://www.w3.org/2004/02/skos/core#closeMatch":[{"uri":"http://d-nb.info/gnd/4155845-5"},{"uri":"http://d-nb.info/gnd/4276935-8"},{"uri":"http://d-nb.info/gnd/4176522-9"},{"uri":"http://d-nb.info/gnd/4175383-5"},{"uri":"http://d-nb.info/gnd/4148701-1"}],"identifier":["148204:"],"inScheme":[{"uri":"http://uri.gbv.de/terminology/rvk/"}],"modified":"2018-12-14","notation":"WD 5380","prefLabel":{"de":"Pyrrolfarbstoffe, Cytochrome, Chromoproteine (Hämoglobin s. WW 8840)"},"type":["http://www.w3.org/2004/02/skos/core#Concept"],"uri":"http://rvk.uni-regensburg.de/nt/WD%205380"},
  {"@context":"https://gbv.github.io/jskos/context.json","broader":[{"uri":"http://rvk.uni-regensburg.de/nt/WW%208840%20-%20WW%208879"}],"created":"2012-07-05","editorialNote":{},"identifier":["152145:13423"],"inScheme":[{"uri":"http://uri.gbv.de/terminology/rvk/"}],"modified":"2018-12-14","notation":"WW 8840","prefLabel":{"de":"Allgemeines"},"type":["http://www.w3.org/2004/02/skos/core#Concept"],"uri":"http://rvk.uni-regensburg.de/nt/WW%208840"}
]

index.add(concepts)

let results
results = index.search("WW 8840")
console.log(results.length) // only matches the second concept (which mentions "WW 8840" in label)

results = index.search("WW 8840", {
  field: "notation"
})
console.log(results.length) // correctly matches two concepts
// with large dataset, also correctly matches the two concepts

Any idea why this is happening? Thanks!

[ts-thomas]

Please change over to the version >= 0.7.x which solves that issue, thanks a lot.

[lludol]

I have the same bug!

To reproduce create documents with multiple attributes. In the search fields put every attributes.

Then put almost the same data in two differents attributes for 3-4 documents.

Exemple

{ id: 1, name: 'fooA', name2: 'fooA456' }
{ id: 1, name: 'fooA', name2: 'fooA456' }
{ id: 1, name: 'fooA', name2: 'fooA456' }

{ id: 1, name: 'fooE', name2: 'fooA123' }
{ id: 1, name: 'fooF', name2: 'fooA123' }
{ id: 1, name: 'fooG', name2: 'fooA123' }

Now you can type in your search something like this: fooA You will only get the first block (the 3 elements in my example).

I have the same bug in production but with more data! Now I am fucked…