Flaky table refresh behaviour

CrateDB version

4.7.0 (also with older versions

CrateDB setup information

single node / multi node

Steps to Reproduce

It is not possible to 100% recreate this every time

1. CREATE TABLE
2. INSERT some date
3. wait a few seconds
4. SELECT on table

CREATE TABLE testtable (
   obj OBJECT(IGNORED)
);

wait a few seconds

INSERT INTO testtable (obj) VALUES ({a=1,b=2}),({c=1,b=2}),({a=1,c=2});

wait a few seconds

SELECT * FROM testtable limit 100;
-- SELECT OK, 0 records returned (0.388 seconds)

I would expect that query to take longer, because a refresh needs to be triggered, however it then should return also the recent data. Typically I would not expect data older than 1 second to be missing from a SELECT

run query again right after

SELECT * FROM testtable limit 100;
-- SELECT OK, 3 records returned (0.002 seconds)

Expected Result

• SELECT takes a bit longer, because refresh needs to be run, before returning results
• All data is returned

Actual Result

• SELECT takes a bit longer, because refresh needs to be run, before returning results
• No data is returned

[remear]

Setting the refresh interval on the table has been a stable workaround so far. Thanks!

[proddata]

Seems like this is a (somewhat flaky) way to reproduce this issue

DROP TABLE IF EXISTS source_table;
DROP TABLE IF EXISTS sink_table;
CREATE TABLE source_table (txt TEXT, db DOUBLE);
CREATE TABLE sink_table (txt TEXT, db DOUBLE);
-- Timeout for 45 seconds ...
INSERT INTO source_table (txt,db) SELECT 'Hello World', 10 FROM generate_series(0,1,1);
--Timeout for 5 seconds ...
INSERT INTO sink_table SELECT txt, db FROM source_table;
--Timeout for 5 seconds ...
SELECT * FROM source_table LIMIT 100;
--Rows:  0
SELECT * FROM sink_table limit 100;
--Rows:  0

Both row counts should be 2 in this case Tested on a 3 node cluster

If the initial (45s) timeout is shorter (e.g. 5s), then both SELECTs return 2

---

test script:

const axios = require("axios");

let user = 'user';
let pwd = 'secret';


const instance = axios.create({
    baseURL: `https://${user}:${pwd}@host:4200/_sql`,
    timeout: 60000
  });


runQueries();

async function runQueries(){
    let res;
    await query(`DROP TABLE IF EXISTS source_table;`);
    await query(`DROP TABLE IF EXISTS sink_table;`);
    await query(`CREATE TABLE source_table (txt TEXT, db DOUBLE);`);
    await query(`CREATE TABLE sink_table (txt TEXT, db DOUBLE);`);
    await timeout(45);

    await query(`INSERT INTO source_table (txt,db) SELECT 'Hello World', 10 FROM generate_series(0,1,1);`);
    await timeout(5);

    await query(`INSERT INTO sink_table SELECT txt, db FROM source_table;`);
    await timeout(5);
    
    res = await query(`SELECT * FROM source_table LIMIT 100;`);
    console.log("Rows: ", res.data.rows.length);
    res = await query(`SELECT * FROM sink_table limit 100;`);
    console.log("Rows: ", res.data.rows.length);
}

async function query(stmt){
    console.log(stmt);
    return await instance.post('/',{stmt});
}

function timeout(seconds) {
    console.log(`Timeout for ${seconds} seconds ...`);
    return new Promise(resolve => setTimeout(resolve, seconds * 1000));
}