Undesired deletion of values depending on start_absolute parameter and datapoint timestamp

Hello,

Scenario

We are using Kairosdb 1.2 with an underlying Scylladb. We are inserting a double of ‘0.01’ two times into the same metric ‘doubledeletiontest’, with the same timestamps, but with different tags (first: ‘tag’ -> ‘child’, second: ‘tag’ -> ‘parent’). After waiting 500ms, we will try and delete the ‘parent’ datapoint with the following payload:

{
  "cache_time": 0,
  "metrics": [
    {
      "name": "doubledeletiontest",
      "tags": {
        "tag": [
          "1"
        ]
      },
      "group_by": [],
      "aggregators": []
    }
  ],
  "start_absolute": 0
}

Problem

But after this, both doubles are deleted. What are we missing here? Isn’t the Delete-Query supposed to work the same way as the Get-Query, i.e. filtering the values by tag? Also, the undesired deletion does not seem to occur if we just send start_absolute of 1000. On further research, we found that the bug seems to happen when deleting datapoints from year 2009, but not so much from 1970.

Proof

I have written a JUnit Test case for better understanding.

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.kairosdb.client.HttpClient;
import org.kairosdb.client.builder.*;
import org.kairosdb.client.response.QueryResponse;
import org.kairosdb.client.response.Result;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.net.URISyntaxException;
import java.time.ZoneId;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;

public class DoubleDeletionFailingTest {

    private static final String TAG_NAME = "tag";
    private static final String KAIROS_URL = "http://localhost:8083";
    private static final String PARENT_ID_TAG = "1";
    private static final String CHILD_ID_TAG = "2";

    private HttpClient kairosClient;
    private HashMap<String, String> parentTags;
    private HashMap<String, String> childTags;
    private long dec2009;
    private long jan1970;

    @Before
    public void setup() throws IOException {
        kairosClient = new HttpClient(KAIROS_URL);

        parentTags = new HashMap<>();
        parentTags.put(TAG_NAME, PARENT_ID_TAG);

        childTags = new HashMap<>();
        childTags.put(TAG_NAME, CHILD_ID_TAG);

        jan1970 = TimeUnit.HOURS.toMillis(24 * 10);
        dec2009 = TimeUnit.HOURS.toMillis(24 * 365 * 40);
    }

    private void insertData(long singleDate) throws URISyntaxException, IOException {
        // create 'child' data and 'parent' data
        MetricBuilder instance = MetricBuilder.getInstance();
        instance.addMetric(getMetricName()).addTag(TAG_NAME, CHILD_ID_TAG).addDataPoint(singleDate, 100.1D);
        kairosClient.pushMetrics(instance);

        instance = MetricBuilder.getInstance();
        instance.addMetric(getMetricName()).addTag(TAG_NAME, PARENT_ID_TAG).addDataPoint(singleDate, 100.1D);
        kairosClient.pushMetrics(instance);
    }

    @Test
    public void deleteSome1970Zero() throws InterruptedException, IOException, URISyntaxException {
        // doesn't work
        deleteAndCheck(0L);
    }

    @Test
    public void deleteSome1970One() throws InterruptedException, IOException, URISyntaxException {
        insertData(jan1970);
        // works
         deleteAndCheck(1L);
    }

    @Test
    public void deleteSome1970Two() throws InterruptedException, IOException, URISyntaxException {
        insertData(jan1970);
        // works
        deleteAndCheck(2L);
    }

    @Test
    public void deleteSome1970SomeSeconds() throws InterruptedException, IOException, URISyntaxException {
        insertData(jan1970);
        // works
        deleteAndCheck(86400L);
    }

    @Test
    public void deleteSome1970OneDay() throws InterruptedException, IOException, URISyntaxException {
        insertData(jan1970);
        // doesn't work
        deleteAndCheck(TimeUnit.HOURS.toMillis(24L));
    }

    @Test
    public void deleteSome1970TwoDays() throws InterruptedException, IOException, URISyntaxException {
        insertData(jan1970);
        // doesn't work
        deleteAndCheck(TimeUnit.HOURS.toMillis(48L));
    }
    
    @Test
    public void deleteSome2009Zero() throws InterruptedException, IOException, URISyntaxException {
        insertData(dec2009);
        // doesn't work
        deleteAndCheck(0L);
    }

    @Test
    public void deleteSome2009One() throws InterruptedException, IOException, URISyntaxException {
        insertData(dec2009);
        // works
         deleteAndCheck(1L);
    }

    @Test
    public void deleteSome2009Two() throws InterruptedException, IOException, URISyntaxException {
        insertData(dec2009);
        // works
        deleteAndCheck(2L);
    }

    @Test
    public void deleteSome2009SomeSeconds() throws InterruptedException, IOException, URISyntaxException {
        insertData(dec2009);
        // works
        deleteAndCheck(86400L);
    }

    @Test
    public void deleteSome2009OneDay() throws InterruptedException, IOException, URISyntaxException {
        insertData(dec2009);
        // doesn't work
        deleteAndCheck(TimeUnit.HOURS.toMillis(24L));
    }

    @Test
    public void deleteSome2009TwoDays() throws InterruptedException, IOException, URISyntaxException {
        insertData(dec2009);
        // doesn't work
        deleteAndCheck(TimeUnit.HOURS.toMillis(48L));
    }

    private void deleteAndCheck(long date) throws InterruptedException, URISyntaxException, IOException {
        TimeUnit.MILLISECONDS.sleep(500L);

        long count = 1L;
        assertThat(count(getMetricName(), childTags), is(count));
        assertThat(count(getMetricName(), parentTags), is(count));

        // delete only 'parent' values
        QueryBuilder deleteQueryBuilder = QueryBuilder.getInstance();
        deleteQueryBuilder.setStart(new Date(date));
        deleteQueryBuilder.addMetric(getMetricName()).addTags(parentTags);

        kairosClient.delete(deleteQueryBuilder);

        TimeUnit.MILLISECONDS.sleep(500L);

        assertThat(count(getMetricName(), parentTags), is(0L));
        // 'child' values must not have been deleted, but are in my environment
        assertThat(count(getMetricName(), childTags), is(count));
    }

    @After
    public void tearDown() throws IOException {
        kairosClient.deleteMetric(getMetricName());

        kairosClient.shutdown();

    }

    public String getMetricName() {
        return "doubledeletiontest";
    }

    private long count(@Nonnull String metricName, @Nonnull Map<String, String> tags) {

        QueryBuilder queryBuilder = QueryBuilder.getInstance();
        queryBuilder.setStart(new Date(1000L));
        queryBuilder.addMetric(metricName).addTags(tags);

        queryBuilder.getMetrics().get(0).addAggregator(AggregatorFactory.createCountAggregator(10, org.kairosdb.client.builder.TimeUnit.YEARS));

        return queryAndTransform(queryBuilder, (results, zoneId) -> {
            if (CollectionUtils.isEmpty(results) || CollectionUtils.isEmpty(results.get(0).getDataPoints())) {
                return 0L;
            }
            DataPoint dataPoint = results.get(0).getDataPoints().get(0);
            try {
                return dataPoint.longValue();
            } catch (DataFormatException e) {
                throw new RuntimeException("Error deserializing values!", e);
            }
        }, ZoneId.of("UTC"));
    }

    private <T> T queryAndTransform(@Nonnull QueryBuilder queryBuilder,
                                    @Nonnull BiFunction<List<Result>, ZoneId, T> transformFunction,
                                    @Nonnull ZoneId timeZone) {
        try {
            queryBuilder.setTimeZone(TimeZone.getTimeZone(timeZone));

            QueryResponse queryResponse = kairosClient.query(queryBuilder);
            if (CollectionUtils.isNotEmpty(queryResponse.getErrors())) {
                // throw IllegalState here to indicate it is most likely the programmer's error
                throw new IllegalStateException("There have been errors upon querying the Kairos database! " + StringUtils.join(queryResponse.getErrors().iterator(), " | "));
            }
            List<Result> results = queryResponse.getQueries().get(0).getResults();
            return transformFunction.apply(results, timeZone);
        } catch (IOException ioex) {
            // we cannot recover here, so we must rethrow this as internal error
            throw new RuntimeException("IO-Exception while querying the Kairosdb database!", ioex);
        } catch (URISyntaxException uri) {
            // we cannot recover here, so we must rethrow this as internal error
            throw new IllegalStateException("Illegal URI configuration led to URISyntaxException!", uri);
        }
    }
}

[brianhks]

I’ve reproduced it and know what the fix is. The problem is if the window for which you delete is larger than the 3 week row it removes the values from the index. The data is still there just the index gets removed. This is a problem with the CassandraDatastore. Working on a fix, stay tuned.

[brianhks]

Sorry, didn’t see you had modified the test. Yes that one just failed for me.