Undesired deletion of values depending on start_absolute parameter and datapoint timestamp
See original GitHub issueHello,
Scenario
We are using Kairosdb 1.2 with an underlying Scylladb. We are inserting a double of ‘0.01’ two times into the same metric ‘doubledeletiontest’, with the same timestamps, but with different tags (first: ‘tag’ -> ‘child’, second: ‘tag’ -> ‘parent’). After waiting 500ms, we will try and delete the ‘parent’ datapoint with the following payload:
{
"cache_time": 0,
"metrics": [
{
"name": "doubledeletiontest",
"tags": {
"tag": [
"1"
]
},
"group_by": [],
"aggregators": []
}
],
"start_absolute": 0
}
Problem
But after this, both doubles are deleted. What are we missing here? Isn’t the Delete-Query supposed to work the same way as the Get-Query, i.e. filtering the values by tag? Also, the undesired deletion does not seem to occur if we just send start_absolute
of 1000. On further research, we found that the bug seems to happen when deleting datapoints from year 2009, but not so much from 1970.
Proof
I have written a JUnit Test case for better understanding.
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.kairosdb.client.HttpClient;
import org.kairosdb.client.builder.*;
import org.kairosdb.client.response.QueryResponse;
import org.kairosdb.client.response.Result;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.net.URISyntaxException;
import java.time.ZoneId;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
public class DoubleDeletionFailingTest {
private static final String TAG_NAME = "tag";
private static final String KAIROS_URL = "http://localhost:8083";
private static final String PARENT_ID_TAG = "1";
private static final String CHILD_ID_TAG = "2";
private HttpClient kairosClient;
private HashMap<String, String> parentTags;
private HashMap<String, String> childTags;
private long dec2009;
private long jan1970;
@Before
public void setup() throws IOException {
kairosClient = new HttpClient(KAIROS_URL);
parentTags = new HashMap<>();
parentTags.put(TAG_NAME, PARENT_ID_TAG);
childTags = new HashMap<>();
childTags.put(TAG_NAME, CHILD_ID_TAG);
jan1970 = TimeUnit.HOURS.toMillis(24 * 10);
dec2009 = TimeUnit.HOURS.toMillis(24 * 365 * 40);
}
private void insertData(long singleDate) throws URISyntaxException, IOException {
// create 'child' data and 'parent' data
MetricBuilder instance = MetricBuilder.getInstance();
instance.addMetric(getMetricName()).addTag(TAG_NAME, CHILD_ID_TAG).addDataPoint(singleDate, 100.1D);
kairosClient.pushMetrics(instance);
instance = MetricBuilder.getInstance();
instance.addMetric(getMetricName()).addTag(TAG_NAME, PARENT_ID_TAG).addDataPoint(singleDate, 100.1D);
kairosClient.pushMetrics(instance);
}
@Test
public void deleteSome1970Zero() throws InterruptedException, IOException, URISyntaxException {
// doesn't work
deleteAndCheck(0L);
}
@Test
public void deleteSome1970One() throws InterruptedException, IOException, URISyntaxException {
insertData(jan1970);
// works
deleteAndCheck(1L);
}
@Test
public void deleteSome1970Two() throws InterruptedException, IOException, URISyntaxException {
insertData(jan1970);
// works
deleteAndCheck(2L);
}
@Test
public void deleteSome1970SomeSeconds() throws InterruptedException, IOException, URISyntaxException {
insertData(jan1970);
// works
deleteAndCheck(86400L);
}
@Test
public void deleteSome1970OneDay() throws InterruptedException, IOException, URISyntaxException {
insertData(jan1970);
// doesn't work
deleteAndCheck(TimeUnit.HOURS.toMillis(24L));
}
@Test
public void deleteSome1970TwoDays() throws InterruptedException, IOException, URISyntaxException {
insertData(jan1970);
// doesn't work
deleteAndCheck(TimeUnit.HOURS.toMillis(48L));
}
@Test
public void deleteSome2009Zero() throws InterruptedException, IOException, URISyntaxException {
insertData(dec2009);
// doesn't work
deleteAndCheck(0L);
}
@Test
public void deleteSome2009One() throws InterruptedException, IOException, URISyntaxException {
insertData(dec2009);
// works
deleteAndCheck(1L);
}
@Test
public void deleteSome2009Two() throws InterruptedException, IOException, URISyntaxException {
insertData(dec2009);
// works
deleteAndCheck(2L);
}
@Test
public void deleteSome2009SomeSeconds() throws InterruptedException, IOException, URISyntaxException {
insertData(dec2009);
// works
deleteAndCheck(86400L);
}
@Test
public void deleteSome2009OneDay() throws InterruptedException, IOException, URISyntaxException {
insertData(dec2009);
// doesn't work
deleteAndCheck(TimeUnit.HOURS.toMillis(24L));
}
@Test
public void deleteSome2009TwoDays() throws InterruptedException, IOException, URISyntaxException {
insertData(dec2009);
// doesn't work
deleteAndCheck(TimeUnit.HOURS.toMillis(48L));
}
private void deleteAndCheck(long date) throws InterruptedException, URISyntaxException, IOException {
TimeUnit.MILLISECONDS.sleep(500L);
long count = 1L;
assertThat(count(getMetricName(), childTags), is(count));
assertThat(count(getMetricName(), parentTags), is(count));
// delete only 'parent' values
QueryBuilder deleteQueryBuilder = QueryBuilder.getInstance();
deleteQueryBuilder.setStart(new Date(date));
deleteQueryBuilder.addMetric(getMetricName()).addTags(parentTags);
kairosClient.delete(deleteQueryBuilder);
TimeUnit.MILLISECONDS.sleep(500L);
assertThat(count(getMetricName(), parentTags), is(0L));
// 'child' values must not have been deleted, but are in my environment
assertThat(count(getMetricName(), childTags), is(count));
}
@After
public void tearDown() throws IOException {
kairosClient.deleteMetric(getMetricName());
kairosClient.shutdown();
}
public String getMetricName() {
return "doubledeletiontest";
}
private long count(@Nonnull String metricName, @Nonnull Map<String, String> tags) {
QueryBuilder queryBuilder = QueryBuilder.getInstance();
queryBuilder.setStart(new Date(1000L));
queryBuilder.addMetric(metricName).addTags(tags);
queryBuilder.getMetrics().get(0).addAggregator(AggregatorFactory.createCountAggregator(10, org.kairosdb.client.builder.TimeUnit.YEARS));
return queryAndTransform(queryBuilder, (results, zoneId) -> {
if (CollectionUtils.isEmpty(results) || CollectionUtils.isEmpty(results.get(0).getDataPoints())) {
return 0L;
}
DataPoint dataPoint = results.get(0).getDataPoints().get(0);
try {
return dataPoint.longValue();
} catch (DataFormatException e) {
throw new RuntimeException("Error deserializing values!", e);
}
}, ZoneId.of("UTC"));
}
private <T> T queryAndTransform(@Nonnull QueryBuilder queryBuilder,
@Nonnull BiFunction<List<Result>, ZoneId, T> transformFunction,
@Nonnull ZoneId timeZone) {
try {
queryBuilder.setTimeZone(TimeZone.getTimeZone(timeZone));
QueryResponse queryResponse = kairosClient.query(queryBuilder);
if (CollectionUtils.isNotEmpty(queryResponse.getErrors())) {
// throw IllegalState here to indicate it is most likely the programmer's error
throw new IllegalStateException("There have been errors upon querying the Kairos database! " + StringUtils.join(queryResponse.getErrors().iterator(), " | "));
}
List<Result> results = queryResponse.getQueries().get(0).getResults();
return transformFunction.apply(results, timeZone);
} catch (IOException ioex) {
// we cannot recover here, so we must rethrow this as internal error
throw new RuntimeException("IO-Exception while querying the Kairosdb database!", ioex);
} catch (URISyntaxException uri) {
// we cannot recover here, so we must rethrow this as internal error
throw new IllegalStateException("Illegal URI configuration led to URISyntaxException!", uri);
}
}
}
Issue Analytics
- State:
- Created 6 years ago
- Reactions:2
- Comments:15 (6 by maintainers)
I’ve reproduced it and know what the fix is. The problem is if the window for which you delete is larger than the 3 week row it removes the values from the index. The data is still there just the index gets removed. This is a problem with the CassandraDatastore. Working on a fix, stay tuned.
Sorry, didn’t see you had modified the test. Yes that one just failed for me.