Sync with filters

Hi,

I’m using Dotmim.Sync v0.8 to sync a mobile device (SQLite db and Xamarin iOS) with a MSSQL db as the server in HTTP mode.

Without any filters everything works as intended. When I try to use filters I’m getting SQLite Error 19: 'FOREIGN KEY constraint failed'

Here is a sample schema of my server database:

CREATE SCHEMA Mobile;

CREATE TABLE [Mobile].[Report]
(
	[Id] INTEGER IDENTITY (1, 1),
	[Title] NVARCHAR(50) NOT NULL,
	[CreatedBy] VARCHAR(255) NOT NULL,
	[CreatedOnUtc] DATETIME NOT NULL,
	[UpdatedBy] VARCHAR(255) CONSTRAINT DF_Report_UpdatedBy DEFAULT NULL,
	[UpdatedOnUtc] DATETIME CONSTRAINT DF_Report_UpdatedOnUtc DEFAULT NULL,
	CONSTRAINT [PK_Mobile_Report] PRIMARY KEY CLUSTERED 
	(
		[Id] ASC
	) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
);

CREATE TABLE [Mobile].[ProductLocation]
(
	[Id] INTEGER,
	[ConstantValue] VARCHAR(25) NOT NULL CONSTRAINT UC_ProductLocation_ConstantValue UNIQUE ([ConstantValue]),
	[Description] VARCHAR(25) NOT NULL,
	CONSTRAINT [PK_Mobile_ProductLocation] PRIMARY KEY CLUSTERED 
	(
		[Id] ASC
	) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
);

INSERT INTO [Mobile].[ProductLocation]
(
	[Id],
	[ConstantValue],
	[Description]
)
VALUES
(
	0,
	'Lower_Shelf',
	'Lower Shelf'
);

INSERT INTO [Mobile].[ProductLocation]
(
	[Id],
	[ConstantValue],
	[Description]
)
VALUES
(
	1,
	'Middle_Shelf',
	'Middle Shelf'
);

INSERT INTO [Mobile].[ProductLocation]
(
	[Id],
	[ConstantValue],
	[Description]
)
VALUES
(
	2,
	'Top_Shelf',
	'Top Shelf'
);

CREATE TABLE [Mobile].[ProductMeasureUnit]
(
	[Id] INTEGER,
	[ConstantValue] VARCHAR(25) NOT NULL CONSTRAINT UC_ProductMeasureUnit_ConstantValue UNIQUE ([ConstantValue]),
	[Description] VARCHAR(25) NOT NULL,
	CONSTRAINT [PK_Mobile_ProductMeasureUnit] PRIMARY KEY CLUSTERED 
	(
		[Id] ASC
	) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
);

INSERT INTO [Mobile].[ProductMeasureUnit]
(
	[Id],
	[ConstantValue],
	[Description]
)
VALUES
(
	0,
	'Weight',
	'WEIGHT (kilos)'
);

INSERT INTO [Mobile].[ProductMeasureUnit]
(
	[Id],
	[ConstantValue],
	[Description]
)
VALUES
(
	1,
	'Volume',
	'VOLUME (liters)'
);

INSERT INTO [Mobile].[ProductMeasureUnit]
(
	[Id],
	[ConstantValue],
	[Description]
)
VALUES
(
	2,
	'Length',
	'LENGTH (meters)'
);

CREATE TABLE [Mobile].[ProductType]
(
	[Description] VARCHAR(50),
	[ProductMeasureUnitId] INTEGER CONSTRAINT FK_Mobile_ProductType_ProductMeasureUnit FOREIGN KEY ([ProductMeasureUnitId])
        REFERENCES [Mobile].[ProductMeasureUnit] ([Id])
        ON DELETE SET NULL
        ON UPDATE CASCADE
	CONSTRAINT [PK_Mobile_ProductType] PRIMARY KEY CLUSTERED 
	(
		[Description]
	) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
);

INSERT INTO [Mobile].[ProductType]
(
	[Description],
	[ProductMeasureUnitId]
)
VALUES
(
	'STONE',
	0
);

INSERT INTO [Mobile].[ProductType]
(
	[Description],
	[ProductMeasureUnitId]
)
VALUES
(
	'OIL',
	1
);

INSERT INTO [Mobile].[ProductType]
(
	[Description],
	[ProductMeasureUnitId]
)
VALUES
(
	'ROPE',
	2
);

CREATE TABLE [Mobile].[ReportLine]
(
	[Id] INTEGER IDENTITY (1, 1),
	[ReportId] INTEGER NOT NULL CONSTRAINT FK_Mobile_ReportLine_Report FOREIGN KEY ([ReportId])
        REFERENCES [Mobile].[Report] ([Id])
        ON DELETE CASCADE
        ON UPDATE CASCADE,
	[ProductLocationId] INTEGER CONSTRAINT FK_Mobile_ReportLine_ProductLocation FOREIGN KEY ([ProductLocationId])
        REFERENCES [Mobile].[ProductLocation] ([Id])
        ON DELETE SET NULL
        ON UPDATE CASCADE,
	[ProductTypeDescription] VARCHAR(50) NOT NULL CONSTRAINT FK_Mobile_ReportLine_ProductType FOREIGN KEY ([ProductTypeDescription])
        REFERENCES [Mobile].[ProductType] ([Description])
        ON DELETE CASCADE
        ON UPDATE CASCADE,
	[Amount] INTEGER NOT NULL
	CONSTRAINT [PK_Mobile_ReportLine] PRIMARY KEY CLUSTERED
	(
		[Id] ASC
	) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
);

INSERT INTO [Mobile].[Report] ([Title], [CreatedBy], [CreatedOnUtc]) VALUES ('TITLE1', 'user1', GETDATE());
INSERT INTO [Mobile].[ReportLine] ([ReportId], [ProductLocationId], [ProductTypeDescription], [Amount]) VALUES (1, 0, 'STONE', 100);

INSERT INTO [Mobile].[Report] ([Title], [CreatedBy], [CreatedOnUtc]) VALUES ('TITLE2', 'user2', GETDATE());
INSERT INTO [Mobile].[ReportLine] ([ReportId], [ProductLocationId], [ProductTypeDescription], [Amount]) VALUES (2, 1, 'OIL', 5);
INSERT INTO [Mobile].[ReportLine] ([ReportId], [ProductLocationId], [ProductTypeDescription], [Amount]) VALUES (2, 2, 'ROPE', 50);

Startup.cs at server side:

services.AddControllers();
services.AddMicrosoftIdentityWebApiAuthentication(Configuration)
        .EnableTokenAcquisitionToCallDownstreamApi()
        .AddInMemoryTokenCaches();

var connectionString = Configuration.GetSection("ConnectionStrings")["devSqlConnection"];

var syncOptions = new SyncOptions
{
    BatchSize = 2000,
    DisableConstraintsOnApplyChanges = false, // Tried with true
    UseBulkOperations = true,
    ConflictResolutionPolicy = Dotmim.Sync.Enumerations.ConflictResolutionPolicy.ClientWins,
    UseVerboseErrors = true
};

var tables = new[] {
    "Mobile.ProductLocation",
    "Mobile.ProductMeasureUnit",
    "Mobile.ProductType",
    "Mobile.Report",
    "Mobile.ReportLine"
};
var syncSetup = new SyncSetup(tables)
{
    StoredProceduresPrefix = "sp",
    StoredProceduresSuffix = "",
    TrackingTablesPrefix = "t",
    TrackingTablesSuffix = ""
};

// Works fine, just one report in the sync process
var reportFilter = new SetupFilter("Report", "Mobile");
reportFilter.AddParameter("CreatedBy", "Report", "Mobile");
reportFilter.AddJoin(Join.Inner, "Mobile.Report").On("Mobile.Report", "Id", "Mobile.ReportLine", "ReportId");
reportFilter.AddWhere("CreatedBy", "Report", "CreatedBy", "Mobile");
syncSetup.Filters.Add(reportFilter);

// This crashes. It's trying to bring all the report lines
var reportLineFilter = new SetupFilter("ReportLine", "Mobile");
reportLineFilter.AddParameter("ReportId", "ReportLine", "Mobile");
reportLineFilter.AddJoin(Join.Left, "Mobile.Report").On("Mobile.ReportLine", "ReportId", "Mobile.Report", "Id");
syncSetup.Filters.Add(reportLineFilter);

services.AddSyncServer<SqlSyncChangeTrackingProvider>(connectionString, syncSetup, syncOptions);

Code at client side:

using (var handler = new HttpClientHandler())
{
    using (var client = new HttpClient(handler))
    {
        handler.AutomaticDecompression = DecompressionMethods.GZip;
        handler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) =>
        {
            if (cert.Issuer.Contains("localhost"))
                return true;
            return errors == System.Net.Security.SslPolicyErrors.None;
        };
        client.DefaultRequestHeaders.Host = $"192.168.1.11:5001";
        client.DefaultRequestHeaders.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));
        client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
        var serverOrchestrator = new WebClientOrchestrator("https://192.168.1.11:5001/api/sync", client: client);
        var clientProvider = new SqliteSyncProvider(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "sqlite.db3"));
        // Tried disabling the constraints
        var clientOptions = new SyncOptions { ConflictResolutionPolicy = ConflictResolutionPolicy.ClientWins, UseBulkOperations = true, DisableConstraintsOnApplyChanges = false, UseVerboseErrors = true };
        var agent = new SyncAgent(clientProvider, serverOrchestrator, clientOptions);
        agent.Parameters.Add("CreatedBy", "user1");
        var progress = new SynchronousProgress<ProgressArgs>(pa => Console.WriteLine($"{pa.PogressPercentageString}\t {pa.Message}"));
        var result = await agent.SynchronizeAsync(progress);
    }
}

What I’m trying to achieve:

• With the first filter, bring to the client all the reports filtered by the “CreatedBy” parameter (“user1” in this case). This works
• With the second filter, bring the report lines linked to the reports from the first filter, ignoring the rest

What am I doing wrong? Is there any way to do this? I tried to disable the constraints, tweak the filters… But I’m not able to finish the process.

Thanks in advance!

[flacidsnake]

Filters are quite complex to handle 😃 It took me 15 minutes to figure out what was the problem

I found the solution, using these 2 filters declaration

syncSetup.Filters.Add("Report", "CreatedBy", "Mobile");

var reportLineFilter = new SetupFilter("ReportLine", "Mobile");
reportLineFilter.AddParameter("CreatedBy", DbType.AnsiString, maxLength:255);
reportLineFilter.AddJoin(Join.Left, "Mobile.Report").On("Mobile.ReportLine", "ReportId", "Mobile.Report", "Id");
reportLineFilter.AddWhere("CreatedBy", "Report", "CreatedBy", "Mobile");
syncSetup.Filters.Add(reportLineFilter);

Let me know if it’s working for you (don’t forget to start from a clean fresh server & client databases)

It works! Merci beaucoup! 😄

[Mimetis]

Filters are quite complex to handle 😃 It took me 15 minutes to figure out what was the problem

I found the solution, using these 2 filters declaration

syncSetup.Filters.Add("Report", "CreatedBy", "Mobile");

var reportLineFilter = new SetupFilter("ReportLine", "Mobile");
reportLineFilter.AddParameter("CreatedBy", DbType.AnsiString, maxLength:255);
reportLineFilter.AddJoin(Join.Left, "Mobile.Report").On("Mobile.ReportLine", "ReportId", "Mobile.Report", "Id");
reportLineFilter.AddWhere("CreatedBy", "Report", "CreatedBy", "Mobile");
syncSetup.Filters.Add(reportLineFilter);

Let me know if it’s working for you (don’t forget to start from a clean fresh server & client databases)