output coords are negative floats

The network is defined by:

class Net(nn.Module):
    
    def __init__(self, layers):
        super(Net, self).__init__()
        if layers == 18:
            model = models.resnet18(pretrained=True)
        elif layers == 34:
            model = models.resnet34(pretrained=True)
        # change the first layer to recieve five channel image
        model.conv1 = nn.Conv2d(5, 64, kernel_size=7, stride=2, padding=3,bias=True)
        # change the last layer to output 32 coordinates
        # model.fc=nn.Linear(512,32)
        # remove final two layers(fc, avepool)
        model = nn.Sequential(*(list(model.children())[:-2]))
        for param in model.parameters():
            param.requires_grad = True
        self.resnet = model
        
    def forward(self, x):
       
        pose_out = self.resnet(x)
        return pose_out

class CoordRegressionNetwork(nn.Module):
    def __init__(self, n_locations, layers):
        super(CoordRegressionNetwork, self).__init__()
        self.resnet = Net(layers)
        self.hm_conv = nn.Conv2d(512, n_locations, kernel_size=1, bias=False)

    def forward(self, images):
        # 1. Run the images through our Resnet
        resnet_out = self.resnet(images)
        # 2. Use a 1x1 conv to get one unnormalized heatmap per location
        unnormalized_heatmaps = self.hm_conv(resnet_out)
        # 3. Normalize the heatmaps
        heatmaps = dsntnn.flat_softmax(unnormalized_heatmaps)
        # 4. Calculate the coordinates
        coords = dsntnn.dsnt(heatmaps)

        return coords, heatmaps

And the training codes are as follows:

for i, data in enumerate(tqdm(train_dataloader)):
            # training
            images, poses = data['image'], data['pose']
            images, poses = images.to(device), poses.to(device)
            coords, heatmaps = net(images)

            # Per-location euclidean losses
            euc_losses = dsntnn.euclidean_losses(coords, poses)
            # Per-location regularization losses
            reg_losses = dsntnn.js_reg_losses(heatmaps, poses, sigma_t=1.0)
            # Combine losses into an overall loss
            loss = dsntnn.average_loss(euc_losses + reg_losses)
        
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            train_loss_epoch.append(loss.item())

I converted keypoint groundtruth into float(-1,1), but all the predicted coords are negative floats:

tensor([[[-0.1286, -0.0830],
         [-0.1169, -0.0810],
         [-0.1205, -0.1476],
         ...,
         [-0.1767, -0.3881],
         [-0.1970, -0.2403],
         [-0.3226, -0.3909]],

        [[-0.0694, -0.0165],
         [-0.0744, -0.0288],
         [-0.1027, -0.0873],
         ...,
         [-0.0766, -0.3926],
         [-0.1146, -0.2482],
         [-0.0907, -0.1812]],

        [[-0.4647, -0.3639],
         [-0.4430, -0.3409],
         [-0.2485, -0.2339],
         ...,
         [-0.2906, -0.4541],
         [-0.3648, -0.3034],
         [-0.4190, -0.3880]],

and the heatmap seems strange:

the visualization results:

[anibali]

No worries, I’m glad that you found your true problem!

[simonhessner]

I have saved the unnormalized heatmaps and wanted to see what the output will be if I load the heatmaps again and then apply dsntnn.flat_softmax and dstnn.dsnt. I was surprised that the coordinates then looked fine. After a bit of debugging I found that the error is not caused by the DSNT layer but by a affine transformation (STN) that I apply to the coordinates after I get them from the DSNT layer.

So my assumption that there is a bug in dsntnn was wrong. Sorry for the false alarm!