I am currently trying out Segment Anything Model (SAM) and my problem needs a multiple boxes with their corresponding points for specificity inside the box. for example box1 = [#, #, # ,#] with its points [x,y] and the class [0 or 1], and then having multiple of those in a single image.

I made it work doing it with multiple bounding boxes only but i want to include points on each boxes.
This is my current code, I’m not sure anymore why it is giving me an error:

RuntimeError: Sizes of tensors must match except in dimension 1. Expected size 1 but got size 3 for tensor number 1 in the list.

import numpy as np
import torch
import matplotlib.pyplot as plt

point = np.array([[330, 370]])
label = np.array([1])


input_point = torch.tensor(point, device=predictor.device)
input_point = input_point.unsqueeze(0)
transformed_point = predictor.transform.apply_coords_torch(input_point, image.shape[:2])

input_label = torch.tensor(label, device=predictor.device)
input_label = input_label.unsqueeze(0)

#yxyx-xyxy
filtered_rois_xyxy = transform_yxyx_to_xyxy(filtered_rois)
input_boxes = torch.tensor(filtered_rois_xyxy, device=predictor.device)
transformed_boxes = predictor.transform.apply_boxes_torch(input_boxes, image.shape[:2])  

masks,_,_ = predictor.predict_torch(
boxes=transformed_boxes,
point_coords=transformed_point,
point_labels=input_label,
multimask_output=False
)

masks.shape

plt.figure(figsize=(10, 10))
plt.imshow(image)
for mask in masks:
    show_mask(mask.cpu().numpy(), plt.gca(), random_color=True)
for box in input_boxes:
    show_box(box.cpu().numpy(), plt.gca())
plt.axis('off')
plt.show()

for debugging purposes, this is the prints of each inputs:

print(input_boxes)
print(input_point)
print(input_label)


tensor([[330, 370, 495, 634],
        [401, 168, 586, 425],
        [  1,   0, 157, 210]], dtype=torch.int32)
tensor([[[330, 370]]])
tensor([[1]])