I was trying to make a project related to cv in which i want to send a video stream got from camera of raspberry pi to a different laptop along with the objects detected in the frames.
each frame takes about 1.7s to be predicted so i want to send the boxes only after prediction.
But it kept giving segmentation error.
Please help
from ultralytics import YOLO
import cv2
import torch
import numpy as np
import pyttsx3
from collections import Counter
from threading import Thread,Event,Lock
from picamera2 import Picamera2
labels={0: 'person',
1: 'bicycle',
2: 'car',
3: 'motorcycle',
4: 'airplane',
5: 'bus',
6: 'train',
7: 'truck',
8: 'boat',
9: 'traffic light',
10: 'fire hydrant',
11: 'stop sign',
12: 'parking meter',
13: 'bench',
14: 'bird',
15: 'cat',
16: 'dog',
17: 'horse',
18: 'sheep',
19: 'cow',
20: 'elephant',
21: 'bear',
22: 'zebra',
23: 'giraffe',
24: 'backpack',
25: 'umbrella',
26: 'handbag',
27: 'tie',
28: 'suitcase',
29: 'frisbee',
30: 'skis',
31: 'snowboard',
32: 'sports ball',
33: 'kite',
34: 'baseball bat',
35: 'baseball glove',
36: 'skateboard',
37: 'surfboard',
38: 'tennis racket',
39: 'bottle',
40: 'wine glass',
41: 'cup',
42: 'fork',
43: 'knife',
44: 'spoon',
45: 'bowl',
46: 'banana',
47: 'apple',
48: 'sandwich',
49: 'orange',
50: 'broccoli',
51: 'carrot',
52: 'hot dog',
53: 'pizza',
54: 'donut',
55: 'cake',
56: 'chair',
57: 'couch',
58: 'potted plant',
59: 'bed',
60: 'dining table',
61: 'toilet',
62: 'tv',
63: 'laptop',
64: 'mouse',
65: 'remote',
66: 'keyboard',
67: 'cell phone',
68: 'microwave',
69: 'oven',
70: 'toaster',
71: 'sink',
72: 'refrigerator',
73: 'book',
74: 'clock',
75: 'vase',
76: 'scissors',
77: 'teddy bear',
78: 'hair drier',
79: 'toothbrush'}
model=YOLO("yolov8n.pt")
midas = torch.hub.load("intel-isl/MiDaS", "MiDaS_small")
engine=pyttsx3.init()
voices = engine.getProperty('voices')
rate = engine.getProperty('rate')
engine.setProperty('voice', voices[1].id)
engine.setProperty('rate', 160)
# engine.say("EyeSpy Welcomes You")
engine.runAndWait()
engine.stop()
device = torch.device("cpu")
midas.to(device)
midas.eval()
midas_transforms = torch.hub.load("intel-isl/MiDaS", "transforms")
transform = midas_transforms.small_transform
# engine.say("The Model has been Loaded")
engine.runAndWait()
engine.stop()
class SharedData:
def __init__(self):
self.image = None
self.box = None
self.label = None
self.prob = None
self.stop_event = Event()
lock=Lock()
def capture_frames(data):
cam=Picamera2()
cam.start()
while not data.stop_event.is_set():
frame=cam.capture_array("main")[:,:,:3]
if frame is not None:
with lock:
data.image = frame
if cv2.waitKey(1) & 0xFF == ord("q") or cv2.waitKey(1) == 27: # 27 is the escape key
break
cam.stop()
cv2.destroyAllWindows()
def process_frames(data):
while not data.stop_event.is_set():
with lock:
if data.image is not None:
frame = data.image
img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
input_batch = transform(img).to(device)
with torch.no_grad():
prediction = midas(input_batch)
prediction = torch.nn.functional.interpolate(
prediction.unsqueeze(1),
size=img.shape[:2],
mode="bicubic",
align_corners=False,
).squeeze()
output = prediction.cpu().numpy()
o=((output-np.min(output))/(np.max(output)-np.min(output)))
overlay=cv2.merge([o,o,o])
feed = (frame*overlay).astype(np.uint8)
results=model(feed)
for result in results:
data.box=result.boxes.xyxy #remove .xyxy to see images
data.prob=result.probs
data.label=result.boxes.cls.numpy()
# for i in range(len(boxes.xyxy)):
# bb=(list(map(int, boxes.xyxy[i])))
# cv2.rectangle(feed, (bb[0], bb[1]), (bb[2], bb[3]), (0, 255, 0), 2)
str=""
for k,v in Counter(result.boxes.cls.numpy()).items():
str+=f"{v} {labels[k]} "
engine.say(str)
engine.runAndWait()
engine.stop()
print(data)
# cv2.imshow("feed", feed)
if (cv2.waitKey(1) & 0xFF == ord("q")) or cv2.waitKey(1)==27: #27 is the escape key
break
cv2.destroyAllWindows()
if __name__ == "__main__":
data = SharedData()
capture_thread = Thread(target=capture_frames, args=(data,))
process_thread = Thread(target=process_frames, args=(data,))
capture_thread.start()
process_thread.start()
try:
while True:
if cv2.waitKey(1) & 0xFF == ord("q") or cv2.waitKey(1) == 27: # 27 is the escape key
break
finally:
data.stop_event.set()
capture_thread.join()
process_thread.join()
I also tried to use multiprocessing but that also kept getting stuck on seg fault
New contributor