Skip to content

Commit 78689e0

Browse files
ThomasJClarkThomasJClark
committed
Merge pull request #510 from JLLeitschuh/fix/cameraSourceNotRestarting
Ensure Camera Sources restart after crash
2 parents 42004a4 + 1ac251e commit 78689e0

File tree

10 files changed

+541
-95
lines changed

10 files changed

+541
-95
lines changed

‎core/src/main/java/edu/wpi/grip/core/PipelineRunner.java‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@
77
import com.google.common.eventbus.EventBus;
88
import com.google.common.eventbus.Subscribe;
99
import com.google.common.util.concurrent.AbstractScheduledService;
10+
import com.google.common.util.concurrent.MoreExecutors;
1011
import com.google.inject.Provider;
1112
import com.google.inject.Singleton;
1213
import edu.wpi.grip.core.events.RenderEvent;
1314
import edu.wpi.grip.core.events.RunPipelineEvent;
1415
import edu.wpi.grip.core.events.StopPipelineEvent;
1516
import edu.wpi.grip.core.util.service.AutoRestartingService;
17+
import edu.wpi.grip.core.util.service.LoggingListener;
1618
import edu.wpi.grip.core.util.service.RestartableService;
1719

1820
import javax.annotation.Nullable;
@@ -52,11 +54,6 @@ public class PipelineRunner implements RestartableService {
5254
this.pipelineService = new AutoRestartingService<>(
5355
() -> new AbstractScheduledService() {
5456

55-
@Override
56-
protected void startUp() {
57-
logger.info("Pipeline Starting");
58-
}
59-
6057
/**
6158
*
6259
* @throws InterruptedException This should never happen.
@@ -81,11 +78,6 @@ protected void runOneIteration() throws InterruptedException {
8178
}
8279
}
8380

84-
@Override
85-
protected void shutDown() {
86-
logger.info("Pipeline Shutting Down");
87-
}
88-
8981
@Override
9082
protected Scheduler scheduler() {
9183
return Scheduler.newFixedRateSchedule(0, 1, TimeUnit.MILLISECONDS);
@@ -97,6 +89,7 @@ protected String serviceName() {
9789
}
9890
}
9991
);
92+
this.pipelineService.addListener(new LoggingListener(logger, PipelineRunner.class), MoreExecutors.directExecutor());
10093
}
10194

10295
/**

‎core/src/main/java/edu/wpi/grip/core/sources/CameraSource.java‎

Lines changed: 30 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@
22

33

44
import com.google.common.base.StandardSystemProperty;
5-
import com.google.common.base.Stopwatch;
65
import com.google.common.eventbus.EventBus;
76
import com.google.common.eventbus.Subscribe;
8-
import com.google.common.math.IntMath;
9-
import com.google.common.util.concurrent.AbstractExecutionThreadService;
7+
import com.google.common.util.concurrent.MoreExecutors;
108
import com.google.inject.assistedinject.Assisted;
119
import com.google.inject.assistedinject.AssistedInject;
1210
import com.thoughtworks.xstream.annotations.XStreamAlias;
@@ -18,23 +16,23 @@
1816
import edu.wpi.grip.core.events.SourceRemovedEvent;
1917
import edu.wpi.grip.core.util.ExceptionWitness;
2018
import edu.wpi.grip.core.util.service.AutoRestartingService;
19+
import edu.wpi.grip.core.util.service.LoggingListener;
2120
import edu.wpi.grip.core.util.service.RestartableService;
2221
import org.bytedeco.javacpp.opencv_core.Mat;
23-
import org.bytedeco.javacv.*;
22+
import org.bytedeco.javacv.FrameGrabber;
23+
import org.bytedeco.javacv.OpenCVFrameGrabber;
24+
import org.bytedeco.javacv.VideoInputFrameGrabber;
2425

2526
import java.io.IOException;
26-
import java.math.RoundingMode;
2727
import java.net.MalformedURLException;
2828
import java.net.URL;
2929
import java.util.Optional;
3030
import java.util.Properties;
3131
import java.util.concurrent.Executor;
32-
3332
import java.util.concurrent.TimeUnit;
3433
import java.util.concurrent.TimeoutException;
3534
import java.util.concurrent.atomic.AtomicBoolean;
3635
import java.util.function.Supplier;
37-
import java.util.logging.Level;
3836
import java.util.logging.Logger;
3937

4038
/**
@@ -94,6 +92,9 @@ public interface Factory {
9492
CameraSource create(Properties properties) throws IOException;
9593
}
9694

95+
/**
96+
* Allows for the creation of a frame grabber using either a device number or URL string address.
97+
*/
9798
public interface FrameGrabberFactory {
9899
FrameGrabber create(int deviceNumber);
99100

@@ -194,90 +195,40 @@ public FrameGrabber create(String addressProperty) throws MalformedURLException
194195
}
195196

196197
/* This must be initialized in the constructor otherwise the grabber supplier won't be present */
197-
this.cameraService = new AutoRestartingService<>(() -> new AbstractExecutionThreadService() {
198-
private final FrameGrabber frameGrabber = grabberSupplier.get();
199-
private boolean failedStartup = false;
200-
201-
/**
202-
* Keep a reference to the thread around so that it can be interrupted when stop is called.
203-
*/
204-
private Optional<Thread> serviceThread = Optional.empty();
205-
198+
this.cameraService = new AutoRestartingService<>(() -> new GrabberService(name, grabberSupplier, new CameraSourceUpdater() {
206199
@Override
207-
protected void startUp() {
208-
serviceThread = Optional.of(Thread.currentThread());
209-
try {
210-
frameGrabber.start();
211-
} catch (FrameGrabber.Exception e) {
212-
failedStartup = true;
213-
getExceptionWitness().flagException(e, "Failed to start");
214-
}
200+
public void setFrameRate(double value) {
201+
CameraSource.this.frameRate = frameRate;
202+
isNewFrame.set(true);
215203
}
216204

217205
@Override
218-
protected void run() throws InterruptedException {
219-
// If we failed to startup don't even try to run. Just give up as soon as possible.
220-
if (failedStartup) return;
221-
222-
final OpenCVFrameConverter.ToMat convertToMat = new OpenCVFrameConverter.ToMat();
223-
final Stopwatch stopwatch = Stopwatch.createStarted();
224-
while (super.isRunning()) {
225-
final Frame videoFrame;
226-
try {
227-
videoFrame = frameGrabber.grab();
228-
} catch (FrameGrabber.Exception e) {
229-
getExceptionWitness().flagException(e, "Failed to grab image");
230-
logger.log(Level.WARNING, "Failed to grab image", e);
231-
break; // We failed on a grab, something went wrong. Bail out and let the service restart.
232-
}
233-
234-
final Mat frameMat = convertToMat.convert(videoFrame);
235-
236-
if (frameMat == null || frameMat.isNull()) {
237-
final String errMsg = "The camera returned a null frame Mat";
238-
getExceptionWitness().flagWarning(errMsg);
239-
logger.log(Level.WARNING, errMsg);
240-
break; // We have a null frame, something external has gone wrong. Bail out and let the service restart.
241-
}
242-
243-
synchronized (currentFrameTransferMat) {
244-
frameMat.copyTo(currentFrameTransferMat);
245-
}
246-
247-
stopwatch.stop();
248-
final long elapsedTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
249-
stopwatch.reset();
250-
stopwatch.start();
251-
if (elapsedTime != 0) frameRate = IntMath.divide(1000, Math.toIntExact(elapsedTime), RoundingMode.DOWN);
252-
getExceptionWitness().clearException();
253-
isNewFrame.set(true);
254-
eventBus.post(new SourceHasPendingUpdateEvent(CameraSource.this));
206+
public void copyNewMat(Mat matToCopy) {
207+
synchronized (CameraSource.this.currentFrameTransferMat) {
208+
matToCopy.copyTo(CameraSource.this.currentFrameTransferMat);
255209
}
210+
isNewFrame.set(true);
256211
}
257212

258213
@Override
259-
protected void shutDown() throws FrameGrabber.Exception {
260-
if (failedStartup) return;
261-
frameRate = 0;
262-
isNewFrame.set(true);
214+
public void updatesComplete() {
263215
eventBus.post(new SourceHasPendingUpdateEvent(CameraSource.this));
264-
frameGrabber.stop();
265216
}
266-
217+
}, getExceptionWitness()::clearException));
218+
this.cameraService.addListener(new Listener() {
267219
@Override
268-
protected void triggerShutdown() {
269-
serviceThread.ifPresent(Thread::interrupt);
270-
}
271-
272-
/*
273-
* Allows us to set our own service name
274-
*/
275-
@Override
276-
protected String serviceName() {
277-
return name + " Service";
220+
public void failed(State from, Throwable failure) {
221+
if (failure instanceof GrabberService.GrabberServiceException) {
222+
// These are expected exceptions. Handle them by flagging an exception
223+
getExceptionWitness().flagException((GrabberService.GrabberServiceException) failure, "Camera service crashed");
224+
} else {
225+
// Rethrow as an uncaught exception if this is not an exception we expected.
226+
Optional.ofNullable(Thread.getDefaultUncaughtExceptionHandler())
227+
.ifPresent(handler -> handler.uncaughtException(Thread.currentThread(), failure));
228+
}
278229
}
279-
280-
});
230+
}, MoreExecutors.directExecutor());
231+
this.cameraService.addListener(new LoggingListener(logger, CameraSource.class), MoreExecutors.directExecutor());
281232
}
282233

283234
@Override

‎core/src/main/java/edu/wpi/grip/core/sources/CameraSourceUpdater.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package edu.wpi.grip.core.sources;
2+
3+
import org.bytedeco.javacpp.opencv_core.Mat;
4+
5+
public interface CameraSourceUpdater {
6+
void setFrameRate(double value);
7+
void copyNewMat(Mat matToCopy);
8+
void updatesComplete();
9+
}

‎core/src/main/java/edu/wpi/grip/core/sources/GrabberService.java‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package edu.wpi.grip.core.sources;
2+
3+
import com.google.common.annotations.VisibleForTesting;
4+
import com.google.common.base.Stopwatch;
5+
import com.google.common.math.IntMath;
6+
import com.google.common.util.concurrent.AbstractExecutionThreadService;
7+
import org.bytedeco.javacpp.opencv_core;
8+
import org.bytedeco.javacv.Frame;
9+
import org.bytedeco.javacv.FrameGrabber;
10+
import org.bytedeco.javacv.OpenCVFrameConverter;
11+
12+
import java.io.IOException;
13+
import java.math.RoundingMode;
14+
import java.util.Optional;
15+
import java.util.concurrent.TimeUnit;
16+
import java.util.function.Supplier;
17+
18+
import static com.google.common.base.Preconditions.checkNotNull;
19+
20+
/**
21+
* A service that manages the lifecycle of a {@link org.bytedeco.javacv.FrameGrabber}.
22+
*/
23+
public class GrabberService extends AbstractExecutionThreadService {
24+
private final String name;
25+
private final Supplier<FrameGrabber> frameGrabberSupplier;
26+
private final CameraSourceUpdater updater;
27+
private final Runnable exceptionClearedCallback;
28+
// Do not set this in the constructor.
29+
private FrameGrabber frameGrabber;
30+
31+
/**
32+
* Keep a reference to the thread around so that it can be interrupted when stop is called.
33+
*/
34+
private Optional<Thread> serviceThread = Optional.empty();
35+
36+
GrabberService(String name, Supplier<FrameGrabber> frameGrabberSupplier, CameraSourceUpdater updater, Runnable exceptionClearedCallback) {
37+
super();
38+
this.name = checkNotNull(name, "Name cannot be null");
39+
this.frameGrabberSupplier = checkNotNull(frameGrabberSupplier, "Factory cannot be null");
40+
this.updater = checkNotNull(updater, "Updater cannot be null");
41+
this.exceptionClearedCallback = checkNotNull(exceptionClearedCallback, "Runnable cannot be null");
42+
}
43+
44+
@Override
45+
protected void startUp() throws GrabberServiceException {
46+
serviceThread = Optional.of(Thread.currentThread());
47+
try {
48+
frameGrabber = frameGrabberSupplier.get();
49+
frameGrabber.start();
50+
} catch (FrameGrabber.Exception e) {
51+
throw new GrabberServiceException("Failed to start", e);
52+
}
53+
}
54+
55+
@Override
56+
protected void run() throws GrabberServiceException {
57+
final OpenCVFrameConverter.ToMat convertToMat = new OpenCVFrameConverter.ToMat();
58+
final Stopwatch stopwatch = Stopwatch.createStarted();
59+
60+
while (super.isRunning()) {
61+
runOneGrab(convertToMat, stopwatch);
62+
}
63+
}
64+
65+
@VisibleForTesting
66+
final void runOneGrab(final OpenCVFrameConverter.ToMat convertToMat, final Stopwatch stopwatch) throws GrabberServiceException {
67+
final Frame videoFrame;
68+
try {
69+
videoFrame = frameGrabber.grab();
70+
} catch (FrameGrabber.Exception e) {
71+
throw new GrabberServiceException("Failed to grab image", e);
72+
}
73+
74+
final opencv_core.Mat frameMat = convertToMat.convert(videoFrame);
75+
76+
if (frameMat == null || frameMat.isNull()) {
77+
throw new GrabberServiceException("Returned a null frame Mat");
78+
}
79+
80+
if (frameMat.empty()) {
81+
throw new GrabberServiceException("Returned an empty Mat");
82+
}
83+
84+
updater.copyNewMat(frameMat);
85+
86+
stopwatch.stop();
87+
final long elapsedTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);
88+
stopwatch.reset();
89+
stopwatch.start();
90+
if (elapsedTime != 0)
91+
updater.setFrameRate(IntMath.divide(1000, Math.toIntExact(elapsedTime), RoundingMode.DOWN));
92+
93+
updater.updatesComplete();
94+
exceptionClearedCallback.run();
95+
}
96+
97+
@Override
98+
protected void shutDown() throws GrabberServiceException {
99+
updater.setFrameRate(0);
100+
updater.updatesComplete();
101+
try {
102+
frameGrabber.stop();
103+
} catch (FrameGrabber.Exception e) {
104+
throw new GrabberServiceException("Failed to stop", e);
105+
}
106+
}
107+
108+
@Override
109+
protected void triggerShutdown() {
110+
serviceThread.ifPresent(Thread::interrupt);
111+
}
112+
113+
/*
114+
* Allows us to set our own service name
115+
*/
116+
@Override
117+
protected String serviceName() {
118+
return name + " Service";
119+
}
120+
121+
public final class GrabberServiceException extends IOException {
122+
123+
GrabberServiceException(String message, Exception cause) {
124+
super("[" + name + "] " + message, cause);
125+
}
126+
127+
GrabberServiceException(String message) {
128+
super("[" + name + "] " + message);
129+
}
130+
}
131+
}

0 commit comments

Comments
 (0)