From 5e01fb2910be9a80c952b0b13317d4d18b46d8f3 Mon Sep 17 00:00:00 2001 From: Paulo Dias Date: Sun, 19 Nov 2023 13:48:44 +0000 Subject: [PATCH 1/8] plugins/videoreader: New video reader with ffmpeg. --- build.gradle | 1 + plugins-dev/videoreader/build.gradle | 4 + plugins-dev/videoreader/plugins.lst | 1 + .../neptus/plugins/videoreader/Camera.java | 37 ++ .../videoreader/DecodeAndPlayVideo.java | 295 +++++++++++ .../videoreader/IpCamManagementPanel.java | 352 +++++++++++++ .../neptus/plugins/videoreader/Player.java | 262 ++++++++++ .../lsts/neptus/plugins/videoreader/Util.java | 321 ++++++++++++ .../plugins/videoreader/VideoReader.java | 484 ++++++++++++++++++ .../videoreader/src/resources/ipUrl.ini | 9 + 10 files changed, 1766 insertions(+) create mode 100644 plugins-dev/videoreader/build.gradle create mode 100644 plugins-dev/videoreader/plugins.lst create mode 100644 plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Camera.java create mode 100644 plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/DecodeAndPlayVideo.java create mode 100644 plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java create mode 100644 plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Player.java create mode 100644 plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Util.java create mode 100644 plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java create mode 100644 plugins-dev/videoreader/src/resources/ipUrl.ini diff --git a/build.gradle b/build.gradle index c05183301e..30317615dc 100644 --- a/build.gradle +++ b/build.gradle @@ -544,6 +544,7 @@ project(':core') { implementation 'com.adobe.xmp:xmpcore:5.1.2' //6.1.10 implementation name: 'xuggle-xuggler-5.4' + implementation 'io.humble:humble-video-all:0.3.0' // Needs to be in the core because of the dll/so libs implementation 'com.google.zxing:javase:3.2.1' //3.4.0 diff --git a/plugins-dev/videoreader/build.gradle b/plugins-dev/videoreader/build.gradle new file mode 100644 index 0000000000..6bea0cebab --- /dev/null +++ b/plugins-dev/videoreader/build.gradle @@ -0,0 +1,4 @@ +dependencies { + //implementation 'io.humble:humble-video-all:0.3.0' // Needs to be in the core because of the dll/so libs +} + diff --git a/plugins-dev/videoreader/plugins.lst b/plugins-dev/videoreader/plugins.lst new file mode 100644 index 0000000000..3461399017 --- /dev/null +++ b/plugins-dev/videoreader/plugins.lst @@ -0,0 +1 @@ +pt.lsts.neptus.plugins.videoreader.VideoReader \ No newline at end of file diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Camera.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Camera.java new file mode 100644 index 0000000000..693b93702f --- /dev/null +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Camera.java @@ -0,0 +1,37 @@ +package pt.lsts.neptus.plugins.videoreader; + +public class Camera { + private String name; + + private String ip; + private String url; + + public Camera(String name, String ip, String url) { + this.name = name; + this.ip = ip; + this.url = url; + } + + public Camera() { + this.name = "Select Device"; + } + + public String getName() { + return name; + } + + public String getIp() { + if (ip == null) return ""; + return ip; + } + + public String getUrl() { + if (url == null) return ""; + return url; + } + + public String toString() { + if (ip == null) return name; + return name + " (" + ip + ")"; + } +} diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/DecodeAndPlayVideo.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/DecodeAndPlayVideo.java new file mode 100644 index 0000000000..61e452bc69 --- /dev/null +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/DecodeAndPlayVideo.java @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2004-2023 Universidade do Porto - Faculdade de Engenharia + * Laboratório de Sistemas e Tecnologia Subaquática (LSTS) + * All rights reserved. + * Rua Dr. Roberto Frias s/n, sala I203, 4200-465 Porto, Portugal + * + * This file is part of Neptus, Command and Control Framework. + * + * Commercial Licence Usage + * Licencees holding valid commercial Neptus licences may use this file + * in accordance with the commercial licence agreement provided with the + * Software or, alternatively, in accordance with the terms contained in a + * written agreement between you and Universidade do Porto. For licensing + * terms, conditions, and further information contact lsts@fe.up.pt. + * + * Modified European Union Public Licence - EUPL v.1.1 Usage + * Alternatively, this file may be used under the terms of the Modified EUPL, + * Version 1.1 only (the "Licence"), appearing in the file LICENSE.md + * included in the packaging of this file. You may not use this work + * except in compliance with the Licence. Unless required by applicable + * law or agreed to in writing, software distributed under the Licence is + * distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF + * ANY KIND, either express or implied. See the Licence for the specific + * language governing permissions and limitations at + * https://github.com/LSTS/neptus/blob/develop/LICENSE.md + * and http://ec.europa.eu/idabc/eupl.html. + * + * For more information please see . + * + * Author: Paulo Dias + * 18/11/2023 + */ +package pt.lsts.neptus.plugins.videoreader; + +import io.humble.video.Decoder; +import io.humble.video.Demuxer; +import io.humble.video.DemuxerStream; +import io.humble.video.Global; +import io.humble.video.Media; +import io.humble.video.MediaDescriptor; +import io.humble.video.MediaPacket; +import io.humble.video.MediaPicture; +import io.humble.video.Rational; +import io.humble.video.awt.ImageFrame; +import io.humble.video.awt.MediaPictureConverter; +import io.humble.video.awt.MediaPictureConverterFactory; + +import java.awt.image.BufferedImage; +import java.io.IOException; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +/** + * Opens a media file, finds the first video stream, and then plays it. + * This is meant as a demonstration program to teach the use of the Humble API. + *

+ * Concepts introduced: + *

+ * + * + *

+ * To run from maven, do: + *

+ * 
+ * mvn install exec:java -Dexec.mainClass="io.humble.video.demos.DecodeAndPlayVideo" -Dexec.args="filename.mp4"
+ *
+ * + * @author aclarke + * + */ +public class DecodeAndPlayVideo { + /** + * Opens a file, and plays the video from it on a screen at the right rate. + * @param filename The file or URL to play. + */ + private static void playVideo(String filename) throws InterruptedException, IOException { + /* + * Start by creating a container object, in this case a demuxer since + * we are reading, to get video data from. + */ + Demuxer demuxer = Demuxer.make(); + + /* + * Open the demuxer with the filename passed on. + */ + demuxer.open(filename, null, false, true, null, null); + + /* + * Query how many streams the call to open found + */ + int numStreams = demuxer.getNumStreams(); + + /* + * Iterate through the streams to find the first video stream + */ + int videoStreamId = -1; + long streamStartTime = Global.NO_PTS; + Decoder videoDecoder = null; + for(int i = 0; i < numStreams; i++) + { + final DemuxerStream stream = demuxer.getStream(i); + streamStartTime = stream.getStartTime(); + final Decoder decoder = stream.getDecoder(); + if (decoder != null && decoder.getCodecType() == MediaDescriptor.Type.MEDIA_VIDEO) { + videoStreamId = i; + videoDecoder = decoder; + // stop at the first one. + break; + } + } + if (videoStreamId == -1) + throw new RuntimeException("could not find video stream in container: "+filename); + + /* + * Now we have found the audio stream in this file. Let's open up our decoder so it can + * do work. + */ + videoDecoder.open(null, null); + + final MediaPicture picture = MediaPicture.make( + videoDecoder.getWidth(), + videoDecoder.getHeight(), + videoDecoder.getPixelFormat()); + + /** A converter object we'll use to convert the picture in the video to a BGR_24 format that Java Swing + * can work with. You can still access the data directly in the MediaPicture if you prefer, but this + * abstracts away from this demo most of that byte-conversion work. Go read the source code for the + * converters if you're a glutton for punishment. + */ + final MediaPictureConverter converter = + MediaPictureConverterFactory.createConverter( + MediaPictureConverterFactory.HUMBLE_BGR_24, + picture); + BufferedImage image = null; + + /** + * This is the Window we will display in. See the code for this if you're curious, but to keep this demo clean + * we're 'simplifying' Java AWT UI updating code. This method just creates a single window on the UI thread, and blocks + * until it is displayed. + */ + final ImageFrame window = ImageFrame.make(); + if (window == null) { + throw new RuntimeException("Attempting this demo on a headless machine, and that will not work. Sad day for you."); + } + + /** + * Media playback, like comedy, is all about timing. Here we're going to introduce very very basic + * timing. This code is deliberately kept simple (i.e. doesn't worry about A/V drift, garbage collection pause time, etc.) + * because that will quickly make things more complicated. + * + * But the basic idea is there are two clocks: + * + * + * And we need to convert between the two units of time. Each MediaPicture and MediaAudio object have associated + * time stamps, and much of the complexity in video players goes into making sure the right picture (or sound) is + * seen (or heard) at the right time. This is actually very tricky and many folks get it wrong -- watch enough + * Netflix and you'll see what I mean -- audio and video slightly out of sync. But for this demo, we're erring for + * 'simplicity' of code, not correctness. It is beyond the scope of this demo to make a full fledged video player. + */ + + // Calculate the time BEFORE we start playing. + long systemStartTime = System.nanoTime(); + // Set units for the system time, which because we used System.nanoTime will be in nanoseconds. + final Rational systemTimeBase = Rational.make(1, 1000000000); + // All the MediaPicture objects decoded from the videoDecoder will share this timebase. + final Rational streamTimebase = videoDecoder.getTimeBase(); + + /** + * Now, we start walking through the container looking at each packet. This + * is a decoding loop, and as you work with Humble you'll write a lot + * of these. + * + * Notice how in this loop we reuse all of our objects to avoid + * reallocating them. Each call to Humble resets objects to avoid + * unnecessary reallocation. + */ + final MediaPacket packet = MediaPacket.make(); + while(demuxer.read(packet) >= 0) { + /** + * Now we have a packet, let's see if it belongs to our video stream + */ + if (packet.getStreamIndex() == videoStreamId) + { + /** + * A packet can actually contain multiple sets of samples (or frames of samples + * in decoding speak). So, we may need to call decode multiple + * times at different offsets in the packet's data. We capture that here. + */ + int offset = 0; + int bytesRead = 0; + do { + bytesRead += videoDecoder.decode(picture, packet, offset); + if (picture.isComplete()) { + image = displayVideoAtCorrectTime(streamStartTime, picture, + converter, image, window, systemStartTime, systemTimeBase, + streamTimebase); + } + offset += bytesRead; + } while (offset < packet.getSize()); + } + } + + // Some video decoders (especially advanced ones) will cache + // video data before they begin decoding, so when you are done you need + // to flush them. The convention to flush Encoders or Decoders in Humble Video + // is to keep passing in null until incomplete samples or packets are returned. + do { + videoDecoder.decode(picture, null, 0); + if (picture.isComplete()) { + image = displayVideoAtCorrectTime(streamStartTime, picture, converter, + image, window, systemStartTime, systemTimeBase, streamTimebase); + } + } while (picture.isComplete()); + + // It is good practice to close demuxers when you're done to free + // up file handles. Humble will EVENTUALLY detect if nothing else + // references this demuxer and close it then, but get in the habit + // of cleaning up after yourself, and your future girlfriend/boyfriend + // will appreciate it. + demuxer.close(); + + // similar with the demuxer, for the windowing system, clean up after yourself. + window.dispose(); + } + + /** + * Takes the video picture and displays it at the right time. + */ + private static BufferedImage displayVideoAtCorrectTime(long streamStartTime, + final MediaPicture picture, final MediaPictureConverter converter, + BufferedImage image, final ImageFrame window, long systemStartTime, + final Rational systemTimeBase, final Rational streamTimebase) + throws InterruptedException { + long streamTimestamp = picture.getTimeStamp(); + // convert streamTimestamp into system units (i.e. nano-seconds) + streamTimestamp = systemTimeBase.rescale(streamTimestamp-streamStartTime, streamTimebase); + // get the current clock time, with our most accurate clock + long systemTimestamp = System.nanoTime(); + // loop in a sleeping loop until we're within 1 ms of the time for that video frame. + // a real video player needs to be much more sophisticated than this. + while (streamTimestamp > (systemTimestamp - systemStartTime + 1000000)) { + Thread.sleep(1); + systemTimestamp = System.nanoTime(); + } + // finally, convert the image from Humble format into Java images. + image = converter.toImage(image, picture); + // And ask the UI thread to repaint with the new image. + window.setImage(image); + return image; + } + + /** + * Takes a media container (file) as the first argument, opens it, + * opens up a window and plays back the video. + * + * @param args Must contain one string which represents a filename + * @throws IOException + * @throws InterruptedException + */ + public static void main(String[] args) throws InterruptedException, IOException + { + final Options options = new Options(); + options.addOption("h", "help", false, "displays help"); + options.addOption("v", "version", false, "version of this library"); + + final CommandLineParser parser = new org.apache.commons.cli.BasicParser(); + try { + final CommandLine cmd = parser.parse(options, args); + if (cmd.hasOption("version")) { + // let's find what version of the library we're running + final String version = io.humble.video_native.Version.getVersionInfo(); + System.out.println("Humble Version: " + version); + } else if (cmd.hasOption("help") || args.length == 0) { + final HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp(DecodeAndPlayVideo.class.getCanonicalName() + " ", options); + } else { + final String[] parsedArgs = cmd.getArgs(); + for(String arg: parsedArgs) + playVideo(arg); + } + } catch (ParseException e) { + System.err.println("Exception parsing command line: " + e.getLocalizedMessage()); + } + } +} \ No newline at end of file diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java new file mode 100644 index 0000000000..52031a9f0d --- /dev/null +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2004-2023 Universidade do Porto - Faculdade de Engenharia + * Laboratório de Sistemas e Tecnologia Subaquática (LSTS) + * All rights reserved. + * Rua Dr. Roberto Frias s/n, sala I203, 4200-465 Porto, Portugal + * + * This file is part of Neptus, Command and Control Framework. + * + * Commercial Licence Usage + * Licencees holding valid commercial Neptus licences may use this file + * in accordance with the commercial licence agreement provided with the + * Software or, alternatively, in accordance with the terms contained in a + * written agreement between you and Universidade do Porto. For licensing + * terms, conditions, and further information contact lsts@fe.up.pt. + * + * Modified European Union Public Licence - EUPL v.1.1 Usage + * Alternatively, this file may be used under the terms of the Modified EUPL, + * Version 1.1 only (the "Licence"), appearing in the file LICENSE.md + * included in the packaging of this file. You may not use this work + * except in compliance with the Licence. Unless required by applicable + * law or agreed to in writing, software distributed under the Licence is + * distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF + * ANY KIND, either express or implied. See the Licence for the specific + * language governing permissions and limitations at + * https://github.com/LSTS/neptus/blob/develop/LICENSE.md + * and http://ec.europa.eu/idabc/eupl.html. + * + * For more information please see . + * + * Author: Pedro Gonçalves + */ +package pt.lsts.neptus.plugins.videoreader; + +import foxtrot.AsyncTask; +import foxtrot.AsyncWorker; +import net.miginfocom.swing.MigLayout; +import pt.lsts.neptus.NeptusLog; +import pt.lsts.neptus.i18n.I18n; +import pt.lsts.neptus.util.ImageUtils; +import pt.lsts.neptus.util.conf.ConfigFetch; + +import javax.swing.ImageIcon; +import javax.swing.JButton; +import javax.swing.JComboBox; +import javax.swing.JDialog; +import javax.swing.JLabel; +import javax.swing.JPanel; +import javax.swing.JTextField; +import javax.swing.SwingUtilities; +import java.awt.Color; +import java.awt.Dialog; +import java.awt.Font; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.KeyAdapter; +import java.awt.event.KeyEvent; +import java.awt.event.MouseAdapter; +import java.awt.event.MouseEvent; +import java.io.File; +import java.net.URI; +import java.util.ArrayList; +import java.util.concurrent.Callable; +import java.util.function.Function; + +public class IpCamManagementPanel extends JPanel { + private final VideoReader videoReader; + private final Function setValidUrlFunction; + private final Runnable connectStreamFunction; + private ArrayList cameraList; + // JPanel for color state of ping to host IPCam + private JPanel colorStateIPCam; + // JButton to confirm IPCam + private JButton selectIPCam; + // JComboBox for list of IPCam in ipUrl.ini + private JComboBox ipCamList; + // row select from string matrix of IPCam List + private int selectedItemIndex; + // JLabel for text IPCam Ping + private JLabel onOffIndicator; + // JTextField for IPCam name + private final JTextField fieldName = new JTextField(I18n.text("Name")); + // JTextField for IPCam ip + private final JTextField fieldIP = new JTextField(I18n.text("IP")); + // JTextField for IPCam url + private final JTextField fieldUrl = new JTextField(I18n.text("URL")); + + public IpCamManagementPanel(VideoReader videoReader, Function setValidUrlFunction, + Runnable connectStreamFunction) { + this.videoReader = videoReader; + this.setValidUrlFunction = setValidUrlFunction; + this.connectStreamFunction = connectStreamFunction; + setLayout(new MigLayout()); + } + + public void show(String camUrl) { + removeAll(); + setLayout(new MigLayout()); + + repaintParametersTextFields(); + cameraList = readIPUrl(); + + URI uri = Util.getCamUrlAsURI(camUrl); + + JDialog ipCamPing = new JDialog(SwingUtilities.getWindowAncestor(videoReader), I18n.text("Select IPCam")); + ipCamPing.setResizable(true); + ipCamPing.setModalityType(Dialog.ModalityType.DOCUMENT_MODAL); + ipCamPing.setSize(440, 200); + ipCamPing.setLocationRelativeTo(videoReader); + + ImageIcon imgIPCam = ImageUtils.createImageIcon("images/menus/camera.png"); + if (imgIPCam != null) { + ipCamPing.setIconImage(imgIPCam.getImage()); + } + ipCamPing.setResizable(false); + //ipCamPing.setBackground(Color.GRAY); + + int sel = 0; + if (uri != null && uri.getScheme() != null) { + String host = uri.getHost(); + String name = "Stream " + uri.getScheme() + "@" + uri.getPort(); + Camera cam = new Camera(name, host, camUrl); + NeptusLog.pub().info("Cam > " + cam + " | host " + host+ " | URI " + camUrl + " | " + cam.getUrl()); + Camera matchCam = cameraList.stream().filter(c -> c.getUrl().equalsIgnoreCase(cam.getUrl())) + .findAny().orElse(null); + + if (matchCam == null) { + cameraList.add(1, cam); + sel = 1; + } + else { + int index = -1; + for (int i = 0; i < cameraList.size(); i++) { + Camera c = cameraList.get(i); + if (c == matchCam) { + index = i; + break; + } + } + sel = index; + if (index < 0) { + cameraList.add(1, cam); + sel = 1; + } + } + } + + ipCamList = new JComboBox(cameraList.toArray()); + ipCamList.setSelectedIndex(0); + ipCamList.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + ipCamList.setEnabled(false); + selectIPCam.setEnabled(false); + selectedItemIndex = ipCamList.getSelectedIndex(); + if (selectedItemIndex > 0) { + Camera selectedCamera = cameraList.get(selectedItemIndex); + colorStateIPCam.setBackground(Color.LIGHT_GRAY); + onOffIndicator.setText("---"); + + repaintParametersTextFields(selectedCamera.getName(), selectedCamera.getIp(), selectedCamera.getUrl()); + + AsyncTask task = new AsyncTask() { + boolean reachable; + + @Override + public Object run() throws Exception { + reachable = Util.hostIsReachable(selectedCamera.getIp()); + return null; + } + + @Override + public void finish() { + if (reachable) { + selectIPCam.setEnabled(true); + //camUrl = selectedCamera.getUrl(); + setValidUrlFunction.apply(selectedCamera.getUrl()); + colorStateIPCam.setBackground(Color.GREEN); + onOffIndicator.setText("ON"); + ipCamList.setEnabled(true); + } + else { + selectIPCam.setEnabled(false); + colorStateIPCam.setBackground(Color.RED); + onOffIndicator.setText("OFF"); + ipCamList.setEnabled(true); + } + selectIPCam.validate(); + selectIPCam.repaint(); + } + }; + AsyncWorker.getWorkerThread().postTask(task); + } + else { + colorStateIPCam.setBackground(Color.RED); + onOffIndicator.setText("OFF"); + ipCamList.setEnabled(true); + repaintParametersTextFields(); + } + } + }); + add(ipCamList, "split 3, width 50:250:250, center"); + + colorStateIPCam = new JPanel(); + onOffIndicator = new JLabel(I18n.text("OFF")); + onOffIndicator.setFont(new Font("Verdana", Font.BOLD, 14)); + colorStateIPCam.setBackground(Color.RED); + colorStateIPCam.add(onOffIndicator); + add(colorStateIPCam, "h 30!, w 30!"); + + selectIPCam = new JButton(I18n.text("Connect"), imgIPCam); + selectIPCam.setEnabled(false); + selectIPCam.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + NeptusLog.pub().info("IPCam Select: " + cameraList.get(selectedItemIndex)); + ipCamPing.setVisible(false); + videoReader.service.execute(connectStreamFunction); + } + }); + fieldIP.setEditable(false); + fieldIP.setFocusable(false); + add(selectIPCam, "h 30!, wrap"); + + JButton addNewIPCam = new JButton(I18n.text("Add New IPCam")); + addNewIPCam.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + // Execute when button is pressed + if (fieldName.getText().trim().isEmpty()) return; + if (fieldIP.getText().trim().isEmpty()) return; + if (fieldUrl.getText().trim().isEmpty()) return; + if (Util.getHostFromURI(fieldUrl.getText().trim()) == null) return; + + Camera camToAdd = Util.parseLineCamera(String.format("%s#%s#%s\n", fieldName.getText().trim(), + fieldIP.getText().trim(), fieldUrl.getText().trim())); + if (camToAdd != null) { + String ipUrlFilename = ConfigFetch.getConfFolder() + "/" + VideoReader.BASE_FOLDER_FOR_URL_INI; + Util.addCamToFile(camToAdd, ipUrlFilename); + reloadIPCamList(); + } + } + }); + + JButton removeIpCam = new JButton(I18n.text("Remove IPCam")); + removeIpCam.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent event) { + Camera camToRemove = (Camera) ipCamList.getSelectedItem(); + String ipUrlFilename = ConfigFetch.getConfFolder() + "/" + VideoReader.BASE_FOLDER_FOR_URL_INI; + // Execute when button is pressed + Util.removeCamFromFile(camToRemove, ipUrlFilename); + reloadIPCamList(); + } + }); + + fieldUrl.addMouseListener(new MouseAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + updateIPFieldFromUrlField(); + } + }); + fieldUrl.addKeyListener(new KeyAdapter() { + @Override + public void keyReleased(KeyEvent e) { + updateIPFieldFromUrlField(); + } + }); + + add(fieldName, "w 410!, wrap"); + add(fieldIP, "w 410!, wrap"); + add(fieldUrl, "w 410!, wrap"); + add(addNewIPCam, "split 2, width 120!, center, gap related"); + add(removeIpCam, "w 120!"); + + ipCamPing.add(this); + ipCamPing.pack(); + + if (sel > 0) { + ipCamList.setSelectedIndex(sel); + } + + // Show dialog + ipCamPing.setVisible(true); + } + + private void updateIPFieldFromUrlField() { + String host = Util.getHostFromURI(fieldUrl.getText()); + if (host != null) { + fieldIP.setText(host); + fieldIP.validate(); + fieldIP.repaint(200); + } + } + + private void repaintParametersTextFields(String name, String ip, String url) { + fieldName.setText(name); + fieldName.validate(); + fieldName.repaint(); + fieldIP.setText(ip); + fieldIP.validate(); + fieldIP.repaint(); + fieldUrl.setText(url); + fieldUrl.validate(); + fieldUrl.repaint(); + } + + private void repaintParametersTextFields() { + repaintParametersTextFields("NAME", "IP", "URL"); + } + + private void reloadIPCamList() { + AsyncTask task = new AsyncTask() { + @Override + public Object run() throws Exception { + cameraList = readIPUrl(); + return null; + } + + @Override + public void finish() { + int itemCount = ipCamList.getItemCount(); + ipCamList.removeAllItems(); + for (Camera camera : cameraList) { + ipCamList.addItem(camera); + } + + // If an item was added select that item + if (itemCount < ipCamList.getItemCount()) { + ipCamList.setSelectedIndex(ipCamList.getItemCount() - 1); + } + } + }; + AsyncWorker.getWorkerThread().postTask(task); + } + + // Read file + private ArrayList readIPUrl() { + Util.createIpUrlFile(); + File confIni = new File(ConfigFetch.getConfFolder() + "/" + VideoReader.BASE_FOLDER_FOR_URL_INI); + return Util.readIpUrl(confIni); + } + + public String getStreamName() { + return fieldName.getText(); + } + + public String getStreamIp() { + return fieldIP.getText(); + } + + public String getStreamUrl() { + return fieldUrl.getText(); + } +} diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Player.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Player.java new file mode 100644 index 0000000000..aa921daf8d --- /dev/null +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Player.java @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2004-2023 Universidade do Porto - Faculdade de Engenharia + * Laboratório de Sistemas e Tecnologia Subaquática (LSTS) + * All rights reserved. + * Rua Dr. Roberto Frias s/n, sala I203, 4200-465 Porto, Portugal + * + * This file is part of Neptus, Command and Control Framework. + * + * Commercial Licence Usage + * Licencees holding valid commercial Neptus licences may use this file + * in accordance with the commercial licence agreement provided with the + * Software or, alternatively, in accordance with the terms contained in a + * written agreement between you and Universidade do Porto. For licensing + * terms, conditions, and further information contact lsts@fe.up.pt. + * + * Modified European Union Public Licence - EUPL v.1.1 Usage + * Alternatively, this file may be used under the terms of the Modified EUPL, + * Version 1.1 only (the "Licence"), appearing in the file LICENSE.md + * included in the packaging of this file. You may not use this work + * except in compliance with the Licence. Unless required by applicable + * law or agreed to in writing, software distributed under the Licence is + * distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF + * ANY KIND, either express or implied. See the Licence for the specific + * language governing permissions and limitations at + * https://github.com/LSTS/neptus/blob/develop/LICENSE.md + * and http://ec.europa.eu/idabc/eupl.html. + * + * For more information please see . + * + * Author: Paulo Dias + * 18/11/2023 + */ +package pt.lsts.neptus.plugins.videoreader; + +import io.humble.video.Decoder; +import io.humble.video.Demuxer; +import io.humble.video.DemuxerStream; +import io.humble.video.Global; +import io.humble.video.MediaDescriptor; +import io.humble.video.MediaPacket; +import io.humble.video.MediaPicture; +import io.humble.video.Rational; +import io.humble.video.awt.ImageFrame; +import io.humble.video.awt.MediaPictureConverter; +import io.humble.video.awt.MediaPictureConverterFactory; +import pt.lsts.neptus.NeptusLog; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.function.Function; + +class Player { + private ExecutorService service; + private final String id; + private Function updateImageFrameFunction; + private Demuxer demuxer; + private int numStreams = -1; + private int videoStreamId = -1; + private long streamStartTime = Global.NO_PTS; + private Decoder videoDecoder = null; + + private ImageFrame.ImageComponent mOnscreenPicture; + + private String url; + + private boolean streamingActive = false; + private boolean streamingFinished = false; + private boolean stopRequest = false; + + public Player(String id, ExecutorService service) { + this.service = service; + this.id = id; + } + + public boolean isStreamingActive() { + return streamingActive; + } + + public boolean isStreamingFinished() { + return streamingFinished; + } + + public boolean isStopRequest() { + return stopRequest; + } + + public void setStopRequest() { + this.stopRequest = true; + updateImageFrameFunction = null; + } + + public boolean start(String url, Function updateImageFrameFunction) throws IOException, InterruptedException { + this.updateImageFrameFunction = updateImageFrameFunction; + this.url = url; + + NeptusLog.pub().warn("Connecting " + ":" + id + ":" + " to " + url); + + demuxer = Demuxer.make(); + demuxer.setReadRetryCount(3); + demuxer.open(url, null, false, true, null, null); + + int numStreams = demuxer.getNumStreams(); + int videoStreamId = -1; + long streamStartTime = Global.NO_PTS; + Decoder videoDecoder = null; + for(int i = 0; i < numStreams; i++) { + final DemuxerStream stream = demuxer.getStream(i); + streamStartTime = stream.getStartTime(); + final Decoder decoder = stream.getDecoder(); + if (decoder != null && decoder.getCodecType() == MediaDescriptor.Type.MEDIA_VIDEO) { + videoStreamId = i; + videoDecoder = decoder; + // stop at the first one. + break; + } + } + if (videoStreamId == -1) { + throw new RuntimeException("could not find video stream in container " + ":" + id + ":" + ": " + url); + } + + videoDecoder.open(null, null); + + final MediaPicture picture = MediaPicture.make( + videoDecoder.getWidth(), + videoDecoder.getHeight(), + videoDecoder.getPixelFormat()); + + /* A converter object we'll use to convert the picture in the video to a BGR_24 format that Java Swing + can work with. You can still access the data directly in the MediaPicture if you prefer, but this + abstracts away from this demo most of that byte-conversion work. Go read the source code for the + converters if you're a glutton for punishment. + */ + final MediaPictureConverter converter = + MediaPictureConverterFactory.createConverter( + MediaPictureConverterFactory.HUMBLE_BGR_24, + picture); + + // TODO + + // Calculate the time BEFORE we start playing. + long systemStartTime = System.nanoTime(); + // Set units for the system time, which because we used System.nanoTime will be in nanoseconds. + final Rational systemTimeBase = Rational.make(1, 1000000000); + // All the MediaPicture objects decoded from the videoDecoder will share this timebase. + final Rational streamTimebase = videoDecoder.getTimeBase(); + + startLoop(videoStreamId, videoDecoder, picture, streamStartTime, converter, systemStartTime, systemTimeBase, streamTimebase); + + return true; + } + + private void startLoop(int videoStreamId, Decoder videoDecoder, MediaPicture picture, + long streamStartTime, MediaPictureConverter converter, long systemStartTime, + Rational systemTimeBase, Rational streamTimebase) { + NeptusLog.pub().warn("Streaming from " + ":" + id + ":" + " " + url); + + final MediaPacket packet = MediaPacket.make(); + streamingActive = true; + service.execute(() -> { + try { + BufferedImage image = null; + while (demuxer.read(packet) >= 0 && !stopRequest) { + /* + * Now we have a packet, let's see if it belongs to our video stream + */ + if (packet.getStreamIndex() == videoStreamId) { + /* + * A packet can actually contain multiple sets of samples (or frames of samples + * in decoding speak). So, we may need to call decode multiple + * times at different offsets in the packet's data. We capture that here. + */ + int offset = 0; + int bytesRead = 0; + do { + bytesRead += videoDecoder.decode(picture, packet, offset); + if (picture.isComplete()) { +// image = displayVideoAtCorrectTime(streamStartTime, picture, +// converter, image, window, systemStartTime, systemTimeBase, +// streamTimebase); + image = adjustToImage(picture, converter, image); + if (updateImageFrameFunction != null) { + updateImageFrameFunction.apply(image); + } + } + offset += bytesRead; + } while (offset < packet.getSize() && !stopRequest); + } + } + + NeptusLog.pub().warn("Stopping" + ":" + id + ":" +" streaming from " + url); + // Some video decoders (especially advanced ones) will cache + // video data before they begin decoding, so when you are done you need + // to flush them. The convention to flush Encoders or Decoders in Humble Video + // is to keep passing in null until incomplete samples or packets are returned. + do { + videoDecoder.decode(picture, null, 0); + if (picture.isComplete()) { +// image = displayVideoAtCorrectTime(streamStartTime, picture, converter, +// image, window, systemStartTime, systemTimeBase, streamTimebase); + image = adjustToImage(picture, converter, image); + if (updateImageFrameFunction != null) { + updateImageFrameFunction.apply(image); + } + } + } while (picture.isComplete()); + } catch (Exception e) { + NeptusLog.pub().error(e.getMessage()); + } finally { + streamingActive = false; + // It is good practice to close demuxers when you're done to free + // up file handles. Humble will EVENTUALLY detect if nothing else + // references this demuxer and close it then, but get in the habit + // of cleaning up after yourself, and your future girlfriend/boyfriend + // will appreciate it. + try { + demuxer.close(); + } + catch (Exception e) { + NeptusLog.pub().error(e.getMessage()); + } + streamingFinished = true; + } + NeptusLog.pub().warn("Streaming " + ":" + id + ":" + " stopped from " + url); + }); + } + + /** + * Takes the video picture and displays it at the right time. + */ + private static BufferedImage displayVideoAtCorrectTime(long streamStartTime, + final MediaPicture picture, final MediaPictureConverter converter, + BufferedImage image, final ImageFrame window, long systemStartTime, + final Rational systemTimeBase, final Rational streamTimebase) + throws InterruptedException { + long streamTimestamp = picture.getTimeStamp(); + // convert streamTimestamp into system units (i.e. nano-seconds) + streamTimestamp = systemTimeBase.rescale(streamTimestamp-streamStartTime, streamTimebase); + // get the current clock time, with our most accurate clock + long systemTimestamp = System.nanoTime(); + // loop in a sleeping loop until we're within 1 ms of the time for that video frame. + // a real video player needs to be much more sophisticated than this. + while (streamTimestamp > (systemTimestamp - systemStartTime + 1000000)) { + Thread.sleep(1); + systemTimestamp = System.nanoTime(); + } + // finally, convert the image from Humble format into Java images. + image = converter.toImage(image, picture); + + // And ask the UI thread to repaint with the new image. + window.setImage(image); + return image; + } + + private static BufferedImage adjustToImage(final MediaPicture picture, + final MediaPictureConverter converter, BufferedImage image) + throws InterruptedException { + // finally, convert the image from Humble format into Java images. + image = converter.toImage(image, picture); + return image; + } +} diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Util.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Util.java new file mode 100644 index 0000000000..d6dc1960c5 --- /dev/null +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Util.java @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2004-2023 Universidade do Porto - Faculdade de Engenharia + * Laboratório de Sistemas e Tecnologia Subaquática (LSTS) + * All rights reserved. + * Rua Dr. Roberto Frias s/n, sala I203, 4200-465 Porto, Portugal + * + * This file is part of Neptus, Command and Control Framework. + * + * Commercial Licence Usage + * Licencees holding valid commercial Neptus licences may use this file + * in accordance with the commercial licence agreement provided with the + * Software or, alternatively, in accordance with the terms contained in a + * written agreement between you and Universidade do Porto. For licensing + * terms, conditions, and further information contact lsts@fe.up.pt. + * + * Modified European Union Public Licence - EUPL v.1.1 Usage + * Alternatively, this file may be used under the terms of the Modified EUPL, + * Version 1.1 only (the "Licence"), appearing in the file LICENSE.md + * included in the packaging of this file. You may not use this work + * except in compliance with the Licence. Unless required by applicable + * law or agreed to in writing, software distributed under the Licence is + * distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF + * ANY KIND, either express or implied. See the Licence for the specific + * language governing permissions and limitations at + * https://github.com/LSTS/neptus/blob/develop/LICENSE.md + * and http://ec.europa.eu/idabc/eupl.html. + * + * For more information please see . + * + * Author: Pedro Gonçalves + */ +package pt.lsts.neptus.plugins.videoreader; + +import org.apache.commons.io.FileUtils; +import pt.lsts.neptus.NeptusLog; +import pt.lsts.neptus.util.FileUtil; +import pt.lsts.neptus.util.conf.ConfigFetch; + +import java.awt.Dimension; +import java.awt.Graphics2D; +import java.awt.Image; +import java.awt.image.BufferedImage; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.concurrent.TimeUnit; + +/** + * @author pedrog + * @author Pedro Costa + * @version 1.0 + */ +public class Util { + private Util() { + } + + public static String getHostFromURI(String camUrl) { + try { + URI uri = new URI(camUrl); + if (uri == null) return null; + + return uri.getHost(); + } + catch (Exception e) { + NeptusLog.pub().warn("Camera URL is not valid: " + camUrl + " :: " + e.getMessage()); + } + return null; + } + + public static URI getCamUrlAsURI(String camUrl) { + try { + URI uri = new URI(camUrl); + return uri; + } + catch (Exception e) { + NeptusLog.pub().warn("Camera URL is not valid: " + camUrl + " :: " + e.getMessage()); + } + return null; + } + + public static ArrayList readIpUrl(File nameFile) { + ArrayList cameraList = new ArrayList<>(); + BufferedReader br = null; + String line; + try { + br = new BufferedReader(new FileReader(nameFile)); + } + catch (FileNotFoundException e) { + e.printStackTrace(); + } + cameraList.add(new Camera()); + + String[] splits; + try { + while ((line = br.readLine()) != null) { + Camera cam = parseLineCamera(line); + if (cam != null) { + cameraList.add(cam); + } + } + } + catch (IOException e) { + e.printStackTrace(); + } + + try { + br.close(); + } + catch (IOException e) { + e.printStackTrace(); + } + return cameraList; + } + + static Camera parseLineCamera(String line) { + if (line.isEmpty() || line.trim().startsWith("#")) return null; + + String[] splits = line.split("#"); + if (splits.length == 3) { + if (splits[0].trim().isEmpty()) return null; + if (splits[1].trim().isEmpty()) return null; + if (splits[2].trim().isEmpty()) return null; + if (getHostFromURI(splits[2].trim().trim()) == null) return null; + + return new Camera(splits[0], splits[1], splits[2]); + } + else if (splits.length == 2) { + if (splits[0].trim().isEmpty()) return null; + if (splits[1].trim().isEmpty()) return null; + String host = getHostFromURI(splits[1].trim().trim()); + if (host == null) return null; + + return new Camera(splits[0], host, splits[1]); + } + + return null; + } + + public static void removeCamFromFile(Camera camToRemove, String fileName) { + File confIni = new File(fileName); + File tempFile = null; + try { + tempFile = File.createTempFile("neptus_", "tmp", new File(ConfigFetch.getNeptusTmpDir())); + tempFile.deleteOnExit(); + } + catch (IOException e) { + e.printStackTrace(); + return; + } + + String currentLine; + try (BufferedReader reader = new BufferedReader(new FileReader(confIni)); + BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile))) { + while ((currentLine = reader.readLine()) != null) { + boolean writeLine = false; + + if (currentLine.isEmpty() || currentLine.trim().startsWith("#")) { + writeLine = true; + } + + if (writeLine == false) { + Camera cam = parseLineCamera(currentLine.trim()); + if (cam != null) { + if (!cam.getName().equalsIgnoreCase(camToRemove.getName())) { + writeLine = true; + } + } + } + + if (writeLine) { + writer.write(currentLine.trim() + System.getProperty("line.separator")); + } + } + } + catch (IOException e) { + e.printStackTrace(); + } + + try { + FileUtils.copyFile(tempFile, confIni); + } + catch (IOException e) { + e.printStackTrace(); + } + } + + public static void addCamToFile(Camera camToAdd, String fileName) { + String iniRsrcPath = FileUtil.getResourceAsFileKeepName(VideoReader.BASE_FOLDER_FOR_URL_INI); + + File confIni = new File(fileName); + if (!confIni.exists()) { + FileUtil.copyFileToDir(iniRsrcPath, ConfigFetch.getConfFolder()); + } + + File tempFile = null; + try { + tempFile = File.createTempFile("neptus_", "tmp", new File(ConfigFetch.getNeptusTmpDir())); + tempFile.deleteOnExit(); + } + catch (IOException e) { + e.printStackTrace(); + return; + } + + ArrayList camsList = readIpUrl(confIni); + boolean updateValueLine = camsList.stream().anyMatch(c -> camToAdd.getName().equalsIgnoreCase(c.getName())); + + if (!updateValueLine) { + String str = String.format("%s#%s\n", camToAdd.getName().trim(), camToAdd.getUrl().trim()); + FileUtil.saveToFile(confIni.getAbsolutePath(), str, "UTF-8", true); + return; + } + + String currentLine; + try (BufferedReader reader = new BufferedReader(new FileReader(confIni)); + BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile))) { + while ((currentLine = reader.readLine()) != null) { + boolean writeLine = false; + + if (currentLine.isEmpty() || currentLine.trim().startsWith("#")) { + writeLine = true; + } + + if (writeLine == false) { + Camera cam = parseLineCamera(currentLine.trim()); + if (cam != null) { + if (cam.getName().equalsIgnoreCase(camToAdd.getName())) { + String str = String.format("%s#%s\n", camToAdd.getName().trim(), camToAdd.getUrl().trim()); + writer.write(str); + } + else { + writeLine = true; + } + } + } + + if (writeLine) { + writer.write(currentLine.trim() + System.getProperty("line.separator")); + } + } + } + catch (IOException e) { + e.printStackTrace(); + } + + try { + FileUtils.copyFile(tempFile, confIni); + } + catch (IOException e) { + e.printStackTrace(); + } + } + + /* + * Checks if a given host is reachable using ping + * @param host + * The ip to check + * @return boolean: true if host is reachable + */ + public static boolean hostIsReachable(String host) { + InetAddress address; + try { + address = InetAddress.getByName(host); + } + catch (UnknownHostException e) { + return false; + } + + // Host must be reachable 3 times to count as reachable + int tries = 3; + boolean isReachable; + while(tries-- > 0) { + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("mm:ss:SSSS"); + LocalDateTime now = LocalDateTime.now(); + try { + isReachable = address.isReachable(1000); + if(!isReachable) + return false; + // Avoid spamming + TimeUnit.MILLISECONDS.sleep(100); + } + catch (Exception e) { + return false; + } + } + + return true; + } + + public static BufferedImage resizeBufferedImage(BufferedImage img, Dimension size) { + if (size != null && size.width != 0 && size.height != 0) { + BufferedImage dimg = new BufferedImage((int) size.width, (int) size.height, img.getType()); + Graphics2D g2d = dimg.createGraphics(); + g2d.drawImage(img.getScaledInstance((int) size.width, (int) size.height, Image.SCALE_SMOOTH), 0, 0, null); + g2d.dispose(); + return dimg; + } + else { + //NeptusLog.pub().warn(I18n.text("Size in resizeBufferedImage must be != NULL and not 0")); + return null; + } + } + + static synchronized void createIpUrlFile() { + String iniRsrcPath = FileUtil.getResourceAsFileKeepName(VideoReader.BASE_FOLDER_FOR_URL_INI); + File confIni = new File(ConfigFetch.getConfFolder() + "/" + VideoReader.BASE_FOLDER_FOR_URL_INI); + if (!confIni.exists()) { + FileUtil.copyFileToDir(iniRsrcPath, ConfigFetch.getConfFolder()); + } + } +} diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java new file mode 100644 index 0000000000..58c7a1e53e --- /dev/null +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2004-2023 Universidade do Porto - Faculdade de Engenharia + * Laboratório de Sistemas e Tecnologia Subaquática (LSTS) + * All rights reserved. + * Rua Dr. Roberto Frias s/n, sala I203, 4200-465 Porto, Portugal + * + * This file is part of Neptus, Command and Control Framework. + * + * Commercial Licence Usage + * Licencees holding valid commercial Neptus licences may use this file + * in accordance with the commercial licence agreement provided with the + * Software or, alternatively, in accordance with the terms contained in a + * written agreement between you and Universidade do Porto. For licensing + * terms, conditions, and further information contact lsts@fe.up.pt. + * + * Modified European Union Public Licence - EUPL v.1.1 Usage + * Alternatively, this file may be used under the terms of the Modified EUPL, + * Version 1.1 only (the "Licence"), appearing in the file LICENSE.md + * included in the packaging of this file. You may not use this work + * except in compliance with the Licence. Unless required by applicable + * law or agreed to in writing, software distributed under the Licence is + * distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF + * ANY KIND, either express or implied. See the Licence for the specific + * language governing permissions and limitations at + * https://github.com/LSTS/neptus/blob/develop/LICENSE.md + * and http://ec.europa.eu/idabc/eupl.html. + * + * For more information please see . + * + * Author: Paulo Dias + * 18/11/2023 + */ +package pt.lsts.neptus.plugins.videoreader; + +import pt.lsts.neptus.NeptusLog; +import pt.lsts.neptus.console.ConsoleLayout; +import pt.lsts.neptus.console.ConsolePanel; +import pt.lsts.neptus.i18n.I18n; +import pt.lsts.neptus.mp.preview.payloads.CameraFOV; +import pt.lsts.neptus.plugins.NeptusProperty; +import pt.lsts.neptus.plugins.PluginDescription; +import pt.lsts.neptus.plugins.Popup; +import pt.lsts.neptus.plugins.update.Periodic; +import pt.lsts.neptus.types.coord.LocationType; +import pt.lsts.neptus.util.ImageUtils; + +import javax.swing.JDialog; +import javax.swing.JLabel; +import javax.swing.JMenuItem; +import javax.swing.JPopupMenu; +import javax.swing.KeyStroke; +import javax.swing.SwingConstants; +import javax.swing.SwingUtilities; +import java.awt.Color; +import java.awt.Component; +import java.awt.Dimension; +import java.awt.Graphics; +import java.awt.Image; +import java.awt.Toolkit; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.ComponentAdapter; +import java.awt.event.ComponentEvent; +import java.awt.event.InputEvent; +import java.awt.event.KeyEvent; +import java.awt.event.MouseAdapter; +import java.awt.event.MouseEvent; +import java.awt.geom.Point2D; +import java.awt.geom.Rectangle2D; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.util.ArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +@PluginDescription(name = "Video Reader", version = "0.1", experimental = true, author = "Paulo Dias", + description = "Plugin to view IP Camera streams using FFMPEG", icon = "images/menus/camera.png", + category = PluginDescription.CATEGORY.INTERFACE) +@Popup(name = "Video Reader", width = 640, height = 480, icon = "images/menus/camera.png") +public class VideoReader extends ConsolePanel { + static final String BASE_FOLDER_FOR_URL_INI = "ipUrl.ini"; + + private static final int DEFAULT_WIDTH_CONSOLE = 640; + private static final int DEFAULT_HEIGHT_CONSOLE = 480; + + private static final int MAX_NULL_FRAMES_FOR_RECONNECT = 10; + public static final String IMAGE_NO_VIDEO = "images/novideo.png"; + + private final Color LABEL_WHITE_COLOR = new Color(255, 255, 255, 200); + + final ExecutorService service = Executors.newCachedThreadPool(new ThreadFactory() { + private final String namePrefix = VideoReader.class.getSimpleName() + "::" + + Integer.toHexString(VideoReader.this.hashCode()); + private final AtomicInteger counter = new AtomicInteger(0); + private final ThreadGroup group = new ThreadGroup(namePrefix); + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r); + t.setName(VideoReader.class.getCanonicalName() + " " + (counter.getAndIncrement())); + t.setDaemon(true); + return t; + } + }); + + @NeptusProperty(name = "Camera URL", editable = false) + private String camUrl = "rtsp://127.0.0.1:8554/"; //rtsp://10.0.20.207:554/live/ch01_0 + + private AtomicInteger emptyFramesCounter = new AtomicInteger(0); + private AtomicInteger threadsIdCounter = new AtomicInteger(0); + + + private Player player; + + private BufferedImage offlineImage; + private BufferedImage onScreenImage; + private BufferedImage onScreenImageLastGood; + + private int widthImgRec; + // Height size of image + private int heightImgRec; + // Width size of Console + private int widthConsole = DEFAULT_WIDTH_CONSOLE; + // Height size of Console + private int heightConsole = DEFAULT_HEIGHT_CONSOLE; + // Scale factor of x pixel + private float xScale; + // Scale factor of y pixel + private float yScale; + private CameraFOV camFov = null; + private Point2D mouseLoc = null; + + private ArrayList cameraList; + private boolean closingPanel = false; + + private boolean refreshTemp; + private boolean paused = false; + + // JPopup Menu + private JPopupMenu popup; + private IpCamManagementPanel ipCamManagementPanel; + // JTextField for IPCam name + private JLabel streamNameJLabel; + private JLabel streamWarnJLabel; + + public VideoReader(ConsoleLayout console) { + this(console, false); + } + + public VideoReader(ConsoleLayout console, boolean usedInsideAnotherConsolePanel) { + super(console, usedInsideAnotherConsolePanel); + + removeAll(); + this.addComponentListener(new ComponentAdapter() { + @Override + public void componentResized(ComponentEvent evt) { + updateSize(evt); + } + + @Override + public void componentShown(ComponentEvent evt) { + updateSize(evt); + } + + private void updateSize(ComponentEvent evt) { + Component c = evt.getComponent(); + updateSizeVariables(c); + if (isDisconnect()) { + setupNoVideoImage(); + } + } + }); + + this.setToolTipText(I18n.text("not connected")); + + ipCamManagementPanel = new IpCamManagementPanel(this, s -> { + camUrl = s; + return null; + }, + this::connectStream); + + // Mouse click + mouseListenerInit(); + + streamNameJLabel = new JLabel(); + streamNameJLabel.setForeground(LABEL_WHITE_COLOR); + streamNameJLabel.setBackground(new Color(0, 0, 0, 80)); + streamNameJLabel.setOpaque(true); + streamNameJLabel.setHorizontalAlignment(SwingConstants.CENTER); + streamNameJLabel.setVerticalAlignment(SwingConstants.TOP); + streamNameJLabel.setVerticalTextPosition(SwingConstants.TOP); + + streamWarnJLabel = new JLabel(); + streamWarnJLabel.setForeground(LABEL_WHITE_COLOR); + streamWarnJLabel.setOpaque(false); + streamWarnJLabel.setHorizontalAlignment(SwingConstants.CENTER); + streamWarnJLabel.setVerticalAlignment(SwingConstants.BOTTOM); + streamWarnJLabel.setVerticalTextPosition(SwingConstants.BOTTOM); + } + + @Override + public void initSubPanel() { + service.execute(Util::createIpUrlFile); + //setMainVehicle(getConsole().getMainSystem()); + } + + @Override + public void cleanSubPanel() { + closingPanel = true; + service.shutdown(); + disconnectStream(); + } + + @Override + protected void paintComponent(Graphics g) { + boolean warn = false; + if (refreshTemp && onScreenImage != null) { + g.drawImage(onScreenImage, 0, 0, this); + refreshTemp = false; + } + else if (onScreenImageLastGood != null && (onScreenImageLastGood.getWidth() == widthConsole + && onScreenImageLastGood.getHeight() == heightConsole)) { + g.drawImage(onScreenImageLastGood, 0, 0, this); + warn = true; + } + else { + NeptusLog.pub().warn("<<<<<<<<<<<<< Black Image >>>>>>>>>>>>>"); + System.out.println("<<<<<<<<<<<<< Black Image >>>>>>>>>>>>>"); + g.setColor(Color.BLACK); + g.fillRect(0, 0, (int) widthConsole, (int) heightConsole); + } + + if (isConnect() || isConnecting() || isDisconnecting()) { + String text = ipCamManagementPanel.getStreamName(); + Rectangle2D bounds = g.getFontMetrics().getStringBounds(text, g); + streamNameJLabel.setText(text); + streamNameJLabel.setSize((int) widthConsole, (int) bounds.getHeight() + 5); + streamNameJLabel.paint(g); + + if (warn) { + String textWarn = "⚠"; + streamWarnJLabel.setText(textWarn); + streamWarnJLabel.setSize((int) widthConsole, (int) heightConsole); + streamWarnJLabel.paint(g); + } + } + } + + private boolean isConnect() { + return player != null && player.isStreamingActive() && !player.isStopRequest(); + } + + private boolean isConnecting() { + return player != null && !player.isStreamingActive() && !player.isStopRequest() + && !player.isStreamingFinished(); + } + + private boolean isDisconnecting() { + return player != null && player.isStreamingActive() && player.isStopRequest(); + } + + private boolean isDisconnect() { + return player == null || player.isStreamingFinished() || player.isStopRequest(); + } + + private void setupNoVideoImage() { + Image noVideoImage = ImageUtils.getImage(IMAGE_NO_VIDEO); + if (noVideoImage == null) { + BufferedImage blackImage = ImageUtils.createCompatibleImage(1, 1, 255); + blackImage.setRGB(0, 0, 0); + } + noVideoImage = noVideoImage != null && noVideoImage.getWidth(null) > 0 + && noVideoImage.getHeight(null) > 0 + && widthConsole >= 0 && heightConsole >= 0 + //? ImageUtils.getScaledImage(noVideoImage, widthConsole, heightConsole, true) + ? Util.resizeBufferedImage(ImageUtils.toBufferedImage(ImageUtils.getImage("images/novideo.png")), new Dimension(widthConsole, heightConsole)) + : noVideoImage; + + BufferedImage onScreenImage = noVideoImage == null + ? null + : ImageUtils.toBufferedImage(noVideoImage); + showImage(onScreenImage); + } + + private void connectStream() { + if (isConnect() || isConnecting()) { + disconnectStream(); + } + + player = new Player(String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()), service); + try { + player.start(camUrl /*fieldUrl.getText()*/, image -> { + BufferedImage scaledImage = ImageUtils.toBufferedImage(ImageUtils.getFastScaledImage(image, widthConsole, heightConsole, true)); + showImage(scaledImage); + return null; + }); + } + catch (IOException | InterruptedException | RuntimeException e) { + player.setStopRequest(); + player = null; + } + + repaint(100); + } + + private void disconnectStream() { + if (player == null) { + return; + } + + System.out.println("disconnectStream"); + + Player playerToDisconnect = player; + player = null; + playerToDisconnect.setStopRequest(); + + onScreenImageLastGood = null; + setupNoVideoImage(); + repaint(100); + } + + private void showImage(BufferedImage image) { + if (!paused) { + if (onScreenImage != null) { + onScreenImageLastGood = onScreenImage; + } + + onScreenImage = image; + } + refreshTemp = true; + repaint(); + } + + @Periodic(millisBetweenUpdates = 1_000) + public void updateToolTip() { + String tooltipText = I18n.text("not connected"); + if (isConnecting()) { + tooltipText = I18n.text("connecting to") + " " + ipCamManagementPanel.getStreamName(); + } + else if (isConnect()) { + tooltipText = I18n.text("streaming from") + " " + ipCamManagementPanel.getStreamName(); + } + else if (isDisconnecting()) { + tooltipText = I18n.text("disconnecting from") + " " + ipCamManagementPanel.getStreamName(); + } + this.setToolTipText(I18n.text(tooltipText)); + + if (isDisconnect()) { + onScreenImageLastGood = null; + setupNoVideoImage(); + } + repaint(500); + } + + private void updateSizeVariables(Component comp) { + widthConsole = comp.getSize().width; + heightConsole = comp.getSize().height; + xScale = (float) widthConsole / widthImgRec; + yScale = (float) heightConsole / heightImgRec; + //size = new Size(widthConsole, heightConsole); + } + + // Mouse click Listener + private void mouseListenerInit() { + addMouseMotionListener(new MouseAdapter() { + @Override + public void mouseMoved(MouseEvent e) { + if (camFov != null) { + double width = ((Component) e.getSource()).getWidth(); + double height = ((Component) e.getSource()).getHeight(); + double x = e.getX(); + double y = height - e.getY(); + mouseLoc = new Point2D.Double((x / width - 0.5) * 2, (y / height - 0.5) * 2); + } + } + }); + + addMouseListener(new MouseAdapter() { + @Override + public void mouseExited(MouseEvent e) { + mouseLoc = null; + //post(new EventMouseLookAt(null)); + } + + public void mouseClicked(MouseEvent e) { + if (SwingUtilities.isLeftMouseButton(e) && e.isControlDown()) { + if (camFov != null) { + double width = ((Component) e.getSource()).getWidth(); + double height = ((Component) e.getSource()).getHeight(); + double x = e.getX(); + double y = height - e.getY(); + mouseLoc = new Point2D.Double((x / width - 0.5) * 2, (y / height - 0.5) * 2); + LocationType loc = camFov.getLookAt(mouseLoc.getX(), mouseLoc.getY()); + loc.convertToAbsoluteLatLonDepth(); + //String id = placeLocationOnMap(loc); +// snap = new StoredSnapshot(id, loc, e.getPoint(), onScreenImage, new Date()); +// snap.setCamFov(camFov); +// try { +// snap.store(); +// } +// catch (Exception ex) { +// NeptusLog.pub().error(ex); +// } + } + } + + if (e.getButton() == MouseEvent.BUTTON3) { + popup = new JPopupMenu(); + JMenuItem item; + + popup.add(item = new JMenuItem(I18n.text("Connect to a IPCam"), + ImageUtils.createImageIcon("images/menus/camera.png"))) + .addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + openIPCamManagementPanel(); + //service.execute(VideoReader.this::connectStream); + } + }); + item.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_I, InputEvent.ALT_MASK)); + + popup.add(item = new JMenuItem(I18n.text("Close connection"), + ImageUtils.createImageIcon("images/menus/exit.png"))) + .addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + NeptusLog.pub().info("Closing video stream"); + service.execute(VideoReader.this::disconnectStream); +// noVideoLogoState = false; +// isCleanTurnOffCam = true; +// state = false; +// ipCam = false; +// closeCapture(capture); + repaint(500); + } + }); + item.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_X, InputEvent.ALT_MASK)); + + popup.addSeparator(); + + popup.add(item = new JMenuItem(I18n.text("Maximize window"), + ImageUtils.createImageIcon("images/menus/maximize.png"))) + .addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + maximizeVideoStreamPanel(); + } + }); + item.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_F, InputEvent.ALT_MASK)); + + popup.addSeparator(); + +// JLabel infoZoom = new JLabel(I18n.text("For zoom use Alt-Z")); +// infoZoom.setEnabled(false); +// popup.add(infoZoom, JMenuItem.CENTER_ALIGNMENT); + + JLabel markSnap = new JLabel(I18n.text("Ctr+Click to mark frame in the map")); + markSnap.setEnabled(false); + popup.add(markSnap, JMenuItem.CENTER_ALIGNMENT); + + popup.show((Component) e.getSource(), e.getX(), e.getY()); + } + } + }); + } + + private void maximizeVideoStreamPanel() { + JDialog dialog = (JDialog) SwingUtilities.getWindowAncestor(VideoReader.this); + Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize().getSize(); + if (dialog.getSize().equals(screenSize)) { + // Already maximized + screenSize = new Dimension(DEFAULT_WIDTH_CONSOLE, DEFAULT_HEIGHT_CONSOLE); + } + dialog.setSize(screenSize); + // We call the resize with its own size to call componentResized + // method of the componentAdapter set in the constructor + VideoReader.this.setSize(VideoReader.this.getSize()); + } + + // Read ipUrl.ini to find IPCam ON + private void openIPCamManagementPanel() { + // JPanel for IPCam Select (MigLayout) + ipCamManagementPanel.show(camUrl); + } +} diff --git a/plugins-dev/videoreader/src/resources/ipUrl.ini b/plugins-dev/videoreader/src/resources/ipUrl.ini new file mode 100644 index 0000000000..ad7212159d --- /dev/null +++ b/plugins-dev/videoreader/src/resources/ipUrl.ini @@ -0,0 +1,9 @@ +SENS-11#rtsp://10.0.20.207:554/live/ch01_0 +Axis#rtsp://10.0.20.102:554/axis-media/media.amp?streamprofile=Mobile +Axis#rtsp://10.0.20.112:554/axis-media/media.amp?streamprofile=Mobile +IRCam##rtsp://10.0.20.113:554/axis-media/media.amp?streamprofile=LSTS +Foxcam#rtsp://usercam1:usercam1@10.0.10.46:88/videoMain +Foxcam#rtsp://usercam2:usercam2@10.0.10.47:88/videoMain +Crawler#rtsp://admin:roboplanet2022@10.0.200.211 +HikVision High#rtsp://admin:pwd4hik!@10.0.20.32/Streaming/Channels/101 +HikVision Low#rtsp://admin:pwd4hik!@10.0.20.32/Streaming/Channels/102 From cc1bca46af3e43946facf0695d51096e18f381e6 Mon Sep 17 00:00:00 2001 From: Paulo Dias Date: Sun, 19 Nov 2023 13:53:29 +0000 Subject: [PATCH 2/8] plugins/videoreader: Cleanup. --- .../lsts/neptus/plugins/videoreader/VideoReader.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java index 58c7a1e53e..01402898dd 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java @@ -226,8 +226,6 @@ else if (onScreenImageLastGood != null && (onScreenImageLastGood.getWidth() == w warn = true; } else { - NeptusLog.pub().warn("<<<<<<<<<<<<< Black Image >>>>>>>>>>>>>"); - System.out.println("<<<<<<<<<<<<< Black Image >>>>>>>>>>>>>"); g.setColor(Color.BLACK); g.fillRect(0, 0, (int) widthConsole, (int) heightConsole); } @@ -447,15 +445,15 @@ public void actionPerformed(ActionEvent e) { }); item.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_F, InputEvent.ALT_MASK)); - popup.addSeparator(); +// popup.addSeparator(); // JLabel infoZoom = new JLabel(I18n.text("For zoom use Alt-Z")); // infoZoom.setEnabled(false); // popup.add(infoZoom, JMenuItem.CENTER_ALIGNMENT); - JLabel markSnap = new JLabel(I18n.text("Ctr+Click to mark frame in the map")); - markSnap.setEnabled(false); - popup.add(markSnap, JMenuItem.CENTER_ALIGNMENT); +// JLabel markSnap = new JLabel(I18n.text("Ctr+Click to mark frame in the map")); +// markSnap.setEnabled(false); +// popup.add(markSnap, JMenuItem.CENTER_ALIGNMENT); popup.show((Component) e.getSource(), e.getX(), e.getY()); } From 0af306a56103c14973d516ba94f1cae0ad4271ea Mon Sep 17 00:00:00 2001 From: Paulo Dias Date: Mon, 20 Nov 2023 12:06:00 +0000 Subject: [PATCH 3/8] plugins/videoreader: Adding output for not supported codec. --- .../src/java/pt/lsts/neptus/plugins/videoreader/Player.java | 4 ++++ .../java/pt/lsts/neptus/plugins/videoreader/VideoReader.java | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Player.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Player.java index aa921daf8d..6a20e6dc76 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Player.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/Player.java @@ -73,6 +73,10 @@ public Player(String id, ExecutorService service) { this.id = id; } + public String getId() { + return id; + } + public boolean isStreamingActive() { return streamingActive; } diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java index 01402898dd..f090b6340c 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java @@ -35,10 +35,12 @@ import pt.lsts.neptus.NeptusLog; import pt.lsts.neptus.console.ConsoleLayout; import pt.lsts.neptus.console.ConsolePanel; +import pt.lsts.neptus.console.notifications.Notification; import pt.lsts.neptus.i18n.I18n; import pt.lsts.neptus.mp.preview.payloads.CameraFOV; import pt.lsts.neptus.plugins.NeptusProperty; import pt.lsts.neptus.plugins.PluginDescription; +import pt.lsts.neptus.plugins.PluginUtils; import pt.lsts.neptus.plugins.Popup; import pt.lsts.neptus.plugins.update.Periodic; import pt.lsts.neptus.types.coord.LocationType; @@ -296,6 +298,9 @@ private void connectStream() { }); } catch (IOException | InterruptedException | RuntimeException e) { + String error = player.getId() + " :: ERROR :: " + e.getMessage(); + NeptusLog.pub().error(error); + getConsole().post(Notification.warning(PluginUtils.getPluginName(this.getClass()), error)); player.setStopRequest(); player = null; } From 56e6244147aa688c1240fda5b0a98aeba75ad170 Mon Sep 17 00:00:00 2001 From: Paulo Dias Date: Tue, 21 Nov 2023 11:53:24 +0000 Subject: [PATCH 4/8] plugins/videoreader/PlayerOpenCv: Added opencv based player. --- .../videoreader/DecodeAndPlayVideo.java | 295 ---------------- .../plugins/videoreader/PlayerOpenCv.java | 314 ++++++++++++++++++ .../plugins/videoreader/VideoReader.java | 19 +- 3 files changed, 327 insertions(+), 301 deletions(-) delete mode 100644 plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/DecodeAndPlayVideo.java create mode 100644 plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/PlayerOpenCv.java diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/DecodeAndPlayVideo.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/DecodeAndPlayVideo.java deleted file mode 100644 index 61e452bc69..0000000000 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/DecodeAndPlayVideo.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright (c) 2004-2023 Universidade do Porto - Faculdade de Engenharia - * Laboratório de Sistemas e Tecnologia Subaquática (LSTS) - * All rights reserved. - * Rua Dr. Roberto Frias s/n, sala I203, 4200-465 Porto, Portugal - * - * This file is part of Neptus, Command and Control Framework. - * - * Commercial Licence Usage - * Licencees holding valid commercial Neptus licences may use this file - * in accordance with the commercial licence agreement provided with the - * Software or, alternatively, in accordance with the terms contained in a - * written agreement between you and Universidade do Porto. For licensing - * terms, conditions, and further information contact lsts@fe.up.pt. - * - * Modified European Union Public Licence - EUPL v.1.1 Usage - * Alternatively, this file may be used under the terms of the Modified EUPL, - * Version 1.1 only (the "Licence"), appearing in the file LICENSE.md - * included in the packaging of this file. You may not use this work - * except in compliance with the Licence. Unless required by applicable - * law or agreed to in writing, software distributed under the Licence is - * distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF - * ANY KIND, either express or implied. See the Licence for the specific - * language governing permissions and limitations at - * https://github.com/LSTS/neptus/blob/develop/LICENSE.md - * and http://ec.europa.eu/idabc/eupl.html. - * - * For more information please see . - * - * Author: Paulo Dias - * 18/11/2023 - */ -package pt.lsts.neptus.plugins.videoreader; - -import io.humble.video.Decoder; -import io.humble.video.Demuxer; -import io.humble.video.DemuxerStream; -import io.humble.video.Global; -import io.humble.video.Media; -import io.humble.video.MediaDescriptor; -import io.humble.video.MediaPacket; -import io.humble.video.MediaPicture; -import io.humble.video.Rational; -import io.humble.video.awt.ImageFrame; -import io.humble.video.awt.MediaPictureConverter; -import io.humble.video.awt.MediaPictureConverterFactory; - -import java.awt.image.BufferedImage; -import java.io.IOException; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; - -/** - * Opens a media file, finds the first video stream, and then plays it. - * This is meant as a demonstration program to teach the use of the Humble API. - *

- * Concepts introduced: - *

- *
    - *
  • MediaPicture: {@link MediaPicture} objects represent uncompressed video in Humble.
    • - *
  • Timestamps: All {@link Media} objects in Humble have a timestamp, and this demonstration introduces the concept of having to worry about when to display information.
    • - *
- * - *

- * To run from maven, do: - *

- * 
- * mvn install exec:java -Dexec.mainClass="io.humble.video.demos.DecodeAndPlayVideo" -Dexec.args="filename.mp4"
- *
- * - * @author aclarke - * - */ -public class DecodeAndPlayVideo { - /** - * Opens a file, and plays the video from it on a screen at the right rate. - * @param filename The file or URL to play. - */ - private static void playVideo(String filename) throws InterruptedException, IOException { - /* - * Start by creating a container object, in this case a demuxer since - * we are reading, to get video data from. - */ - Demuxer demuxer = Demuxer.make(); - - /* - * Open the demuxer with the filename passed on. - */ - demuxer.open(filename, null, false, true, null, null); - - /* - * Query how many streams the call to open found - */ - int numStreams = demuxer.getNumStreams(); - - /* - * Iterate through the streams to find the first video stream - */ - int videoStreamId = -1; - long streamStartTime = Global.NO_PTS; - Decoder videoDecoder = null; - for(int i = 0; i < numStreams; i++) - { - final DemuxerStream stream = demuxer.getStream(i); - streamStartTime = stream.getStartTime(); - final Decoder decoder = stream.getDecoder(); - if (decoder != null && decoder.getCodecType() == MediaDescriptor.Type.MEDIA_VIDEO) { - videoStreamId = i; - videoDecoder = decoder; - // stop at the first one. - break; - } - } - if (videoStreamId == -1) - throw new RuntimeException("could not find video stream in container: "+filename); - - /* - * Now we have found the audio stream in this file. Let's open up our decoder so it can - * do work. - */ - videoDecoder.open(null, null); - - final MediaPicture picture = MediaPicture.make( - videoDecoder.getWidth(), - videoDecoder.getHeight(), - videoDecoder.getPixelFormat()); - - /** A converter object we'll use to convert the picture in the video to a BGR_24 format that Java Swing - * can work with. You can still access the data directly in the MediaPicture if you prefer, but this - * abstracts away from this demo most of that byte-conversion work. Go read the source code for the - * converters if you're a glutton for punishment. - */ - final MediaPictureConverter converter = - MediaPictureConverterFactory.createConverter( - MediaPictureConverterFactory.HUMBLE_BGR_24, - picture); - BufferedImage image = null; - - /** - * This is the Window we will display in. See the code for this if you're curious, but to keep this demo clean - * we're 'simplifying' Java AWT UI updating code. This method just creates a single window on the UI thread, and blocks - * until it is displayed. - */ - final ImageFrame window = ImageFrame.make(); - if (window == null) { - throw new RuntimeException("Attempting this demo on a headless machine, and that will not work. Sad day for you."); - } - - /** - * Media playback, like comedy, is all about timing. Here we're going to introduce very very basic - * timing. This code is deliberately kept simple (i.e. doesn't worry about A/V drift, garbage collection pause time, etc.) - * because that will quickly make things more complicated. - * - * But the basic idea is there are two clocks: - *
    - *
  • Player Clock: The time that the player sees (relative to the system clock).
    • - *
  • Stream Clock: Each stream has its own clock, and the ticks are measured in units of time-bases
    • - *
- * - * And we need to convert between the two units of time. Each MediaPicture and MediaAudio object have associated - * time stamps, and much of the complexity in video players goes into making sure the right picture (or sound) is - * seen (or heard) at the right time. This is actually very tricky and many folks get it wrong -- watch enough - * Netflix and you'll see what I mean -- audio and video slightly out of sync. But for this demo, we're erring for - * 'simplicity' of code, not correctness. It is beyond the scope of this demo to make a full fledged video player. - */ - - // Calculate the time BEFORE we start playing. - long systemStartTime = System.nanoTime(); - // Set units for the system time, which because we used System.nanoTime will be in nanoseconds. - final Rational systemTimeBase = Rational.make(1, 1000000000); - // All the MediaPicture objects decoded from the videoDecoder will share this timebase. - final Rational streamTimebase = videoDecoder.getTimeBase(); - - /** - * Now, we start walking through the container looking at each packet. This - * is a decoding loop, and as you work with Humble you'll write a lot - * of these. - * - * Notice how in this loop we reuse all of our objects to avoid - * reallocating them. Each call to Humble resets objects to avoid - * unnecessary reallocation. - */ - final MediaPacket packet = MediaPacket.make(); - while(demuxer.read(packet) >= 0) { - /** - * Now we have a packet, let's see if it belongs to our video stream - */ - if (packet.getStreamIndex() == videoStreamId) - { - /** - * A packet can actually contain multiple sets of samples (or frames of samples - * in decoding speak). So, we may need to call decode multiple - * times at different offsets in the packet's data. We capture that here. - */ - int offset = 0; - int bytesRead = 0; - do { - bytesRead += videoDecoder.decode(picture, packet, offset); - if (picture.isComplete()) { - image = displayVideoAtCorrectTime(streamStartTime, picture, - converter, image, window, systemStartTime, systemTimeBase, - streamTimebase); - } - offset += bytesRead; - } while (offset < packet.getSize()); - } - } - - // Some video decoders (especially advanced ones) will cache - // video data before they begin decoding, so when you are done you need - // to flush them. The convention to flush Encoders or Decoders in Humble Video - // is to keep passing in null until incomplete samples or packets are returned. - do { - videoDecoder.decode(picture, null, 0); - if (picture.isComplete()) { - image = displayVideoAtCorrectTime(streamStartTime, picture, converter, - image, window, systemStartTime, systemTimeBase, streamTimebase); - } - } while (picture.isComplete()); - - // It is good practice to close demuxers when you're done to free - // up file handles. Humble will EVENTUALLY detect if nothing else - // references this demuxer and close it then, but get in the habit - // of cleaning up after yourself, and your future girlfriend/boyfriend - // will appreciate it. - demuxer.close(); - - // similar with the demuxer, for the windowing system, clean up after yourself. - window.dispose(); - } - - /** - * Takes the video picture and displays it at the right time. - */ - private static BufferedImage displayVideoAtCorrectTime(long streamStartTime, - final MediaPicture picture, final MediaPictureConverter converter, - BufferedImage image, final ImageFrame window, long systemStartTime, - final Rational systemTimeBase, final Rational streamTimebase) - throws InterruptedException { - long streamTimestamp = picture.getTimeStamp(); - // convert streamTimestamp into system units (i.e. nano-seconds) - streamTimestamp = systemTimeBase.rescale(streamTimestamp-streamStartTime, streamTimebase); - // get the current clock time, with our most accurate clock - long systemTimestamp = System.nanoTime(); - // loop in a sleeping loop until we're within 1 ms of the time for that video frame. - // a real video player needs to be much more sophisticated than this. - while (streamTimestamp > (systemTimestamp - systemStartTime + 1000000)) { - Thread.sleep(1); - systemTimestamp = System.nanoTime(); - } - // finally, convert the image from Humble format into Java images. - image = converter.toImage(image, picture); - // And ask the UI thread to repaint with the new image. - window.setImage(image); - return image; - } - - /** - * Takes a media container (file) as the first argument, opens it, - * opens up a window and plays back the video. - * - * @param args Must contain one string which represents a filename - * @throws IOException - * @throws InterruptedException - */ - public static void main(String[] args) throws InterruptedException, IOException - { - final Options options = new Options(); - options.addOption("h", "help", false, "displays help"); - options.addOption("v", "version", false, "version of this library"); - - final CommandLineParser parser = new org.apache.commons.cli.BasicParser(); - try { - final CommandLine cmd = parser.parse(options, args); - if (cmd.hasOption("version")) { - // let's find what version of the library we're running - final String version = io.humble.video_native.Version.getVersionInfo(); - System.out.println("Humble Version: " + version); - } else if (cmd.hasOption("help") || args.length == 0) { - final HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp(DecodeAndPlayVideo.class.getCanonicalName() + " ", options); - } else { - final String[] parsedArgs = cmd.getArgs(); - for(String arg: parsedArgs) - playVideo(arg); - } - } catch (ParseException e) { - System.err.println("Exception parsing command line: " + e.getLocalizedMessage()); - } - } -} \ No newline at end of file diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/PlayerOpenCv.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/PlayerOpenCv.java new file mode 100644 index 0000000000..b2db128dbb --- /dev/null +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/PlayerOpenCv.java @@ -0,0 +1,314 @@ +/* + * Copyright (c) 2004-2023 Universidade do Porto - Faculdade de Engenharia + * Laboratório de Sistemas e Tecnologia Subaquática (LSTS) + * All rights reserved. + * Rua Dr. Roberto Frias s/n, sala I203, 4200-465 Porto, Portugal + * + * This file is part of Neptus, Command and Control Framework. + * + * Commercial Licence Usage + * Licencees holding valid commercial Neptus licences may use this file + * in accordance with the commercial licence agreement provided with the + * Software or, alternatively, in accordance with the terms contained in a + * written agreement between you and Universidade do Porto. For licensing + * terms, conditions, and further information contact lsts@fe.up.pt. + * + * Modified European Union Public Licence - EUPL v.1.1 Usage + * Alternatively, this file may be used under the terms of the Modified EUPL, + * Version 1.1 only (the "Licence"), appearing in the file LICENSE.md + * included in the packaging of this file. You may not use this work + * except in compliance with the Licence. Unless required by applicable + * law or agreed to in writing, software distributed under the Licence is + * distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF + * ANY KIND, either express or implied. See the Licence for the specific + * language governing permissions and limitations at + * https://github.com/LSTS/neptus/blob/develop/LICENSE.md + * and http://ec.europa.eu/idabc/eupl.html. + * + * For more information please see . + * + * Author: Paulo Dias + * 18/11/2023 + */ +package pt.lsts.neptus.plugins.videoreader; + +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.Scalar; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; +import org.opencv.videoio.VideoCapture; +import pt.lsts.neptus.NeptusLog; +import pt.lsts.neptus.i18n.I18n; +import pt.lsts.neptus.util.SearchOpenCv; +import pt.lsts.neptus.util.UtilCv; + +import java.awt.Dimension; +import java.awt.image.BufferedImage; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; + +class PlayerOpenCv { + private static boolean tryToLoadOpenCVLibs = false; + private static boolean loadedOpenCVLibs = false; + + // Timeout for watchDogThread in milliseconds + private static final int WATCH_DOG_TIMEOUT_MILLIS = 4000; + private static final int WATCH_DOG_LOOP_THREAD_TIMEOUT_MILLIS = 10_000; + + private static final int MAX_NULL_FRAMES_FOR_RECONNECT = 10; + + private static final int DEFAULT_WIDTH_CONSOLE = 640; + private static final int DEFAULT_HEIGHT_CONSOLE = 480; + + private ExecutorService service; + private final String id; + private Function sizeChangeFunction; + private VideoCapture capture; + // Image resize + private Mat matResize; + // Image receive + private Mat mat; + + private String url; + private String infoSizeStream; + + private Function updateImageFrameFunction; + private AtomicInteger emptyFramesCounter = new AtomicInteger(0); + private AtomicInteger threadsIdCounter = new AtomicInteger(0); + + private Scalar black = new Scalar(0); + // Size of output frame + private Dimension size = null; + + // Width size of image + private int widthImgRec; + // Height size of image + private int heightImgRec; + // Width size of Console + private int widthConsole = DEFAULT_WIDTH_CONSOLE; + // Height size of Console + private int heightConsole = DEFAULT_HEIGHT_CONSOLE; + // flag for state of neptus logo + private boolean noVideoLogoState = false; + // Scale factor of x pixel + private float xScale; + // Scale factor of y pixel + private float yScale; + + private BufferedImage frameImage; + + // counter for frame tag ID + private short frameTagID = 1; + private AtomicLong captureLoopAtomicLongMillis = new AtomicLong(-1); + + private boolean histogramFlag = false; + + private boolean streamingActive = false; + private boolean streamingFinished = false; + private boolean stopRequest = false; + + public PlayerOpenCv(String id, ExecutorService service) { + this.service = service; + this.id = id; + + if (findOpenCV()) { + NeptusLog.pub().info(I18n.text("OpenCv-4.x.x found.")); + } + } + + void sizeChange(Dimension size) { + updateSizeVariables(size); + matResize = new Mat((int) size.height, (int) size.width, CvType.CV_8UC3); + } + + public boolean isHistogramFlag() { + return histogramFlag; + } + + public void setHistogramFlag(boolean histogramFlag) { + this.histogramFlag = histogramFlag; + } + + private void updateSizeVariables(Dimension size) { + widthConsole = size.width; + heightConsole = size.height; + xScale = (float) widthConsole / widthImgRec; + yScale = (float) heightConsole / heightImgRec; + this.size = size; + } + + private static boolean findOpenCV() { + if (tryToLoadOpenCVLibs) { + return loadedOpenCVLibs; + } + + tryToLoadOpenCVLibs = true; + loadedOpenCVLibs = SearchOpenCv.searchJni(); + return loadedOpenCVLibs; + } + + public String getId() { + return id; + } + + public boolean isStreamingActive() { + return streamingActive; + } + + public boolean isStreamingFinished() { + return streamingFinished; + } + + public boolean isStopRequest() { + return stopRequest; + } + + public void setStopRequest() { + this.stopRequest = true; + updateImageFrameFunction = null; + } + + public boolean start(String url, Function updateImageFrameFunction) throws Exception { + this.updateImageFrameFunction = updateImageFrameFunction; + this.url = url; + + NeptusLog.pub().warn("Connecting " + ":" + id + ":" + " to " + url); + + // just to initialize size vars + updateSizeVariables(new Dimension(widthConsole, heightConsole)); + + // Create Buffer (type MAT) for Image receive + mat = new Mat(heightImgRec, widthImgRec, CvType.CV_8UC3); + capture = new VideoCapture(); + capture.setExceptionMode(true); + + try { + NeptusLog.pub().info("Video Stream from IPCam capturing - tid::" + id); + boolean res = capture.open(url); + if (false && !res) { + if (capture.isOpened()) { + capture.release(); + } + + setStopRequest(); + streamingFinished = true; + return false; + } + } catch (Exception | Error e) { + NeptusLog.pub().error("Video Stream from IPCam open error - tid::" + id + + " :: " + e.getMessage()); + + setStopRequest(); + streamingFinished = true; + return false; + } + + if (capture != null && capture.isOpened()) { + streamingActive = true; + NeptusLog.pub().info("Video Stream from IPCam is captured - tid::" + id); + //startWatchDog(); + emptyFramesCounter.set(0); + } + + startLoop(); + + return true; + } + + private void startLoop() { + NeptusLog.pub().warn("Streaming from " + ":" + id + ":" + " " + url); + long startTime = System.currentTimeMillis(); + streamingActive = true; + + //resetWatchDog(4000); + + service.execute(() -> { + try { + BufferedImage image = null; + while (/*watchDog.isAlive() &&*/ streamingActive && capture != null && capture.isOpened() && !stopRequest) { + try { + boolean ret = capture.read(mat); + if (stopRequest) { + NeptusLog.pub().warn("Streaming exiting connection tid::" + id + " by stop request"); + break; + } + + if (ret) { + //resetWatchDog(4_000); + } else { + break; + } + } catch (Exception | Error e) { + NeptusLog.pub().debug(e.getMessage()); + break; + } + + long stopTime = System.currentTimeMillis(); + if ((stopTime - startTime) != 0) { + infoSizeStream = String.format("Size(%d x %d) | Scale(%.2f x %.2f) | FPS:%d |\t\t\t", + mat.cols(), mat.rows(), xScale, yScale, (int) (1000 / (stopTime - startTime))); + } + + if (mat.empty()) { + NeptusLog.pub().warn(I18n.text("ERROR capturing, empty img of IPCam - tid::" + id)); + //repaint(); + emptyFramesCounter.incrementAndGet(); + continue; + } + + emptyFramesCounter.set(0); + + xScale = (float) widthConsole / mat.cols(); + yScale = (float) heightConsole / mat.rows(); + Imgproc.resize(mat, matResize, new Size(widthConsole, heightConsole)); + // Convert Mat to BufferedImage + frameImage = UtilCv.matToBufferedImage(matResize); + // Display image in JFrame + if (histogramFlag) { +// if (saveSnapshot) { +// UtilCv.saveSnapshot(UtilCv.addText(UtilCv.histogramCv(frameImage), +// I18n.text("Histogram - On"), VideoReader.LABEL_WHITE_COLOR, +// frameImage.getWidth() - 5, 20), +// String.format(logDir + "/snapshotImage")); +// saveSnapshot = false; +// } + if (updateImageFrameFunction != null) { + updateImageFrameFunction.apply(UtilCv.addText(UtilCv.histogramCv(frameImage), + I18n.text("Histogram - On"), + VideoReader.LABEL_WHITE_COLOR, frameImage.getWidth() - 5, 20)); + } + } + else { + +// if (saveSnapshot) { +// UtilCv.saveSnapshot(offlineImage, +// String.format(logDir + "/snapshotImage")); +// saveSnapshot = false; +// } + if (updateImageFrameFunction != null) { + updateImageFrameFunction.apply(frameImage); + } + } + } + } catch (Exception e) { + NeptusLog.pub().error(e.getMessage()); + e.printStackTrace(); + } finally { + streamingActive = false; + try { + if (capture != null && capture.isOpened()) { + capture.release(); + } + } + catch (Exception e) { + NeptusLog.pub().error(e.getMessage()); + } + streamingFinished = true; + } + NeptusLog.pub().warn("Streaming " + ":" + id + ":" + " stopped from " + url); + }); + } +} diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java index f090b6340c..e91407a3c3 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java @@ -90,7 +90,7 @@ public class VideoReader extends ConsolePanel { private static final int MAX_NULL_FRAMES_FOR_RECONNECT = 10; public static final String IMAGE_NO_VIDEO = "images/novideo.png"; - private final Color LABEL_WHITE_COLOR = new Color(255, 255, 255, 200); + final static Color LABEL_WHITE_COLOR = new Color(255, 255, 255, 200); final ExecutorService service = Executors.newCachedThreadPool(new ThreadFactory() { private final String namePrefix = VideoReader.class.getSimpleName() + "::" @@ -114,7 +114,7 @@ public Thread newThread(Runnable r) { private AtomicInteger threadsIdCounter = new AtomicInteger(0); - private Player player; + private PlayerOpenCv player; private BufferedImage offlineImage; private BufferedImage onScreenImage; @@ -172,6 +172,10 @@ private void updateSize(ComponentEvent evt) { if (isDisconnect()) { setupNoVideoImage(); } + + if (player != null) { + player.sizeChange(c.getSize()); + } } }); @@ -289,15 +293,18 @@ private void connectStream() { disconnectStream(); } - player = new Player(String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()), service); + //player = new Player(String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()), service); + player = new PlayerOpenCv(String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()), service); + player.sizeChange(new Dimension(widthConsole, heightConsole)); try { player.start(camUrl /*fieldUrl.getText()*/, image -> { - BufferedImage scaledImage = ImageUtils.toBufferedImage(ImageUtils.getFastScaledImage(image, widthConsole, heightConsole, true)); + //BufferedImage scaledImage = ImageUtils.toBufferedImage(ImageUtils.getFastScaledImage(image, widthConsole, heightConsole, true)); + BufferedImage scaledImage = image; showImage(scaledImage); return null; }); } - catch (IOException | InterruptedException | RuntimeException e) { + catch (Exception | Error e) { String error = player.getId() + " :: ERROR :: " + e.getMessage(); NeptusLog.pub().error(error); getConsole().post(Notification.warning(PluginUtils.getPluginName(this.getClass()), error)); @@ -315,7 +322,7 @@ private void disconnectStream() { System.out.println("disconnectStream"); - Player playerToDisconnect = player; + PlayerOpenCv playerToDisconnect = player; player = null; playerToDisconnect.setStopRequest(); From 06afe78926723d42b9ea76a5871a75f032cccd3e Mon Sep 17 00:00:00 2001 From: Paulo Dias Date: Tue, 21 Nov 2023 12:32:41 +0000 Subject: [PATCH 5/8] plugins/videoreader: Histogram flag addition. --- .../neptus/plugins/videoreader/VideoReader.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java index e91407a3c3..630f96335e 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java @@ -120,6 +120,9 @@ public Thread newThread(Runnable r) { private BufferedImage onScreenImage; private BufferedImage onScreenImageLastGood; + // Flag for Histogram image + private boolean histogramFlag = false; + private int widthImgRec; // Height size of image private int heightImgRec; @@ -296,6 +299,7 @@ private void connectStream() { //player = new Player(String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()), service); player = new PlayerOpenCv(String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()), service); player.sizeChange(new Dimension(widthConsole, heightConsole)); + player.setHistogramFlag(histogramFlag); try { player.start(camUrl /*fieldUrl.getText()*/, image -> { //BufferedImage scaledImage = ImageUtils.toBufferedImage(ImageUtils.getFastScaledImage(image, widthConsole, heightConsole, true)); @@ -448,6 +452,18 @@ public void actionPerformed(ActionEvent e) { popup.addSeparator(); + popup.add(item = new JMenuItem(I18n.text("Toggle Histogram filter"), + ImageUtils.createImageIcon("images/menus/histogram.png"))) + .addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + histogramFlag = !histogramFlag; + if (player != null) { + player.setHistogramFlag(histogramFlag); + } + } + }); + item.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_H, InputEvent.ALT_MASK)); + popup.add(item = new JMenuItem(I18n.text("Maximize window"), ImageUtils.createImageIcon("images/menus/maximize.png"))) .addActionListener(new ActionListener() { From c9588826bb0a862b5a0ef551b27f0408fd13f05e Mon Sep 17 00:00:00 2001 From: Paulo Dias Date: Tue, 21 Nov 2023 12:32:51 +0000 Subject: [PATCH 6/8] plugins/videoreader: Cleanup --- .../videoreader/IpCamManagementPanel.java | 12 ++-- .../plugins/videoreader/VideoReader.java | 64 +++++++++---------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java index 52031a9f0d..115b97fe6f 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java @@ -60,11 +60,12 @@ import java.net.URI; import java.util.ArrayList; import java.util.concurrent.Callable; +import java.util.function.BiFunction; import java.util.function.Function; public class IpCamManagementPanel extends JPanel { private final VideoReader videoReader; - private final Function setValidUrlFunction; + private final BiFunction setValidUrlFunction; private final Runnable connectStreamFunction; private ArrayList cameraList; // JPanel for color state of ping to host IPCam @@ -84,7 +85,7 @@ public class IpCamManagementPanel extends JPanel { // JTextField for IPCam url private final JTextField fieldUrl = new JTextField(I18n.text("URL")); - public IpCamManagementPanel(VideoReader videoReader, Function setValidUrlFunction, + public IpCamManagementPanel(VideoReader videoReader, BiFunction setValidUrlFunction, Runnable connectStreamFunction) { this.videoReader = videoReader; this.setValidUrlFunction = setValidUrlFunction; @@ -172,8 +173,7 @@ public Object run() throws Exception { public void finish() { if (reachable) { selectIPCam.setEnabled(true); - //camUrl = selectedCamera.getUrl(); - setValidUrlFunction.apply(selectedCamera.getUrl()); + //setValidUrlFunction.apply(selectedCamera.getName(), selectedCamera.getUrl()); colorStateIPCam.setBackground(Color.GREEN); onOffIndicator.setText("ON"); ipCamList.setEnabled(true); @@ -213,9 +213,11 @@ public void finish() { public void actionPerformed(ActionEvent e) { NeptusLog.pub().info("IPCam Select: " + cameraList.get(selectedItemIndex)); ipCamPing.setVisible(false); + Camera selectedCamera = cameraList.get(selectedItemIndex); + setValidUrlFunction.apply(selectedCamera.getName(), selectedCamera.getUrl()); videoReader.service.execute(connectStreamFunction); } - }); + });dis fieldIP.setEditable(false); fieldIP.setFocusable(false); add(selectIPCam, "h 30!, wrap"); diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java index 630f96335e..8df6369583 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java @@ -37,13 +37,11 @@ import pt.lsts.neptus.console.ConsolePanel; import pt.lsts.neptus.console.notifications.Notification; import pt.lsts.neptus.i18n.I18n; -import pt.lsts.neptus.mp.preview.payloads.CameraFOV; import pt.lsts.neptus.plugins.NeptusProperty; import pt.lsts.neptus.plugins.PluginDescription; import pt.lsts.neptus.plugins.PluginUtils; import pt.lsts.neptus.plugins.Popup; import pt.lsts.neptus.plugins.update.Periodic; -import pt.lsts.neptus.types.coord.LocationType; import pt.lsts.neptus.util.ImageUtils; import javax.swing.JDialog; @@ -70,7 +68,6 @@ import java.awt.geom.Point2D; import java.awt.geom.Rectangle2D; import java.awt.image.BufferedImage; -import java.io.IOException; import java.util.ArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -108,9 +105,9 @@ public Thread newThread(Runnable r) { }); @NeptusProperty(name = "Camera URL", editable = false) - private String camUrl = "rtsp://127.0.0.1:8554/"; //rtsp://10.0.20.207:554/live/ch01_0 + private String camUrl = ""; //rtsp://10.0.20.207:554/live/ch01_0 + private String camName = ""; - private AtomicInteger emptyFramesCounter = new AtomicInteger(0); private AtomicInteger threadsIdCounter = new AtomicInteger(0); @@ -134,7 +131,7 @@ public Thread newThread(Runnable r) { private float xScale; // Scale factor of y pixel private float yScale; - private CameraFOV camFov = null; + //private CameraFOV camFov = null; private Point2D mouseLoc = null; private ArrayList cameraList; @@ -145,10 +142,10 @@ public Thread newThread(Runnable r) { // JPopup Menu private JPopupMenu popup; - private IpCamManagementPanel ipCamManagementPanel; + private final IpCamManagementPanel ipCamManagementPanel; // JTextField for IPCam name - private JLabel streamNameJLabel; - private JLabel streamWarnJLabel; + private final JLabel streamNameJLabel; + private final JLabel streamWarnJLabel; public VideoReader(ConsoleLayout console) { this(console, false); @@ -184,8 +181,9 @@ private void updateSize(ComponentEvent evt) { this.setToolTipText(I18n.text("not connected")); - ipCamManagementPanel = new IpCamManagementPanel(this, s -> { - camUrl = s; + ipCamManagementPanel = new IpCamManagementPanel(this, (name, url) -> { + camName = name; + camUrl = url; return null; }, this::connectStream); @@ -240,7 +238,7 @@ else if (onScreenImageLastGood != null && (onScreenImageLastGood.getWidth() == w } if (isConnect() || isConnecting() || isDisconnecting()) { - String text = ipCamManagementPanel.getStreamName(); + String text = camName; //ipCamManagementPanel.getStreamName(); Rectangle2D bounds = g.getFontMetrics().getStringBounds(text, g); streamNameJLabel.setText(text); streamNameJLabel.setSize((int) widthConsole, (int) bounds.getHeight() + 5); @@ -381,13 +379,13 @@ private void mouseListenerInit() { addMouseMotionListener(new MouseAdapter() { @Override public void mouseMoved(MouseEvent e) { - if (camFov != null) { - double width = ((Component) e.getSource()).getWidth(); - double height = ((Component) e.getSource()).getHeight(); - double x = e.getX(); - double y = height - e.getY(); - mouseLoc = new Point2D.Double((x / width - 0.5) * 2, (y / height - 0.5) * 2); - } +// if (camFov != null) { +// double width = ((Component) e.getSource()).getWidth(); +// double height = ((Component) e.getSource()).getHeight(); +// double x = e.getX(); +// double y = height - e.getY(); +// mouseLoc = new Point2D.Double((x / width - 0.5) * 2, (y / height - 0.5) * 2); +// } } }); @@ -399,16 +397,16 @@ public void mouseExited(MouseEvent e) { } public void mouseClicked(MouseEvent e) { - if (SwingUtilities.isLeftMouseButton(e) && e.isControlDown()) { - if (camFov != null) { - double width = ((Component) e.getSource()).getWidth(); - double height = ((Component) e.getSource()).getHeight(); - double x = e.getX(); - double y = height - e.getY(); - mouseLoc = new Point2D.Double((x / width - 0.5) * 2, (y / height - 0.5) * 2); - LocationType loc = camFov.getLookAt(mouseLoc.getX(), mouseLoc.getY()); - loc.convertToAbsoluteLatLonDepth(); - //String id = placeLocationOnMap(loc); +// if (SwingUtilities.isLeftMouseButton(e) && e.isControlDown()) { +// if (camFov != null) { +// double width = ((Component) e.getSource()).getWidth(); +// double height = ((Component) e.getSource()).getHeight(); +// double x = e.getX(); +// double y = height - e.getY(); +// mouseLoc = new Point2D.Double((x / width - 0.5) * 2, (y / height - 0.5) * 2); +// LocationType loc = camFov.getLookAt(mouseLoc.getX(), mouseLoc.getY()); +// loc.convertToAbsoluteLatLonDepth(); +// String id = placeLocationOnMap(loc); // snap = new StoredSnapshot(id, loc, e.getPoint(), onScreenImage, new Date()); // snap.setCamFov(camFov); // try { @@ -417,14 +415,14 @@ public void mouseClicked(MouseEvent e) { // catch (Exception ex) { // NeptusLog.pub().error(ex); // } - } - } +// } +// } if (e.getButton() == MouseEvent.BUTTON3) { popup = new JPopupMenu(); JMenuItem item; - popup.add(item = new JMenuItem(I18n.text("Connect to a IPCam"), + popup.add(item = new JMenuItem(I18n.text("Connect to stream"), ImageUtils.createImageIcon("images/menus/camera.png"))) .addActionListener(new ActionListener() { public void actionPerformed(ActionEvent e) { @@ -434,7 +432,7 @@ public void actionPerformed(ActionEvent e) { }); item.setAccelerator(KeyStroke.getKeyStroke(KeyEvent.VK_I, InputEvent.ALT_MASK)); - popup.add(item = new JMenuItem(I18n.text("Close connection"), + popup.add(item = new JMenuItem(I18n.text("Close stream connection"), ImageUtils.createImageIcon("images/menus/exit.png"))) .addActionListener(new ActionListener() { public void actionPerformed(ActionEvent e) { From 548b268e8e27eed90eb14cba2c9773bfbb21ac91 Mon Sep 17 00:00:00 2001 From: Paulo Dias Date: Tue, 21 Nov 2023 12:44:41 +0000 Subject: [PATCH 7/8] plugins/videoreader: Cleanup --- .../videoreader/IpCamManagementPanel.java | 2 +- .../plugins/videoreader/PlayerOpenCv.java | 11 +---------- .../neptus/plugins/videoreader/VideoReader.java | 17 ++++++++++------- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java index 115b97fe6f..c4767a4281 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/IpCamManagementPanel.java @@ -217,7 +217,7 @@ public void actionPerformed(ActionEvent e) { setValidUrlFunction.apply(selectedCamera.getName(), selectedCamera.getUrl()); videoReader.service.execute(connectStreamFunction); } - });dis + }); fieldIP.setEditable(false); fieldIP.setFocusable(false); add(selectIPCam, "h 30!, wrap"); diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/PlayerOpenCv.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/PlayerOpenCv.java index b2db128dbb..9f3db1cafe 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/PlayerOpenCv.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/PlayerOpenCv.java @@ -47,7 +47,6 @@ import java.awt.image.BufferedImage; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; class PlayerOpenCv { @@ -63,7 +62,7 @@ class PlayerOpenCv { private static final int DEFAULT_WIDTH_CONSOLE = 640; private static final int DEFAULT_HEIGHT_CONSOLE = 480; - private ExecutorService service; + private final ExecutorService service; private final String id; private Function sizeChangeFunction; private VideoCapture capture; @@ -77,9 +76,7 @@ class PlayerOpenCv { private Function updateImageFrameFunction; private AtomicInteger emptyFramesCounter = new AtomicInteger(0); - private AtomicInteger threadsIdCounter = new AtomicInteger(0); - private Scalar black = new Scalar(0); // Size of output frame private Dimension size = null; @@ -91,8 +88,6 @@ class PlayerOpenCv { private int widthConsole = DEFAULT_WIDTH_CONSOLE; // Height size of Console private int heightConsole = DEFAULT_HEIGHT_CONSOLE; - // flag for state of neptus logo - private boolean noVideoLogoState = false; // Scale factor of x pixel private float xScale; // Scale factor of y pixel @@ -100,10 +95,6 @@ class PlayerOpenCv { private BufferedImage frameImage; - // counter for frame tag ID - private short frameTagID = 1; - private AtomicLong captureLoopAtomicLongMillis = new AtomicLong(-1); - private boolean histogramFlag = false; private boolean streamingActive = false; diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java index 8df6369583..511abfa18a 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java @@ -294,11 +294,12 @@ private void connectStream() { disconnectStream(); } - //player = new Player(String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()), service); - player = new PlayerOpenCv(String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()), service); - player.sizeChange(new Dimension(widthConsole, heightConsole)); - player.setHistogramFlag(histogramFlag); + String cid = String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()); try { + //player = new Player(String.format("%05X-%d", VideoReader.this.hashCode(), threadsIdCounter.getAndIncrement()), service); + player = new PlayerOpenCv(cid, service); + player.sizeChange(new Dimension(widthConsole, heightConsole)); + player.setHistogramFlag(histogramFlag); player.start(camUrl /*fieldUrl.getText()*/, image -> { //BufferedImage scaledImage = ImageUtils.toBufferedImage(ImageUtils.getFastScaledImage(image, widthConsole, heightConsole, true)); BufferedImage scaledImage = image; @@ -307,11 +308,13 @@ private void connectStream() { }); } catch (Exception | Error e) { - String error = player.getId() + " :: ERROR :: " + e.getMessage(); + String error = cid + " :: ERROR :: " + e.getMessage(); NeptusLog.pub().error(error); getConsole().post(Notification.warning(PluginUtils.getPluginName(this.getClass()), error)); - player.setStopRequest(); - player = null; + if (player != null) { + player.setStopRequest(); + player = null; + } } repaint(100); From 91d24751a339d48de130481c18d5ec02b07a6e91 Mon Sep 17 00:00:00 2001 From: Paulo Dias Date: Tue, 21 Nov 2023 12:46:33 +0000 Subject: [PATCH 8/8] plugins/videoreader: Cleanup --- .../java/pt/lsts/neptus/plugins/videoreader/VideoReader.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java index 511abfa18a..a3120ac74e 100644 --- a/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java +++ b/plugins-dev/videoreader/src/java/pt/lsts/neptus/plugins/videoreader/VideoReader.java @@ -325,8 +325,6 @@ private void disconnectStream() { return; } - System.out.println("disconnectStream"); - PlayerOpenCv playerToDisconnect = player; player = null; playerToDisconnect.setStopRequest();