OpenShot Library | libopenshot  0.2.7
FFmpegReader.h
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Header file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
12  * (http://www.openshotstudios.com). This file is part of
13  * OpenShot Library (http://www.openshot.org), an open-source project
14  * dedicated to delivering high quality video editing and animation solutions
15  * to the world.
16  *
17  * This file is originally based on the Libavformat API example, and then modified
18  * by the libopenshot project.
19  *
20  * OpenShot Library (libopenshot) is free software: you can redistribute it
21  * and/or modify it under the terms of the GNU Lesser General Public License
22  * as published by the Free Software Foundation, either version 3 of the
23  * License, or (at your option) any later version.
24  *
25  * OpenShot Library (libopenshot) is distributed in the hope that it will be
26  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28  * GNU Lesser General Public License for more details.
29  *
30  * You should have received a copy of the GNU Lesser General Public License
31  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
32  */
33 
34 #ifndef OPENSHOT_FFMPEG_READER_H
35 #define OPENSHOT_FFMPEG_READER_H
36 
37 #include "ReaderBase.h"
38 
39 // Include FFmpeg headers and macros
40 #include "FFmpegUtilities.h"
41 
42 #include <cmath>
43 #include <ctime>
44 #include <iostream>
45 #include <stdio.h>
46 #include <memory>
47 #include "CacheMemory.h"
48 #include "Clip.h"
49 #include "OpenMPUtilities.h"
50 #include "Settings.h"
51 
52 
53 namespace openshot {
54  /**
55  * @brief This struct holds the associated video frame and starting sample # for an audio packet.
56  *
57  * Because audio packets do not match up with video frames, this helps determine exactly
58  * where the audio packet's samples belong.
59  */
60  struct AudioLocation {
61  int64_t frame;
63 
64  bool is_near(AudioLocation location, int samples_per_frame, int64_t amount);
65  };
66 
67  /**
68  * @brief This class uses the FFmpeg libraries, to open video files and audio files, and return
69  * openshot::Frame objects for any frame in the file.
70  *
71  * All seeking and caching is handled internally, and the primary public interface is the GetFrame()
72  * method. To use this reader, simply create an instance of this class, and call the GetFrame method
73  * to start retrieving frames. Use the <b>info</b> struct to obtain information on the file, such as the length
74  * (# of frames), height, width, bit rate, frames per second (fps), etc...
75  *
76  * @code
77  * // Create a reader for a video
78  * openshot::FFmpegReader r("MyAwesomeVideo.webm");
79  * r.Open(); // Open the reader
80  *
81  * // Get frame number 1 from the video
82  * std::shared_ptr<openshot::Frame> f = r.GetFrame(1);
83  *
84  * // Now that we have an openshot::Frame object, lets have some fun!
85  * f->Display(); // Display the frame on the screen
86  * f->DisplayWaveform(); // Display the audio waveform as an image
87  * f->Play(); // Play the audio through your speaker
88  *
89  * // Close the reader
90  * r.Close();
91  * @endcode
92  */
93  class FFmpegReader : public ReaderBase {
94  private:
95  std::string path;
96 
97  AVFormatContext *pFormatCtx;
98  int i, videoStream, audioStream;
99  AVCodecContext *pCodecCtx, *aCodecCtx;
100 #if USE_HW_ACCEL
101  AVBufferRef *hw_device_ctx = NULL; //PM
102 #endif
103  AVStream *pStream, *aStream;
104  AVPacket *packet;
105  AVFrame *pFrame;
106  bool is_open;
107  bool is_duration_known;
108  bool check_interlace;
109  bool check_fps;
110  bool has_missing_frames;
111  int max_concurrent_frames;
112 
113  CacheMemory working_cache;
114  CacheMemory missing_frames;
115  std::map<int64_t, int64_t> processing_video_frames;
116  std::multimap<int64_t, int64_t> processing_audio_frames;
117  std::map<int64_t, int64_t> processed_video_frames;
118  std::map<int64_t, int64_t> processed_audio_frames;
119  std::multimap<int64_t, int64_t> missing_video_frames;
120  std::multimap<int64_t, int64_t> missing_video_frames_source;
121  std::multimap<int64_t, int64_t> missing_audio_frames;
122  std::multimap<int64_t, int64_t> missing_audio_frames_source;
123  std::map<int64_t, int> checked_frames;
124  AudioLocation previous_packet_location;
125 
126  // DEBUG VARIABLES (FOR AUDIO ISSUES)
127  int prev_samples;
128  int64_t prev_pts;
129  int64_t pts_total;
130  int64_t pts_counter;
131  int64_t num_packets_since_video_frame;
132  int64_t num_checks_since_final;
133  std::shared_ptr<openshot::Frame> last_video_frame;
134 
135  bool is_seeking;
136  int64_t seeking_pts;
137  int64_t seeking_frame;
138  bool is_video_seek;
139  int seek_count;
140  int64_t seek_audio_frame_found;
141  int64_t seek_video_frame_found;
142 
143  int64_t audio_pts_offset;
144  int64_t video_pts_offset;
145  int64_t last_frame;
146  int64_t largest_frame_processed;
147  int64_t current_video_frame; // can't reliably use PTS of video to determine this
148 
149  int hw_de_supported = 0; // Is set by FFmpegReader
150 #if USE_HW_ACCEL
151  AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE;
152  AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE;
153  int IsHardwareDecodeSupported(int codecid);
154 #endif
155 
156  /// Check for the correct frames per second value by scanning the 1st few seconds of video packets.
157  void CheckFPS();
158 
159  /// Check the current seek position and determine if we need to seek again
160  bool CheckSeek(bool is_video);
161 
162  /// Check if a frame is missing and attempt to replace its frame image (and
163  bool CheckMissingFrame(int64_t requested_frame);
164 
165  /// Check the working queue, and move finished frames to the finished queue
166  void CheckWorkingFrames(bool end_of_stream, int64_t requested_frame);
167 
168  /// Convert Frame Number into Audio PTS
169  int64_t ConvertFrameToAudioPTS(int64_t frame_number);
170 
171  /// Convert Frame Number into Video PTS
172  int64_t ConvertFrameToVideoPTS(int64_t frame_number);
173 
174  /// Convert Video PTS into Frame Number
175  int64_t ConvertVideoPTStoFrame(int64_t pts);
176 
177  /// Create a new Frame (or return an existing one) and add it to the working queue.
178  std::shared_ptr<openshot::Frame> CreateFrame(int64_t requested_frame);
179 
180  /// Calculate Starting video frame and sample # for an audio PTS
181  AudioLocation GetAudioPTSLocation(int64_t pts);
182 
183  /// Get an AVFrame (if any)
184  bool GetAVFrame();
185 
186  /// Get the next packet (if any)
187  int GetNextPacket();
188 
189  /// Get the smallest video frame that is still being processed
190  int64_t GetSmallestVideoFrame();
191 
192  /// Get the smallest audio frame that is still being processed
193  int64_t GetSmallestAudioFrame();
194 
195  /// Get the PTS for the current video packet
196  int64_t GetVideoPTS();
197 
198  /// Check if there's an album art
199  bool HasAlbumArt();
200 
201  /// Remove partial frames due to seek
202  bool IsPartialFrame(int64_t requested_frame);
203 
204  /// Process a video packet
205  void ProcessVideoPacket(int64_t requested_frame);
206 
207  /// Process an audio packet
208  void ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample);
209 
210  /// Read the stream until we find the requested Frame
211  std::shared_ptr<openshot::Frame> ReadStream(int64_t requested_frame);
212 
213  /// Remove AVFrame from cache (and deallocate its memory)
214  void RemoveAVFrame(AVFrame *);
215 
216  /// Remove AVPacket from cache (and deallocate its memory)
217  void RemoveAVPacket(AVPacket *);
218 
219  /// Seek to a specific Frame. This is not always frame accurate, it's more of an estimation on many codecs.
220  void Seek(int64_t requested_frame);
221 
222  /// Update PTS Offset (if any)
223  void UpdatePTSOffset(bool is_video);
224 
225  /// Update File Info for audio streams
226  void UpdateAudioInfo();
227 
228  /// Update File Info for video streams
229  void UpdateVideoInfo();
230 
231  public:
232  /// Final cache object used to hold final frames
234 
235  /// Enable or disable seeking. Seeking can more quickly locate the requested frame, but some
236  /// codecs have trouble seeking, and can introduce artifacts or blank images into the video.
238 
239  /// @brief Constructor for FFmpegReader.
240  ///
241  /// Sets (and possibly opens) the media file path,
242  /// or throws an exception.
243  /// @param path The filesystem location to load
244  /// @param inspect_reader if true (the default), automatically open the media file and loads frame 1.
245  FFmpegReader(const std::string& path, bool inspect_reader=true);
246 
247  /// Destructor
248  virtual ~FFmpegReader();
249 
250  /// Close File
251  void Close() override;
252 
253  /// Get the cache object used by this reader
254  CacheMemory *GetCache() override { return &final_cache; };
255 
256  /// Get a shared pointer to a openshot::Frame object for a specific frame number of this reader.
257  ///
258  /// @returns The requested frame of video
259  /// @param requested_frame The frame number that is requested.
260  std::shared_ptr<openshot::Frame> GetFrame(int64_t requested_frame) override;
261 
262  /// Determine if reader is open or closed
263  bool IsOpen() override { return is_open; };
264 
265  /// Return the type name of the class
266  std::string Name() override { return "FFmpegReader"; };
267 
268  // Get and Set JSON methods
269  std::string Json() const override; ///< Generate JSON string of this object
270  void SetJson(const std::string value) override; ///< Load JSON string into this object
271  Json::Value JsonValue() const override; ///< Generate Json::Value for this object
272  void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object
273 
274  /// Open File - which is called by the constructor automatically
275  void Open() override;
276 
277  /// Return true if frame can be read with GetFrame()
278  bool GetIsDurationKnown();
279  };
280 
281 }
282 
283 #endif
Header file for CacheMemory class.
Header file for Clip class.
Header file for FFmpegUtilities.
Header file for OpenMPUtilities (set some common macros)
Header file for ReaderBase class.
Header file for global Settings class.
This class is a memory-based cache manager for Frame objects.
Definition: CacheMemory.h:50
This class uses the FFmpeg libraries, to open video files and audio files, and return openshot::Frame...
Definition: FFmpegReader.h:93
CacheMemory * GetCache() override
Get the cache object used by this reader.
Definition: FFmpegReader.h:254
void Open() override
Open File - which is called by the constructor automatically.
FFmpegReader(const std::string &path, bool inspect_reader=true)
Constructor for FFmpegReader.
Json::Value JsonValue() const override
Generate Json::Value for this object.
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:233
std::string Name() override
Return the type name of the class.
Definition: FFmpegReader.h:266
virtual ~FFmpegReader()
Destructor.
std::string Json() const override
Generate JSON string of this object.
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
void Close() override
Close File.
void SetJson(const std::string value) override
Load JSON string into this object.
bool IsOpen() override
Determine if reader is open or closed.
Definition: FFmpegReader.h:263
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:98
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:60
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)