Skip to content

Commit

Permalink
Un-reverting previous commit, removing queuing from FFmpegWriter for …
Browse files Browse the repository at this point in the history 
…simplicity. Increasing margin on webm unit tests, when checking pixel color.
  • Loading branch information
@jonoomph
jonoomph committed Feb 11, 2024
1 parent 038dd17 commit d87e9e4
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 148 deletions.
175 changes: 50 additions & 125 deletions src/FFmpegWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int6
FFmpegWriter::FFmpegWriter(const std::string& path) :
path(path), oc(NULL), audio_st(NULL), video_st(NULL), samples(NULL),
audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(1), num_of_rescalers(1),
initial_audio_input_frame_size(0), img_convert_ctx(NULL), num_of_rescalers(1),
rescaler_position(0), video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0),
original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) {
Expand Down Expand Up @@ -679,118 +679,50 @@ void FFmpegWriter::WriteFrame(std::shared_ptr<openshot::Frame> frame) {
if (!is_open)
throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);

// Add frame pointer to "queue", waiting to be processed the next
// time the WriteFrames() method is called.
if (info.has_video && video_st)
spooled_video_frames.push_back(frame);

if (info.has_audio && audio_st)
spooled_audio_frames.push_back(frame);

ZmqLogger::Instance()->AppendDebugMethod(
"FFmpegWriter::WriteFrame",
"frame->number", frame->number,
"spooled_video_frames.size()", spooled_video_frames.size(),
"spooled_audio_frames.size()", spooled_audio_frames.size(),
"cache_size", cache_size,
"is_writing", is_writing);

// Write the frames once it reaches the correct cache size
if ((int)spooled_video_frames.size() == cache_size || (int)spooled_audio_frames.size() == cache_size) {
// Write frames to video file
write_queued_frames();
}
// Write frames to video file
write_frame(frame);

// Keep track of the last frame added
last_frame = frame;
}

// Write all frames in the queue to the video file.
void FFmpegWriter::write_queued_frames() {
ZmqLogger::Instance()->AppendDebugMethod(
"FFmpegWriter::write_queued_frames",
"spooled_video_frames.size()", spooled_video_frames.size(),
"spooled_audio_frames.size()", spooled_audio_frames.size());

void FFmpegWriter::write_frame(std::shared_ptr<Frame> frame) {
// Flip writing flag
is_writing = true;

// Transfer spool to queue
queued_video_frames = spooled_video_frames;
queued_audio_frames = spooled_audio_frames;

// Empty spool
spooled_video_frames.clear();
spooled_audio_frames.clear();

// Create blank exception
bool has_error_encoding_video = false;

// Process all audio frames (in a separate thread)
if (info.has_audio && audio_st && !queued_audio_frames.empty())
write_audio_packets(false);

// Loop through each queued image frame
while (!queued_video_frames.empty()) {
// Get front frame (from the queue)
std::shared_ptr<Frame> frame = queued_video_frames.front();

// Add to processed queue
processed_frames.push_back(frame);

// Encode and add the frame to the output file
if (info.has_video && video_st)
process_video_packet(frame);

// Remove front item
queued_video_frames.pop_front();

} // end while


// Loop back through the frames (in order), and write them to the video file
while (!processed_frames.empty()) {
// Get front frame (from the queue)
std::shared_ptr<Frame> frame = processed_frames.front();

if (info.has_video && video_st) {
// Add to deallocate queue (so we can remove the AVFrames when we are done)
deallocate_frames.push_back(frame);

// Does this frame's AVFrame still exist
if (av_frames.count(frame)) {
// Get AVFrame
AVFrame *frame_final = av_frames[frame];

// Write frame to video file
bool success = write_video_packet(frame, frame_final);
if (!success)
has_error_encoding_video = true;
}
}

// Remove front item
processed_frames.pop_front();
}
// Process audio frame
if (info.has_audio && audio_st)
write_audio_packets(false, frame);

// Loop through, and deallocate AVFrames
while (!deallocate_frames.empty()) {
// Get front frame (from the queue)
std::shared_ptr<Frame> frame = deallocate_frames.front();
// Process video frame
if (info.has_video && video_st)
process_video_packet(frame);

if (info.has_video && video_st) {
// Does this frame's AVFrame still exist
if (av_frames.count(frame)) {
// Get AVFrame
AVFrame *av_frame = av_frames[frame];
AVFrame *frame_final = av_frames[frame];

// Write frame to video file
if (!write_video_packet(frame, frame_final)) {
has_error_encoding_video = true;
}

// Deallocate buffer and AVFrame
av_freep(&(av_frame->data[0]));
AV_FREE_FRAME(&av_frame);
av_freep(&(frame_final->data[0]));
AV_FREE_FRAME(&frame_final);
av_frames.erase(frame);
}

// Remove front item
deallocate_frames.pop_front();
}

// Done writing
Expand Down Expand Up @@ -820,12 +752,9 @@ void FFmpegWriter::WriteFrame(ReaderBase *reader, int64_t start, int64_t length)

// Write the file trailer (after all frames are written)
void FFmpegWriter::WriteTrailer() {
// Write any remaining queued frames to video file
write_queued_frames();

// Process final audio frame (if any)
if (info.has_audio && audio_st)
write_audio_packets(true);
write_audio_packets(true, NULL);

// Flush encoders (who sometimes hold on to frames)
flush_encoders();
Expand Down Expand Up @@ -1598,7 +1527,10 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
}

// write all queued frames' audio to the video file
void FFmpegWriter::write_audio_packets(bool is_final) {
void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptr<openshot::Frame> frame) {
if (!frame && !is_final)
return;

// Init audio buffers / variables
int total_frame_samples = 0;
int frame_position = 0;
Expand All @@ -1608,56 +1540,49 @@ void FFmpegWriter::write_audio_packets(bool is_final) {
ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout

// Create a new array (to hold all S16 audio samples, for the current queued frames
unsigned int all_queued_samples_size = sizeof(int16_t) * (queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE);
unsigned int all_queued_samples_size = sizeof(int16_t) * AVCODEC_MAX_AUDIO_FRAME_SIZE;
int16_t *all_queued_samples = (int16_t *) av_malloc(all_queued_samples_size);
int16_t *all_resampled_samples = NULL;
int16_t *final_samples_planar = NULL;
int16_t *final_samples = NULL;

// Loop through each queued audio frame
while (!queued_audio_frames.empty()) {
// Get front frame (from the queue)
std::shared_ptr<Frame> frame = queued_audio_frames.front();
// Get audio sample array
float *frame_samples_float = NULL;

// Get the audio details from this frame
// Get the audio details from this frame
if (frame) {
sample_rate_in_frame = frame->SampleRate();
samples_in_frame = frame->GetAudioSamplesCount();
channels_in_frame = frame->GetAudioChannelsCount();
channel_layout_in_frame = frame->ChannelsLayout();

// Get audio sample array
float *frame_samples_float = NULL;
// Get samples interleaved together (c1 c2 c1 c2 c1 c2)
frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);
}

// Calculate total samples
total_frame_samples = samples_in_frame * channels_in_frame;

// Translate audio sample values back to 16 bit integers with saturation
const int16_t max16 = 32767;
const int16_t min16 = -32768;
for (int s = 0; s < total_frame_samples; s++, frame_position++) {
float valF = frame_samples_float[s] * (1 << 15);
int16_t conv;
if (valF > max16) {
conv = max16;
} else if (valF < min16) {
conv = min16;
} else {
conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
}

// Copy into buffer
all_queued_samples[frame_position] = conv;
// Calculate total samples
total_frame_samples = samples_in_frame * channels_in_frame;

// Translate audio sample values back to 16 bit integers with saturation
const int16_t max16 = 32767;
const int16_t min16 = -32768;
for (int s = 0; s < total_frame_samples; s++, frame_position++) {
float valF = frame_samples_float[s] * (1 << 15);
int16_t conv;
if (valF > max16) {
conv = max16;
} else if (valF < min16) {
conv = min16;
} else {
conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
}

// Deallocate float array
delete[] frame_samples_float;

// Remove front item
queued_audio_frames.pop_front();
// Copy into buffer
all_queued_samples[frame_position] = conv;
}

} // end while
// Deallocate float array
delete[] frame_samples_float;


// Update total samples (since we've combined all queued frames)
Expand Down
21 changes: 2 additions & 19 deletions src/FFmpegWriter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,6 @@ namespace openshot {
class FFmpegWriter : public WriterBase {
private:
std::string path;
int cache_size;
bool is_writing;
bool is_open;
int64_t video_timestamp;
Expand Down Expand Up @@ -152,15 +151,6 @@ namespace openshot {
int original_channels;

std::shared_ptr<openshot::Frame> last_frame;
std::deque<std::shared_ptr<openshot::Frame> > spooled_audio_frames;
std::deque<std::shared_ptr<openshot::Frame> > spooled_video_frames;

std::deque<std::shared_ptr<openshot::Frame> > queued_audio_frames;
std::deque<std::shared_ptr<openshot::Frame> > queued_video_frames;

std::deque<std::shared_ptr<openshot::Frame> > processed_frames;
std::deque<std::shared_ptr<openshot::Frame> > deallocate_frames;

std::map<std::shared_ptr<openshot::Frame>, AVFrame *> av_frames;

/// Add an AVFrame to the cache
Expand Down Expand Up @@ -205,13 +195,13 @@ namespace openshot {
void process_video_packet(std::shared_ptr<openshot::Frame> frame);

/// write all queued frames' audio to the video file
void write_audio_packets(bool is_final);
void write_audio_packets(bool is_final, std::shared_ptr<openshot::Frame> frame);

/// write video frame
bool write_video_packet(std::shared_ptr<openshot::Frame> frame, AVFrame *frame_final);

/// write all queued frames
void write_queued_frames();
void write_frame(std::shared_ptr<Frame> frame);

public:

Expand All @@ -224,9 +214,6 @@ namespace openshot {
/// Close the writer
void Close();

/// Get the cache size (number of frames to queue before writing)
int GetCacheSize() { return cache_size; };

/// Determine if writer is open or closed
bool IsOpen() { return is_open; };

Expand Down Expand Up @@ -273,10 +260,6 @@ namespace openshot {
/// \note This is an overloaded function.
void SetAudioOptions(std::string codec, int sample_rate, int bit_rate);

/// @brief Set the cache size
/// @param new_size The number of frames to queue before writing to the file
void SetCacheSize(int new_size) { cache_size = new_size; };

/// @brief Set video export options
/// @param has_video Does this file need a video stream
/// @param codec The codec used to encode the images in this video
Expand Down
8 changes: 4 additions & 4 deletions tests/FFmpegWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,10 @@ TEST_CASE( "Webm", "[libopenshot][ffmpegwriter]" )
int pixel_index = 112 * 4; // pixel 112 (4 bytes per pixel)

// Check image properties on scanline 10, pixel 112
CHECK((int)pixels[pixel_index] == Approx(23).margin(5));
CHECK((int)pixels[pixel_index + 1] == Approx(23).margin(5));
CHECK((int)pixels[pixel_index + 2] == Approx(23).margin(5));
CHECK((int)pixels[pixel_index + 3] == Approx(255).margin(5));
CHECK((int)pixels[pixel_index] == Approx(23).margin(7));
CHECK((int)pixels[pixel_index + 1] == Approx(23).margin(7));
CHECK((int)pixels[pixel_index + 2] == Approx(23).margin(7));
CHECK((int)pixels[pixel_index + 3] == Approx(255).margin(7));
}

TEST_CASE( "Options_Overloads", "[libopenshot][ffmpegwriter]" )
Expand Down

0 comments on commit d87e9e4

Please sign in to comment.