audio_frame.h

Go to the documentation of this file.

/*******************************************************************************
 * Copyright (C) 2018-2021 Intel Corporation
 *
 * SPDX-License-Identifier: MIT
 ******************************************************************************/
 
#pragma once
 
#include "audio_event.h"
 
#include "metadata/gva_json_meta.h"
#include "metadata/gva_tensor_meta.h"
 
#include <gst/audio/audio.h>
#include <gst/audio/gstaudiometa.h>
#include <gst/gstbuffer.h>
 
#include <algorithm>
#include <assert.h>
#include <functional>
#include <memory>
#include <stdexcept>
#include <string>
#include <vector>
 
namespace GVA {
 
class AudioFrame {
  protected:
    GstBuffer *buffer;
 
    std::unique_ptr<GstAudioInfo, std::function<void(GstAudioInfo *)>> info;
 
  public:
    AudioFrame(GstBuffer *buffer, GstAudioInfo *info)
        : buffer(buffer), info(gst_audio_info_copy(info), gst_audio_info_free) {
        if (not buffer or not info) {
            throw std::invalid_argument("GVA::AudioFrame: buffer or info nullptr");
        }
    }
 
    AudioFrame(GstBuffer *buffer, const GstCaps *caps) : buffer(buffer) {
        if (not buffer or not caps) {
            throw std::invalid_argument("GVA::AudioFrame: buffer or caps nullptr");
        }
        info = std::unique_ptr<GstAudioInfo, std::function<void(GstAudioInfo *)>>(gst_audio_info_new(),
                                                                                  gst_audio_info_free);
        if (!gst_audio_info_from_caps(info.get(), caps)) {
            throw std::runtime_error("GVA::AudioFrame: gst_audio_info_from_caps failed");
        }
    }
 
    AudioFrame(GstBuffer *buffer) : buffer(buffer) {
        if (not buffer)
            throw std::invalid_argument("GVA::AudioFrame: buffer is nullptr");
 
        GstAudioMeta *meta = audio_meta();
        if (not meta)
            throw std::logic_error("GVA::AudioFrame: audio_meta() is nullptr");
 
        info = std::unique_ptr<GstAudioInfo, std::function<void(GstAudioInfo *)>>(gst_audio_info_new(),
                                                                                  gst_audio_info_free);
        if (not info.get())
            throw std::logic_error("GVA::AudioFrame: gst_audio_info_new() failed");
 
        memcpy(info.get(), &meta->info, sizeof(meta->info));
    }
 
    GstAudioMeta *audio_meta() {
        return gst_buffer_get_audio_meta(buffer);
    }
 
    GstAudioInfo *audio_info() {
        return info.get();
    }
 
    std::vector<AudioEvent> events() {
        return get_events();
    }
 
    const std::vector<AudioEvent> events() const {
        return get_events();
    }
 
    std::vector<Tensor> tensors() {
        return get_tensors();
    }
 
    const std::vector<Tensor> tensors() const {
        return get_tensors();
    }
 
    std::vector<std::string> messages() {
        std::vector<std::string> json_messages;
        GstGVAJSONMeta *meta = NULL;
        gpointer state = NULL;
        GType meta_api_type = g_type_from_name(GVA_JSON_META_API_NAME);
        while ((meta = (GstGVAJSONMeta *)gst_buffer_iterate_meta_filtered(buffer, &state, meta_api_type))) {
            json_messages.emplace_back(meta->message);
        }
        return json_messages;
    }
 
    AudioEvent add_event(long start_time, long end_time, std::string label = std::string(), double confidence = 0.0) {
 
        GstGVAAudioEventMeta *meta = gst_gva_buffer_add_audio_event_meta(buffer, label.c_str(), start_time, end_time);
 
        // Add detection tensor
        GstStructure *detection = gst_structure_new("detection", "start_timestamp", G_TYPE_LONG, start_time,
                                                    "end_timestamp", G_TYPE_LONG, end_time, NULL);
        if (confidence) {
            gst_structure_set(detection, "confidence", G_TYPE_DOUBLE, confidence, NULL);
        }
        gst_gva_audio_event_meta_add_param(meta, detection);
 
        return AudioEvent(meta);
    }
 
    Tensor add_tensor() {
        const GstMetaInfo *meta_info = gst_meta_get_info(GVA_TENSOR_META_IMPL_NAME);
 
        if (!gst_buffer_is_writable(buffer))
            throw std::runtime_error("Buffer is not writable.");
 
        GstGVATensorMeta *tensor_meta = (GstGVATensorMeta *)gst_buffer_add_meta(buffer, meta_info, NULL);
 
        return Tensor(tensor_meta->data);
    }
 
    void add_message(const std::string &message) {
        const GstMetaInfo *meta_info = gst_meta_get_info(GVA_JSON_META_IMPL_NAME);
 
        if (!gst_buffer_is_writable(buffer))
            throw std::runtime_error("Buffer is not writable.");
 
        GstGVAJSONMeta *json_meta = (GstGVAJSONMeta *)gst_buffer_add_meta(buffer, meta_info, NULL);
        json_meta->message = g_strdup(message.c_str());
    }
 
    void remove_event(const AudioEvent &event) {
        if (!gst_buffer_is_writable(buffer))
            throw std::runtime_error("Buffer is not writable.");
 
        if (!gst_buffer_remove_meta(buffer, (GstMeta *)event._meta())) {
            throw std::out_of_range("GVA::AudioFrame: AudioEvent doesn't belong to this frame");
        }
    }
 
    void remove_tensor(const Tensor &tensor) {
        GstGVATensorMeta *meta = NULL;
        gpointer state = NULL;
        while ((meta = GST_GVA_TENSOR_META_ITERATE(buffer, &state))) {
            if (meta->data == tensor._structure) {
                if (!gst_buffer_is_writable(buffer))
                    throw std::runtime_error("Buffer is not writable.");
 
                if (gst_buffer_remove_meta(buffer, (GstMeta *)meta))
                    return;
            }
        }
        throw std::out_of_range("GVA::AudioFrame: Tensor doesn't belong to this frame");
    }
 
  private:
    std::vector<AudioEvent> get_events() const {
        std::vector<AudioEvent> events;
        GstMeta *meta = NULL;
        gpointer state = NULL;
 
        while ((meta = gst_buffer_iterate_meta_filtered(buffer, &state, GST_GVA_AUDIO_EVENT_META_API_TYPE)))
            events.emplace_back((GstGVAAudioEventMeta *)meta);
        return events;
    }
 
    std::vector<Tensor> get_tensors() const {
        std::vector<Tensor> tensors;
        GstGVATensorMeta *meta = NULL;
        gpointer state = NULL;
        GType meta_api_type = g_type_from_name("GstGVATensorMetaAPI");
        while ((meta = (GstGVATensorMeta *)gst_buffer_iterate_meta_filtered(buffer, &state, meta_api_type)))
            tensors.emplace_back(meta->data);
        return tensors;
    }
};
 
} // namespace GVA