iT邦幫忙

2026 iThome 鐵人賽
DAY 17
0
Software Development

深入認識DeepStream，不只是停在執行範例系列 第 17 篇

[Day 17] 認識GStreamer

18th鐵人賽 deepstream
69 瀏覽

  • 分享至 

  • xImage
    •  

我們現在對GLib有了基礎的認識，今天就將先前的範例用C寫出來，進一步的了解GObject、GMainLoop是什麼，以及過去提到的Element、Pad、Pipeline怎麼用程式碼操作。

範例程式

先上完整的程式再分段介紹。建立main.c貼上以下程式

#include <gst/gst.h>

#define DEMO_VIDEO_FILE "/opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4"
#define OUTPUT_FILE "/ironman/day_17_demo.mp4"
#define PGIE_CONFIG_FILE "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test1/dstest1_pgie_config.txt"

static gboolean pad_is_nvmm_video(GstPad* pad) {
    GstCaps* caps = gst_pad_get_current_caps(pad);
    if (!caps) {
        caps = gst_pad_query_caps(pad, NULL);
    }
    g_assert(caps != NULL);

    GstStructure* str = gst_caps_get_structure(caps, 0);
    const gchar* name = gst_structure_get_name(str);

    if (g_strcmp0(name, "video/x-raw") == 0) {
        GstCapsFeatures* features = gst_caps_get_features(caps, 0);
        if (features && gst_caps_features_contains(features, "memory:NVMM")) {
            gst_caps_unref(caps);
            return TRUE;
        }
    }

    gst_caps_unref(caps);
    return FALSE;
}

static void on_srcbin_pad_added(GstElement*, GstPad* pad, gpointer user_data) {
    if (!pad_is_nvmm_video(pad)) {
        g_print("WARNING: ignore the pad is not a NVMM video\n");
        return;
    }

    GstElement* streammux = user_data;
    GstPad* sink_pad = gst_element_request_pad_simple(streammux, "sink_0");
    if (!sink_pad) {
        g_printerr("Failed to request sink_0 pad from streammux\n");
        return;
    }

    if (gst_pad_link(pad, sink_pad) != GST_PAD_LINK_OK) {
        g_printerr("Failed to link srcbin and streammux\n");
        gst_object_unref(sink_pad);
        return;
    }

    gst_object_unref(sink_pad);
    g_print("Link srcbin to streammux successfully\n");
}

static gboolean on_bus_call(GstBus*, GstMessage* message, gpointer user_data) {

    switch (message->type) {
        case GST_MESSAGE_ERROR: {
            GError* err = NULL;
            gchar* debug_info = NULL;

            gst_message_parse_error(message, &err, &debug_info);
            g_printerr("Error from element %s: %s\n", GST_OBJECT_NAME(message->src), err->message);
            g_printerr("Debug info: %s\n", debug_info ? debug_info : "none");

            g_free(debug_info);
            g_error_free(err);
            break;
        }
        case GST_MESSAGE_EOS: {
            GMainLoop* loop = user_data;
            /* Stop the main loop to end the application */
            g_print("Got EOS from the pipeline, stop the main loop\n");
            g_main_loop_quit(loop);

            /* Return FALSE to remove the bus watch */
            return FALSE;
        }
        default:
            break;
    }

    return TRUE;
}

static GstElement* create_demo_pipeline(const gchar* demo_video_file, const gchar* output_file) {
    GstElement* pipeline = gst_pipeline_new("ds-demo");

    GstElement* srcbin = gst_element_factory_make("nvurisrcbin", NULL);
    GstElement* streammux = gst_element_factory_make("nvstreammux", NULL);
    GstElement* pgie = gst_element_factory_make("nvinfer", NULL);
    GstElement* osd = gst_element_factory_make("nvdsosd", NULL);
    GstElement* transform = gst_element_factory_make("nvvideoconvert", NULL);
    GstElement* enc = gst_element_factory_make("nvv4l2h264enc", NULL);
    GstElement* h264parse = gst_element_factory_make("h264parse", NULL);
    GstElement* mp4mux = gst_element_factory_make("mp4mux", NULL);
    GstElement* sink = gst_element_factory_make("filesink", NULL);

    if (!srcbin || !streammux || !pgie || !osd || !transform || !enc || !h264parse || !mp4mux || !sink) {
        g_printerr("Failed to create one of elements\n");
        gst_object_unref(pipeline);
        return NULL;
    }

    /* Ownership of elements is fully transfer to the pipeline */
    gst_bin_add_many(GST_BIN(pipeline), srcbin, streammux, pgie, osd, transform, enc, h264parse, mp4mux, sink, NULL);

    /* Link elements after they are added to the pipeline.
     * Because the sink pad of nvurisrcbin will be added when playing,
     * we can't link srcbin to streammux at this moment.
     */
    if (!gst_element_link_many(streammux, pgie, osd, transform, enc, h264parse, mp4mux, sink, NULL)) {
        gst_object_unref(pipeline);
        g_printerr("Failed to link elements\n");
        return NULL;
    }

    /* Set the source file */
    GError* error = NULL;
    gchar* uri = g_filename_to_uri(demo_video_file, NULL, &error);
    if (!uri) {
        g_printerr("Can't convert '%s' to an uri: %s", demo_video_file, error->message);
        g_error_free(error);
        gst_object_unref(pipeline);
        return NULL;
    }
    g_object_set(srcbin, "uri", uri, NULL);
    g_free(uri);

    /* Set the output file */
    g_object_set(sink, "location", output_file, NULL);

    /* Misc. */
    g_object_set(pgie, "config-file-path", PGIE_CONFIG_FILE, NULL);
    g_object_set(streammux,
        "batch-size", 1,
        "width", 1920,
        "height", 1080,
        NULL);

    /* Set signal callback for srcbin */
    (void)g_signal_connect(srcbin, "pad-added", G_CALLBACK(on_srcbin_pad_added), streammux);

    return pipeline;
}

int main(int argc, char** argv) {
    gst_init(&argc, &argv);

    GMainLoop* loop = g_main_loop_new(NULL, FALSE);

    GstElement* pipeline = create_demo_pipeline(DEMO_VIDEO_FILE, OUTPUT_FILE);
    if (!pipeline) {
        g_printerr("Failed to create the demo pipeline\n");
        g_main_loop_unref(loop);
        return EXIT_FAILURE;
    }

    /* Add bus watch to handle an error message and EOS */
    GstBus* bus = gst_element_get_bus(pipeline);
    (void)gst_bus_add_watch(bus, on_bus_call, loop);
    gst_object_unref(bus);


    /* Play the pipeline */
    if (gst_element_set_state(pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
        g_printerr("Failed to play the pipeline\n");
        gst_object_unref(pipeline);
        g_main_loop_unref(loop);
        return EXIT_FAILURE;
    }

    /* Run the main loop to let GStreamer pumping message automatically.
     * The application blocks in this line and will continue after receiving
     * an EOS message.
     */
    g_main_loop_run(loop);

    /* Stop the pipeline */
    gst_element_set_state(pipeline, GST_STATE_NULL);

    gst_object_unref(pipeline);
    g_main_loop_unref(loop);

    g_print("Complete successfully\n");

    return EXIT_SUCCESS;
}

建立Makefile

CC := gcc
TARGET := demo_app
SRCS := main.c
OBJS := $(SRCS:.c=.o)

PKGS := gstreamer-1.0

CFLAGS += -Wall -Wextra $(shell pkg-config --cflags $(PKGS))
LIBS += $(shell pkg-config --libs $(PKGS))

all: $(TARGET)

$(TARGET): $(OBJS)
	$(CC) $(OBJS) -o $@ $(LIBS)

%.o: %.c
	$(CC) $(CFLAGS) -c $< -o $@

clean:
	rm -f $(OBJS) $(TARGET)

.PHONY: all clean

編譯範例程式

make

範例剖析

建立元件

GstElement* srcbin = gst_element_factory_make("nvurisrcbin", NULL);
    GstElement* streammux = gst_element_factory_make("nvstreammux", NULL);

[...]

建立元件統一使用gst_element_factory_make這個函式

GstElement *
gst_element_factory_make (const gchar * factoryname,
                          const gchar * name)

factoryname就是元件名稱，跟我們使用gst-inspect-1.0查詢的名稱是相同的。name是實例(instance)的名稱，每個GstElement*可以有一個獨立的名稱，通常輸入NULL自動命名就好。如果GStreamer從所有插件中都找不到該元件，函式會回傳NULL，記得要檢查。

if (!srcbin || !streammux || !pgie || !osd || !transform || !enc || !h264parse || !mp4mux || !sink) {
        g_printerr("Failed to create one of elements\n");
        gst_object_unref(pipeline);
        return NULL;
    }

建立pipeline及連接元件

GstElement* pipeline = gst_pipeline_new("ds-demo");

[...]

gst_bin_add_many(GST_BIN(pipeline), srcbin, streammux, pgie, osd, transform, enc, h264parse, mp4mux, sink, NULL);

[...]

if (!gst_element_link_many(streammux, pgie, osd, transform, enc, h264parse, mp4mux, sink, NULL)) {
        gst_object_unref(pipeline);
        g_printerr("Failed to link elements\n");
        return NULL;
    }

pipeline由gst_pipeline_new建立，回傳不會是NULL

GstElement *
gst_pipeline_new (const gchar * name)

有了pipeline之後我們得先將所有元件加入pipeline，因為前面已經檢查過每一個元件不為NULL，所以可以使用gst_bin_add_many這個高階函式

gst_bin_add_many (GstBin * bin,
                  GstElement * element_1,
                  ... ...)

而pipeline在GStreamer當中是屬於GstBin物件，GstBin視為單一元件，可以包含一個到數個子元件。只有已經加入pipeline的元件才可以連結，為了方便我們使用高階函式gst_element_link_many。它的優點是一行就可以連接所有元件，缺點是失敗時你沒辦法排查問題出在哪兩個元件之間。

gboolean
gst_element_link_many (GstElement * element_1,
                       GstElement * element_2,
                       ... ...)

另外要留意的是我們是使用nvurisrcbin，它的Sink Pad要在運行期間才會產生，所以我們只能先連接nvstreammux到後面的元件，nvusisrcbin和nvstreammux的連接必須在callback內部完成，也就是on_srcbin_pad_added。註冊callback要使用到g_signal_connect

#define g_signal_connect (
  instance,
  detailed_signal,
  c_handler,
  data
)

它背後是呼叫g_signal_connect_data，可以看到註冊後是有回傳值的，這個回傳值可以用來取消註冊，但因為這邊pipeline用完之後就要直接銷毀了，所以就偷懶一點忽略回傳值，否則得寫額外的結構保留到最後取消註冊。

gulong
g_signal_connect_data (
  GObject* instance,
  const gchar* detailed_signal,
  GCallback c_handler,
  gpointer data,
  GClosureNotify destroy_data,
  GConnectFlags connect_flags
)

Return valueThe handler ID (always greater than 0).

設定屬性

統一使用g_object_set，可設定的屬性就是從gst-inspect-1.0查詢，大家可以去對照之前用gst-launch-1.0執行的時候設定了哪些屬性。

void
g_object_set (
  GObject* object,
  const gchar* first_property_name,
  ...
)

另外留意一下這段，g_object_set會複製輸入值，而不是轉移Ownership，如果輸入值是要手動清除的，記得在調用之後清除。

gchar* uri = g_filename_to_uri(demo_video_file, NULL, &error);

[...]

g_object_set(srcbin, "uri", uri, NULL);
g_free(uri);

動態連接Pad

static void on_srcbin_pad_added(G_GNUC_UNUSED GstElement* element, GstPad* pad, gpointer user_data) {
    if (!pad_is_nvmm_video(pad)) {
        g_print("WARNING: ignore the pad is not a NVMM video\n");
        return;
    }

    GstElement* streammux = user_data;
    GstPad* sink_pad = gst_element_request_pad_simple(streammux, "sink_0");
    if (!sink_pad) {
        g_printerr("Failed to request sink_0 pad from streammux\n");
        return;
    }

    if (gst_pad_link(pad, sink_pad) != GST_PAD_LINK_OK) {
        g_printerr("Failed to link srcbin and streammux\n");
        gst_object_unref(sink_pad);
        return;
    }

    gst_object_unref(sink_pad);
    g_print("Link srcbin to streammux successfully\n");
}

nvurisrcbin會動態新增的Pad有asrc_%u和vsrc_%u，因為我們只要影片的輸入，所以得預防性忽略進來的Pad是asrc_%u的情況，詳細的檢查方式就參考pad_is_nvmm_video。

nvstreammux的Sink Pad是屬於On request，得主動跟元件請求，當上游的Src Pad產生後，用gst_element_request_pad_simple這個函式請求，連接完後記得減少引用。

GstPad *
gst_element_request_pad_simple (GstElement * element,
                                const gchar * name)

監聽GstBus

GstBus負責將訊息由"Streaming Thread"傳遞到"Main Thread"，訊息發送是非同步的，這麼做是為了確保"Main Thread"的執行緒安全，大多數的UI框架都限制資料變動只能在UI Thread完成，GstBus的設計就簡化了訊息轉發(Dispatch)的工作

GstBus* bus = gst_element_get_bus(pipeline);
(void)gst_bus_add_watch(bus, on_bus_call, loop);
gst_object_unref(bus);

範例當中有兩個要監聽的重點，一個是錯誤訊息，另一個是End Of Stream (EOS)。

static gboolean on_bus_call(G_GNUC_UNUSED GstBus* bus, GstMessage* message, gpointer user_data) {

    switch (message->type) {
        case GST_MESSAGE_ERROR: {
            GError* err = NULL;
            gchar* debug_info = NULL;

            gst_message_parse_error(message, &err, &debug_info);
            g_printerr("Error from element %s: %s\n", GST_OBJECT_NAME(message->src), err->message);
            g_printerr("Debug info: %s\n", debug_info ? debug_info : "none");

            g_free(debug_info);
            g_error_free(err);
            break;
        }
        case GST_MESSAGE_EOS: {
            GMainLoop* loop = user_data;
            /* Stop the main loop to end the application */
            g_print("Got EOS from the pipeline, stop the main loop\n");
            g_main_loop_quit(loop);

            /* Return FALSE to remove the bus watch */
            return FALSE;
        }
        default:
            break;
    }

    return TRUE;
}

每個GstMessage如果有夾帶資料的話，都會提供gst_message_parse_<message_type>函式把資料解出，只要去查官方文件就可以。至於要監聽EOS是因為GstBus會將事件送進預設的GMainContext，必須要預設的GMainLoop在運行狀態才能自動收到訊息，所以當串流結束後得手動停止GMainLoop，否則應用程式就會卡死在g_main_loop_run那一行。

pipeline播放和停止

/* Play the pipeline */
if (gst_element_set_state(pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
    g_printerr("Failed to play the pipeline\n");
    gst_object_unref(pipeline);
    g_main_loop_unref(loop);
    return EXIT_FAILURE;
}

/* Run the main loop to let GStreamer pumping message automatically.
    * The application blocks in this line and will continue after receiving
    * an EOS message.
    */
g_main_loop_run(loop);

/* Stop the pipeline */
gst_element_set_state(pipeline, GST_STATE_NULL);

GstElement有四種狀態，NULL、READY、PAUSED、PLAYING。NULL是初始化物件，READY是分配串流狀態要使用的資源，PAUSED已經開始播放但處於暫停狀態，PLAYING則是播放狀態，直接設定狀態到PLAYING會自動過度中間的每一個狀態。

GStreamer會建立"Streaming Thread"處理播放相關的任務，"Main Thread"保留給呼叫端自行應用，所以這邊就是讓GMainLoop跑起來確保可以收到GstBus的訊息。當收到EOS後記得把pipeline設定為NULL狀態，清除串流要使用的資源。

結語

之前看似非常容易的範例要用C實作出來就變得非常繁瑣，這也是正式進入GStreamer後要面臨的一個門檻，每一步的操作都要大量查詢GLib和GStreamer的文件，否則容易造成資源沒有被正確釋放。


上一篇
[Day 16] 認識GLib
下一篇
[Day 18] GstProbe和迭代NvDsBatchMeta
系列文
深入認識DeepStream，不只是停在執行範例 共 22 篇
  1. 18
    [Day 18] GstProbe和迭代NvDsBatchMeta
  2. 19
    [Day 19] 細說各種Metadata
  3. 20
    [Day 20] 如何附加新的Meta Data
  4. 21
    [Day 21] 如何建立User Metadata
  5. 22
    [Day 22] 實作GStreamer插件的優勢
完整目錄
圖片
  熱門推薦
圖片
{{ item.channelVendor }} | {{ item.webinarstarted }} |
{{ formatDate(item.duration) }}
直播中

尚未有邦友留言

立即登入留言
iThome鐵人賽
參賽組數
929 組
團體組數
32 組
累計文章數
16009 篇
完賽人數
221 人
iT+看影片追技術 看影片追技術 看更多
熱門tag
15th鐵人賽 16th鐵人賽 13th鐵人賽 17th鐵人賽 14th鐵人賽 12th鐵人賽 18th鐵人賽 11th鐵人賽 鐵人賽 2019鐵人賽 javascript 2018鐵人賽 python 2017鐵人賽 windows php c# linux windows server css
熱門問題
熱門回答
熱門文章
IT邦幫忙