gstreamer example / some progress but not workiny yet

[Robert O.]

Heyho,

Im working on the gstreamer example.

It kind of works and streams to texture.
But there are two major problems:
1) The picture is just in black an white: I have no idea why, and also I am working mainly on the other, much bigger problem:
2) The application fills all the memory until system crashes.
I tried to fix this by adding:
"(gst_app_sink_set_max_buffers (cast sink i8*) 8)
(gst_app_sink_set_drop (cast sink i8*) #t)"
, but that had no effect. At least not the desired one.

After some googeling it seems that we need to call "gst_buffer_unref" in the new-sample callback function to free the memory every new-sampe callback. The problem is I cant bind-lib "gst_buffer_unref" which seems very odd to me since most other gst_buffer* functions are available and they are in the gst base library.
So my main question is: Has anybody any idea why I cant bind-lib "gst_buffer_unref"???

Has anybody managed to stream v4l data into extempore in some other way (opencv, custom c script with direct acess to device,...)?


Here is my changed and kind of working but not working version of the gstvideo.xtm example:

(sys:load "libs/external/glfw3.xtm")

(bind-val width i32 640)
(bind-val height i32 480)
(bind-val datsize i64 (* 3 (i32toi64 width) (i32toi64 height)))

(bind-val window GLFWwindow*
  (begin
    (glfwSetErrorCallback (convert (get_native_fptr glfw_error_callback)))
    (glfw_init_and_create_interaction_window width height)))

(sys:load "libs/external/gl/gl-objects.xtm")
(sys:load "libs/external/gstreamer.xtm")

(bind-val vao VAO* (VAO_create_ss_quad))
(bind-val tex Texture (Texture_create))
(bind-val sp ShaderProgram
  (ShaderProgram_create
   (sys_slurp_file "examples/external/shader-tutorials/texture.vert")
   (sys_slurp_file "examples/external/shader-tutorials/texture.frag")))

(bind-func xtm_frame_data_to_tex
  (lambda (tex data)
    (glActiveTexture GL_TEXTURE0)
    (glBindTexture GL_TEXTURE_2D tex)
    (glTexImage2D GL_TEXTURE_2D
                  0
                  GL_RGB
                  width height
                  0
                  GL_RGB
                  GL_UNSIGNED_BYTE
                  data)
    (glTexParameteri GL_TEXTURE_2D GL_TEXTURE_WRAP_S GL_CLAMP_TO_EDGE)
    (glTexParameteri GL_TEXTURE_2D GL_TEXTURE_WRAP_T GL_CLAMP_TO_EDGE)
    (glTexParameteri GL_TEXTURE_2D GL_TEXTURE_MIN_FILTER GL_NEAREST)
    (glTexParameteri GL_TEXTURE_2D GL_TEXTURE_MAG_FILTER GL_NEAREST)
    (gl_print_error "in xtm_frame_data_to_tex")))

(bind-val vplay1 i1 0)
(bind-val sampledata1 i8*)
(bind-val sampledata1dirty i1 0)
(bind-val texid1 i32 0)

(bind-type userdata <i32,GstElement*>)

(bind-func init_sample_data
  (lambda ()
    (set! vplay1 #f)
    (set! sampledata1dirty #f)
    (set! sampledata1 (malloc datsize))
    void))

(bind-func gl_load_movie_textures
  (lambda ()
    (if sampledata1dirty
        (xtm_frame_data_to_tex tex sampledata1))
    void))

(bind-func playbin_bus_call:[void,GstElement*,GstMessage*,userdata*]*
  (lambda (bus msg data)
    (let ((playz (tref (cast data userdata*) 1))
          (msgtype (tref msg 1))
          (time (tref msg 2))
          (seq (tref msg 4)))
      (cond ((<> (& GST_MESSAGE_EOS msgtype) 0)
             (printf "END OF STREAM!\n")
             (gst_element_set_state playz GST_STATE_NULL)
             (gst_object_unref (cast playz gpointer)) ;; does playz release sink?
             (free data)
             1))
      void)))

(bind-func new_preroll
  (lambda (sink:GstAppSink* data:gpointer)
    (let ((sample (gst_app_sink_pull_preroll (cast sink i8*)))
          (caps (gst_sample_get_caps sample))
          (st (gst_caps_get_structure caps 0))
          (gv (gst_structure_get_value st "framerate"))
          (num (gst_value_get_fraction_numerator gv))
          (denom (gst_value_get_fraction_denominator gv))
          (playz (tref (cast data userdata*) 1))
          (quad (tref (cast data userdata*) 0)))
      (cond ((= quad 1) (set! vplay1 #t)))
      (printf "Preroll[%d]: %s\n" quad (gst_caps_to_string caps))
      GST_FLOW_OK)))

(bind-func new_sample
  (lambda (sink:GstAppSink* data:gpointer)
    (let ((sample (gst_app_sink_pull_sample (cast sink i8*)))
          (caps (gst_sample_get_caps sample))
          (buffer (gst_sample_get_buffer sample))
          (dat:i8* null)
          (xsize:gsize 0)
          (quad (tref (cast data userdata*) 0))
          )
      ;;(printf "Playing back on quad:%d\n" quad)
      (set! dat sampledata1)
      (set! xsize (gst_buffer_extract buffer 0 dat datsize))
      (gst_buffer_remove_all_memory buffer)
      ;;here I think we should add: (gst_buffer_unref buffer) but (bind-lib gstlib gst_buffer_unref [void,GstBuffer*]*) throws an error :-(
      (if (= xsize datsize)
          (cond ((= quad 1) (set! sampledata1dirty #t))
                )
          (cond ((= quad 1) (set! sampledata1dirty #f))
                ))
      GST_FLOW_OK)))
($ (println (gst_version_string)))
(bind-func eos
  (lambda (sink:GstAppSink* data:gpointer)
    (let ((quad (tref (cast data userdata*) 0))
          (playz (tref (cast data userdata*) 1)))
      (cond ((= quad 1) (set! vplay1 #f) (set! sampledata1dirty #f)))
      (printf "Finished Decoding[%d]!\n" quad)
      void)))

;; path must be a valid URI to media

(bind-func xtm_play
  (lambda ()
    (if vplay1
        (println "Wait to finish the running video first!\n")
        (let ((gstErr:GError* (halloc))
              (pipeline (gst_parse_launch "v4l2src device=/dev/video0 ! videoflip method=5 ! appsink name=sink caps=video/x-raw,format=RGB,width=640,height=480" gstErr));;
              (sink (gst_bin_get_by_name (cast pipeline GstBin*) "sink"))
              (bus (gst_pipeline_get_bus (cast pipeline GstPipeline*)))
              (appBin:GstElement* (gst_bin_new "app_bin"))
              (data:userdata* (halloc))
              (callbacks:GstAppSinkCallbacks* (halloc))
              (gv:GValue* (halloc))
              (maxbuffers:guint (i64toi32 8))
              (drop:gboolean (i1toi32 #t))
              (emit_signals:gboolean (i1toi32 #f))
              )

          (gst_bus_add_watch bus (cast (get_native_fptr playbin_bus_call) GstBusFunc) (cast data gpointer))
          (println "sink: " (not (null? sink)))
          (println "pipeline: "  (not (null? pipeline)))
          (gst_object_unref bus)
          (tfill! callbacks
                  (get_native_fptr eos)
                  (get_native_fptr new_preroll)
                  (get_native_fptr new_sample))
          (tset! data 0 1)
          (tset! data 1 (cast pipeline GstElement*))
          (gst_app_sink_set_emit_signals (cast sink i8*) emit_signals)
          (gst_app_sink_set_max_buffers (cast sink i8*) maxbuffers)
          (gst_app_sink_set_drop (cast sink i8*) drop)
          (gst_app_sink_set_callbacks (cast sink i8*) callbacks (cast data gpointer) null)

          (println "maxbuffers: " (gst_app_sink_get_max_buffers (cast sink i8*)))
          (println "drop frames?: " (gst_app_sink_get_drop (cast sink i8*)))
          (println "emit signals?: " (gst_app_sink_get_emit_signals (cast sink i8*)))
          ;; start decoding the media!
          (let ((res (gst_element_set_state pipeline GST_STATE_PLAYING)))

            (if (or (= res 1) (= res 2))
                (begin (printf "Start Decoding[%d]\n" (tref data 0)) #t)
                (begin (printf "Failed To Open[%d]\n" (tref data 0))
                       (gst_element_set_state pipeline GST_STATE_NULL)
                       (gst_object_unref (cast pipeline i8*))
                       (free data)
                       #f)))
          void))))

(bind-func _glib_main_loop_call
  (lambda ()
    (g_main_context_iteration null 0)))

(bind-func gl_draw_loop
  (lambda (time:i64 delta_t:double)
    ;; step glib event loop
    ;; draw texture
    (gl_load_movie_textures)
    (glClear GL_COLOR_BUFFER_BIT)
    ;; draw the world
    (let ((texloc (glGetUniformLocation sp "tex")))
      (glUseProgram sp)
      (glUniform1i texloc 0))
    (VAO_bind_and_draw_arrays vao GL_TRIANGLE_STRIP 0 4)
    (glfwSwapBuffers window)

    (let ((next_time (+ time (convert (* 44100. delta_t)))))
      (callback next_time gl_draw_loop next_time delta_t))))

(bind-func glib_main_loop
  (lambda ()
    (_glib_main_loop_call)
    (callback (+ (now) 2000) glib_main_loop)))

;; start your movies here!
 ($ (begin
      (glib_main_loop)
      (init_sample_data)
      (gst_init_check null null null)
      (gl_draw_loop (now) (/ 1. 60.))
      (xtm_play)))



greetings, Robert Oetting

[Andrew Sorensen]

Hi Robert,

My first guess would be that gst_buffer_unref is defined as a macro?

--
Sent from my Android device with K-9 Mail. Please excuse my brevity.

[Robert O.]

Its defined as static! Seems like there has to be another way except using gst_buffer_unref.

thx for your help.

[Robert O.]

Okok,  i finally got the gst-example working.

It was necessary to change the buffer.c and buffer.h file in the gstreamer sources from https://gstreamer.freedesktop.org/src/gstreamer/. I used the 1.2.0 version. The files are found in gstreamer-1.2.0/gs/.
I added the the function in gstreamer-1.2.0/gst/buffer.c

void
gst_buffer_unref (GstBuffer * buf)
{
  gst_mini_object_unref (GST_MINI_OBJECT_CAST (buf));
}

And changed this lines in buffer.h:

#ifdef _FOOL_GTK_DOC_
G_INLINE_FUNC void gst_buffer_unref (GstBuffer * buf);
#endif

static inline void
gst_buffer_unref (GstBuffer * buf)
{
  gst_mini_object_unref (GST_MINI_OBJECT_CAST (buf));
}

into this:

void gst_buffer_unref (GstBuffer * buf);

I found a helpful shellscript for compiling gst here: http://kacianka.at/?p=145. I split it in two files, one for downloading and one for compiling so I can edit the files in between. This script does not add the .so files to the system. So its necessary to add the exact path to the new binaries in gst.xtm found in extempore/libs/external/. The shared libraries are found in the new directory created by the script gstreamer_build/lib/. Of course the last thing I had to do was adding the gst_buffer_unref - function to gst.xtm so I can access it in extempore by adding the line:

(bind-lib gstlib gst_buffer_unref [void,GstBuffer*]*)

Now the follwing code reads a v4l device and streams the video into "inframedata". This array can be uploaded to a gl texture,...


(bind-val inwidth i32 1024) (bind-val inheight i32 768)
(bind-val datsize i64 (* 3 (i32toi64 inwidth) (i32toi64 inheight)))
(bind-val inframedata i8*)
(bind-val vplay1 i1 0)

(bind-type userdata <i32,GstElement*>)

(bind-func init_sample_data
  (lambda ()

    (set! inframedata (malloc datsize))
    void))

(bind-func init
  (lambda ()
    (gst_init null null)))

(bind-func new_preroll
  (lambda (sink:GstAppSink* data:gpointer)
    (let ((sample (gst_app_sink_pull_preroll (cast sink i8*)))
          (caps (gst_sample_get_caps sample))
          (st (gst_caps_get_structure caps 0))
          (gv (gst_structure_get_value st "framerate"))
          (num (gst_value_get_fraction_numerator gv))
          (denom (gst_value_get_fraction_denominator gv))
          (playz (tref (cast data userdata*) 1))
          (quad (tref (cast data userdata*) 0)))
      (cond ((= quad 1) (set! vplay1 #t)))
      ;;(printf "Preroll[%d]: %s\n" quad (gst_caps_to_string caps))
      GST_FLOW_OK)))

(bind-func new_sample
  (lambda (sink:GstAppSink* data:gpointer)
    (let ((sample (gst_app_sink_pull_sample (cast sink i8*)))
          (caps (gst_sample_get_caps sample))
          (buffer (gst_sample_get_buffer sample))
          (dat:i8* null)
          (xsize:gsize 0)
          (quad (tref (cast data userdata*) 0)))
      ;;(printf "Playing back on quad:%d\n" quad)

      (set! dat inframedata)

      (set! xsize (gst_buffer_extract buffer 0 dat datsize))

      (gst_buffer_unref buffer)
    ;  (gst_caps_unref caps)

      ;;(if (= xsize datsize) (printf "ok\n")(printf "not ok\n"))
      GST_FLOW_OK)))

(bind-func eos
  (lambda (sink:GstAppSink* data:gpointer)
    (let ((quad (tref (cast data userdata*) 0))
          (playz (tref (cast data userdata*) 1)))
      ;(cond ((= quad 1) (set! vplay1 #f) (set! sampledata1dirty #f)))
      (printf "Finished Decoding[%d]!\n" quad)
      void)))

(bind-func xtm_play
      (lambda ()
      (let (
        (gstErr:GError* (halloc))
          (pipeline (gst_parse_launch "v4l2src device=/dev/video1 ! videoscale ! videoflip method=5 ! appsink name=sink caps=video/x-raw,format=RGB,width=1024,height=768" gstErr))

        (sink (gst_bin_get_by_name (cast pipeline GstBin*) "sink"))

        (data:userdata* (halloc))
        (callbacks:GstAppSinkCallbacks* (halloc))

        (xsize:gsize 0)
        (maxbuffers:guint 1)
        )

      (if (null? pipeline) (printf "problem: cant get pipeline\n") (printf "pipeline created...\n"))
      (if (null? sink) (printf "problem: cant get sink\n") (printf "got sink from pipeline...\n"))

      (gst_app_sink_set_max_buffers (cast sink i8*) maxbuffers)

      (gst_app_sink_set_emit_signals (cast sink i8*) 1)
      (gst_app_sink_set_drop (cast sink i8*) 1)

      (tfill! callbacks
          (get_native_fptr eos)
          (get_native_fptr new_preroll)
          (get_native_fptr new_sample))
      (tset! data 0 1)
      (tset! data 1 (cast pipeline GstElement*))

      (gst_app_sink_set_callbacks (cast sink i8*) callbacks (cast data gpointer) null)

      (let ((res (gst_element_set_state pipeline GST_STATE_PLAYING)))
          (if (or (= res 1) (= res 2))

                (begin (printf "Pipeline set to playing state\n")
                #t)
                (begin (printf "Unable to set the pipeline to playing state\n")

                       (gst_element_set_state pipeline GST_STATE_NULL)
                       (gst_object_unref (cast pipeline i8*))

                       #f)))
                       (cast sink i8*)

                       )))

(bind-func _glib_main_loop_call
  (lambda ()
    (g_main_context_iteration null 0)))

(bind-func glib_main_loop
  (lambda ()
    (_glib_main_loop_call)

    (callback (+ (now) 1470) glib_main_loop))) ;;1470 makes 30 fps

($ (init_sample_data)
   (init)
   (glib_main_loop)
   (xtm_play)
   (printf "video input from v4l-device activated!\n")
   )

Changing the sources of gst is unpractical. I could not find another way, and as my c skills are limted, I am not shure if the changes in buffer.* have unwanted side effects. For me it works fine.


It would be nice to add gstramer to the core libraries. Is that legally possible?

[digego]

Hey Robert,

Good stuff, I haven't looked at gstreamer for quite some time (as you can tell).  I'll will try to find some time to take a look soonish  ;)