Versions Compared

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

To enable Rialto in WebKit, apply patch below and rebuild webkit using bitbake command.
Solution tested with Broadcom device only.Important note: Rialto RIALTO_ENABLE_DECRYPT_BUFFER flag has to be enabled to support EME.

Expand
titleWebKit 2.38.2

Change tested with WebKit 2.38.2 (e585b369185b29fbc514afd959295efd02df9426)

Code Block
languagediff
titlewebkit.patch
diff --git a/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp b/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp
index 08bdac7e1e45..5aa17911bbfc 100644
--- a/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp
+++ b/Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp
@@ -402,12 +402,16 @@ void registerWebKitGStreamerElements()
         // The VAAPI plugin is not much maintained anymore and prone to rendering issues. In the
         // mid-term we will leverage the new stateless VA decoders. Disable the legacy plugin,
         // unless the WEBKIT_GST_ENABLE_LEGACY_VAAPI environment variable is set to 1.
+        auto* registry = gst_registry_get();
         const char* enableLegacyVAAPIPlugin = getenv("WEBKIT_GST_ENABLE_LEGACY_VAAPI");
         if (!enableLegacyVAAPIPlugin || !strcmp(enableLegacyVAAPIPlugin, "0")) {
-            auto* registry = gst_registry_get();
             if (auto vaapiPlugin = adoptGRef(gst_registry_find_plugin(registry, "vaapi")))
                 gst_registry_remove_plugin(registry, vaapiPlugin.get());
         }
+        if (auto brcmVidFilterPlugin = adoptGRef(gst_registry_find_plugin(registry, "brcmvidfilter")))
+            gst_registry_remove_plugin(registry, brcmVidFilterPlugin.get());
+        if (auto brcmAudFilterPlugin = adoptGRef(gst_registry_find_plugin(registry, "brcmaudfilter")))
+            gst_registry_remove_plugin(registry, brcmAudFilterPlugin.get());
         registryWasUpdated = true;
     });
 
diff --git a/Source/WebCore/platform/graphics/gstreamer/GStreamerRegistryScanner.cpp b/Source/WebCore/platform/graphics/gstreamer/GStreamerRegistryScanner.cpp
index d2bbf1416e9b..6b4a5ddd6c45 100644
--- a/Source/WebCore/platform/graphics/gstreamer/GStreamerRegistryScanner.cpp
+++ b/Source/WebCore/platform/graphics/gstreamer/GStreamerRegistryScanner.cpp
@@ -77,17 +77,10 @@ void GStreamerRegistryScanner::getSupportedDecodingTypes(HashSet<String, ASCIICa
 
 GStreamerRegistryScanner::ElementFactories::ElementFactories(OptionSet<ElementFactories::Type> types)
 {
-#if PLATFORM(BCM_NEXUS) || PLATFORM(BROADCOM)
-    if (types.contains(Type::AudioDecoder))
-        audioDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_MARGINAL);
-    if (types.contains(Type::VideoDecoder))
-        videoDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
-#else
     if (types.contains(Type::AudioDecoder))
         audioDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_MARGINAL);
     if (types.contains(Type::VideoDecoder))
         videoDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
-#endif
     if (types.contains(Type::AudioParser))
         audioParserFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_NONE);
     if (types.contains(Type::VideoParser))
diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp
index 6e46736d07d0..8855840217ce 100644
--- a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp
+++ b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp
@@ -1321,28 +1321,12 @@ void MediaPlayerPrivateGStreamer::loadingFailed(MediaPlayer::NetworkState networ
 
 GstElement* MediaPlayerPrivateGStreamer::createAudioSink()
 {
-#if PLATFORM(BROADCOM) || USE(WESTEROS_SINK) || PLATFORM(AMLOGIC) || PLATFORM(REALTEK)
-    // If audio is being controlled by an another pipeline, creating sink here may interfere with
-    // audio playback. Instead, check if an audio sink was setup in handleMessage and use it.
-    return nullptr;
-#endif
-
-    // For platform specific audio sinks, they need to be properly upranked so that they get properly autoplugged.
-
-    auto role = m_player->isVideoPlayer() ? "video"_s : "music"_s;
-    GstElement* audioSink = createPlatformAudioSink(role);
-    RELEASE_ASSERT(audioSink);
-    if (!audioSink)
-        return nullptr;
-
-#if ENABLE(WEB_AUDIO)
-    GstElement* audioSinkBin = gst_bin_new("audio-sink");
-    ensureAudioSourceProvider();
-    m_audioSourceProvider->configureAudioBin(audioSinkBin, audioSink);
-    return audioSinkBin;
-#else
-    return audioSink;
-#endif
+// #if ENABLE(WEB_AUDIO)
+//     // Rialto webaudio doesnt work, investigation needed
+//     return gst_element_factory_make("rialtowebaudiosink", "rialtowebaudiosink");
+// #else
+    return gst_element_factory_make("rialtomseaudiosink", "rialtomseaudiosink");
+// #endif
 }
 
 GstElement* MediaPlayerPrivateGStreamer::audioSink() const
@@ -1941,7 +1925,6 @@ void MediaPlayerPrivateGStreamer::handleMessage(GstMessage* message)
             }
         }
 #endif
-
 #if PLATFORM(BROADCOM) || USE(WESTEROS_SINK) || PLATFORM(AMLOGIC) || PLATFORM(REALTEK)
         if (currentState <= GST_STATE_READY && newState >= GST_STATE_READY) {
             // Detect an audio sink element and store reference to it if it supersedes what we currently have.
@@ -4150,7 +4133,7 @@ GstElement* MediaPlayerPrivateGStreamer::createHolePunchVideoSink()
     if (m_isLegacyPlaybin && !isPIPRequested)
         return nullptr;
     // Westeros using holepunch.
-    GstElement* videoSink = makeGStreamerElement("westerossink", "WesterosVideoSink");
+    GstElement* videoSink = gst_element_factory_make("rialtomsevideosink", "rialtomsevideosink");
     g_object_set(G_OBJECT(videoSink), "zorder", 0.0f, nullptr);
     if (isPIPRequested) {
         g_object_set(G_OBJECT(videoSink), "res-usage", 0u, nullptr);
@@ -4572,7 +4555,6 @@ std::optional<VideoFrameMetadata> MediaPlayerPrivateGStreamer::videoFrameMetadat
 
 void MediaPlayerPrivateGStreamer::setPageIsVisible(bool visible)
 {
-
     if (m_visible != visible) {
 #if USE(GSTREAMER_HOLEPUNCH)
         Locker locker { m_holePunchLock };
diff --git a/Source/
WebCore/platform/graphics/gstreamer/mse/MediaPlayerPrivateGStreamerMSE.cpp
cmake/FindThunder.cmake b/Source/
WebCore/platform/graphics/gstreamer/mse/MediaPlayerPrivateGStreamerMSE.cpp
cmake/FindThunder.cmake
index 
fc7241444484
ab7064e3988b..
ad28e3722888
bc086ad9105d 100644
--- a/Source/
WebCore/platform/graphics/gstreamer/mse/MediaPlayerPrivateGStreamerMSE.cpp
cmake/FindThunder.cmake
+++ b/Source/
WebCore/platform/graphics/gstreamer/mse/MediaPlayerPrivateGStreamerMSE.cpp
cmake/FindThunder.cmake
@@ -
229
56,
6
9 +
229
56,
30
7 @@ 
MediaTime MediaPlayerPrivateGStreamerMSE::durationMediaTime() const
find_path(THUNDER_INCLUDE_DIR
 )
 
 
void MediaPlayerPrivateGStreamerMSE::seek(const MediaTime& time) { +
find_library(THUNDER_LIBRARY
-    NAMES 
//
ocdm
- 
Rialto
 
doesnt
 
like
 
when seek is performed below paused state +
HINTS ${PC_THUNDER_LIBDIR}
-         
if (m_isSeeking) {
 ${PC_THUNDER_LIBRARY_DIRS}
+    NAMES 
if (m_isSeekPending) { + m_seekTime = time; + return; + } + } + + if (pipeline()) + { + GstState state; +
ocdmRialto
 )
 
 include(FindPackageHandleStandardArgs)


Expand
titleWebKit 2.38.5
  1. Apply change with Rialto Quirks: https://github.com/WebPlatformForEmbedded/WPEWebKit/pull/1323 (this change has been merged in 1.38.6-r2 and maybe earlier)
  2. Apply this patch to enable EME:
    Code Block
    languagediff
    titleRialto EME
    diff --git a/Source/cmake/FindThunder.cmake b/Source/cmake/FindThunder.cmake
    index ab7064e3988b..bc086ad9105d 100644
    --- a/Source/cmake/FindThunder.cmake
    +++ b/Source/cmake/FindThunder.cmake
    @@ -56,9 +56,7 @@ find_path(THUNDER_INCLUDE_DIR
     )
     
     find_library(THUNDER_LIBRARY
    -    NAMES ocdm
    -    HINTS ${PC_THUNDER_LIBDIR}
    -        
GstStateChangeReturn
  1.  
getStateResult = gst_element_get_state(pipeline(), &state, nullptr, 0); + if (getStateResult == GST_STATE_CHANGE_FAILURE || getStateResult == GST_STATE_CHANGE_NO_PREROLL) { + GST_DEBUG_OBJECT(pipeline(), "[Seek] cannot seek, current state change is %s", gst_element_state_change_return_get_name(getStateResult)); +
  1.  ${PC_THUNDER_LIBRARY_DIRS}
    +    NAMES ocdmRialto
     )
     
     include(FindPackageHandleStandardArgs)
    
    
  2. SVPPAY is not supported in Rialto, so this patch has to be applied for AmLogic devices:
    Code Block
    languagediff
    titleSvppay removal
    diff --git a/Source/WebCore/platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp
    index df182ae06550..49004c73feda 100644
    --- a/Source/WebCore/platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp
    +++ b/Source/WebCore/platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp
    @@ -400,25 +400,7 @@ GstPadProbeReturn onWebKitMediaSourcePadEvent(GstPad* pad, GstPadProbeInfo* info
         if (probData->didTryCreatePayloader == false)
         {
             probData->didTryCreatePayloader = 
return
  1. true;
    
+
  1. -        
} + if (getStateResult == GST_STATE_CHANGE_ASYNC || state < GST_STATE_PAUSED || m_isEndReached
  1. if (WebCore::doCapsHaveType(caps, GST_VIDEO_CAPS_TYPE_PREFIX)) {
    
+
  1. -            
m_isSeekPending = true; +
  1. // svppay element is always inserted even for non encrypted caps
    -      
m_isSeeking
  1.    
=
  1.  
true; +
  1.   // to force secure memory video parser so we 
m_seekTime = time; +
  1. don't need to replace
    -            // parser when 
m_isEndReached
  1. encrypted 
= false; +
  1. samples arrive
    -
    -            // 
return; + } + }
  1. svppay payloader is available on Amlogic platform only
    -            GRefPtr<GstElementFactory> payloader_factory = adoptGRef(gst_element_factory_find("svppay"));
    -            if (payloader_factory) {
    -                probData->payloader = gst_element_factory_create(payloader_factory.get(), nullptr);
    -                payloader = probData->payloader.get();
    -            }
    -            if (payloader) {
    -                gst_bin_add(GST_BIN(parent_bin.get()), payloader);
    -                GST_INFO("svppay %s added to %s for pad %s",
    -                         GST_ELEMENT_NAME(payloader), GST_ELEMENT_NAME(parent_bin.get()), GST_PAD_NAME(pad));
    -            } else {
    -                GST_INFO("svppay not found -> continuing without payloader");
    -            }
    -        }
    +        GST_INFO("SVP PAY DISABLED FOR RIALTO");
         }
     
         if(!decryptorAttached && WebCore::areEncryptedCaps(caps))

    Set two environment variables for widget/application:
    Code Block
    languagebash
    titleEnvironment variables
    WEBKIT_GST_QUIRKS=rialto
    WEBKIT_GST_HOLE_PUNCH_QUIRK=rialto
Expand
titleWebKit 2.38.6-r2
  1. Apply these patches (this adds together steps 2 and 3 of the change given in section 2.38.5)
    in the following sub-directory:- build-xione-sercomm/tmp/work/armv7vet2hf-neon-rdkmllib32-linux-gnueabi/lib32-wpe-webkit/2.38.6+gitAUTOINC+59d63a5298-r2/git
    Code Block
    languagediff
    titleRialto EME
    diff -NarU 6 a/Source/WebCore/platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp
    --- a/Source/WebCore/platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp	2024-07-31 10:05:28.000000000 +0100
    +++ b/Source/WebCore/platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp	2024-08-05 16:59:58.906225172 +0100
    @@ -398,31 +398,13 @@
         GstElement* payloader = probData->payloader.get();
         bool payloaderAttached = payloader && probData->payloaderAttached;
     
         if (probData->didTryCreatePayloader == false)
         {
             probData->didTryCreatePayloader = true;
    -        if (WebCore::doCapsHaveType(caps, GST_VIDEO_CAPS_TYPE_PREFIX)) {
    -            // svppay element is always inserted even for non encrypted caps
    -            // to force secure memory video parser so we don't need to replace
    -            // parser when encrypted samples arrive
    -
    -            // svppay payloader is available on Amlogic platform only
    -            GRefPtr<GstElementFactory> payloader_factory = adoptGRef(gst_element_factory_find("svppay"));
    -
GST_DEBUG_OBJECT(pipeline(), "Requested seek to %s", time.toString().utf8().data()); doSeek(time, m_playbackRate, GST_SEEK_FLAG_FLUSH); } @@ -257,10 +281,21 @@ bool MediaPlayerPrivateGStreamerMSE::doSeek(const MediaTime& position, float rat m_isWaitingForPreroll = true; m_isEndReached = false; - // Important: In order to ensure correct propagation whether pre-roll has happened or not, we send the seek directly - // to the source element, rather than letting playbin do the routing. - gst_element_seek(m_source.get(), rate, GST_FORMAT_TIME, seekFlags, - GST_SEEK_TYPE_SET, toGstClockTime(m_seekTime), GST_SEEK_TYPE_NONE, 0); + // Seek with m_source doesnt always forward seek event to all rialto sinks which causes seek problem. + if (pipeline()) + { + fprintf(stderr, "Using pipeline to seek"); + gst_element_seek(pipeline(), rate, GST_FORMAT_TIME, seekFlags, +
  1.             
GST_SEEK_TYPE_SET, toGstClockTime(m_seekTime), GST_SEEK_TYPE_NONE, 0); +
  1. if (payloader_factory) {
    -          
} +
  1.     
else +
  1.   probData->payloader 
{ +
  1. = gst_element_factory_create(payloader_factory.get(), nullptr);
    -         
//
  1.  
Important:
  1.  
In
  1.  
order
  1.  
to
  1.  
ensure
  1.  
correct
  1.  
propagation
  1. payloader 
whether
  1. = 
pre-roll has happened or not, we send the seek directly +
  1. probData->payloader.get();
    -            
  1. }
    -   
//
  1.  
to
  1.  
the
  1.  
source
  1.  
element,
  1.  
rather
  1.  
than
  1.  
letting
  1.  
playbin
  1.  
do
  1. if 
the routing. +
  1. (payloader) {
    -        
fprintf(stderr,
  1.  
"Using
  1.  
source
  1.  
to
  1.  
seek"); +
  1.     gst_bin_
element
  1. add(GST_
seek
  1. BIN(
m
  1. parent_
source
  1. bin.get()), 
rate, GST_FORMAT_TIME, seekFlags, +
  1. payloader);
    -                GST_
SEEK_TYPE_SET, toGstClockTime(m_seekTime), GST_SEEK_TYPE_NONE, 0); + }
  1. INFO("svppay %s added to %s for pad %s",
    -       
invalidateCachedPosition();
  1.         
//
  1.  
Notify
  1.  
MediaSource
  1.  
and
  1.  
have
  1.  
new
  1.  
frames
  1.  
enqueued
  1.  
(when
  1.  
they're available). @@ -409,6 +444,17 @@ void MediaPlayerPrivateGStreamerMSE::updateStates()
  1.  GST_ELEMENT_NAME(payloader), GST_ELEMENT_NAME(parent_bin.get()), GST_PAD_NAME(pad));
    -            
GST_ERROR_OBJECT(pipeline(), "Setting the pipeline to PAUSED failed");
  1. } else {
    -         
m_isPipelinePlaying
  1.  
=
  1.  
false;
  1.      
} + if (getStateResult == GST_STATE_CHANGE_SUCCESS && m_currentState >= GST_STATE_PAUSED) { +
  1. GST_INFO("svppay not found -> continuing without payloader");
    -            }
    - 
if
  1.  
(m_isSeekPending)
  1.  
{ +
  1.      }
    +        GST_
DEBUG_OBJECT(pipeline(), "[Seek] committing pending seek to %s", toString(m_seekTime).utf8().data()); +
  1. INFO("SVP PAY DISABLED FOR RIALTO");
         }
     
         if(!decryptorAttached && WebCore::areEncryptedCaps(caps))
         {
           
m_isSeekPending
  1.  
= false; +
  1.  if(!decryptor)
             {
    diff -NarU 
m_isSeeking = doSeek(m_seekTime, m_player->rate(), static_cast<GstSeekFlags>(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_ACCURATE)); +
  1. 6 a/Source/cmake/FindThunder.cmake b/Source/cmake/FindThunder.cmake
    --- a/Source/cmake/FindThunder.cmake	2024-07-31 10:04:55.000000000 +0100
    +++ b/Source/cmake/FindThunder.cmake	2024-08-05 16:59:58.906225172 +0100
    @@ -53,15 +53,13 @@
         
if (!m_isSeeking) { +
  1. HINTS ${PC_THUNDER_INCLUDEDIR}
               
invalidateCachedPosition(); +
  1. ${PC_THUNDER_INCLUDE_DIRS}
         PATH_SUFFIXES "WPEFramework/ocdm/" "Thunder/ocdm/"
     )
     
     find_library(THUNDER_LIBRARY
    -    NAMES 
GST_DEBUG_OBJECT(pipeline(), "[Seek] seeking to %s failed", toString(m_seekTime).utf8().data()); +
  1. ocdm
    -    HINTS ${PC_THUNDER_LIBDIR}
    -          
  1. ${PC_THUNDER_LIBRARY_DIRS}
    +    NAMES ocdmRialto
     )
     
} +
  1. 
     include(FindPackageHandleStandardArgs)
     
     find_package_handle_standard_args(Thunder
       
} } bool MediaPlayerPrivateGStreamerMSE::isTimeBuffered(const MediaTime &time) const
  1. FOUND_VAR THUNDER_FOUND
    
    
  2. Set two environment variables for widget/application:
    Code Block
    languagebash
    titleEnvironment variables
    WEBKIT_GST_QUIRKS=rialto
    WEBKIT_GST_HOLE_PUNCH_QUIRK=rialto