Port playlist script from iOS

testing

original script

https://github.com/brave/brave-ios/blob/development/Client/Frontend/UserContent/UserScripts/Playlist.js

the variable wrapped with $< > is replaced with security token from UserS

We can simply remove those.

test script

open script

// Copyright 2021 The Brave Authors. All rights reserved.
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// MARK: - Media Detection

(function() {
    function is_nan(value) {
        return typeof value === "number" && value !== value;
    }
    
    function is_infinite(value) {
        return typeof value === "number" && (value === Infinity || value === -Infinity);
    }
    
    function clamp_duration(value) {
        if (is_nan(value)) {
            return 0.0;
        }
        
        if (is_infinite(value)) {
            return Number.MAX_VALUE;
        }
        return value;
    }
    
    // Algorithm:
    // Generate a random number from 0 to 256
    // Roll-Over clamp to the range [0, 15]
    // If the index is 13, set it to 4.
    // If the index is 17, clamp it to [0, 3]
    // Subtract that number from 15 (XOR) and convert the result to hex.
    function uuid_v4() {
        // X >> 2 = X / 4 (integer division)
        
        // AND-ing (15 >> 0) roll-over clamps to 15
        // AND-ing (15 >> 2) roll-over clamps to 3
        // So '8' digit is clamped to 3 (inclusive) and all others clamped to 15 (inclusive).
        
        // 0 XOR 15 = 15
        // 1 XOR 15 = 14
        // 8 XOR 15 = 7
        // So N XOR 15 = 15 - N

        // UUID string format generated with array appending
        // Results in "10000000-1000-4000-8000-100000000000".replace(...)
        return ([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g, (X) => {
            return (X ^ (crypto.getRandomValues(new Uint8Array(1))[0] & (15 >> (X >> 2)))).toString(16);
        });
    }
    
    function tagNode(node) {
        if (node) {
            if (!node.tagUUID) {
                node.tagUUID = uuid_v4();
                node.addEventListener('webkitpresentationmodechanged', (e) => e.stopPropagation(), true);
            }
        }
    }
    
    function sendMessage(message) {
        console.log(message)
    }
    
    function notify(target, type) {
        if (target) {
            var name = target.title;
            if (!name) {
                name = document.title;
            }
            
            if (target.src && target.src !== "") {
                tagNode(target);
                sendMessage({
                    "securitytoken": "$<security_token>",
                    "name": name,
                    "src": target.src,
                    "pageSrc": window.location.href,
                    "pageTitle": document.title,
                    "mimeType": type,
                    "duration": clamp_duration(target.duration),
                    "detected": false,
                    "tagId": target.tagUUID
                });
            }
            else {
                target.querySelectorAll('source').forEach(function(node) {
                    if (node.src && node.src !== "") {
                        if (node.closest('video') === target) {
                            tagNode(target);
                            sendMessage({
                                "securitytoken": "$<security_token>",
                                "name": name,
                                "src": node.src,
                                "pageSrc": window.location.href,
                                "pageTitle": document.title,
                                "mimeType": type,
                                "duration": clamp_duration(target.duration),
                                "detected": false,
                                "tagId": target.tagUUID
                            });
                        }
                        
                        if (node.closest('audio') === target) {
                            tagNode(target);
                            sendMessage({
                                "securitytoken": "$<security_token>",
                                "name": name,
                                "src": node.src,
                                "pageSrc": window.location.href,
                                "pageTitle": document.title,
                                "mimeType": type,
                                "duration": clamp_duration(target.duration),
                                "detected": false,
                                "tagId": target.tagUUID
                            });
                        }
                    }
                });
            }
        }
    }
    
    // TODO(sko) context menu item with extension feature?
    function setupLongPress() {
        // TODO(sko) remove define property here
        Object.defineProperty(window, 'onLongPressActivated', {
            enumerable: false,
            configurable: true,
            value:
            function(localX, localY) {
                function execute(page, offsetX, offsetY) {
                    var target = page.document.elementFromPoint(localX - offsetX, localY - offsetY);
                    var targetVideo = target ? target.closest("video") : null;
                    var targetAudio = target ? target.closest("audio") : null;

                    // Video or Audio might have some sort of overlay..
                    // Like player controls for pause/play, etc..
                    // So we search for video/audio elements relative to touch position.
                    if (!targetVideo && !targetAudio) {
                        var touchX = localX + (page.scrollX + offsetX);
                        var touchY = localY + (page.scrollY + offsetY);
                    
                        var videoElements = page.document.querySelectorAll('video');
                        for (element of videoElements) {
                            var rect = element.getBoundingClientRect();
                            var x = rect.left + (page.scrollX + offsetX);
                            var y = rect.top + (page.scrollY + offsetY);
                            var w = rect.right - rect.left;
                            var h = rect.bottom - rect.top;
                            
                            if (touchX >= x && touchX <= (x + w) && touchY >= y && touchY <= (y + h)) {
                                targetVideo = element;
                                break;
                            }
                        }
                        
                        var audioElements = page.document.querySelectorAll('audio');
                        for (element of audioElements) {
                            var rect = element.getBoundingClientRect();
                            var x = rect.left + (page.scrollX + offsetX);
                            var y = rect.top + (page.scrollY + offsetY);
                            var w = rect.right - rect.left;
                            var h = rect.bottom - rect.top;
                            
                            if (touchX >= x && touchX <= (x + w) && touchY >= y && touchY <= (y + h)) {
                                targetAudio = element;
                                break;
                            }
                        }
                        
                        // No elements found nearby so do nothing..
                        if (!targetVideo && !targetAudio) {
                            // webkit.messageHandlers.handler.postMessage({});
                            return;
                        }
                    }
                    
                    // Elements found
                    if (targetVideo) {
                        tagNode(targetVideo);
                        notify(targetVideo, 'video');
                    }

                    if (targetAudio) {
                        tagNode(targetAudio);
                        notify(targetAudio, 'audio');
                    }
                }
                
                // Any videos in the current `window.document`
                // will have an offset of (0, 0) relative to the window.
                execute(window, 0, 0);
                
                // Any videos in a `iframe.contentWindow.document`
                // will have an offset of (0, 0) relative to its contentWindow.
                // However, it will have an offset of (X, Y) relative to the current window.
                for (frame of document.querySelectorAll('iframe')) {
                    // Get the frame's bounds relative to the current window.
                    var bounds = frame.getBoundingClientRect();
                    execute(frame.contentWindow, bounds.left, bounds.top);
                }
            }
        });
    }
    
    // MARK: ---------------------------------------
    
    function setupDetector() {
        function notifyNodeSource(node, src, mimeType) {
            var name = node.title;
            if (name == null || typeof name == 'undefined' || name == "") {
                name = document.title;
            }

            if (mimeType == null || typeof mimeType == 'undefined' || mimeType == "") {
                if (node.constructor.name == 'HTMLVideoElement') {
                    mimeType = 'video';
                }

                if (node.constructor.name == 'HTMLAudioElement') {
                    mimeType = 'audio';
                }
                
                if (node.constructor.name == 'HTMLSourceElement') {
                    videoNode = node.closest('video');
                    if (videoNode != null && typeof videoNode != 'undefined') {
                        mimeType = 'video'
                    } else {
                        mimeType = 'audio'
                    }
                }
            }

            if (src && src !== "") {
                tagNode(node);
                sendMessage({
                    "securitytoken": "$<security_token>",
                    "name": name,
                    "src": src,
                    "pageSrc": window.location.href,
                    "pageTitle": document.title,
                    "mimeType": mimeType,
                    "duration": clamp_duration(node.duration),
                    "detected": true,
                    "tagId": node.tagUUID
                });
            } else {
                var target = node;
                document.querySelectorAll('source').forEach(function(node) {
                    if (node.src !== "") {
                        if (node.closest('video') === target) {
                            tagNode(target);
                            sendMessage({
                                "securitytoken": "$<security_token>",
                                "name": name,
                                "src": node.src,
                                "pageSrc": window.location.href,
                                "pageTitle": document.title,
                                "mimeType": mimeType,
                                "duration": clamp_duration(target.duration),
                                "detected": true,
                                "tagId": target.tagUUID
                            });
                        }
                        
                        if (node.closest('audio') === target) {
                            tagNode(target);
                            sendMessage({
                                "securitytoken": "$<security_token>",
                                "name": name,
                                "src": node.src,
                                "pageSrc": window.location.href,
                                "pageTitle": document.title,
                                "mimeType": mimeType,
                                "duration": clamp_duration(target.duration),
                                "detected": true,
                                "tagId": target.tagUUID
                            });
                        }
                    }
                });
            }
        }

        function notifyNode(node) {
            notifyNodeSource(node, node.src, node.type);
        }

        function getAllVideoElements() {
            return document.querySelectorAll('video');
        }

        function getAllAudioElements() {
            return document.querySelectorAll('audio');
        }
        
        function requestWhenIdleShim(fn) {
          var start = Date.now()
          return setTimeout(function () {
            fn({
              didTimeout: false,
              timeRemaining: function () {
                return Math.max(0, 50 - (Date.now() - start))
              },
            })
          }, 2000);  // Resolution of 1000ms is fine for us.
        }

        function onReady(fn) {
            if (document.readyState === "complete" || document.readyState === "ready") {
                setTimeout(fn, 1);
            } else {
                document.addEventListener("DOMContentLoaded", fn);
            }
        }
        
        function observePage() {
            console.log("observe page");
            Object.defineProperty(HTMLMediaElement.prototype, 'tagUUID', {
                enumerable: false,
                configurable: false,
                writable: true,
                value: null
            });
            
            var descriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'src');
            Object.defineProperty(HTMLMediaElement.prototype, 'src', {
                enumerable: descriptor.enumerable,
                configurable: descriptor.configurable,
                get: function(){
                    return this.getAttribute('src')
                },
                set: function(value) {
                    // Typically we'd call the original setter.
                    // But since the property represents an attribute, this is okay.
                    this.setAttribute('src', value);
                    //notifyNode(this); // Handled by `setVideoAttribute`
                }
            });
            
            var setVideoAttribute = HTMLVideoElement.prototype.setAttribute;
            HTMLVideoElement.prototype.setAttribute = function(key, value) {
                setVideoAttribute.call(this, key, value);
                if (key.toLowerCase() == 'src') {
                    notifyNode(this);
                }
            };
            
            HTMLVideoElement.prototype.setAttribute.toString = function() {
                return "function () { [native code] }";
            };
            
            var setAudioAttribute = HTMLAudioElement.prototype.setAttribute;
            HTMLAudioElement.prototype.setAttribute = function(key, value) {
                setAudioAttribute.call(this, key, value);
                if (key.toLowerCase() == 'src') {
                    notifyNode(this);
                }
            };
            
            HTMLAudioElement.prototype.setAttribute.toString = function() {
                return "function () { [native code] }";
            };
            
            // When the page is idle
            // Fetch static video and audio elements
            var fetchExistingNodes = () => {
                requestWhenIdleShim((deadline) => {
                    var videos = getAllVideoElements();
                    var audios = getAllAudioElements();
                    if (!videos) {
                        videos = [];
                    }
                    
                    if (!audios) {
                        audios = [];
                    }
                    
                    // Only on the next frame/vsync we notify the nodes
                    requestAnimationFrame(() => {
                        videos.forEach((e) => {
                            notifyNode(e);
                        });
                    });
                    
                    // Only on the next frame/vsync we notify the nodes
                    requestAnimationFrame(() => {
                        audios.forEach((e) => {
                            notifyNode(e);
                        });
                    });
                });
                
                // This function runs only once, so we remove as soon as the page is ready or complete
                document.removeEventListener("DOMContentLoaded", fetchExistingNodes);
            };
            
            // Listen for when the page is ready or complete
            onReady(fetchExistingNodes);
        }

        observePage();
    }
    
    function setupTagNode() {
        // TODO(sko) Remove defineProperty
        Object.defineProperty(window, 'mediaCurrentTimeFromTag', {
            enumerable: false,
            configurable: true,
            value:
            function(tag) {
                for (element of document.querySelectorAll('video')) {
                    if (element.tagUUID == tag) {
                        return clamp_duration(element.currentTime);
                    }
                }
                
                for (element of document.querySelectorAll('audio')) {
                    if (element.tagUUID == tag) {
                        return clamp_duration(element.currentTime);
                    }
                }
                
                return 0.0;
            }
        });
        
        // TODO(sko) Remove defineProperty
        Object.defineProperty(window, 'stopMediaPlayback', {
            enumerable: false,
            configurable: true,
            value:
            function(tag) {
                for (element of document.querySelectorAll('video')) {
                    element.pause();
                }
                
                for (element of document.querySelectorAll('audio')) {
                    element.pause();
                }
                
                return 0.0;
            }
        });
    }
    
    // MARK: -----------------------------
    
    setupLongPress();
    setupDetector();
    setupTagNode();
})();

simplified script: return videos/audios

(function() {
  console.log("HI!!!!");

  function getNodeSource(node, src, mimeType) {
    var name = node.title;
    if (name == null || typeof name == 'undefined' || name == "") {
      name = document.title;
    }

    if (mimeType == null || typeof mimeType == 'undefined' || mimeType == "") {
      if (node.constructor.name == 'HTMLVideoElement') {
        mimeType = 'video';
      }

      if (node.constructor.name == 'HTMLAudioElement') {
        mimeType = 'audio';
      }

      if (node.constructor.name == 'HTMLSourceElement') {
        videoNode = node.closest('video');
        if (videoNode != null && typeof videoNode != 'undefined') {
          mimeType = 'video'
        } else {
          mimeType = 'audio'
        }
      }
    }

    if (src && src !== "") {
      // tagNode(node);
      return {
        "securitytoken": "$<security_token>",
        "name": name,
        "src": src,
        "pageSrc": window.location.href,
        "pageTitle": document.title,
        "mimeType": mimeType,
        //"duration": clamp_duration(node.duration),
        "detected": true,
        "tagId": node.tagUUID
      };
    } else {
      var target = node;
      document.querySelectorAll('source').forEach(function(node) {
        if (node.src !== "") {
          if (node.closest('video') === target) {
            // tagNode(target);
            return {
              "securitytoken": "$<security_token>",
              "name": name,
              "src": node.src,
              "pageSrc": window.location.href,
              "pageTitle": document.title,
              "mimeType": mimeType,
              //"duration": clamp_duration(target.duration),
              "detected": true,
              "tagId": target.tagUUID
            };
          }

          if (node.closest('audio') === target) {
            tagNode(target);
            return {
              "securitytoken": "$<security_token>",
              "name": name,
              "src": node.src,
              "pageSrc": window.location.href,
              "pageTitle": document.title,
              "mimeType": mimeType,
              //"duration": clamp_duration(target.duration),
              "detected": true,
              "tagId": target.tagUUID
            };
          }
        }
      });
    }
  }

  function getNodeData(node) {
    return getNodeSource(node, node.src, node.type);
  }

  function getAllVideoElements() {
    return document.querySelectorAll('video');
  }

  function getAllAudioElements() {
    return document.querySelectorAll('audio');
  }


  let videoElements = getAllVideoElements();
  let audioElements = getAllAudioElements();
  if (!videoElements) {
    videoElements = [];
  }

  if (!audioElements) {
    audioElements = [];
  }
  

  let medias = [...videoElements].map(e => getNodeData(e));
  medias = medias.concat([...audioElements].map(e => getNodeData(e)));
  return medias;

})();

This script will observe a web contents and print an object on console when it finds videos. In youtube, when you play a video, you will see a log on console.

returned object

When “blob:” returned, we can’t useit

{
    "securitytoken": "$<security_token>",
    "name": "brave browser - YouTube",
    "src": "blob:https://www.youtube.com/f4d3b0ce-5fbd-4a9a-b3ae-28e879a22f59",
    "pageSrc": "https://www.youtube.com/watch?v=2KQxl_TwQE8",
    "pageTitle": "brave browser - YouTube",
    "mimeType": "video",
    "duration": 0,
    "detected": true,
    "tagId": "9c8eab2d-57b9-4235-a6f1-40ad459a3c96"
}

iOS workaound : hide mediaSrc api so that we can get downloadable url

https://github.com/brave/brave-ios/blob/development/Client/Frontend/UserContent/UserScripts/PlaylistSwizzler.js

{
    "securitytoken": "$<security_token>",
    "name": "Dave Grohl - Smells Like Teen Spirit (@ the Ford) - YouTube",
    "src": "https://rr1---sn-3u-bh2zs.googlevideo.com/videoplayback?expire=1656356657&ei=0aq5YrbpDe6ZsfIPnZ-RkA8&ip=125.132.182.165&id=o-AAATZdWXcuUX8ZWlz_88Y1HoP8kNvdBHVmXZ8nsZJU_K&itag=18&source=youtube&requiressl=yes&mh=o7&mm=31%2C26&mn=sn-3u-bh2zs%2Csn-oguesnd6&ms=au%2Conr&mv=m&mvi=1&pl=18&ctier=A&pfa=5&initcwndbps=813750&hightc=yes&spc=4ocVCwCFvU919sJikIzNdcAkwfcNsLzUXD17H94uFdCm&vprv=1&mime=video%2Fmp4&ns=2iMPEKuf4G8LcGeZ1w5MftsG&gir=yes&clen=26336521&ratebypass=yes&dur=295.938&lmt=1637412870456462&mt=1656334611&fvip=3&fexp=24001373%2C24007246&c=WEB&txp=5538434&n=bWhyrGYikl844Q&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cctier%2Cpfa%2Chightc%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cratebypass%2Cdur%2Clmt&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&lsig=AG3C_xAwRQIhANLNwwFgbxZXw6GSeYR2dqkA-VIq1iV0vrnfv7cgH9zBAiAwsoDlP5-TPktu9iUB-oetzmySmI1L_W20ivHcay2-9Q%3D%3D&sig=AOq0QJ8wRQIhAPhnrOSiKheu7ijysNgO9CpFJHvUQ8_sUACyE5BMZkORAiB0WcSfHDpTdJoDKn340ZSJfqH_rhou12MzPx4X0tWtpQ%3D%3D&cpn=ZMRfHEli1t6DtWHe&cver=2.20220623.00.00&ptk=youtube_single&oid=ghgcd_xPPR_URSW5U_cwtQ&pltype=contentugc",
    "pageSrc": "https://www.youtube.com/watch?v=oKU1HXMZYm4",
    "pageTitle": "Dave Grohl - Smells Like Teen Spirit (@ the Ford) - YouTube",
    "mimeType": "video",
    "duration": 295.938322,
    "detected": true,
    "tagId": "5bc285eb-b4e3-448d-83a6-d42041e9409d"
}

---

Things to consider

Problem1: when and how we communicate with js.

Playlist Desktop implementation uses ExecuteJavascript() API which is designed to return only once. But our new script installs sort of video detector and can fire events whenever it finds videos. This is useful when DOM is manipulated and <video is created. But as mentioned above, ExecuteJavascript() API is a kind of ‘one-shot’ stuff, we need to consider how we’ll handle this.

other option

#### Option 1: install detector and observe For this purpose, we usually use extension APIs like `chrome.runtime.sendMessage` or custom extension API. And we're thinking of serving the script as "component extension", it may go well with this way. So, my rough idea is

Video Detector Extension (component extension)
- `content script` or `chrome.scripting`: loaded on tab's contents, installs the Detector on loading. 
- background script: communicates with contents script and native side. we may need custom API for native side.
- extension item: when context menu for video element is about to be shown, we can inject extension's menu item.

reference:greaselion

• using brave extension for messaging
• sender: https://github.com/brave/brave-site-specific-scripts/blob/83ce8cb32803189746f4883459ddfadf50afe4fc/scripts/brave_rewards/publisher/common/messaging.ts#L19
• receiver: https://github.com/brave/brave-core/blob/cf8240170e1fd742385864a42785185b7610b256/components/brave_extension/extension/brave_extension/background/greaselion.ts#L329
• greaselion is disabled on Android(enable_extension)

This looks promising for desktop but I’m not sure we can do this on Android.

Note: But we’re not going to use Extension api for “Playlist webui”. The webui should communicate via mojo.

Option 1-1

Replae “sendMessage” from the script based on platform as iOS does.

• desktop: s/sendMessage/chrome.runtime.sendMessage()
• android: s/sendMessage/{TODO}

Option 1-2

Can we fire events from js to native without extension API?

• web api(v8->blink->content)? -> this can have potential security issue.
• js_injection component: haven’t used (components/js_injection/browser/js_communication_host.h
• others??

Option 2: execute script whenever we need

If we’re going to use ExecuteJavascript() we need to execute the script properly. Typically, when LoadFinished we should execute the script. And when dom is change, it’d be good to do this again. Unless we can do this srmartly, we should execute the script when users click a button on location bar.

But I think this way is more easy to apply to Android.

---

Problem2: Thumbnail

Currently, the script that iOS uses doesn’t return us thumbnail path.

Option1: og:image

• Youtube doesn’t provide proper og:image tag for Browser. I think they send diffrent og tag info per user agent.
• Fortunately, when request goes via ApiRequestHelper, proper thumbnail is downloaded.
• But as our script may return multiple videos on a page.

other options

##### Option2: Generate thumbnail url based on rules per sites. * Youtube -> https://img.youtube.com/vi/{video id}/0.jpg

Option3. send another request for the url? pretending other user agent?

Problem3 : adblocker data controller doesn’t exist.

blocking itself is natural.

ibchrome_dll.dylib                 0x000000010b22cc64 brave_shields::BraveShieldsWebContentsObserver::DispatchBlockedEvent(GURL const&, int, std::__Cr::basic_string<char, std::__Cr::char_traits<char>, std::__Cr::allocator<char>> const&) + 480
6   libchrome_dll.dylib                 0x000000010ade2c90 brave::OnShouldBlockRequestResult(bool, scoped_refptr<base::SequencedTaskRunner>, base::RepeatingCallback<void ()> const&, std::__Cr::shared_ptr<brave::BraveRequestInfo>, brave::EngineFlags) + 332
7   libchrome_dll.dylib                 0x000000010ade4bf0 void base::internal::ReplyAdapter<brave::EngineFlags, brave::EngineFlags>(base::OnceCallback<void (brave::EngineFlags)>, std::__Cr::unique_ptr<brave::EngineFlags, std::__Cr::default_delete<brave::EngineFlags>>*) + 172
8   libchrome_dll.dylib                 0x0000000109eced94 base::internal::Invoker<base::internal::BindState<void (*)(base::OnceCallback<web_app::(anonymous namespace)::LoadedConfigs ()>, std::__Cr::unique_ptr<web_app::(anonymous namespace)::LoadedConfigs, std::__Cr::default_delete<web_app::(anonymous namespace)::LoadedConfigs>>*), base::OnceCallback<web_app::(anonymous namespace)::LoadedConfigs ()>, base::internal::UnretainedWrapper<std::__Cr::unique_ptr<web_app::(anonymous namespace)::LoadedConfigs, std::__Cr::default_delete<web_app::(anonymous namespace)::LoadedConfigs>>>>, void ()>::RunOnce(base::internal::BindStateBase*) + 36
9   libbase.dylib                       0x0000000100ce2c44 base::(anonymous namespace)::PostTaskAndReplyRelay::RunReply(base::(anonymous namespace)::PostTaskAndReplyRelay) + 208
10  libbase.dylib                       0x0000000100ce2cbc base::internal::Invoker<base::internal::BindState<void (*)(base::(anonymous namespace)::PostTaskAndReplyRelay), base::(anonymous namespace)::PostTaskAndReplyRelay>, void ()>::RunOnce(base::internal::BindStateBase*) + 68
11  libbase.dylib                       0x0000000100c94324 base::TaskAnnotator::RunTaskImpl(base::PendingTask&) + 328
12  libbase.dylib                       0x0000000100cb9da4 base::sequence_manager::internal::ThreadControllerWithMessagePumpImpl::DoWorkImpl(base::sequence_manager::LazyNow*) + 860

[sangwoo108]

@petemill @simonhong Could you take a look at Things to consider part? I wonder what you think about the messaging.

[sangwoo108]

Let me close this as https://github.com/brave/brave-core/pull/14100 was merged! I’ll file new issue wheeI find something to improve.