added an expert option (eg : await start(expert=True)), which does so…

…me hacking for more experienced users. It disables web security and origin-trials, as well as ensures shadow-roots are always open.

added logic which fixes the user agent in headless mode

docs/_build/html/_modules/index.html

@@ -317,6 +317,7 @@ <h1>All modules for which code is available</h1>
 <li><a href="nodriver/core/_contradict.html">nodriver.core._contradict</a></li>
 <li><a href="nodriver/core/browser.html">nodriver.core.browser</a></li>
 <li><a href="nodriver/core/config.html">nodriver.core.config</a></li>
+<li><a href="nodriver/core/connection.html">nodriver.core.connection</a></li>
 <li><a href="nodriver/core/element.html">nodriver.core.element</a></li>
 <li><a href="nodriver/core/tab.html">nodriver.core.tab</a></li>
 </ul>

docs/_build/html/_modules/nodriver/core/browser.html

@@ -474,7 +474,7 @@ <h1>Source code for nodriver.core.browser</h1><div class="highlight"><pre>
             <span class="n">new_target</span> <span class="o">=</span> <span class="n">Tab</span><span class="p">(</span>
                 <span class="p">(</span>
                     <span class="sa">f</span><span class="s2">&quot;ws://</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">host</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">port</span><span class="si">}</span><span class="s2">&quot;</span>
-                    <span class="sa">f</span><span class="s2">&quot;/devtools/page&quot;</span>  <span class="c1"># all types are &#39;page&#39; internally in chrome apparently</span>
+                    <span class="sa">f</span><span class="s2">&quot;/devtools/</span><span class="si">{</span><span class="n">target_info</span><span class="o">.</span><span class="n">type_</span><span class="w"> </span><span class="ow">or</span><span class="w"> </span><span class="s1">&#39;page&#39;</span><span class="si">}</span><span class="s2">&quot;</span>  <span class="c1"># all types are &#39;page&#39; internally in chrome apparently</span>
                     <span class="sa">f</span><span class="s2">&quot;/</span><span class="si">{</span><span class="n">target_info</span><span class="o">.</span><span class="n">target_id</span><span class="si">}</span><span class="s2">&quot;</span>
                 <span class="p">),</span>
                 <span class="n">target</span><span class="o">=</span><span class="n">target_info</span><span class="p">,</span>
@@ -519,20 +519,24 @@ <h1>Source code for nodriver.core.browser</h1><div class="highlight"><pre>
                 <span class="p">)</span>
             <span class="p">)</span>
             <span class="c1"># get the connection matching the new target_id from our inventory</span>
-            <span class="n">connection</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
+            <span class="n">connection</span><span class="p">:</span> <span class="n">tab</span><span class="o">.</span><span class="n">Tab</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
                 <span class="nb">filter</span><span class="p">(</span>
                     <span class="k">lambda</span> <span class="n">item</span><span class="p">:</span> <span class="n">item</span><span class="o">.</span><span class="n">type_</span> <span class="o">==</span> <span class="s2">&quot;page&quot;</span> <span class="ow">and</span> <span class="n">item</span><span class="o">.</span><span class="n">target_id</span> <span class="o">==</span> <span class="n">target_id</span><span class="p">,</span>
                     <span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">,</span>
                 <span class="p">)</span>
             <span class="p">)</span>
+            <span class="n">connection</span><span class="o">.</span><span class="n">browser</span> <span class="o">=</span> <span class="bp">self</span>
 
         <span class="k">else</span><span class="p">:</span>
             <span class="c1"># first tab from browser.tabs</span>
-            <span class="n">connection</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">item</span><span class="p">:</span> <span class="n">item</span><span class="o">.</span><span class="n">type_</span> <span class="o">==</span> <span class="s2">&quot;page&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">))</span>
+            <span class="n">connection</span><span class="p">:</span> <span class="n">tab</span><span class="o">.</span><span class="n">Tab</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
+                <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">item</span><span class="p">:</span> <span class="n">item</span><span class="o">.</span><span class="n">type_</span> <span class="o">==</span> <span class="s2">&quot;page&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">)</span>
+            <span class="p">)</span>
             <span class="c1"># use the tab to navigate to new url</span>
             <span class="n">frame_id</span><span class="p">,</span> <span class="n">loader_id</span><span class="p">,</span> <span class="o">*</span><span class="n">_</span> <span class="o">=</span> <span class="k">await</span> <span class="n">connection</span><span class="o">.</span><span class="n">send</span><span class="p">(</span><span class="n">cdp</span><span class="o">.</span><span class="n">page</span><span class="o">.</span><span class="n">navigate</span><span class="p">(</span><span class="n">url</span><span class="p">))</span>
             <span class="c1"># update the frame_id on the tab</span>
             <span class="n">connection</span><span class="o">.</span><span class="n">frame_id</span> <span class="o">=</span> <span class="n">frame_id</span>
+            <span class="n">connection</span><span class="o">.</span><span class="n">browser</span> <span class="o">=</span> <span class="bp">self</span>
 
         <span class="k">await</span> <span class="n">connection</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mf">0.25</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">connection</span></div>
@@ -545,6 +549,7 @@ <h1>Source code for nodriver.core.browser</h1><div class="highlight"><pre>
         <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="p">:</span>
             <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;use ``await Browser.create()`` to create a new instance&quot;</span><span class="p">)</span>
             <span class="k">return</span>
+
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_pid</span><span class="p">:</span>
             <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process</span><span class="o">.</span><span class="n">returncode</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                 <span class="k">return</span> <span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">config</span><span class="p">)</span>
@@ -777,7 +782,8 @@ <h1>Source code for nodriver.core.browser</h1><div class="highlight"><pre>
     <span class="k">async</span> <span class="k">def</span> <span class="nf">update_targets</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="n">targets</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">cdp</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">TargetInfo</span><span class="p">]</span>
         <span class="n">targets</span> <span class="o">=</span> <span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_targets</span><span class="p">()</span>
-
+        <span class="n">target_ids</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span><span class="o">.</span><span class="n">target_id</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">targets</span><span class="p">]</span>
+        <span class="n">existing_target_ids</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span><span class="o">.</span><span class="n">target_id</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">]</span>
         <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">targets</span><span class="p">:</span>
             <span class="k">for</span> <span class="n">existing_tab</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">:</span>
                 <span class="n">existing_target</span> <span class="o">=</span> <span class="n">existing_tab</span><span class="o">.</span><span class="n">target</span>

docs/_build/html/_modules/nodriver/core/config.html

@@ -307,6 +307,7 @@ <h1>Source code for nodriver.core.config</h1><div class="highlight"><pre>
         <span class="n">lang</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;en-US&quot;</span><span class="p">,</span>
         <span class="n">host</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">AUTO</span><span class="p">,</span>
         <span class="n">port</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">AUTO</span><span class="p">,</span>
+        <span class="n">expert</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">AUTO</span><span class="p">,</span>
         <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="nb">dict</span><span class="p">,</span>
     <span class="p">):</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
@@ -327,6 +328,10 @@ <h1>Source code for nodriver.core.config</h1><div class="highlight"><pre>
 <span class="sd">        :param sandbox: disables sandbox</span>
 <span class="sd">        :param autodiscover_targets: use autodiscovery of targets</span>
 <span class="sd">        :param lang: language string to use other than the default &quot;en-US,en;q=0.9&quot;</span>
+<span class="sd">        :param expert: when set to True, enabled &quot;expert&quot; mode.</span>
+<span class="sd">               This conveys, the inclusion of parameters: --disable-web-security ----disable-site-isolation-trials,</span>
+<span class="sd">               as well as some scripts and patching useful for debugging (for example, ensuring shadow-root is always in &quot;open&quot; mode)</span>
+
 <span class="sd">        :param kwargs:</span>
 
 <span class="sd">        :type user_data_dir: PathLike</span>
@@ -357,6 +362,7 @@ <h1>Source code for nodriver.core.config</h1><div class="highlight"><pre>
         <span class="bp">self</span><span class="o">.</span><span class="n">sandbox</span> <span class="o">=</span> <span class="n">sandbox</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">host</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">port</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">expert</span> <span class="o">=</span> <span class="n">expert</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">_extensions</span> <span class="o">=</span> <span class="p">[]</span>
         <span class="c1"># when using posix-ish operating system and running as root</span>
         <span class="c1"># you must use no_sandbox = True, which in case is corrected here</span>
@@ -444,10 +450,12 @@ <h1>Source code for nodriver.core.config</h1><div class="highlight"><pre>
         <span class="c1"># the browser, as by the time it starts, the port</span>
         <span class="c1"># is probably already taken</span>
         <span class="n">args</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_default_browser_args</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+
         <span class="n">args</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">&quot;--user-data-dir=</span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">user_data_dir</span><span class="p">]</span>
         <span class="n">args</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">&quot;--disable-features=IsolateOrigins,site-per-process&quot;</span><span class="p">]</span>
         <span class="n">args</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">&quot;--disable-session-crashed-bubble&quot;</span><span class="p">]</span>
-
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">expert</span><span class="p">:</span>
+            <span class="n">args</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">&quot;--disable-web-security&quot;</span><span class="p">,</span> <span class="s2">&quot;--disable-site-isolation-trials&quot;</span><span class="p">]</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_browser_args</span><span class="p">:</span>
             <span class="n">args</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="n">arg</span> <span class="k">for</span> <span class="n">arg</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_browser_args</span> <span class="k">if</span> <span class="n">arg</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">args</span><span class="p">])</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">headless</span><span class="p">:</span>