BrowserHelper.cs

// =============================================================================
// GPAL - Generally Positive Automation Library
// Copyright © 2026 Software Decisions, Inc. All rights reserved.
//
// This file is part of GPAL.
// Licensed under the Business Source License 1.1
//
// Primary development, architecture, and vision by Michael B. Vederman,
// CEO of Software Decisions, Inc., Texas.
//
// Internal development maintained privately.
// Public releases appear on GitHub: https://github.com/SoftwareDecisionsInc/GPAL.
//
// See LICENSE for full terms, including Additional Use Grant.
// =============================================================================
 
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Sockets;
using System.Net.WebSockets;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Controls.Primitives;
using System.Windows.Forms;
using DocumentFormat.OpenXml.Bibliography;
using Microsoft.Win32;
using Newtonsoft.Json.Linq;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.Chromium;
using OpenQA.Selenium.DevTools;
using OpenQA.Selenium.Edge;
using OpenQA.Selenium.Firefox;
using OpenQA.Selenium.Support.UI;
using static GenerallyPositive.Browser.BrowserHelper;
using static GenerallyPositive.Enums;
using System.Net.NetworkInformation;
using System.Net;
using System.Text.RegularExpressions;
 
 
namespace GenerallyPositive.Browser
{
    public class BrowserHelper
    {
        static bool alreadyUpdating = false;
 
        static List<string> lastErrorMessage = new List<string>();
        static bool supressedMessage = false;
        static string lastSessionToken;

        public static IWebDriver GetBrowserDriver(BrowserSettings browserSettings)
        {
            IWebDriver tmpDriver = null;
            ChromiumOptions options = null;
            bool waitForDebugger = false;
            string userAgent = GPAL.GPALSettings.UserAgent;
 
            if (null != browserSettings.BrowserDriver)
                return browserSettings.BrowserDriver;
 
            // every launch, not only the ones that ask for a temp profile, so a run that supplies its own profile
            // still clears what a killed run left behind
            ChromeProfileManager.SweepAbandonedTempProfiles();
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Running workflow for Automation Engine [{browserSettings.AutomationEngine}]", null, GPALObjectType.None);
 
            if (null == browserSettings.DriverLocation)
                // driver is located with executable
                browserSettings.DriverLocation = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().GetName().CodeBase).Substring(6);    // skip file:\\
 
            if (true == GPAL.GPALSettings.AutoUpdateWebDriver && false == alreadyUpdating)
            {
                alreadyUpdating = true;
                DriverHelper.UpdateDriver(browserSettings);
                // NOTE: no more port overlap, can have multiple running, each one exits when its own browser closes
                // its stdin, no need to kill any longer, we'll just get the next available port
                // BrowserHelper.KillProcess("GPALRestAPI");   // could be started by browser that has ottomagic installed
            }
 
            try
            {
                if (BrowserType.Chrome == browserSettings.BrowserType || BrowserType.Edge == browserSettings.BrowserType)
                {
                    if (BrowserType.Chrome == browserSettings.BrowserType)
                        options = new ChromeOptions();
                    else
                        options = new EdgeOptions();
 
                    // the driver answers alert, confirm and prompt itself, which is a webdriver capability and
                    // not something the page can see. nothing is overridden and nothing is injected
                    if (null != browserSettings.DialogsAccepted)
                        options.UnhandledPromptBehavior = true == browserSettings.DialogsAccepted
                            ? UnhandledPromptBehavior.Accept
                            : UnhandledPromptBehavior.Dismiss;
 
                    if (true == browserSettings.UseExistingBrowser)
                    {
                        options.DebuggerAddress = "127.0.0.1:" + browserSettings.ExistingBrowserPort.ToString();
                        waitForDebugger = true;
                    }
                    else
                    {
                        if (null == browserSettings.DebugPort && true == browserSettings.UsePuppeteer && false == browserSettings.DebugPipe)  // make sure puppeteerURL is properly set - if debugport is set, it will update url, if not, set the debugport to default
                        {
                            // the first free port from 0xdead up, so this is still 57005 on a quiet machine and
                            // moves along when a browser from another run or another bot is already on it
                            browserSettings.DebugPort = FindFreePort();
                            browserSettings.PuppeteerUrl = $"http://127.0.0.1:{browserSettings.DebugPort}";
                        }

                        if (false == string.IsNullOrEmpty(browserSettings.DownloadLocation))
                            options.AddUserProfilePreference("download.default_directory", browserSettings.DownloadLocation);
 
                        // NOTE: chrome 136 requires a user-data-dir for security reasons
                        // Profile settings
                        if (false == string.IsNullOrEmpty(browserSettings.ProfileUserName) || false == string.IsNullOrEmpty(browserSettings.ProfileName))
                        {
                            browserSettings.ProfileDataDirectory = GetBrowserProfileDirectory(browserSettings);
 
                            if (false == string.IsNullOrEmpty(browserSettings.ProfileName))
                            {
                                // the profile name might not be what we expect, which is a directory name.
                                // this will iterate thru all directories under user-data-dir, open the Preferences file and try to match token profile.name
                                // if it matches, then the directory containing that Preferences file is returned
                                // FindProfileDirectory returns a directory or throws
                                options.AddArguments($"--profile-directory={FindProfileDirectory(browserSettings.ProfileDataDirectory, browserSettings.ProfileName)}");
                            }
                        }
                        else if (true == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory))
                        {
                            // Create temporary profile if none specified
                            browserSettings.ProfileDataDirectory = ChromeProfileManager.CreateTempUserProfile(
                                browserSettings.DownloadLocation ?? FileHelper.GetDefaultDownloadDirectory(browserSettings.Browser),
                                browserSettings.PromptForDownload,
                                browserSettings.OpenPDFExternally,
                                browserSettings.LoadImages,
                                browserSettings.UseOttoMagic
                            );
                            browserSettings.TempProfileCreated = true;
                        }
 
                        // a supplied profile is the user's own. selenium writes the download preferences into it at
                        // launch and chrome keeps them, so read what is there now and Browser.Close puts it back.
                        // a temp profile is deleted on close, so there is nothing to preserve there
                        if (false == browserSettings.TempProfileCreated && false == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory))
                            browserSettings.PreviousDownloadPreferences = ChromeProfileManager.ApplyDownloadPreferences(
                                browserSettings.ProfileDataDirectory,
                                browserSettings.DownloadLocation,
                                browserSettings.PromptForDownload,
                                browserSettings.OpenPDFExternally);
 
                        // a profile can only be owned by one browser. launching onto one that is already open hands
                        // the command line to that browser and exits, so the debug port never opens and the run
                        // waits for a browser that was never started. said here, where it can still be acted on
                        if (false == browserSettings.TempProfileCreated)
                            RefuseIfProfileIsOpen(browserSettings);
 
                        if (false == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory))
                            options.AddArgument($"--user-data-dir={browserSettings.ProfileDataDirectory.Replace("\\", "/")}");
                        /* prolly no longer a use case for this - edge has some annoying popups that block selenium unless logged in as guest
                                                else if (true == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory) && BrowserType.Edge == browserSettings.BrowserType)
                                                    options.AddArgument("--guest");     // in msedge, an annoyting 'personalize your experience' nag and options bar pops up blocking automation, this prevents that
                        */
 
                        // *******************************************************************
                        // NOTE: CAVEAT: suppose a time comes to run with some sort of extension, this isn't causing any harm, just annoying...
                        // not keen on disabling extensions, they make the bot look alive, but somehow we score low on recaptcha with ottomagic, no idea why, puppeteer is awesome
                        // *******************************************************************
                        options.AddArguments("--disable-extensions");    // no ottomagic - selenium and gpalrestapi/ottomagic DO NOT get along at all
                                                                         // *******************************************************************
                                                                         // *******************************************************************
                        // one --disable-features switch only, chromium takes the last one rather than merging them.
                        // edge's shopping content scripts mutate retail pages constantly, which is what wedges
                        // DOM.getDocument on an item page that chrome handles fine
                        string disableFeatures = "TranslateUI,Translate";
 
                        if (BrowserType.Edge == browserSettings.BrowserType)
                        {
                            disableFeatures += ",msShoppingTrigger,msShopping,msEdgeShoppingUI,msEdgeShoppingList";
 
                            // Edge implicit sign-in, which --disable-sync does not cover
                            if (true == browserSettings.TempProfileCreated)
                                disableFeatures += ",msImplicitSignin,msEdgeImplicitSignin,msIdentityImplicitSignin";
                        }
 
                        options.AddArgument($"--disable-features={disableFeatures}");
 
                        // Edge signs a new profile in with the Windows account and syncs favourites, settings
                        // and extensions down onto it. Applied to temp profiles only; a supplied profile keeps
                        // whatever sync settings it has
                        if (true == browserSettings.TempProfileCreated)
                        {
                            options.AddArgument("--disable-sync");
                            options.AddArgument("--disable-signin-promo");
                        }
 
                        // Disable infobars and any chrome overlays
                        options.AddArgument("--disable-infobars");
                        options.AddArgument("--disable-session-crashed-bubble");
                        options.AddArgument("--hide-crash-restore-bubble");
                        options.AddArgument("--noerrdialogs");
 
                        if (true == browserSettings.UseHeadless)
                        {
                            // Set headless mode
                            options.AddArgument("--headless=new");
                            // Disable GPU
                            //options.AddArgument("--disable-gpu");
                            // Disable software rasterizer
                            //options.AddArgument("--disable-software-rasterizer");
                            // Disable sandbox
                            //options.AddArgument("--no-sandbox");
 
                            // Disable notifications
                            options.AddArgument("--disable-notifications");
                            // Disable popup blocking
                            options.AddArgument("--disable-popup-blocking");
                            // Disable setuid sandbox
                            //options.AddArgument("--disable-setuid-sandbox");
 
                            // a headless browser says so in its user agent, so the word has to come out of it.
                            // Reading it from a real browser costs a whole launch, so that only happens when the
                            // workflow asked for it; otherwise the answer comes from what is already known
                            string currUserAgent;
 
                            if (false == browserSettings.UserAgentFromBrowser)
                                currUserAgent = ResolveUserAgent(browserSettings);
                            else
                            {
                                if (BrowserType.Chrome == browserSettings.BrowserType)
                                    tmpDriver = new ChromeDriver(browserSettings.DriverLocation, (ChromeOptions)options);
                                else
                                    tmpDriver = new EdgeDriver(browserSettings.DriverLocation, (EdgeOptions)options);
 
                                IJavaScriptExecutor js = (IJavaScriptExecutor)tmpDriver;
                                currUserAgent = (String)js.ExecuteScript("return navigator.userAgent");
                                tmpDriver.Quit();
                            }
 
                            options.AddArgument($"--user-agent={currUserAgent.Replace("Headless", "")}");
 
                            options.AddUserProfilePreference("download.prompt_for_download", false);
                        }
                        else
                        {
                            if (true == browserSettings.PromptForDownload)
                                options.AddUserProfilePreference("download.prompt_for_download", true);
                            else
                                options.AddUserProfilePreference("download.prompt_for_download", false);
                        }
 
                        // default for chrome is false - so turn on this safeguard
                        // Purpose: When set to true, it allows Chrome to automatically handle cases where the specified download directory
                        // (set via "download.default_directory") is invalid, inaccessible, or doesn't exist. Chrome may attempt to upgrade
                        // or fall back to a valid directory, ensuring downloads proceed without errors.
                        options.AddUserProfilePreference("download.directory_upgrade", true);
 
                        options.AddUserProfilePreference("plugins.always_open_pdf_externally", browserSettings.OpenPDFExternally);
                        options.AddArguments($"--blink-settings=imagesEnabled={browserSettings.LoadImages.ToString().ToLower()}");
                        options.AddUserProfilePreference("disable-popup-blocking", $"{!browserSettings.BlockPopUps}");
 
                        // turn on dark mode
                        if (browserSettings.StealthType.HasFlag(StealthType.DarkMode))
                            options.AddArgument("--force-dark-mode");
 
                        // Set remote debugging pipe or portt
                        if (true == browserSettings.DebugPipe)
                            options.AddArgument($"--remote-debugging-pipe  --enable-unsafe-extension-debugging");
                        else if (null != browserSettings.DebugPort)
                            options.AddArgument($"--remote-debugging-port={browserSettings.DebugPort.Value}");
 
                        //options.AddUserProfilePreference("safebrowsing.enabled", true); // CAVEAT: should this be configurable? disables warnings while downloading files... 'this may harm' seems we don't want that during automation
 
                        // obscure anti-bot detections
                        // Adding argument to disable the AutomationControlled flag 
                        // no longer supported, throws infobar message 5/12/25
                        options.AddArguments("--disable-blink-features=AutomationControlled");
 
                        // Exclude the collection of enable-automation switches
                        options.AddExcludedArgument("enable-automation");
                        if (options is ChromeOptions chromeOpts)
                            chromeOpts.AddAdditionalChromeOption("useAutomationExtension", false);
                        else if (options is EdgeOptions edgeOpts)
                            edgeOpts.AddAdditionalEdgeOption("useAutomationExtension", false);
                    }
 
                    if (browserSettings.StealthType.HasFlag(StealthType.PatchDriver))
                    {
                        if (BrowserType.Edge == browserSettings.BrowserType)
                            EdgePatcher.PatchEdgeDriver(browserSettings.Browser);
                        else
                            ChromePatcher.PatchChromeDriver(browserSettings.Browser);
                    }
 
                    // selenium will not accept a url as a commandline option, adds -- in front, there is no
                    // options.AddArgument(browserSettings.CurrentURL);    // doesn't always work, some enterprise browsers have a mandatory first page that bypasses commandline url
 
                    // Use this to query the logs to get the server response code for the goto
                    // NOTE: CAVEAT: this might make us more detectable in selenium??
                    // this is not a required feature, to return the navigation status code in SeleniumGoto
                    options.SetLoggingPreference(OpenQA.Selenium.LogType.Performance, LogLevel.All);
 
                    tmpDriver = GetWebDriver(browserSettings, options);
 
                    //if (BrowserType.Chrome == browserSettings.BrowserType)
                    //    tmpDriver = new ChromeDriver(browserSettings.DriverLocation, (ChromeOptions)options);
                    //else
                    //    tmpDriver = new EdgeDriver(browserSettings.DriverLocation, (EdgeOptions)options);
                }
                else if (BrowserType.FireFox == browserSettings.BrowserType)
                {
                    // read more: https://github.com/mozilla/geckodriver/issues/430
                    FirefoxDriverService firefoxDriverService = FirefoxDriverService.CreateDefaultService(browserSettings.DriverLocation);
                    FirefoxOptions ffOptions = new FirefoxOptions();
 
                    if (null != browserSettings.DialogsAccepted)
                        ffOptions.UnhandledPromptBehavior = true == browserSettings.DialogsAccepted
                            ? UnhandledPromptBehavior.Accept
                            : UnhandledPromptBehavior.Dismiss;
                    if (true == browserSettings.UseExistingBrowser)
                    {
                        firefoxDriverService.HideCommandPromptWindow = true;
                        firefoxDriverService.Port = 0;                          // let the driver take any free port for its own http server
                        firefoxDriverService.Host = "127.0.0.1";
                        firefoxDriverService.ConnectToRunningBrowser = true;
                        // marionette is Firefox's side of the connection, the same role the debugger port plays for
                        // Chrome and Edge above. 2828 is Firefox's own default when the workflow did not name one
                        firefoxDriverService.BrowserCommunicationPort = browserSettings.ExistingBrowserPort ?? 2828;
 
                        tmpDriver = new FirefoxDriver(firefoxDriverService, ffOptions);
 
                        waitForDebugger = true;
                    }
                    else
                    {
                        // Set window size
                        ffOptions.AddArgument($"-width={browserSettings.WindowSize.Width}");
                        ffOptions.AddArgument($"-height={browserSettings.WindowSize.Height}");
 
                        if (true == browserSettings.UseHeadless)
                        {
 
                            // Set headless mode
                            ffOptions.AddArgument("-headless");
 
                            // Disable notifications
                            ffOptions.AddArgument("-disable-notifications");
 
                            // Set user agent - this makes firefox not start byt gives error
                            // if (null != userAgent)
                            //   ffOptions.AddArgument($"--user-agent={userAgent}");
                        }
 
                        if (null != browserSettings.DebugPort)
                        {
                            firefoxDriverService.Port = (int)browserSettings.DebugPort.Value;
 
                            if (true == browserSettings.UsePuppeteer)
                                browserSettings.PuppeteerUrl = $"http://127.0.0.1:{browserSettings.DebugPort}";
 
                            if (string.IsNullOrEmpty(browserSettings.ProfileDataDirectory) && string.IsNullOrEmpty(browserSettings.ProfileName) && string.IsNullOrEmpty(browserSettings.ProfileUserName))
                                ffOptions.Profile = new OpenQA.Selenium.Firefox.FirefoxProfileManager().GetProfile("default");
                        }
                        else if (true == browserSettings.UsePuppeteer)  // make sure puppeteerURL is properly set - if debugport is set, it will update url, if not, set the debugport to default
                        {
                            // the first free port from 0xdead up, so this is still 57005 on a quiet machine and
                            // Skips ports already bound by another process. Called once per browser: a second
                            // call would reserve a port nothing uses
                            if (null == browserSettings.DebugPort)
                                browserSettings.DebugPort = FindFreePort();
 
                            browserSettings.PuppeteerUrl = $"http://127.0.0.1:{browserSettings.DebugPort}";
                        }
 
                        if (null != browserSettings.DebugPort)
                            ffOptions.AddArgument($"--remote-debugging-port={browserSettings.DebugPort.Value} ");
 
                        ffOptions.SetPreference("print.always_print_silent", true);
                        ffOptions.SetPreference("print.show_print_progress", false);
 
 
                        // headless turns the viewer off whatever the workflow asked. there is nobody to render a
                        // pdf for, and Get() sets OpenPDFExternally false for headless, which would otherwise
                        // switch pdfjs back on and leave every pdf displayed to no one instead of downloaded
                        ffOptions.SetPreference("pdfjs.disabled", browserSettings.OpenPDFExternally || browserSettings.UseHeadless);
 
                        if (false == browserSettings.PromptForDownload)
                            ffOptions.SetPreference("browser.helperApps.neverAsk.saveToDisk", GPAL.GPALSettings.FirefoxDirectDownloadMimeTypes);
                        else
                            ffOptions.SetPreference("browser.helperApps.neverAsk.saveToDisk", "");
 
                        // false is firefox for "ask where to save every file", which is the save dialog. the
                        // OttoMagic path writes this one too; without it here a prompted download on selenium
                        // firefox depended on whatever the profile already said
                        ffOptions.SetPreference("browser.download.useDownloadDir", false == browserSettings.PromptForDownload);
 
                        if (false == browserSettings.LoadImages)
                            ffOptions.SetPreference("permissions.default.image", 0);
                        else
                            ffOptions.SetPreference("permissions.default.image", 1);
 
                        // how many popups one event may open. zero is none, and twenty is what firefox ships
                        // with, so blocking is the low number and the sense of this reads backwards
                        ffOptions.SetPreference("dom.popup_maximum", true == browserSettings.BlockPopUps ? 0 : 20);
 
                        ffOptions.SetPreference("safebrowsing.enabled", false); // TODO: should this be configurable? disables warnings while downloading files... 'this may harm' seems we don't want that during automation
 
                        if (false == string.IsNullOrEmpty(browserSettings.DownloadLocation))
                        {
                            ffOptions.SetPreference("browser.download.dir", browserSettings.DownloadLocation);
                            ffOptions.SetPreference("browser.download.folderList", 2); // 2: Use the custom download directory
                        }
 
                        ffOptions.SetPreference("dom.webdriver.enabled", false);
 
                        // -profile and the path go in as two arguments. One argument holding both reads as an
                        // unknown flag to firefox, which then starts on a profile of its own while geckodriver
                        // watches the named one for the marionette port. A trailing separator escapes the closing
                        // quote windows puts around the argument, so it comes off first
                        if (false == string.IsNullOrEmpty(browserSettings.ProfileUserName) || false == string.IsNullOrEmpty(browserSettings.ProfileName))
                            // browserSettings.ProfileDataDirectory = GetBrowserProfileDirectory(browserSettings); // not sure this is applicable to firefox
                            ffOptions.AddArguments("-profile", (browserSettings.ProfileName ?? browserSettings.ProfileUserName).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
                        else if (false == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory))
                            ffOptions.AddArguments("-profile", browserSettings.ProfileDataDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
 
                        // Ensure that debugging is enabled in Firefox, F12, ..., settings, enable browser chrome and bugging toolboxes checkbox, enable remote debugging checkbox
                        //options.AddArguments("-foreground");
                        //options.AddArguments("-start-debugger-server 2828");
                        //tmpDriver = new FirefoxDriver(browserSettings.driverLocation, options);
 
                        // kludge on opening first window
                        //Uri safe = RemoteWebDriverExtended.FirefoxURL;
                        //RemoteWebDriverExtended.FirefoxURL = new Uri(browserSettings.currentURL);
                        //tmpDriver = RemoteWebDriverExtended.GetFireFoxDriver(browserSettings.driverLocation);
                        //RemoteWebDriverExtended.FirefoxURL = safe;
                    }
 
                    ffOptions.BinaryLocation = FindFirefoxBinaryLocation();
 
//                    tmpDriver = new FirefoxDriver(browserSettings.DriverLocation, ffOptions);   // would ever start up a session with no options?
                                                                                                // Explicit service for Firefox to get geckodriver PID
                    if (true == browserSettings.HiddenDesktop)
                    {
                        // the driver is started by hand so it can be named a desktop, and Selenium is handed the
                        // port rather than a service. StartHiddenDriver sets ServiceDriverPid from the launch
                        int hiddenPort = StartHiddenDriver(browserSettings);
 
                        tmpDriver = new OpenQA.Selenium.Remote.RemoteWebDriver(new Uri($"http://127.0.0.1:{hiddenPort}"), ffOptions);
                    }
                    else
                    {
                        FirefoxDriverService firefoxService = FirefoxDriverService.CreateDefaultService(browserSettings.DriverLocation);
                        firefoxService.HideCommandPromptWindow = true;  // hides the console window
 
                        tmpDriver = new FirefoxDriver(firefoxService, ffOptions);
 
                        // Capture geckodriver.exe PID — reuse the same property
                        browserSettings.ServiceDriverPid = firefoxService.ProcessId;
                    }
 
                    // and firefox's own, which it reports as a capability. the driver launches the browser, so
                    // nothing else here has a handle on it, and every fallback in the teardown is written around
                    // having one: with no handle there is no wait for it to go, no warning that it did not, and
                    // no kill. geckodriver is killed by pid at the end, and a firefox that outlived its driver is
                    // then orphaned and runs on
                    browserSettings.Process = SeleniumBrowserProcess(tmpDriver, "moz:processID");
 
                    //tmpDriver = new FirefoxDriver(firefoxDriverService, ffOptions);
                }
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to start browser driver for [{browserSettings.BrowserType}].", browserSettings.Browser, GPALObjectType.Browser, ex);
                // unrecoverable error, the only time we rethrow
                // return null;
                throw;
            }
            if (null != tmpDriver)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Browser [{browserSettings.BrowserType}] launched.", browserSettings, GPALObjectType.Other);
 
                browserSettings.BrowserDriver = tmpDriver;
 
                // Edge preloads its New Tab Page into a hidden background window that appears asynchronously as an
                // extra WindowHandle and corrupts our sequential tab-index tracking. At this instant (before we
                // navigate) the driver's url is that preload's url, so capture it as a runtime signature - later we
                // positively match the phantom tab by this url, never by position (which races the async insert)
                // nor a hardcoded string.
                var edgeGhostBrowser = browserSettings.Browser as Browser;
                if (BrowserType.Edge == browserSettings.BrowserType && null != edgeGhostBrowser)
                    try { edgeGhostBrowser.GhostTabUrl = tmpDriver.Url; }
                    catch { /* non-fatal: without a signature we simply never skip a ghost */ }
 
                // pageload timeout to connect 30 seconds
                tmpDriver.Manage().Timeouts().PageLoad.Add(System.TimeSpan.FromSeconds(30));    // NOTE: hardcoded value
 
                //BrowserHelper.SeleniumGoToUrl(browserSettings.CurrentURL, browserSettings);
 
                browserSettings.CurrentURL = "about:blank";
 
                if (true == browserSettings.Maximize)
                    tmpDriver.Manage().Window.Maximize();
                else if (true == browserSettings.Minimize)
                    tmpDriver.Manage().Window.Minimize();
                else if (true == browserSettings.FullScreen)
                    tmpDriver.Manage().Window.FullScreen();
                else if (false == browserSettings.WindowSize.IsEmpty)
                {
                    // Set the desired window size
                    var windowSize = new Size(browserSettings.WindowSize.Width, browserSettings.WindowSize.Height);
                    tmpDriver.Manage().Window.Size = windowSize;
 
                    // Set the desired window position (top-left corner). Point is x then y, which is Left then Top
                    var windowPosition = new Point(browserSettings.WindowSize.Left, browserSettings.WindowSize.Top);
                    tmpDriver.Manage().Window.Position = windowPosition;
                }
 
                if (true == waitForDebugger)
                    Thread.Sleep(2000);
 
                if (BrowserType.Chrome == browserSettings.BrowserType)
                {
                    DriverHelper.GetChromeVersion(out browserSettings.Version);
                }
                else if (BrowserType.Edge == browserSettings.BrowserType)
                {
                    DriverHelper.GetEdgeVersion(out browserSettings.Version);
                }
                else if (BrowserType.FireFox == browserSettings.BrowserType)
                {
                    DriverHelper.GetFirefoxVersion(out browserSettings.Version);
                }
 
                CheckDocumentReady(((Browser)browserSettings.Browser), true);
            }
 
            return tmpDriver;
        }

        public static void AllowDownloads(BrowserSettings browserSettings, string directory)
        {
            var parameters = new Dictionary<string, object>
            {
                { "behavior", "allow" },
                { "downloadPath", directory }
            };
 
            ((OpenQA.Selenium.Chromium.ChromiumDriver)browserSettings.BrowserDriver).ExecuteCdpCommand("Page.setDownloadBehavior", parameters);
        }

        internal static string ResolveUserAgent(BrowserSettings browserSettings)
        {
            string userAgent = browserSettings.OverrideUserAgent;
 
            if (true == string.IsNullOrWhiteSpace(userAgent))
                userAgent = GPAL.GPALSettings.UserAgent;
 
            if (true == string.IsNullOrWhiteSpace(userAgent))
                userAgent = MagicHelper.GetUserAgentString(browserSettings.BrowserType);
 
            return userAgent;
        }
        internal static void PresentCredentials(Browser browser, GPALUrl url)
        {
            Credentials credential = (Credentials)browser.BrowserSettings.Credentials;
            WebAuthType authType = credential.WebAuthType;
            string header = AuthorizationHeader(credential, authType);
 
            browser.BrowserSettings.CredentialsPresented = DateTime.UtcNow;
 
            // a form is a page, and a page is the workflow's job. Holding the credential is all that was asked for
            if (WebAuthType.Form == authType || WebAuthType.None == authType)
                return;
 
            if (true == string.IsNullOrEmpty(header) || BrowserType.FireFox == browser.BrowserSettings.BrowserType || true == browser.UseOttoMagic)
                CredentialsInUrl(browser, url, credential);
            else
            {
                string name = AuthorizationHeaderName(authType);
                Dictionary<string, object> headers = new Dictionary<string, object> { { name, header } };
 
                if (true == browser.UsePuppeteer)
                    browser.PuppeteerCommunicator.SetExtraHttpHeaders(headers).GetAwaiter().GetResult();
                else
                    ((OpenQA.Selenium.Chromium.ChromiumDriver)browser.BrowserDriver).ExecuteCdpCommand("Network.setExtraHTTPHeaders",
                        new Dictionary<string, object> { { "headers", headers } });
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Credentials presented as [{authType}] for [{url?.Url}]", browser, GPALObjectType.Browser);
            }
        }
        internal static string AuthorizationHeader(Credentials credential, WebAuthType authType)
        {
            string value = null;
 
            if (WebAuthType.Basic == authType || WebAuthType.Proxy == authType)
                value = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{credential.Username}:{credential.Password}"));
            else if (WebAuthType.Bearer == authType)
                value = "Bearer " + credential.AccessToken;
            else if (WebAuthType.XAuthToken == authType || WebAuthType.XApiKey == authType || WebAuthType.ApiKey == authType)
                value = credential.AccessToken;      // their own header, so no scheme word in front of it
 
            return value;
        }

        internal static string AuthorizationHeaderName(WebAuthType authType)
        {
            string name = "Authorization";
 
            if (WebAuthType.Proxy == authType)
                name = "Proxy-Authorization";
            else if (WebAuthType.XAuthToken == authType)
                name = "X-Auth-Token";
            else if (WebAuthType.XApiKey == authType)
                name = "X-API-Key";
            else if (WebAuthType.ApiKey == authType)
                name = "api-key";
 
            return name;
        }
        static void CredentialsInUrl(Browser browser, GPALUrl url, Credentials credential)
        {
            UriBuilder builder = new UriBuilder(url?.Url)
            {
                UserName = Uri.EscapeDataString(credential.Username ?? string.Empty),
                Password = Uri.EscapeDataString(credential.Password ?? string.Empty)
            };
            GPALUrl credentialed = new GPALUrl(builder.Uri.ToString());
 
            // the engines are called directly rather than through .GoTo, because .GoTo announces where it is
            // going and this url is a password
            if (true == browser.UseOttoMagic)
                browser.MagicHelper.GoTo(credentialed);
            else if (true == browser.UsePuppeteer)
                browser.PuppeteerClient.GoTo(credentialed).Execute();
            else
                SeleniumGoToUrl(credentialed, browser.BrowserSettings);
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Credentials presented to [{builder.Host}], answering its challenge", browser, GPALObjectType.Browser);
        }

        private static Process SeleniumBrowserProcess(IWebDriver driver, string capability)
        {
            Process retVal = null;
 
            try
            {
                object pid = (driver as IHasCapabilities)?.Capabilities?.GetCapability(capability);
 
                if (null != pid)
                    retVal = Process.GetProcessById(Convert.ToInt32(pid));
            }
            catch (Exception ex)
            {
                // a pid that names nothing is a browser that is already gone, which is not worth a warning here.
                // the teardown treats a null the same as it always did
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"No browser process behind the driver for [{capability}]", null, GPALObjectType.Browser, ex);
            }
 
            return retVal;
        }
        internal static void WaitForBrowserToExit(BrowserSettings browserSettings, int timeoutMs)
        {
            try
            {
                browserSettings.Process?.WaitForExit(timeoutMs);
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Could not wait on the browser process before restoring [{browserSettings.ProfileDataDirectory}]", browserSettings.Browser, GPALObjectType.Browser, ex);
            }
        }

        public static void SetHeadlessDownload(BrowserSettings browserSettings, string destination)
        {
            string directory = Path.GetDirectoryName(destination);
            string filename = Path.GetFileName(destination);
 
            var enableDownloadCommandParameters = new Dictionary<string, object>
            {
                { "behavior", "allow" },
                { "downloadPath", directory },
                { "filename", filename }
            };
 
            if (BrowserType.Chrome == browserSettings.BrowserType || BrowserType.Edge == browserSettings.BrowserType)
                ((OpenQA.Selenium.Chromium.ChromiumDriver)browserSettings.BrowserDriver).ExecuteCdpCommand("Page.setDownloadBehavior", enableDownloadCommandParameters);
        }

        internal const string CallRecorderScript = @"
            (function () {
                if (window.__gpalCalls) return;
 
                window.__gpalCalls = [];
 
                var record = function (url, method, headers, body, type) {
                    var call = { url: String(url), method: method || 'GET', type: type, headers: headers || {}, postData: body || null, status: 0 };
                    window.__gpalCalls.push(call);
                    return call;
                };
 
                var nativeFetch = window.fetch;
 
                if (nativeFetch)
                    window.fetch = function (input, init) {
                        var url = input && input.url ? input.url : input;
                        var headers = {};
                        var options = init || {};
 
                        // a Headers object, a plain object or an array of pairs, all of which fetch accepts
                        if (options.headers && options.headers.forEach)
                            options.headers.forEach(function (v, k) { headers[k] = v; });
                        else if (options.headers)
                            Object.keys(options.headers).forEach(function (k) { headers[k] = options.headers[k]; });
 
                        var call = record(url, options.method, headers, options.body, 'Fetch');
 
                        return nativeFetch.apply(this, arguments).then(function (response) {
                            call.status = response.status;
                            return response;
                        }, function (error) {
                            call.status = 0;
                            throw error;
                        });
                    };
 
                var nativeOpen = XMLHttpRequest.prototype.open;
                var nativeSend = XMLHttpRequest.prototype.send;
                var nativeSetHeader = XMLHttpRequest.prototype.setRequestHeader;
 
                XMLHttpRequest.prototype.open = function (method, url) {
                    this.__gpalCall = { method: method, url: url, headers: {} };
                    return nativeOpen.apply(this, arguments);
                };
 
                XMLHttpRequest.prototype.setRequestHeader = function (name, value) {
                    if (this.__gpalCall) this.__gpalCall.headers[name] = value;
                    return nativeSetHeader.apply(this, arguments);
                };
 
                XMLHttpRequest.prototype.send = function (body) {
                    var pending = this.__gpalCall;
 
                    if (pending) {
                        var call = record(pending.url, pending.method, pending.headers, body, 'XHR');
                        this.addEventListener('loadend', function () { call.status = this.status; });
                    }
 
                    return nativeSend.apply(this, arguments);
                };
            })();
        ";
 
        public static string SeleniumAddScriptToEvaluateOnNewDocument(BrowserSettings browserSettings, string script)
        {
            if (BrowserType.Chrome != browserSettings.BrowserType && BrowserType.Edge != browserSettings.BrowserType)
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{browserSettings.BrowserType}] does not support Page.addScriptToEvaluateOnNewDocument", browserSettings.Browser, GPALObjectType.Browser);
                return null;
            }
 
            var result = ((OpenQA.Selenium.Chromium.ChromiumDriver)browserSettings.BrowserDriver).ExecuteCdpCommand("Page.addScriptToEvaluateOnNewDocument", new Dictionary<string, object>
            {
                { "source", script }
            }) as Dictionary<string, object>;
 
            return result?["identifier"]?.ToString();
        }

        public static void SeleniumRemoveScriptToEvaluateOnNewDocument(BrowserSettings browserSettings, string identifier)
        {
            if (BrowserType.Chrome != browserSettings.BrowserType && BrowserType.Edge != browserSettings.BrowserType)
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{browserSettings.BrowserType}] does not support Page.removeScriptToEvaluateOnNewDocument", browserSettings.Browser, GPALObjectType.Browser);
                return;
            }
 
            ((OpenQA.Selenium.Chromium.ChromiumDriver)browserSettings.BrowserDriver).ExecuteCdpCommand("Page.removeScriptToEvaluateOnNewDocument", new Dictionary<string, object>
            {
                { "identifier", identifier }
            });
        }

        public static string GetBrowserProfileDirectory(BrowserSettings browserSettings, bool getDefaultDirectory = false)
        {
            // Chromium profiles live under LocalApplicationData, Firefox profiles under ApplicationData
            // (Roaming). There is no single base for all three
            string localAppData = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
            string roamingAppData = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
            string profilePath = string.Empty;
 
            switch (browserSettings.BrowserType)
            {
                case BrowserType.Chrome:
                    profilePath = Path.Combine(localAppData, "Google", "Chrome", "User Data");
                    break;
                case BrowserType.FireFox:
                    profilePath = Path.Combine(roamingAppData, "Mozilla", "Firefox", "Profiles");
                    break;
                case BrowserType.Edge:
                    profilePath = Path.Combine(localAppData, "Microsoft", "Edge", "User Data");
                    break;
                default:
                    Exception ex = new Exception("Unsupported Browser Type");   // let's force this as an exception to post to the exception channel
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unsupported browser type for loading user profile [{browserSettings.ProfileUserName}]: [{browserSettings.BrowserType.ToString()}]", ex);
                    break;
            }
 
            string newProfilePath = profilePath;
            // chrome 136 will not let the default directory be specified as the user-data-dir if running via selenium (no access to user cookies)
            // - copy to a new folder is a workaround, we will guard against defining them the same
            if (false == string.IsNullOrEmpty(browserSettings.ProfileUserName) && false == getDefaultDirectory)
            {
                newProfilePath = Path.Combine(profilePath, browserSettings.ProfileUserName);
                // Check if the profile directory exists
                // Throws rather than falling back to the user data root, which would silently run the default
                // profile instead of the one named
                if (!Directory.Exists(newProfilePath))
                    RefuseProfile($"[{browserSettings.BrowserType}] has no profile directory [{newProfilePath}] for user name [{browserSettings.ProfileUserName}]");
            }
            return newProfilePath;
        }

        public static string FindProfileDirectory(string userDataDirectory, string profileName)
        {
            // Checked before GetDirectories, which throws DirectoryNotFoundException without naming the
            // profile that was asked for
            if (false == Directory.Exists(userDataDirectory))
                RefuseProfile($"Looked for profile [{profileName}] in [{userDataDirectory}], which is not there");
 
            string[] profileDirectories = Directory.GetDirectories(userDataDirectory);
            List<string> matches = new List<string>();
 
            foreach (string profileDirectory in profileDirectories)
            {
                string preferencesFile = Path.Combine(profileDirectory, "Preferences");
 
                if (File.Exists(preferencesFile))
                {
                    string profileNameFromFile = GetProfileNameFromPreferencesFile(preferencesFile);
 
                    if (null != profileNameFromFile && profileNameFromFile.Equals(profileName, StringComparison.OrdinalIgnoreCase))
                        matches.Add(new DirectoryInfo(profileDirectory).Name);
                }
            }
 
            if (0 < matches.Count)
            {
                // Chrome allows duplicate display names. The first match is used; the warning names the
                // alternatives and their sign-in state, which is usually what separates them
                if (1 < matches.Count)
                {
                    string choices = string.Join(", ", matches.Select(directory =>
                        $"[{directory}] ({SignedInSaid(Path.Combine(userDataDirectory, directory))})"));
 
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING,
                        $"[{matches.Count}] profiles are named [{profileName}]: {choices}. Using [{matches[0]}]. Name the directory instead of the display name to say which one you meant",
                        null, GPALObjectType.Browser);
                }
 
                return matches[0];
            }
            // Accepts a directory name as well as a display name. Chrome puts the directory name on
            // --profile-directory, so both forms are in circulation
            if (true == Directory.Exists(Path.Combine(userDataDirectory, profileName)))
                return profileName;
 
            // Chrome creates an empty profile rather than refusing an unknown --profile-directory, so an
            // unmatched name is thrown rather than passed through
            RefuseProfile($"No profile named [{profileName}] in [{userDataDirectory}], by display name or directory name");
 
            return null;
        }
 
        // Sign-in state from account_info in the profile Preferences. Reported in the duplicate-name warning
        private static string SignedInSaid(string profileDirectory)
        {
            string retVal = "cannot tell";
 
            try
            {
                string preferencesFile = Path.Combine(profileDirectory, "Preferences");
 
                if (true == File.Exists(preferencesFile))
                {
                    JObject preferences = JObject.Parse(File.ReadAllText(preferencesFile));
                    JArray accounts = preferences.SelectToken("account_info") as JArray;
 
                    retVal = null != accounts && 0 < accounts.Count ? "signed in" : "not signed in";
                }
            }
            catch (Exception)
            {
                // An unparseable Preferences leaves the state unknown; the directory name is still reported
            }
 
            return retVal;
        }
 
        // Publishes the ERROR and throws, matching RefuseIfProfileIsOpen
        private static void RefuseProfile(string message)
        {
            GPAL.PublishSimpleEvent(GPALEventType.ERROR, message, null, GPALObjectType.Browser);
 
            throw new GPALException($"{GPAL.MyMethodName()}: {message}");
        }

        static string GetProfileNameFromPreferencesFile(string preferencesFilePath)
        {
            try
            {
                string preferencesContent = File.ReadAllText(preferencesFilePath);
                JObject preferencesJson = JObject.Parse(preferencesContent);
 
                // Extract the given_name from the JSON structure
                JToken givenNameToken = preferencesJson.SelectToken("profile.name");
                if (givenNameToken != null)
                {
                    return givenNameToken.ToString();
                }
 
                return null; // given_name not found
            }
            catch
            {
                return null; // ERROR occurred while parsing JSON or accessing values
            }
        }

        public static string FindFirefoxBinaryLocation()
        {
            // Open the registry key corresponding to Firefox installation
            using (RegistryKey key = Registry.LocalMachine.OpenSubKey(GPAL.GPALSettings.FirefoxBinaryPathRegistry))
            {
                if (key != null)
                {
                    // Retrieve the version of Firefox installed
                    string firefoxPath = key.GetValue(null)?.ToString();    // get (Default) value
                    if (!string.IsNullOrEmpty(firefoxPath))
                        return firefoxPath;
                }
            }
 
            return null; // Firefox binary location not found
        }

        public static string FindTab(BrowserSettings browserSettings, string URL)
        {
            var handles = browserSettings.BrowserDriver.WindowHandles;
            foreach (var handle in handles)
            {
                browserSettings.BrowserDriver.SwitchTo().Window(handle);
                if (URL.Contains(browserSettings.BrowserDriver.Url) || browserSettings.BrowserDriver.Url.Contains(URL))
                {
                    return handle;
                }
            }
            return null;
        }

        public static bool GotoNextPage(Browser browser)
        {
 
            List<GPALElement> matchedElems = null;
            ReadOnlyCollection<GPALElement> elems = null;
            bool clicked = false;
            byte VK_END = 0x23;
            int withAllThatMatchSave = browser.CurrentUOW.WithAllThatMatch;
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, "Getting next page.", browser, GPALObjectType.Browser);
            browser.WithAllThatMatch(1);
 
 
            // CAVEAT: TODO: rework, scroll and scrape, scrolling might scroll elements off page or unrender them
            if (false == browser.CurrentUOW.InfiniteScroll)
            {
                // we cannot used cached results knowing we are going to a new page
                browser.CurrentUOW.NextPageButton.WebSelectorFoundResults = null;
                browser.CurrentUOW.NextPageButton.WebSelectorMatchedResults = null;
                foreach (var sel in browser.CurrentUOW.WithSelectorList)
                {
                    sel.WebSelectorFoundResults = null;
                    sel.WebSelectorMatchedResults = null;
                }
 
                elems = ElementHelper.FindWebElements(browser, browser.CurrentUOW, browser.CurrentUOW.NextPageButton, out bool matchedAll, out matchedElems, null, false);
 
                // can only be one match
                if (null != elems && 0 < elems.Count && true == matchedAll)
                {
                    try
                    {
 
                        if (false == string.IsNullOrEmpty(elems[0].Href))
                        {
                            string href = elems[0].Href;
                            UnitOfWork saveUOW = browser.CurrentUOW;    // keep this in
                            bool obeySave = browser.BrowserSettings.ObeyRobotsTxt;
                            browser.BrowserSettings.ObeyRobotsTxt = false;
                            browser.GoTo(href);    // checkdocumentready is in goto - do not perform robots check on navigation
 
                            // the site can put an interstitial in the way, a device check or similar. It is a page
                            // of its own at a url of its own, and it is ready, so document.ready says nothing. The
                            // page asked for is the one at the url asked for, so wait until that is where we are
                            Stopwatch arriving = Stopwatch.StartNew();
                            string here = GetCurrentUrl(browser.BrowserSettings);
 
                            if (false == href.Equals(here, StringComparison.OrdinalIgnoreCase))
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Waiting for [{href}], the page showing is [{here}]", browser, GPALObjectType.Browser);
 
                            while (false == href.Equals(here, StringComparison.OrdinalIgnoreCase)
                                && browser.BrowserSettings.WaitForDocumentReadyTimeoutMs > arriving.ElapsedMilliseconds)
                            {
                                System.Threading.Thread.Sleep(250);
 
                                here = GetCurrentUrl(browser.BrowserSettings);
                            }
 
                            if (false == href.Equals(here, StringComparison.OrdinalIgnoreCase))
                                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Gave up waiting for [{href}] after [{arriving.ElapsedMilliseconds}] ms, the page showing is [{here}]", browser, GPALObjectType.Browser);
 
                            browser.CurrentUOW = saveUOW;
                            browser.BrowserSettings.ObeyRobotsTxt = obeySave;
                            browser.BrowserSettings.CurrentURL = href;
                        }
                        else
                        {
                            // can be only one next page button elems[0] - on ebay, the dimensions are weird > 10k
                            elems[0].Click();
                            //CheckDocumentReady(browser);
                        }
 
                        clicked = true;
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Exception thrown", browser, GPALObjectType.Browser, ex);
                    }
                }
                else
                {
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, "Unable to find Next Page Button", browser, GPALObjectType.Browser);
                }
            }
            else // infinite scroll, go to page end and wait for load
            {
                if (false == browser.BrowserSettings.UseHeadless && (true == browser.BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware))
                    HardwareHelper.SendKey(VK_END);
                else
                    _ = browser.PageEnd;
 
                CheckDocumentReady(browser);
 
                clicked = true;
            }
 
            browser.WithAllThatMatch(withAllThatMatchSave);
 
            return clicked;
        }

        public static void FillInWithTokens(Browser browser, IGPALGrid<string> tokens, WriteMode writeMode)
        {
            int tokenIdx = 0;
            List<ReadOnlyCollection<GPALElement>> rowsOfColumns = new List<ReadOnlyCollection<GPALElement>>();
            List<ReadOnlyCollection<GPALElement>> rowsOfElements = new List<ReadOnlyCollection<GPALElement>>();
            // a column-wise list of the selector info to match with the column of matched/returned elements to know how to interact with that column of elemnts
            // of course selectors are always a 'column'
            List<SmallSelectorNode> interactionInfo = new List<SmallSelectorNode>();
 
            while (true)
            {
 
                // waitfor did not match if returns false
                int rowCnt = 0;
                int lastRowCnt = 0;
                int elementIdx = 0;
 
                interactionInfo.Clear();    // are we looping? of course, the CallAfterFillIn method workflow has returned for the next iteration.
                rowsOfElements.Clear();
                rowsOfColumns.Clear();
                browser.CurrentUOW.MatchedRowIndexes.Clear();
                browser.CurrentUOW.ColCount = 0;
                browser.CurrentUOW.RowCount = 0;    // Math.Max below only ever raises it, so a pass that finds fewer elements than the last would size the grid for rows it has no elements to fill
                browser.CurrentUOW.PartialMatch = false;    // only ever |= true below, so one pass that did not match everything would put every later pass down the partial-match path
 
                foreach (Selector sel in browser.CurrentUOW.WithSelectorList)
                {
                    if (SelectorType.Selector != sel.SelectorType)
                        continue;   // we don't look up data literals
 
                    // TODO: what to do on partial match?
                    List<GPALElement> matchedElems = null;
                    ReadOnlyCollection<GPALElement> tmpElems = ElementHelper.FindWebElements(browser, browser.CurrentUOW, sel, out bool matchedAll, out matchedElems);
                    rowsOfElements.Add(tmpElems);
                    browser.CurrentUOW.ColCount++;
 
                    if (false == matchedAll)
                        browser.CurrentUOW.PartialMatch |= true;
 
                    if (0 < tmpElems?.Count && true == matchedAll)
                    {
                        // RowCount is the total count of elements found or just the WithAllThatMatch row count
                        browser.CurrentUOW.RowCount = Math.Max(browser.CurrentUOW.RowCount, (int.MaxValue == browser.CurrentUOW.WithAllThatMatch ? tmpElems.Count : browser.CurrentUOW.WithAllThatMatch));
                    }
                    else if (0 < matchedElems?.Count && false == matchedAll)
                    {
                        // partial match
                        int tmpIdx = 0;
                        // add the indices of the matched items
                        // when we build the grid, we will only get rows that matched
                        // NO 'AND' on selector MATCH
                        // only 'OR' operator, all those that match on different selectors
                        foreach (GPALElement webElement in matchedElems)
                            if (-1 != (tmpIdx = tmpElems.IndexOf(webElement)))
                                if (false == browser.CurrentUOW.MatchedRowIndexes.Contains(new KeyValuePair<int, string>(tmpIdx, sel.AttributeName)))
                                    browser.CurrentUOW.MatchedRowIndexes.Add(tmpIdx, sel.AttributeName);
 
                        // matched row indexes is the row count we will extract data from on partial matches, so however many indices we have, that is our MAX row count
                        // up to WithAllThatMatch row count if not int.MaxValue
                        browser.CurrentUOW.RowCount = (int.MaxValue == browser.CurrentUOW.WithAllThatMatch ? browser.CurrentUOW.MatchedRowIndexes.Count : browser.CurrentUOW.WithAllThatMatch);
                    }
 
                    // if we returned elements, add an entry for this selector in interactionInfo, we consume it below
                    if (0 < matchedElems?.Count || 0 < tmpElems?.Count)
                        interactionInfo.Add(new SmallSelectorNode() { InteractionType = sel.InteractionType, OffsetX = sel.OffsetX, OffsetY = sel.OffsetY, DeltaX = sel.DeltaX, DeltaY = sel.DeltaY });
                }
 
                // we have partial matches, so construct the grid only from matched rows
                if (true == browser.CurrentUOW.PartialMatch)
                {
                    List<GPALElement> newColumn = new List<GPALElement>();
 
                    browser.CurrentUOW.RowCount = browser.CurrentUOW.MatchedRowIndexes.Count;
                    // get a list of all the elements in one column
                    // iterate thru each column pulling out matched rows
                    // create the return grid as only matched elements
                    foreach (ReadOnlyCollection<GPALElement> webElements in rowsOfElements)
                    {
                        // create a newCOlumn from all elements via indexes where rows 'matched' on some selector value
                        // we have multiple selectors contributing to matched rows based upon that selectors 'match' criteria
                        foreach (KeyValuePair<int, string> idx in browser.CurrentUOW.MatchedRowIndexes)
                            newColumn.Add(webElements[idx.Key]);
                        rowsOfColumns.Add(new ReadOnlyCollection<GPALElement>(newColumn));
                        newColumn = new List<GPALElement>();
                    }
                }
                else
                {
                    foreach (ReadOnlyCollection<GPALElement> webElements in rowsOfElements)
                        rowsOfColumns.Add(webElements);
                }
 
                // if we are looping, we have a new 'page' and new elements to interact with, our old list is stale
                browser.CurrentUOW.ElementGrid.Clear();
 
                // we should have one column for each selector (that found elements)
                if (0 != browser.CurrentUOW.ColCount)
                {
                    for (int cnt = 0; cnt < browser.CurrentUOW.RowCount; cnt++)
                        browser.CurrentUOW.ElementGrid.AddRow(new List<UnitOfWork.ElementNode>(browser.CurrentUOW.ColCount));    // create a row of columns for the number of elements found
 
                    int columnIdx = 0; ;  // start with the first column in the row
 
                    // grab a row of columns 
                    // construct our grid by pivoting the data from columns, to rows
                    // iterate each column elements (rows) add an element to each row of the element grid
                    foreach (ReadOnlyCollection<GPALElement> column in rowsOfColumns)
                    {
                        rowCnt = lastRowCnt;    // if we are on subsequent pages, we have to add at the end of the list, up to WithAllThatMatch row count (or all rows)
                        foreach (GPALElement elem in column)
                        {
                            if (true == ElementHelper.IsElementEditable(elem))
                                browser.CurrentUOW.ElementGrid[rowCnt].Add(
                                    new UnitOfWork.ElementNode()
                                    {
                                        GPALElement = elem,
                                        InteractionType = interactionInfo[columnIdx].InteractionType,
                                        OffsetX = interactionInfo[columnIdx].OffsetX,
                                        OffsetY = interactionInfo[columnIdx].OffsetY
                                    });
 
                            // only add as many rows as was asked for
                            // all or only save up to withallthatmatch row count
                            if (int.MaxValue != browser.CurrentUOW.WithAllThatMatch && rowCnt == browser.CurrentUOW.WithAllThatMatch)   // only save the first one, we do not have the WithAll directive
                                break;
 
                            rowCnt++;
                        }
                    }
                }
 
                // TODO: CAVEAT: do we have concept of next page in apps? prolly? if we still have tokens and are done, shouldn't we get a next page or bail
                // BUG: IF WE consume all our tokens, why would we loop again (while true)
 
                // keep looping while we have tokens
                // but after dealing with the first round of tokens, with more to go
                if (tokenIdx < tokens.Count())
                {
                    // we now have our elements in rows and columns
                    // iterate over the tokens, in rows and columns to fill in the text
                    foreach (List<UnitOfWork.ElementNode> elementNode in browser.CurrentUOW.ElementGrid)    // get one row of elements
                    {
                        elementIdx = 0;
                        List<string> currentRow = tokens[tokenIdx];
 
                        foreach (string token in currentRow)                                            // iterate over tokens, filling in the corresponding input/textarea webelement
                        {
                            if (elementIdx < elementNode.Count && null != elementNode[elementIdx])
                            {
                                ElementHelper.ScrollIntoView(browser, elementNode[elementIdx].GPALElement, elementNode[elementIdx].InteractionType);
 
                                if (true == browser.UseOttoMagic)
                                {
                                    if (InteractionType.Hardware == elementNode[elementIdx].InteractionType)
                                        ElementHelper.HardwareFillInFrom(browser, elementNode[elementIdx].GPALElement, elementNode[elementIdx].OffsetX, elementNode[elementIdx].OffsetY, token.ToString(), writeMode);
                                    else
                                        switch (writeMode)
                                        {
                                            case WriteMode.Append:
                                                browser.BrowserSettings.MagicHelper.FillInAppend(elementNode[elementIdx].GPALElement.Css, token.ToString());
                                                break;
 
                                            case WriteMode.Insert:
                                                browser.BrowserSettings.MagicHelper.FillInInsert(elementNode[elementIdx].GPALElement.Css, token.ToString());
                                                break;
 
                                            case WriteMode.Overwrite:
                                                browser.BrowserSettings.MagicHelper.FillInOverwrite(elementNode[elementIdx].GPALElement.Css, token.ToString());
                                                break;
                                        }
                                }
                                else if (true == browser.UsePuppeteer)
                                {
                                    if (InteractionType.Hardware == elementNode[elementIdx].InteractionType)
                                        ElementHelper.HardwareFillInFrom(browser, elementNode[elementIdx].GPALElement, elementNode[elementIdx].OffsetX, elementNode[elementIdx].OffsetY, token.ToString(), writeMode);
                                    else
                                        switch (writeMode)
                                        {
                                            case WriteMode.Append:
                                                browser.PuppeteerClient.FillInAppend(elementNode[elementIdx].GPALElement.Css).WithText(token.ToString()).Execute();
                                                break;
 
                                            case WriteMode.Insert:
                                                browser.PuppeteerClient.FillInInsert(elementNode[elementIdx].GPALElement.Css).WithText(token.ToString()).Execute();
                                                break;
 
                                            case WriteMode.Overwrite:
                                                browser.PuppeteerClient.FillInOverwrite(elementNode[elementIdx].GPALElement.Css).WithText(token.ToString()).Execute();
                                                break;
                                        }
                                }
                                else    // selenium
                                {
                                    if (InteractionType.Hardware == elementNode[elementIdx].InteractionType)
                                        ElementHelper.HardwareFillInFrom(browser, elementNode[elementIdx].GPALElement, elementNode[elementIdx].OffsetX, elementNode[elementIdx].OffsetY, token.ToString(), writeMode);
                                    else if (InteractionType.JavaScript == elementNode[elementIdx].InteractionType)
                                        ElementHelper.JavaScriptFillInFrom(browser, elementNode[elementIdx].GPALElement, token.ToString(), writeMode);
                                    else
                                        ElementHelper.SeleniumFillInFrom(browser, elementNode[elementIdx].GPALElement, token.ToString(), writeMode);
                                }
 
                                elementIdx++;   // increment the column [element] count for each token. if we have a token, we are expecting elements for it.
                            }
                            else
                                // detect if we have too many tokens to consume and publish an information event [entirely possible this scenario is fine]
                                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Unused token [{token}] from tokens [{string.Join(",", currentRow)}] in row [{tokenIdx + 1}]", browser, GPALObjectType.Browser);
 
                            try
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{writeMode}] text [{token.ToString()}] in element [{elementNode[elementIdx - 1].GPALElement.TagName}]", browser, GPALObjectType.Browser);
                            }
                            catch
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"NOT WRITTEN: [{writeMode}] text [{token.ToString()}] in element [NOT FOUND].", browser, GPALObjectType.Browser);
                            }
                        }
 
                        // detect if we do not have enough tokens for all the form fields [entirely possible this scenario is fine]
                        // we got here and finished our tokens but have not exhausted our columns [elements]
                        while (elementIdx < elementNode.Count && null != elementNode[elementIdx++])
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"No token for control: <[{elementNode[elementIdx - 1].GPALElement.GetType()}]> : [{elementNode[elementIdx - 1].GPALElement.Text}]", browser, GPALObjectType.Browser);
                    }
 
                    UnitOfWork safeUOW = browser.CurrentUOW;
                    CallIfStatus handled = 0;
 
                    // CAVEAT: there is no concept of 'handled (1)' vs 'not handled (0)' but definitely can request to exit
                    // TODO: CAVEAT: is cast to dynamic best?
                    if (null != browser.CurrentUOW.CallAfterFillIn && true == rowsOfColumns.Any())
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking CallAfterFillIn [{browser.CurrentUOW.CallAfterFillIn.Method.Name}]", browser, GPALObjectType.Browser);
                        handled = browser.CurrentUOW.CallAfterFillIn(browser, tokens, tokenIdx);
                    }
 
                    // CAVEAT: kludge - the call after handler can and probably will set a new current unit of work, but we need our old current unit
                    // the call after handler UOW is no longer in scope, so restore our UOW
                    browser.CurrentUOW = safeUOW;
 
                    // CAVEAT: elements may go stale because of callafterfillin handler actions
                    // test if elements still valid by testing the first one
                    //try
                    //{
                    //    ((UnitOfWork.ElementNode)(browser.CurrentUOW.ElementGrid[0])[0]).GPALElement.GetAttribute("tag");
                    //} 
                    //catch (Exception ex)   // failed, invalidate elementgrid
                    //{
                    //    browser.CurrentUOW.ElementGrid.Clear();
                    //    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"CallAfterFillIn handler [{browser.CurrentUOW.CallAfterFillIn.GetInvocationList()[0].Method.Name}] invalidated browser elements, clearing cache. Exception follows: [{ex.Message}]", browser, GPALObjectType.Browser, ex);
                    //    break;
                    //}
 
                    if (CallIfStatus.Terminate == handled)
                    {
                        string str = $"CallAfterFillIn handler [{browser.CurrentUOW.CallAfterFillIn.GetInvocationList()[0].Method.Name}] requested program termination.";
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
                        throw new GPALException($"{GPAL.MyMethodName()}: " + str);
                    }
 
                    tokenIdx++;
                }
                else
                    break;
 
                lastRowCnt += rowCnt;
                // TODO: are we done or do we have to go to the next page?
 
                // we are out of tokens, break out
                // if we have more tokens, shouldn't we go to another page?
                // CAVEAT: BUG: we are just looping back thru the same selectors and consuming more tokens, so does that mean CallAfterFillIn will do something and this is the behavior we want?
                // or do we break because we have more tokens then elements? or should we loop?
                // why did i put this in a while true loop?
                if (tokenIdx == tokens.Count())
                    break;
            }
        }
 
        // this is failing after clicks on wells fargo because it's checking faster than the page can load, but i'm reticent to add too much of a delay in jswait
        // since this is not entirely critical, we are just going to not print out any messages about this failing, but we also updated execute javascript to return 'fail' instead of 'complete' on catching an exception
        // that will result in this waiting the full 30 seconds timeout. not sure if that is good or bad, we will see.
 
        // gap between url reads while waiting to see if a click navigated. Each read is a driver round trip, so this
        // is not a hot loop the way WaitForValueOrTime is
        private const int NavigationPollMs = 50;
 
        // chromium window handles are the CDP target id behind this prefix; firefox reports its pid in capabilities
        private const string CdpWindowPrefix = "CDwindow-";
        private const string MozProcessIdCapability = "moz:processID";
        static string ReadyStatusOrNavError(Browser browser, string sessionToken)
        {
            string status = browser.MagicHelper.GetReadyStatus(sessionToken)?.Trim('"');
 
            if (true == status?.ToLower().Contains("nav-error"))
            {
                // GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"The page did not load, so there is nothing to work with. [{status}]", browser, GPALObjectType.Browser);
                browser.RaiseOnFail(GPALFailure.Navigation, status);
            }
 
            return status;
        }
        public static void CheckDocumentReady(Browser browser, bool overrideSetting = false)
        {
            bool hasURL = null != browser?.BrowserSettings.CurrentURL && false == "about:blank".Equals(browser.BrowserSettings.CurrentURL);
 
            // A loading page rejects script, which is the condition this method waits out. Restored rather
            // than cleared so a nested call does not re-enable it, and held in a finally so a throw does not
            // leave the browser suppressed
            bool wasSuppressed = true == browser?.BrowserSettings.SuppressJavaScriptFailures;
            string sessionToken = Guid.NewGuid().ToString();
 
            if (null != browser)
                browser.BrowserSettings.SuppressJavaScriptFailures = true;
 
            try
            {
                if ((true == browser?.BrowserSettings.WaitOnDocumentReady && true == hasURL) || true == overrideSetting)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Waiting up to [{browser.BrowserSettings.WaitForDocumentReadyTimeoutMs}] ms on document.ready");
 
                    // nav-error is reported instead of a ready state when the navigation failed outright, so
                    // the caller stops rather than polling to the timeout
                    if (true == browser?.UseOttoMagic)
                        WaitForValueOrTime(browser, "complete, interactive, nav-error", () => ReadyStatusOrNavError(browser, sessionToken), browser.BrowserSettings.WaitForDocumentReadyTimeoutMs);
                    else if (true == browser?.UsePuppeteer)
                        WaitForValueOrTime(browser, "complete, interactive", () => ((PuppeteerClient)browser.PuppeteerClient).GetReadyStatus(sessionToken).Execute<object>(), browser.BrowserSettings.WaitForDocumentReadyTimeoutMs);
                    else    // handles selenium
                        WaitForValueOrTime(browser, "complete, interactive", () => JsCheckReadyState(browser, sessionToken), browser.BrowserSettings.WaitForDocumentReadyTimeoutMs);
                }
                else if (true == browser.BrowserSettings.WaitOnNetworkIdle && true == hasURL)
                {
                    if (true == browser.UsePuppeteer)
                        WaitForValueOrTime(browser, "true", () => browser.PuppeteerClient.CheckNetworkIdle().WithSessionToken(sessionToken).Execute<bool>().ToString(), browser.BrowserSettings.WaitForDocumentReadyTimeoutMs);
                    else if (true == browser.BrowserSettings.UseOttoMagic)
                        WaitForValueOrTime(browser, "true", () => WaitForNetworkIdle(browser, browser.BrowserSettings.NetworkIdleTimeoutMs, browser.BrowserSettings.NetworkIdleMaxConnections, browser.BrowserSettings.NetworkIdlePruneMs, sessionToken).ToString(), browser.BrowserSettings.WaitForDocumentReadyTimeoutMs);
                    else
                        WaitForValueOrTime(browser, "true", () => WaitForNetworkIdle(browser, sessionToken).ToString(), browser.BrowserSettings.WaitForDocumentReadyTimeoutMs);
                    }
                }
            finally
            {
                if (null != browser)
                    browser.BrowserSettings.SuppressJavaScriptFailures = wasSuppressed;
            }
        }

        internal static void FetchWithTokens(Browser browser, GPALRequest request)
        {
            IGPALGrid<string> tokens = browser.CurrentUOW.FetchTokens;
            int required = request.TokenCount();
            string name = request.Name ?? request.Path;
 
            if (null == tokens && 0 < required)
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Fetch of [{name}] uses [{required}] tokens but no token source was given. Use .WithTokensFrom before .Fetch.", browser, GPALObjectType.Browser);
            else
            {
                // no token source is still one pass, the request exactly as it was written
                int rowCount = tokens?.Rows ?? 1;
 
                for (int tokenIdx = 0; tokenIdx < rowCount; tokenIdx++)
                {
                    List<string> row = null == tokens ? null : tokens[tokenIdx];
 
                    // a row with a hole in it would send a body with a token still in it, which is worse than not
                    // sending it at all
                    if (null != row && row.Count < required)
                        GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Fetch of [{name}] needs [{required}] tokens but row [{tokenIdx + 1}] has [{row.Count}]. Skipping the row.", browser, GPALObjectType.Browser);
                    else
                        FetchRow(browser, request, row, tokenIdx);
                }
            }
        }
        static void FetchRow(Browser browser, GPALRequest request, List<string> row, int tokenIdx)
        {
            int mark = browser.CurrentUOW.FetchResults.Count;
            string name = request.Name ?? request.Path;
 
            for (int page = 0; page < browser.CurrentUOW.PageCount; page++)
            {
                string where = null == row ? $"Page [{page + 1}]" : $"Row [{tokenIdx + 1}] page [{page + 1}]";
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Fetching [{name}]. {where}.", browser, GPALObjectType.Browser);
 
                string body = FetchInPage(browser, request, page, row);
 
                // nothing here stops the run. one page failing says nothing about the next one, or about the rows
                // after it, and whatever went wrong is written into the results so it reaches .SaveTo and
                // CallAfterFetch rather than living only in the log. every page still produces an entry, so
                // position in the results always maps to the row and page that made it
                if (null == body)
                {
                    string why = $"Fetch of [{name}] returned nothing on {where.ToLower()}.";
 
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, why, browser, GPALObjectType.Browser);
                    body = why;
                }
                else if (200 > browser.ServerResponseCode || 300 <= browser.ServerResponseCode)
                {
                    // a server that answered is data whatever it answered, and the body is usually the only thing
                    // that says why, so it is kept with the reason in front of it
                    string why = $"Fetch of [{name}] returned HTTP [{browser.ServerResponseCode}] on {where.ToLower()}.";
 
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, why, browser, GPALObjectType.Browser);
                    body = why + Environment.NewLine + body;
                }
 
                browser.CurrentUOW.FetchResults.Add(body);
            }
 
            if (null != request.AfterFetch)
            {
                UnitOfWork safeUOW = browser.CurrentUOW;
                List<string> results = browser.CurrentUOW.FetchResults.GetRange(mark, browser.CurrentUOW.FetchResults.Count - mark);
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking CallAfterFetch [{request.AfterFetch.Method.Name}]", browser, GPALObjectType.Browser);
 
                CallIfStatus handled = request.AfterFetch(browser, results, browser.CurrentUOW.FetchTokens, tokenIdx);
 
                // the handler can and probably will start its own unit of work, so restore ours
                browser.CurrentUOW = safeUOW;
 
                if (CallIfStatus.Terminate == handled)
                {
                    string str = $"CallAfterFetch handler [{request.AfterFetch.Method.Name}] requested program termination.";
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
                    throw new GPALException($"{GPAL.MyMethodName()}: " + str);
                }
            }
        }
        internal static string FetchInPage(Browser browser, GPALRequest request, int page, List<string> row)
        {
            // GPALRequest, paging and tokens are GPAL ideas, so they are resolved here and the layers below only
            // ever see a finished url and plain values, the same as every other endpoint
            string envelope = FetchEnvelope(browser, request.ResolveUrl(page, row), request.Method, request.ResolveBody(page, row), request.ContentType, request.HeaderPairs(page, row), false);
 
            return UnpackFetchResponse(browser, envelope);
        }
        static string FetchEnvelope(Browser browser, string url, string method, string body, string contentType, string[] headers, bool asBytes)
        {
            string envelope = null;
 
            if (true == browser.UseSelenium)
                envelope = ((IJavaScriptExecutor)browser.BrowserDriver).ExecuteAsyncScript(
                    FetchSetupScript(url, method, body, contentType, headers, asBytes) + @"
                    var done = arguments[arguments.length - 1];
 
                    fetch(target.toString(), options)
                        .then(gpalEnvelope)
                        .then(done)
                        .catch(function (error) { done(gpalFailure(error)); });
                ") as string;
            else if (true == browser.UsePuppeteer)
                envelope = browser.PuppeteerClient
                    .Fetch(url)
                    .WithVerb(method)
                    .WithBody(body)
                    .WithContentType(contentType)
                    .WithHeaders(headers)
                    .WithBytes(asBytes)
                    .Execute();
            else if (true == browser.UseOttoMagic)
                envelope = browser.MagicHelper.Fetch(url, method, body, contentType, headers, asBytes);
            else
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Fetch is not implemented for [{browser.BrowserSettings.AutomationEngine}] yet.", browser, GPALObjectType.Browser);
 
            return envelope;
        }
        internal static string LiveUserAgent(Browser browser)
        {
            string retVal = null;
 
            if (true == browser.UseSelenium)
                retVal = ((IJavaScriptExecutor)browser.BrowserDriver).ExecuteScript("return navigator.userAgent") as string;
            else if (true == browser.UsePuppeteer)
                retVal = browser.PuppeteerClient.GetUserAgent().Execute();
            else if (true == browser.UseOttoMagic)
                retVal = Unquoted(browser.MagicHelper.GetUserAgent());
 
            return retVal;
        }

        static string Unquoted(string value)
        {
            string retVal = value;
 
            while (false == string.IsNullOrEmpty(retVal) && true == retVal.StartsWith("\"") && true == retVal.EndsWith("\"") && 1 < retVal.Length)
            {
                string unwrapped = Newtonsoft.Json.JsonConvert.DeserializeObject<string>(retVal);
 
                if (unwrapped == retVal)
                    break;
 
                retVal = unwrapped;
            }
 
            return retVal;
        }
        internal static string LiveAcceptLanguage(Browser browser)
        {
            string languages = null;
            string retVal = null;
 
            if (true == browser.UseSelenium)
                languages = ((IJavaScriptExecutor)browser.BrowserDriver).ExecuteScript("return navigator.languages.join(',')") as string;
            else if (true == browser.UsePuppeteer)
                languages = browser.PuppeteerClient.GetLanguages().Execute();
            else if (true == browser.UseOttoMagic)
                languages = Unquoted(browser.MagicHelper.GetLanguages());
 
            if (false == string.IsNullOrWhiteSpace(languages))
                retVal = AcceptLanguageFrom(languages.Split(','), browser.BrowserSettings.BrowserType);
 
            return retVal;
        }

        static string AcceptLanguageFrom(string[] languages, Enums.BrowserType browserType)
        {
            List<string> weighted = new List<string>();
 
            for (int index = 0; index < languages.Length; index++)
            {
                string language = languages[index].Trim();
 
                if (true == string.IsNullOrEmpty(language))
                    continue;
 
                if (Enums.BrowserType.FireFox == browserType)
                    language = language.ToLowerInvariant();
 
                if (0 == index)
                    weighted.Add(language);
                else
                {
                    // a tenth is as fine as the header goes, and a language weighted to nothing is left off
                    // rather than sent as q=0.0, which asks the server not to answer in it at all
                    double quality = Math.Round(1.0 - (0.1 * index), 1);
 
                    if (0 < quality)
                        weighted.Add($"{language};q={quality.ToString("0.#", System.Globalization.CultureInfo.InvariantCulture)}");
                }
            }
 
            return string.Join(",", weighted);
        }
        internal static byte[] FetchBytes(Browser browser, string url, string[] headers, out string lastModified)
        {
            byte[] bytes = null;
            JObject response = UnpackFetchEnvelope(browser, FetchEnvelope(browser, url, "GET", null, null, headers, true));
            string encoded = response?["body"]?.ToString();
 
            lastModified = response?["lastModified"]?.ToString();
 
            if (false == string.IsNullOrEmpty(encoded))
                bytes = Convert.FromBase64String(encoded);
 
            return bytes;
        }
        static string UnpackFetchResponse(Browser browser, string envelope)
        {
            return UnpackFetchEnvelope(browser, envelope)?["body"]?.ToString();
        }
        static JObject UnpackFetchEnvelope(Browser browser, string envelope)
        {
            JObject response = null;
 
            // a fetch that could not be made has no status of its own, and leaving the last one standing would
            // let a caller read a stale 200 or 304 as though it belonged to this request
            browser.ServerResponseCode = 0;
 
            if (true == string.IsNullOrEmpty(envelope))
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"[{browser.BrowserSettings.AutomationEngine}] returned nothing for the fetch, so the page never answered", browser, GPALObjectType.Browser);
            else
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEEPDEBUG, $"Fetch envelope [{envelope}]", browser, GPALObjectType.Browser);
 
                // never parse blind. a shape we do not recognize is reported with what actually arrived, because
                // a stack trace out of a json parser says nothing about what the page sent
                if (false == envelope.TrimStart().StartsWith("{"))
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Fetch response was not understood. Received [{envelope}]", browser, GPALObjectType.Browser);
                else
                {
                    response = JObject.Parse(envelope);
                    browser.ServerResponseCode = response["status"]?.Value<int>() ?? 0;
 
                    // the page could not make the request at all and said why. Without this the caller only knows
                    // that nothing came back, which is the same thing a refusal and a bug look like
                    if (false == string.IsNullOrEmpty(response["error"]?.ToString()))
                        GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"The page could not make the request [{response["error"]}]", browser, GPALObjectType.Browser);
                }
            }
 
            return response;
        }
        internal static string FetchSetupScript(string url, string method, string body, string contentType, string[] headers, bool asBytes)
        {
            return $@"
                var target = new URL({Newtonsoft.Json.JsonConvert.SerializeObject(url)}, location.href);
                var reqBody = {Newtonsoft.Json.JsonConvert.SerializeObject(body)};
                var contentType = {Newtonsoft.Json.JsonConvert.SerializeObject(contentType)};
                var headerPairs = {Newtonsoft.Json.JsonConvert.SerializeObject(headers ?? new string[0])};
                var asBytes = {(true == asBytes ? "true" : "false")};
 
                var options = {{ method: {Newtonsoft.Json.JsonConvert.SerializeObject(method ?? "GET")}, credentials: 'include', headers: {{}} }};
                for (var h = 0; h < headerPairs.length; h += 2)
                    options.headers[headerPairs[h]] = headerPairs[h + 1];
 
                if (reqBody) {{
                    options.body = reqBody;
                    if (!options.headers['Content-Type'])
                        options.headers['Content-Type'] = contentType;
                }}
 
                // one envelope for every engine, so the C# side has a single shape to unpack. text() would put a
                // replacement character wherever a byte is not valid utf-8, which is most of any real file, so a
                // file is read as an arraybuffer and base64 encoded instead. Written with promises rather than
                // await so the same function serves selenium's callback and puppeteer's awaited expression.
                // String.fromCharCode is applied over slices because one call with a few hundred thousand
                // arguments overflows the stack
                function gpalEnvelope(response) {{
                    return (asBytes
                        ? response.arrayBuffer().then(function (buffer) {{
                            var bytes = new Uint8Array(buffer), binary = '', slice = 0x8000;
                            for (var i = 0; i < bytes.length; i += slice)
                                binary += String.fromCharCode.apply(null, bytes.subarray(i, i + slice));
                            return btoa(binary);
                        }})
                        : response.text()
                    ).then(function (payload) {{
                        return JSON.stringify({{
                            status: response.status,
                            body: payload,
                            encoding: asBytes ? 'base64' : null,
                            lastModified: response.headers.get('Last-Modified')
                        }});
                    }});
                }}
 
                // a fetch the page could not make still answers in the envelope shape, carrying the reason.
                // returning nothing here is what made a refused request and a bug look identical from C#
                function gpalFailure(error) {{
                    return JSON.stringify({{
                        status: 0,
                        body: null,
                        encoding: null,
                        lastModified: null,
                        error: String((error && error.message) || error) + ' requesting ' + target.toString()
                    }});
                }}
            ";
        }
        public static object ExecuteJavaScriptObj(string executeMe, IBrowser browser, params object[] args)
        {
            dynamic retVal = null;
 
            try
            {
                if (browser.UseOttoMagic)
                {
                    retVal = browser.MagicHelper.ExecuteJavaScript(executeMe);
                }
                else if (true == browser.UsePuppeteer)
                {
                    retVal = browser.PuppeteerClient.ExecuteJavaScript(executeMe).WithParameters(args).Execute<object>();
                    retVal = retVal.result.value;
                }
                else // beginning chrome 136, dynamic javascript execution, even by a browser extension, is no longer possihle. has to be baked in code.
                {
                    IJavaScriptExecutor js = (IJavaScriptExecutor)browser.BrowserDriver;
                    retVal = js.ExecuteScript(executeMe, args);
                }
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                // Read from the browser that failed. A non-Browser argument is not suppressed
                bool suppressed = true == (browser as Browser)?.BrowserSettings.SuppressJavaScriptFailures;
 
                if (false == suppressed)
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $@"Failed to execute javascript. Continuing.", browser, GPALObjectType.Browser, ex);
            }
 
            return retVal;
        }
        // NOTE: if this fails we could fallback to puppeteer, but could we do anything with selenium if this doesn't work?
        private static string UnwrapScriptResult(string result)
        {
            string retVal = result;
 
 
            // the RemoteObject describes the result: a type, and the value it is describing. anything else is a
            // returned object that happens to be json, and belongs to the caller untouched
            if (false == string.IsNullOrWhiteSpace(retVal) && true == retVal.TrimStart().StartsWith("{"))
            {
                try
                {
                    Newtonsoft.Json.Linq.JObject remote = Newtonsoft.Json.Linq.JObject.Parse(retVal);
 
                    if (null != remote["type"] && null != remote["value"]
                        && remote.Properties().All(each => "type" == each.Name || "value" == each.Name || "description" == each.Name || "className" == each.Name || "subtype" == each.Name))
                    {
                        Newtonsoft.Json.Linq.JToken value = remote["value"];
 
                        retVal = Newtonsoft.Json.Linq.JTokenType.String == value.Type
                            ? value.ToString()
                            : Newtonsoft.Json.JsonConvert.SerializeObject(value);
                    }
                }
                catch (Exception)
                {
                    // shaping the answer is never a reason to fail the call
                }
            }
 
            // a json encoded string, one layer at a time, until it stops being one
            while (false == string.IsNullOrEmpty(retVal) && retVal.StartsWith("\"") && retVal.EndsWith("\""))
            {
                try
                {
                    string peeled = Newtonsoft.Json.JsonConvert.DeserializeObject<string>(retVal);
 
                    if (peeled == retVal)
                        break;
 
                    retVal = peeled;
                }
                catch (Exception)
                {
                    break;   // not an encoded string after all, so it is the value already
                }
            }
 
            return retVal;
        }

        public static string ExecuteJavaScript(IBrowser browser, string executeMe, params object[] args)
        {
            string retVal = RunScript(browser, executeMe, args);
 
            // V8 raises this message only for a return outside a function
            if (false == browser.UseSelenium && true == retVal?.Contains("Illegal return statement"))
                retVal = RunScript(browser, "(function(){" + Environment.NewLine + executeMe + Environment.NewLine + "})()", args);
 
            return retVal;
        }
 
        private static string RunScript(IBrowser browser, string executeMe, params object[] args)
        {
            dynamic retVal = null;
 
            try
            {
                if (browser.UseOttoMagic)
                {
                    retVal = browser.MagicHelper.ExecuteJavaScript(executeMe);
                }
                else if (true == browser.UsePuppeteer)
                {
                    if (true == args.Any())
                        retVal = browser.PuppeteerClient.ExecuteJavaScript(executeMe).WithParameters(args).Execute<object>();
                    else
                        retVal = browser.PuppeteerClient.ExecuteJavaScript(executeMe).Execute<object>();
 
                    if (Puppeteer.resultExtractors != null && Puppeteer.resultExtractors.TryGetValue(DevToolsMethods.RuntimeEvaluate, out var extractor))
                        retVal = extractor((Newtonsoft.Json.Linq.JObject)retVal.result);
                    else
                        retVal = retVal.result.ToObject();
                }
                else // beginning chrome 136, dynamic javascript execution, even by a browser extension, is no longer possihle. has to be baked in code.
                {
                    IJavaScriptExecutor js = (IJavaScriptExecutor)browser.BrowserDriver;
                    if (true == args?.Any())
                        retVal = js.ExecuteScript(executeMe, args);
                    else
                        retVal = js.ExecuteScript(executeMe);
 
 
                }
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                // Read from the browser that failed. A non-Browser argument is not suppressed
                bool suppressed = true == (browser as Browser)?.BrowserSettings.SuppressJavaScriptFailures;
 
                if (false == suppressed)
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $@"Failed to execute javascript. Continuing.", browser, GPALObjectType.Browser, ex);
                retVal = "fail";
            }
 
            return UnwrapScriptResult(retVal?.ToString());
        }

        public static string GetCurrentUrl(BrowserSettings browserSettings)
        {
            string currentUrl = null;
 
            if (true == browserSettings.UseOttoMagic)
                currentUrl = browserSettings.MagicHelper.GetCurrentUrl();
            else if (true == browserSettings.UsePuppeteer)
                currentUrl = browserSettings.PuppeteerClient.GetCurrentUrl().Execute();
            else
                currentUrl = browserSettings.Browser.BrowserDriver.Url;
 
            // ottomagic and puppeteer answer over the rest layer, whose untyped Execute hands back the response
            // body as it stands - a json string, quotes and all. deserializing rather than trimming so an escaped
            // quote inside the url comes back as the character it is
            if (true == currentUrl?.StartsWith("\"") && true == currentUrl.EndsWith("\""))
                currentUrl = Newtonsoft.Json.JsonConvert.DeserializeObject<string>(currentUrl);
 
            return currentUrl;
        }
        public static string JsCheckReadyState(dynamic browser, string sessionToken)
        {
            string readyState = "loading";
 
            if (lastSessionToken != sessionToken)
            {
                lastErrorMessage.Clear();
                supressedMessage = false;
                lastSessionToken = sessionToken;
            }
 
            try
            {
                if (browser.UseOttoMagic)
                {
                    readyState = browser.BrowserSettings.MagicHelper.GetReadyStatus();
                    if (null == readyState)
                        readyState = @"loading";
                    else
                        readyState = readyState.Trim('"');
                }
                else
                {
                    bool withReturn = false == browser.UsePuppeteer;
 
                    // use javascript to get our html document location, but it's off by 2 pixels compared to uiautomation. uiautomation, however, failed so often that is was unreliable
                    readyState = ExecuteJavaScript(browser, $"{(true == withReturn ? "return " : "")}document.readyState");
                }
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION,
                    $"Error executing JavaScript",
                    null, GPALObjectType.None, ex);
            }
 
            string currentErrorMessage = $"Document.Ready status [{readyState}]";
 
            if (false == lastErrorMessage.Contains(currentErrorMessage))
            {
                lastErrorMessage.Add(currentErrorMessage);
                GPAL.PublishSimpleEvent(GPALEventType.INFO, currentErrorMessage);
                supressedMessage = false;
            }
            else if (false == supressedMessage)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Supressing repeat messages", currentErrorMessage, GPALObjectType.Other);
                supressedMessage = true;
            }
 
            return readyState;
        }
 
        // written by grok
        public static void SeleniumGoToUrl(GPALUrl url, BrowserSettings browserSettings)
        {
            string normalizedTarget = url.ToString().TrimEnd('/').ToLowerInvariant();
 
            try
            {
                // 1. Perform navigation
                browserSettings.BrowserDriver.Navigate().GoToUrl(url);
 
                // Give Chrome a brief moment for network events (helps on fast 304s / cached pages)
                System.Threading.Thread.Sleep(400); // 400–800 ms usually enough; can be replaced with WebDriverWait if preferred
 
                // 2. Fetch performance logs (Chrome/Edge only — geckodriver does not support getLog)
                if (browserSettings.BrowserType != BrowserType.FireFox)
                {
 
                var performanceLogs = browserSettings.BrowserDriver.Manage().Logs.GetLog("performance");
 
                int? bestStatus = null;
                string bestMatchedUrl = null;
                int candidatePriority = int.MaxValue;
 
                foreach (var entry in performanceLogs)
                {
                    string message = entry.Message;
 
                    if (message.IndexOf("Network.responseReceived", StringComparison.Ordinal) == -1)
                        continue;
 
                    // Quick filter to reduce sub-resource noise
                    if (message.IndexOf("\"url\":\"", StringComparison.Ordinal) == -1 ||
                        message.IndexOf(normalizedTarget, StringComparison.OrdinalIgnoreCase) == -1)
                        continue;
 
                    // Extract status
                    var statusMatch = System.Text.RegularExpressions.Regex.Match(
                        message,
                        @"""status(?:Code)?""\s*:\s*(\d+)",
                        System.Text.RegularExpressions.RegexOptions.IgnoreCase | System.Text.RegularExpressions.RegexOptions.CultureInvariant
                    );
 
                    if (!statusMatch.Success)
                        continue;
 
                    int status;
                    if (!int.TryParse(statusMatch.Groups[1].Value, out status))
                        continue;
 
                    if (status < 100 || status > 599)
                        continue;
 
                    // Priority: lower number = better candidate
                    int priority;
                    if (status == 304)
                        priority = 0;           // jackpot: cache hit
                    else if (status >= 200 && status < 300)
                        priority = 10;          // success
                    else if (status >= 300 && status < 400)
                        priority = 20;          // redirect (often intermediate)
                    else if (status >= 400 && status < 500)
                        priority = 30;          // client error (useful to know)
                    else if (status >= 500)
                        priority = 40;          // server error
                    else
                        priority = 50;
 
                    bool shouldUpdate = false;
 
                    if (!bestStatus.HasValue)
                    {
                        shouldUpdate = true;
                    }
                    else if (priority < candidatePriority)
                    {
                        shouldUpdate = true;
                    }
                    else if (priority == candidatePriority && status == 304)
                    {
                        shouldUpdate = true;
                    }
 
                    if (shouldUpdate)
                    {
                        bestStatus = status;
                        candidatePriority = priority;
 
                        // Extract matched URL for debug (optional)
                        var urlMatch = System.Text.RegularExpressions.Regex.Match(message, @"""url"":""([^""]+)""");
                        if (urlMatch.Success)
                        {
                            bestMatchedUrl = urlMatch.Groups[1].Value;
                        }
                    }
                }
 
                if (bestStatus.HasValue)
                {
                    browserSettings.ServerResponseCode = bestStatus.Value;
 
                    // Optional: log what we decided (uncomment for debugging)
                    /*
                    GPAL.PublishSimpleEvent(
                        GPALEventType.DEBUG,
                        $"Captured final status {bestStatus.Value} (priority {candidatePriority}) for ~{bestMatchedUrl ?? "unknown"}",
                        browserSettings,
                        GPALObjectType.Other
                    );
                    */
                }
                else
                {
                    // Rare: no plausible status found despite navigation
                    browserSettings.ServerResponseCode = 0;
                    GPAL.PublishSimpleEvent(
                        GPALEventType.DEBUG,
                        $"No valid HTTP status captured in performance logs for [{url}]",
                        browserSettings,
                        GPALObjectType.Other
                    );
                }
 
                } // end Chrome/Edge only block
                else
                {
                    try
                    {
                        var result = ((OpenQA.Selenium.IJavaScriptExecutor)browserSettings.BrowserDriver)
                            .ExecuteScript("return performance.getEntriesByType('navigation')[0]?.responseStatus ?? 0;");
                        browserSettings.ServerResponseCode = result != null && int.TryParse(result.ToString(), out int ffStatus) && ffStatus > 0 ? ffStatus : 0;
                    }
                    catch
                    {
                        browserSettings.ServerResponseCode = 0;
                    }
                }
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                browserSettings.ServerResponseCode = -1;
                GPAL.PublishSimpleEvent(
                    GPALEventType.EXCEPTION,
                    $"Unexpected error navigting to [{url}]",
                    browserSettings,
                    GPALObjectType.Other,
                    ex
                );
            }
        }


        public static KillProcessesResult KillProcessesByName(params string[] namesOfProcessesToKill)
        {
            KillProcessesResult result = null;
 
            foreach (var name in namesOfProcessesToKill)
                result = KillProcess(name);
 
            return result;
        }
        public static string GetUserAgent(IBrowser browser)
        {
            string userAgent = null;
 
            try
            {
                if (true == browser.UseOttoMagic)
                    userAgent = ((Browser)browser).MagicHelper.GetUserAgent();
                else if (true == browser.UsePuppeteer)
                    userAgent = browser.PuppeteerClient.GetUserAgent().Execute<string>();
                else
                    userAgent = ExecuteJavaScript(browser, "return navigator.userAgent");
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to get live user agent.", browser, GPALObjectType.Browser, ex);
            }
 
            if (true == string.IsNullOrEmpty(userAgent))
                userAgent = MagicHelper.GetUserAgentString(browser.BrowserType);
 
            // ottomagic and puppeteer answer over the rest layer, whose untyped Execute hands back the response
            // body as it stands - a json string, quotes and all - and a quote is not a legal header value
            if (true == userAgent?.StartsWith("\"") && true == userAgent.EndsWith("\""))
                userAgent = Newtonsoft.Json.JsonConvert.DeserializeObject<string>(userAgent);
 
            return userAgent;
        }
        public static bool TestUrlForPDF(Browser browser, string url)
        {
            if (null == url)
                return false;
 
            bool retVal = false;
 
            // does the url contain .pdf?
            if (0 <= url.IndexOf(".pdf", 0, StringComparison.OrdinalIgnoreCase))
                retVal = true;
            else
                using (HttpClient client = new HttpClient())
                {
                    client.DefaultRequestHeaders.Add("User-Agent", GetUserAgent(browser));
 
                    try
                    {
                        // Use HttpMethod.Head to retrieve headers ONLY (minimal overhead)
                        using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Head, url))
                        {
                            HttpResponseMessage response = client.SendAsync(request).Result;
 
                            if (response.IsSuccessStatusCode)
                            {
                                // 1. Check Content-Type for application/pdf
                                string contentType = response.Content.Headers.ContentType?.MediaType;
                                if (contentType != null && contentType.Equals("application/pdf", StringComparison.OrdinalIgnoreCase))
                                {
                                    retVal = true;
                                }
                                // 2. Fallback: Check Content-Disposition for a .pdf filename
                                else if (response.Content.Headers.ContentDisposition != null)
                                {
                                    string fileName = response.Content.Headers.ContentDisposition.FileName;
                                    if (!string.IsNullOrEmpty(fileName) && fileName.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase))
                                    {
                                        retVal = true;
                                    }
                                }
                            }
                            else
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Failed to verify URL [{url}] via HEAD request. Status code [{response.StatusCode}]");
                            }
                        }
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Exception during HEAD request for [{url}]", null, GPALObjectType.None, ex);
                    }
                }
 
            if (true == retVal)
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"GPAL detected PDF file opened in browser [{url}]");
 
            return retVal;
        }
        // how many times the file is asked for before giving up on the page in front of it clearing, and how long
        // to leave between asks
        internal const int PdfFetchAttempts = 3;
        internal const int PdfFetchWaitMs = 1500;

        internal static bool IsPdf(byte[] content)
        {
            return 4 < content?.Length
                && '%' == content[0] && 'P' == content[1] && 'D' == content[2] && 'F' == content[3];
        }
        public static bool DownloadPdfFile(string pdfUrl, string savePath, IBrowser browser)
        {
            // we clicked first, it could've downloaded the file, so look 
            string filePath = $@"{FileHelper.EnsureDirectoryEndsWithBackslash(Path.GetDirectoryName(savePath))}{System.IO.Path.GetFileName(pdfUrl)}";
            if (true == File.Exists(filePath))
            {
                File.Move(filePath, savePath);
                return true;
            }
            else
            {
                filePath = $@"{FileHelper.GetDefaultDownloadDirectory(browser)}{Path.GetFileName(pdfUrl)}";
                if (true == File.Exists(filePath))
                {
                    File.Move(filePath, savePath);
                    return true;
                }
            }
 
            // fetch first, from inside the page, so the file comes down the connection the browser already has:
            // its cookies, its tls fingerprint, its clearance. a url the site signed for this session only is
            // served to that connection and refused to the bare client below.
            // the click may still be on its way to the file, and a site that shows a redirect page first serves
            // that page at this very url until it is. so what came back has to be a pdf before it is written, and
            // when it is not we wait, ask where the browser is now, and try that
            for (int attempt = 0; attempt < PdfFetchAttempts; attempt++)
            {
                byte[] fetched = FetchBytes((Browser)browser, pdfUrl, null, out string _);
 
                if (true == IsPdf(fetched))
                {
                    File.WriteAllBytes(savePath, fetched);
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"PDF file [{pdfUrl}] fetched from the page to [{savePath}]");
                    return true;
                }
 
                if (attempt < PdfFetchAttempts - 1)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{pdfUrl}] answered with [{fetched?.Length ?? 0}] bytes that are not a pdf, waiting for the file itself", browser, GPALObjectType.Browser);
                    Thread.Sleep(PdfFetchWaitMs);
                    pdfUrl = GetCurrentUrl(((Browser)browser).BrowserSettings) ?? pdfUrl;
                }
                else
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{pdfUrl}] never answered with a pdf, the page in front of the file did. Nothing written to [{savePath}]", browser, GPALObjectType.Browser);
            }
 
            // fetch is subject to the page's origin, so a pdf served from somewhere else can be refused where a
            // plain request is not. that is what this is here for
            using (HttpClient client = new HttpClient())
            {
                client.DefaultRequestHeaders.Add("User-Agent", GetUserAgent(browser));
 
                try
                {
                    // Download the PDF file content synchronously
                    byte[] pdfContent = client.GetByteArrayAsync(pdfUrl).Result;
 
                    if (false == IsPdf(pdfContent))
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{pdfUrl}] answered with [{pdfContent?.Length ?? 0}] bytes that are not a pdf. Nothing written to [{savePath}]", browser, GPALObjectType.Browser);
                        return false;
                    }
 
                    // Save the PDF file to disk
                    File.WriteAllBytes(savePath, pdfContent);
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"PDF file [{pdfUrl}] downloaded successfully to [{savePath}]");
                    return true;
                }
                catch (GPALException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"PDF file NOT downloaded from [{pdfUrl}]", ex);
                }
            }
            return false;
        }

        internal static IWebDriver GetWebDriver(BrowserSettings browserSettings, dynamic options)
        {
            IWebDriver tmpDriver = null;
 
            try
            {
                if (true == browserSettings.HiddenDesktop)
                {
                    // the driver is started by hand so it can be named a desktop, and Selenium is handed the port
                    // rather than a service. StartHiddenDriver sets ServiceDriverPid from the launch
                    int hiddenPort = StartHiddenDriver(browserSettings);
 
                    tmpDriver = new OpenQA.Selenium.Remote.RemoteWebDriver(new Uri($"http://127.0.0.1:{hiddenPort}"), options);
                }
                else if (BrowserType.Chrome == browserSettings.BrowserType)
                {
                    // Create the service explicitly so we can get the PID
                    ChromeDriverService chromeService = ChromeDriverService.CreateDefaultService(browserSettings.DriverLocation);
                    chromeService.HideCommandPromptWindow = true; // optional but recommended
 
                    ChromeOptions chromeOptions = (ChromeOptions)options;
 
                    tmpDriver = new ChromeDriver(chromeService, chromeOptions);
 
                    // *** Capture the chromedriver.exe PID ***
                    browserSettings.ServiceDriverPid = chromeService.ProcessId;
                }
                else // Edge
                {
                    EdgeDriverService edgeService = EdgeDriverService.CreateDefaultService(browserSettings.DriverLocation);
                    edgeService.HideCommandPromptWindow = true; // optional
 
                    EdgeOptions edgeOptions = (EdgeOptions)options;
 
                    tmpDriver = new EdgeDriver(edgeService, edgeOptions);
 
                    // *** Capture the msedgedriver.exe PID (reuse the same property) ***
                    browserSettings.ServiceDriverPid = edgeService.ProcessId;
                }
            } catch (Exception ex)
            {
                string url = string.Empty;
                switch (browserSettings.BrowserType)
                {
                    case Enums.BrowserType.Chrome:
                        url = GPAL.GPALSettings.ChromeDriverUpdateURL;
                        break;
                    case Enums.BrowserType.Edge:
                        url = GPAL.GPALSettings.EdgeDriverUpdateURL;
                        break;
                    case Enums.BrowserType.FireFox:
                        url = GPAL.GPALSettings.FirefoxDriverUpdateURL;
                        break;
                }
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Please manually update the driver @ [{url}]..", browserSettings.Browser, GPALObjectType.Browser, ex);
            }
            return tmpDriver;
        }
        #region <KillProcess>

 
        private static class DriverInfos
        {
            public static DriverInfo Chrome = new DriverInfo(
                executableFileName: GPAL.GPALSettings.ChromeDriverFilename,
                processName: "chromedriver",
                browserName: "chrome"
                );
 
            public static DriverInfo FireFox = new DriverInfo(
                executableFileName: GPAL.GPALSettings.FirefoxDriverFilename,
                processName: "geckodriver",
                browserName: "firefox"
                );
 
            public static DriverInfo Edge = new DriverInfo(
                executableFileName: GPAL.GPALSettings.EdgeDriverFilename,
                processName: "MSEdgeDriver",
                browserName: "edge"
                );
 
        }

        private class DriverInfo
        {

            public DriverInfo(string executableFileName, string processName, string browserName)
            {
                ExecutableFileName = executableFileName;
                ProcessName = processName;
                BrowserName = browserName;
            }
            public string ExecutableFileName { get; private set; }
            public string ProcessName { get; private set; }
            public string BrowserName { get; private set; }
        }
        public class KillProcessesResult
        {
            public string Name { get; set; }
            public int Found { get; set; } = 0;
            public int Killed { get; set; } = 0;
            public bool Success { get; set; } = false;
        }

        public static KillProcessesResult[] KillAllRunningWebDrivers()
        {
            List<KillProcessesResult> result = new List<KillProcessesResult>();
            KillProcessesResult[] kpa = result.ToArray();
            alreadyUpdating = false;
 
            try
            {
                result.Add(KillProcess(DriverInfos.FireFox.ProcessName));
                result.Add(KillProcess(DriverInfos.Chrome.ProcessName));
                result.Add(KillProcess(DriverInfos.Edge.ProcessName));
 
                // zero out all browsers
                //GPAL.Browsers.Clear();
            } catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to kill processes", ex);
            }
 
            return kpa;
        }
        private const int MaxParentHops = 8;
 
        internal const int GracefulExitMs = 10000;   // long enough for a browser to write its profile out, short enough not to hold up a run
        internal const int ProfileReleaseWaitMs = 15000;   // a browser asked to close is still letting its profile go
        internal const int ProfileReleaseCheckMs = 250;

        private static void CloseBrowserGracefully(Browser aBrowser, bool endItAll)
        {
            try
            {
                if (true == aBrowser.UsePuppeteer)
                {
                    // Browser.close is CDP's own orderly shutdown
                    if (false == aBrowser.BrowserSettings.Process?.HasExited)
                    {
                        aBrowser.BrowserSettings.PuppeteerCommunicator.SendCommand(DevToolsMethods.BrowserClose, new Dictionary<string, object>(), null).GetAwaiter().GetResult();
                        aBrowser.PuppeteerCommunicator._readerReadyTcs.SetCanceled();
 
                        if (false == aBrowser.PuppeteerCommunicator._usePipes)
                            aBrowser.PuppeteerCommunicator.CloseOutputWebSocketAsync().GetAwaiter();
                    }
                }
                else if (true == aBrowser.UseSelenium && null != aBrowser.BrowserDriver)
                {
                    // Quit ends the session and closes every window; Close leaves the others alone when this is
                    // one of several GPAL browsers sharing the driver.
                    // EdgeDriver has a habit of sitting on Quit for its whole command timeout, which is what had
                    // this skipped altogether, so it gets its own budget here and the caller kills what is left
                    Task closing = Task.Run(() =>
                    {
                        if (true == endItAll)
                            aBrowser.BrowserDriver?.Quit();
                        else
                            aBrowser.BrowserDriver?.Close();     // NOTE: CAVEAT: closes the currently focused window
                    });
 
                    closing.Wait(GracefulExitMs);
                }
                else
                {
                    // ottomagic. an extension cannot close the browser hosting it, and a headless browser has no
                    // window for a message to reach. the browser's own windows it can close, and they are there
                    // in headless the same as anywhere, so those go first and the window message is left as the
                    // fallback for whatever survives it
                    // an extension cannot quit the browser hosting it, but it can close every window it has,
                    // and closing the last one is an ordinary quit: the profile is written and the exit is
                    // recorded clean rather than as a crash. this reaches headless, where a window message has
                    // nothing to arrive at.
                    // the extension enumerates them, so a window a page opened counts the same as one GPAL did,
                    // and nothing here has to know how many there are
                    // asking a browser that has already gone costs a full round of retries against a port
                    // nothing is listening on, and that shows up as a slow start on the next run
                    if (true == aBrowser.UseOttoMagic && true == aBrowser.IsAlive)
                        aBrowser.MagicHelper.CloseBrowser();
 
                    if (false == aBrowser.BrowserSettings.Process?.HasExited)
                        aBrowser.BrowserSettings.Process?.CloseMainWindow();
                }
            }
            catch
            {
            }
        }
 
        public static KillProcessesResult[] KillAllRunningProcesses(bool killWebDrivers, IBrowser browser)
        {
            List<KillProcessesResult> result = new List<KillProcessesResult>();
            KillProcessesResult[] kpa = result.ToArray();
 
            try
            {
                int browserCount = GPAL.Browsers.Count;
 
                if (0 == browserCount)
                {
                    // forms or converter or application, this is expected...
                    //GPAL.PublishSimpleEvent(GPALEventType.INFO, "No browsers to kill? How'd we get here?", browser, GPALObjectType.Browser);
                    return kpa;
                }
 
                // suppose we have two windows open, we will have two GPAL browsers in the list, both with the same driver
                // what happens if one workflow issues a .close() which ends the workflow instead of a .CloseWindow()
                // we can't perform any operation on the browserdriver as that will affect both windows
                // we could see the browser count > 1 and if killWebDrivers == false, assume it's not the end
                // then we can look at the browsersettings, get the CUrrentUrl and then CloseWindow(browserSettings.CurrentUrl)?
                foreach (Browser aBrowser in GPAL.Browsers)
                {
                    if (aBrowser == browser || null == browser) // null browser means unhandled exception, kill it all; browser not null = close(), killwebdrivers ends it all
                        try
                        {
                            bool endItAll = (aBrowser == browser && true == killWebDrivers) || null == browser;
 
                            // ask before killing. a browser that is killed records its own exit as a crash, and
                            // chromium reopens the previous session after a crash whatever session.restore_on_startup
                            // says, so every run GPAL killed taught the next one to come up holding its tabs. a
                            // killed browser also lingers holding the profile, long enough to write its own
                            // Preferences back over what GPAL set before launch
                            CloseBrowserGracefully(aBrowser, endItAll);
 
                            if (null != aBrowser.BrowserSettings.Process)
                                try
                                {
                                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{aBrowser.BrowserSettings.BrowserType}] waiting on process [{aBrowser.BrowserSettings.Process.Id}] to exit", aBrowser, GPALObjectType.Browser);
 
                                    // it was asked and it went, so there is nothing to kill. the kill is the
                                    // fallback it was always meant to be rather than the way this normally ends
                                    if (false == aBrowser.BrowserSettings.Process.WaitForExit(GracefulExitMs))
                                    {
                                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{aBrowser.BrowserSettings.BrowserType}] did not shut down within [{GracefulExitMs / 1000}] seconds, killing it. Its profile will read as crashed.", aBrowser, GPALObjectType.Browser);
                                        result.Add(KillProcess(aBrowser.BrowserSettings.Process));
                                    }
                                }
                                catch
                                { }
 
                            // the webdriver service is ours, not the user's, and holds no profile to leave in a
                            // bad state, so it is killed outright once the browser it was driving is gone
                            if (true == aBrowser.UseSelenium && true == endItAll)
                            {
                                int pid = aBrowser.BrowserSettings.ServiceDriverPid;
 
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, ">>> Disposing browser driver");
 
                                try
                                {
                                    if (0 < pid)
                                        KillProcessTree(pid);
 
                                    // fallback - kill any stray driver processes by name
                                    var kpaDrivers = KillAllRunningWebDrivers();
 
                                    foreach (KillProcessesResult killProcessesResult in kpaDrivers)
                                        result.Add(killProcessesResult);
                                }
                                catch { /* ignore */ }
                                finally
                                {
                                    aBrowser.BrowserSettings.ServiceDriverPid = 0;
                                }
                            }
 
                            Thread.Sleep(1_000);
                        }
                        catch
                        {
 
                        }
                        finally
                        {
                            if (true == aBrowser.BrowserSettings.TempProfileCreated)
                                ChromeProfileManager.RemoveTempUserProfile(aBrowser.BrowserSettings.ProfileDataDirectory);
 
                            // this browser's own profile is handled above. the sweep is for the ones a run that
                            // never reached here left behind
                            ChromeProfileManager.SweepAbandonedTempProfiles();
                        }
                }
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to kill processes", ex);
            }
 
            // obsolete now that we have kill process tree since capturing the driver pid
            //try
            //{
            //    if (true == killWebDrivers)
            //    {
            //        var kpaDrivers = KillAllRunningWebDrivers();
 
            //        foreach (KillProcessesResult killProcessesResult in kpaDrivers)
            //            result.Add(killProcessesResult);
 
            //        foreach (IBrowser aBrowser in GPAL.Browsers)
            //            switch (aBrowser.BrowserType)
            //            {
            //                case BrowserType.Chrome:
            //                    KillProcess(DriverInfos.Chrome.BrowserName);
            //                    break;
 
            //                case BrowserType.Edge:
            //                    KillProcess(DriverInfos.Edge.BrowserName);
            //                    break;
 
            //                case BrowserType.FireFox:
            //                    KillProcess(DriverInfos.FireFox.BrowserName);
            //                    break;
            //            }
            //    }
            //}
            //catch { }
 
            return result.ToArray();
        }
        internal static void EndWindowlessSuccessor(BrowserSettings browserSettings)
        {
            int ourPid = browserSettings.Process?.Id ?? 0;
 
            if (0 < ourPid)
            {
                // the replacement is started as our browser goes, so there is nothing to find until it has gone
                WaitForBrowserToExit(browserSettings, Browser.ProfileRestoreWaitMs);
 
                // caught rather than thrown: WMI is not always answerable, and a browser we could not ask about
                // is not a reason to fail a close that has otherwise finished
                try
                {
                    using (var search = new System.Management.ManagementObjectSearcher(
                        $"SELECT ProcessId, CommandLine FROM Win32_Process WHERE ParentProcessId = {ourPid}"))
                        foreach (System.Management.ManagementObject found in search.Get())
                        {
                            string commandLine = found["CommandLine"]?.ToString() ?? string.Empty;
 
                            if (true == commandLine.Contains("--no-startup-window"))
                            {
                                int successor = Convert.ToInt32(found["ProcessId"]);
 
                                GPAL.PublishSimpleEvent(GPALEventType.INFO,
                                    $"Ending the windowless browser [{successor}] left behind by [{ourPid}], which would hold the profile open",
                                    browserSettings, GPALObjectType.Browser);
 
                                KillProcessTree(successor);
                            }
                        }
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Could not look for a windowless browser left behind by [{ourPid}]",
                        browserSettings, GPALObjectType.Browser, ex);
                }
            }
        }
 
        private static void KillProcessTree(int pid)
        {
            if (pid <= 0) return;
 
            try
            {
                var startInfo = new ProcessStartInfo
                {
                    FileName = "taskkill",
                    Arguments = $"/PID {pid} /T /F",  // /T = tree, /F = force
                    WindowStyle = ProcessWindowStyle.Hidden,
                    CreateNoWindow = true,
                    UseShellExecute = false
                };
 
                using (Process taskkillProcess = Process.Start(startInfo))
                {
                    taskkillProcess?.WaitForExit(5000); // optional timeout
                }
            }
            catch
            {
                // Ignore errors during cleanup (access denied, already dead, etc.)
            }
        }
        public static KillProcessesResult KillProcess(Process process)
        {
            if (null != process)
            {
                KillProcessesResult result = new KillProcessesResult();
 
                string Name = "";
                try
                {
                    Name = process.ProcessName;
                    result.Name = Name;
                }
                catch (Exception)
                {
                    // process has already exited, nothing to do
                    result.Success = true;
                    return result;
                }
 
                process.Kill();
                process.WaitForExit();
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Killed process [{result.Name}]");
                return result;
            }
            return null;
        }

        public static KillProcessesResult KillProcess(string processName)
        {
            KillProcessesResult result = new KillProcessesResult { Name = processName };
 
            var processes = Process.GetProcessesByName(processName);
 
            result.Found = processes.Count();
 
            if (result.Found > 0)
            {
                foreach (var process in processes)
                {
                    do
                    {
                        try
                        {
                            process.Kill();
                            process.WaitForExit(); // Wait for the process to finish
                        }
                        catch
                        {
 
                        }
                    } while (false == process.HasExited);
 
                    result.Killed++;
                    result.Success = true;
                    result.Found++;
 
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Killed process [{processName}]");
                }
            }
            else
            {
                result.Success = true;
            }
 
            return result;
        }
        #endregion <KillProcess>
 
        private static Stopwatch watch = new System.Diagnostics.Stopwatch();
        private static int lastX = 0, lastY = 0;
 
        // https://stackoverflow.com/questions/8438483/bring-browser-from-back-to-front-selenium-web-driver-java
        // NOTE: we prolly should restore the window, too, if it's minimized? or do a hardware top for all?
        // NOTE: maybe we only need to top if we have image selectors? otherwise, we don't need to be on top...
        private const string PortsLogName = "ports-in-use.log";

        private const string NativeHostName = "gpal.rest.api.nativeapp";

        private static readonly string[] NativeHostKeys =
        {
            @"Software\\Google\\Chrome\\NativeMessagingHosts\\" + NativeHostName,
            @"Software\\Chromium\\NativeMessagingHosts\\" + NativeHostName,
            @"Software\\Microsoft\\Edge\\NativeMessagingHosts\\" + NativeHostName,
            @"Software\\WOW6432Node\\Google\\Chrome\\NativeMessagingHosts\\" + NativeHostName,
            @"Software\\WOW6432Node\\Chromium\\NativeMessagingHosts\\" + NativeHostName,
            @"Software\\WOW6432Node\\Microsoft\\Edge\\NativeMessagingHosts\\" + NativeHostName
        };

        public static string RestApiLocation()
        {
            string retVal = null;
 
            foreach (RegistryKey root in new[] { Registry.CurrentUser, Registry.LocalMachine })
            {
                foreach (string keyPath in NativeHostKeys)
                    try
                    {
                        using (RegistryKey key = root.OpenSubKey(keyPath))
                        {
                            string manifestPath = key?.GetValue(null) as string;
 
                            if (true == string.IsNullOrWhiteSpace(manifestPath) || false == File.Exists(manifestPath))
                                continue;
 
                            // the manifest is the browser's own record of where the host lives, so it is the
                            // one place that cannot disagree with what the browser will actually launch
                            dynamic manifest = Newtonsoft.Json.JsonConvert.DeserializeObject(File.ReadAllText(manifestPath));
                            string exePath = manifest?.path;
 
                            if (false == string.IsNullOrWhiteSpace(exePath) && true == File.Exists(exePath))
                            {
                                retVal = exePath;
                                break;
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Could not read the native host registration at [{keyPath}]", null, GPALObjectType.None, ex);
                    }
 
                if (false == string.IsNullOrWhiteSpace(retVal))
                    break;
            }
 
            return retVal;
        }

        public static int LiveRestApiHosts(out List<(int processId, int port)> hosts)
        {
            hosts = new List<(int processId, int port)>();
 
            string exePath = RestApiLocation();
            string folder = true == string.IsNullOrWhiteSpace(exePath) ? null : Path.GetDirectoryName(exePath);
            string path = true == string.IsNullOrWhiteSpace(folder) ? null : Path.Combine(folder, PortsLogName);
 
            // no file is the ordinary answer when no ottomagic browser is up, because the last host out deletes it
            if (true == string.IsNullOrWhiteSpace(path) || false == File.Exists(path))
                return hosts.Count;
 
            try
            {
                // opened sharing rather than locking, so reading never holds a host up mid write
                using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                using (StreamReader reader = new StreamReader(stream))
                    foreach (string line in reader.ReadToEnd().Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries))
                    {
                        string[] halves = line.Split(':');
 
                        if (2 == halves.Length
                            && true == int.TryParse(halves[0], out int pid)
                            && true == int.TryParse(halves[1], out int port)
                            && false == hosts.Any(each => each.processId == pid))
                            hosts.Add((pid, port));
                    }
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Could not read the host list at [{path}]", null, GPALObjectType.None, ex);
            }
 
            return hosts.Count;
        }
 
        internal static bool RestApiAnswering(string baseUrl)
        {
            bool retVal = false;
 
            if (false == string.IsNullOrWhiteSpace(baseUrl))
                try
                {
                    System.Net.HttpWebRequest ask = (System.Net.HttpWebRequest)System.Net.WebRequest.Create($"{baseUrl}status");
                    ask.Timeout = 1000;
                    ask.ReadWriteTimeout = 1000;
 
                    using (System.Net.HttpWebResponse answer = (System.Net.HttpWebResponse)ask.GetResponse())
                        retVal = System.Net.HttpStatusCode.OK == answer.StatusCode;
                }
                catch (System.Net.WebException)
                {
                    // refused, timed out, or answered something other than a running server
                }
 
            return retVal;
        }
 
        public static void TopBrowser(Browser browser, bool moveDontClick, bool forceClick = false) // AFTER HARDWARE CLICK, we want to MOVE NOT CLICK mouse off browser into a safe area, so reusing this method
        {
            int x, y;
 
            // only top if we are interacting with the page using hardware or image mathcing on the current selector
            // NOTE: null-safe throughout - CurrentUOW/CurrentSelector are null when we are not on a real page, and
            // a (bool) cast on a null bool? (from the ?. chain) would throw.
            bool needsTopping = (browser.CurrentUOW?.CurrentSelector?.SelectorSettings?.SelectorPaths
                    ?.Any(path => path.SelectorPathType == SelectorPathType.Image) ?? false)
                    || InteractionType.Hardware == browser.CurrentUOW?.CurrentSelector?.InteractionType
                    || true == browser.BrowserSettings.UseHardware;
 
            // don't do anything, we are always in control and only need it for image matching potentially
            if (true == browser.BrowserSettings.UseHeadless || false == needsTopping)
                return;
 
            // NOTE: moving the mouse off the browser after a click exists only to keep stray :hover effects
            // (tooltips, dropdowns, sticky menus) from interfering with subsequent IMAGE selector matching -
            // CSS/DOM selectors don't care about hover state. If this unit of work has no image selectors,
            // skip the move entirely: teleporting the cursor off-viewport after every single click is itself
            // a mechanical, easily-fingerprinted pattern, so we only pay that cost when it's actually needed.
            if (true == moveDontClick || true == forceClick)   // selenium control - click the topmost left corner of the browser to top it, only thing that works 100%
            {
                if (true == watch.IsRunning)
                {
                    if (1000 > watch.ElapsedMilliseconds)   // don't do anything if we were called less than 1 second ago, avoid double clicking the browser (max window)
                        return;
                    watch.Stop();
                }
                watch = System.Diagnostics.Stopwatch.StartNew();
 
                HardwareHelper.GetCursorPos(out Point currCursorPos);
 
                if (true == browser.BrowserSettings.UseHardware || true == GPAL.UseHardware || Enums.BrowserType.FireFox == browser.BrowserSettings.BrowserType || true == moveDontClick || true == forceClick)
                {
                    try
                    {
                        Rectangle rect = ElementHelper.GetWindowRectangle(browser);
                        // true screen origin/size of the window (window.screenLeft/screenTop + outer size), so
                        // nothing has to guess at the height of a browser's chrome
                        Rectangle outerRect = ElementHelper.GetWindowOuterRectangle(browser);
 
                        if (true == moveDontClick)
                        {
                            // NOTE: park the mouse at a randomly chosen edge/corner of the window (above, below,
                            // left, or right) instead of always "above the window" - landing in the same relative
                            // spot after every click is itself a mechanical, easily-fingerprinted pattern.
                            var rnd = new Random();
                            switch (rnd.Next(4))
                            {
                                case 0: x = rect.Left + rnd.Next(-50, rect.Width + 50); y = outerRect.Top - 20; break;   // above the true window top
                                case 1: x = rect.Left + rnd.Next(-50, rect.Width + 50); y = rect.Bottom + 50; break;    // below
                                case 2: x = rect.Left - 60; y = rect.Top + rnd.Next(0, rect.Height); break;             // left
                                default: x = rect.Right + 60; y = rect.Top + rnd.Next(0, rect.Height); break;          // right
                            }
                            // keep the parked cursor on the desktop working area - "below" a maximized window is
                            // the taskbar, and hovering there pops window previews that then sit over the page
                            // and corrupt the very image matching this parking exists to protect
                            Rectangle workingArea = Screen.PrimaryScreen.WorkingArea;
                            x = Math.Min(Math.Max(workingArea.Left, x), workingArea.Right - 1);
                            y = Math.Min(Math.Max(workingArea.Top, y), workingArea.Bottom - 1);
                        }
                        else
                        {
                            // click a few pixels into the true top-left corner of the browser window to top it -
                            // just inside the corner so we land on the window frame/title, off the resize hotzone,
                            // and never on a tab or page content
                            x = outerRect.Left + 3;
                            y = outerRect.Top + 3;
                        }
                        lastX = x;
                        lastY = y;
                        //x = browser.BrowserDriver.Manage().Window.Position.X + 6;
                        //y = x = browser.BrowserDriver.Manage().Window.Position.Y + 6;
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to get browser coords. Retrying.", browser, GPALObjectType.Browser, ex);
                        x = lastX;
                        y = lastY;
                    }
 
                    if (true == GPAL.SimulateMouse || true == browser.CurrentUOW.CurrentSelector?.SimulateMouse)
                        HardwareHelper.MoveMouse(x, y, 10, 10);    // if we are moving the mouse out of the way, just move it up from the current spot
                    else
                        HardwareHelper.MoveMouse(x, y);
 
                    if (false == moveDontClick)
                    {
                        HardwareHelper.HardwareClick(x, y, ClickType.LeftClick);     // firefox will not take software topping requests, only hardware clicks
                        Thread.Sleep(10);
 
                        if (true == GPAL.SimulateMouse || true == browser.CurrentUOW.CurrentSelector?.SimulateMouse)
                            HardwareHelper.MoveMouse(currCursorPos.X, currCursorPos.Y, 10, 10);
                        else
                            HardwareHelper.MoveMouse(currCursorPos.X, currCursorPos.Y);
                    }
                }
                else if (true == browser.UseSelenium)
                {
                    try
                    {
                        // Top the browser by re-focusing whatever tab Selenium currently has active - a no-op for
                        // tab selection but it still brings the window forward. Do NOT switch by our own
                        // CurrentTabIdx here: if that index is stale it yanks focus to the wrong tab right before
                        // a click. If even this is problematic, the catch falls back to the WindowHelper route.
                        browser.BrowserDriver.SwitchTo().Window(browser.BrowserDriver.CurrentWindowHandle);
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        string tmpStr = "Faling back to windows topping.";
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to top browser using Selenium. {(false == GPAL.NoFallbackRecoveryActions ? tmpStr : String.Empty)})", browser, GPALObjectType.Browser, ex);
                        if (false == GPAL.NoFallbackRecoveryActions)
                            WindowHelper.TopBrowser(browser.BrowserSettings.Process, browser.BrowserSettings.ServiceDriverPid);
                    }
                }
            }
            else
            {   // we have an image selector or we use ottomagic or puppeteer so use windows to top the browser
                if (false == WindowHelper.TopBrowser(browser.BrowserSettings.Process, browser.BrowserSettings.ServiceDriverPid))
                    throw new GPALException("Browser has exited");
            }
        }
 
        // New utility method to find a free port
        internal static bool CameFromProcess(int processId, int ancestorId)
        {
            bool retVal = false;
            int walking = processId;
 
            for (int hop = 0; hop < MaxParentHops && 0 != walking && false == retVal; hop++)
            {
                if (walking == ancestorId)
                    retVal = true;
                else
                    walking = ParentOf(walking);
            }
 
            return retVal;
        }

        private static int ParentOf(int processId)
        {
            int retVal = 0;
 
            try
            {
                using (var searcher = new System.Management.ManagementObjectSearcher(
                    $"SELECT ParentProcessId FROM Win32_Process WHERE ProcessId = {processId}"))
                    foreach (System.Management.ManagementObject each in searcher.Get())
                        using (each)
                            retVal = Convert.ToInt32(each["ParentProcessId"]);
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Could not ask who started process [{processId}]", null, GPALObjectType.Browser, ex);
            }
 
            return retVal;
        }

        internal static void RefuseIfProfileIsNotARoot(BrowserSettings browserSettings)
        {
            string given = browserSettings.ProfileDataDirectory;
 
            if (true == string.IsNullOrEmpty(given) || BrowserType.FireFox == browserSettings.BrowserType) return;
 
            // A user data root holds profile directories; a profile directory holds a Preferences file
            if (false == File.Exists(Path.Combine(given, "Preferences"))) return;
 
            string message = $"[{given}] is a browser profile, not a user data directory. Chromium would build a new empty profile inside it and run that instead. Give the directory that holds it and name the profile, or copy it out to a directory of its own";
 
            GPAL.PublishSimpleEvent(GPALEventType.ERROR, message, null, GPALObjectType.Browser);
 
            throw new GPALException($"{GPAL.MyMethodName()}: {message}");
        }
 
        internal static void RefuseIfProfileIsOpen(BrowserSettings browserSettings)
        {
            RefuseIfProfileIsNotARoot(browserSettings);
 
            string profile = browserSettings.ProfileDataDirectory;
 
            if (false == string.IsNullOrEmpty(profile))
            {
                string wanted = NormalizedDirectory(profile);
                bool isFirefox = BrowserType.FireFox == browserSettings.BrowserType;
 
                string processName = true == isFirefox ? "firefox.exe"
                                   : BrowserType.Edge == browserSettings.BrowserType ? "msedge.exe"
                                   : "chrome.exe";
 
                // chromium names the profile with --user-data-dir=, firefox with -profile and the path after it
                string profileArgument = true == isFirefox
                                       ? "-profile\\s+(\"[^\"]*\"|[^\\s]*)"
                                       : "--user-data-dir=(\"[^\"]*\"|[^\\s]*)";
                int holder = 0;
 
                // A browser started without --user-data-dir owns the default root. Chromium only: Firefox
                // names a profile rather than a root, and its default cannot be derived from the path
                string defaultRoot = true == isFirefox
                                   ? null
                                   : NormalizedDirectory(GetBrowserProfileDirectory(browserSettings, true));
 
                holder = ProfileHolder(profile, processName, wanted, profileArgument, defaultRoot);
 
                // a browser closed a moment ago is still letting the profile go, and a run that follows another
                // on the same profile arrives during exactly that. Only a profile still held after the wait is
                // one somebody else is using
                if (0 != holder)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Profile [{profile}] is still held by [{processName}] process [{holder}], waiting up to [{ProfileReleaseWaitMs / 1000}] seconds for it to be let go", null, GPALObjectType.Browser);
 
                    Stopwatch waited = Stopwatch.StartNew();
 
                    while (0 != holder && ProfileReleaseWaitMs > waited.ElapsedMilliseconds)
                    {
                        System.Threading.Thread.Sleep(ProfileReleaseCheckMs);
 
                        holder = ProfileHolder(profile, processName, wanted, profileArgument, defaultRoot);
                    }
 
                    if (0 == holder)
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Profile [{profile}] was let go after [{waited.ElapsedMilliseconds}] ms", null, GPALObjectType.Browser);
                }
 
                if (0 != holder)
                {
                    string message = $"Profile [{profile}] is already open in [{processName}] process [{holder}]. Close it, or give this run a profile of its own";
 
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, message, null, GPALObjectType.Browser);
 
                    throw new GPALException($"{GPAL.MyMethodName()}: {message}");
                }
            }
        }

        private static int ProfileHolder(string profile, string processName, string wanted, string profileArgument, string defaultRoot)
        {
            int retVal = 0;
 
            try
            {
                using (var search = new System.Management.ManagementObjectSearcher(
                    $"SELECT ProcessId, CommandLine FROM Win32_Process WHERE Name = '{processName}'"))
                {
                    foreach (System.Management.ManagementObject found in search.Get())
                    {
                        string commandLine = found["CommandLine"]?.ToString();
 
                        if (true == string.IsNullOrEmpty(commandLine)) continue;
 
                        // Only the browser process holds the profile. Renderer, GPU, utility and crashpad
                        // processes carry --type= and no --user-data-dir, so without this every child of
                        // every browser reads as holding the default root
                        if (true == commandLine.Contains("--type=")) continue;
 
                        Match match = Regex.Match(commandLine, profileArgument);
 
                        // The root it named, or the default root when it named none
                        string holds = true == match.Success
                                     ? NormalizedDirectory(match.Groups[1].Value.Trim('"'))
                                     : defaultRoot;
 
                        if (true == string.IsNullOrEmpty(holds)) continue;
 
                        if (true == wanted.Equals(holds, StringComparison.OrdinalIgnoreCase))
                        {
                            retVal = Convert.ToInt32(found["ProcessId"]);
                            break;
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                // asking is a courtesy, not the job. a machine that will not answer wmi gets the old behavior
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Could not check whether [{profile}] is already open", null, GPALObjectType.Browser, ex);
            }
 
            return retVal;
        }
        // trailing slashes and relative segments make two spellings of one directory, and a command line can carry
        // either. compared as full paths, they are the same directory or they are not
        private static string NormalizedDirectory(string path)
        {
            string retVal = path;
 
            try
            {
                retVal = Path.GetFullPath(path).TrimEnd('\\', '/');
            }
            catch
            {
                retVal = path.TrimEnd('\\', '/');
            }
 
            return retVal;
        }
 
        // a browser sets these for itself, and a fetch issued in the page refuses to be told otherwise. copying
        // them into a request achieves nothing; the ones that matter are whatever the site's own code added
        private static readonly string[] browserOwnedHeaders =
        {
            "referer", "user-agent", "host", "origin", "cookie", "content-length",
            "connection", "accept-encoding", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform",
            "sec-ch-ua-arch", "sec-ch-ua-model", "sec-ch-ua-full-version-list", "sec-ch-device-memory",
            "sec-fetch-dest", "sec-fetch-mode", "sec-fetch-site"
        };

        internal static bool BrowserOwnedHeader(string name)
        {
            return Array.Exists(browserOwnedHeaders, owned => owned.Equals(name, StringComparison.OrdinalIgnoreCase));
        }
 
        // ports already handed out, and when. a browser takes seconds to start listening, and in that gap the
        // port it was given still looks free to anyone who asks. holding the number for a while is what stops
        // the second browser being sent to the first one's port
        // Every port handed out by this process. Nothing is removed: a browser may not bind for some seconds
        // and stays running afterwards, and reusing a port would put two browsers on one session
        private static readonly HashSet<int> portsHandedOut = new HashSet<int>();
 
        // Where the next search starts, so ports climb instead of rescanning from the base
        private static int lastPortHandedOut = 0;

        internal static int StartHiddenDriver(BrowserSettings browserSettings)
        {
            int retVal = FindFreePort();
            string driverName;
            string portArgument;
 
            switch (browserSettings.BrowserType)
            {
                case BrowserType.FireFox:
                    driverName = "geckodriver.exe";
                    portArgument = $"--port {retVal}";
                    break;
                case BrowserType.Edge:
                    driverName = "msedgedriver.exe";
                    portArgument = $"--port={retVal}";
                    break;
                default:
                    driverName = "chromedriver.exe";
                    portArgument = $"--port={retVal}";
                    break;
            }
 
            string driverPath = Path.Combine(browserSettings.DriverLocation ?? AppDomain.CurrentDomain.BaseDirectory, driverName);
 
            // Selenium Manager fetches a driver that is not there, and it is not on this path. A hidden run needs
            // the driver already in place, so it says which one and where rather than failing further along
            if (false == File.Exists(driverPath))
                throw new GPALException($"No [{driverName}] in [{Path.GetDirectoryName(driverPath)}], which a hidden run needs because Selenium Manager cannot fetch one here");
 
            Native.STARTUPINFO startup = new Native.STARTUPINFO
            {
                cb = (uint)Marshal.SizeOf<Native.STARTUPINFO>(),
                lpDesktop = HiddenDesktop.StartupNameFor(browserSettings.HiddenDesktopName)
            };
 
            if (false == Native.CreateProcess(null, $@"""{driverPath}"" {portArgument}", IntPtr.Zero, IntPtr.Zero, false,
                                              Native.CREATE_NO_WINDOW, IntPtr.Zero, null, ref startup, out var started))
                throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error(), $"Could not start [{driverName}] on [{browserSettings.HiddenDesktopName}]");
 
            Native.CloseHandle(started.hProcess);
            Native.CloseHandle(started.hThread);
 
            // the pid the teardown kills by, taken straight from the launch rather than out of a DriverService.
            // nothing else holds the browser the driver launches, so a lost pid is a browser that outlives the run
            browserSettings.ServiceDriverPid = (int)started.dwProcessId;
 
            if (false == WaitForPort(retVal, 30_000))
                throw new GPALException($"[{driverName}] never answered on port [{retVal}]");
 
            return retVal;
        }

        private static bool WaitForPort(int port, int timeoutInMs)
        {
            bool retVal = false;
            DateTime giveUpAt = DateTime.Now.AddMilliseconds(timeoutInMs);
 
            while (false == retVal && DateTime.Now < giveUpAt)
            {
                try
                {
                    using (TcpClient probe = new TcpClient())
                    {
                        probe.Connect("127.0.0.1", port);
                        retVal = true;
                    }
                }
                catch (SocketException)
                {
                    Thread.Sleep(200);
                }
            }
 
            return retVal;
        }

        internal static int FindFreePort(int startPort = 0xdead)
        {
            int retVal = 0;
 
            lock (portsHandedOut)
            {
                // Continues from the last port handed out, so one not yet bound is not offered again
                int from = Math.Max(startPort, lastPortHandedOut + 1);
 
                for (int port = from; port <= 65535 && 0 == retVal; port++)
                {
                    if (true == portsHandedOut.Contains(port))
                        continue;
 
                    // binding is the only question that answers itself. connecting says free for a port nobody
                    // is listening on yet, which is every port a browser has been told to use and not reached
                    TcpListener probe = null;
 
                    try
                    {
                        probe = new TcpListener(System.Net.IPAddress.Loopback, port);
                        probe.Start();
 
                        // claimed while the lock is still held, so the next thread in never sees this one free
                        portsHandedOut.Add(port);
                        lastPortHandedOut = port;
                        retVal = port;
                    }
                    catch (SocketException)
                    {
                        // somebody holds it, so move along
                    }
                    finally
                    {
                        probe?.Stop();
                    }
                }
            }
 
            if (0 == retVal)
                throw new GPALException($"No free ports available between [{startPort}] and [65535].");
 
            return retVal;
        }

        internal static string DocumentStamp(Browser browser)
        {
            // OttoMagic uses its own endpoint. ExecuteJavaScript there runs through userScripts.execute, which
            // requires the profile to allow user scripts
            string stamp = true == browser.UseOttoMagic
                ? browser.MagicHelper.GetDocumentStamp()
                : ExecuteJavaScriptObj("return performance.timeOrigin + '|' + location.href;", browser)?.ToString();
 
            return stamp;
        }
 
        // timeOrigin is first so the split is on the first bar: a url may contain one, a number may not
        private static string OriginFromStamp(string stamp)
        {
            int bar = stamp?.IndexOf('|') ?? -1;
            string origin = 0 <= bar ? stamp.Substring(0, bar) : null;
 
            return origin;
        }
 
        private static string UrlFromStamp(string stamp)
        {
            int bar = stamp?.IndexOf('|') ?? -1;
            string url = 0 <= bar ? stamp.Substring(bar + 1) : null;
 
            return url;
        }
 
        internal static object MarkDocument(Browser browser)
        {
            object mark = null;
 
            // Selenium marks the document with the html element and detects replacement by staleness, which
            // needs no script. The others take a stamp, which also carries the url the poll would otherwise
            // fetch separately
            if (true == browser.UseSelenium)
                mark = browser.BrowserDriver.FindElement(By.TagName("html"));
            else if (true == browser.UsePuppeteer || true == browser.UseOttoMagic)
                mark = DocumentStamp(browser);
 
            return mark;
        }
        internal static bool DocumentReplaced(Browser browser, object mark)
        {
            bool replaced = false;
 
            if (true == browser.UseSelenium && mark is IWebElement html)
                try { _ = html.TagName; }
                catch (StaleElementReferenceException) { replaced = true; }
                catch (NoSuchElementException) { replaced = true; }
            else if (mark is string priorStamp)
            {
                string nowOrigin = OriginFromStamp(DocumentStamp(browser));
                string wasOrigin = OriginFromStamp(priorStamp);
 
                replaced = false == string.IsNullOrEmpty(nowOrigin)
                        && false == string.IsNullOrEmpty(wasOrigin)
                        && wasOrigin != nowOrigin;
            }
 
            return replaced;
        }
        internal static bool WaitForNavigationOrTime(Browser browser, string priorUrl, object documentMark, int timeOutAfterMs)
        {
            bool navigated = false;
            bool documentGone = false;
            bool urlChanged = false;
            string lastUrl = priorUrl;
            DateTime timeout = DateTime.Now.AddMilliseconds(timeOutAfterMs);
            GPALEventType publishToConsole = GPAL.GPALSettings.ConsoleEvents;
            GPALEventType publishToDebug = GPAL.GPALSettings.DebugEvents;
 
            GPAL.GPALSettings.ConsoleEvents = GPALEventType.NONE;
            GPAL.GPALSettings.DebugEvents = GPALEventType.NONE;
 
            try
            {
                if (true == browser.IsAlive)
                    do
                    {
                        // One round trip for both signals. A stamp that cannot be read leaves the last known
                        // url in place and the loop repeats
                        if (documentMark is string priorStamp)
                        {
                            string nowStamp = DocumentStamp(browser);
                            string nowOrigin = OriginFromStamp(nowStamp);
                            string wasOrigin = OriginFromStamp(priorStamp);
 
                            // A missing origin on either side means undecided. Only two origins that both
                            // arrived and differ count as a replaced document
                            documentGone = false == string.IsNullOrEmpty(nowOrigin)
                                        && false == string.IsNullOrEmpty(wasOrigin)
                                        && wasOrigin != nowOrigin;
 
                            lastUrl = UrlFromStamp(nowStamp) ?? lastUrl;
                        }
                        else
                        {
                            documentGone = DocumentReplaced(browser, documentMark);
                            lastUrl = browser.GetSetCurrentUrl();
                        }
 
                        urlChanged = false == UrlHelper.AreEquivalent(priorUrl, lastUrl);
 
                        navigated = true == documentGone || true == urlChanged;
 
                        // only sleep if we are going round again, so a grace of 0 costs exactly one look
                        if (false == navigated && DateTime.Now < timeout)
                            Thread.Sleep(NavigationPollMs);
                    }
                    while (false == navigated && DateTime.Now < timeout);
            }
            finally
            {
                GPAL.GPALSettings.ConsoleEvents = publishToConsole;
                GPAL.GPALSettings.DebugEvents = publishToDebug;
            }
 
            // said once per click, after publishing is back on, so we can see which of the two signals answered and
            // whether the engine gave us a document to watch at all
            GPAL.PublishSimpleEvent(GPALEventType.DEBUG,
                $"Navigation watch [{(navigated ? "navigated" : "stayed")}] document [{(documentGone ? "replaced" : "same")}] url [{(urlChanged ? "changed" : "same")}] mark [{(documentMark is long mark ? mark.ToString() : null == documentMark ? "none" : "held")}] from [{priorUrl}] to [{lastUrl}]",
                browser, GPALObjectType.Browser);
 
            return navigated;
        }
        // TODO - pass in a unique GUID to each method (add a parm to them) so the called methods can suppress repeat messages knowing they are called with the same token (so the same session)
        // gap between asks while waiting on a page state. short enough that a page which is already there is not
        // held up noticeably, long enough that a page taking seconds is asked tens of times rather than hundreds
        internal const int WaitForValuePollMs = 250;
 
        internal static void WaitForValueOrTime(Browser browser, string resultsExpected, Func<dynamic> action, int timeOutAfterMs)
        {
            DateTime timeout = DateTime.Now.AddMilliseconds(timeOutAfterMs);
            string taskReturnValue;
            // the expected values are written with a space after each comma, so every token but the first
            // carried a leading space and could never be found in an answer. "interactive" and "nav-error"
            // were unreachable, and a failed navigation polled to the full timeout instead of stopping
            string[] tokens = resultsExpected.Split(',').Select(token => token.Trim()).ToArray();
 
            if (true == browser.IsAlive)
                do
                {
                    dynamic result = action();
 
                    try
                    {
                        // Try to treat as Task
                        var task = (Task<string>)result; // will throw if not a Task
                        task.GetAwaiter().GetResult(); // block until complete
 
                        // If generic Task<T>, get the Result property
                        var taskType = task.GetType();
                        if (taskType.IsGenericType)
                        {
                            var taskResult = taskType.GetProperty("Result")?.GetValue(task);
                            taskReturnValue = taskResult?.ToString() ?? resultsExpected;
                        }
                        else
                        {
                            taskReturnValue = tokens[0]; // non-generic Task
                        }
                    }
                    catch
                    {
                        // Not a Task — treat as a string
                        taskReturnValue = result?.ToString() ?? resultsExpected;
                    }
 
                    foreach (string token in tokens)
                        if (true == taskReturnValue.ToLower().Contains(token))
                            return;
 
                    // the loop had no pace of its own, so it asked again the instant the last answer landed and
                    // a slow page took hundreds of round trips to report one state change. a page does not become
                    // ready any sooner for being asked more often
                    Thread.Sleep(WaitForValuePollMs);
                }
                while (DateTime.Now < timeout);
        }
 
        #region NetworkHelpers
        public static bool WaitForNetworkIdle(Browser browser, string sessionToken)
        {
            int timeoutMs = browser.BrowserSettings.NetworkIdleTimeoutMs;
            int maxConnections = browser.BrowserSettings.NetworkIdleMaxConnections;
            int pruneMs = browser.BrowserSettings.NetworkIdlePruneMs;
 
            DateTime timeout = DateTime.Now.AddMilliseconds(timeoutMs);
            DateTime lastChange = DateTime.Now;
 
            if (lastSessionToken != sessionToken)
            {
                lastErrorMessage.Clear();
                supressedMessage = false;
                lastSessionToken = sessionToken;
            }
 
            int lastCount = -1;
 
            string currentErrorMessage = $"Waiting up to [{timeoutMs}] ms for network to idle to [{maxConnections}] connections for [{pruneMs}] ms";
 
            if (false == lastErrorMessage.Contains(currentErrorMessage))
            {
                lastErrorMessage.Add(currentErrorMessage);
                GPAL.PublishSimpleEvent(GPALEventType.INFO, currentErrorMessage, browser, GPALObjectType.Browser);
                supressedMessage = false;
            }
            else if (false == supressedMessage)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Supressing repeat messages", currentErrorMessage, GPALObjectType.Other);
                supressedMessage = true;
            }
 
            while (DateTime.Now < timeout)
            {
                // Get current inflight requests
                int inflight = Convert.ToInt32(BrowserHelper.ExecuteJavaScriptObj(
                    "return window.__gpalNetworkMonitor ? window.__gpalNetworkMonitor.inflight : 0;",
                    browser
                ));
 
                if (inflight != lastCount)
                {
                    lastCount = inflight;
                    lastChange = DateTime.Now;
                }
 
                // If inflight is 0 for pruneMs, we consider network idle
                if (inflight == 0 && (DateTime.Now - lastChange).TotalMilliseconds >= pruneMs)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Network IS Idle. Status [true]");
                    return true;
                }
 
                Thread.Sleep(100); // Poll every 100ms
            }
 
            currentErrorMessage = $"Network IS NOT Idle. Status [false]";
 
            if (false == lastErrorMessage.Contains(currentErrorMessage))
            {
                lastErrorMessage.Add(currentErrorMessage);
                GPAL.PublishSimpleEvent(GPALEventType.INFO, currentErrorMessage, browser, GPALObjectType.Browser);
                supressedMessage = false;
            }
            else if (false == supressedMessage)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Supressing repeat messages", currentErrorMessage, GPALObjectType.Other);
                supressedMessage = true;
            }
            return false; // Timeout
        }
 
        // requires selenium.webdriver update, we are using the framework version :(
        //public static string WaitForNetworkIdle(Browser browser)
        //{
        //    var task = WaitForNetworkIdleAsyncInternal(browser);
        //    return task.GetAwaiter().GetResult();
        //}
        //public static async Task<string> WaitForNetworkIdleAsyncInternal(Browser browser)
        //{ 
        //    int timeoutMs = browser.BrowserSettings.NetworkIdleTimeoutMs;
        //    int maxConnections = browser.BrowserSettings.NetworkIdleMaxConnections;
        //    int pruneMs = browser.BrowserSettings.NetworkIdlePruneMs;
        //    IWebDriver driver = browser.BrowserDriver;
 
 
        //    if (!(driver is IDevTools devToolsDriver))
        //    {
        //        GPAL.PublishSimpleEvent(GPALEventType.ERROR, "This driver does not support check network idle.", browser, GPALObjectType.Browser);
        //        return "true";
        //    }
 
        //    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Waiting up to [{timeoutMs}] ms for network to idle to [{maxConnections}] connections for [{pruneMs}] ms", browser, GPALObjectType.Browser);
 
        //    var network = driver.Manage().Network;
        //    var requests = new ConcurrentBag<string>();
        //    bool isIdle = false;
 
        //    var activeRequests = new ConcurrentDictionary<string, byte>();
 
        //    // Handlers
        //    EventHandler<NetworkRequestSentEventArgs> requestSentHandler = (sender, e) =>
        //    {
        //        activeRequests[e.RequestId] = 0;
        //    };
 
        //    EventHandler<NetworkResponseReceivedEventArgs> responseReceivedHandler = (sender, e) =>
        //    {
        //        activeRequests.TryRemove(e.RequestId, out _);
        //    };
 
        //    // Attach
        //    network.NetworkRequestSent += requestSentHandler;
        //    network.NetworkResponseReceived += responseReceivedHandler;
 
        //    try
        //    {
        //        await network.StartMonitoring();
 
        //        var overallTimeout = DateTime.UtcNow.AddMilliseconds(timeoutMs);
        //        DateTime? idleSince = null;
 
        //        while (DateTime.UtcNow < overallTimeout)
        //        {
        //            int openConnections = activeRequests.Count;
 
        //            if (openConnections <= maxConnections)
        //            {
        //                idleSince ??= DateTime.UtcNow; // mark when idle started
        //                if ((DateTime.UtcNow - idleSince.Value).TotalMilliseconds >= pruneMs)
        //                {
        //                    isIdle = true;
        //                    break;
        //                }
        //            }
        //            else
        //            {
        //                idleSince = null; // reset idle timer if activity resumed
        //            }
 
        //            await Task.Delay(250); // small polling interval
        //        }
 
        //        if (!isIdle)
        //            GPAL.PublishSimpleEvent(GPALEventType.ERROR, "Network idle timeout reached.", browser, GPALObjectType.Browser);
        //    }
        //    finally
        //    {
        //        network.NetworkRequestSent -= requestSentHandler;
        //        network.NetworkResponseReceived -= responseReceivedHandler;
        //        await network.StopMonitoring();
        //    }
 
        //    return isIdle.ToString();
        //}
        public static bool WaitForNetworkIdle(
            Browser browser,
            int timeoutMs = 500,
            int maxConnections = 0,
            int pruneMs = 3000,
            string sessionToken = null)
        {
            Exception innerEx = null;
            try
            {
                if (lastSessionToken != sessionToken)
                {
                    lastErrorMessage.Clear();
                    supressedMessage = false;
                    lastSessionToken = sessionToken;
                }
 
                string currentErrorMessage = $"Waiting up to [{timeoutMs}] ms for network to idle to [{maxConnections}] connections for [{pruneMs}] ms";
 
                if (false == lastErrorMessage.Contains(currentErrorMessage))
                {
                    lastErrorMessage.Add(currentErrorMessage);
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, currentErrorMessage);
                    supressedMessage = false;
                }
                else if (false == supressedMessage)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, "Supressing repeat messages", currentErrorMessage, GPALObjectType.Other);
                    supressedMessage = true;
                }
 
                DateTime startTime = DateTime.Now;
                while ((DateTime.Now - startTime).TotalMilliseconds < timeoutMs)
                {
                    string response = browser.OttoMagicClient.WithEndpoint(ApiEndpoint.CheckNetworkIdle).WithMaxConnections(maxConnections).WithPruneMs(pruneMs).WithTimeoutMs(timeoutMs).Execute();
 
                    if (false == string.IsNullOrEmpty(response))
                    {
                        bool retVal = response.Contains("idle");
 
                        currentErrorMessage = $"Network IS{(retVal ? "" : " NOT")} Idle. Status [{retVal}]";
 
                        if (false == lastErrorMessage.Contains(currentErrorMessage))
                        {
                            lastErrorMessage.Add(currentErrorMessage);
                            GPAL.PublishSimpleEvent(GPALEventType.INFO, currentErrorMessage);
                            supressedMessage = false;
                        }
                        else if (false == supressedMessage)
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.INFO, "Supressing repeat messages", currentErrorMessage, GPALObjectType.Other);
                            supressedMessage = true;
                        }
 
                        return retVal;
                    }
 
                    Thread.Sleep(100);
                }
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                innerEx = ex;
            }
            // Log timeout
            GPAL.PublishSimpleEvent(null != innerEx ? GPALEventType.EXCEPTION : GPALEventType.WARNING,
                $"Page did not reach network idle state within [{timeoutMs}] milliseconds",
                null, GPALObjectType.None, innerEx);
            return false;
        }
        public static IPEndPoint[] GetAllTcpListeners()
        {
            IPGlobalProperties ipGlobalProperties = IPGlobalProperties.GetIPGlobalProperties();
            return ipGlobalProperties.GetActiveTcpListeners();
        }
        public static bool IsPortListening(int port)
        {
            IPEndPoint[] tcpListeners = GetAllTcpListeners();
            return tcpListeners.Any(ep => ep.Port == port);
        }        
        #endregion NetworkHelpers
    }
}