Browser.cs

// =============================================================================
// GPAL - Generally Positive Automation Library
// Copyright © 2026 Software Decisions, Inc. All rights reserved.
//
// This file is part of GPAL.
// Licensed under the Business Source License 1.1
//
// Primary development, architecture, and vision by Michael B. Vederman,
// CEO of Software Decisions, Inc., Texas.
//
// Internal development maintained privately.
// Public releases appear on GitHub: https://github.com/SoftwareDecisionsInc/GPAL.
//
// See LICENSE for full terms, including Additional Use Grant.
// =============================================================================
 
#define DONOTUSECDP // which solution to use for printDomElement
 
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using DocumentFormat.OpenXml.Bibliography;
using GenerallyPositive.GPALForm;
using Newtonsoft.Json;
using OpenQA.Selenium;
using OpenQA.Selenium.Interactions;
using static GenerallyPositive.Enums;
using static GenerallyPositive.SelectorSettings;
using HWND = System.IntPtr;
 
// the browser has a HiddenDesktop property now, which hides the static class of the same name in this
// file. the alias says which one is meant at the four places that want the static one
using Desktops = GenerallyPositive.Browser.HiddenDesktop;
 
namespace GenerallyPositive.Browser
{
    public class Browser
        : IBrowser
    {
        public delegate CallIfStatus CallIfDelegate(IBrowser browser, List<IGPALElement> foundElements, List<IGPALElement> matchedElements, Selector selector, bool matchedAll);
        public delegate CallIfStatus CallAfterFillInDelegate(IBrowser browser, IGPALGrid<string> tokens, int tokenIdx);
        public delegate CallIfStatus CallAfterFetchDelegate(IBrowser browser, List<string> results, IGPALGrid<string> tokens, int tokenIdx);

        public delegate CallIfStatus CallOnFailDelegate(IBrowser browser, GPALFailure failure, string detail);

        public IBrowser ToGPALObject()
        {
            return this;
        }
 
        #region <Working Vars>
        BrowserSettings _browserSettings = null;
        internal UnitOfWork persistentUOW { get; set; } = null;
        // When the browser was launched, null until it is. GoTo only ever asked whether the tab tally was zero to
        // find this out, which made a counter answer a question that is not about counting
        internal DateTime? BrowserLaunched { get; set; } = null;
 
        // Tabs GPAL opened itself, which is not the same as the tabs the browser has: ResolveGhostTabHandle finds
        // Edge's phantom tab precisely by noticing there are more real tabs than tabs we opened. Only counted where
        // the answer is free, which is why no engine is asked to go and look
        internal int tabsWeOpened { get; set; } = 0;
        internal string GhostTabUrl { get; set; } = null;
        internal string GhostTabHandle { get; set; } = null;
        public int CurrentTabIdx { get; internal set; } = 0;
 
        // Which window each tab belongs to, tab handle to window handle. Selenium hands back one flat list of every
        // tab in every window and will not say which belongs to which, but GPAL opened them so GPAL knows. Recorded
        // once where a tab is born and never updated after. A tab that appeared on its own joins the window we were
        // looking at when we first noticed it, which keeps it reachable by cycling.
        private readonly Dictionary<string, string> tabWindows = new Dictionary<string, string>();
 
        internal bool _areRobotsAllowed = true;
        public bool AreRobotsAllowed { get => _areRobotsAllowed; internal set => _areRobotsAllowed = value; }
 
        #endregion <Working Vars>
        internal Browser()
        {
            this.BrowserSettings = new BrowserSettings(this)
            {
                DriverLocation = GPAL.DriverLocation,
                DownloadTimeoutInSec = GPAL.GPALSettings.DownloadTimeoutInSec
            };
 
            CurrentUOW = new UnitOfWork() { Browser = this };
            persistentUOW = new UnitOfWork() { Browser = this };
            WorkflowManager = new WorkflowManager(this); // Pass current Browser instance
        }
 
        #region <Browser Settings>
        public IAllowBrowserSettingsOrGoTo WithDriverLocation(string directory)
        {
            BrowserSettings.DriverLocation = FileHelper.EnsureDirectoryEndsWithBackslash(Environment.ExpandEnvironmentVariables(directory));
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithUseDirectDownload(bool trueFalse)
        {
            BrowserSettings.UseDirectDownload = trueFalse;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithDownloadTimeoutInSec(int seconds)
        {
            BrowserSettings.DownloadTimeoutInSec = seconds;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithDownloadLocation(string directory)
        {
            BrowserSettings.DownloadLocation = FileHelper.EnsureDirectoryEndsWithBackslash(Environment.ExpandEnvironmentVariables(directory));
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithBlockPopUps(bool blockPopUp)
        {
            BrowserSettings.BlockPopUps = blockPopUp;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithHiddenDesktop(bool hiddenDesktop)
        {
            BrowserSettings.HiddenDesktop = hiddenDesktop;
 
            // a browser that asks for a hidden desktop without saying which is asking for one of its own, so it
            // gets a name nothing else will be handed. two browsers sharing is what naming one is for
            if (true == hiddenDesktop)
                BrowserSettings.HiddenDesktopName = Desktops.NextName();
 
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithHiddenDesktop(string desktopName)
        {
            BrowserSettings.HiddenDesktop = true;
 
            // calling this at all is asking for a hidden desktop. a name that turned out to be nothing is a
            // desktop of this browser's own, the same as asking without a name. not wanting one is not calling
            BrowserSettings.HiddenDesktopName = true == string.IsNullOrWhiteSpace(desktopName)
                                              ? Desktops.NextName()
                                              : desktopName;
 
            return this;
        }

        private T LaunchWhereAsked<T>(Func<T> launch)
        {
            T retVal;
 
            if (true == BrowserSettings.HiddenDesktop)
                retVal = Desktops.Launch(BrowserSettings.HiddenDesktopName, launch);
            else
                retVal = launch();
 
            return retVal;
        }
        private static AutomationEngine FirefoxStepDown(AutomationEngine engine)
        {
            AutomationEngine retVal = engine;
 
            switch (engine)
            {
                case AutomationEngine.PuppeteerPort:
                case AutomationEngine.PuppeteerPipe:
                    retVal = AutomationEngine.Selenium;
                    break;
                case AutomationEngine.PuppeteerPortHW:
                case AutomationEngine.PuppeteerPipeHW:
                    retVal = AutomationEngine.SeleniumHW;
                    break;
            }
 
            return retVal;
        }
 
        private static AutomationEngine HardwareStepDown(AutomationEngine engine)
        {
            AutomationEngine retVal = engine;
 
            switch (engine)
            {
                case AutomationEngine.OttoMagicHW:
                    retVal = AutomationEngine.OttoMagic;
                    break;
                case AutomationEngine.PuppeteerPortHW:
                    retVal = AutomationEngine.PuppeteerPort;
                    break;
                case AutomationEngine.PuppeteerPipeHW:
                    retVal = AutomationEngine.PuppeteerPipe;
                    break;
                case AutomationEngine.SeleniumHW:
                    retVal = AutomationEngine.Selenium;
                    break;
            }
 
            return retVal;
        }
        [BrowserType()]
        public IAllowBrowserSettingsOrGoTo WithBrowserType(BrowserType browserType)
        {
            BrowserSettings.BrowserType = browserType;
 
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithOverwriteExistingFile(bool overwriteFile)
        {
            BrowserSettings.DeleteFileBeforeDownload = overwriteFile;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithLoadImages(bool loadImages)
        {
            BrowserSettings.LoadImages = loadImages;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithScrollIntoView(bool trueFalse = true)
        {
            BrowserSettings.ScrollIntoView = trueFalse;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithPromptForDownload(bool promptForDownload)
        {
            BrowserSettings.PromptForDownload = promptForDownload;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithOpenPDFExternally(bool openPDFExternally)
        {
            BrowserSettings.OpenPDFExternally = openPDFExternally;
            return this;
        }
 
        //public IAllowBrowserSettingsOrGoTo WithDownloadFileTypes(string csvList)
        //{
        //    string[] fileTypes = csvList.Split(',');
        //    foreach (string fileType in fileTypes)
        //        BrowserSettings.DownloadFileTypeList.Add(fileType);
        //    return this;
        //}

        public IAllowWithHeaderOrFileActions WithGridToSave(IGPALGrid<string> inputGrid)
        {
            if (null != inputGrid && 0 < inputGrid.Count())
            {
                // TODO: CAVEAT: is this best? we can define any type Grid but then saving it to dynamic?
                CurrentUOW.RetGrid = inputGrid;
                CurrentUOW.ColCount = inputGrid.Columns;
            }
            else
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Input IGPALGrid<T> is empty.", this, GPALObjectType.Browser);
 
            return this;
        }
 
        //public IAllowBrowserSettingsOrGoTo WithOpenFileTypes(string csvList)
        //{
        //    string[] fileTypes = csvList.Split(',');
        //    foreach (string fileType in fileTypes)
        //        BrowserSettings.OpenFileTypeList.Add(fileType);
        //    return this;
        //}

        public IAllowBrowserActionOrAnySelector WithExistingBrowser(int port)
        {
            BrowserSettings.UseExistingBrowser = true;
            BrowserSettings.ExistingBrowserPort = port;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithRestApiUrl(string restApiUrl)
        {
            if (true == string.IsNullOrWhiteSpace(restApiUrl))
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"No address given to WithRestApiUrl, so a browser will be launched as usual", this, GPALObjectType.Browser);
                return this;
            }
 
            // the engine is what decides how a browser is driven, and only OttoMagic is driven over REST
            if (false == UseOttoMagic)
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"WithRestApiUrl drives a browser over GPALRestAPI, which is the OttoMagic engine. The engine is currently [{BrowserSettings.AutomationEngine}]", this, GPALObjectType.Browser);
 
            BrowserSettings.RestApiBaseUrl = restApiUrl.EndsWith("/") ? restApiUrl : restApiUrl + "/";
            BrowserSettings.AttachedRestApi = true;
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Driving the browser already running on [{BrowserSettings.RestApiBaseUrl}].", this, GPALObjectType.Browser);
 
            return this;
        }
 
        public IAllowBrowserSettingsOrGoTo WithUseDebugPort(int port = 0xdead) // will actually take default from interface and is not required, but for clarity
        {
            if (port < 1024 || port > 65535)
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Invalid debug port [{port}]. Must be between 1024 and 65535. Using current port [{BrowserSettings.DebugPort}].", this, GPALObjectType.Browser);
                return this;
            }
 
            // if the port is already in use on the computer, keep bumping it up one until we find an open port
            while (true == BrowserHelper.IsPortListening(port))
                port++;
 
            BrowserSettings.DebugPort = port;
 
            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Puppeteer using port [{port}]", this, GPALObjectType.Browser);
 
            // NOTE: CAVEAT: existing port generally should be the same as debug port - do we want to set this?
            //  BUT, it is possible we are automating multiple browsers, so these have to be separate for each browser, but if it's not yet set...
            //      so, if we are open in debugger port, we could then connect to another browser on another existing port and copy data over.
            // automating multiple browsers is problematic, tabs is the way to go, if using puppeteer, we could in theory handle multiple browsers.
            if (null == BrowserSettings.ExistingBrowserPort)
                BrowserSettings.ExistingBrowserPort = port;
 
            AutomationEngine = AutomationEngine.PuppeteerPort;
            BrowserSettings.DebugPipe = false;
 
            return this;
        }
        public IAllowBrowserSettingsOrGoTo WithAutomationEngine(AutomationEngine automationEngine)
        {
            if (BrowserType.FireFox == BrowserSettings.BrowserType && 
                (AutomationEngine.PuppeteerPort == automationEngine || AutomationEngine.PuppeteerPortHW == automationEngine || 
                AutomationEngine.PuppeteerPipe == automationEngine || AutomationEngine.PuppeteerPipeHW == automationEngine))
            {
                AutomationEngine steppedDownTo = FirefoxStepDown(automationEngine);
 
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Firefox cannot be driven with [{automationEngine}], so this browser runs on [{steppedDownTo}]", this, GPALObjectType.Browser);
 
                automationEngine = steppedDownTo;
            }
 
            AutomationEngine = automationEngine;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithUseDebugPipe(bool trueFalse = false) // will actually take default from interface and is not required, but for clarity
        {
            AutomationEngine = AutomationEngine.PuppeteerPipe;
            BrowserSettings.DebugPipe = true;
            BrowserSettings.DebugPort = null;
 
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithWaitOnDocumentReady(int timeoutInMs)
        {
            BrowserSettings.WaitOnDocumentReady = true;
            BrowserSettings.WaitOnNetworkIdle = false;
            BrowserSettings.WaitForDocumentReadyTimeoutMs = timeoutInMs;
 
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithNavigationGrace(int graceInMs)
        {
            BrowserSettings.NavigationGraceMs = graceInMs;
 
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithWaitOnIdleConnection(bool trueFalse = true)
        {
            BrowserSettings.WaitOnNetworkIdle = trueFalse;
            BrowserSettings.WaitOnDocumentReady = trueFalse ? false : BrowserSettings.WaitOnDocumentReady;
 
            return this;
        }

        public IAllowNetworkIdleSettings WithNetworkIdleTimeoutMs(int networkIdleTimeoutMs = 500)
        {
            BrowserSettings.NetworkIdleTimeoutMs = networkIdleTimeoutMs;
            return this;
        }

        public IAllowNetworkIdleSettings WithNetworkIdleMaxConnections(int maxConnections = 0)
        {
            BrowserSettings.NetworkIdleMaxConnections = maxConnections;
            return this;
        }

        public IAllowNetworkIdleSettings WithNetworkIdlePruneMs(int networkIdlePruneMs = 3_000)
        {
            BrowserSettings.NetworkIdlePruneMs = networkIdlePruneMs;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithProfileDataDirectory(string profileDataDirectory)
        {
            // we get what the system thinks is the default profile used by chrome
            // there is some issue with selenium if it runs under the default profile it creates - since 136 tho a profile directory has to be specified
            // this check might still be required. if so we will make defaultdirectory a local variable.
            //if (null == BrowserSettings.DefaultProfileDataDirectory)
            //    BrowserSettings.DefaultProfileDataDirectory = FileHelper.EnsureDirectoryEndsWithBackslash(BrowserHelper.GetBrowserProfileDirectory(BrowserSettings, true));
 
            string tmpProfileDirectory = FileHelper.EnsureDirectoryEndsWithBackslash(Environment.ExpandEnvironmentVariables(profileDataDirectory));
 
            // cannot use the default profile under bot control due to access to secure user cookies
            // OttoMagic will not be affected because it does not specify a debugger port.
            //if (true == BrowserSettings.DefaultProfileDataDirectory.Equals(tmpProfileDirectory) && null == GPAL.OttoMagicExtensionPath)
            //    GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Specified user data dir [{tmpProfileDirectory}] cannot be the default profile location using Selenium.</br>Use OttoMagic instead.", this, GPALObjectType.Browser);
            //else
                BrowserSettings.ProfileDataDirectory = tmpProfileDirectory;
 
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithProfileUserName(string profileUserName)
        {
            BrowserSettings.ProfileUserName = profileUserName;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithProfileName(string profileName)
        {
            BrowserSettings.ProfileName = profileName;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithUseStealth(StealthType steathType)
        {
            BrowserSettings.StealthType = steathType;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithUseReferrer(string referrer = "")
        {
            BrowserSettings.Referrer = referrer;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithUseUserAgent(string userAgent = "")
        {
            BrowserSettings.OverrideUserAgent = userAgent;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithUserAgentFromBrowser(bool userAgentFromBrowser = true)
        {
            BrowserSettings.UserAgentFromBrowser = userAgentFromBrowser;
            return this;
        }

        public IAllowAfterAnySelector WithSelector(Selector selector)
        {
            selector.ElementsFoundAndMatchedCount = 0;  // in case it's being reused
            selector.DeleteMe = false;                  // Remove() latches this on and nothing else clears it, so a reused selector would be swept on its first maintenance pass
            selector.RemovedMethods.Clear();            // a handler that took itself out of play in an earlier workflow is back in play for this one
            if (true == CurrentUOW.ActionCalled)
            {
                // NOTE: CAVEAT: should this a list to have nested iframes?
                var iframeRoot = CurrentUOW.ContextPath;
                var shadowRoot = CurrentUOW.ShadowRoot;
 
                CurrentUOW = new UnitOfWork
                {
                    ContextPath = iframeRoot,
                    ShadowRoot = shadowRoot
                };
 
                CurrentUOW.Browser = this;
                selector.Browser = this;
            }
 
            ResolveInteractionType(selector);
 
            // if we are reusing, we can't use cached results in a new uow
            selector.WebSelectorFoundResults = null;
            selector.WebSelectorMatchedResults = null;
 
            //CurrentUOW.SelectorType = SelectorType.WithSelector;
            // clone so if this selector is reused, each UOW list will be unique and we won't change the selector by changing the browser it is associated with
            // NOTE: cloning disassociates with actual program selector, so we can't set any queryable properties on Selector like 'ElementsFoundAndMatchedCount'
            // BrowserSettings.MySelector = new Selector(this, selector);    // MySelector temp hold for other browser syntax
            BrowserSettings.CurrentSelector = selector;
 
            CurrentUOW.WithSelectorList.Add(BrowserSettings.CurrentSelector);
            return this;
        }

        public IAllowAfterAnySelector WithSelector(string literalData)
        {
            if (true == CurrentUOW.ActionCalled)
            {
                // NOTE: CAVEAT: should this a list to have nested iframes?
                var iframeRoot = CurrentUOW.ContextPath;
                var shadowRoot = CurrentUOW.ShadowRoot;
                //                var shadowRootElement = CurrentUOW.ShadowRootGPALElement;
 
                CurrentUOW = new UnitOfWork
                {
                    ContextPath = iframeRoot,
                    ShadowRoot = shadowRoot
                };
 
                persistentUOW = new UnitOfWork();
 
                CurrentUOW.Browser = this;
                persistentUOW.Browser = this;
            }
 
            Selector tmpSel = GPAL.Selector.WithText(literalData).ToGPALObject();
            tmpSel.selectorSettings.SelectorType = SelectorType.Data;
            tmpSel.selectorSettings.Name = "Literal" + CurrentUOW.WithSelectorList.Select(s => SelectorType.Data == s.SelectorSettings.SelectorType).Count() + 1;
            CurrentUOW.WithSelectorList.Add(tmpSel);
            return this;
        }

        public IAllowAfterAnySelector WithSelector(Func<string> dataFunction)
        {
            Selector tmpSel = GPAL.Selector.ToGPALObject();
            tmpSel.selectorSettings.SelectorType = SelectorType.DataFunc;
            tmpSel.selectorSettings.Name = dataFunction.Method.Name;
            tmpSel.selectorSettings.DataFunction = dataFunction;
            CurrentUOW.WithSelectorList.Add(tmpSel);
            return this;
        }
 
        #region Storage
        public IAllowBrowserActionOrAnySelector ContinueAsRESTClient(out IRESTClient client)
        {
            // the browser's own origin, so the client is ready to be given an endpoint rather than the caller
            // restating what the browser already knows
            string origin = UrlHelper.GetOrigin(BrowserHelper.GetCurrentUrl(BrowserSettings));
            RESTClient retVal = (RESTClient)GPAL.RESTClient.WithAPIBase(origin);
 
            // a browser that has not been anywhere has no origin to talk to and no cookies to carry, so the
            // client would come back empty and fail later at the request instead of here at the call
            if (true == string.IsNullOrEmpty(origin))
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"No page loaded, so there is no session to carry. Navigate first, then continue as a RESTClient", this, GPALObjectType.Browser);
 
            // a cookie get scoped to this origin, declared and run the way any storage action is. it is declared
            // on a url of its own so a workflow that declared its own actions still runs only those
            GPALUrl cookieUrl = new GPALUrl(origin);
 
            cookieUrl.Add(new StorageAction { StorageType = WebsiteStorageType.cookie, Action = WebsiteStorageAction.get });
 
            UrlHelper.TryPerformAction(cookieUrl, BrowserSettings,
                new StorageAction { StorageType = WebsiteStorageType.cookie, Action = WebsiteStorageAction.get },
                out string cookieJson);
 
            string cookieHeader = CookieHeaderFrom(cookieJson, origin, out int cookiesRead);
 
            if (false == string.IsNullOrEmpty(cookieHeader))
                retVal.WithHeader("Cookie", cookieHeader);
 
            // asked of the browser rather than taken from what it was configured with, because a site can bind a
            // session to the agent it saw
            string sentUserAgent = BrowserHelper.LiveUserAgent(this);
            string sentAcceptLanguage = BrowserHelper.LiveAcceptLanguage(this);
 
            retVal.WithHeader("User-Agent", sentUserAgent ?? BrowserHelper.ResolveUserAgent(BrowserSettings));
 
            if (false == string.IsNullOrEmpty(sentAcceptLanguage))
                retVal.WithHeader("Accept-Language", sentAcceptLanguage);
 
            // the credential the browser is actually using, whichever way it got one. WithCredentials is the
            // common case and basic auth never becomes a cookie - the browser resends the header on every
            // request - so without this a client carrying only cookies gets a flat 401
            ICredentials browserCredential = BrowserSettings.Credentials;
 
            if (null != browserCredential)
            {
                Credentials credential = (Credentials)browserCredential;
                string authValue = BrowserHelper.AuthorizationHeader(credential, credential.WebAuthType);
 
                if (false == string.IsNullOrEmpty(authValue))
                    retVal.WithHeader(BrowserHelper.AuthorizationHeaderName(credential.WebAuthType), authValue);
 
            }
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Continuing as a RESTClient from [{BrowserSettings.BrowserType}], carrying its [{(null == sentUserAgent ? "configured" : "live")}] user agent, [{(string.IsNullOrEmpty(sentAcceptLanguage) ? "no accept language" : sentAcceptLanguage)}], [{cookieHeader?.Split(';').Length ?? 0}] of [{cookiesRead}] cookies for [{origin}] and [{(null == browserCredential ? "no credential" : ((Credentials)browserCredential).WebAuthType.ToString())}]", this, GPALObjectType.Browser);
 
            client = retVal;
 
            return this;
        }

        private static string CookieHeaderFrom(string cookieJson, string origin, out int cookiesRead)
        {
            string retVal = null;
 
            cookiesRead = 0;
 
            try
            {
                if (false == string.IsNullOrEmpty(cookieJson))
                {
                    string host = true == Uri.TryCreate(origin, UriKind.Absolute, out Uri originUri) ? originUri.Host : null;
                    Newtonsoft.Json.Linq.JArray cookies = Newtonsoft.Json.Linq.JArray.Parse(cookieJson);
                    List<string> pairs = new List<string>();
 
                    cookiesRead = cookies.Count;
 
                    foreach (Newtonsoft.Json.Linq.JToken cookie in cookies)
                    {
                        // puppeteer calls the name Key, the other two call it name
                        string name = (string)(cookie["name"] ?? cookie["Name"] ?? cookie["Key"]);
                        string value = (string)(cookie["value"] ?? cookie["Value"]);
                        string domain = (string)(cookie["domain"] ?? cookie["Domain"]);
 
                        if (false == string.IsNullOrEmpty(name) && true == UrlHelper.CookieAppliesToHost(domain, host))
                            pairs.Add($"{name}={value}");
                    }
 
                    if (0 < pairs.Count)
                        retVal = string.Join("; ", pairs);
                }
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Could not read the cookies into a header", null, GPALObjectType.Browser, ex);
            }
 
            return retVal;
        }
 

        public IAllowStorageOutData RunGet(WebsiteStorageType storageType)
        {
            BrowserSettings.StorageActionCalled = true;
            BrowserSettings.StorageActionToRun.Action = WebsiteStorageAction.get;
            BrowserSettings.StorageActionToRun.StorageType = storageType;
            UrlHelper.TryPerformAction(BrowserSettings.CurrentGPALUrl, BrowserSettings, BrowserSettings.StorageActionToRun, out BrowserSettings.StorageActionToRun._data);
            return this;
        }

        public IAllowBrowserActionOrAnySelector RunSet(WebsiteStorageType storageType)
        {
            BrowserSettings.StorageActionCalled = true;
            BrowserSettings.StorageActionToRun.Action = WebsiteStorageAction.set;
            BrowserSettings.StorageActionToRun.StorageType = storageType;
            UrlHelper.TryPerformAction(BrowserSettings.CurrentGPALUrl, BrowserSettings, BrowserSettings.StorageActionToRun, out _);
            return this;
        }

        public IAllowBrowserActionOrAnySelector RunDelete(WebsiteStorageType storageType)
        {
            BrowserSettings.StorageActionCalled = true;
            BrowserSettings.StorageActionToRun.Action = WebsiteStorageAction.delete;
            BrowserSettings.StorageActionToRun.StorageType = storageType;
            UrlHelper.TryPerformAction(BrowserSettings.CurrentGPALUrl, BrowserSettings, BrowserSettings.StorageActionToRun, out _);
            return this;
        }

        public IAllowStorageOptions WithStorageDomain(string domain)
        {
            if (true == BrowserSettings.StorageActionCalled)
            {
                BrowserSettings.StorageActionToRun = new StorageAction();
                BrowserSettings.StorageActionCalled = false;
            }
 
            BrowserSettings.StorageActionToRun.Domain = domain;
 
            return this;
        }

        public IAllowStorageOptions WithStoragePath(string path)
        {
            if (true == BrowserSettings.StorageActionCalled)
            {
                BrowserSettings.StorageActionToRun = new StorageAction();
                BrowserSettings.StorageActionCalled = false;
            }
 
            BrowserSettings.StorageActionToRun.Path = path;
 
            return this;
        }

        public IAllowStorageOptions WithStorageKey(string key)
        {
            if (true == BrowserSettings.StorageActionCalled)
            {
                BrowserSettings.StorageActionToRun = new StorageAction();
                BrowserSettings.StorageActionCalled = false;
            }
 
            BrowserSettings.StorageActionToRun.Key = key;
 
            return this;
        }

        public IAllowStorageOptions WithStorageStoreName(string storeName)
        {
            if (true == BrowserSettings.StorageActionCalled)
            {
                BrowserSettings.StorageActionToRun = new StorageAction();
                BrowserSettings.StorageActionCalled = false;
            }
 
            BrowserSettings.StorageActionToRun.StoreName = storeName;
 
            return this;
        }

        public IAllowStorageOptions WithStorageData(string data)
        {
            if (true == BrowserSettings.StorageActionCalled)
            {
                BrowserSettings.StorageActionToRun = new StorageAction();
                BrowserSettings.StorageActionCalled = false;
            }
 
            BrowserSettings.StorageActionToRun.Data = data;
 
            return this;
        }

        public IAllowStorageOptions WithUserDefinedFilter(string userDefined)
        {
            if (true == BrowserSettings.StorageActionCalled)
            {
                BrowserSettings.StorageActionToRun = new StorageAction();
                BrowserSettings.StorageActionCalled = false;
            }
 
            BrowserSettings.StorageActionToRun.UserDefined = userDefined;
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector GetStorageData(out string data)
        {
            data = BrowserSettings.StorageActionToRun.Data;
 
            return this;
        }
        #endregion Storage

        public IAllowAfterAnySelector InElement(Selector selector)
        {
            selector.ElementsFoundAndMatchedCount = 0;
            selector.WebSelectorFoundResults = null;
            selector.WebSelectorMatchedResults = null;
 
            // Fresh UOW after any action (your standard reset)
            if (CurrentUOW.ActionCalled)
            {
                var previousShadowRoot = CurrentUOW.ShadowRoot;
                var previousContextPath = CurrentUOW.ContextPath;
 
                CurrentUOW = new UnitOfWork
                {
                    Browser = this,
                    ShadowRoot = previousShadowRoot,
                    ContextPath = previousContextPath  // preserve chain
                };
 
                persistentUOW = new UnitOfWork { Browser = this };
            }
 
            BrowserSettings.CurrentSelector = selector;
            CurrentUOW.InSelectorList.Add(selector);
 
            // --- Switch to clean context first ---
            if (true == UseOttoMagic)
                MagicHelper?.SwitchToDefaultContent();
            else if (true == UsePuppeteer)
                PuppeteerClient.SwitchToDefaultContent().Execute();
            else
                BrowserDriver.SwitchTo().DefaultContent();
 
            // Walk existing context
            IEnumerable<UnitOfWork.WebElementWithType> stepsToProcess;
 
            //if (true == UseOttoMagic)
            //{
            //    int lastIframeIndex = CurrentUOW.ContextPath
            //        .Select((step, index) => new { step, index })
            //        .LastOrDefault(x => x.step.elementType == ElementType.IFrame)?.index ?? -1;
 
            //    stepsToProcess = CurrentUOW.ContextPath
            //        .Skip(lastIframeIndex + 1)
            //        .Where(step => step.elementType == ElementType.ShadowRoot ||
            //                       step.elementType == ElementType.Element);
            //}
            //else
            {
                stepsToProcess = CurrentUOW.ContextPath;
            }
 
            GPALElement inElement = null;
 
            foreach (UnitOfWork.WebElementWithType contextType in stepsToProcess)
            {
                if (ElementType.IFrame == contextType.elementType)
                {
                    if (UsePuppeteer)
                        PuppeteerClient.SwitchToFrame(contextType.gPalElement.Css).Execute();
                    else
                        BrowserDriver.SwitchTo().Frame((IWebElement)contextType.gPalElement.WebElement);
 
                    // a search context belongs to one document - anything scoped in the frame we just left is
                    // stale, and a shadow root from the parent throws "no such shadow root" if reused here
                    inElement = null;
                }
                else if (ElementType.ShadowRoot == contextType.elementType)
                {
                    if (UseOttoMagic)
                        MagicHelper.SwitchToShadowRoot(contextType.gPalElement.Css);
                    else if (UsePuppeteer)
                        PuppeteerClient.SwitchToShadowRoot(contextType.gPalElement.Css).Execute();
 
                    // scope to THIS step's shadow root rather than the last one added to the unit of work, so a
                    // path with more than one shadow root walks through each in turn instead of collapsing them
                    inElement = contextType.gPalElement;
                }
                else if (ElementType.Element == contextType.elementType)
                {
                    inElement = contextType.gPalElement;   // this becomes the new root
                }
            }
 
            // --- Actually find the InElement target ---
            ElementHelper.FindWebElements(
                this, CurrentUOW, selector, out _, out var tempElems, inElement); // shadow still passed for Otto
 
            if (tempElems?.Count > 0)
            {
                var elementEntry = new UnitOfWork.WebElementWithType(
                    selector, tempElems[0], ElementType.Element);
 
                CurrentUOW.ContextPath ??= new List<UnitOfWork.WebElementWithType>();
                CurrentUOW.ContextPath.Add(elementEntry);
            }
            else
            {
                // Optional: log / throw depending on your strictness
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"InElement selector returned no results [{selector.Name}]", this);
            }
 
            return this;
        }

        public IAllowAfterAnySelector InFrame(Selector selector)
        {
            selector.ElementsFoundAndMatchedCount = 0;
            selector.WebSelectorFoundResults = null;
            selector.WebSelectorMatchedResults = null;
 
            // Fresh UOW after any action (standard fluent reset)
            if (CurrentUOW.ActionCalled)
            {
                var previousShadowRoot = CurrentUOW.ShadowRoot;
                var previousContextPath = CurrentUOW.ContextPath;
 
                CurrentUOW = new UnitOfWork
                {
                    Browser = this,
                    ShadowRoot = previousShadowRoot,
                    ContextPath = previousContextPath
                };
 
                persistentUOW = new UnitOfWork { Browser = this };
            }
 
            BrowserSettings.CurrentSelector = selector;
            CurrentUOW.InSelectorList.Add(selector);
 
            if (true == UseOttoMagic)
                MagicHelper.SwitchToDefaultContent();
            else if (true == UsePuppeteer)
                PuppeteerClient.SwitchToDefaultContent().Execute();
            else
                this.BrowserDriver.SwitchTo().DefaultContent();
 
            IEnumerable<UnitOfWork.WebElementWithType> stepsToProcess;
            GPALElement inElement = null;
 
            //if (true == UseOttoMagic)
            //{
            //    // OttoMagic special rule:
            //    // Skip ALL IFrames, and only keep ShadowRoot steps that come AFTER the last IFrame in the list.
            //    int lastIframeIndex = CurrentUOW.ContextPath
            //        .Select((step, index) => new { step, index })
            //        .LastOrDefault(x => x.step.elementType == ElementType.IFrame)?.index ?? -1;
 
            //    stepsToProcess = CurrentUOW.ContextPath
            //        .Skip(lastIframeIndex + 1)                    // everything after the last iframe
            //        .Where(step => step.elementType == ElementType.ShadowRoot); // NOTE: only shadow roots - redundant, but in case we add other types
            //}
            //else
            {
                // puppeteer or selenium have to be iframe aware: process the full original path
                stepsToProcess = CurrentUOW.ContextPath;
            }
 
            // Now walk the (possibly filtered) steps
            foreach (UnitOfWork.WebElementWithType contextType in stepsToProcess)
            {
                if (ElementType.IFrame == contextType.elementType)
                {
                    if (true == UseOttoMagic)
                        MagicHelper.SwitchToElement(contextType.gPalElement.Css);
                    else if(true == UsePuppeteer)
                        PuppeteerClient.SwitchToFrame(contextType.gPalElement.Css).Execute();
                    else
                        BrowserDriver.SwitchTo().Frame((IWebElement)contextType.gPalElement.WebElement);
 
                    // a search context belongs to one document - anything scoped in the frame we just left is
                    // stale, and a shadow root from the parent throws "no such shadow root" if reused here
                    inElement = null;
                }
                else if (ElementType.ShadowRoot == contextType.elementType)
                {
                    if (true == UseOttoMagic)
                        MagicHelper.SwitchToShadowRoot(contextType.gPalElement.Css);
                    else if (true == UsePuppeteer)
                        PuppeteerClient.SwitchToShadowRoot(contextType.gPalElement.Css).Execute();
 
                    // scope to THIS step's shadow root rather than the last one added to the unit of work, so a
                    // path with more than one shadow root walks through each in turn instead of collapsing them
                    inElement = contextType.gPalElement;
                }
            }
            
            // Find the iframe
            ElementHelper.FindWebElements(
                this, CurrentUOW, selector, out _, out var tempElems, CurrentUOW.ShadowRoot);
 
            if (tempElems?.Count > 0)
            {
                var iframeElement = new UnitOfWork.WebElementWithType(
                    selector, tempElems[0], ElementType.IFrame);
 
                CurrentUOW.ContextPath ??= new List<UnitOfWork.WebElementWithType>();
                CurrentUOW.ContextPath.Add(iframeElement);
            }
 
            return this;
        }

        public IAllowAfterAnySelector InShadowDom(Selector shadowDomSelector)
        {
            shadowDomSelector.ElementsFoundAndMatchedCount = 0;
            shadowDomSelector.WebSelectorFoundResults = null;
            shadowDomSelector.WebSelectorMatchedResults = null;
 
            if (CurrentUOW.ActionCalled)
            {
                var previousShadowRoot = CurrentUOW.ShadowRoot;
                var previousContextPath = CurrentUOW.ContextPath;
 
                CurrentUOW = new UnitOfWork
                {
                    Browser = this,
                    ShadowRoot = previousShadowRoot,
                    ContextPath = previousContextPath
                };
 
                persistentUOW = new UnitOfWork { Browser = this };
            }
 
            BrowserSettings.CurrentSelector = shadowDomSelector;
            CurrentUOW.InSelectorList.Add(shadowDomSelector);
 
            if (true == UseOttoMagic)
                MagicHelper.SwitchToDefaultContent();
            else if (true == UsePuppeteer)
                PuppeteerClient.SwitchToDefaultContent().Execute();
            else
                this.BrowserDriver.SwitchTo().DefaultContent();
 
            GPALElement inElement = null;
 
            // Now walk the (possibly filtered) steps
            foreach (UnitOfWork.WebElementWithType contextType in CurrentUOW.ContextPath)
            {
                if (ElementType.IFrame == contextType.elementType)
                {
                    if (true == UseOttoMagic)
                        MagicHelper.InFrame(contextType.gPalElement.Css);
                    else if (true == UsePuppeteer)
                        PuppeteerClient.SwitchToFrame(contextType.gPalElement.Css).Execute();
                    else
                        BrowserDriver.SwitchTo().Frame((IWebElement)contextType.gPalElement.WebElement);
 
                    // a search context belongs to one document - anything scoped in the frame we just left is
                    // stale, and a shadow root from the parent throws "no such shadow root" if reused here
                    inElement = null;
                }
                else if (ElementType.ShadowRoot == contextType.elementType)
                {
                    if (true == UseOttoMagic)
                        MagicHelper.SwitchToShadowRoot(contextType.gPalElement.Css);
                    else if (true == UsePuppeteer)
                        PuppeteerClient.SwitchToShadowRoot(contextType.gPalElement.Css).Execute();
 
                    // scope to THIS step's shadow root rather than the last one added to the unit of work, so a
                    // path with more than one shadow root walks through each in turn instead of collapsing them
                    inElement = contextType.gPalElement;
                }
            }
 
            ElementHelper.FindWebElements(
                this, CurrentUOW, shadowDomSelector, out _, out var tempElems, inElement);
 
            if (tempElems?.Count > 0)
            {
                var shadowElement = new UnitOfWork.WebElementWithType(
                    shadowDomSelector, tempElems[0], ElementType.ShadowRoot);
 
                CurrentUOW.ShadowRoot = tempElems[0];
                CurrentUOW.ShadowRoot.IsShadowRoot = true;
 
                CurrentUOW.ContextPath ??= new List<UnitOfWork.WebElementWithType>();
                CurrentUOW.ContextPath.Add(shadowElement);
 
                // Existing shadow root plumbing (unchanged)
                if (true == UseOttoMagic)
                {
                    // CurrentUOW.ShadowRoot.ShadowRoot = MagicHelper.GetShadowRoot(shadowDomSelector.SelectorPaths[0].SelectorPath); // this doesn't make sense, nothing useful can be returned
                    // tempElems[0].ShadowRoot = CurrentUOW.ShadowRoot;
                }
                else if (true == UsePuppeteer)
                {
                    //CurrentUOW.ShadowRoot.ShadowRoot = PuppeteerClient.GetShadowRoot(tempElems[0].Css).Execute<GPALElement>();
                    //tempElems[0].ShadowRoot = CurrentUOW.ShadowRoot;
                }
                else if (true == UseSelenium)
                {
                    ISearchContext shadowRoot = null;
 
                    // GetShadowRoot is webdriver's own call for this and throws only when the host genuinely has
                    // no open root. asking javascript instead cannot tell a closed root from a script that did not
                    // run, because ExecuteJavaScriptObj swallows its own failures and hands back null either way -
                    // and a null here marks an open root closed, which sends every search inside it back the host
                    // instead of the children.
                    try
                    {
                        shadowRoot = ((IWebElement)tempElems[0].WebElement).GetShadowRoot();
                    }
                    catch
                    {
                        // closed, handled below
                    }
 
                    if (null != shadowRoot)
                        CurrentUOW.ShadowRoot.WebElement = shadowRoot;  // dynamic - the search dispatches to it at runtime
                    else
                    {
                        CurrentUOW.ShadowRoot.IsClosedShadowRoot = true;
                        // a closed shadow root is never handed to webdriver - arguments[0].shadowRoot is null and
                        // GetShadowRoot throws. leave the host element in place so the root still has a real
                        // position to click relative to, and so searches inside it stay scoped to the host rather
                        // than silently falling back to the whole document and matching something unrelated.
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Closed shadow root for selector [{shadowDomSelector.Name}]. Will return host element for searches inside.", this, GPALObjectType.Browser);
                    }
                }
            }
 
            return this;
        }

        public IAllowAfterAnySelector InMainDom()
        {
            CurrentUOW.ShadowRoot = null;
            CurrentUOW.ContextPath?.Clear();
 
            if (true == UseOttoMagic)
                MagicHelper.SwitchToDefaultContent();
            else if (true == UsePuppeteer)
                PuppeteerClient.SwitchToDefaultContent().Execute();
            else if (true == UseSelenium)
                this.BrowserDriver.SwitchTo().DefaultContent();
 
            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, "Returned to main document", CurrentUOW);
            return this;
        }

        public IAllowWithPagesAndGridActions WithNextPageButton(Selector nextPageButtonselector)
        {
            CurrentUOW.NextPageButton = nextPageButtonselector;
            BrowserSettings.CurrentSelector = nextPageButtonselector;  // for use by .WithHardware
            return this;
        }

        public IAllowGetGridAndFillInFrom WithPages(int pageCount)
        {
            CurrentUOW.PageCount = pageCount;
            return this;
        }

        public IAllowWithPagesAndFetch WithTokensFrom(IGPALGrid<string> inputGrid)
        {
            CurrentUOW.FetchTokens = inputGrid;
            return this;
        }

        public IAllowWithPagesAndFetch WithTokensFrom(GPALFile inputFile)
        {
            List<IGPALGrid<string>> tokenList = FileHelper.TokenizeFile(CurrentUOW, inputFile);
 
            CurrentUOW.FetchTokens = 0 == tokenList?.Count ? null : tokenList[0];
            return this;
        }

        public IAllowWithPagesAndFetch WithTokensFrom(GPALDatabase inputDatabase)
        {
            DatabaseHelper.TokenizeDatabase(CurrentUOW, inputDatabase);
 
            CurrentUOW.FetchTokens = CurrentUOW.InputDatabase?.Tokens;
            return this;
        }

        public IAllowWithPagesAndFetch WithTokensFrom(string tokens)
        {
            IGPALGrid<string> grid = GPAL.GridForType<string>();
 
            grid.AddRow(new List<string>(tokens?.Split(',') ?? new string[0]));
            CurrentUOW.FetchTokens = grid;
            return this;
        }

        public IAllowAfterAnySelector CallAfterFillIn(Browser.CallAfterFillInDelegate callAfterFillIn)
        {
            CurrentUOW.CallAfterFillIn = callAfterFillIn;
 
            return this;
        }

        public IAllowAfterAnySelector CallIfFound(Browser.CallIfDelegate callIfFoundDelegate)
        {
            CurrentUOW.CallIfFound.Add(callIfFoundDelegate);
 
            return this;
        }

        public IAllowAfterAnySelector CallIfNotFound(Browser.CallIfDelegate callIfNotFound)
        {
            CurrentUOW.CallIfNotFound.Add(callIfNotFound);
 
            return this;
        }

        public IAllowAfterWaitForAndSizeControl WithWaitForWindowTimeout(int waitTimeInSeconds)
        {
            BrowserSettings.WaitForWindowTimeoutInSeconds = waitTimeInSeconds;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo PersistentCallIfFound(CallIfDelegate persistentCallIfFound)
        {
            persistentUOW.CallIfFound.Add(persistentCallIfFound);
 
            return this;
        }

        public IAllowBrowserSettingsOrGoTo PersistentCallIfNotFound(CallIfDelegate persistentCallIfNotFound)
        {
            persistentUOW.CallIfNotFound.Add(persistentCallIfNotFound);
 
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithCredentials(ICredentials credentials)
        {
            BrowserSettings.Credentials = credentials;
 
            return this;
        }
        public IAllowBrowserSettingsOrGoTo CallOnFail(CallOnFailDelegate callOnFail)
        {
            BrowserSettings.CallOnFail.Add(callOnFail);
 
            return this;
        }
        internal CallIfStatus RaiseOnFail(GPALFailure failure, string detail)
        {
            CallIfStatus handled = CallIfStatus.NotHandled;
            List<CallOnFailDelegate> handlers = 0 < BrowserSettings.CallOnFail.Count ? BrowserSettings.CallOnFail : GPAL.GPALSettings.CallOnFailHandlers;
 
            foreach (CallOnFailDelegate handler in handlers)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking CallOnFail [{handler.Method.Name}] for [{failure}]", this, GPALObjectType.Browser);
 
                handled = handler(this, failure, detail);
 
                if (CallIfStatus.Terminate == handled)
                {
                    string str = $"CallOnFail handler [{handler.Method.Name}] requested program termination.";
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, str, this, GPALObjectType.Browser);
                    throw new GPALException($"{GPAL.MyMethodName()}: " + str);
                }
 
                if (CallIfStatus.Handled == handled)
                    break;
            }
 
            return handled;
        }
        public IAllowSelectorInFrameBrowserSettingsOrGoto WithPersistentSelector(Selector selector)
        {
            // when would we ever add another persistent uow?
            if (true == CurrentUOW.ActionCalled)
            {
                // Preserve context for new UnitOfWork
                var iframeRoot = CurrentUOW.ContextPath;
                var shadowRoot = CurrentUOW.ShadowRoot;
 
                CurrentUOW = new UnitOfWork
                {
                    ContextPath = iframeRoot,
                    ShadowRoot = shadowRoot,
                    Browser = this
                };
 
                persistentUOW = new UnitOfWork() { Browser = this };
            }
 
            selector.Browser = this;
            selector.DeleteMe = false;         // Remove() latches this on and nothing else clears it, so a reused persistent selector would be swept on its first maintenance pass
            selector.RemovedMethods.Clear();   // a handler that took itself out of play in an earlier workflow is back in play for this one
 
            ResolveInteractionType(selector);
 
            // if we are reusing, we can't use cached results in a new uow
            selector.WebSelectorFoundResults = null;
            selector.WebSelectorMatchedResults = null;
            
            persistentUOW.WithSelectorList.Add(selector);   // CAVEAT: there is no way to know when this is defined that we are headless, so this selector could be hardware emulaiton which won't work
            return this;
        }

        IAllowBrowserSettingsOrGoTo IAllowSelectorInFrameBrowserSettingsOrGoto.InPersistentIframe(Selector iFrame)
        {
            persistentUOW.WithSelectorList.Last().selectorSettings.InFrame = iFrame;
            return this;
        }

        public IAllowWithHeaderOrFileActions WithHeader(string header)
        {
            if (true == CurrentUOW.GetGridCalled)
            {
                CurrentUOW.GetGridCalled = false;
                CurrentUOW.HeaderList.Clear();
            }
            CurrentUOW.HeaderList.Add(header);
            return this;
        }

        public IAllowAfterAnySelectorExceptWithAll WithAllThatMatch(int rowCount = int.MaxValue)
        {
            // if we haven't started yet, this is called on the browser to set the per browser value
            // if called after an action, it's the same thing, it's for the whole browser
            if (0 == CurrentUOW.WithSelectorList.Count || true == CurrentUOW.ActionCalled)
                BrowserSettings.WithAllThatMatch = rowCount;
            else
                CurrentUOW.WithAllThatMatch = rowCount;
            return this;
        }

        public IAllowAllBrowserAndAllSelector WithWindowSize(Rectangle windowSize)
        {
            int screenWidth = Puppeteer.GetScreenWidthInt();
            int screenHeight = Puppeteer.GetScreenHeightInt();
 
            if (100 > windowSize.Width || 100 > windowSize.Height)
            {
                int widthToUse = Math.Min(windowSize.Width, 100);
                int heightToUse = Math.Min(windowSize.Height, 100);
 
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Requested browser sizes [{windowSize.Width}x{windowSize.Height}] cannot be less than 100. Using [{widthToUse}x{heightToUse}].", this, GPALObjectType.Browser);
 
                windowSize.Width = widthToUse;
                windowSize.Height = heightToUse;
            }
            else if (screenWidth < windowSize.Width || screenHeight < windowSize.Height)
            {
                int widthToUse = Math.Min(windowSize.Width, screenWidth);
                int heightToUse = Math.Min(windowSize.Height, screenHeight);
 
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Requested browser size [{windowSize.Width}x{windowSize.Height}] exceeds screen size [{screenWidth}x{screenHeight}]. Using [{widthToUse}x{heightToUse}].", this, GPALObjectType.Browser);
 
                windowSize.Width = widthToUse;
                windowSize.Height = heightToUse;
 
            }
 
            BrowserSettings.WindowSize = windowSize;
 
            return this;
        }

        public IAllowAfterAnySelector WithHardware
        {
            get
            {
                // set this in the selector by calling it fluently
                _ = BrowserSettings.CurrentSelector.WithHardware;
                return this;
            }
        }

        public IAllowAfterAnySelector WithJavaScript
        {
            get
            {
                // set this in the selector by calling it fluently
                _ = BrowserSettings.CurrentSelector.WithJavaScript;
                return this;
            }
        }

        public IAllowWithPagesAndGridActions WithInfiniteScroll // TODO: add aprameter ScrolThenScrape, ScrapeThenScroll to indicate how the pave behaves (1 page in mem or all pages)
        {
            get
            {
                CurrentUOW.InfiniteScroll = true;
                return this;
            }
        }
        #region ACTION GETTERS
        public IAllowBrowserActionOrAnySelector Back
        {
            get
            {
                InAction(false, false);
                // selenium doesn't know what happened
                // BrowserHelper.ExecuteJavaScript(this, "return history.back();");
                if (true == BrowserSettings.UseOttoMagic)
                    MagicHelper.Back();
                else if (true == BrowserSettings.UsePuppeteer)
                    PuppeteerClient.Back().Execute();
                else
                    BrowserDriver.Navigate().Back();
 
                return this;
            }
        }

        public IAllowBrowserActionOrAnySelector Forward
        {
            get
            {
                InAction(false, false);
                // selenium doesn't know what happened
                // BrowserHelper.ExecuteJavaScript(this, "return history.forward();");
                if (true == BrowserSettings.UseOttoMagic)
                    MagicHelper.Forward();
                else if (true == BrowserSettings.UsePuppeteer)
                    PuppeteerClient.Forward().Execute();
                else
                    BrowserDriver.Navigate().Forward();
 
                return this;
            }
        }

        public IAllowBrowserActionOrAnySelector Refresh
        {
            get
            {
                InAction(false, false);
 
                if (true == BrowserSettings.UseOttoMagic)
                {
                    MagicHelper.Refresh();
                    Thread.Sleep(500);  // for some reason we check document ready before it even refreshes
                }
                else if (true == UsePuppeteer)
                    PuppeteerClient.Refresh().Execute();
                else
                    BrowserDriver.Navigate().Refresh();
 
                BrowserHelper.CheckDocumentReady(this);
 
                return this;
            }
        }

        public IAllowBrowserActionOrAnySelector PageDown
        {
            get
            {
                InAction(false, false);
 
                if (true == BrowserSettings.UseOttoMagic)
                    MagicHelper.PageDown();
                else if (true == UsePuppeteer)
                    PuppeteerClient.PageDown().Execute();
                else
                    ScrollPage("down");
 
                return this;
            }
        }

        public IAllowBrowserActionOrAnySelector PageEnd
        {
            get
            {
                InAction(false, false);
 
                if (true == BrowserSettings.UseOttoMagic)
                    MagicHelper.PageEnd();
                else if (true == UsePuppeteer)
                    PuppeteerClient.PageEnd().Execute();
                else
                    ScrollToPosition("end");
 
                return this;
            }
        }

        public IAllowBrowserActionOrAnySelector PageTop
        {
            get
            {
                InAction(false, false);
 
                if (true == BrowserSettings.UseOttoMagic)
                    MagicHelper.PageTop(); // Assuming you have a PageUp() method in MagicHelper
                else if (true == UsePuppeteer)
                    PuppeteerClient.PageTop().Execute();
                else
                    ScrollToPosition("top");
 
                return this;
            }
        }

        public IAllowBrowserActionOrAnySelector PageUp
        {
            get
            {
                InAction(false, false);
 
                if (true == BrowserSettings.UseOttoMagic)
                    MagicHelper.PageUp();
                else if (true == UsePuppeteer)
                    PuppeteerClient.PageUp().Execute();
                else
                    ScrollPage("up");
 
                return this;
            }
        }

        public IAllowAllBrowserAndAllSelector Restore
        {
            get
            {
                BrowserSettings.FullScreen = false;
                BrowserSettings.Maximize = false;
                BrowserSettings.Minimize = false;
 
                //InAction(false, false);
 
                if (null != BrowserSettings.Process || 0 != BrowserSettings.ServiceDriverPid)
                    if (true == BrowserSettings.UseOttoMagic)
                    {
                        if (true == BrowserSettings.FullScreen)
                            MagicHelper.Normal();
                        else
                            MagicHelper.Restore();
                    }
                    else if (true == BrowserSettings.UsePuppeteer)
                    {
                        if (true == BrowserSettings.FullScreen)
                            PuppeteerClient.Normal().Execute();
                        else
                            PuppeteerClient.Restore().Execute();
                    }
                    else if (null != BrowserSettings.BrowserDriver)
                        BrowserSettings.BrowserDriver.Manage().Window.Size = BrowserSettings.SavedWindowSize;
 
                return this;
            }
        }

        public IAllowAllBrowserAndAllSelector FullScreen
        {
            get
            {
                BrowserSettings.Minimize = false;
                BrowserSettings.Maximize = false;
                BrowserSettings.FullScreen = true;
 
                //InAction(false, false);
 
                if (null != BrowserSettings.Process || 0 != BrowserSettings.ServiceDriverPid)
                    if (true == BrowserSettings.UseOttoMagic)
                    {
                        BrowserSettings.SavedWindowSize = new Size(MagicHelper.WindowOuterWidth(), MagicHelper.WindowOuterHeight());
                        MagicHelper.FullScreen();
                    }
                    else if (true == BrowserSettings.UsePuppeteer)
                    {
                        BrowserSettings.SavedWindowSize = new Size(PuppeteerClient.WindowOuterWidth().Execute<int>(), PuppeteerClient.WindowOuterHeight().Execute<int>());
                        MagicHelper.FullScreen();
                    }
                    else if (null != BrowserSettings.BrowserDriver)
                    {
                        BrowserSettings.SavedWindowSize = BrowserSettings.BrowserDriver.Manage().Window.Size;
                        BrowserSettings.BrowserDriver?.Manage().Window.FullScreen();
                    }
 
                return this;
            }
        }

        public IAllowAllBrowserAndAllSelector Maximize
        {
            get
            {
                BrowserSettings.Maximize = true;
                BrowserSettings.Minimize = false;
                BrowserSettings.FullScreen = false;
 
                //InAction(false, false);
 
                if (null != BrowserSettings.Process || 0 != BrowserSettings.ServiceDriverPid)
                    if (true == BrowserSettings.UseOttoMagic)
                    {
                        if (0 == BrowserSettings.WindowSize.Width && 0 == BrowserSettings.WindowSize.Height)
                            BrowserSettings.SavedWindowSize = new Size(MagicHelper.WindowOuterWidth(), MagicHelper.WindowOuterHeight());
                        MagicHelper.Maximize();
                    }
                    else if (true == BrowserSettings.UsePuppeteer)
                    {
                        if (0 == BrowserSettings.WindowSize.Width && 0 == BrowserSettings.WindowSize.Height)
                            BrowserSettings.SavedWindowSize = new Size(PuppeteerClient.WindowOuterWidth().Execute<int>(), PuppeteerClient.WindowOuterHeight().Execute<int>());
                        PuppeteerClient.Maximize().Execute();
                    }
                    else if (null != BrowserSettings.BrowserDriver)
                    {
                        // get current window size if one was not defined on startup
                        BrowserSettings.SavedWindowSize = BrowserSettings.BrowserDriver.Manage().Window.Size;
                        BrowserSettings.BrowserDriver?.Manage().Window.Maximize();
                    }
 
                return this;
            }
        }

        public IAllowAllBrowserAndAllSelector Minimize
        {
            get
            {
                BrowserSettings.Minimize = true;
                BrowserSettings.Maximize = false;
                BrowserSettings.FullScreen = false;
 
                //InAction(false);
 
                if (null != BrowserSettings.Process || 0 != BrowserSettings.ServiceDriverPid)
                    if (true == BrowserSettings.UseOttoMagic)
                    {
                        BrowserSettings.SavedWindowSize = new Size(MagicHelper.WindowOuterWidth(), MagicHelper.WindowOuterHeight());
                        MagicHelper.Minimize();
                    }
                    else if (true == BrowserSettings.UsePuppeteer)
                    {
                        BrowserSettings.SavedWindowSize = new Size(PuppeteerClient.WindowOuterWidth().Execute<int>(), PuppeteerClient.WindowOuterHeight().Execute<int>());
                        PuppeteerClient.Minimize().Execute();
                    }
                    else if (null != BrowserSettings.BrowserDriver)
                    {
                        // get current window size if one was not defined on startup
                        BrowserSettings.SavedWindowSize = BrowserSettings.BrowserDriver.Manage().Window.Size;
                        BrowserSettings.BrowserDriver?.Manage().Window.Minimize();
                    }
 
                return this;
            }
        }
        #endregion ACTION GETTERS
        #endregion <Browser Settings>
        #region <Browser Actions>
        public IAllowBrowserSettingsOrGoTo WithObeyRobotsTxt(bool trueFalse = false)
        {
            BrowserSettings.ObeyRobotsTxt = trueFalse;
            return this;
        }

        public IAllowBrowserSettingsOrGoTo WithRespectRobotMetaTags(bool trueFalse = false)
        {
            BrowserSettings.RespectRobotMetaTags = trueFalse;
            return this;
        }
 

        public IAllowBrowserActionOrAnySelector Get(GPALUrl URL)
        {
            try
            {
                BrowserSettings.UseHeadless = true;
 
                // configure headless options
                BrowserSettings.OpenPDFExternally = false;  // these two settings will download any pdf file to the specified "download.default_directory" (see WithDownloadDirectory and SetDownloadFilename)
                BrowserSettings.PromptForDownload = false;
 
                BrowserSettings.OpenInTab = false;
                GoTo(URL);
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to GET from [{URL}]", this, GPALObjectType.Browser, ex);
            }
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector GoTo(GPALUrl URL)
        {
            bool firstRunSave = firstRun;
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Visiting [{URL?.Url}]", this, GPALObjectType.Browser);
 
            if (BrowserType.FireFox == BrowserSettings.BrowserType &&
                (AutomationEngine.PuppeteerPort == AutomationEngine || AutomationEngine.PuppeteerPortHW == AutomationEngine ||
                AutomationEngine.PuppeteerPipe == AutomationEngine || AutomationEngine.PuppeteerPipeHW == AutomationEngine))
            {
                AutomationEngine steppedDownTo = FirefoxStepDown(BrowserSettings.AutomationEngine);
 
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Firefox cannot be driven with [{BrowserSettings.AutomationEngine}], so this browser runs on [{steppedDownTo}]", this, GPALObjectType.Browser);
 
                AutomationEngine = steppedDownTo;
            }
 
            URL.ForUrl(MagicHelper.GetFullUrl(URL?.Url, this, out _areRobotsAllowed));
 
            if (false == AreRobotsAllowed && true == BrowserSettings.ObeyRobotsTxt)
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Visiting [{URL?.Url}] is disallowed by robots.txt and your request to honor it. Not going to URL. Workflow will fail.", this, GPALObjectType.Browser);
                URL.ForUrl("https://google.com");
            }
 
            // where we were, so a navigation that turns out to be a download rather than a page can put both the
            // browser and what we say about it back
            string priorUrl = BrowserSettings.CurrentURL;
            GPALUrl priorGPALUrl = BrowserSettings.CurrentGPALUrl;
            TabTuple priorTab = true == UseOttoMagic ? new TabTuple(GetActiveTabId(), priorUrl) : null;
            string priorHandle = null;   // the tab we are leaving, by id, filled in only when we open a new one
            int tabsBefore = -1;
 
            BrowserSettings.CurrentURL = URL?.Url;
            BrowserSettings.CurrentGPALUrl = URL;
 
            // keep this in, new page, new uow
            CurrentUOW = new UnitOfWork() { Browser = this }; // so an immediate .inshadowdom or .inframe starts new and doesn't carry forward
 
            // DO NOT PASS IN URL for first run startup, it doesn't like it in headless mode puppeteer
            if (true == InAction(false, false, null))   // actions have ended but we do have to signal how to denote a new UOW, too
            {
                try
                {
                    if (true == BrowserSettings.WaitOnNetworkIdle && true == UseSelenium)
                        ApplyNetworkMonitor();
 
                    CheckAndApplyStealth();
                    SetReferrer();
                    SetUserAgent();
 
                    // said once, before the first navigation, because every mechanism that carries a credential
                    // holds for the session: a header keeps being sent, and a challenge the browser has already
                    // answered stays answered for that origin
                    if (null != BrowserSettings.Credentials && null == BrowserSettings.CredentialsPresented)
                        BrowserHelper.PresentCredentials(this, URL);
 
                    BrowserHelper.CheckDocumentReady(this);
 
                    // do we have an open browser? no
                    if (null == BrowserLaunched || false == BrowserSettings.OpenInTab)
                    {
 
                        if (true == UseOttoMagic)
                        {
                            //if (false == firstRunSave)  // url passed on command line
                            {
                                //MagicHelper.InMainDom();
                                TabTuple tabTuple = MagicHelper.GoTo(URL);
                                UpdateActiveTab(tabTuple);
                            }
                            //else
                            //    UpdateActiveTab(MagicHelper.NextTab());     // get the active tab tuple, no other tabs, will return current
                        }
                        else if (true == UsePuppeteer)
                        {
                            PuppeteerClient.SwitchToDefaultContent().Execute();
                            //if (false == firstRunSave)  // url passed on command line
                            PuppeteerClient.GoTo(URL).Execute();
                        }
                        else
                        {
                            BrowserDriver.SwitchTo().DefaultContent();
                            BrowserHelper.SeleniumGoToUrl(URL, BrowserSettings);
                        }
                        if (null == BrowserLaunched)
                        {
                            BrowserLaunched = DateTime.Now;
                            tabsWeOpened++;
                        }
                    }
                    else
                    {
                        // what the browser has before we add ours, so we can tell afterwards whether ours survived
                        tabsBefore = RealTabCount();
                        priorHandle = true == UseSelenium ? BrowserDriver.CurrentWindowHandle : null;   // to get back by id, not by position
                        tabsWeOpened++;
                        if (true == UseOttoMagic)
                        {
                            TabTuple tabTuple = MagicHelper.NewTab(URL);
                            if (true == string.IsNullOrEmpty(tabTuple?.Url))
                                tabTuple.Url = URL;
                            GoToTab(tabTuple);
                        }
                        else if (true == UsePuppeteer)
                        {
                            PuppeteerClient.NewTab().Execute();
                            _ = NextTab;
                            PuppeteerClient.GoTo(URL).Execute();
                        }
                        else
                        {
                            BrowserDriver.SwitchTo().NewWindow(WindowType.Tab);
                            RecordTabWindow(BrowserDriver.CurrentWindowHandle, priorHandle);   // a new tab of the window we were in
                            //_ = NextTab;
                            BrowserHelper.SeleniumGoToUrl(URL, BrowserSettings);
                        }
                    }
 
                    // the browser saying the page never loaded. Selenium leaves ServerResponseCode at -1, puppeteer
                    // answers Page.navigate with an errorText, and OttoMagic reports it through get-ready-status,
                    // which is where its own raise happens. GPAL fails forward from here either way: the workflow
                    // carries on against a page that is not there, its selectors find nothing and CallIfNotFound
                    // fires, unless a CallOnFail handler decides otherwise
                    bool navigationFailed = (true == UseSelenium && -1 == BrowserSettings.ServerResponseCode)
                        || (true == UsePuppeteer && false == string.IsNullOrEmpty(PuppeteerCommunicator.LastNavigationError));
 
                    if (true == UseSelenium && -1 == BrowserSettings.ServerResponseCode)
                        RaiseOnFail(GPALFailure.Navigation, $"[{URL?.Url}] did not load.");
                    else if (true == UsePuppeteer && false == string.IsNullOrEmpty(PuppeteerCommunicator.LastNavigationError))
                        RaiseOnFail(GPALFailure.Navigation, $"[{URL?.Url}] did not load. [{PuppeteerCommunicator.LastNavigationError}]");
 
                    // a url that downloads gives back no page at all, and a tab opened for one has no document to
                    // fall back on, so the browser disposes it and tells nobody. Only the difference between the
                    // two readings is trusted, which leaves any tab we did not open sitting in both and cancelling
                    if (-1 != tabsBefore && RealTabCount() == tabsBefore)
                        BackOutOfSelfClosedTab(URL, priorUrl, priorGPALUrl, priorTab, priorHandle);
                    else
                    {
                        BrowserHelper.CheckDocumentReady(this);
 
                        // use shortcuiting, if it's not set, why check?
                        if (true == BrowserSettings.RespectRobotMetaTags && false == UrlHelper.CheckRobotMetaTags(URL, this))
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Visiting [{BrowserSettings.CurrentURL}] is disallowed by robot meta tags and your request to honor them..");
                            URL.ForUrl("https://google.com");
 
                            if (true == UseOttoMagic)
                            {
                                //MagicHelper.InMainDom();
                                TabTuple tabTuple = MagicHelper.GoTo(URL);
                                UpdateActiveTab(tabTuple);
                            }
                            else if (true == UsePuppeteer)
                            {
                                PuppeteerClient.SwitchToDefaultContent().Execute();
                                PuppeteerClient.GoTo(URL).Execute();
                            }
                            else
                            {
                                BrowserDriver.SwitchTo().DefaultContent();
                                BrowserHelper.SeleniumGoToUrl(URL, BrowserSettings);
                            }
                        }
                        else
                        {
                            BrowserSettings.CurrentURL = URL.Url;
 
                            // RaiseOnFail says nothing of its own when no CallOnFail handler is registered, so
                            // without this a navigation that ended on the browser's error page read as a load
                            if (true == navigationFailed)
                                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{BrowserSettings.CurrentURL}] did not load. Continuing against whatever the browser is showing.");
                            else
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{BrowserSettings.CurrentURL}] loaded.");
                        }
 
                        BrowserSettings.GPALUrlList.Add(URL);
 
                        // there is a document now, which is what the dialog override needs before it can be put
                        // on the page in front of us rather than only on the ones still to load
                        _navigated = true;
                    }
                }
                catch (GPALException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to navigate to [{URL?.Url}]", this, GPALObjectType.Browser, ex);
                    // internet explorer throws a timeout even tho it does navigate
                }
 
                //int retries = 3;
 
                //if (false == URL.Contains(GetSetCurrentUrl()) && 0 < retries--)
                //    Thread.Sleep(500);
 
                BrowserSettings.OpenInTab = false;  // .WithNewTab applies only once, so it must appear before each .GoTo
            }
            else // we had a driver initialization falure
                throw new GPALException("Driver initialization error. Please see mesages.");
 
            return this;
        }

        private void ApplyGoogleReferrer()
        {
            if (true == UseOttoMagic)
                MagicHelper.StealthOverrideReferrer();
            else if (true == UsePuppeteer)
                PuppeteerCommunicator.StealthOverrideReferrer().GetAwaiter().GetResult();
            else if (BrowserType.FireFox != BrowserSettings.BrowserType)
                ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand("Network.setExtraHTTPHeaders", new Dictionary<string, object>
                {
                    {
                        "headers", new Dictionary<string, object>
                        {
                            { "Referer", "https://www.google.com/" }    // NOTE: the referrer misspelling is baked into the http/1.0 protocol spec
                        }
                    }
                });
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, "Stealth Google Referrer applied.", this, GPALObjectType.Browser);
        }
        private void ApplyNetworkMonitor()
        {
            string networkMonitorScript = @"
                    (function() {
                        if (!window.__gpalNetworkMonitor) {
                            window.__gpalNetworkMonitor = { inflight: 0 };
 
                            const origOpen = XMLHttpRequest.prototype.open;
                            XMLHttpRequest.prototype.open = function() {
                                this.addEventListener('loadstart', () => { window.__gpalNetworkMonitor.inflight++; });
                                this.addEventListener('loadend', () => { window.__gpalNetworkMonitor.inflight--; });
                                origOpen.apply(this, arguments);
                            };
 
                            const origFetch = window.fetch;
                            window.fetch = function(...args) {
                                window.__gpalNetworkMonitor.inflight++;
                                return origFetch(...args).finally(() => { window.__gpalNetworkMonitor.inflight--; });
                            };
                        }
                    })();
                ";
 
            if (UsePuppeteer)
            {
                PuppeteerCommunicator.EnableFetchScriptInjection(networkMonitorScript, null)
                    .GetAwaiter().GetResult();
            }
            else if (BrowserType.FireFox == BrowserSettings.BrowserType)
            {
                // Firefox has no CDP preload script support — inject post-load via ExecuteScript instead.
                // The guard (function(){if(!window.__gpalNetworkMonitor)...}) makes re-injection safe on each navigation.
                ((OpenQA.Selenium.IJavaScriptExecutor)BrowserDriver).ExecuteScript(networkMonitorScript);
            }
            else
            {
                ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand(
                    "Page.addScriptToEvaluateOnNewDocument",
                    new Dictionary<string, object>
                    {
                        { "source", networkMonitorScript }
                    }
                );
            }
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, "Network idle monitor script injected.", this, GPALObjectType.Browser);
        }
        private void ApplyCDPProtection()
        {
            // hide CDP port detection by adding a script that will always run when a page loads (before the website loads any detection scripts)
            // https://www.reddit.com/r/webscraping/comments/1evht3i/help_in_bypassing_cdp_detection/
            // https://github.com/rebrowser/rebrowser-patches
            // https://github.com/kaliiiiiiiiii/brotector/blob/master/brotector.js#L377 // string detections
            string antiAntiBotScript = @"
                        (function() {
                            var originalError = Error;
                            Object.defineProperty(Error.prototype, 'stack', {
                                configurable: false,
                                enumerable: true,
                                writable: true,
                                value: (function() {
                                    try {
                                        throw new originalError();
                                    } catch (e) {
                                        return e.stack;
                                    }
                                })()
                            });
 
                            window.Error = new Proxy(originalError, {
                                construct(target, args) {
                                    var instance = new target(...args);
                                    return Object.freeze(instance);
                                }
                            });
                        })();
                    ";
 
            if (true == UsePuppeteer)
            {
                PuppeteerCommunicator.EnableFetchScriptInjection(antiAntiBotScript, null).GetAwaiter().GetResult();
            }
            else if (true == UseSelenium && BrowserType.FireFox != BrowserSettings.BrowserType)
            {
                ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand("Page.addScriptToEvaluateOnNewDocument", new Dictionary<string, object>
                {
                    { "source", antiAntiBotScript }
                });
            }
 
            string antiAntiBotScript2 = "Object.defineProperty(navigator, 'webdriver', { get: () => undefined })";
            if (true == UseSelenium && BrowserType.FireFox != BrowserSettings.BrowserType)
            {
                ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand("Page.addScriptToEvaluateOnNewDocument", new Dictionary<string, object>
                {
                    { "source", antiAntiBotScript2 }
                });
            }
            else if (true == UsePuppeteer)
            {
                PuppeteerCommunicator.EnableFetchScriptInjection(antiAntiBotScript2, null).GetAwaiter().GetResult();
            }
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, "Stealth CDP protection applied.", this, GPALObjectType.Browser);
        }
        private void ApplyToStringOverride()
        {
            string toStringOverrideScript = @"
                (function() {
                    // Store the original toString
                    const originalToString = Object.prototype.toString;
 
                    // Create a proxy handler to intercept toString calls
                    const handler = {
                        apply: function(target, thisArg, argumentsList) {
                            if (thisArg instanceof Function && /antiAntiBotScript|CDP/.test(originalToString.call(thisArg))) {
                                return 'function () { [native code] }';
                            }
                            return target.apply(thisArg, argumentsList);
                        }
                    };
 
                    // Wrap toString with a proxy
                    Object.prototype.toString = new Proxy(originalToString, handler);
 
                    console.log('toString override applied via Proxy');
                })();
            ";
 
            if (true == UseSelenium && BrowserType.FireFox != BrowserSettings.BrowserType)
            {
                ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand("Page.addScriptToEvaluateOnNewDocument", new Dictionary<string, object>
                {
                    { "source", toStringOverrideScript }
                });
            }
            else if (true == UsePuppeteer)
            {
                PuppeteerCommunicator.EnableFetchScriptInjection(toStringOverrideScript, null).GetAwaiter().GetResult();
            }
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, "Stealth toString override applied.", this, GPALObjectType.Browser);
        }
        private void CheckAndApplyStealth()
        {
            // hide CDP port detection by adding a script that will always run when a page loads (before the website loads any detection scripts)
            // https://www.reddit.com/r/webscraping/comments/1evht3i/help_in_bypassing_cdp_detection/
            // https://github.com/rebrowser/rebrowser-patches
            // https://github.com/kaliiiiiiiiii/brotector/blob/master/brotector.js#L377 // string detections
 
            if (false == BrowserSettings.StealthApplied)
            {
                if (false == UseOttoMagic && BrowserType.FireFox != BrowserType)
                {
                    if (BrowserSettings.StealthType.HasFlag(StealthType.CDP))
                        ApplyCDPProtection();
 
                    if (BrowserSettings.StealthType.HasFlag(StealthType.ToStringOverride))
                        ApplyToStringOverride();
                }
 
                if (BrowserSettings.StealthType.HasFlag(StealthType.GoogleReferrer))
                {
                    // set referer to google to make us look like we are coming in from search results click
                    try 
                    { 
                        ApplyGoogleReferrer(); 
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex) 
                    { 
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Google Referrer stealth skipped", this, GPALObjectType.Browser, ex); 
                    }
                }
            }
            BrowserSettings.StealthApplied = true;
        }

        public IAllowBrowserActionOrAnySelector CloseTab(dynamic URLorTabId = null)
        {
            // special use case closing a tab; can't call previoustab because we aren't in a good window yet and it will try to top
            if (true == UsePuppeteer)
            {
                if (URLorTabId is string s)
                    PuppeteerClient.CloseTab(s).Execute();
                else if (URLorTabId is GPALUrl url)
                    PuppeteerClient.CloseTab(url).Execute();
                else if (URLorTabId is int i)
                    PuppeteerClient.CloseTab(i).Execute();
                else
                    PuppeteerClient.CloseTab().Execute();
            }
            else if (true == UseOttoMagic)
            {
                string currentUrl = BrowserSettings.CurrentURL;
 
                int currentTabId = GetActiveTabId();
                TabTuple tabTuple = MagicHelper.CloseTab(URLorTabId);   // tabtuple is new active tab
 
                if (true == string.IsNullOrEmpty(URLorTabId))
                {
                    BrowserSettings.TabIdsToURL.Remove(currentTabId);
                    BrowserSettings.URLsToTabId.Remove(currentUrl);
                }
                else if (URLorTabId is int tabId)
                {
                    BrowserSettings.URLsToTabId.Remove(BrowserSettings.TabIdsToURL[tabId].URL);
                    BrowserSettings.TabIdsToURL.Remove(tabId);
                }
                else if (URLorTabId is string url)
                {
                    BrowserSettings.URLsToTabId.Remove(url);
                    BrowserSettings.TabIdsToURL.Remove(currentTabId);
                }
 
                UpdateActiveTab(tabTuple);
            }
            else
            {
                if (true == string.IsNullOrWhiteSpace(URLorTabId))
                {
                    ResolveGhostTabHandle();   // ensure Edge's phantom tab is known before we pick a tab to land on
 
                    // work out where we are going BEFORE closing, so the handle list is the one that still has
                    // this tab in it and the index means what it says
                    string closing = null;
                    string landOn = null;
 
                    try
                    {
                        var before = BrowserDriver.WindowHandles;
                        closing = BrowserDriver.CurrentWindowHandle;
 
                        // the tab to the left of the one closing, skipping Edge's ghost, else anything that is not us
                        int closingIdx = before.IndexOf(closing);
                        for (int idx = closingIdx - 1; 0 <= idx && null == landOn; idx--)
                            if (before[idx] != GhostTabHandle)
                                landOn = before[idx];
 
                        if (null == landOn)
                            landOn = before.FirstOrDefault(h => h != closing && h != GhostTabHandle);
                    }
                    catch (Exception exRead)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"CloseTab: could not read the tab list before closing, will fall back to whatever the driver leaves us on.", this, GPALObjectType.Browser, exRead);
                    }
 
                    // guarded: a flaky/off-screen EdgeDriver can hang the full 20s command timeout here and throw,
                    // which must not crash the workflow - the tab is being torn down regardless.
                    try { BrowserSettings.BrowserDriver.Close(); }  // close current tab
                    catch (Exception exClose) { GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"CloseTab: driver failed to close the current tab, continuing.", this, GPALObjectType.Browser, exClose); }
 
                    // closing the last tab leaves nothing to land on, which is a legitimate end of the workflow
                    if (false == string.IsNullOrEmpty(landOn))
                        try
                        {
                            BrowserDriver.SwitchTo().Window(landOn);
                            CurrentTabIdx = BrowserDriver.WindowHandles.IndexOf(landOn);   // re-seat against the list we actually have now
 
                        }
                        catch (Exception exSwitch)
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"CloseTab: could not switch to tab [{landOn}] after closing.", this, GPALObjectType.Browser, exSwitch);
                        }
                }
                else    // close a tab with a particular url (we don't use handles on selenium side)
                {
                    // the tab we are on now, so closing some other tab does not move the workflow. FindTab walks
                    // the tabs to read their urls, so where the driver ends up afterwards is not where it started
                    string staying = BrowserDriver.CurrentWindowHandle;
                    var handle = BrowserHelper.FindTab(BrowserSettings, URLorTabId);
 
                    // no tab is sitting on that url any more, which happens whenever the tab has navigated on, a
                    // download being the obvious case. Selenium answers a null handle with "'handle' must be a
                    // string" and takes the workflow down, so say what happened instead
                    if (true == string.IsNullOrEmpty(handle))
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"CloseTab: no tab is on [{URLorTabId}], nothing closed. Call .CloseTab() with no url to close the current tab.", this, GPALObjectType.Browser);
                    else
                    {
                        BrowserSettings.BrowserDriver.SwitchTo().Window(handle);
 
                        // guarded for the same reason as the no-url case above
                        try { BrowserSettings.BrowserDriver.Close(); }
                        catch (Exception exClose) { GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"CloseTab: driver failed to close tab [{URLorTabId}], continuing.", this, GPALObjectType.Browser, exClose); }
 
                        // go back to where the workflow was. Closing the tab we were on ourselves leaves nothing to
                        // go back to, so take the last one open, and closing the only tab leaves nothing at all
                        var after = BrowserDriver.WindowHandles;
                        string landOn = true == after.Contains(staying) ? staying : after.LastOrDefault(h => h != GhostTabHandle);
 
                        if (false == string.IsNullOrEmpty(landOn))
                        {
                            BrowserDriver.SwitchTo().Window(landOn);
                            CurrentTabIdx = after.IndexOf(landOn);
 
                        }
                    }
                }
            }
 
            // did that leave us anything to be on? ask the browser rather than reading our own tally, which cannot
            // see tabs a page opened or closed by itself. Firefox answers 1 here, it keeps the browser alive on an
            // about:blank tab, where chrome and edge answer 0 or have already torn the session down
            int tabsLeft = 0;
 
            try
            {
                tabsLeft = RealTabCount();
 
                if (0 < tabsLeft)
                    GetSetCurrentUrl();
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Browser stopped answering after closing the tab.", this, GPALObjectType.Browser, ex);
            }
 
            if (0 == tabsLeft)
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Closed the last tab, the browser closed with it.", this, GPALObjectType.Browser);
 
            tabsWeOpened--;
 
            //CheckPersistentSelectors();
 
            return this;
        }

        private List<string> TabsInThisWindow(IList<string> handles, string current)
        {
            List<string> siblings = new List<string>();
 
            // the tab we are on decides which window we are talking about, and it is its own window if this is the
            // first we have heard of it
            if (false == tabWindows.ContainsKey(current))
                tabWindows[current] = current;
 
            string thisWindow = tabWindows[current];
 
            foreach (string handle in handles)
            {
                if (false == tabWindows.ContainsKey(handle))
                    tabWindows[handle] = thisWindow;
 
                if (tabWindows[handle] == thisWindow && handle != GhostTabHandle)
                    siblings.Add(handle);
            }
 
            return siblings;
        }
        private void RecordTabWindow(string newHandle, string openedFrom)
        {
            if (true == string.IsNullOrEmpty(newHandle))
                return;
 
            if (false == string.IsNullOrEmpty(openedFrom) && true == tabWindows.ContainsKey(openedFrom))
                tabWindows[newHandle] = tabWindows[openedFrom];
            else
                tabWindows[newHandle] = string.IsNullOrEmpty(openedFrom) ? newHandle : openedFrom;
        }
        internal string ResolveGhostTabHandle()
        {
            // only Selenium switches by handle, and only Edge injects the preload ghost
            if (false == UseSelenium || BrowserType.Edge != BrowserSettings.BrowserType || true == string.IsNullOrEmpty(GhostTabUrl))
                return GhostTabHandle;
 
            // already found - cached for the life of the session
            if (false == string.IsNullOrEmpty(GhostTabHandle))
                return GhostTabHandle;
 
            var handles = BrowserDriver.WindowHandles;
 
            // nothing foreign present: no more open handles than tabs we opened ourselves
            if (handles.Count <= tabsWeOpened)
                return null;
 
            string current;
            try { current = BrowserDriver.CurrentWindowHandle; }
            catch { return null; }
 
            foreach (string handle in handles)
            {
                if (handle == current) continue;   // the tab we are on is never the ghost
 
                string handleUrl = null;
                try
                {
                    BrowserDriver.SwitchTo().Window(handle);
                    handleUrl = BrowserDriver.Url;
                }
                catch { /* can't read it, skip */ }
                finally { try { BrowserDriver.SwitchTo().Window(current); } catch { } }
 
                // positive identity match against the launch signature - a real tab will not match.
                // Match on the base url (scheme+host+path) only: Edge's launch url and the settled ghost url
                // share the same NTP path but differ in query (prerender=1 vs startpage=1, space encoding, an
                // extra invalidaterender token), so a full-url compare would never match.
                if (false == string.IsNullOrEmpty(handleUrl) && true == UrlHelper.SameBaseUrl(handleUrl, GhostTabUrl))
                {
                    GhostTabHandle = handle;
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Ghost tab found, will always skip [{handleUrl}]", this, GPALObjectType.Browser);
                    break;
                }
            }
 
            return GhostTabHandle;
        }
        public IAllowBrowserActionOrAnySelector NextTab
        {
            get
            {
                // due to clicks opening new tabs and other reasons, we can't track open tab count unless we have our own browser extension (can't do it in javascript, only extension)
 
                // browser will be topped below by selenium when we switch tabs
                // BrowserHelper.TopBrowser(this);
 
                // next tab - control page-down
                // when we move away from selenium, we may use this
                // HardwareHelper.SendKey(GPAL.VK_NEXT, ModifierKeys.Control);   // page down
 
                // using selenium, it must know which tab we are on to function properly, it doesn't know we switched tabs
                if (true == UseOttoMagic)
                {
                    TabTuple tabTuple = MagicHelper.NextTab();
                    UpdateActiveTab(tabTuple);
                }
                else if (true == UsePuppeteer)
                {
                    PuppeteerClient.NextTab().Execute();
                }
                else
                {
                    ResolveGhostTabHandle();   // Edge injects a phantom NTP handle; identify it so we never land on it
 
                    var handles = BrowserDriver.WindowHandles;
                    string current = BrowserDriver.CurrentWindowHandle;
 
                    // cycle the tabs of the window we are in, not the driver's flat list of every tab in every
                    // window. Ask the driver where we are rather than remembering it: a remembered position points
                    // at the wrong tab the moment any tab opens or closes
                    List<string> siblings = TabsInThisWindow(handles, current);
 
                    int idx = siblings.IndexOf(current);
                    if (0 > idx) idx = 0;
 
                    idx = (idx + 1) % siblings.Count;
 
                    CurrentTabIdx = handles.IndexOf(siblings[idx]);   // reported to the workflow, never read back as our position
 
                    BrowserDriver.SwitchTo().Window(siblings[idx]);
                    if (BrowserDriver.CurrentWindowHandle != siblings[idx])
                        BrowserDriver.SwitchTo().Window(siblings[idx]);
 
                }
 
                Thread.Sleep(50);
                //BrowserHelper.CheckDocumentReady(this, true);   // check document.ready
 
                // get set url, set gpalcurrenturl
                // NOTE: add search gpalurllist and setting gpalcurrenturl
 
                return this;
            }
        }

        public IAllowBrowserActionOrAnySelector NewTab(GPALUrl url)
        {
            BrowserSettings.OpenInTab = true;
            BrowserSettings.Browser.GoTo(url);  // will set currentgpalurl and add url to list
            return this;
        }

        public IAllowBrowserActionOrAnySelector PreviousTab
        {
            get
            {
                // due to clicks opening new tabs and other reasons, we can't track open tab count unless we have our own browser extension (can't do it in javascript, only extension)
 
                // browser will be topped below by selenium when we switch tabs
                // BrowserHelper.TopBrowser(this);
 
                // previous tab using ctrl-page up
                // when we move away from selenium, we may use this
                // HardwareHelper.SendKey(GPAL.VK_PRIOR, ModifierKeys.Control);   // page up
 
                // using selenium, it must know which tab we are on to function properly, it doesn't know we switched tabs above
                if (true == UseOttoMagic)
                {
                    UpdateActiveTab(MagicHelper.PreviousTab());
                }
                else if (true == UsePuppeteer)
                {
                    PuppeteerClient.PreviousTab().Execute();
                }
                else
                {
                    ResolveGhostTabHandle();   // Edge injects a phantom NTP handle; identify it so we never land on it
 
                    var handles = BrowserDriver.WindowHandles;
                    string current = BrowserDriver.CurrentWindowHandle;
 
                    // same as NextTab: the tabs of this window only, and the driver is asked where we are
                    List<string> siblings = TabsInThisWindow(handles, current);
 
                    int idx = siblings.IndexOf(current);
                    if (0 > idx) idx = 0;
 
                    // (i - 1 + n) % n - add the raw count before the modulo so a -1 wrap stays positive
                    idx = (idx - 1 + siblings.Count) % siblings.Count;
 
                    CurrentTabIdx = handles.IndexOf(siblings[idx]);   // reported to the workflow, never read back as our position
 
                    BrowserDriver.SwitchTo().Window(siblings[idx]);
                    if (BrowserDriver.CurrentWindowHandle != siblings[idx])
                        BrowserDriver.SwitchTo().Window(siblings[idx]);
 
                }
 
                //BrowserHelper.CheckDocumentReady(this, true);   // check idle
 
                return this;
            }
        }

        public IAllowBrowserActionOrAnySelector LeftClick()
        {
            return LeftClick(ModifierKeys.NONE);
        }

        public IAllowBrowserActionOrAnySelector LeftClick(ModifierKeys modifierKeys)
        {
            string priorUrl = BrowserSettings.CurrentURL;
            object priorDocument = BrowserHelper.MarkDocument(this);   // the page we are clicking on, to tell later if we left it
 
            // snapshot open handles before the click so we can tell if the click opened a new tab (target=_blank)
            var handlesBefore = (true == UseSelenium) ? new HashSet<string>(BrowserDriver.WindowHandles) : null;
 
            // waitfor did not match if returns false, so we have no element to click
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.LeftClick(this, modifierKeys);
 
            if (true == UseSelenium)
            {
                ResolveGhostTabHandle();   // make sure Edge's phantom tab is identified before we count new tabs
 
                // in case this opens a new tab
                var handles = BrowserDriver.WindowHandles.ToList();
                string currentHandle = BrowserDriver.CurrentWindowHandle;
 
                CurrentTabIdx = handles.IndexOf(currentHandle);
 
                // reliable our-tabs counter: add only genuinely new tabs this click opened, never Edge's ghost
                int opened = 0;
 
                foreach (string handle in handles.Where(h => false == handlesBefore.Contains(h) && h != GhostTabHandle))
                {
                    RecordTabWindow(handle, currentHandle);   // a click opens into the window it was clicked in
                    opened++;
                }
 
                if (0 < opened) tabsWeOpened += opened;
            }
 
            // the click returns as soon as the event is dispatched, so give the browser a moment to commit the
            // navigation before deciding it did not go anywhere. Reading the url once here saw the old page
            if (true == BrowserHelper.WaitForNavigationOrTime(this, priorUrl, priorDocument, BrowserSettings.NavigationGraceMs))
            {
                BrowserHelper.CheckDocumentReady(this);
 
                // read the url again now the document is committed, not while it was still tearing down
                string currentUrl = GetSetCurrentUrl();
 
                if (true == UseOttoMagic)
                {
                    int tabId = GetActiveTabId();
                    TabTuple tabTuple = new TabTuple() { TabId = tabId, Url = currentUrl };
                    UpdateActiveTab(tabTuple);
                }
                else if (true == UsePuppeteer)
                    PuppeteerClient.SwitchToDefaultContent().Execute();
                else
                    BrowserDriver.SwitchTo().DefaultContent();
            }
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector LeftClick(Selector selector)
        {
            if (null != selector)
                WithSelector(selector);
 
            return LeftClick(ModifierKeys.NONE);
        }

        public IAllowBrowserActionOrAnySelector SelectClick(SelectClickType selectClickType)
        {
            // waitfor did not match if returns false, so we have no element to click
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                string priorUrl = BrowserSettings.CurrentURL;
                object priorDocument = BrowserHelper.MarkDocument(this);   // the page we are clicking on, to tell later if we left it
                ElementHelper.GenericClick(this, selectClickType);
 
                if (true == BrowserHelper.WaitForNavigationOrTime(this, priorUrl, priorDocument, BrowserSettings.NavigationGraceMs))
                    BrowserHelper.CheckDocumentReady(this);
            }
 
            GetSetCurrentUrl();
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector StealthLeftClick(Selector selector)
        {
            if (null != selector)
                WithSelector(selector);
 
            // waitfor did not match if returns false, so we have no element to click
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                // we never enable runtime, so should not be needed
                if (true == UseOttoMagic || true == UsePuppeteer)
                    ElementHelper.LeftClick(this);
                else if (BrowserType.FireFox != BrowserSettings.BrowserType)
                {
                    ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand("Runtime.disable", new Dictionary<string, object> { });
                    ElementHelper.LeftClick(this);
                    ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand("Runtime.enable", new Dictionary<string, object> { });
                }
                else
                    ElementHelper.LeftClick(this);
            }
 
            GetSetCurrentUrl();
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector LeftDoubleClick()
        {
            // waitfor did not match if returns false
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                ElementHelper.GenericClick(this, ClickType.LeftDoubleClick);
                BrowserHelper.CheckDocumentReady(this);
            }
            // NOTE: should we expect this to possibly change the url? how many websites do you interact that take double clicks? only a custom interface, i think.
            return this;
        }

        public IAllowBrowserActionOrAnySelector MiddleClick()
        {
            string priorUrl = BrowserSettings.CurrentURL;
 
            // snapshot open handles before the click so we can tell which tab(s) the middle-click opened
            var handlesBefore = (true == UseSelenium) ? new HashSet<string>(BrowserDriver.WindowHandles) : null;
 
            // waitfor did not match if returns false, so we have no element to click
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.MiddleClick(this);
 
            string currentUrl = GetSetCurrentUrl();
 
            if (true == UseSelenium)
            {
                ResolveGhostTabHandle();   // make sure Edge's phantom tab is identified before we count new tabs
 
                // in case this opens a new tab
                var handles = BrowserDriver.WindowHandles.ToList();
                string currentHandle = BrowserDriver.CurrentWindowHandle;
 
                CurrentTabIdx = handles.IndexOf(currentHandle);
 
                // reliable our-tabs counter: add only genuinely new tabs this middle-click opened, never Edge's ghost
                int opened = 0;
 
                foreach (string handle in handles.Where(h => false == handlesBefore.Contains(h) && h != GhostTabHandle))
                {
                    RecordTabWindow(handle, currentHandle);   // a middle click opens into the window it was clicked in
                    opened++;
                }
 
                if (0 < opened) tabsWeOpened += opened;
            }
 
            if (false == UrlHelper.AreEquivalent(priorUrl, currentUrl))
            {
                if (true == UseOttoMagic)
                {
                    int tabId = GetActiveTabId();
                    TabTuple tabTuple = new TabTuple() { TabId = tabId, Url = currentUrl };
                    UpdateActiveTab(tabTuple);
                }
                else if (true == UsePuppeteer)
                    PuppeteerClient.SwitchToDefaultContent().Execute();
                else
                    BrowserDriver.SwitchTo().DefaultContent();
            }
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector RightClick(Selector selector = null)
        {
            if (null != selector)
            {
                // if we are using GET, we are headless, there is no hardware interaction, force selenium on any selector, overriding definitions
                if (true == BrowserSettings.UseHeadless)
                    selector.InteractionType = InteractionType.Selenium;
                WithSelector(selector);
            }
            // waitfor did not match if returns false
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                ElementHelper.GenericClick(this, ClickType.RightClick);
                BrowserHelper.CheckDocumentReady(this);
            }
            // NOTE: should we expect this to possibly change the url? how many websites do you interact that take double clicks? only a custom interface, i think.
            return this;
        }


        bool useEAvars = false;
        GPALElement EAelement = null;
        Selector EAselector = null;
        internal void LeftClickAndDownload(GPALFile gPalFile, GPALElement webElement, Selector selector)
        {
            useEAvars = true;
            EAelement = webElement;
            EAselector = selector;
 
            LeftClickAndDownload(gPalFile);
 
        }
        // long enough for a dialog that is opening to be there, short enough that finding none costs nothing
        internal const int UnexpectedSaveAsWaitSeconds = 2;
        internal const int ProfileRestoreWaitMs = 5000;      // how long to give the browser to shut down before the profile goes back

        public IAllowBrowserActionOrAnySelector LeftClickAndDownload(GPALFile gPalFileToDownloadTo)
        {
            string fullFilePathToSave;
            HWND windowOpened = IntPtr.Zero;
            List<string> returnFilename = new List<string>();
            bool noSaveAsDialog = false;
            FileDownloaded = false;
            int rowCount = 0;
 
            if (true == useEAvars)
            {
                DownloadWorkflow(EAelement, EAselector);
                return this;
            }
 
            int DownloadWorkflow(GPALElement webElement, Selector selector)
            {
                GPALFile fileToSave = gPalFileToDownloadTo.Next;
                fullFilePathToSave = FileHelper.CleanUpFileDestination(fileToSave.Filename, out string newFilename, BrowserSettings.DeleteFileBeforeDownload);
                returnFilename.Add(fullFilePathToSave);  // return the filenames generated
 
                string currentUrl = null, currentUrl2 = null;
 
                currentUrl = BrowserHelper.GetCurrentUrl(BrowserSettings);
 
                FileDownloaded = null;
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Downloading to [{fullFilePathToSave}]", this, GPALObjectType.Browser);
 
                if (true == BrowserSettings.UseDirectDownload)
                {
                    FileHelper.DownloadToFile(webElement, fullFilePathToSave);
                }
                // no prompt means no Save As dialog is coming, so there is nothing to wait for or type into - watch
                // the download directory instead, the same as headless does. OpenPDFExternally is not part of this
                // any more: it is a profile preference neither cdp nor the extension can set, so it never changes
                else if (true == BrowserSettings.UseHeadless || false == BrowserSettings.PromptForDownload)    // headless mode or no prompt for filename, monitor default download dir
                {
                    // so assume the file will download
                    string watchedDirectory = FileHelper.SetupDownloadWatcher(fullFilePathToSave, BrowserSettings);
 
                    // BOTH OF THESE do not check for completion, just sets download path and clicks
                    if (true == BrowserSettings.UsePuppeteer)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Element [{webElement.TagName}] for selector [{selector.Name}] [{selector.InteractionType.ToString()}][Left] clicked with modifiers [NONE].");
                        PuppeteerClient.LeftClickAndDownload(webElement.ElementHandle).DownloadTo(fileToSave).Execute();
                    }
                    else if (true == BrowserSettings.UseOttoMagic)
                    {
                        // these two arms click through their own client rather than ElementHelper.Click, so the
                        // click never reaches the event that method publishes and has to be said here. without it
                        // a download that produced nothing cannot be told apart from a click that never landed
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Element [{webElement.TagName}] for selector [{selector.Name}] [{selector.InteractionType.ToString()}][Left] clicked with modifiers [NONE].");
                        MagicHelper.LeftClickAndDownload(webElement.Css, fileToSave);
                    }
                    else   // selenium
                    {
                        BrowserHelper.SetHeadlessDownload(BrowserSettings, fullFilePathToSave);
                        ElementHelper.Click(this, selector, webElement, ClickType.LeftClick, ModifierKeys.NONE);
                    }
 
                    Thread.Sleep(250);
                    // NOTE: new scenario, some nag modal pops up before the download starts
                    // define those as persistent selectors and check persistent selectors here after we click to automatically dismiss them
                    string urlAfterClick = BrowserHelper.GetCurrentUrl(BrowserSettings);
 
                    // the persistent selectors belong to the page the click was made on. a click that navigated
                    // left that page, so there is nothing of ours to look for where we landed
                    if (true == UrlHelper.AreEquivalent(currentUrl, urlAfterClick))
                        CheckPersistentSelectors();
 
                    if (false == UrlHelper.AreEquivalent(currentUrl, urlAfterClick)
                        && true == fileToSave.Filename.EndsWith(".pdf")
                        && true == BrowserHelper.TestUrlForPDF(this, urlAfterClick))
                    {
                        //GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{urlAfterClick}] is the pdf itself, fetching it rather than waiting on a Save As dialog.", this, GPALObjectType.Browser);
                        FileDownloaded = BrowserHelper.DownloadPdfFile(urlAfterClick, fullFilePathToSave, this);
                    }
 
                    // PromptForDownload is a claim about the browser's configuration that cannot be read back from
                    // any of the three engines, and on ottomagic and puppeteer cannot be set either. so rather than
                    // trust it, look: a dialog that did open blocks the browser for everything after it, and a
                    // couple of seconds spent finding out beats losing the rest of the run to a modal nobody typed
                    // into. nothing appears when the claim was right, which is the ordinary case
                    if (false == BrowserSettings.UseHeadless)
                    {
                        HWND unexpectedDialog = IntPtr.Zero;
 
                        if (BrowserType.Chrome == BrowserSettings.BrowserType || BrowserType.Edge == BrowserSettings.BrowserType)
                            unexpectedDialog = FormHelper.WaitForWindow("Save As", UnexpectedSaveAsWaitSeconds);
                        else if (BrowserType.FireFox == BrowserSettings.BrowserType)
                            // NOTE: that is an elipses character used, not ...
                            unexpectedDialog = FormHelper.WaitForWindow("Enter name of file to save to…", UnexpectedSaveAsWaitSeconds);
 
                        if (IntPtr.Zero != unexpectedDialog)
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{BrowserSettings.BrowserType}] asked where to save the file even though prompting was turned off. Answering the dialog with [{newFilename}].", this, GPALObjectType.Browser);
 
                            HardwareHelper.SendString(newFilename);
                            HardwareHelper.SendKey(GPAL.VK_RETURN);
                        }
                    }
 
                    // null is the file handler not having reached an answer yet, which is the only thing worth
                    // waiting on. true and false are both answers and it shuts the watcher down as it gives one,
                    // so nothing can change after that.
                    // DownloadTimeoutInSec bounds how long nothing happens, not how long the file takes. every time
                    // the bytes on disk change the clock starts over, so a transfer is never cut off while it is
                    // moving, and a wait where nothing is moving always ends - whether the download never started
                    // or it died partway. stopping the clock on start instead left no way out of here at all
                    Stopwatch sinceProgress = Stopwatch.StartNew();
                    long lastSeen = long.MinValue;
 
                    // false is not final. these events arrive many times over one download and an early one can
                    // conclude nothing while a later one is still resolving the file, so a false here is answered
                    // by waiting rather than by giving up. what ends this loop is the allowance below
                    while (null == FileDownloaded || false == FileDownloaded)
                    {
                        // NOTE: it is entirely possible, since we do not destroy the filewatcher, that the file could still come in and be handled, i've not seen it, but it's possible
                        long progress = FileHelper.DownloadProgress(watchedDirectory, fullFilePathToSave);
 
                        if (progress != lastSeen)
                        {
                            lastSeen = progress;
                            sinceProgress.Restart();
                        }
                        else if (sinceProgress.Elapsed.TotalSeconds > FileHelper.AllowanceFor(watchedDirectory, BrowserSettings))
                            break;
 
                        Thread.Sleep(100); // Prevents 100% CPU usage
                    }
 
                    currentUrl2 = BrowserHelper.GetCurrentUrl(BrowserSettings);
 
                    // but maybe it's a pdf opening in the browser itself, so check to see if that happened and if the url is a pdf file
                    // of course it is possible the url won't contain .pdf and served up via a header? we might have to chack that
                    if (false == (bool)(FileDownloaded ?? false) && false == BrowserSettings.OpenPDFExternally && true == fileToSave.Filename.EndsWith(".pdf"))
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Testing [{currentUrl2}] for PDF file", this, GPALObjectType.Browser);
                        if (false == UrlHelper.AreEquivalent(currentUrl, currentUrl2) && true == BrowserHelper.TestUrlForPDF(this, currentUrl2))
                        {
                            FileDownloaded = BrowserHelper.DownloadPdfFile(currentUrl2, fullFilePathToSave, this);
                            noSaveAsDialog = !(bool)FileDownloaded;
                        }
                    }
 
                    // we're headless, no fallback, if it didn't download it's not going someplace other than where we specified
                    if (false == (bool)(FileDownloaded ?? false))
                        GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"File not downloaded to [{fullFilePathToSave}].", this, GPALObjectType.Browser);
                        FileHelper.DiscardAbandonedDownload(BrowserSettings);
 
                }
                else if (false == BrowserSettings.UseHeadless)    // if we are not headless, we can handle the dialog or direct download
                {
                    ElementHelper.Click(this, selector, webElement, ClickType.LeftClick, ModifierKeys.NONE);
 
                    string urlAfterClick = BrowserHelper.GetCurrentUrl(BrowserSettings);
 
                    // the persistent selectors belong to the page the click was made on. a click that navigated
                    // left that page, so there is nothing of ours to look for where we landed
                    // NOTE: new scenario, some nag modal pops up before the download starts
                    // define those as persistent selectors and check persistent selectors here after we click to automatically dismiss them
                    if (true == UrlHelper.AreEquivalent(currentUrl, urlAfterClick))
                        CheckPersistentSelectors();
 
                    // the click landed on the pdf itself, so the browser is showing it and no Save As is ever coming.
                    // fetching it now is the whole of the work, and skips the wait for a dialog that will not appear.
                    // OpenPDFExternally is not consulted: a browser honoring it would have prompted or downloaded
                    // rather than navigated, so arriving here on a pdf url says it did not take
                    if (false == UrlHelper.AreEquivalent(currentUrl, urlAfterClick)
                        && true == fileToSave.Filename.EndsWith(".pdf")
                        && true == BrowserHelper.TestUrlForPDF(this, urlAfterClick))
                    {
                        //GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{urlAfterClick}] is the pdf itself, fetching it rather than waiting on a Save As dialog.", this, GPALObjectType.Browser);
                        FileDownloaded = BrowserHelper.DownloadPdfFile(urlAfterClick, fullFilePathToSave, this);
                    }
 
                    if (false == (bool)(FileDownloaded ?? false) && true == BrowserSettings.PromptForDownload)
                        if (BrowserType.Chrome == BrowserSettings.BrowserType || BrowserType.Edge == BrowserSettings.BrowserType)
                            windowOpened = FormHelper.WaitForWindow("Save As", BrowserSettings.WaitForWindowTimeoutInSeconds);
                        else if (BrowserType.FireFox == BrowserSettings.BrowserType)
                            // NOTE: that is an elipses character used, not ...
                            windowOpened = FormHelper.WaitForWindow("Enter name of file to save to…", BrowserSettings.WaitForWindowTimeoutInSeconds);
 
                    // nothing below applies once the pdf is already on disk. it is all recovery for a Save As that
                    // never came, and running it would report a failure for a file we have and then reach for the
                    // element the navigation took with it
                    if (true == (bool)(FileDownloaded ?? false))
                        noSaveAsDialog = false;
                    // save as window didn't open in the allotted timeout, or we set to no prompt and a direct download
                    else if (IntPtr.Zero != windowOpened)
                    {
                        HardwareHelper.SendString(newFilename);
                        HardwareHelper.SendKey(GPAL.VK_RETURN);
 
                        // is the file save dialog still open? send ENTER again
                        if (BrowserType.Chrome == BrowserSettings.BrowserType || BrowserType.Edge == BrowserSettings.BrowserType)
                            windowOpened = FormHelper.WaitForWindow("Save As", 1);
                        else if (BrowserType.FireFox == BrowserSettings.BrowserType)
                            windowOpened = FormHelper.WaitForWindow("Enter name of file to save to…", 1);    // NOTE: that is an elipses character used, not ...
 
                        if (IntPtr.Zero != windowOpened)
                            HardwareHelper.SendKey(GPAL.VK_RETURN);
 
                        string actualDownloadedFilename = null;
 
                        actualDownloadedFilename = FileHelper.WaitForDownloadToFinish(newFilename, BrowserSettings.DownloadTimeoutInSec);
 
                        FileDownloaded = true != string.IsNullOrEmpty(actualDownloadedFilename);
 
                        if (false == FileDownloaded)
                            GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"File not downloaded to [{fullFilePathToSave}]", this, GPALObjectType.Browser);
                            FileHelper.DiscardAbandonedDownload(BrowserSettings);
 
                    }
                    else
                    {
                        if (true == BrowserSettings.PromptForDownload)
                            GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Failed to handle Save As... dialog [{fullFilePathToSave}][{selector.Name}]", BrowserSettings, GPALObjectType.Other);
 
                        noSaveAsDialog = true;
 
                        // NOTE: maybe it's a pdf opening in the browser itself, so check to see if that happened and if the url is a pdf file
                        currentUrl2 = BrowserHelper.GetCurrentUrl(BrowserSettings);
 
                        // OpenPDFExternally is not consulted here either. the Save As wait has already come and gone
                        // with nothing, so whatever the setting asked for is not what the browser did
                        if (false == (bool)(FileDownloaded ?? false) && true == fileToSave.Filename.EndsWith(".pdf"))
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Testing [{currentUrl2}] for PDF file", this, GPALObjectType.Browser);
                            if (false == UrlHelper.AreEquivalent(currentUrl, currentUrl2) && true == BrowserHelper.TestUrlForPDF(this, currentUrl2))
                            {
                                FileDownloaded = BrowserHelper.DownloadPdfFile(currentUrl2, fullFilePathToSave, this);
                                noSaveAsDialog = !(bool)FileDownloaded;
                            }
                        }
                    }
                }
 
                // we didn't get a save as dialog within BrowserSettings.WaitForWindowTimeoutInSeconds which defaults to 30 seconds
                // so why? well, the website might have downloaded directly using the current browser default download directory
                // so we will attempt to see if a file was downloaded matching the file extension of the GPALFile the user specified
                if (true == noSaveAsDialog)
                {
                    string downloadDirectory = FileHelper.GetDefaultDownloadDirectory(this);
                    string downloadUrl = FileHelper.GetDownloadUrl(webElement);
                    string filename = Path.GetFileName(downloadUrl) ?? fullFilePathToSave;
 
                    string wildcardFilename = "*" + Path.GetExtension(filename);
                    string systemDownloadedTo = Path.Combine(downloadDirectory, wildcardFilename);
                    string actualDownloadedFilename = null;
 
                    actualDownloadedFilename = FileHelper.WaitForDownloadToFinish(systemDownloadedTo, BrowserSettings.DownloadTimeoutInSec);
 
                    FileDownloaded = true != string.IsNullOrEmpty(actualDownloadedFilename);
 
                    if (true == FileDownloaded)
                    {
                        if (false == fullFilePathToSave.Equals(actualDownloadedFilename))
                        {
                            File.Move(actualDownloadedFilename, fullFilePathToSave);
                            if (false == actualDownloadedFilename.Equals(fullFilePathToSave) && true == File.Exists(actualDownloadedFilename))
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Downloaded [{actualDownloadedFilename}] unable to move to [{fullFilePathToSave}]", null, GPALObjectType.None);
                            else
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{actualDownloadedFilename}] moved to [{fullFilePathToSave}]", this, GPALObjectType.Browser);
                        }
                    }
                    else
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{filename}] not found in [{downloadDirectory}]", this, GPALObjectType.Browser);
                }
 
                if (null != FileHelper.watcher)    // we might not have disposed of watcher
                {
                    try
                    {
                        FileHelper.watcher.Dispose();
                        FileHelper.watcher = null;
                    }
                    catch { }
                }
 
                if (false == BrowserSettings.UseHeadless && BrowserType.Edge == BrowserSettings.BrowserType)    // dismiss the 'downloads' popup which can steal keys, clicks or get in the way
                {
                    // KLUDGE: must be a hardware top to dismiss 'downloads' by clicking the browser chrome
                    BrowserHelper.TopBrowser(this, false, true);
                }
 
                if (int.MaxValue != CurrentUOW.WithAllThatMatch && ++rowCount == CurrentUOW.WithAllThatMatch)
                    return -1;
                else 
                    return 0;
            }
 
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                foreach (Selector selector in CurrentUOW.WithSelectorList)
                {
                    if (SelectorType.Selector != selector.SelectorType)
                        continue;   // we don't look up data literals
 
                    ReadOnlyCollection<GPALElement> webelements = ElementHelper.FindWebElements(this, CurrentUOW, selector, out bool matchedAll, out List<GPALElement> matchedElements);
                    if (null != webelements)
                    {
                        // the page these came from is the one to come back to. a download click that navigates
                        // takes that document with it, and every element still waiting in the list belongs to it
                        string pageOfTheElements = BrowserHelper.GetCurrentUrl(BrowserSettings);
                        int matchCount = matchedElements.Count;
 
                        for (int elementIdx = 0; elementIdx < matchCount; elementIdx++)
                        {
                            string urlNow = BrowserHelper.GetCurrentUrl(BrowserSettings);
 
                            // back to where the elements live, and find them again - the ones in hand are gone with
                            // their document, and asking one of them anything raises an unknown reference
                            if (false == UrlHelper.AreEquivalent(pageOfTheElements, urlNow))
                            {
                                GoTo(pageOfTheElements);
                                ElementHelper.FindWebElements(this, CurrentUOW, selector, out bool _, out matchedElements);
 
                                if (matchedElements.Count != matchCount)
                                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{selector.Name}] matched [{matchedElements.Count}] on the way back rather than [{matchCount}], so the remaining downloads may not be the ones first found.", this, GPALObjectType.Browser);
                            }
 
                            if (elementIdx >= matchedElements.Count)
                                break;
 
                            // int.MaxValue tells us we have met our .WithAllThatMatch elements to process
                            if (-1 == DownloadWorkflow(matchedElements[elementIdx], selector))
                                break;
                        }
                    }
                }
 
                foreach (string filepath in returnFilename)
                {
                    gPalFileToDownloadTo.ReturnFilenames.Add(filepath); // return the filenames we generated
//                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"File saved to [{filepath}]");
                }
            }
 
            GetSetCurrentUrl();
 
            return this;
        }

        internal void LeftClickAndUpload(GPALFile gPalFile, GPALElement webElement, Selector selector)
        {
            useEAvars = true;
            EAelement = webElement;
            EAselector = selector;
 
            LeftClickAndUpload(gPalFile);
 
        }
        public IAllowBrowserActionOrAnySelector LeftClickAndUpload(GPALFile gPalFileToUpload)
        {
            HWND windowOpened = IntPtr.Zero;
 
            if (true == useEAvars)
            {
                UploadWorkflow(EAelement, EAselector, false);   // NOTE: CAVEAT: not sure this makes sense, once again, trying to interpret the concept of gpalfile having more than one file
                return this;
            }
 
            // NOTE: if you are providing a generic selector, say it finds 3 upload buttons, if you only supply 2 filenames in gg
            int UploadWorkflow(GPALElement webElement, Selector selector, bool uploadAllFiles)
            {
                string fileToUpload = null;
 
                // asked once, here, while the element is still the one we just found. reading it inside the loop
                // reads a stale element, because the first SendKeys is what makes it stale.
                // the .multiple property rather than the attribute: multiple is a bare boolean attribute carrying
                // no value in the markup, and GetAttribute capitalises the name before asking the driver, which is
                // enough to miss the special case that would have turned it into "true"
                bool sendAllAtOnce = true == UseSelenium
                                     && 1 < gPalFileToUpload.Count
                                     && true == Convert.ToBoolean(BrowserHelper.ExecuteJavaScript(this, "return arguments[0].multiple;", webElement.WebElement) ?? false);
 
                for (int fileCount = 0; fileCount < gPalFileToUpload.Count; fileCount++)
                {
                    fileToUpload = gPalFileToUpload.Filenames[fileCount];
 
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Uploading [{fileToUpload}]", this, GPALObjectType.Browser);
 
                    // while we can deal with the input dialog, why bother? this is cleaner
                    // if (true == BrowserSettings.UseHeadless || true == BrowserSettings.UseOttoMagic)    // headless mode or no prompt for filename, alternate methods to add file to input
 
                    // if the user wants to use hardware input, we can do that, otherwise direct is best - but hardware works for selenium headful mode
                    if (false == BrowserSettings.UseHardware && false == GPAL.GPALSettings.UseHardware && InteractionType.Hardware != selector.InteractionType)
                    {
 
                        if (true == string.IsNullOrEmpty(fileToUpload))
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, "For some reason we have an empty filename", gPalFileToUpload, GPALObjectType.Other);
 
                        // BOTH OF THESE do not check for completion, just sets download path and clicks
                        if (true == BrowserSettings.UsePuppeteer)
                        {
                            if (true == uploadAllFiles)
                            {
                                PuppeteerClient.LeftClickAndUpload(webElement.ElementHandle).UploadFrom(gPalFileToUpload).Execute();
                                break;
                            }
                            PuppeteerClient.LeftClickAndUpload(webElement.ElementHandle).UploadFrom(fileToUpload).Execute();
                        }
                        else if (true == BrowserSettings.UseOttoMagic)
                            MagicHelper.LeftClickAndUpload(webElement.Css, fileToUpload);
                        else   // selenium
                        {
                            // chromedriver appends a file on every SendKeys, geckodriver throws on the second one
                            // instead. all of them take the whole set in one call with the paths newline separated,
                            // so a multiple input is filled that way and there is nothing left to loop over
                            string filesToSend = true == sendAllAtOnce ? string.Join("\n", gPalFileToUpload.Filenames) : fileToUpload;
 
                            try
                            {
                                ((IWebElement)webElement.iWebElement).SendKeys(filesToSend);   // NOTE: special case in selenium sendkeys on input[type=files] adds a file
                            }
                            catch    // stale element
                            {
                                ElementHelper.FindWebElements(this, CurrentUOW, selector, out bool _, out List<GPALElement> matchedElements);
                                webElement = 0 < matchedElements.Count ? matchedElements[0] : null;
                                webElement?.SendKeys(filesToSend.Replace("\\", "/"));
                            }
 
                            if (true == sendAllAtOnce)
                                break;
                        }
 
                        Thread.Sleep(250);
                    }
                    else if (false == BrowserSettings.UseHeadless)    // NOTE: HARDWARE insisted upon for this selector, so it is clickable
                    {
                        ElementHelper.Click(this, selector, webElement, ClickType.LeftClick, ModifierKeys.NONE);    // click the input[type=file] button
 
                        if (BrowserType.Chrome == BrowserSettings.BrowserType || BrowserType.Edge == BrowserSettings.BrowserType)
                            windowOpened = FormHelper.WaitForWindow("Open", BrowserSettings.WaitForWindowTimeoutInSeconds);
                        else if (BrowserType.FireFox == BrowserSettings.BrowserType)
                            // NOTE: that is an elipses character used, not ...
                            windowOpened = FormHelper.WaitForWindow("File Upload", BrowserSettings.WaitForWindowTimeoutInSeconds);
 
                        // save as window didn't open in the allotted timeout, or we set to no prompt and a direct download
                        if (IntPtr.Zero != windowOpened)
                        {
                            int filesToSend = 1;
 
                            if (true == webElement.GetAttribute("multiple")?.ToLower().Equals("true"))
                                filesToSend = gPalFileToUpload.Count;
 
                            if (true == uploadAllFiles)
                                for (int idx = 0; idx < gPalFileToUpload.Count; idx++)
                                {
                                    fileToUpload = gPalFileToUpload.Filenames[idx];
                                    HardwareHelper.SendString($@"""{fileToUpload}"" ");
                                }
                            else
                                HardwareHelper.SendString($@"""{fileToUpload}"" ");
 
                            HardwareHelper.SendKey(GPAL.VK_RETURN);
 
                            // is the file save dialog still open? send ENTER again
                            if (BrowserType.Chrome == BrowserSettings.BrowserType || BrowserType.Edge == BrowserSettings.BrowserType)
                                windowOpened = FormHelper.WaitForWindow("Open", 1);
                            else if (BrowserType.FireFox == BrowserSettings.BrowserType)
                                windowOpened = FormHelper.WaitForWindow("File Upload", 1);    // NOTE: that is an elipses character used, not ...
 
                            if (IntPtr.Zero != windowOpened)
                                HardwareHelper.SendKey(GPAL.VK_RETURN);
 
                            if (true == uploadAllFiles)
                                break;  // we send all the filenames, so long as they are not too long
                        }
                        else
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Failed to handle Open File dialog [{fileToUpload}][{selector.Name}] not upoloaded.", BrowserSettings, GPALObjectType.Other);
                            return -1;
                        }
                    }
 
                    //if (false == BrowserSettings.UseHeadless && BrowserType.Edge == BrowserSettings.BrowserType)    // dismiss the 'downloads' popup which can steal keys, clicks or get in the way
                    //{
                    //    // KLUDGE: must be a hardware top to dismiss 'downloads' by clicking the browser chrome
                    //    bool tmpTop = GPAL.SeleniumTopsBrowser;
                    //    GPAL.WithSeleniumToppingBrowser(false);
                    //    BrowserHelper.TopBrowser(this);
                    //    GPAL.WithSeleniumToppingBrowser(tmpTop);
                    //}
 
                    // NOTE: CAVEAT: not sure how to use WithAllThatMatch, seems cumbersome?
                    if (int.MaxValue != CurrentUOW.WithAllThatMatch && fileCount == CurrentUOW.WithAllThatMatch && false == uploadAllFiles)
                        return -1;
                    else if (false == uploadAllFiles)
                        return 0;
                }
 
                return -1;
            }
 
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                foreach (Selector selector in CurrentUOW.WithSelectorList)
                {
                    if (SelectorType.Selector != selector.SelectorType)
                        continue;   // we don't look up data literals
 
                    ReadOnlyCollection<GPALElement> webelements = ElementHelper.FindWebElements(this, CurrentUOW, selector, out bool matchedAll, out List<GPALElement> matchedElements);
                    if (null != webelements)
                    {
                        foreach (GPALElement webElement in matchedElements)
                        {
                            // int.MaxValue tells us we have met our .WithAllThatMatch elements to process 
                            if (-1 == UploadWorkflow(webElement, selector, 1 == matchedElements.Count))
                                break;
                        }
                    }
                }
            }
 
            GetSetCurrentUrl();
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector RightClickAndDownload(GPALFile filenames)
        {
            string fullFilePathToSave;
            HWND windowOpened = IntPtr.Zero;
            List<string> returnFilename = new List<string>();
            FileDownloaded = false;
 
            // waitfor did not match if returns false
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                foreach (Selector selector in CurrentUOW.WithSelectorList)
                {
                    if (SelectorType.Selector != selector.SelectorType)
                        continue;   // we don't look up data literals
 
                    ReadOnlyCollection<GPALElement> webelements = ElementHelper.FindWebElements(this, CurrentUOW, selector, out bool matchedAll, out List<GPALElement> matchedElements);
                    if (null != webelements)
                    {
                        foreach (GPALElement webElement in matchedElements)
                        {
                            int rowCount = 0;
 
                            fullFilePathToSave = FileHelper.CleanUpFileDestination(filenames.Next, out string newFilename, BrowserSettings.DeleteFileBeforeDownload);
                            returnFilename.Add(fullFilePathToSave);  // return the filenames generated
 
                            // this will iterate over all selectors in the CurrentUOW and right click each and attempt to download from each
                            // CAVEAT: this is untested with multiple selector elements
 
                            // CAVEAT: KLUDGE: Selenium cannot interact with context menu, so switch to javascript click, then back, altho it's prolly pointless to restore it at this point
                            Enums.InteractionType interactionType = selector.InteractionType;
                            if (Enums.InteractionType.Selenium == selector.InteractionType && "GPALElement" != webElement.TagName)
                                selector.InteractionType = InteractionType.JavaScript;
                            else if ("GPALElement" != webElement.TagName && false == BrowserSettings.UseHeadless)
                                selector.InteractionType = InteractionType.Hardware;
 
                            if (false == BrowserSettings.UseHeadless && (true == BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware))    // if we are not headless, we will handle the dialog, otherwise set using selenium and click
                            {
                                // NOTE: CAVEAT: if DownloadToFile always works, why would we care about using the save as dialog?
                                bool noSaveAsDialog = false;
                                string currentUrl = null, currentUrl2 = null;
 
                                if (true == BrowserSettings.UseOttoMagic)
                                    currentUrl = MagicHelper.GetCurrentUrl();
                                else if (true == BrowserSettings.UsePuppeteer)
                                    currentUrl = PuppeteerClient.GetCurrentUrl().Execute();
                                else
                                    currentUrl = BrowserDriver.Url;
 
                                ElementHelper.Click(this, selector, webElement, ClickType.RightClick, ModifierKeys.NONE);
 
                                // MS EDGE and context menu is weird under driver control, it seems to need time to settle in because it resets to the top of the context menu
                                // so we sleep 5 seconds
                                // we also give it a home key to gobble up cause it's just unpredictable
                                Thread.Sleep(5000);
 
                                // arrow down menu, then enter to select entry
                                // 2 or 3 arrows down. if it goes too far, it copies the url to the clipboard, so if the save as... doesn't open, we will try with 3 arrow down
                                HardwareHelper.SendKey(GPAL.VK_HOME);   // home key press is first element
                                Thread.Sleep(150);
                                HardwareHelper.SendKey(GPAL.VK_DOWN);
                                Thread.Sleep(150);
                                HardwareHelper.SendKey(GPAL.VK_DOWN);
                                Thread.Sleep(150);
                                HardwareHelper.SendKey(GPAL.VK_DOWN);
                                Thread.Sleep(150);
 
                                if (BrowserType.FireFox == BrowserSettings.BrowserType || BrowserType.Edge == BrowserSettings.BrowserType)
                                    HardwareHelper.SendKey(GPAL.VK_DOWN);
                                Thread.Sleep(150);
                                HardwareHelper.SendKey(GPAL.VK_RETURN);
 
                                // CAVEAT: KLUDGE - see above, restore to selenium
                                selector.InteractionType = interactionType;
 
                                if (BrowserType.Chrome == BrowserSettings.BrowserType || BrowserType.Edge == BrowserSettings.BrowserType)
                                    windowOpened = FormHelper.WaitForWindow("Save As", BrowserSettings.WaitForWindowTimeoutInSeconds);
                                else if (BrowserType.FireFox == BrowserSettings.BrowserType)
                                {
                                    windowOpened = FormHelper.WaitForWindow("Enter name of file to save to…", BrowserSettings.WaitForWindowTimeoutInSeconds);    // NOTE: that is an elipses character used, not ...
 
                                    if (IntPtr.Zero == windowOpened)    // could we have a misclick and saving this page? then the dialog title is 'Save As'
                                    {
                                        windowOpened = FormHelper.WaitForWindow("Save As", BrowserSettings.WaitForWindowTimeoutInSeconds);
 
                                        if (IntPtr.Zero != windowOpened)
                                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"GPAL thinks a misclick happened and the wrong item is saved. Please verify [{fullFilePathToSave}].", this, GPALObjectType.Browser);
                                    }
                                }
 
                                if (IntPtr.Zero != windowOpened)
                                {
                                    HardwareHelper.SendString(fullFilePathToSave);
                                    HardwareHelper.SendKey(GPAL.VK_RETURN);
 
                                    // is the file save dialog still open? send ENTER again
                                    // TODO: make strings configurable
                                    if (BrowserType.Chrome == BrowserSettings.BrowserType || BrowserType.Edge == BrowserSettings.BrowserType)
                                        windowOpened = FormHelper.WaitForWindow("Save As", 5);
                                    else if (BrowserType.FireFox == BrowserSettings.BrowserType)
                                        windowOpened = FormHelper.WaitForWindow("Enter name of file to save to…", 5);    // NOTE: that is an elipses character used, not ...
 
                                    if (IntPtr.Zero != windowOpened)
                                        HardwareHelper.SendKey(GPAL.VK_RETURN);
 
                                    // wait for file to download
                                    string actualDownloadedFilename2 = null;
 
                                    actualDownloadedFilename2 = FileHelper.WaitForDownloadToFinish(fullFilePathToSave, BrowserSettings.DownloadTimeoutInSec);
 
                                    FileDownloaded = true != string.IsNullOrEmpty(actualDownloadedFilename2);
 
                                    if (BrowserType.Edge == BrowserSettings.BrowserType)    // dismiss the 'downloads' popup which can steal keys, clicks or get in the way
                                    {
                                        // KLUDGE: must be a hardware top to dismiss 'downloads' by clicking the browser chrome
                                        BrowserHelper.TopBrowser(this, false, true);
                                    }
                                }
                                else
                                {
                                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Failed to handle Save As... dialog [{fullFilePathToSave}][{selector.Name}]", BrowserSettings, GPALObjectType.Other);
                                    noSaveAsDialog = true;
 
                                    // but maybe it's a pdf opening in the browser itself, so check to see if that happened and if the url is a pdf file
                                    // of course it is possible the url won't contain .pdf and served up via a header? we might have to chack that
 
                                    if (true == BrowserSettings.UseOttoMagic)
                                        currentUrl2 = MagicHelper.GetCurrentUrl();
                                    else if (true == BrowserSettings.UsePuppeteer)
                                        currentUrl2 = PuppeteerClient.GetCurrentUrl().Execute();
                                    else
                                        currentUrl2 = BrowserDriver.Url;
 
                                    // but maybe it's a pdf opening in the browser itself, so check to see if that happened and if the url is a pdf file
                                    // of course it is possible the url won't contain .pdf and served up via a header? we might have to chack that
                                    if (false == BrowserSettings.OpenPDFExternally && true == fullFilePathToSave.EndsWith(".pdf"))
                                    {
                                        GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Testing [{currentUrl2}] for PDF file", this, GPALObjectType.Browser);
                                        if (false == UrlHelper.AreEquivalent(currentUrl, currentUrl2) && true == BrowserHelper.TestUrlForPDF(this, currentUrl2))
                                        {
                                            FileDownloaded = BrowserHelper.DownloadPdfFile(currentUrl2, fullFilePathToSave, this);
                                            noSaveAsDialog = !(bool)FileDownloaded;
                                        }
                                    }
                                }
 
                                // we didn't get a save as dialog within BrowserSettings.WaitForWindowTimeoutInSeconds which defaults to 30 seconds
                                // so why? well, the website might have downloaded directly using the current browser default download directory
                                // so we will attempt to see if a file was downloaded matching the file extension of the GPALFile the user specified
                                if (true == noSaveAsDialog)
                                {
                                    string downloadDirectory = FileHelper.GetDefaultDownloadDirectory(this);
                                    string downloadUrl = FileHelper.GetDownloadUrl(webElement);
                                    string filename = Path.GetFileName(downloadUrl) ?? fullFilePathToSave;
 
                                    string wildcardFilename = "*" + Path.GetExtension(filename);
                                    string systemDownloadedTo = Path.Combine(downloadDirectory, wildcardFilename);
                                    string actualDownloadedFilename2 = null;
 
                                    actualDownloadedFilename2 = FileHelper.WaitForDownloadToFinish(systemDownloadedTo, BrowserSettings.DownloadTimeoutInSec);
 
                                    FileDownloaded = true != string.IsNullOrEmpty(actualDownloadedFilename2);
 
                                    if (true == FileDownloaded)
                                    {
                                        if (false == fullFilePathToSave.Equals(actualDownloadedFilename2))
                                        {
                                            File.Move(actualDownloadedFilename2, fullFilePathToSave);
                                            if (false == actualDownloadedFilename2.Equals(fullFilePathToSave) && true == File.Exists(actualDownloadedFilename2))
                                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Downloaded file [{actualDownloadedFilename2}] unable to move to [{fullFilePathToSave}]", null, GPALObjectType.None);
                                            else
                                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{actualDownloadedFilename2}] moved to [{fullFilePathToSave}]", this, GPALObjectType.Browser);
                                        }
                                    }
                                }
                            }
                            else    // set download destination then click
                                FileHelper.DownloadToFile(webElement, fullFilePathToSave);
                            /* selenium navigate the context menu, untested
                            else
                            {
                                SetDownloadFilename(fullFilePath);
 
                                // Create an Actions object
                                Actions actions = new Actions(browserSettings.BrowserDriver);
                                actions.SendKeys(Keys.Home);
 
                                // Simulate pressing the down arrow key multiple times
                                actions.SendKeys(Keys.ArrowDown);
                                actions.SendKeys(Keys.ArrowDown);
                                actions.SendKeys(Keys.ArrowDown);
 
                                // If using Firefox, simulate pressing the down arrow key one more time to get to FF command
                                if (BrowserType.FireFox == BrowserSettings.BrowserType)
                                    actions.SendKeys(Keys.ArrowDown);
 
                                // Simulate pressing the return key
                                actions.SendKeys(Keys.Enter);
 
                                // Simulate typing the filename
                                if (BrowserType.FireFox == browserSettings.BrowserType || BrowserType.Edge == browserSettings.BrowserType)
                                    actions.SendKeys(Keys.ArrowDown);
 
                                // Simulate pressing the return key
                                actions.SendKeys(Keys.Enter).Build().Perform();
                            }
                            */
                            
                            // how many of the found elements are we downloading from?
                            if (int.MaxValue != CurrentUOW.WithAllThatMatch && ++rowCount == CurrentUOW.WithAllThatMatch)
                                break;
                        }
                    }
                    else
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.ERROR, "Failed to detect Save As open window.", this, GPALObjectType.Browser);
                    }
                }
 
                foreach (string filepath in returnFilename)
                {
                    filenames.ReturnFilenames.Add(filepath); // return the filenames we generated
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"File saved to [{filepath}]");
                }
            }
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector AppendFrom(GPALDatabase inputDatabase)
        {
            DatabaseHelper.TokenizeDatabase(CurrentUOW, inputDatabase);
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                BrowserHelper.FillInWithTokens(this, CurrentUOW.InputDatabase.Tokens, WriteMode.Append);
            return this;
        }

        public IAllowBrowserActionOrAnySelector InsertFrom(GPALDatabase inputDatabase)
        {
            DatabaseHelper.TokenizeDatabase(CurrentUOW, inputDatabase);
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                BrowserHelper.FillInWithTokens(this, CurrentUOW.InputDatabase.Tokens, WriteMode.Insert);
            return this;
        }

        public IAllowBrowserActionOrAnySelector FillInFrom(GPALDatabase inputDatabase)
        {
            DatabaseHelper.TokenizeDatabase(CurrentUOW, inputDatabase);
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                BrowserHelper.FillInWithTokens(this, CurrentUOW.InputDatabase.Tokens, WriteMode.Overwrite);
            return this;
        }

        public IAllowBrowserActionOrAnySelector AppendFrom(GPALFile inputFile)
        {
            ((IGPALFileInternal)inputFile).TokenList = FileHelper.TokenizeFile(CurrentUOW, inputFile);
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                foreach (IGPALGrid<string> tokens in ((IGPALFileInternal)inputFile).TokenList)
                    BrowserHelper.FillInWithTokens(this, tokens, WriteMode.Append);
            return this;
        }

        public IAllowBrowserActionOrAnySelector FillInFrom(GPALFile inputFile)
        {
            ((IGPALFileInternal)inputFile).TokenList = FileHelper.TokenizeFile(CurrentUOW, inputFile);
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                foreach (IGPALGrid<string> tokens in ((IGPALFileInternal)CurrentUOW.InputFile).TokenList)
                    BrowserHelper.FillInWithTokens(this, tokens, WriteMode.Overwrite);
            return this;
        }

        public IAllowBrowserActionOrAnySelector InsertFrom(GPALFile inputFile)
        {
            ((IGPALFileInternal)inputFile).TokenList = FileHelper.TokenizeFile(CurrentUOW, inputFile);
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                foreach (IGPALGrid<string> tokens in ((IGPALFileInternal)inputFile).TokenList)
                    BrowserHelper.FillInWithTokens(this, tokens, WriteMode.Insert);
            return this;
        }

        public IAllowBrowserActionOrAnySelector AppendFrom(IGPALGrid<string> inputGrid)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                BrowserHelper.FillInWithTokens(this, inputGrid, WriteMode.Append);
            return this;
        }

        public IAllowBrowserActionOrAnySelector FillInFrom(IGPALGrid<string> inputGrid)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                BrowserHelper.FillInWithTokens(this, inputGrid, WriteMode.Overwrite);
            return this;
        }

        public IAllowBrowserActionOrAnySelector InsertFrom(IGPALGrid<string> inputGrid)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                BrowserHelper.FillInWithTokens(this, inputGrid, WriteMode.Insert);
            return this;
        }

        public IAllowBrowserActionOrAnySelector AppendFrom(string text)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.FillInFrom(this, text, WriteMode.Append);
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector FillInFrom(string text)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.FillInFrom(this, text, WriteMode.Overwrite);
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector InsertFrom(string textToUse)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.FillInFrom(this, textToUse, WriteMode.Insert);
            return this;
        }

        public IAllowBrowserActionOrAnySelector Focus(Selector selector)
        {
            if (null != selector)
                WithSelector(selector);
            // waitfor did not match if returns false
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.Focus(this);
            return this;
        }

        public IAllowBrowserActionOrAnySelector Hide(Selector selector = null)
        {
            if (null != selector)
                WithSelector(selector);
 
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.Hide(this);
 
            return this;
        }

        private string _pendingSetAttributeName;

        public IAllowSetAttributeValue WithAttribute(string attribute)
        {
            _pendingSetAttributeName = attribute;
            return this;
        }

        public IAllowBrowserActionOrAnySelector SetAttribute(string value)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.SetAttribute(this, _pendingSetAttributeName, value);
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector SetValueFrom(Selector selector)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.SetValueFromElement(this, selector);
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector Hover(Selector selector = null)
        {
            return MoveTo(selector);
        }

        public IAllowBrowserActionOrAnySelector MoveTo(Selector selector = null)
        {
            if (null != selector)
                WithSelector(selector);
 
            // waitfor did not match if returns false
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                ElementHelper.MoveTo(this);
            return this;
        }

        public IAllowBrowserActionOrAnySelector StartWorkflow()
        {
            ReadOnlyCollection<GPALElement> tmpElems;
 
            if (true == CurrentUOW.ActionCalled)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Action has already been called. This workflow has already run. Continuing.", this, GPALObjectType.Browser);
                return this;
            }
            else
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Workflow starting, you should be careful if you follow this with actions. CallIf handlers will be invoked again.", this, GPALObjectType.Browser);
 
            if (true == WorkflowSetup(WaitTime.Never < CurrentUOW.WaitForInMs, true))
                for (int pages = 0; pages < CurrentUOW.PageCount; pages++)  // is start workflow is used, then we are using callif/callafter handlers with no main workflow
                {
                    foreach (Selector sel in CurrentUOW.WithSelectorList)
                        if (SelectorType.Selector == sel.SelectorType)
                            tmpElems = ElementHelper.FindWebElements(this, CurrentUOW, sel, out bool matchedAll, out List<GPALElement> matchedElems);
 
                    if (pages < CurrentUOW.PageCount - 1)
                    {
                        if (false == BrowserSettings.AttachedRestApi)
                BrowserHelper.TopBrowser(this, false); // ensure the browser window is on top, can use selenium or hardware (mouse click) topping. firefox will only top with hardware
                        if (false == BrowserHelper.GotoNextPage(this))
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"No next page, breaking loop. Retrieved [{pages + 1}] out of [{CurrentUOW.PageCount}] pages.", this, GPALObjectType.Browser);
                            break;
                        }
                    }
                }
 
            return this;
        }
        public IAllowBrowserActionOrAnySelector GetElements(out List<GPALElement> elements)
        {
            elements = new List<GPALElement>();
 
            if (true == CurrentUOW.ActionCalled)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Action has already been called. Not retrieving elements.", this, GPALObjectType.Browser);
                return this;
            }
 
            if (true == WorkflowSetup(WaitTime.Never < CurrentUOW.WaitForInMs, true))
                for (int pages = 0; pages < CurrentUOW.PageCount; pages++)
                {
                    foreach (Selector sel in CurrentUOW.WithSelectorList)
                        if (SelectorType.Selector == sel.SelectorType)
                        {
                            ElementHelper.FindWebElements(this, CurrentUOW, sel, out bool matchedAll, out List<GPALElement> matchedElems);
                            if (sel.WebSelectorFoundResults != null)
                                elements.AddRange(sel.WebSelectorFoundResults);
                        }
 
                    if (pages < CurrentUOW.PageCount - 1)
                    {
                        BrowserHelper.TopBrowser(this, false);
                        if (false == BrowserHelper.GotoNextPage(this))
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"No next page, breaking loop. Retrieved [{pages + 1}] out of [{CurrentUOW.PageCount}] pages.", this, GPALObjectType.Browser);
                            break;
                        }
                    }
                }
 
            return this;
        }
        public IAllowWithHeaderOrFileActions GetGrid(ref IGPALGrid<string> returnGrid)
        {
            List<ReadOnlyCollection<GPALElement>> columns = new List<ReadOnlyCollection<GPALElement>>();
            List<ReadOnlyCollection<GPALElement>> tmpColumns = new List<ReadOnlyCollection<GPALElement>>();
            var selectorMeta = new List<(string Name, string AttributeName)>();
            List<GPALElement> matchedElems = new List<GPALElement>();
            dynamic foundElements = null;
 
            CurrentUOW.GetGridCalled = true;
            CurrentUOW.RetGrid.Clear();
 
            // Table-native path (engine-agnostic navigation). When a single selector resolves to a
            // <table> or its <tr> rows, the grid is extracted structurally below and we return early.
            // Everything else falls through to the unchanged column-per-selector zip.
            //if (TryGetGridFromTable(ref returnGrid))
            //    return this;
 
            // Content-based row dedup, opt-in via .WithDedupeData(true) on the grid the caller passed in.
            // seenRows holds a key per row's extracted values. We seed it from the rows already in the caller's
            // grid so accumulation across GetGrid calls stays duplicate-free, not just within this one call.
            bool useDedupe = (returnGrid as GPALGrid<string>)?.UseDedupe ?? false;
            var seenRows = new HashSet<string>();
            if (true == useDedupe && null != returnGrid)
                foreach (List<string> existingRow in returnGrid)
                    seenRows.Add(RowDedupeKey(existingRow));
            bool capReached = false;
 
            for (int pages = 0; pages < CurrentUOW.PageCount && !capReached; pages++)
            {
                // check for persistent selectors on each new page
                if (!InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                {
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, "WaitFor did not find the elements for the grid. Not retrieving anything.", this, GPALObjectType.Browser);
                    return this;
                }
 
                tmpColumns.Clear();
                columns.Clear();
                CurrentUOW.MatchedRowIndexes.Clear();
                selectorMeta.Clear();
 
                BrowserHelper.CheckDocumentReady(this);
 
                // iterate over selectors
                foreach (Selector sel in CurrentUOW.WithSelectorList)
                {
                    // cache is no longer used, but maybe the user might wanna look at tham for debugging?
                    sel.WebSelectorFoundResults = null;
                    sel.WebSelectorMatchedResults = null;
 
                    selectorMeta.Add((sel.Name, sel.AttributeName));
 
                    // we have selectors for elements and data/datafunction literals
                    if (SelectorType.Selector == sel.selectorSettings.SelectorType)
                    {
                        bool matchedAll = false;
 
                        foundElements = ElementHelper.FindWebElements(this, CurrentUOW, sel, out matchedAll, out matchedElems);
 
                        // the only instance where matchedElems does not have what we want is if custom match doesn't return any...
                        // if nothing matches, then create a psuedo "NOT FOUND" element for the grid
                        if (matchedElems?.Count > 0)
                        {
                            foreach (GPALElement element in matchedElems)
                                element.AttributeName = sel.AttributeName;
 
                            tmpColumns.Add(new ReadOnlyCollection<GPALElement>(matchedElems));
                        }
                        //else if (foundElements?.Count > 0)    // silly check
                        //    tmpColumns.Add(new ReadOnlyCollection<GPALElement>(foundElements));
                        else
                        {
                            var placeholder = new List<GPALElement>();
                            int rows = CurrentUOW.RowCount > 0 ? CurrentUOW.RowCount : 0;   // set from previous iteration, how many found for that selector?
                            if (0 < rows)
                            {
                                for (int i = 0; i < rows; i++)
                                    placeholder.Add(new GPALElement(new Point(0, 0), new Size(1, 1), $"{GPAL.ErrorPlaceholder} : {sel.Name}", BrowserDriver, "GPALElement") { AttributeName = sel.AttributeName, Browser = this, Selector = sel });
                                tmpColumns.Add(new ReadOnlyCollection<GPALElement>(placeholder));
                            }
                        }
 
                        if (matchedElems?.Count > 0)
                        {
                            CurrentUOW.ColCount++;
 
                            // set rowcount (found elements) for this iteration
                            if (matchedAll)     // found = matched
                                CurrentUOW.RowCount = Math.Max(CurrentUOW.RowCount, matchedElems.Count);
                            else if (matchedElems?.Count > 0)   // matched match criteria
                            {
                                CurrentUOW.PartialMatch = true;
                                int tmpIdx = 0;
                                foreach (var elem in matchedElems)
                                    if ((tmpIdx = matchedElems.IndexOf(elem)) != -1 && !CurrentUOW.MatchedRowIndexes.ContainsKey(tmpIdx))
                                        CurrentUOW.MatchedRowIndexes.Add(tmpIdx, sel.AttributeName);
                                CurrentUOW.RowCount = CurrentUOW.MatchedRowIndexes.Count;
                            }
                            else    // we found no elements but need placeholders if the workflow specifies WithAllThatMatch (int.MaxValue means all, no end, limited by pages hopefully)
                                CurrentUOW.RowCount = int.MaxValue != CurrentUOW.WithAllThatMatch ? CurrentUOW.WithAllThatMatch : 0;
                        }
                    }
                    else if (SelectorType.Data == sel.selectorSettings.SelectorType)    // use literal data
                    {
                        CurrentUOW.ColCount++;
                        GPALElement tmpElement = GPAL.Element;
                        tmpElement.Text = sel.SelectorPath;
                        tmpColumns.Add(new ReadOnlyCollection<GPALElement>(new List<GPALElement>(1) { tmpElement }));
                    }
                    else if (SelectorType.DataFunc == sel.selectorSettings.SelectorType)    // use literal data provided by function
                    {
                        CurrentUOW.ColCount++;
                        GPALElement tmpElement = GPAL.Element;
                        tmpElement.Text = sel.selectorSettings.DataFunction();
                        tmpColumns.Add(new ReadOnlyCollection<GPALElement>(new List<GPALElement>(1) { tmpElement }));
                    }
 
                    // one line per selector instead of one going in and one coming out. Not guarded on finding
                    // something: a selector that found nothing is the case you most want to see in the log
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Grid for [{sel.Name}] page [{pages + 1}] returned [{CurrentUOW.RowCount}] rows of [{tmpColumns.Count()}] columns", this, GPALObjectType.Browser);
                }
 
                if (CurrentUOW.PartialMatch)
                {
                    columns.Clear();
                    foreach (var colElems in tmpColumns)
                    {
                        var newCol = new List<GPALElement>();
                        // the matched indexes come from one selector, but they are applied to every column, and
                        // columns are not all the same length yet - a literal data selector is one element, a
                        // placeholder column can be shorter. an index past the end is always past the end, so
                        // skipping it just leaves the tail short, and the padding loop below fills it out to
                        // targetRows with placeholders exactly as it does for every other short column.
                        foreach (var idx in CurrentUOW.MatchedRowIndexes.Keys)
                            if (idx < colElems.Count)
                                newCol.Add(colElems[idx]);
                        columns.Add(new ReadOnlyCollection<GPALElement>(newCol));
                    }
                }
                else
                    columns = tmpColumns;
 
                // how many elements (rows of data) did we find on this page
                int pageRows = 0 < columns.Count ? columns.Max(c => c.Count) : 0;
                int targetRows = CurrentUOW.RowCount > 0 ? CurrentUOW.RowCount : pageRows;
 
                // apply WithAllThatMatch per-page cap (only when n fits within one page)
                // total-cap and dedup are handled row-by-row during population below
                if (int.MaxValue != CurrentUOW.WithAllThatMatch && CurrentUOW.WithAllThatMatch <= pageRows)
                    targetRows = Math.Min(targetRows, CurrentUOW.WithAllThatMatch);
 
                // how many colums did we get? should be the number of selectors
                // now create our columns to iterate, whatever had the most, that is what we will fill all rows to
                for (int i = 0; i < columns.Count; i++)
                {
                    var col = columns[i].ToList();
                    while (col.Count < targetRows)  // one of the selectors has more rows than this one, so create a placeholder
                    {
                        // for scrollintoview, we need something...
                        var firstNonEmpty = columns.FirstOrDefault(c => c.Count > 0)?.FirstOrDefault();
                        var meta = selectorMeta[i];
                        col.Add(new GPALElement(new Point(0, 0), new Size(1, 1), $"{GPAL.ErrorPlaceholder} : {meta.Name}", BrowserDriver, "GPALElement") { AttributeName = meta.AttributeName, Browser = this, Css = firstNonEmpty?.Css, ElementBackendNodeId = (int)(firstNonEmpty?.ElementBackendNodeId) });
                    }
                    columns[i] = new ReadOnlyCollection<GPALElement>(col);
                }
                CurrentUOW.RowCount = targetRows;
 
                // Populate RetGrid row-major. Dedup (when opted in) is content-based: we key each row by its
                // extracted values, so a row is dropped if its data duplicates one already accumulated (seenRows
                // is seeded from the caller's grid) or one seen earlier this call. This composes with the
                // grid-accumulation contract and needs no per-element attribute hashing.
                // WithAllThatMatch total-cap stops accumulation once we have enough unique rows
                for (int r = 0; r < CurrentUOW.RowCount && !capReached; r++)
                {
                    var row = new List<string>();
                    for (int c = 0; c < columns.Count; c++)
                    {
                        var elem = columns[c][r];
 
                        // whatever item we are scraping, bring it into view
                        // for UX
                        if (true == BrowserSettings.ScrollIntoView)
                            if (false == BrowserSettings.UseHeadless && false == "GPALElement".Equals(elem.TagName))
                                if (true == UseOttoMagic)
                                {
                                    if (false == string.IsNullOrEmpty(elem.Css))
                                        MagicHelper.ScrollIntoView(elem.Css);
                                }
                                else if (true == UsePuppeteer)
                                    PuppeteerClient.ScrollIntoView(elem).Execute();
                                else
                                    ElementHelper.ScrollIntoView(this, elem);
 
                        if (!string.IsNullOrEmpty(elem.AttributeName))
                            row.Add(elem.GetAttribute(elem.AttributeName));
                        else if (null != elem.TagName)
                        {
                            switch (elem.TagName?.ToLower())
                            {
                                case "a": row.Add(elem.GetAttribute("href")); break;
                                case "img": row.Add(elem.GetAttribute("src")); break;
                                default: row.Add(elem.Text); break;
                            }
                        }
                        else // literal data
                            row.Add(elem.Text);
                    }
 
                    // DEDUPE: skip this row if its data matches one already in the grid or seen this call
                    if (true == useDedupe && false == seenRows.Add(RowDedupeKey(row)))
                        continue;
 
                    CurrentUOW.RetGrid.AddRow(row);
                    if (int.MaxValue != CurrentUOW.WithAllThatMatch && CurrentUOW.WithAllThatMatch > pageRows
                        && CurrentUOW.RetGrid.Count() >= CurrentUOW.WithAllThatMatch)
                        capReached = true;
                }
 
                if (!capReached && pages < CurrentUOW.PageCount - 1)
                {
                    BrowserHelper.TopBrowser(this, false);
                    if (!BrowserHelper.GotoNextPage(this))
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"No next page, breaking loop. Retrieved [{pages + 1}] out of [{CurrentUOW.PageCount}] pages.", this, GPALObjectType.Browser);
                        break;
                    }
                }
            }
 
            // Hand back the results without taking ownership of a grid the caller passed in:
            //  - null in  -> the caller just wants a grid, so give them GPAL's internal one.
            //  - grid in   -> the caller owns it (and may have configured it, e.g. WithDedupeData); add our rows
            //                 into THEIR grid and keep their reference so repeated GetGrid calls can accumulate.
            if (null == returnGrid)
                returnGrid = (IGPALGrid<string>)CurrentUOW.RetGrid;
            else
                returnGrid.Add((IGPALGrid<string>)CurrentUOW.RetGrid);
 
            return this;
        }

        private static string RowDedupeKey(List<string> row)
        {
            return string.Join("\u001F", row);
        }
 
        // ─────────────────────────────────────────────────────────────────────────────────────
        // Table-native GetGrid (engine-agnostic). Wedged into GetGrid above; does not touch the
        // column-per-selector zip. A single selector that resolves to a <table> (or its <tr> rows)
        // is walked structurally with the same ElementHelper.FindWebElements dispatch used
        // everywhere else, honoring colspan/rowspan, deriving headers from <th>, filling gaps with
        // GPAL.ErrorPlaceholder, and populating CurrentUOW.RetGrid so SaveTo*/pagination just work.
        // NOTE: OttoMagic/Puppeteer scope a child query by the parent element's Css, so the table
        // and its rows must carry a usable Css for those engines (Selenium scopes by IWebElement).
        // NOTE: deeply nested tables are a known limitation (descendant tr/td/th matches inner cells).
        // ─────────────────────────────────────────────────────────────────────────────────────
        private bool TryGetGridFromTable(ref IGPALGrid<string> returnGrid)
        {
            // Only a single element-selector request can be table-native. Multiple selectors are the
            // column-per-selector zip, and literal Data/DataFunc columns are never tables.
            if (1 != CurrentUOW.WithSelectorList.Count
                || SelectorType.Selector != CurrentUOW.WithSelectorList[0].selectorSettings.SelectorType)
                return false;
 
            Selector sel = CurrentUOW.WithSelectorList[0];
 
            // Probe once to learn the target type. If it is not a <table> or a set of <tr> rows,
            // this is not table-native; return false and let the untouched column-zip path run.
            bool matchedAll; List<GPALElement> matchedElems;
            ReadOnlyCollection<GPALElement> seeds = ElementHelper.FindWebElements(this, CurrentUOW, sel, out matchedAll, out matchedElems);
            string tag = (null != seeds && seeds.Count > 0) ? seeds[0].TagName?.ToLowerInvariant() : null;
            bool isTable = "table" == tag;
            bool isRows = "tr" == tag;
            if (!isTable && !isRows)
                return false;
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"GetGrid: [{sel.Name}] resolved to <{tag}>; extracting the table structurally.", this, GPALObjectType.Browser);
 
            var seenRows = new HashSet<string>();
            List<string> headers = null;
            bool capReached = false;
            int cap = (int.MaxValue == CurrentUOW.WithAllThatMatch)
                        ? int.MaxValue : CurrentUOW.WithAllThatMatch;
 
            for (int page = 0; page < CurrentUOW.PageCount && !capReached; page++)
            {
                if (page > 0)
                {
                    seeds = ElementHelper.FindWebElements(this, CurrentUOW, sel, out matchedAll, out matchedElems);
                    if (null == seeds || 0 == seeds.Count)
                        break;
                }
                BrowserHelper.CheckDocumentReady(this);
 
                // Gather the body row elements for this page (and headers, once).
                var bodyRows = new List<GPALElement>();
                if (isTable)
                {
                    foreach (GPALElement table in seeds)
                    {
                        var headerRows = FindWithin(table, "thead tr");
                        var tableBody = FindWithin(table, "tbody tr");
                        if (0 == tableBody.Count && 0 == headerRows.Count)
                        {
                            // no thead/tbody: treat a leading all-<th> row as the header
                            var all = FindWithin(table, "tr");
                            if (all.Count > 0 && RowIsAllHeader(all[0]))
                            {
                                headerRows.Add(all[0]);
                                tableBody.AddRange(all.Skip(1));
                            }
                            else
                                tableBody.AddRange(all);
                        }
                        if (null == headers && headerRows.Count > 0)
                        {
                            var hmatrix = BuildTableMatrix(headerRows);
                            if (hmatrix.Count > 0)
                                headers = hmatrix[0];   // first header row, span-aligned to the data
                        }
                        bodyRows.AddRange(tableBody);
                    }
                }
                else // the seeds themselves are the rows
                {
                    bodyRows.AddRange(seeds);
                }
 
                // Build a rectangular, span-aware matrix of cell text for these rows.
                List<List<string>> matrix = BuildTableMatrix(bodyRows);
 
                foreach (List<string> row in matrix)
                {
                    string key = string.Join("␁", row);
                    if (!seenRows.Add(key))
                        continue;   // duplicate row from a prior page/scroll pass
                    CurrentUOW.RetGrid.AddRow(row);
                    if (CurrentUOW.RetGrid.Count() >= cap)
                    {
                        capReached = true;
                        break;
                    }
                }
 
                if (!capReached && page < CurrentUOW.PageCount - 1)
                {
                    BrowserHelper.TopBrowser(this, false);
                    if (!BrowserHelper.GotoNextPage(this))
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"GetGrid: no next page after page [{page + 1}].", this, GPALObjectType.Browser);
                        break;
                    }
                }
            }
 
            if (null != headers && headers.Count > 0
                && (null == CurrentUOW.HeaderList || 0 == CurrentUOW.HeaderList.Count))
                CurrentUOW.HeaderList = headers;
 
            returnGrid = (IGPALGrid<string>)CurrentUOW.RetGrid;
            return true;
        }
 
        // Engine-agnostic scoped query: elements matching css *within* a parent element, using the
        // same ElementHelper.FindWebElements dispatch (Selenium/Puppeteer/OttoMagic) as the rest of GPAL.
        private List<GPALElement> FindWithin(GPALElement parent, string css)
        {
            Selector scoped = GPAL.Selector.WithCSS(css).ToGPALObject();
            bool matchedAll; List<GPALElement> matchedElems;
            ReadOnlyCollection<GPALElement> found = ElementHelper.FindWebElements(this, CurrentUOW, scoped, out matchedAll, out matchedElems, webElement: parent);
            return (null != found) ? found.ToList() : new List<GPALElement>();
        }
 
        // True when a <tr>'s cells are all <th> (a header row when there is no <thead>).
        private bool RowIsAllHeader(GPALElement tr)
        {
            var cells = FindWithin(tr, "td, th");
            if (0 == cells.Count)
                return false;
            foreach (GPALElement c in cells)
                if (!"th".Equals(c.TagName, StringComparison.OrdinalIgnoreCase))
                    return false;
            return true;
        }
 
        // Turn <tr> elements into a rectangular grid of cell strings, honoring colspan and rowspan.
        // Spanned cells repeat their text across covered positions; structural gaps and short rows
        // are filled with GPAL.ErrorPlaceholder so every row has the same column count.
        private List<List<string>> BuildTableMatrix(List<GPALElement> rowElems)
        {
            var matrix = new List<List<string>>();
            // pending rowspans: column index -> (remaining future rows, text)
            var pending = new Dictionary<int, KeyValuePair<int, string>>();
 
            foreach (GPALElement tr in rowElems)
            {
                var cells = FindWithin(tr, "td, th");
                var row = new List<string>();
                int col = 0, ci = 0;
 
                while (ci < cells.Count || pending.Keys.Any(k => k >= col))
                {
                    KeyValuePair<int, string> carry;
                    if (pending.TryGetValue(col, out carry))
                    {
                        row.Add(carry.Value);
                        int left = carry.Key - 1;
                        if (left > 0) pending[col] = new KeyValuePair<int, string>(left, carry.Value);
                        else pending.Remove(col);
                        col++;
                        continue;
                    }
 
                    if (ci < cells.Count)
                    {
                        GPALElement cell = cells[ci++];
                        string text = CellText(cell);
                        int cspan = ParseSpan(cell, "colspan");
                        int rspan = ParseSpan(cell, "rowspan");
                        for (int c = 0; c < cspan; c++)
                        {
                            row.Add(text);
                            if (rspan > 1) pending[col] = new KeyValuePair<int, string>(rspan - 1, text);
                            col++;
                        }
                    }
                    else
                    {
                        // a rowspan carry sits further right but there is no cell here: keep alignment
                        row.Add(GPAL.ErrorPlaceholder);
                        col++;
                    }
                }
 
                matrix.Add(row);
            }
 
            // pad every row to the widest so the grid is rectangular
            int width = 0;
            foreach (var r in matrix) if (r.Count > width) width = r.Count;
            foreach (var r in matrix) while (r.Count < width) r.Add(GPAL.ErrorPlaceholder);
 
            return matrix;
        }
 
        private static int ParseSpan(GPALElement cell, string attr)
        {
            int n;
            string v = cell.GetAttribute(attr);
            return (int.TryParse(v, out n) && n > 0) ? n : 1;
        }
 
        // Cell text read the same way the column-zip path reads cells, so both paths agree.
        private string CellText(GPALElement cell)
        {
            if (!string.IsNullOrEmpty(cell.AttributeName))
                return cell.GetAttribute(cell.AttributeName);
            switch (cell.TagName?.ToLowerInvariant())
            {
                case "a": return cell.GetAttribute("href");
                case "img": return cell.GetAttribute("src");
                default: return cell.Text ?? string.Empty;
            }
        }

        public IAllowCallTemplate WithCallFilter(string urlFragment)
        {
            BrowserSettings.CallFilter = urlFragment;
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector CaptureCallTemplate(out GPALRequest template)
        {
            template = null;
 
            // waiting is pointless where nothing is recording, so this says so now rather than in thirty seconds
            if (false == CanCaptureCalls)
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"No template: capturing calls is not available on [{BrowserSettings.AutomationEngine}]", this, GPALObjectType.Browser);
 
                return this;
            }
 
            CaptureCalls(true);
 
            GPALCall found = null;
            Stopwatch waited = Stopwatch.StartNew();
 
            while (null == found && CallTemplateWaitMilliseconds > waited.ElapsedMilliseconds)
            {
                CaptureCalls(out List<GPALCall> calls);
 
                // the last match, not the last call. the filter narrows what is recorded from here on, but
                // anything captured before it was set is still in the list, so it is applied again here
                foreach (GPALCall call in calls)
                    if (true == string.IsNullOrEmpty(BrowserSettings.CallFilter) || true == call.Url?.Contains(BrowserSettings.CallFilter))
                        found = call;
 
                if (null == found)
                    System.Threading.Thread.Sleep(250);
            }
 
            waited.Stop();
 
            if (null == found)
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"No call matching [{BrowserSettings.CallFilter}] in [{waited.ElapsedMilliseconds}]ms, so there is no template", this, GPALObjectType.Browser);
 
                return this;
            }
 
            template = (GPALRequest)GPAL.Request
                .WithPath(found.Url)
                .WithHttpMethod("POST".Equals(found.Method, StringComparison.OrdinalIgnoreCase) ? Enums.HttpVerb.Post : Enums.HttpVerb.Get)
                .WithBody(found.PostData)
                .WithName(BrowserSettings.CallFilter ?? found.Url);
 
            // a browser sets these for itself and a fetch refuses to be told otherwise, so a template carrying
            // them would be a template that cannot be sent
            foreach (KeyValuePair<string, string> header in found.Headers)
                if (false == BrowserHelper.BrowserOwnedHeader(header.Key))
                    template.WithHeader($"{header.Key}: {header.Value}");
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Template taken from the page's own [{found.Method}] to [{found.Url}] after [{waited.ElapsedMilliseconds}]ms, with [{template.Headers.Count}] of its headers", this, GPALObjectType.Browser);
 
            return this;
        }
 
        // how long to wait for the page to make the call a template is wanted from
        private const int CallTemplateWaitMilliseconds = 30_000;
 
        // puppeteer reports requests over CDP, the extension watches with webRequest, and selenium records them
        // in the page and needs a driver to inject through. all three can, by different means
        private bool CanCaptureCalls => true == BrowserSettings.UsePuppeteer
                                     || true == BrowserSettings.UseOttoMagic
                                     || null != BrowserSettings.BrowserDriver;
 
        public IAllowBrowserActionOrAnySelector CaptureCalls(bool capture = true)
        {
            // the browser reports its requests over CDP, which is Puppeteer's channel. selenium would need CDP
            // wired up the same way and the extension would need its own webRequest listener, so this says so
            // rather than recording nothing and looking like a page that asked for nothing
            // said once per browser, not once per ask. a workflow waiting for a template asks in a loop, and an
            // engine that cannot capture would otherwise say so a hundred times while it waits for nothing
            if (false == CanCaptureCalls)
            {
                if (false == BrowserSettings.CaptureUnavailableReported)
                {
                    BrowserSettings.CaptureUnavailableReported = true;
 
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Capturing calls is not available on [{BrowserSettings.AutomationEngine}]. It needs a browser that reports its requests, and this one has none started", this, GPALObjectType.Browser);
                }
 
                return this;
            }
 
            if (true == BrowserSettings.UseOttoMagic)
            {
                // the extension watches with webRequest, from outside the page. nothing is injected, so this
                // survives a navigation on its own and sees the requests no page script issued
                BrowserSettings.CaptureCalls = capture;
 
                MagicHelper.CaptureCalls(capture, BrowserSettings.CallFilter);
            }
            else if (false == BrowserSettings.UsePuppeteer && null != BrowserSettings.BrowserDriver)
            {
                // selenium has no CDP event stream, so the page records for itself. registered to run on every
                // document where chromium allows it, and injected into the page in hand either way, so saying
                // this after a navigation still starts recording
                BrowserSettings.CaptureCalls = capture;
 
                if (true == capture)
                {
                    if (BrowserType.Chrome == BrowserSettings.BrowserType || BrowserType.Edge == BrowserSettings.BrowserType)
                        BrowserHelper.SeleniumAddScriptToEvaluateOnNewDocument(BrowserSettings, BrowserHelper.CallRecorderScript);
 
                    BrowserHelper.ExecuteJavaScript(this, BrowserHelper.CallRecorderScript);
                }
            }
            else
            {
                // recorded as a setting first. saying this before going anywhere is the ordinary way to write it,
                // and reaching for the communicator now would build one to talk to a browser that has not started
                BrowserSettings.CaptureCalls = capture;
 
                // a communicator that exists is a browser that is up, so it can be told now. asking for one that
                // does not exist would build it, and the Puppeteer engine has no Selenium driver to test instead
                if (true == BrowserSettings.HasPuppeteerCommunicator)
                    BrowserSettings.PuppeteerCommunicator.CaptureCalls(capture).GetAwaiter().GetResult();
            }
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector CaptureCalls(out List<GPALCall> calls)
        {
            CaptureCalls(true);
 
            calls = true == BrowserSettings.UsePuppeteer
                ? true == BrowserSettings.HasPuppeteerCommunicator
                    ? new List<GPALCall>(BrowserSettings.PuppeteerCommunicator.CapturedCalls)
                    : new List<GPALCall>()
                : true == BrowserSettings.UseOttoMagic
                    ? ExtensionRecordedCalls()
                    : RecordedCalls();
 
            // debug, not info. taking a snapshot is something a workflow does in a loop while it waits, and a
            // line per attempt buries the run. the template says the thing worth reading
            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Handing back [{calls.Count}] captured calls.", this, GPALObjectType.Browser);
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector ClearCapturedCalls()
        {
            if (true == BrowserSettings.UseOttoMagic)
                // the recorder keeps recording, and clears what it holds on the way through
                MagicHelper.CaptureCalls(BrowserSettings.CaptureCalls, BrowserSettings.CallFilter, true);
            else if (true == BrowserSettings.UsePuppeteer)
            {
                if (true == BrowserSettings.HasPuppeteerCommunicator)
                    BrowserSettings.PuppeteerCommunicator.ClearCapturedCalls();
            }
            else if (null != BrowserSettings.BrowserDriver)
                // the page holds its own, so the page is where it is emptied
                BrowserHelper.ExecuteJavaScript(this, "window.__gpalCalls = []; return true;");
 
            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, "Forgot the calls captured so far.", this, GPALObjectType.Browser);
 
            return this;
        }
 
        // webRequest and CDP name the same kinds of request differently: main_frame against Document,
        // xmlhttprequest against XHR. a workflow that filters on the type should not have to know which engine
        // recorded it, so the extension's names are put into CDP's before anyone sees them
        private static string CdpResourceType(string webRequestType)
        {
            string retVal = webRequestType;
 
            switch (webRequestType)
            {
                case "main_frame":
                case "sub_frame": retVal = "Document"; break;
                case "xmlhttprequest": retVal = "XHR"; break;
                case "script": retVal = "Script"; break;
                case "image":
                case "imageset": retVal = "Image"; break;
                case "stylesheet": retVal = "Stylesheet"; break;
                case "font": retVal = "Font"; break;
                case "media": retVal = "Media"; break;
                case "websocket": retVal = "WebSocket"; break;
                case "ping": retVal = "Ping"; break;
                case "csp_report": retVal = "CSPViolationReport"; break;
 
                // webRequest puts both XHR and fetch under xmlhttprequest, where CDP tells them apart. a
                // workflow filtering for the site's own api wants both either way, so nothing is lost.
                // "other" is the catch-all for what fits nowhere else, and is left saying exactly that
                case "other": retVal = "Other"; break;
            }
 
            return retVal;
        }
 
        // what the extension recorded, read back as the same shape CDP produces. webRequest watches the browser
        // rather than the page, so unlike the selenium recorder this holds the documents, scripts and images too
        private List<GPALCall> ExtensionRecordedCalls()
        {
            List<GPALCall> retVal = new List<GPALCall>();
 
            string json = MagicHelper.GetCapturedCalls();
 
            if (true == string.IsNullOrWhiteSpace(json)) return retVal;
 
            json = json.Trim();
 
            // native messaging hands the extension's answer over as a json string, so what arrives here is
            // sometimes the object and sometimes that object encoded inside a string. unwrap the second before
            // parsing, because the escaped quotes inside it are not something JObject.Parse will take
            if (true == json.StartsWith("\""))
                json = Newtonsoft.Json.JsonConvert.DeserializeObject<string>(json);
 
            Newtonsoft.Json.Linq.JObject snapshot;
 
            // a snapshot that cannot be read is the one failure that looks exactly like a page that made no
            // calls, so it says what it got rather than handing back an empty list
            try
            {
                snapshot = Newtonsoft.Json.Linq.JObject.Parse(json);
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"The captured calls could not be read. [{json.Length}] characters starting [{json.Substring(0, Math.Min(120, json.Length))}]", this, GPALObjectType.Browser, ex);
 
                return retVal;
            }
 
            // the recorder keeps the newest and counts what fell off, so a workflow that finds nothing can tell
            // a page that never called it from a page that called too much
            int dropped = (int?)snapshot["dropped"] ?? 0;
 
            if (0 < dropped)
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"The recorder dropped [{dropped}] older calls to stay within its limit", this, GPALObjectType.Browser);
 
            foreach (Newtonsoft.Json.Linq.JObject recorded in snapshot["calls"] ?? new Newtonsoft.Json.Linq.JArray())
            {
                GPALCall call = new GPALCall
                {
                    Url = recorded["url"]?.ToString(),
                    Method = recorded["method"]?.ToString(),
                    ResourceType = CdpResourceType(recorded["type"]?.ToString()),
                    PostData = recorded["postData"]?.ToString(),
                    Initiator = recorded["initiator"]?.ToString(),
                    Status = (int?)recorded["status"] ?? 0
                };
 
                if (recorded["headers"] is Newtonsoft.Json.Linq.JObject headers)
                    foreach (KeyValuePair<string, Newtonsoft.Json.Linq.JToken> header in headers)
                        call.Headers[header.Key] = header.Value?.ToString();
 
                retVal.Add(call);
            }
 
            return retVal;
        }
 
        // what the page recorded for itself, read back as the same shape CDP produces. the browser's own requests
        // for documents, scripts and images never went through fetch or XMLHttpRequest, so they are not here
        private List<GPALCall> RecordedCalls()
        {
            List<GPALCall> retVal = new List<GPALCall>();
 
            if (null == BrowserSettings.BrowserDriver) return retVal;
 
            string json = BrowserHelper.ExecuteJavaScript(this, "return JSON.stringify(window.__gpalCalls || []);");
 
            if (true == string.IsNullOrWhiteSpace(json)) return retVal;
 
            foreach (Newtonsoft.Json.Linq.JObject recorded in Newtonsoft.Json.Linq.JArray.Parse(json))
            {
                GPALCall call = new GPALCall
                {
                    Url = recorded["url"]?.ToString(),
                    Method = recorded["method"]?.ToString(),
                    ResourceType = recorded["type"]?.ToString(),
                    PostData = recorded["postData"]?.ToString(),
                    Initiator = "script",
                    Status = (int?)recorded["status"] ?? 0
                };
 
                if (recorded["headers"] is Newtonsoft.Json.Linq.JObject headers)
                    foreach (KeyValuePair<string, Newtonsoft.Json.Linq.JToken> header in headers)
                        call.Headers[header.Key] = header.Value?.ToString();
 
                retVal.Add(call);
            }
 
            return retVal;
        }
 
        public IAllowWithHeaderOrFileActions Fetch(GPALRequest request)
        {
            CurrentUOW.FetchResults.Clear();
 
            if (null == request)
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, "No request given to fetch.", this, GPALObjectType.Browser);
            else
                BrowserHelper.FetchWithTokens(this, request);
 
            InAction(false);
            return this;
        }
        public IAllowBrowserActionOrAnySelector SaveTo(ref string data)
        {
            List<string> values = new List<string>();
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Handing back [{CurrentUOW.FetchResults.Count}] fetch responses.", this, GPALObjectType.Browser);
 
            // the array has to parse whatever came back, or saving it was pointless. a json body is spliced in as
            // it stands, and anything else, an html deny page or a body carrying the http warning in front of it,
            // goes in as a json string. still readable, still an array
            foreach (string body in CurrentUOW.FetchResults)
                values.Add(true == StringFormatDetector.IsJson(body.Trim()) ? body : Newtonsoft.Json.JsonConvert.SerializeObject(body));
 
            data = "[" + string.Join(",", values) + "]";
            return this;
        }
        public IAllowBrowserActionOrAnySelector SaveTo(GPALFile file)
        {
            //GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Saving [{CurrentUOW.RetGrid?.Rows ?? 0}] rows of page data to [{file?.Filename}].", this, GPALObjectType.Browser);
 
            var converter = GPAL.Converter.WithInput(CurrentUOW.RetGrid).WithColumnNames(ResolveHeaderColumns(file));
            // Naming headers via WithHeader is clear intent to write them; declare it so the output emits a
            // header row unless the output file explicitly turns it off (WithFirstLineIsColumnNames(false)).
            if (null != CurrentUOW.HeaderList && 0 < CurrentUOW.HeaderList.Count)
                converter.WithFirstLineHasColumnNames(true);
            converter.SaveTo(file);
            return this;
        }

        public IAllowBrowserActionOrAnySelector AppendTo(GPALFile file)
        {
            //GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Appending [{CurrentUOW.RetGrid?.Rows ?? 0}] rows of page data to [{file?.Filename}].", this, GPALObjectType.Browser);
 
            GPAL.Converter.WithInput(CurrentUOW.RetGrid).WithColumnNames(ResolveHeaderColumns(file)).AppendTo(file);
            return this;
        }
 
        // Resolve the output column names for a grid save, handed to the Converter as column metadata
        // (WithColumnNames), NOT injected as a data row. Precedence, highest first, with a short higher list
        // topped up from the next level down:
        //   1. CurrentUOW.HeaderList  - the workflow's .WithHeader columns (override)
        //   2. the output GPALFile's ColumnList - the file's own columns (wildcard/.Next replication of the
        //      first entry happens at expansion time, so entry 0 is representative here)
        //   3. selector names - the ultimate global, always defined
        // Whether the names are written is governed by the output file's FirstLineIsColumnNames.
        private List<string> ResolveHeaderColumns(GPALFile file)
        {
            List<string> selectorNames = CurrentUOW.WithSelectorList?.Where(s => null != s.Name).Select(s => s.Name).ToList()
                ?? new List<string>();
 
            List<string> fileColumns = new List<string>();
            IGPALGrid<string> columnList = ((IGPALFileInternal)file).FileSettings.ColumnList;
            if (null != columnList && 0 < columnList.Count() && null != columnList[0])
                fileColumns = new List<string>(columnList[0]);
 
            List<string> resolved = (null != CurrentUOW.HeaderList && 0 < CurrentUOW.HeaderList.Count)
                ? new List<string>(CurrentUOW.HeaderList)
                : new List<string>();
            for (int i = resolved.Count; i < fileColumns.Count; i++)
                resolved.Add(fileColumns[i]);
            for (int i = resolved.Count; i < selectorNames.Count; i++)
                resolved.Add(selectorNames[i]);
            return resolved;
        }
        public IAllowAfterWaitFor WaitFor(WaitTime waitForTimeInMs)
        {
            // we are done waiting on any selectors since we performed an action
            // so, we just wait/sleep
            if (true == CurrentUOW.ActionCalled)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Sleeping for [{waitForTimeInMs}] ms", this, GPALObjectType.Browser);
                Thread.Sleep(waitForTimeInMs);
            }
            else
                CurrentUOW.WaitForInMs = waitForTimeInMs;
 
            return this;
        }

        public IAllowAfterWaitFor WaitFor(Selector waitForSelector)
        {
            bool weSetWaitFor = false;
            WaitTime safe = WaitTime.Never;
 
            // never (can't wait on a selector 'never') so we will set a time and restore
            if (WaitTime.Never == CurrentUOW.WaitForInMs)
            {
                safe = CurrentUOW.WaitForInMs;
                CurrentUOW.WaitForInMs = WaitTime.Immediate;
                weSetWaitFor = true;
            }
 
            WithSelector(waitForSelector);  // this will create a new UOW with waittimeinms = never (or GPAL.WaitFor global setting)
            WorkflowSetup(true, true);  // this will set CurrentUOW.ActionCalled = true, so this is it's own action
 
            if (true == weSetWaitFor)
                CurrentUOW.WaitForInMs = safe;
 
            return this;
        }

        public IAllowAfterWaitFor WaitFor(ElementState elementState)
        {
            if (ElementState.NotSet != elementState)
            {
                CurrentUOW.WaitForElementState = elementState;
 
                if (true == CurrentUOW.ActionCalled)    // do this now, we are running the workflow
                    ElementHelper.WaitFor(this, CurrentUOW.WaitForInMs, out bool matchedAll);
            }
            else
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, "ElementState NOTSET is not a valid wait condition. Please select a valid ElementState.", null, GPALObjectType.None);
 
            return this;
        }

        public IAllowAfterWaitForAndSizeControl WaitForWindow(string waitForTitle)
        {
            FormHelper.WaitForWindow(waitForTitle, BrowserSettings.WaitForWindowTimeoutInSeconds);
            return this;
        }

        public IAllowAfterWaitForAndSizeControl WaitForWindowRegex(string waitForTitleRegex)
        {
            FormHelper.WaitForWindow(waitForTitleRegex, BrowserSettings.WaitForWindowTimeoutInSeconds);
            return this;
        }

        public IAllowBrowserActionOrAnySelector WithPageOrientation(PageOrientation pageOrientation)
        {
            BrowserSettings.PageOrientation = pageOrientation;
            return this;
        }
        // todo: iterate over UOW selectors
        // GPAL.Browser.GoTo("ebay.com").WithSelector(selectorToPrint).PrintToPDF(GPAL.FileFor("filename.pdf")); // print element and contents
        // GPAL.Browser.GoTo("ebay.com").PrintToPDF(GPAL.FileFor("filename.pdf")); // print whole page
        public IAllowBrowserActionOrAnySelector PrintToPDF(GPALFile gpalFile)
        {
            string fullHtml = null;
            // If no .withselector/selector set, print the whole page
            string selectorPath = string.IsNullOrEmpty(CurrentUOW.CurrentSelector?.SelectorPaths[0].SelectorPath)
                ? "html"
                : CurrentUOW.CurrentSelector?.SelectorPaths[0].SelectorPath;
 
            if (true == BrowserSettings.UseOttoMagic)
                fullHtml = MagicHelper.GetContentAndCss(selectorPath);
            else
            {
                string elementContent = BrowserHelper.ExecuteJavaScriptObj(
                    $@"var div = document.querySelector('{selectorPath}'); return div.outerHTML;",
                    BrowserSettings.Browser
                ).ToString();
 
                // Inject the div content & CSS used
                string cssContent = BrowserHelper.ExecuteJavaScriptObj($@"
                function getAllCss() {{
                        var css = '';
                        var stylesheets = document.styleSheets;
                        for (var i = 0; i < stylesheets.length; i++) {{
                            try {{
                                var rules = stylesheets[i].cssRules;
                                for (var j = 0; j < rules.length; j++) {{
                                    css += rules[j].cssText + ' ';
                                }}
                            }} catch (e) {{ /* Ignore cross-origin stylesheet errors */ }}
                        }}
                        return css;
                    }}
 
                    function getElementCss(element) {{
                        var css = '';
                        var style = window.getComputedStyle(element);
                        for (var i = 0; i < style.length; i++) {{
                            css += style[i] + ': ' + style.getPropertyValue(style[i]) + '; ';
                        }}
                        return css;
                    }}
 
                    var allCss = getAllCss();
                    var div = document.querySelector('{selectorPath}');
                    var elementCss = getElementCss(div);
 
                    return '<style>' + allCss + ' ' + elementCss + '</style>';
                ", BrowserSettings.Browser).ToString();
                fullHtml = $@"<html><head>{cssContent}</head><body>{elementContent}</body></html>";
            }
 
#if DONOTUSECDP
        #region NEW get the browser to print to pdf without CDP
            // Step 1: Combine content into full HTML
            string outputHtmlPath = $"{gpalFile.Filename}.html";
 
            // Step 2: Save the HTML to a file
            File.WriteAllText(outputHtmlPath, fullHtml);
 
            // Step 3: Create JSON configuration file for PDF options
            string pdfOptionsPath = $"{gpalFile.Filename}_pdf_options.json";
            string pdfOptionsJson = @"{
                ""printBackground"": true,
                ""landscape"":" + (PageOrientation.Landscape == BrowserSettings.PageOrientation ? "true" : "false") 
                +@"
                ""marginTop"": 0,
                ""marginBottom"": 0,
                ""marginLeft"": 0,
                ""marginRight"": 0
            }";
            File.WriteAllText(pdfOptionsPath, pdfOptionsJson);
 
            // Step 4: Launch browser to print the HTML file to PDF
            List<string> addlArgs = new List<string>
            {
                "--no-pdf-header-footer",
                $"--print-to-pdf=\"{gpalFile.Filename}\"",
                $"--print-to-pdf-options=\"{pdfOptionsPath}\""
            };
 
            // Use file:// URL to load the local HTML file
            string fileUrl = $"file:///{Path.GetFullPath(outputHtmlPath).Replace("\\", "/")}";
            System.Diagnostics.Process pdfPrinter = MagicHelper.LaunchBrowser(this, fileUrl, addlArgs);
 
            // Wait for natural exit instead of blind sleep + kill
            bool exitedCleanly = pdfPrinter.WaitForExit(30000);   // generous timeout, e.g. 30s
 
            if (!exitedCleanly)
            {
                try
                {
                    if (!pdfPrinter.HasExited)
                        pdfPrinter.Kill();
                }
                catch { }
 
                GPAL.PublishSimpleEvent(GPALEventType.WARNING,
                    "PrintToPDF process did not exit cleanly (timeout)",
                    BrowserSettings.Browser, GPALObjectType.Browser);
            }
            else if (pdfPrinter.ExitCode != 0)
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING,
                    $"PrintToPDF exited with code [{pdfPrinter.ExitCode}]",
                    BrowserSettings.Browser, GPALObjectType.Browser);
            }
 
            // Step 5: Clean up temporary files
            try
            {
                File.Delete(outputHtmlPath);
                File.Delete(pdfOptionsPath);
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                // Log warning if file deletion fails (e.g., file in use)
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Failed to delete temporary files", BrowserSettings.Browser, GPALObjectType.Browser, ex);
            }
        #endregion NEW get the browser to print to pdf without CDP
#else
    #region OLD use CDP to page.PrintToPDF a window
    BrowserHelper.ExecuteJavaScript(@"
        var newWindow = window.open('', '_blank');
        newWindow.document.open();
        newWindow.document.write('<html><head>' + arguments[0] + '</head><body>' + arguments[1] + '</body></html>');
        newWindow.document.close();
        newWindow.focus();
    ", browserSettings.Browser, new { cssContent, elementContent });
 
    // Wait for the new window to load
    System.Threading.Thread.Sleep(2000);
 
    // Switch to the new window
    var windows = browserSettings.BrowserDriver.WindowHandles;
    browserSettings.BrowserDriver.SwitchTo().Window(windows[windows.Count - 1]);
 
    // Define print settings
    var printSettings = new Dictionary<string, object>
    {
        { "printBackground", true },
        { "landscape", false }
    };
 
    // Execute the print command
    dynamic result = null;
 
    if (BrowserType.Chrome == browserSettings.BrowserType)
        result = ((OpenQA.Selenium.Chromium.ChromiumDriver)browserSettings.BrowserDriver).ExecuteCdpCommand("Page.printToPDF", printSettings);
    else if (BrowserType.Edge == browserSettings.BrowserType)
        result = ((OpenQA.Selenium.Edge.EdgeDriver)browserSettings.BrowserDriver).ExecuteCdpCommand("Page.printToPDF", printSettings);
    else
        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{browserSettings.BrowserType}] does not support Page.printToPDF", browserSettings.Browser, GPALObjectType.Browser);
 
    var resultDict = result as IDictionary<string, object>;
 
    if (resultDict != null && resultDict.ContainsKey("data"))
    {
        // Decode the PDF data
        var pdfData = Convert.FromBase64String(resultDict["data"].ToString());
        // Save the PDF to a file
        System.IO.File.WriteAllBytes(gpalFile.Filename, pdfData);
    }
 
    browserSettings.BrowserDriver.Close();
    windows = browserSettings.BrowserDriver.WindowHandles;
    browserSettings.BrowserDriver.SwitchTo().Window(windows[windows.Count - 1]);
    #endregion OLD use CDP to page.PrintToPDF a window
#endif
            return this;
        }
        /// </summary>
        /// <param name="textToSend">Text to type</param>
        /// <returns>Fluent interface to write your workflow</returns>
        public IAllowBrowserActionOrAnySelector SendString(string textToSend)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                ElementHelper.SendString(this, textToSend);
            }
 
            return this;
        }
        public IAllowBrowserActionOrAnySelector PressModifierKey(ModifierKeys modifierKeys)
        {
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Pressed [{GetModifierKeys(modifierKeys)}]", this, GPALObjectType.Browser);
 
            if (ModifierKeys.NONE != modifierKeys)
                if (false == BrowserSettings.UseHeadless && (true == BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware))
                {
                    HardwareHelper.PressModifierKey(modifierKeys);
                }
                else if (true == UseOttoMagic)
                {
                    MagicHelper.PressModifierKey(modifierKeys);
                }
                else if (true == UsePuppeteer || (true == BrowserSettings.UseHeadless && (true == BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware)))
                {
                    PuppeteerClient.PressModifierKey(modifierKeys);
                }
                else
                {
                    Actions actions = new Actions(BrowserDriver);
                    bool anyKeySet = false;
 
                    if (modifierKeys.HasFlag(ModifierKeys.Alt))
                    {
                        actions.KeyDown(Keys.Alt);
                        anyKeySet = true;
                    }
                    if (modifierKeys.HasFlag(ModifierKeys.Control))
                    {
                        actions.KeyDown(Keys.Control);
                        anyKeySet = true;
                    }
                    if (modifierKeys.HasFlag(ModifierKeys.Shift))
                    {
                        actions.KeyDown(Keys.Shift);
                        anyKeySet = true;
                    }
                    if (modifierKeys.HasFlag(ModifierKeys.Windows))
                    {
                        actions.KeyDown(Keys.Meta);
                        anyKeySet = true;
                    }
 
                    ModifierKeys unsupported = modifierKeys & (ModifierKeys.Application | ModifierKeys.ScrollLock);
                    if (ModifierKeys.NONE != unsupported)
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"PressModifierKey: [{GetModifierKeys(unsupported)}] is not supported via Selenium/WebDriver Actions and was ignored.", this, GPALObjectType.Browser);
 
                    if (true == anyKeySet)
                        actions.Perform();
                }
 
            return this;
        }
        public IAllowBrowserActionOrAnySelector ReleaseModifierKey(ModifierKeys modifierKeys)
        {
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Released [{GetModifierKeys(modifierKeys)}]", this, GPALObjectType.Browser);
 
            if (ModifierKeys.NONE != modifierKeys)
                if (false == BrowserSettings.UseHeadless && (true == BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware))
                {
                    HardwareHelper.ReleaseModifierKeys(modifierKeys);
                }
                else if (true == UseOttoMagic)
                {
                    MagicHelper.ReleaseModifierKey(modifierKeys);
                }
                else if (true == UsePuppeteer || (true == BrowserSettings.UseHeadless && (true == BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware)))
                {
                    PuppeteerClient.ReleaseModifierKey(modifierKeys);
                }
                else
                {
                    Actions actions = new Actions(BrowserDriver);
                    bool anyKeySet = false;
 
                    if (modifierKeys.HasFlag(ModifierKeys.Alt))
                    {
                        actions.KeyUp(Keys.Alt);
                        anyKeySet = true;
                    }
                    if (modifierKeys.HasFlag(ModifierKeys.Control))
                    {
                        actions.KeyUp(Keys.Control);
                        anyKeySet = true;
                    }
                    if (modifierKeys.HasFlag(ModifierKeys.Shift))
                    {
                        actions.KeyUp(Keys.Shift);
                        anyKeySet = true;
                    }
                    if (modifierKeys.HasFlag(ModifierKeys.Windows))
                    {
                        actions.KeyUp(Keys.Meta);
                        anyKeySet = true;
                    }
 
                    ModifierKeys unsupported = modifierKeys & (ModifierKeys.Application | ModifierKeys.ScrollLock);
                    if (ModifierKeys.NONE != unsupported)
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"ReleaseModifierKey: [{GetModifierKeys(unsupported)}] is not supported via Selenium/WebDriver Actions and was ignored.", this, GPALObjectType.Browser);
 
                    if (true == anyKeySet)
                        actions.Perform();
                }
 
            return this;
        }
        /// </summary>
        /// <example>
        /// <code>
        /// // send CTRL-HOME
        ///     ...
        ///     myBrowser
        ///         .PressModifierKey(ModifierKeys.Control)
        ///         .SendKey(GPAL.VK_HOME);
        ///         .WaitFor(150)
        ///         .ReleaseModifierKey(ModifierKeys.Control)
        /// </code>
        /// </example>
        /// <param name="VKCode">Virtual Keycode</param>
        /// <returns>Fluent interface to write your workflow</returns>
        public IAllowBrowserActionOrAnySelector SendKey(byte VKCode)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                ElementHelper.SendKey(this, VKCode);
            }
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector ScrollWindowByHorizontal(int scrollAmountInPixels)
        {
            if (true == UseOttoMagic)
                MagicHelper.ScrollWindowByHorizontal(scrollAmountInPixels);
            else if (true == UsePuppeteer)
                PuppeteerClient.ScrollWindowByHorizontal(scrollAmountInPixels).Execute();
            else
                BrowserHelper.ExecuteJavaScript(this, $"window.scrollBy({{left: {scrollAmountInPixels}, top: 0, behavior: 'instant'}})");
 
            return this;
        }
        /// <returns>Fluent interface to write your workflow</returns>
        public IAllowBrowserActionOrAnySelector ScrollWindowByVertical(int scrollAmountInPixels)
        {
            if (true == UseOttoMagic)
                MagicHelper.ScrollWindowByVertical(scrollAmountInPixels);
            else if (true == UsePuppeteer)
                PuppeteerClient.ScrollWindowByVertical(scrollAmountInPixels).Execute();
            else
                BrowserHelper.ExecuteJavaScript(this, $"window.scrollBy({{left: 0, top: {scrollAmountInPixels}, behavior: 'instant'}})");
 
            return this;
        }
        public IAllowBrowserActionOrAnySelector ScrollElementHorizontalByPixels(int scrollAmountInPixels)
        {
            ElementHelper.ScrollElement(this, scrollAmountInPixels, ScrollTypes.Horizontal);
            return this;
        }
        public IAllowBrowserActionOrAnySelector ScrollElementVerticalByPixels(int scrollAmountInPixels)
        {
            ElementHelper.ScrollElement(this, scrollAmountInPixels, ScrollTypes.Vertical);
            return this;
        }

        public IAllowBrowserActionOrAnySelector GoToTab(dynamic URLorTabTuple = null)
        {
            if (true == UseOttoMagic)
            {
                TabTuple gotoTabTuple = MagicHelper.GotoTab(URLorTabTuple);
                if (URLorTabTuple is TabTuple)
                    UpdateActiveTab(URLorTabTuple);
                else
                    UpdateActiveTab(gotoTabTuple);
            }
            else if (true == UsePuppeteer)
            {
                if (URLorTabTuple is GPALUrl url)
                    PuppeteerClient.GoToTab(url.Url).Execute();
                else if (URLorTabTuple is int tabId)
                    PuppeteerClient.GoToTab(tabId).Execute();
            }
            else
            {
                foreach (var handle in this.BrowserDriver.WindowHandles)
                {
                    string currentUrl = this.BrowserDriver.SwitchTo().Window(handle).Url;
                    if (currentUrl == URLorTabTuple)
                    {
                        this.BrowserDriver.SwitchTo().Window(handle);
                        BrowserSettings.CurrentURL = URLorTabTuple;
 
                        var handles = BrowserDriver.WindowHandles.ToList();
                        string currentHandle = BrowserDriver.CurrentWindowHandle;
 
                        CurrentTabIdx = handles.IndexOf(currentHandle);
 
                        return this;
                    }
                }
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Failed to detect tab with URL [{URLorTabTuple}]", this, GPALObjectType.Browser);
            }
 
            return this;
        }


        public IAllowBrowserActionOrAnySelector DragAndDrop(ModifierKeys modifierKeys)
        {
            // waitfor did not match if returns false, so we have no element to click
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                ElementHelper.DragAndDrop(this, modifierKeys);
            }
            return this;
        }


        public IAllowBrowserActionOrAnySelector DragAndDrop(Selector selector = null)
        {
            if (null != selector)
                WithSelector(selector);
 
            // waitfor did not match if returns false
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
            {
                ElementHelper.DragAndDrop(this);
            }
            return this;
        }

        public IAllowBrowserActionOrAnySelector ExecuteJavaScriptObj(string javascript)
        {
            JavaScriptResultObj = BrowserHelper.ExecuteJavaScriptObj(javascript, this);
            return this;
        }

        public IAllowBrowserActionOrAnySelector ExecuteJavaScriptStr(string javascript)
        {
            JavaScriptResultStr = BrowserHelper.ExecuteJavaScript(this, javascript);
            return this;
        }
 
        private readonly List<string> _injectedScriptIdentifiers = new List<string>();
 
        // whether this browser has registered a script with whatever is driving it, which decides whether there
        // is anything to take away when it closes
        private bool _scriptsRegistered = false;
        private readonly List<string> _pendingInjectedScripts = new List<string>();
 
        // the override goes on once. after that the answers are written to the page rather than registered again
        private bool _dialogRegistered = false;
 
        // the answers and the page they were written to, so nothing is written twice for no reason
        private string _dialogPushed = null;
 
        // counts up per registration so the newest override wins wherever the scripts run in
        private int _dialogSeq = 0;

        private bool EngineReachable
        {
            get
            {
                bool retVal;
 
                if (true == UseOttoMagic)
                    retVal = false == string.IsNullOrEmpty(BrowserSettings.RestApiBaseUrl);
                else if (true == UsePuppeteer)
                    retVal = false == string.IsNullOrEmpty(BrowserSettings.PuppeteerUrl);
                else // selenium
                    retVal = null != BrowserSettings.BrowserDriver;
 
                return retVal;
            }
        }

        private void RegisterInjectedScript(string script)
        {
            _scriptsRegistered = true;
 
            if (true == UseOttoMagic)
            {
                MagicHelper.InjectScript(script);
            }
            else if (true == UsePuppeteer)
            {
                PuppeteerCommunicator.InjectScript(script, null).GetAwaiter().GetResult();
            }
            else // selenium
            {
                var identifier = BrowserHelper.SeleniumAddScriptToEvaluateOnNewDocument(BrowserSettings, script);
                if (false == string.IsNullOrEmpty(identifier))
                    _injectedScriptIdentifiers.Add(identifier);
            }
        }

        /// Called from WorkflowSetup once the engine answers and before the first navigation, so a script queued
        /// ahead of .GoTo() still runs on the first document.
        /// </summary>
        internal void ApplyPendingInjectedScripts()
        {
            if (0 < _pendingInjectedScripts.Count && true == EngineReachable)
            {
                foreach (var script in _pendingInjectedScripts)
                    RegisterInjectedScript(script);
 
                _pendingInjectedScripts.Clear();
            }
        }


        /// over CDP, and the page's own alert, confirm and prompt are untouched, so nothing about it is visible
        /// to the page. OttoMagic cannot do that, because an open dialog stops the page thread the extension
        /// talks through, so there GPAL replaces those three functions before the page runs and no dialog is
        /// ever raised. That replacement is the one thing here a page could notice, which is why it is only
        /// done on the engine that has no other way.
        /// </summary>
        /// <param name="accept">True to accept, false to dismiss.</param>
        /// <returns>Fluent interface to write your workflow</returns>
        public IAllowBrowserActionOrAnySelector WithDialogsAccepted(bool accept)
        {
            // the driver reads this as a capability when it starts, so a browser that is already up was built
            // without it and nothing here can change that. said rather than quietly doing nothing, and only
            // where the driver is the one answering: naming prompt text moves Selenium into the page, where
            // changing the answer works like it does everywhere else
            if (true == UseSelenium && null != BrowserSettings.BrowserDriver && false == InPageDialogs)
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selenium takes this when the browser starts, and this one is already running, so dialogs are not answered. Call WithDialogsAccepted before the first GoTo", this, GPALObjectType.Browser);
 
            BrowserSettings.DialogsAccepted = accept;
 
            ArmDialogHandling();
 
            return this;
        }

        /// What a prompt hands back when dialogs are accepted. Left unsaid, the prompt's own default stands.
        /// </summary>
        /// <param name="text">The text to answer a prompt with.</param>
        /// <returns>Fluent interface to write your workflow</returns>
        public IAllowBrowserActionOrAnySelector WithDialogText(string text)
        {
            BrowserSettings.DialogText = text;
 
            // a workflow that says what to answer has said to answer, so this turns it on rather than sitting
            // there doing nothing until WithDialogsAccepted is also called
            if (null == BrowserSettings.DialogsAccepted)
                BrowserSettings.DialogsAccepted = true;
 
            ArmDialogHandling();
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector ClearDialogHandling()
        {
            BrowserSettings.DialogsAccepted = null;
            BrowserSettings.DialogText = null;
 
            ArmDialogHandling();
 
            return this;
        }

        internal void ArmDialogHandling()
        {
            // still runs with nothing to answer with, because turning it off is something the page has to be
            // told about too, and only if it was ever told to start
            if (false == EngineReachable || (null == BrowserSettings.DialogsAccepted && false == _dialogRegistered))
                return;
 
            if (true == InPageDialogs)
            {
                if (false == _dialogRegistered && null != BrowserSettings.DialogsAccepted)
                {
                    string script = DialogScript();
 
                    _dialogRegistered = true;
 
                    // registering covers every document from here on, which before the first navigation is all
                    // of them. running it directly is for the other case, arming partway through a workflow,
                    // where the page being stood on has already loaded and would otherwise be the one page not
                    // covered
                    InjectScript(script);
 
                    if (true == _navigated)
                        BrowserHelper.ExecuteJavaScript(this, script);
 
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Answering dialogs in the page, accept [{BrowserSettings.DialogsAccepted}]", this, GPALObjectType.Browser);
                }
 
                // every document runs its own copy of the script, which sets the answers back to the ones it was
                // registered with, so what is on the page is stale when the workflow changes its mind and stale
                // again when the page moves. both are known here without asking the browser anything
                string written = DialogAnswers() + "@" + BrowserSettings.CurrentURL;
 
                if (true == _navigated && written != _dialogPushed)
                {
                    _dialogPushed = written;
                    PushDialogAnswers();
                }
 
                // asked once, after the first navigation, because a script that never registered and one that
                // registered and never ran look the same from here: both leave the page raising real dialogs,
                // and the first one to appear stops the workflow with no idea why
                // asked of OttoMagic only, and once. registering there goes through the extension and can be
                // refused, which looks from here exactly like a registration that worked. Selenium registers the
                // same script over CDP, where there is nothing to refuse it
                if (true == UseOttoMagic && true == _navigated && false == _dialogChecked)
                {
                    _dialogChecked = true;
 
                    // an expression, not a statement: what OttoMagic runs this through takes the value of the
                    // last expression, and a top level return is a syntax error there
                    string seen = BrowserHelper.ExecuteJavaScript(this, "window.__gpalDialogSeq");
 
                    if (true == string.IsNullOrEmpty(seen) || "null" == seen)
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"The dialog override is not on the page, so dialogs will stop this workflow. The usual cause is Allow user scripts being off for the extension", this, GPALObjectType.Browser);
                }
            }
            else if (true == UsePuppeteer && null != BrowserSettings.DialogsAccepted && false == _dialogRegistered)
            {
                // Page.javascriptDialogOpening does not arrive until the domain is on, and the launch paths do
                // not turn it on
                _dialogRegistered = true;
 
                PuppeteerCommunicator.EnablePageEvents().GetAwaiter().GetResult();
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Answering dialogs over CDP, accept [{BrowserSettings.DialogsAccepted}]", this, GPALObjectType.Browser);
            }
 
            // selenium answering yes or no is armed at launch, because UnhandledPromptBehavior is a capability
            // on the driver options and nothing after that can change it
        }

        private bool InPageDialogs => true == UseOttoMagic
            || (true == UseSelenium && null != BrowserSettings.DialogText);
 
        // set when a navigation has finished rather than when one has been asked for. BrowserSettings.CurrentURL
        // is the url GoTo is heading to and is written before it goes, so it says nothing about what is on screen
        private bool _navigated = false;
 
        // whether the override was found in place on the page after the first navigation
        private bool _dialogChecked = false;

        private void PushDialogAnswers()
        {
            // only where the override is already installed, so this is never what puts the globals there
            BrowserHelper.ExecuteJavaScript(this, "if (window.__gpalDialogSeq) { " + DialogAnswers() + " }");
        }

        private string DialogAnswers()
        {
            string accept = null == BrowserSettings.DialogsAccepted
                ? "null"
                : true == BrowserSettings.DialogsAccepted ? "true" : "false";
            string typed = null == BrowserSettings.DialogText ? "null" : JsonConvert.SerializeObject(BrowserSettings.DialogText);
 
            return "window.__gpalDialogAccept = " + accept + "; window.__gpalDialogAnswer = " + typed + ";";
        }

        private string DialogScript()
        {
            return
                "(function () {" +
                "  var seq = " + (++_dialogSeq) + ";" +
                "  if (window.__gpalDialogSeq >= seq) return;" +
                "  window.__gpalDialogSeq = seq;" +
                "  " + DialogAnswers() +
                "  var asText = Function.prototype.toString;" +
                "  function stand(name, fn) {" +
                "    Object.defineProperty(fn, 'name', { value: name, configurable: true });" +
                // put where the real one is rather than where the spec says it should be. chrome carries
                // these as own properties of window, and one defined on Window.prototype sits behind the
                // native it was meant to replace: installed, never called. asking first also keeps the
                // shape a page would see, since ours ends up in the same place as the one it replaced
                "    var onto = Object.getOwnPropertyDescriptor(window, name) ? window : Window.prototype;" +
                "    Object.defineProperty(onto, name," +
                "      { value: fn, writable: true, enumerable: true, configurable: true });" +
                "  }" +
                // kept so turning it off can hand the page back what it started with, rather than leaving it
                // with three functions that answer nothing
                "  var real = { alert: window.alert, confirm: window.confirm, prompt: window.prompt };" +
                "  function off() { return null === window.__gpalDialogAccept; }" +
                "  stand('alert', function (message) {" +
                "    if (off()) return real.alert.call(window, message);" +
                "  });" +
                "  stand('confirm', function (message) {" +
                "    if (off()) return real.confirm.call(window, message);" +
                "    return true === window.__gpalDialogAccept;" +
                "  });" +
                "  stand('prompt', function (message, preset) {" +
                "    if (off()) return real.prompt.call(window, message, preset);" +
                "    if (true !== window.__gpalDialogAccept) return null;" +
                "    return null !== window.__gpalDialogAnswer ? window.__gpalDialogAnswer : (preset || '');" +
                "  });" +
                // a page that asks what these are reads what it would read of the real ones. it raises the bar
                // rather than clearing it: this is the part of the feature a page can still catch out
                "  Function.prototype.toString = function () {" +
                "    if (this === window.alert || this === window.confirm || this === window.prompt" +
                "        || this === Function.prototype.toString)" +
                "      return 'function ' + this.name + '() { [native code] }';" +
                "    return asText.call(this);" +
                "  };" +
                "})();";
        }

        /// <example>
        /// <code>
        ///     browser.GetWindowRectangle(out Rectangle window);
        /// </code>
        /// </example>
        /// <param name="window">Receives the window rectangle in screen coordinates.</param>
        /// <returns>Fluent interface to write your workflow</returns>
        public IAllowBrowserActionOrAnySelector GetWindowRectangle(out Rectangle window)
        {
            window = Rectangle.Empty;
 
            if (true == InAction(false))
            {
                if (true == UsePuppeteer)
                    window = PuppeteerCommunicator.GetWindowRectangle().GetAwaiter().GetResult();
                else
                    ElementHelper.GetWindowRectangle(this, out window);
            }
 
            return this;
        }


        public IAllowBrowserActionOrAnySelector InjectScript(string script)
        {
            // no browser to inject into until the first .GoTo()/.Get() launches one, so the script is held
            // and registered the moment the engine answers, ahead of the first navigation
            if (false == EngineReachable)
                _pendingInjectedScripts.Add(script);
            else
                RegisterInjectedScript(script);
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector InjectScript(GPALFile scriptFile)
        {
            foreach (var filename in scriptFile.Filenames)
                InjectScript(File.ReadAllText(filename));
            return this;
        }


        public IAllowBrowserActionOrAnySelector ClearInjectedScripts()
        {
            // scripts still waiting on a browser go first, so one queued and then cleared never reaches the page
            _pendingInjectedScripts.Clear();
 
            if (true == EngineReachable)
            {
                if (true == UseOttoMagic)
                {
                    MagicHelper.ClearInjectedScripts();
                }
                else if (true == UsePuppeteer)
                {
                    PuppeteerCommunicator.ClearInjectedScripts().GetAwaiter().GetResult();
                }
                else // selenium
                {
                    foreach (var identifier in _injectedScriptIdentifiers)
                        BrowserHelper.SeleniumRemoveScriptToEvaluateOnNewDocument(BrowserSettings, identifier);
                    _injectedScriptIdentifiers.Clear();
                }
            }
            return this;
        }

        public IAllowBrowserActionOrAnySelector SetRange(int rangeValue)
        {
            if (true == InAction(WaitTime.Never < CurrentUOW.WaitForInMs))
                foreach (Selector selector in CurrentUOW.WithSelectorList)
                {
                    ElementHelper.FindWebElements(this, CurrentUOW, selector, out _, out List<GPALElement> elems);
                    foreach (GPALElement elem in elems)
                    {
                        ElementHelper.ScrollIntoView(this, elem);
 
                        if (true == UseOttoMagic)
                            MagicHelper.SetRange(elem.Css, rangeValue);
                        else if (true == UsePuppeteer)
                            PuppeteerClient.SetRange(elem.Css).WithRangeValue(rangeValue).Execute();
                        else if (true == UseSelenium)
                        {
                            // Set the value and dispatch the change events so the UI updates
                            string script = @"
                            arguments[0].value = arguments[1];
                            arguments[0].dispatchEvent(new Event('input', { bubbles: true }));
                            arguments[0].dispatchEvent(new Event('change', { bubbles: true }));
                        ";
 
                            BrowserHelper.ExecuteJavaScript(this, script, elem.WebElement, rangeValue);
                        }
                    }
                }
 
            return this;
        }
        #endregion <Browser Actions>
        #region <Getters>
        private bool _started = false;
        internal bool Started
        {
            get => _started;
            set => _started = value;
        }

        public int ServerResponseCode
        {
            get => BrowserSettings.ServerResponseCode;
            set => BrowserSettings.ServerResponseCode = value;
        }


        public string CurrentUrl
        {
            get => BrowserSettings.CurrentURL;
            internal set => BrowserSettings.CurrentURL = value;
        }
        /// </summary>
        public bool UseSelenium
        {
            get => BrowserSettings.UseSelenium;
        }

        public bool UsePuppeteer
        {
            get => BrowserSettings.UsePuppeteer;
        }
        public bool UseOttoMagic
        {
            get => BrowserSettings.UseOttoMagic;
        }

        public Process Process
        {
            get => BrowserSettings.Process;
        }
        public AutomationEngine AutomationEngine
        {
            get => BrowserSettings.AutomationEngine;
            internal set => BrowserSettings.AutomationEngine = value;
        }
        public bool? FileDownloaded
        {
            get
            {
                return BrowserSettings.FileDownloaded;
            }
 
            set
            {
                BrowserSettings.FileDownloaded = value;
            }
        }

        public BrowserType BrowserType
        {
            get
            {
                return BrowserSettings.BrowserType;
            }
        }
        internal UnitOfWork CurrentUOW { get; set; } = null;
        internal WorkflowManager WorkflowManager { get; set; }
        internal string Name
        {
            get
            {
                return BrowserSettings.BrowserName;
            }
 
            set
            {
                BrowserSettings.BrowserName = value;
            }
        }
        /// Only available after the first .GoTo<br/>
        /// <b>NOTE:</b> Only for extended Selenium control access. Be careful.
        /// </summary>
        public IWebDriver BrowserDriver
        {
            get
            {
                return BrowserSettings.BrowserDriver;
            }
 
            internal set
            {
                BrowserSettings.BrowserDriver = value;
            }
        }
        public string RestApiBaseUrl
        {
            get
            {
                return BrowserSettings.RestApiBaseUrl;
            }
        }
        public BrowserSettings BrowserSettings
        {
            get
            {
                return _browserSettings;
            }
 
            internal set
            {
                _browserSettings = value;
            }
        }
        internal string DownloadLocation
        {
            get
            {
                return BrowserSettings.DownloadLocation;
            }
        }
        /// </summary>
        public PuppeteerCommunicator PuppeteerCommunicator
        {
            get => BrowserSettings.PuppeteerCommunicator;
            internal set => BrowserSettings.PuppeteerCommunicator = value;
        }
 
        public bool IsAlive
        {
            get
            {
                // nothing of ours was launched, so ask its GPALRestAPI instead. that exits with the browser
                // it belongs to, so an answer means the browser is still there
                if (true == BrowserSettings.AttachedRestApi)
                    return BrowserHelper.RestApiAnswering(BrowserSettings.RestApiBaseUrl);
 
                if (BrowserSettings.Process != null)
                    return !BrowserSettings.Process.HasExited;
                if (BrowserSettings.ServiceDriverPid != 0)
                {
                    try { System.Diagnostics.Process.GetProcessById(BrowserSettings.ServiceDriverPid); return true; }
                    catch (ArgumentException) { return false; }
                }
                return true;
            }
        }
        public IPuppeteerClient PuppeteerClient
        {
            get => BrowserSettings.PuppeteerClient;
            internal set => BrowserSettings.PuppeteerClient = value;
        }
        public MagicHelper MagicHelper
        {
            get => (MagicHelper)BrowserSettings.MagicHelper;
        }
 
        public IRESTClient OttoMagicClient => MagicHelper.Client;

        public IHiddenDesktop HiddenDesktop => true == BrowserSettings.HiddenDesktop
            ? new GPALHiddenDesktop(BrowserSettings.HiddenDesktopName)
            : null;
        #endregion <Getters>
        #region <Helpers>
        #region <Private>
        bool ScrollPage(string direction)
        {
            try
            {
                // Validate direction
                if (direction != "up" && direction != "down")
                {
                    return false;
                }
 
                // JavaScript code with direction parameter
                string script = @"
                    (function(direction) {
                        function scrollPage(direction) {
                        try {
                            const isDown = direction === 'down';
                            const scrollAmount = isDown ? window.innerHeight : -window.innerHeight;
                            const keyName = isDown ? 'PageDown' : 'PageUp';
                            const keyCode = isDown ? 34 : 33;
 
                            // Check overflow
                            const overflow = getComputedStyle(document.documentElement).overflow;
 
                            // Try window.scrollBy if overflow is not 'hidden'
                            const initialScrollY = window.scrollY;
                            if (overflow !== 'hidden') {
                            window.scrollBy({ left: 0, top: scrollAmount, behavior: 'instant' });
                            if (window.scrollY !== initialScrollY) {
                                return {
                                status: 'success',
                                method: 'window.scrollBy',
                                scrollPosition: window.scrollY,
                                elementClass: 'window'
                                };
                            }
                            }
 
                            // Try scrolling the scrollable element
                            const scrollable = Array.from(document.querySelectorAll('*')).find(
                            el => el.scrollHeight > el.clientHeight && getComputedStyle(el).overflowY !== 'hidden'
                            ) || document.body;
                            const initialScrollTop = scrollable.scrollTop;
                            scrollable.scrollBy({ left: 0, top: scrollAmount, behavior: 'instant' });
                            const elementClass = scrollable.className || 'body';
 
                            if (scrollable.scrollTop !== initialScrollTop) {
                            return {
                                status: 'success',
                                method: 'element.scrollBy',
                                scrollPosition: scrollable.scrollTop,
                                elementClass
                            };
                            }
 
                            // Fallback to Page Up/Down key press
                            scrollable.focus();
                            const event = new KeyboardEvent('keydown', {
                            key: keyName,
                            code: keyName,
                            keyCode,
                            which: keyCode,
                            bubbles: true,
                            cancelable: true
                            });
                            scrollable.dispatchEvent(event);
 
                            if (scrollable.scrollTop !== initialScrollTop) {
                            return {
                                status: 'success',
                                method: keyName,
                                scrollPosition: scrollable.scrollTop,
                                elementClass
                            };
                            }
 
                            // No scroll method worked
                            return {
                            status: 'failed',
                            message: 'No scroll method worked',
                            scrollPosition: scrollable.scrollTop,
                            elementClass
                            };
                        } catch (error) {
                            return {
                            status: 'error',
                            message: error.message,
                            scrollPosition: 0,
                            elementClass: 'none'
                            };
                        }
                        }
                        return scrollPage(direction);
                    })('" + direction + "')";
 
                // Execute the script
                var result = BrowserHelper.ExecuteJavaScript(this, script);
 
                // Parse result (adjust based on BrowserHelper's return type)
                dynamic response = result; // Assumes result is a dynamic object or JSON
                return response.status == "success";
            }
            catch
            {
                return false;
            }
        }
        private bool ScrollToPosition(string position)
        {
            try
            {
                // Validate position
                if (position != "top" && position != "end")
                {
                    return false;
                }
 
                // JavaScript code with position parameter
                string script = @"
                    (function(position) {
                        function scrollToPosition(position) {
                            try {
                                const isEnd = position === 'end';
                                const scrollTarget = isEnd ? document.body.scrollHeight : 0;
                                const keyName = isEnd ? 'End' : 'Home';
                                const keyCode = isEnd ? 35 : 36;
 
                                // Check overflow
                                const overflow = getComputedStyle(document.documentElement).overflow;
 
                                // Try window.scrollTo if overflow is not 'hidden'
                                const initialScrollY = window.scrollY;
                                if (overflow !== 'hidden') {
                                    window.scrollTo({ left: 0, top: scrollTarget, behavior: 'instant' });
                                    if (window.scrollY !== initialScrollY) {
                                        return {
                                            status: 'success',
                                            method: 'window.scrollTo',
                                            scrollPosition: window.scrollY,
                                            elementClass: 'window'
                                        };
                                    }
                                }
 
                                // Try scrolling the scrollable element
                                const scrollable = Array.from(document.querySelectorAll('*')).find(
                                    el => el.scrollHeight > el.clientHeight && getComputedStyle(el).overflowY !== 'hidden'
                                ) || document.body;
                                const initialScrollTop = scrollable.scrollTop;
                                const elementScrollTarget = isEnd ? scrollable.scrollHeight : 0;
                                scrollable.scrollTo({ left: 0, top: elementScrollTarget, behavior: 'instant' });
                                const elementClass = scrollable.className || 'body';
 
                                if (scrollable.scrollTop !== initialScrollTop) {
                                    return {
                                        status: 'success',
                                        method: 'element.scrollTo',
                                        scrollPosition: scrollable.scrollTop,
                                        elementClass
                                    };
                                }
 
                                // Fallback to Home/End key press
                                scrollable.focus();
                                const event = new KeyboardEvent('keydown', {
                                    key: keyName,
                                    code: keyName,
                                    keyCode,
                                    which: keyCode,
                                    bubbles: true,
                                    cancelable: true
                                });
                                scrollable.dispatchEvent(event);
 
                                if (scrollable.scrollTop !== initialScrollTop) {
                                    return {
                                        status: 'success',
                                        method: keyName,
                                        scrollPosition: scrollable.scrollTop,
                                        elementClass
                                    };
                                }
 
                                // No scroll method worked
                                return {
                                    status: 'failed',
                                    message: 'No scroll method worked',
                                    scrollPosition: scrollable.scrollTop,
                                    elementClass
                                };
                            } catch (error) {
                                return {
                                    status: 'error',
                                    message: error.message,
                                    scrollPosition: 0,
                                    elementClass: 'none'
                                };
                            }
                        }
                        return scrollToPosition(position);
                    })('" + position + "')";
 
                // Execute the script
                var result = BrowserHelper.ExecuteJavaScript(this, script);
 
                // Parse result (adjust based on BrowserHelper's return type)
                dynamic response = result; // Assumes result is a dynamic object or JSON
                return response.status == "success";
            }
            catch
            {
                return false;
            }
        }
        private static void SelectorListMaintenanceSweep(UnitOfWork CurrentUOW)
        {
            for (int idx = 0; idx < CurrentUOW.WithSelectorList.Count; idx++)
                if (true == CurrentUOW.WithSelectorList[idx].DeleteMe)
                    CurrentUOW.WithSelectorList.RemoveAt(idx);
 
            for (int idx = 0; idx < CurrentUOW.InSelectorList.Count; idx++)
                if (true == CurrentUOW.InSelectorList[idx].DeleteMe)
                    CurrentUOW.InSelectorList.RemoveAt(idx);
        }
        private void CheckPersistentSelectors()
        {
            UnitOfWork safeUOW = CurrentUOW;
 
            // all we have to do is walk the list, try to find the element and the callif callbacks will do the work
            if (0 < (persistentUOW.WithSelectorList?.Count ?? 0))
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Checking [{persistentUOW.WithSelectorList?.Count}] persistent selector(s)");
 
                foreach (Selector selector in persistentUOW.WithSelectorList)
                {
                    if (SelectorType.Selector != selector.SelectorType)
                        continue;   // we don't look up data literals
 
                    // we cannot use cached results knowing we are a new uow/action
                    selector.WebSelectorFoundResults = null;
                    selector.WebSelectorMatchedResults = null;
                    CurrentUOW.CurrentSelector = selector;
 
                    // **************** persistent selector could be in an iframe ****************
                    if (null != selector.selectorSettings.InFrame)
                    {
                        // locate the iframe itself - iframes live in the main document, so the
                        // persistent (frame-0) query is correct here for all engines.
                        ElementHelper.FindWebElements(this, persistentUOW, selector.selectorSettings.InFrame, out _, out var iFrameMatches, null, false, true);
 
                        if (0 == (iFrameMatches?.Count ?? 0))
                            continue;   // the iframe isn't on the page right now - nothing to check this pass
 
                        // Puppeteer/Selenium: actually switch the driver into the iframe so the
                        // search runs at that frame's document root, then switch back.
                        if (true == UseOttoMagic)
                            MagicHelper.SwitchToElement(iFrameMatches[0].Css);
                        else if(true == UsePuppeteer)
                            PuppeteerClient.SwitchToFrame(iFrameMatches[0].Css).Execute();
                        else
                            BrowserDriver.SwitchTo().Frame((IWebElement)iFrameMatches[0].WebElement);
 
                        ElementHelper.FindWebElements(this, persistentUOW, selector, out _, out _, null, true, true);
 
                        if (true == UseOttoMagic)
                            MagicHelper.SwitchToDefaultContent();
                        else if (true == UsePuppeteer)
                            PuppeteerClient.SwitchToDefaultContent().Execute();
                        else
                            BrowserDriver.SwitchTo().DefaultContent();
 
                        // restore the driver to wherever the main workflow's context path left it -
                        // FindElements only replays CurrentUOW.ContextPath when it is non-empty, so an
                        // empty ContextPath means the main document is already the correct state.
                        if (0 < (CurrentUOW.ContextPath?.Count ?? 0))
                            foreach (UnitOfWork.WebElementWithType contextType in CurrentUOW.ContextPath)
                            {
                                if (ElementType.IFrame == contextType.elementType)
                                {
                                    if (true == UseOttoMagic)
                                        MagicHelper.SwitchToElement(iFrameMatches[0].Css);
                                    else if (true == UsePuppeteer)
                                        PuppeteerClient.SwitchToFrame(contextType.gPalElement.Css).Execute();
                                    else
                                        BrowserDriver.SwitchTo().Frame((IWebElement)contextType.gPalElement.WebElement);
                                }
                                else if (ElementType.ShadowRoot == contextType.elementType)
                                {
                                    if (true == UseOttoMagic)
                                        MagicHelper.SwitchToElement(contextType.gPalElement.Css);
                                    else if (true == UsePuppeteer)
                                        PuppeteerClient.SwitchToShadowRoot(contextType.gPalElement.Css).Execute();
                                }
                            }
 
                        continue;
                    }
 
                    ElementHelper.FindWebElements(this, persistentUOW, selector, out _, out _, null, true, true);
                }
            }
            CurrentUOW = safeUOW;
        }
        private bool CheckWaitFor()
        {
            bool matchFound = true;
            List<GPALElement> foundElements = null;
 
            // an earlier action on this same unit of work already waited these selectors in successfully, so
            // waiting again is redundant - the page cannot have moved under us, since any selector added after
            // an action starts a fresh unit of work. the action still resolves its own elements either way.
            if (null != CurrentUOW.WaitForSatisfied)
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Selectors for this unit of work already found at [{CurrentUOW.WaitForSatisfied:HH:mm:ss.fff}], skipping redundant wait.", this, GPALObjectType.Browser);
                return true;
            }
 
            // -1 means wait forever
            // -2 never
            if (WaitTime.Never < CurrentUOW.WaitForInMs)
            {
                // we are waiting on selector(s)
                foundElements = ElementHelper.WaitFor(this, CurrentUOW.WaitForInMs, out matchFound);
 
                if (0 < foundElements?.Count)
                    CurrentUOW.WaitForSatisfied = DateTime.Now;
            }
 
            return (0 < foundElements?.Count);
        }
        internal bool WorkflowSetup(bool checkWaitFor, bool doMaintenance, GPALUrl URL = null)
        {
            // settled before anything is launched, because the engine chosen here decides which of the branches
            // below runs
            if (true == BrowserSettings.HiddenDesktop && false == BrowserSettings.HiddenDesktopReported)
            {
                BrowserSettings.HiddenDesktopReported = true;
 
                // SendInput reaches the desktop the user is on and no other, so a hardware engine on a desktop of
                // its own would be clicking at nothing. it drives through its protocol instead, and the workflow
                // is told which engine it is actually running rather than finding out from the results
                if (true == BrowserSettings.UseHardware)
                {
                    AutomationEngine steppedDownTo = HardwareStepDown(BrowserSettings.AutomationEngine);
 
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{BrowserSettings.AutomationEngine}] cannot drive a hidden desktop, so this browser runs on [{steppedDownTo}]", this, GPALObjectType.Browser);
 
                    BrowserSettings.AutomationEngine = steppedDownTo;
                }
 
                // a desktop that cannot be made is a browser that opens in front of the user, so the workflow is
                // told and this browser runs on the desktop it can have
                if (false == Desktops.Ensure(BrowserSettings.HiddenDesktopName))
                {
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"No hidden desktop could be made, so this browser opens on the desktop you are working on", this, GPALObjectType.Browser);
 
                    BrowserSettings.HiddenDesktop = false;
                }
            }
 
            // also checked in WithProfileDataDirectory as vital for that operation, but if that is not specified then this is the best location to set this
            // NOTE: CAVEAT: it's not clear this is even required here, but we may find a use for it
            //if (null == BrowserSettings.ProfileDataDirectory)
            //    BrowserSettings.ProfileDataDirectory = FileHelper.EnsureDirectoryEndsWithBackslash(BrowserHelper.GetBrowserProfileDirectory(BrowserSettings, true));
 
            // first time init is the only time URL is used
            // but due to window tracking, we will always go to https://google.com first then an explicit goto
            if (true == UseSelenium)
            {
                // instatiate a selenium driver and launch chrome headless
                if (null == BrowserDriver)
                {
                    if (null == (BrowserDriver = LaunchWhereAsked(() => BrowserHelper.GetBrowserDriver(BrowserSettings))))
                        return false;    // NOTE: CAVEAT: will never return false, getbrowserdiver will now throw an exception and end the workflow if it can't start the browser
 
                    // selenium allows runtime setting of prompt for download and for chromium browsers, we can turn it off runtime via CDP
                    // if we are running under a temp profile (none specified) we write out preferences to handle prompting to save
                    // but if a profile is loaded and it doesn't have it set in the profile, we have to toggle it based upon the workflow setting
                    //  firefox has no runtime equivalent, so it is set in the profile prefs when the driver is built(BrowserHelper GetBrowserDriver).
                    if (false == BrowserSettings.PromptForDownload && BrowserType.FireFox != BrowserSettings.BrowserType)
                        BrowserHelper.AllowDownloads(BrowserSettings, BrowserSettings.DownloadLocation ?? FileHelper.GetDefaultDownloadDirectory(this));
                }
            }
            else if (null == BrowserSettings.Process && false == BrowserSettings.AttachedRestApi)   // use browser extension/puppeteer to control browser, not selenium
            {
                if (true == UseOttoMagic)
                    // we have to ensure we always open to a page that loads a content page.
                    BrowserSettings.Process = LaunchWhereAsked(() => MagicHelper.LaunchBrowser(this, URL));
                else
                    BrowserSettings.Process = LaunchWhereAsked(() => Puppeteer.LaunchBrowser(this, URL));
            }
 
            // the desktop is asked what is on it, once, after the first launch. a driver that did not inherit the
            // desktop starts the browser back on the visible one and reports a clean launch either way, so the
            // only thing that tells the two apart is looking at where the window actually went
            if (true == BrowserSettings.HiddenDesktop && false == BrowserSettings.HiddenDesktopChecked)
            {
                BrowserSettings.HiddenDesktopChecked = true;
 
                if (false == Desktops.HasWindows(BrowserSettings.HiddenDesktopName, 10_000))
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Nothing arrived on [{BrowserSettings.HiddenDesktopName}], so this browser is on the desktop you are working on", this, GPALObjectType.Browser);
            }
 
            // topping a window means nothing on a desktop nobody is looking at, and the hardware path it can fall
            // back to cannot reach one
            if (false == BrowserSettings.HiddenDesktop)
                BrowserHelper.TopBrowser(this, false); // ensure the browser window is on top, can use selenium or hardware (mouse click) topping. firefox will only top with hardware
 
            CurrentUOW.ActionCalled = true;
 
            if (true == doMaintenance)
            {
                SelectorListMaintenanceSweep(CurrentUOW); // purge any nodes marked 'deleteme' before iterating list
                SelectorListMaintenanceSweep(persistentUOW); // purge any nodes marked 'deleteme' before iterating list
                CheckPersistentSelectors(); // check if there are persistent selectors to deal with
            }
 
            if (true == checkWaitFor)
                CheckWaitFor();
 
            if (true == UsePuppeteer && null == BrowserSettings.PuppeteerUrl)
            {
                BrowserSettings.PuppeteerUrl = $"http://localhost:{BrowserSettings.DebugPort}";
                _ = PuppeteerCommunicator;  // init
 
                // Selenium writes download.prompt_for_download as a profile preference on every launch, but the
                // puppeteer path only gets it when LaunchBrowser built the profile itself. Setting it here covers
                // every profile, and being browser scoped it holds for tabs opened later
                if (false == BrowserSettings.PromptForDownload)
                {
                    PuppeteerCommunicator.SetDownloadBehavior(BrowserSettings.DownloadLocation ?? FileHelper.GetDefaultDownloadDirectory(this)).GetAwaiter().GetResult();
                    BrowserSettings.DownloadBehaviorSet = true;
                }
            }
 
            // every engine is reachable and GoTo has not navigated yet, so held scripts registered here still
            // run on the first document
            ApplyPendingInjectedScripts();
            ArmDialogHandling();
 
            return true;
        }
        bool firstRun { get; set; } = true;
        bool inAction = false;
        internal bool InAction(bool checkWaitFor, bool doMaintenance = true, GPALUrl URL = null)
        {
            bool retval = true;
            if (false == inAction)
            {
                string saveURL = URL;
                inAction = true;
                FileDownloaded = false; // any action will clear out this flag, but we can leftclickanddownload and then .WIthSelector(noDataSelector) and preserve it.
 
                //if (true == firstRun && BrowserSettings.StealthType.HasFlag(StealthType.GenerateRealCookies))
                //    URL = null;
                //else
                //    firstRun = false;
 
                // we have to return true otherwise callIfFound/NotFound are never invoked - waitfor just waits for the element to show up, but doesn't abend the workflow
                // getBrowserDriver should not return null now, should throw, so this should always return true
                retval = WorkflowSetup(checkWaitFor, doMaintenance, URL);
                
                /*
                if (true == firstRun && BrowserSettings.StealthType.HasFlag(StealthType.GenerateRealCookies))
                {
                    firstRun = false;
                    MagicHelper.LetsDoSomeSurfing(BrowserSettings);
                    //if (true == MagicHelper.LetsDoSomeSurfing(BrowserSettings))    // stealth type - generate some cookies and history...
                    //    GoTo(saveURL);
                }
                */
                inAction = false;
            }
            return retval;    // NOTE: CAVEAT: useless anymore
        }
        private void BackOutOfSelfClosedTab(GPALUrl URL, string priorUrl, GPALUrl priorGPALUrl, TabTuple priorTab, string priorHandle)
        {
            tabsWeOpened--;
 
            // each engine finds its way back the way it already knows how. The tab we came from never navigated,
            // so it is still open and still sitting on the url it was on
            if (true == UseOttoMagic)
                GoToTab(priorTab);
            else if (true == UsePuppeteer)
                PuppeteerClient.GoToTab(priorGPALUrl).Execute();
            else
            {
                // back to the exact tab we left, by id. It never navigated, so it is still open unless the user
                // closed it themselves, in which case the last tab in the list is as good an answer as any
                var handles = BrowserDriver.WindowHandles;
                string landOn = true == handles.Contains(priorHandle) ? priorHandle : handles[handles.Count - 1];
 
                BrowserDriver.SwitchTo().Window(landOn);
                CurrentTabIdx = handles.IndexOf(landOn);
            }
 
            BrowserSettings.CurrentURL = priorUrl;
            BrowserSettings.CurrentGPALUrl = priorGPALUrl;
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{URL?.Url}] downloaded rather than opening a page, so its tab closed itself. Back on [{priorUrl}].", this, GPALObjectType.Browser);
        }
        internal int RealTabCount()
        {
            int realCount;
 
            if (true == UseOttoMagic)
                realCount = MagicHelper.TabCount();
            else if (true == UsePuppeteer)
                realCount = PuppeteerCommunicator.TabCount().GetAwaiter().GetResult();
            else
                // the plural handles command does not need a live current window, unlike CurrentWindowHandle, so
                // this still answers after a tab has closed itself, which is the whole point of asking
                realCount = BrowserDriver.WindowHandles.Count;
 
            return realCount;
        }
        private int GetActiveTabId()
        {
            foreach(var key in BrowserSettings.TabIdsToURL.Keys.Where(k => BrowserSettings.TabIdsToURL[k].ActiveTab == true))
                return key;
 
            return 0;
        }
        private void UpdateActiveTab(TabTuple activeTabTuple)
        {
            string priorUrl = null;
            if (null != activeTabTuple)
            {
                // First, update or add the current tabId with its tabInfo
                if (BrowserSettings.TabIdsToURL.ContainsKey(activeTabTuple.TabId))
                {
                    priorUrl = BrowserSettings.TabIdsToURL[activeTabTuple.TabId].URL;
                    BrowserSettings.TabIdsToURL[activeTabTuple.TabId].URL = activeTabTuple.Url ?? BrowserSettings.TabIdsToURL[activeTabTuple.TabId].URL;
                    BrowserSettings.TabIdsToURL[activeTabTuple.TabId].ActiveTab = true;
                }
                else
                {
                    priorUrl = activeTabTuple.Url;
                    BrowserSettings.TabIdsToURL.Add(activeTabTuple.TabId, new TabInfo { URL = activeTabTuple.Url, ActiveTab = true });
                }
 
                // Second, set ActiveTab = false for all other tabs
                foreach (var key in BrowserSettings.TabIdsToURL.Keys.Where(k => k != activeTabTuple.TabId))
                    BrowserSettings.TabIdsToURL[key].ActiveTab = false;
 
                if (null != activeTabTuple.Url)
                {
                    URLInfo uRLInfo = new URLInfo { TabId = activeTabTuple.TabId, ActiveTab = true };
                    // Next, update or add the current URL with its urlInfo
                    if (BrowserSettings.URLsToTabId.ContainsKey(priorUrl))
                    {
                        BrowserSettings.URLsToTabId[activeTabTuple.Url] = uRLInfo;
                        BrowserSettings.URLsToTabId[activeTabTuple.Url].ActiveTab = true;
                        if (false == priorUrl.Equals(activeTabTuple.Url))
                            BrowserSettings.URLsToTabId.Remove(priorUrl);
                    }
                    else
                        BrowserSettings.URLsToTabId.Add(activeTabTuple.Url, new URLInfo { TabId = activeTabTuple.TabId, ActiveTab = true });
 
                    // last, set ActiveTab = false for all other tabs
                    foreach (var key in BrowserSettings.URLsToTabId.Keys.Where(k => k != activeTabTuple.Url))
                        BrowserSettings.URLsToTabId[key].ActiveTab = false;
 
                    BrowserSettings.CurrentURL = activeTabTuple.Url;
                    // NOTE: add search gpalurllist and setting gpalcurrenturl
                }
            }
            else
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, "ActiveTabTuple from last operation is null", this, GPALObjectType.Browser);
        }
        // NOTE: not implemented. the concept was TabIds for ottomagic, for precise tab switching where the
        // server rewrites urls
        internal bool TryDecodeNumberToIpAddress(string number, out string ipAddress)
        {
            bool retVal = false;
            uint numberUInt;
 
            ipAddress = null;
 
            if (true == UInt32.TryParse(number, out numberUInt))
            {
 
                // Convert number to IPv4 bytes (network byte order)
                byte[] bytes = BitConverter.GetBytes(numberUInt);
                if (BitConverter.IsLittleEndian)
                    Array.Reverse(bytes); // Ensure big-endian for IP
 
                // Create IP address from bytes
                try
                {
                    var ip = new System.Net.IPAddress(bytes);
                    ipAddress = ip.ToString(); // e.g., "192.168.1.1"
 
                    // Validate: Ensure it's a valid IPv4 address (4 octets, 0-255 range)
                    var octets = ipAddress.Split('.');
                    if (octets.Length != 4) return false;
                    if (octets.All(o => byte.TryParse(o, out byte value) && value <= 255))
                    {
                        try
                        {
                            var host = System.Net.Dns.GetHostEntry(ip);
                            //                            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{number}] resolves to [{ip}][{host.HostName}].", this, GPALObjectType.Browser);
                            retVal = true;
                        }
                        catch (GPALException)
                        {
                            throw;
                        }
                        catch (Exception ex)
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{number}][{ip}] doesn't resolve to a hostname.", this, GPALObjectType.Browser, ex);
                        }
                    }
 
                }
                catch (GPALException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"[{number}] doesn't resolve to an IP address", this, GPALObjectType.Browser, ex);
                }
            }
 
            return retVal;
        }
        #endregion <Private>
        public void Close(bool killWebDrivers = false)
        {
            InAction(false, false);
 
            tabsWeOpened = CurrentTabIdx = 0;
            BrowserLaunched = null;
 
            // a script registered through the extension is registered in the profile, not in this browser, and
            // stays there once the browser has gone. the next browser on that profile runs it, whatever engine
            // drives it and whatever it was asked to do, so what this workflow put there it takes away
            if (true == UseOttoMagic && true == _scriptsRegistered)
                ClearInjectedScripts();
 
            StopCasting();
 
            // the download directory was pointed at ours for the workflow, and it is the user's browser
            if (true == BrowserSettings.DownloadBehaviorSet)
            {
                BrowserSettings.DownloadBehaviorSet = false;
                PuppeteerCommunicator.RestoreDownloadBehavior().GetAwaiter().GetResult();
            }
 
            if (true == BrowserSettings.AttachedRestApi)
            {
                // someone else started it and may still be using it, so the window goes and nothing else
                MagicHelper.CloseWindow(BrowserSettings.CurrentURL);
            }
            else if (1 == GPAL.Browsers.Count || true == killWebDrivers)
            {
                if (true == UsePuppeteer)
                    BrowserSettings.PuppeteerCommunicator._isRunning = false;
 
                // kill all removes temp profile directory if it exists
                BrowserHelper.KillAllRunningProcesses(killWebDrivers, this);
            }
            else
            {
                if (true == UseSelenium)
                    BrowserDriver.Close();
                else if (true == UseOttoMagic && true == IsAlive)
                    MagicHelper.CloseWindow(BrowserSettings.CurrentURL);
                else if (true == UsePuppeteer)
                    PuppeteerClient.CloseWindow().Execute();
            }
 
            // firefox rewrites its prefs.js on the way out the same as chrome does, so this waits for the same
            // reason: the process has to be gone first
            if (null != BrowserSettings.PreviousFirefoxPreferences)
            {
                BrowserHelper.WaitForBrowserToExit(BrowserSettings, ProfileRestoreWaitMs);
                FirefoxProfileManager.RestoreUserJs(BrowserSettings.ProfileDataDirectory, BrowserSettings.PreviousFirefoxPreferences);
                BrowserSettings.PreviousFirefoxPreferences = null;
            }
 
            // chrome writes Preferences as it shuts down, so anything we put back while it is still running is
            // overwritten. the profile only goes back once the process is gone
            if (null != BrowserSettings.PreviousDownloadPreferences)
            {
                BrowserHelper.WaitForBrowserToExit(BrowserSettings, ProfileRestoreWaitMs);
                ChromeProfileManager.RestoreDownloadPreferences(BrowserSettings.ProfileDataDirectory, BrowserSettings.PreviousDownloadPreferences);
                BrowserSettings.PreviousDownloadPreferences = null;
            }
 
            // a chromium browser hosting an extension's background page does not stop when its last window
            // closes; it starts a windowless replacement that holds the profile open. ours started it, so ours
            // ends it, rather than leaving it for the next run to fail on
            BrowserHelper.EndWindowlessSuccessor(BrowserSettings);
 
            BrowserSettings.Process = null;
 
            GPAL.Browsers.Remove(this);
        }
        internal void ResolveInteractionType(Selector selector)
        {
            InteractionType engineDefault = true == UseOttoMagic ? InteractionType.OttoMagic
                                          : true == UsePuppeteer ? InteractionType.Puppeteer
                                          : InteractionType.Selenium;
            InteractionType resolved = engineDefault;
            bool wantsHardware = InteractionType.Hardware == selector.RequestedInteractionType || true == BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware;
 
            if (true == wantsHardware)
            {
                if (false == BrowserSettings.UseHeadless)
                    resolved = InteractionType.Hardware;
                else if (InteractionType.Hardware == selector.RequestedInteractionType)
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selector [{selector.Name}] asked for hardware interaction, which a headless browser has no mouse or keyboard to perform. Using [{engineDefault}].", this, GPALObjectType.Browser);
            }
            else if (InteractionType.JavaScript == selector.RequestedInteractionType || true == BrowserSettings.UseJavaScript)
                // ottomagic already drives the page through injected javascript, so its own interaction is the answer
                resolved = true == UseOttoMagic ? InteractionType.OttoMagic : InteractionType.JavaScript;
            else if (InteractionType.Selenium == selector.RequestedInteractionType && false == UseSelenium)
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selector [{selector.Name}] asked for selenium interaction, which the [{AutomationEngine}] engine has no driver for. Using [{engineDefault}].", this, GPALObjectType.Browser);
 
            selector.InteractionType = resolved;
        }
        public void RemoveCallIfHandlerEverywhere(CallIfDelegate func)
        {
            // soft delete. the handler stays in the list and gets skipped when the chain runs, which keeps a
            // reused selector's handlers intact for the next workflow and keeps us from mutating a list
            // CallIfHandlers is in the middle of iterating - this is nearly always called from inside a handler
            foreach (Selector selector in CurrentUOW.WithSelectorList)
                if (false == selector.RemovedMethods.Contains(func))
                    selector.RemovedMethods.Add(func);
 
            foreach (Selector selector in CurrentUOW.InSelectorList)
                if (false == selector.RemovedMethods.Contains(func))
                    selector.RemovedMethods.Add(func);
 
            foreach (Selector selector in persistentUOW.WithSelectorList)
                if (false == selector.RemovedMethods.Contains(func))
                    selector.RemovedMethods.Add(func);
        }
        public bool IsEndOfPage()
        {
            if (true == UseOttoMagic)
                return MagicHelper.IsEndOfPage();
            else if (true == UsePuppeteer)
            {
                try
                {
                    bool tmp = PuppeteerClient.IsEndOfPage().Execute<bool>();
                    return tmp;
                }
                catch
                {
                    return true;
                }
            }
            else
            {
                try
                {
                    // JavaScript code to check if at end of page
                    string script = @"
                        return (function() {
                          try {
                            const overflow = getComputedStyle(document.documentElement).overflow;
                            if (overflow !== 'hidden') {
                              return (window.innerHeight + window.scrollY) >= document.body.scrollHeight;
                            }
                            const scrollable = Array.from(document.querySelectorAll('*')).find(
                              el => el.scrollHeight > el.clientHeight && getComputedStyle(el).overflowY !== 'hidden'
                            ) || document.body;
                            return (scrollable.clientHeight + scrollable.scrollTop) >= scrollable.scrollHeight;
                          } catch {
                            return false;
                          }
                        })()";
 
                    // Execute the script
                    ExecuteJavaScriptObj(script);
 
                    // Parse result
                    if (bool.TryParse(JavaScriptResultObj?.ToString(), out bool val))
                    {
                        return val;
                    }
                    return false;
                }
                catch
                {
                    return false;
                }
            }
        }
        #endregion <Helpers>
        #region Window Handling
        public IAllowBrowserActionOrAnySelector NextWindow
        {
            get
            {
                InAction(false);
                if (true == UseOttoMagic)
                {
                    WindowTuple window = MagicHelper.NextWindow();
                    foreach (var key in BrowserSettings.WindowIdsToURL.Keys)
                        BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
                    BrowserSettings.WindowIdsToURL[window.WindowId] = new WindowInfo { URL = window.Url, ActiveWindow = true };
                    BrowserSettings.CurrentURL = window.Url;
                }
                else if (true == UsePuppeteer)
                {
                    PuppeteerClient.NextWindow().Execute();
                }
                else // Selenium
                {
                    // Get all tracked windows from your dictionary
                    var trackedWindows = BrowserSettings.WindowIdsToURL.Keys.ToList();
 
                    if (trackedWindows.Count <= 1)
                        return this; // Nothing to cycle to
 
                    // Find the current active one
                    string currentHandle = trackedWindows.FirstOrDefault(h =>
                        BrowserSettings.WindowIdsToURL[h].ActiveWindow);
 
                    // If none marked active (edge case), fall back to Selenium's current handle
                    if (string.IsNullOrEmpty(currentHandle))
                        currentHandle = BrowserDriver.CurrentWindowHandle;
 
                    // Find its index in the list
                    int currentIndex = trackedWindows.IndexOf(currentHandle);
                    if (currentIndex == -1) currentIndex = 0; // safety
 
                    // Calculate next index (cycle)
                    int nextIndex = (currentIndex + 1) % trackedWindows.Count;
                    string nextHandle = trackedWindows[nextIndex];
 
                    // Switch to it in Selenium
                    BrowserDriver.SwitchTo().Window(nextHandle);
 
                    // Update active flags
                    foreach (var key in BrowserSettings.WindowIdsToURL.Keys)
                        BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
 
                    BrowserSettings.WindowIdsToURL[nextHandle].ActiveWindow = true;
 
                    // Optional: refresh URL in case page navigated independently
                    BrowserSettings.WindowIdsToURL[nextHandle].URL = BrowserDriver.Url;
 
                    BrowserSettings.CurrentURL = BrowserDriver.Url;
                }
 
                GetSetCurrentUrl();
                return this;
            }
        }

        public IAllowBrowserActionOrAnySelector OpenWindow(GPALUrl URL)
        {
            InAction(false);
 
            string urlString = MagicHelper.GetFullUrl(URL?.Url, this, out _areRobotsAllowed);
 
            if (false == AreRobotsAllowed && true == BrowserSettings.ObeyRobotsTxt)
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Visiting [{URL?.Url}] is disallowed by robots.txt and your request to honor it. Not going to URL. Workflow will fail.", this, GPALObjectType.Browser);
                URL.ForUrl("https://google.com");
            }
 
            if (UseOttoMagic)
            {
                WindowTuple window = MagicHelper.OpenWindow(urlString);
 
                // Deactivate all others
                foreach (var key in BrowserSettings.WindowIdsToURL.Keys.ToList())
                {
                    BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
                }
 
                // Add or update the new one
                BrowserSettings.WindowIdsToURL[window.WindowId] = new WindowInfo
                {
                    URL = window.Url,
                    ActiveWindow = true
                };
 
                BrowserSettings.CurrentURL = window.Url ?? urlString;
            }
            else if (UsePuppeteer)
            {
                PuppeteerClient.OpenWindow(urlString).Execute();
                // Assume Puppeteer updates CurrentURL internally, or fetch it later
                BrowserSettings.CurrentURL = urlString; // fallback if not updated elsewhere
            }
            else // Selenium
            {
                string originalHandle = BrowserDriver.CurrentWindowHandle;
 
                // This creates a new top-level window and switches to it automatically
                BrowserDriver.SwitchTo().NewWindow(WindowType.Window);
 
                // Now we're in the new window
                string newHandle = BrowserDriver.CurrentWindowHandle;
 
                tabWindows[newHandle] = newHandle;   // a window is its own window, and the first tab in it
 
                // Navigate to the desired URL
                BrowserHelper.SeleniumGoToUrl(urlString, BrowserSettings);
 
                // Update your internal tracking
                foreach (var key in BrowserSettings.WindowIdsToURL.Keys.ToList())
                {
                    BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
                }
 
                BrowserSettings.WindowIdsToURL[newHandle] = new WindowInfo
                {
                    URL = urlString,
                    ActiveWindow = true
                };
 
                BrowserSettings.CurrentURL = BrowserDriver.Url; // should be the url now
            }
 
            URL = GetSetCurrentUrl();
 
            // NOTE: use evaluation shortcuiting, if it's not set, don't even check, it has a performance cost
            if (true == BrowserSettings.RespectRobotMetaTags && false == UrlHelper.CheckRobotMetaTags(URL, this))
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Visiting [{BrowserSettings.CurrentURL}] is disallowed by robot meta tags and your request to honor them..");
                URL.ForUrl("https://google.com");
 
                if (true == UseOttoMagic)
                {
                    TabTuple tabTuple = MagicHelper.GoTo(URL);
                    UpdateActiveTab(tabTuple);
                }
                else if (true == UsePuppeteer)
                {
                    PuppeteerClient.SwitchToDefaultContent().Execute();
                    PuppeteerClient.GoTo(URL).Execute();
                }
                else
                {
                    BrowserDriver.SwitchTo().DefaultContent();
                    BrowserHelper.SeleniumGoToUrl(URL, BrowserSettings);
                }
            }
 
            BrowserSettings.GPALUrlList.Add(URL);
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector PreviousWindow
        {
            get
            {
                InAction(false);
 
                if (UseOttoMagic)
                {
                    WindowTuple window = MagicHelper.PreviousWindow();
 
                    foreach (var key in BrowserSettings.WindowIdsToURL.Keys)
                        BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
 
                    BrowserSettings.WindowIdsToURL[window.WindowId] = new WindowInfo
                    {
                        URL = window.Url,
                        ActiveWindow = true
                    };
 
                    BrowserSettings.CurrentURL = window.Url;
                }
                else if (UsePuppeteer)
                {
                    PuppeteerClient.PreviousWindow().Execute();
                }
                else // Selenium
                {
                    // Get all tracked windows from dictionary (preserves insertion order)
                    var trackedWindows = BrowserSettings.WindowIdsToURL.Keys.ToList();
 
                    if (trackedWindows.Count <= 1)
                        return this; // Nothing to cycle to
 
                    // Find the current active one
                    string currentHandle = trackedWindows.FirstOrDefault(h =>
                        BrowserSettings.WindowIdsToURL[h].ActiveWindow);
 
                    // Fallback to Selenium's current handle if none marked active (safety)
                    if (string.IsNullOrEmpty(currentHandle))
                        currentHandle = BrowserDriver.CurrentWindowHandle;
 
                    // Find its index
                    int currentIndex = trackedWindows.IndexOf(currentHandle);
                    if (currentIndex == -1) currentIndex = 0;
 
                    // Calculate previous index (wrap around to the last one if at the beginning)
                    int previousIndex = (currentIndex - 1 + trackedWindows.Count) % trackedWindows.Count;
                    string previousHandle = trackedWindows[previousIndex];
 
                    // Switch to it in Selenium
                    BrowserDriver.SwitchTo().Window(previousHandle);
 
                    // Update active flags
                    foreach (var key in BrowserSettings.WindowIdsToURL.Keys)
                        BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
 
                    BrowserSettings.WindowIdsToURL[previousHandle].ActiveWindow = true;
 
                    // Optional: refresh stored URL in case the page changed
                    BrowserSettings.WindowIdsToURL[previousHandle].URL = BrowserDriver.Url;
 
                    BrowserSettings.CurrentURL = BrowserDriver.Url;
                }
 
                GetSetCurrentUrl();
                return this;
            }
        }
        
        public IAllowBrowserActionOrAnySelector GoToWindow(dynamic urlOrId)
        {
            InAction(false);
            if (true == UseOttoMagic)
            {
                string windowId = null;
                WindowTuple window = null;
 
                if (urlOrId is GPALUrl url)
                {
                    windowId = BrowserSettings.WindowIdsToURL.FirstOrDefault(x => x.Value.URL.Contains(url.Url) || url.Url.Contains(x.Value.URL)).Key;
                    BrowserSettings.CurrentGPALUrl = url;
                }
                else if (urlOrId is string urlStr)
                    windowId = BrowserSettings.WindowIdsToURL.FirstOrDefault(x => x.Value.URL.Contains(urlStr) || urlStr.Contains(x.Value.URL)).Key;
 
                if (true == string.IsNullOrEmpty(windowId))
                    window = MagicHelper.GoToWindow(urlOrId);
                else
                    window = MagicHelper.GoToWindow(windowId);
 
                foreach (var key in BrowserSettings.WindowIdsToURL.Keys)
                    BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
 
                if (false == BrowserSettings.WindowIdsToURL.ContainsKey(window.WindowId))
                    BrowserSettings.WindowIdsToURL[window.WindowId] = new WindowInfo { URL = window.Url, ActiveWindow = true };
                else
                    BrowserSettings.WindowIdsToURL[window.WindowId].ActiveWindow = true;
 
                BrowserSettings.CurrentURL = window.Url;
            }
            else if (true == UsePuppeteer)
            {
                string windowId = null;
                WindowTuple window = null;
 
                if (urlOrId is GPALUrl url)
                {
                    windowId = BrowserSettings.WindowIdsToURL.FirstOrDefault(x => x.Value.URL.Contains(url.Url) || url.Url.Contains(x.Value.URL)).Key;
                    BrowserSettings.CurrentGPALUrl = url;
                }
                else if (urlOrId is string urlStr)
                    windowId = BrowserSettings.WindowIdsToURL.FirstOrDefault(x => x.Value.URL.Contains(urlStr) || urlStr.Contains(x.Value.URL)).Key;
 
                if (true == string.IsNullOrEmpty(windowId))
                    window = PuppeteerClient.GoToWindow(urlOrId).Execute<WindowTuple>();
                else if (true == Int32.TryParse(windowId, out int windowIdInt))
                    window = PuppeteerClient.GoToWindow(windowIdInt).Execute<WindowTuple>();
 
                foreach (var key in BrowserSettings.WindowIdsToURL.Keys)
                    BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
 
                if (false == BrowserSettings.WindowIdsToURL.ContainsKey(window.WindowId))
                    BrowserSettings.WindowIdsToURL[window.WindowId] = new WindowInfo { URL = window.Url, ActiveWindow = true };
                else
                    BrowserSettings.WindowIdsToURL[window.WindowId].ActiveWindow = true;
 
                BrowserSettings.CurrentURL = window.Url;
            }
            else if (true == UseSelenium)
            {
                string theUrl = null;
                string targetHandle = null;
 
                if (urlOrId is GPALUrl url)
                    theUrl = url.Url;
 
                foreach (var handle in BrowserDriver.WindowHandles)
                {
                    BrowserDriver.SwitchTo().Window(handle);
                    if (urlOrId is string str && (handle == str || BrowserDriver.Url.Contains(str)))
                    {
                        targetHandle = handle;
                        break;
                    }
                }
 
                if (targetHandle != null)
                {
                    BrowserDriver.SwitchTo().Window(targetHandle);
                    BrowserSettings.CurrentURL = BrowserDriver.Url;
                }
                else
                {
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Failed to detect window with URL/ID [{urlOrId}]", this, GPALObjectType.Browser);
                }
            }
 
            GetSetCurrentUrl();
            return this;
        }

        public IAllowBrowserActionOrAnySelector CloseWindow(dynamic urlOrId = null)
        {
            // NOTE: CAVEAT: we aren't currently using the id we planned for with ottomagic, but this will cause problems if we do
            urlOrId = MagicHelper.GetFullUrl(urlOrId, this, out bool _);
 
            InAction(false);
 
            BrowserSettings.CurrentGPALUrl = null;
 
            if (true == UseOttoMagic)
            {
                string windowIdToClose = null;
 
                if (urlOrId is GPALUrl url)
                {
                    if (false == string.IsNullOrEmpty(url.Url))
                        // Find by partial URL match
                        windowIdToClose = BrowserSettings.WindowIdsToURL
                            .FirstOrDefault(x => x.Value.URL.Contains(url.Url) || url.Url.Contains(x.Value.URL))  // NOTE: CAVEAT: is doing contains both ways safe?
                            .Key;
                }
                else if (urlOrId is string urlStr)
                {
                    if (false == string.IsNullOrEmpty(urlStr))
                        // Find by partial URL match
                        windowIdToClose = BrowserSettings.WindowIdsToURL
                            .FirstOrDefault(x => x.Value.URL.Contains(urlStr) || urlStr.Contains(x.Value.URL))  // NOTE: CAVEAT: is doing contains both ways safe?
                            .Key;
                }
                else
                {
                    // Default: close the currently active window
                    windowIdToClose = BrowserSettings.WindowIdsToURL
                        .FirstOrDefault(x => true == x.Value.ActiveWindow)
                        .Key;
                }
 
                if (string.IsNullOrEmpty(windowIdToClose))
                {
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR,
                        $"No window found to close (URL containing [{urlOrId}])", this, GPALObjectType.Browser);
                    return this;
                }
 
                // Let MagicHelper close it and return the new active window
                WindowTuple newActiveWindow = MagicHelper.CloseWindow(windowIdToClose);
 
                // Remove from tracking
                BrowserSettings.WindowIdsToURL.Remove(windowIdToClose);
 
                // Update active status for the new one (if any windows remain)
                if (newActiveWindow != null && !string.IsNullOrEmpty(newActiveWindow.WindowId))
                {
                    foreach (var key in BrowserSettings.WindowIdsToURL.Keys)
                        BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
 
                    BrowserSettings.WindowIdsToURL[newActiveWindow.WindowId] = new WindowInfo
                    {
                        URL = newActiveWindow.Url,
                        ActiveWindow = true
                    };
 
                    BrowserSettings.CurrentURL = newActiveWindow.Url;
                }
                else if (BrowserSettings.WindowIdsToURL.Count == 0)
                {
                    BrowserSettings.CurrentURL = null;
                }
            }
            else if (true == UsePuppeteer)
            {
                // Puppeteer handles its own tracking and current URL update
                if (urlOrId is GPALUrl url)
                    PuppeteerClient.CloseWindow(url).Execute();
                else if (urlOrId is string urlStr)
                    PuppeteerClient.CloseWindow(urlStr).Execute();
                else if (urlOrId is int id)
                    PuppeteerClient.CloseWindow(id).Execute();
                else
                    PuppeteerClient.CloseWindow().Execute();
            }
            else // Selenium
            {
                string handleToClose = null;
 
                if (urlOrId is string urlStr)
                {
                    // Search by partial URL match
                    handleToClose = BrowserSettings.WindowIdsToURL
                        .FirstOrDefault(x => x.Value.URL.Contains(urlStr) || urlStr.Contains(x.Value.URL))
                        .Key;
                }
                else
                {
                    // Default: close the currently active window
                    handleToClose = BrowserSettings.WindowIdsToURL
                        .FirstOrDefault(x => x.Value.ActiveWindow)
                        .Key;
 
                    // Fallback to Selenium's current handle if not marked
                    if (string.IsNullOrEmpty(handleToClose))
                        handleToClose = BrowserDriver.CurrentWindowHandle;
                }
 
                if (string.IsNullOrEmpty(handleToClose) ||
                    !BrowserSettings.WindowIdsToURL.ContainsKey(handleToClose))
                {
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR,
                        $"No window found to close (URL containing [{urlOrId}])", this, GPALObjectType.Browser);
                    return this;
                }
 
                // Switch to the target window and close it
                BrowserDriver.SwitchTo().Window(handleToClose);
                BrowserDriver.Close(); // Closes the current window/tab
 
                // Remove from internal tracking
                BrowserSettings.WindowIdsToURL.Remove(handleToClose);
 
                // If no windows left
                if (BrowserDriver.WindowHandles.Count == 0)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, "No windows remain after closing.", this, GPALObjectType.Browser);
                    return this;
                }
 
                // Determine which window Selenium switched to automatically
                string newActiveHandle = BrowserDriver.CurrentWindowHandle;
 
                // Update active flags
                foreach (var key in BrowserSettings.WindowIdsToURL.Keys)
                    BrowserSettings.WindowIdsToURL[key].ActiveWindow = false;
 
                if (BrowserSettings.WindowIdsToURL.ContainsKey(newActiveHandle))
                {
                    BrowserSettings.WindowIdsToURL[newActiveHandle].ActiveWindow = true;
                    BrowserSettings.WindowIdsToURL[newActiveHandle].URL = BrowserDriver.Url; // refresh
                }
                else
                {
                    // Fallback: pick first remaining and mark active (shouldn't happen normally)
                    var firstRemaining = BrowserSettings.WindowIdsToURL.Keys.FirstOrDefault();
                    if (firstRemaining != null)
                    {
                        BrowserSettings.WindowIdsToURL[firstRemaining].ActiveWindow = true;
                        BrowserDriver.SwitchTo().Window(firstRemaining);
                    }
                }
            }
 
            GetSetCurrentUrl();
            return this;
        }
 
        #endregion Window Handling
        #region WithWorkFlow, Run, While, Until, Generated by Grok
        public IAllowBrowserActionOrAnySelector Run()
        {
            WorkflowManager.Run();
            return this;
        }
        /// browser handed to each is this one.
        /// </summary>
        /// <remarks>
        /// The list is what <see cref="While"/> and <see cref="Until(Selector)"/> repeat, which is what makes it
        /// worth having: a page of results and a Next button is one workflow run until the button is gone.
        /// <br/><br/>
        /// To run several workflows at once, each with a browser of its own and possibly a different browser or
        /// engine, use <see cref="GPAL.Workflow"/>. This browser runs its workflows sequentially and always has.
        /// </remarks>
        /// <example>
        /// Page through search results, gathering each page, until the Next button is no longer there:
        /// <code>
        /// browser
        ///     .GoTo("https://www.example.com/search?q=laptop")
        ///     .WithWorkflow(b =&gt;
        ///     {
        ///         b.WithSelector(GPAL.CssSelector(".result-title"))
        ///          .WithAllThatMatch()
        ///          .GetElements(out List&lt;GPALElement&gt; found);
        ///
        ///         results.AddRange(found);
        ///
        ///         b.WithSelector(GPAL.CssSelector("a.next")).LeftClick();
        ///     })
        ///     .Until(b =&gt; 0 == b.CurrentUOW.WebSelectorFoundResults.Count);
        /// </code>
        /// </example>
        /// <param name="workflow">The workflow to run. The browser passed in is this browser.</param>
        /// <returns>Fluent interface to add another workflow, or to run them</returns>
        public IAllowWorkflowExecution WithWorkflow(Action<IBrowser> workflow)
        {
            return WorkflowManager.WithWorkflow(workflow);
        }

        public IAllowWorkflowExecution WhileLoopTimeout(int timeoutMs)
        {
            if (timeoutMs <= 0)
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Invalid timeout ([{timeoutMs}]ms) specified. Using default [{BrowserSettings.WhileLoopTimeoutMs}]ms.", this, GPALObjectType.Browser);
            }
            else
            {
                BrowserSettings.WhileLoopTimeoutMs = timeoutMs;
            }
            return this;
        }

        public IAllowWorkflowExecution WhileLoopMaxIterations(int maxIterations)
        {
            if (maxIterations <= 0)
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Invalid max iterations ([{maxIterations}]) specified. Using default [{BrowserSettings.WhileLoopMaxIterations}].", this, GPALObjectType.Browser);
            }
            else
            {
                BrowserSettings.WhileLoopMaxIterations = maxIterations;
            }
            return this;
        }


        public IAllowBrowserActionOrAnySelector While(Func<Browser, bool> predicate)
        {
            return WorkflowManager.While(predicate);
        }


        public IAllowBrowserActionOrAnySelector Until(Selector selector)
        {
            return WorkflowManager.Until(selector);
        }


        /// <param name="predicate">A function evaluated against the <see cref="Browser"/> after each iteration; the loop stops once it returns true</param>
        /// <returns>Fluent interface to write your workflow</returns>
        public IAllowBrowserActionOrAnySelector Until(Func<Browser, bool> predicate)
        {
            return WorkflowManager.Until(predicate);
        }
        #endregion WithWorkFlow, Run, While, Until, Generated by Grok
        #region OCR
        #endregion OCR
        #region Getters/Setters
        public List<string> RobotsTxt
        {
            get => BrowserSettings.RobotsTxt;
            internal set => BrowserSettings.RobotsTxt = value;
        }
        public object JavaScriptResultObj
        {
            get => BrowserSettings.JavaScriptResultObj;
            internal set => BrowserSettings.JavaScriptResultObj = value;
        }
        public string JavaScriptResultStr
        {
            get => BrowserSettings.JavaScriptResultStr;
            internal set => BrowserSettings.JavaScriptResultStr = value;
        }
        public string BrowserVersion
        {
            get => BrowserSettings.Version;
            internal set => BrowserSettings.Version = value;
        }
 
        #endregion
        #region Helpers
        internal static string GetModifierKeys(ModifierKeys modifierKeys)
        {
            if (modifierKeys == ModifierKeys.NONE)
                return "None";
 
            var setFlags = Enum.GetValues(typeof(ModifierKeys))
                .Cast<ModifierKeys>()
                .Where(flag => flag != ModifierKeys.NONE && modifierKeys.HasFlag(flag))
                .Select(flag => flag.ToString());
 
            return string.Join(", ", setFlags);
        }
        internal string GetSetCurrentUrl()
        {
            string currentUrl = null;
 
            try
            {
                if (true == BrowserSettings.UseOttoMagic)
                    currentUrl = MagicHelper.GetCurrentUrl();
                else if (true == UsePuppeteer)
                    currentUrl = BrowserSettings.Browser.PuppeteerClient.GetCurrentUrl().Execute<dynamic>();
                else
                    // read the url of whatever tab Selenium currently has active - do NOT switch by our own
                    // CurrentTabIdx, which if stale would switch to the wrong tab and read the wrong url
                    currentUrl = BrowserDriver.SwitchTo().Window(BrowserDriver.CurrentWindowHandle).Url;
 
                currentUrl = currentUrl?.Trim('"');
 
                if (false == UrlHelper.AreEquivalent(currentUrl, BrowserSettings.CurrentURL))
                {
                    BrowserSettings.CurrentURL = currentUrl;
 
                    // invalidate cached elements, we are on a new page
                    foreach (Selector selector in this.CurrentUOW.WithSelectorList)
                    {
                        selector.WebSelectorFoundResults = null;
                        selector.WebSelectorMatchedResults = null;
                    }
                }
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to get URL, returning [{BrowserSettings.CurrentURL}]", this, GPALObjectType.Browser, ex);
            }
            return BrowserSettings.CurrentURL;
        }
        internal void SetReferrer()
        {
            string message = $"[{BrowserSettings.Referrer}] referrer applied.";
            var parms = new Dictionary<string, object>
                        {
                            {
                                "headers", new Dictionary<string, object>
                                {
                                    { "Referer", BrowserSettings.Referrer }    // NOTE: the referrer misspelling is baked into the http/1.0 protocol spec
                                }
                            }
                        };
 
            if (null != BrowserSettings.Referrer)
            {
                if (true == UsePuppeteer)
                    PuppeteerClient.OverrideReferrer(BrowserSettings.Referrer).Execute();
                else if (true == BrowserSettings.UseOttoMagic)
                    MagicHelper.OverrideReferrer(BrowserSettings.Referrer);
                else if (BrowserType.FireFox != BrowserSettings.BrowserType)
                    ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand("Network.setExtraHTTPHeaders", parms);
 
                // clear it, then ignore it.
                if (true == string.Empty.Equals(BrowserSettings.Referrer))
                {
                    BrowserSettings.Referrer = null;
                    message = $"Cleared referrer.";
                }
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, message, this, GPALObjectType.Browser);
            }
        }
        internal void SetUserAgent()
        {
            string message = $"[{BrowserSettings.OverrideUserAgent}] user agent applied.";
            var parms = new Dictionary<string, object> { { "userAgent", BrowserSettings.OverrideUserAgent } };
 
            if (null != BrowserSettings.OverrideUserAgent)
            {
                if (true == UsePuppeteer)
                    PuppeteerClient.SetUserAgent(BrowserSettings.OverrideUserAgent).Execute();
                else if (true == BrowserSettings.UseOttoMagic)
                    MagicHelper.SetUserAgent(BrowserSettings.OverrideUserAgent);
                else if (BrowserType.FireFox != BrowserSettings.BrowserType)
                    ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand("Network.setUserAgentOverride", parms);
 
                // clear it, then ignore it.
                if (true == string.Empty.Equals(BrowserSettings.OverrideUserAgent))
                {
                    BrowserSettings.OverrideUserAgent = null;
                    message = $"Cleared user agent override.";
                }
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, message, this, GPALObjectType.Browser);
            }
        }
        #endregion Helpers
        #region Sitep Operations
        public IAllowBrowserActionOrAnySelector GetHydratedData(out NextJsHydrationResult hydrationResult)
        {
            hydrationResult = CachedHydration();
            return this;
        }

        private NextJsHydrationResult CachedHydration()
        {
            string key = GetSetCurrentUrl();
 
            if (false == string.IsNullOrEmpty(key) && true == _hydrationCache.TryGetValue(key, out NextJsHydrationResult cached))
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Hydrated data cache hit for [{UrlHelper.GetOrigin(CurrentUrl)}]", this, GPALObjectType.Browser);
                return cached;
            }
 
            var result = NextJsHydrationExtractor.Extract(this, GetHydrationSource());
 
            if (false == string.IsNullOrEmpty(key))
                _hydrationCache[key] = result;
 
            return result;
        }
        private string GetHydrationSource()
        {
            string raw = string.Empty;
 
            if (true == UseOttoMagic)
                raw = MagicHelper.GetPageSource();
            else if (true == UsePuppeteer)
                raw = PuppeteerClient.GetPageSource().Execute();
            else
            {
                try { raw = BrowserDriver.PageSource; }
                catch (GPALException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, "Failed to read PageSource for hydration [" + ex.Message + "]", this, GPALObjectType.Browser);
                }
            }
 
            if (string.IsNullOrWhiteSpace(raw))
                return string.Empty;
 
            string peeled = JsonUnescapeToStable(raw.Trim().Trim('\"').Trim('\''));
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Hydrated data created for [{UrlHelper.GetOrigin(CurrentUrl)}]", this, GPALObjectType.Browser);
 
            return Regex.Replace(peeled, @"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "");
        }

        public IAllowBrowserActionOrAnySelector SaveHydratedData(GPALFile hydrationFile)
        {
            NextJsHydrationExtractor.SaveToFile(CachedHydration(), hydrationFile.Filename);
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Saved hydrated data [{hydrationFile.Filename}]", this, GPALObjectType.Browser);
 
            return this;
        }
        public IAllowBrowserActionOrAnySelector GetLLMDigest(out LLMDigestResult digestResult, string ruleSetName = null)
        {
            digestResult = CachedDigest(ruleSetName);
            return this;
        }

        /// GetLLMDigest followed by SaveLLMDigest does not fetch and convert the page twice.
        /// </summary>
        private LLMDigestResult CachedDigest(string ruleSetName)
        {
            string key = GetSetCurrentUrl() + "|" + (ruleSetName ?? string.Empty);
 
            if (true == _digestCache.TryGetValue(key, out LLMDigestResult cached))
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"LLM Digest cache hit for [{UrlHelper.GetOrigin(CurrentUrl)}]", this, GPALObjectType.Browser);
                return cached;
            }
 
            GetPageSource(out string pageSource);
            LLMDigestResult digestResult = HtmlToOptimizedMarkdown.CreateDigest(this, pageSource, ruleSetName);
            _digestCache[key] = digestResult;
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"LLM Digest created for [{UrlHelper.GetOrigin(CurrentUrl)}]", this, GPALObjectType.Browser);
 
            return digestResult;
        }
        public IAllowBrowserActionOrAnySelector SaveLLMDigest(GPALFile llmDigestFile, string ruleSetName = null)
        {
            HtmlToOptimizedMarkdown.SaveToFile(CachedDigest(ruleSetName), llmDigestFile.Filename);
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Saved LLM Digest [{llmDigestFile.Filename}]", this, GPALObjectType.Browser);
 
            return this;
        }
        public IAllowBrowserActionOrAnySelector GetPageSource(out string pageSource)
        {
            pageSource = string.Empty;
 
            if (true == UseOttoMagic)
            {
                pageSource = MagicHelper.GetPageSource();
            }
            else if (true == UsePuppeteer)
            {
                pageSource = PuppeteerClient.GetPageSource().Execute();
            }
            else
            {
                try
                {
                    pageSource = BrowserDriver.PageSource;
                }
                catch (GPALException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(
                        GPALEventType.ERROR,
                        "Failed to read page source in Selenium [" + ex.Message + "]",
                        this,
                        GPALObjectType.Browser);
                }
            }
 
            pageSource = NormalizePageSource(pageSource);
 
            Uri currUrl = new Uri(CurrentUrl, UriKind.Absolute);
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Retrieved page source for [{UrlHelper.NormalizePath(currUrl.AbsolutePath)}]", this, GPALObjectType.Browser);
 
            return this;
        }
        private static string NormalizePageSource(string raw)
        {
            if (string.IsNullOrWhiteSpace(raw))
                return string.Empty;
 
            // JS/JSON-unescape first so the XML-viewer marker (real quotes) is detectable
            // regardless of how the engine transported the source (e.g. OttoMagic returns it escaped).
            string working = SafeJsUnescape(raw.Trim().Trim('\"').Trim('\''));
 
            if (working.IndexOf("webkit-xml-viewer-source-xml", StringComparison.Ordinal) != -1)
            {
                string fileSource = ConverterHelper.ExtractRawXmlFromBrowserSource(working);
                if (!string.IsNullOrWhiteSpace(fileSource))
                {
                    // Strip control characters only; do NOT HTML-decode, that would corrupt escaped
                    // entities (e.g. &amp;) that are a genuine part of the recovered file source.
                    return Regex.Replace(fileSource.Trim(), @"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "");
                }
            }
 
            // Ordinary page: return the honest page source with the standard cleanup.
            return SafeCleanPageSource(raw);
        }
        private static string SafeCleanPageSource(string input)
        {
            if (string.IsNullOrWhiteSpace(input))
                return string.Empty;
 
            string working = input.Trim().Trim('\"').Trim('\'');
 
            // Peel the transport's JSON-string escaping FIRST. OttoMagic sends outerHTML as a JSON string
            // (leading quote, \" for attribute quotes, \n for newlines); HtmlDecode must run AFTER this,
            // because decoding entities injects raw quotes that make the still-escaped payload un-peelable.
            working = SafeJsUnescape(working);
 
            working = System.Web.HttpUtility.HtmlDecode(working);
 
            // Remove control characters only
            working = Regex.Replace(working, @"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "");
 
            return working.Trim();
        }

        private static string SafeJsUnescape(string input)
        {
            if (string.IsNullOrEmpty(input))
                return input;
 
            // Step 1: peel the transport's JSON-string escaping. OttoMagic's native messaging hands us the
            // source as a JSON string (leading quote, \" for attribute quotes, \n for newlines), sometimes
            // even double-escaped. Peel each valid JSON-string layer with a real parser instead of guessing
            // with String.Replace. This self-terminates: once the text holds raw quotes/newlines it is no
            // longer a JSON string token and parsing stops - that is exactly the decoded page source. For
            // Selenium/Puppeteer, whose source is already clean HTML, this is a no-op (peel fails immediately).
            string result = JsonUnescapeToStable(input);
 
            // Step 2: ALWAYS run the lenient character-level unescape on the result, so every engine converges
            // on identical output. Without this, OttoMagic (whose peel succeeds) would keep the single-level \"
            // left in the page's own embedded strings (e.g. __NEXT_DATA__ JSON), while Selenium/Puppeteer
            // (whose peel is a no-op) collapse those \" here - producing different source for the same page.
 
            // Basic escapes - but PROTECT newlines and carriage returns so they stay escaped for JSON
            result = result.Replace(@"\""", "\"")
                           .Replace(@"\'", "'")
                           .Replace(@"\\", "\\")     // keep single backslash
                           .Replace(@"\/", "/");
 
            // Safe unicode only
            result = Regex.Replace(result, @"\\u([0-9a-fA-F]{4})", m =>
            {
                try
                {
                    int code = int.Parse(m.Groups[1].Value, System.Globalization.NumberStyles.HexNumber);
                    if (code < 0 || code > 0x10FFFF || (code >= 0xD800 && code <= 0xDFFF))
                        return m.Value;        // keep original if invalid
                    return char.ConvertFromUtf32(code);
                }
                catch
                {
                    return m.Value;
                }
            });
 
            // Remove unknown escapes (but protect " \ ' \ / \b \f \t)
            result = Regex.Replace(result, @"\\‍([^""'\\/bfnrtu])", "$1");
 
            return result;
        }

        private static string JsonUnescapeToStable(string input)
        {
            string current = input;
 
            // Cap the passes: real content peels in 1-2 layers; the cap only guards against pathological input.
            for (int pass = 0; pass < 5; pass++)
            {
                string next = TryJsonStringDecode(current);
                if (null == next || next == current)
                    break;
                current = next;
            }
 
            return current;
        }


        private static string TryJsonStringDecode(string s)
        {
            if (string.IsNullOrEmpty(s))
                return null;
 
            try
            {
                return JsonConvert.DeserializeObject("\"" + s + "\"") as string;
            }
            catch
            {
                return null;
            }
        }

        public IAllowBrowserActionOrAnySelector GetSiteMap(out string sitemapXML)
        {
            GetPageSource(out sitemapXML);
            return this;
        }
 
        Dictionary<string, List<string>> siteMaps = new Dictionary<string, List<string>>();
        List<string> suggestedSitemaps = new List<string>();
 
        // Result caches so a Save* reuses what a preceding Get* already produced (and so we do not repeatedly
        // walk the same site). The sitemap is site-wide, keyed by origin; the digest/hydration results are
        // page-level, keyed by the full URL. Instance-scoped, so each browser/engine still does the work once;
        // call ClearResultCache() to force a refresh.
        private readonly Dictionary<string, List<string>> _sitemapCache = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase);
        private readonly Dictionary<string, LLMDigestResult> _digestCache = new Dictionary<string, LLMDigestResult>(StringComparer.OrdinalIgnoreCase);
        private readonly Dictionary<string, NextJsHydrationResult> _hydrationCache = new Dictionary<string, NextJsHydrationResult>(StringComparer.OrdinalIgnoreCase);

        public IAllowBrowserActionOrAnySelector ClearResultCache()
        {
            _sitemapCache.Clear();
            _digestCache.Clear();
            _hydrationCache.Clear();
            GPAL.PublishSimpleEvent(GPALEventType.INFO, "Cleared sitemap/digest/hydration result cache.", this, GPALObjectType.Browser);
            return this;
        }

        public IAllowBrowserActionOrAnySelector GetSiteMapUrls(out List<string> sitemapUrls)
        {
            sitemapUrls = CachedSitemapUrls();
            return this;
        }

        private List<string> CachedSitemapUrls()
        {
            string origin = UrlHelper.GetOrigin(CurrentUrl);
 
            if (false == string.IsNullOrEmpty(origin) && true == _sitemapCache.TryGetValue(origin, out List<string> cached))
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Sitemap cache hit for [{origin}] ([{cached.Count}] URLs).", this, GPALObjectType.Browser);
                return cached;
            }
 
            List<string> walked = WalkSitemapUrls();
 
            if (false == string.IsNullOrEmpty(origin))
                _sitemapCache[origin] = walked;
 
            return walked;
        }
 
        private List<string> WalkSitemapUrls()
        {
            List<string> sitemapUrls = new List<string>();
            string currentUrl = CurrentUrl;
            bool isFromRobotsTxt = false;
            SitemapExtractor sme = new SitemapExtractor(this);
            List<string> potentialSitemapUrls = sme.GetSitemapUrl(currentUrl, out isFromRobotsTxt);
            bool urlIsSitemap = false;
            IGPALGrid<string> retGrid = new GPALGrid<string>();
            int idx = 0;
            string pageSource = string.Empty;
 
            do
            {
                string potentialSitemapUrl = string.Empty;
                if (idx < potentialSitemapUrls.Count)
                    potentialSitemapUrl = potentialSitemapUrls[idx++];
 
                // before we walk to any potential sitemaps, is the current page the user directed us to a sitemap?
                // get the source and check if it looks like it, if so, follow it
                // if not, try the sitemaps we came up with, which might be from robots.txt
                if (false == isFromRobotsTxt)
                {
                    // a .gz current page is a binary download, not a page - fetch + inflate it instead of reading the tab
                    if (true == SitemapExtractor.IsGzipSitemapUrl(currentUrl))
                        pageSource = SitemapExtractor.FetchAndInflateGzipSitemap(currentUrl, this);
                    else
                        GetPageSource(out pageSource);
                }
 
                // IF NOT sitemap in robots.txt, and if the url comtains the literal word "sitemap" treat the current url as a sitemap
                // else run a heuristic upon it to see if it looks like one, it's not great...
                // NOTE: CAVEAT: if there is a sitemap in robots.txt, the page we are on is not examined to be a sitemap
                if (true == isFromRobotsTxt ||
                    (false == currentUrl.ToLower().Contains("sitemap") &&    // does url contain 'sitemap' it just might be
                    false == currentUrl.ToLower().EndsWith(".xml") &&       // does the url end in .xml, it better be a sitemap
                    false == SitemapExtractor.IsValidSitemapContent(pageSource) &&       // no sitemap name, no .xml, last check does it look like a sitemap? 10 links and 2 two end in xml? // NOTE: not sure of the heuristic
                    false == suggestedSitemaps.Contains(potentialSitemapUrl) // we've already suggestted this and decided against it...
                    ))
                {
                    // gzipped sitemaps (often the big ones) can't be navigated to - the browser downloads the
                    // binary and the WebDriver session hangs. Download + inflate directly instead.
                    if (true == SitemapExtractor.IsGzipSitemapUrl(potentialSitemapUrl))
                        pageSource = SitemapExtractor.FetchAndInflateGzipSitemap(potentialSitemapUrl, this);
                    else
                    {
                        GoTo(potentialSitemapUrl);
                        GetPageSource(out pageSource);
                    }
                }
                if (false == suggestedSitemaps.Contains(potentialSitemapUrl))
                    suggestedSitemaps.Add(potentialSitemapUrl);
 
                // a bot-check interstitial (e.g. PerimeterX "Robot or human?") is not a sitemap - recognize it,
                // warn, and stop walking this origin rather than parsing the challenge page as xml.
                if (true == SitemapExtractor.IsBotCheckPage(pageSource))
                {
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Bot-check interstitial detected at [{currentUrl}], skipping sitemap walk.", this, GPALObjectType.Browser);
                    break;
                }
 
                sitemapUrls = sme.ExtractSitemapUrls(pageSource, Int32.MaxValue, out urlIsSitemap);   // NOTE: CAVEAT: hardcoded value - we want them all - we need a setting for this? with all that match?
                retGrid.AddRow(sitemapUrls);
                string currentUrl2 = string.Empty;
 
                if (true == UseOttoMagic)
                    currentUrl2 = MagicHelper.GetCurrentUrl();
                else if (true == BrowserSettings.UsePuppeteer)
                    currentUrl2 = PuppeteerClient.GetCurrentUrl().Execute();
                else
                    currentUrl2 = BrowserDriver.Url;
 
                currentUrl2 = currentUrl2.Trim('\"');
 
                // how to break out of the loop?
                // 1 - we have no sitemap urls
                // 2 - we have a list and have walked all of them and accumulated all of the sitemap information walking all sitemaps
                if (0 == potentialSitemapUrls.Count || potentialSitemapUrls.Count == idx)
                {
                    if (false == UrlHelper.AreEquivalent(currentUrl, currentUrl2))
                        GoTo(currentUrl);
                    break;
                }
 
            } while (true);
 
            sitemapUrls = retGrid.ToList();
 
            return sitemapUrls;
        }
        public IAllowBrowserActionOrAnySelector SaveSiteMapUrls(GPALFile sitemapUrlsFile)
        {
            List<string> sitemapUrls = CachedSitemapUrls();
 
            // one URL per row, single "url" column, so the converter can emit any format from the file extension
            IGPALGrid<string> grid = new GPALGrid<string>();
            foreach (string url in sitemapUrls)
                grid.AddRow(new List<string> { url });
 
            GPAL.Converter
                .WithInput(grid)
                .WithColumnName("url")
                .SaveTo(sitemapUrlsFile);
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Saved [{sitemapUrls.Count}] sitemap URLs to [{sitemapUrlsFile.Filename}]", this, GPALObjectType.Browser);
 
            return this;
        }
        #endregion Sitemap Operations
        #region Cast Operations
        private string SinkName { get; set; }

        public IAllowBrowserActionOrAnySelector CastTab(string sinkName)
        {
            SinkName = sinkName;
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Casting Tab to [{sinkName}]");
 
            if (true == UsePuppeteer)
                PuppeteerClient.CastTab(sinkName).Execute();
            else if (true == UseSelenium)
                CastTo_Selenium(sinkName, CastMode.Tab);
            else if (true == UseOttoMagic)
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, "Casting is not available via OttoMagic");
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector CastDesktop(string sinkName)
        {
            SinkName = sinkName;
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Casting Desktop to [{sinkName}]");
 
            if (true == UsePuppeteer)
                PuppeteerClient.CastDesktop(sinkName).Execute();
            else if (true == UseSelenium)
                CastTo_Selenium(sinkName, CastMode.Desktop);
            else if (true == UseOttoMagic)
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, "Casting is not available via OttoMagic");
 
            return this;
        }

        public IAllowBrowserActionOrAnySelector StopCasting()
        {
            if (false == string.IsNullOrEmpty(SinkName))
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Stopping cast to [{SinkName}]");
 
                if (true == UsePuppeteer)
                {
                    if (null != PuppeteerCommunicator.SinkName)
                    {
                        PuppeteerClient.StopCasting().Execute();
                        PuppeteerCommunicator.SinkName = null;
                    }
                }
                else if (true == UseSelenium)
                {
                    var driver = (OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver;
 
                    var param = new Dictionary<string, object> { ["sinkName"] = SinkName };
 
                    if (null != SinkName)
                        try
                        {
                            // 1. Try official stop command
                            driver.ExecuteCdpCommand("Cast.stopCasting", param);
                            SinkName = null;
                        }
                        catch (Exception ex)
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Cast.stopCasting failed, which it does sometimes", this, GPALObjectType.Browser, ex);
                        }
                }
            }
 
            return this;
        }

        private IAllowBrowserActionOrAnySelector CastTo_Selenium(string sinkName, CastMode mode)
        {
            int retries = 10;
 
            if (string.IsNullOrEmpty(sinkName))
                return this;
 
            ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand("Cast.enable", 
                new Dictionary<string, object>());
 
            Thread.Sleep(3_000);
            var param = new Dictionary<string, object> { ["sinkName"] = sinkName };
 
            string command = (mode == CastMode.Tab)
                ? "Cast.startTabMirroring"
                : "Cast.startDesktopMirroring";
 
            while (0 < retries)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Searching for [{sinkName}].");
                try
                {
                    ((OpenQA.Selenium.Chromium.ChromiumDriver)BrowserDriver).ExecuteCdpCommand(command, param);
 
                    GPAL.PublishSimpleEvent(GPALEventType.INFO,
                        $"[{sinkName}] found.", this, GPALObjectType.Browser);
                    break;
                }
                catch
                {
                    retries--;
                }
                Thread.Sleep(3_000);
            }
            return this;
        }
        #endregion Cast Operations
    }
}