MagicHelper.cs

// =============================================================================
// GPAL - Generally Positive Automation Library
// Copyright © 2026 Software Decisions, Inc. All rights reserved.
//
// This file is part of GPAL.
// Licensed under the Business Source License 1.1
//
// Primary development, architecture, and vision by Michael B. Vederman,
// CEO of Software Decisions, Inc., Texas.
//
// Internal development maintained privately.
// Public releases appear on GitHub: https://github.com/SoftwareDecisionsInc/GPAL.
//
// See LICENSE for full terms, including Additional Use Grant.
// =============================================================================
 
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.IO.Pipes;
using System.Linq;
using System.Management;
using System.Net;
using System.Net.Sockets;
using System.Runtime.InteropServices;
using System.Security.Policy;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using DocumentFormat.OpenXml.Office2010.PowerPoint;
using GenerallyPositive.Application;
using Microsoft.Win32.SafeHandles;
using OpenQA.Selenium;
using OpenQA.Selenium.DevTools;
using static GenerallyPositive.Browser.BrowserHelper;
using static GenerallyPositive.Enums;
using static OpenCvSharp.XImgProc.CvXImgProc;
 
namespace GenerallyPositive.Browser
{
    public static class Native
    {
        // Constants
        public const uint DUPLICATE_SAME_ACCESS = 0x00000002;
        public const uint HANDLE_FLAG_INHERIT = 0x00000001;
        public const uint STARTF_USESTDHANDLES = 0x00000100;
        public const uint STARTF_USESHOWWINDOW = 0x00000001;
        public const uint CREATE_NO_WINDOW = 0x08000000;
        public const uint STILL_ACTIVE = 259;
        public const ushort SW_HIDE = 0;
        public const ushort SW_SHOWNORMAL = 1;
 
        // Proc/thread attribute constant (if using STARTUPINFOEX)
        public const int PROC_THREAD_ATTRIBUTE_HANDLE_LIST = 0x00020002;
 
        // SECURITY_ATTRIBUTES
        [StructLayout(LayoutKind.Sequential)]
        public struct SECURITY_ATTRIBUTES
        {
            public int nLength;
            public IntPtr lpSecurityDescriptor;
            public int bInheritHandle; // BOOL as int
        }
 
        // STARTUPINFO (standard)
        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
        public struct STARTUPINFO
        {
            public uint cb;
            public string lpReserved;
            public string lpDesktop;
            public string lpTitle;
            public uint dwX;
            public uint dwY;
            public uint dwXSize;
            public uint dwYSize;
            public uint dwXCountChars;
            public uint dwYCountChars;
            public uint dwFillAttribute;
            public uint dwFlags;
            public ushort wShowWindow;
            public ushort cbReserved2;
            public IntPtr lpReserved2;
            public IntPtr hStdInput;
            public IntPtr hStdOutput;
            public IntPtr hStdError;
        }
 
        // Extended STARTUPINFO for attribute list
        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
        public struct STARTUPINFOEX
        {
            public STARTUPINFO StartupInfo;
            public IntPtr lpAttributeList;
        }
 
        // PROCESS_INFORMATION
        [StructLayout(LayoutKind.Sequential)]
        public struct PROCESS_INFORMATION
        {
            public IntPtr hProcess;
            public IntPtr hThread;
            public uint dwProcessId;
            public uint dwThreadId;
        }
 
        // P/Invokes
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool CreatePipe(
            out IntPtr hReadPipe,
            out IntPtr hWritePipe,
            ref SECURITY_ATTRIBUTES lpPipeAttributes,
            uint nSize);
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool SetHandleInformation(
            IntPtr hObject,
            uint dwMask,
            uint dwFlags);
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool GetHandleInformation(
            IntPtr hObject,
            out uint lpdwFlags);
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool DuplicateHandle(
            IntPtr hSourceProcessHandle,
            IntPtr hSourceHandle,
            IntPtr hTargetProcessHandle,
            out IntPtr lpTargetHandle,
            uint dwDesiredAccess,
            bool bInheritHandle,
            uint dwOptions);
 
        [DllImport("kernel32.dll")]
        public static extern IntPtr GetCurrentProcess();
 
        [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
        public static extern bool CreateProcess(
            string lpApplicationName,
            string lpCommandLine,
            IntPtr lpProcessAttributes,
            IntPtr lpThreadAttributes,
            bool bInheritHandles,
            uint dwCreationFlags,
            IntPtr lpEnvironment,
            string lpCurrentDirectory,
            ref STARTUPINFO lpStartupInfo,
            out PROCESS_INFORMATION lpProcessInformation);
 
        [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
        public static extern bool CreateProcess(
            string lpApplicationName,
            string lpCommandLine,
            IntPtr lpProcessAttributes,
            IntPtr lpThreadAttributes,
            bool bInheritHandles,
            uint dwCreationFlags,
            IntPtr lpEnvironment,
            string lpCurrentDirectory,
            ref STARTUPINFOEX lpStartupInfoEx,
            out PROCESS_INFORMATION lpProcessInformation);
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool InitializeProcThreadAttributeList(
            IntPtr lpAttributeList,
            int dwAttributeCount,
            int dwFlags,
            ref IntPtr lpSize);
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool UpdateProcThreadAttribute(
            IntPtr lpAttributeList,
            uint dwFlags,
            IntPtr attribute,
            IntPtr lpValue,
            IntPtr cbSize,
            IntPtr lpPreviousValue,
            IntPtr lpReturnSize);
 
        [DllImport("kernel32.dll")]
        public static extern void DeleteProcThreadAttributeList(
            IntPtr lpAttributeList);
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool PeekNamedPipe(
            IntPtr hNamedPipe,
            IntPtr lpBuffer,
            uint nBufferSize,
            IntPtr lpBytesRead,
            out uint lpTotalBytesAvail,
            IntPtr lpBytesLeftThisMessage);
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool GetExitCodeProcess(
            IntPtr hProcess,
            out uint lpExitCode);
 
        // Added: TerminateProcess to allow force-killing a process handle
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool TerminateProcess(
            IntPtr hProcess,
            uint uExitCode);
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern bool CloseHandle(
            IntPtr hObject);
 
        [DllImport("kernel32.dll", SetLastError = true)]
        public static extern IntPtr GetStdHandle(int nStdHandle);
    }
    public class WindowInfo
    {
        public string URL { get; set; }
        public bool ActiveWindow { get; set; }
    }
    public class WindowTuple
    {
        public string WindowId { get; set; }
        public string Url { get; set; }
 
        public WindowTuple(string windowId, string url)
        {
            WindowId = windowId;
            Url = url;
        }
 
        public WindowTuple()
        { }
    }
 
    public class TabTuple
    {
        public int TabId { get; set; }
        public string Url { get; set; }
 
        public TabTuple(int TabId, string Url)
        {
            this.TabId = TabId;
            this.Url = Url;
        }
        public TabTuple()
        { }
    }
 
    public class MagicHelper : IMagicHelper
    {
        internal MagicHelper(Browser browser)
        {
            this.browser = browser;
        }
 
        // The extension announces the port it picked by broadcasting to a fixed udp port, and the message is a bare
        // number with nothing in it that says which browser sent it. So only one browser can be launching and
        // listening at a time: a second listener cannot bind the socket, and if it could it would have no way to
        // tell whose port it just read. This gate holds a launch from starting until the one before it has its port.
        // Machine wide by name, because two GPAL applications race each other exactly the way two browsers do.
        private static readonly Mutex ottoHandshakeGate = new Mutex(false, "GPAL.OttoMagic.PortHandshake");
        private const int OttoHandshakeGateTimeoutMs = 60_000;
 
        List<string> lastErrorMessage = new List<string>();
        bool supressedMessage = false;
        string lastSessionToken;
        IBrowser browser { get; set; } = null;
 
        private IRESTClient _client;
        private string _clientApiBase;

        internal IRESTClient Client
        {
            get
            {
                if (null == _client)
                    _client = (IRESTClient)GPAL.RESTClient;
 
                string url = ((Browser)browser)?.BrowserSettings.RestApiBaseUrl;
 
                // re-pointed rather than set once: a browser that relaunches lands on a new port, and the
                // client the workflow is holding has to follow it
                if (false == string.IsNullOrEmpty(url) && url != _clientApiBase)
                {
                    _client.WithAPIBase(url);
                    _clientApiBase = url;
                }
 
                return _client;
            }
        }
 
        #region Commands
        public void Back()
        {
            Client.Back().Execute();
        }
 
        public string CaptureVisibleTab()
        {
            return Client.CaptureVisibleTab().Execute();
        }
        public string CheckNetworkIdle(int? maxConnections = null, int? timeoutMs = null, int? pruneMs = null)
        {
            BrowserSettings settings = ((Browser)browser)?.BrowserSettings;
 
            // the shorthand CheckNetworkIdle(int) hands back a type the other two cannot be said on, so
            // the endpoint is named and all three go on together
            string retVal = Client
                .WithEndpoint(ApiEndpoint.CheckNetworkIdle)
                .WithMaxConnections(maxConnections ?? settings?.NetworkIdleMaxConnections ?? 0)
                .WithTimeoutMs(timeoutMs ?? settings?.NetworkIdleTimeoutMs ?? 30_000)
                .WithPruneMs(pruneMs ?? settings?.NetworkIdlePruneMs ?? 3_000)
                .Execute();
 
            return retVal;
        }

        public TabTuple CloseTab(dynamic URLorTabId = null)
        {
            if (null == URLorTabId)
                return Client.CloseTab().Execute<TabTuple>();
            else if (URLorTabId is string v)
                return Client.CloseTab(v).Execute<TabTuple>();
            else if (URLorTabId is GPALUrl u)
                return Client.CloseTab(u).Execute<TabTuple>();
            else if (URLorTabId is int i)
                return Client.CloseTab(URLorTabId).Execute<TabTuple>();
            else
                return Client.CloseTab().Execute<TabTuple>();
 
        }

        public bool DeleteStorage(WebsiteStorageType storageType, bool deleteAcrossOrigins = false, string domain = null, string path = null, string key = null, string storeName = null)
        {
            IAllowRESTStorageOptions fluentObject = null;
 
            fluentObject = Client.DeleteStorage(storageType);
 
            if (false == string.IsNullOrEmpty(domain))
                fluentObject.WithStorageDomain(domain);
 
            if (false == string.IsNullOrEmpty(path))
                fluentObject.WithStoragePath(path);
 
            if (false == string.IsNullOrEmpty(key))
                fluentObject.WithStorageKey(key);
 
            if (false == string.IsNullOrEmpty(storeName))
                fluentObject.WithStorageStoreName(storeName);
 
            fluentObject.WithDeleteAcrossOrigins(deleteAcrossOrigins);
 
            fluentObject.Execute();
 
            return true;
        }

        public string GetStorage(WebsiteStorageType storageType, string domain = null, string path = null, string key = null, string storeName = null)
        {
            IAllowRESTStorageOptions fluentObject = null;
 
            fluentObject = Client.GetStorage(storageType);
 
            if (false == string.IsNullOrEmpty(domain))
                fluentObject.WithStorageDomain(domain);
 
            if (false == string.IsNullOrEmpty(path))
                fluentObject.WithStoragePath(path);
 
            if (false == string.IsNullOrEmpty(key))
                fluentObject.WithStorageKey(key);
 
            if (false == string.IsNullOrEmpty(storeName))
                fluentObject.WithStorageStoreName(storeName);
 
            return fluentObject.Execute();
        }
 
        public bool SetStorage(WebsiteStorageType storageType, string data, string domain = null, string path = null, string key = null, string storeName = null)
        {
 
            IAllowRESTStorageOptions fluentObject = null;
 
            fluentObject = Client.SetStorage(storageType);
 
            if (false == string.IsNullOrEmpty(domain))
                fluentObject.WithStorageDomain(domain);
 
            if (false == string.IsNullOrEmpty(path))
                fluentObject.WithStoragePath(path);
 
            if (false == string.IsNullOrEmpty(key))
                fluentObject.WithStorageKey(key);
 
            if (false == string.IsNullOrEmpty(storeName))
                fluentObject.WithStorageStoreName(storeName);
 
            if (false == string.IsNullOrEmpty(data))
                fluentObject.WithData(data);
 
            fluentObject.Execute<bool>();
 
            return true;
        }

        public void Forward()
        {
            Client.Forward().Execute();
        }

        public void FullScreen()
        {
            Client.FullScreen().Execute();
        }

        public string Get(GPALUrl URL)
        {
            bool areRobotsAllowed = true;
 
            URL.ForUrl(GetFullUrl(URL?.Url, browser, out areRobotsAllowed));
 
            if (null != browser)
                ((Browser)browser)._areRobotsAllowed = areRobotsAllowed;
 
            var result = Client.GoTo(URL?.Url).Execute();
            browser.ServerResponseCode = ((RESTClient)Client).StatusCode;
 
            return result;
        }

        public string GetContentAndCss(string elementId)
        {
            return Client.GetContentAndCss(elementId).Execute();
        }
        public Dictionary<string, object> GetCssAttributes(string elementId)
        {
            return Client.GetCssAttributes(elementId).Execute<Dictionary<string, object>>();
        }
        public Dictionary<string, object> GetDomAttributes(string elementId)
        {
            return Client.GetDomAttributes(elementId).Execute<Dictionary<string, object>>();
        }
        public Dictionary<string, object> GetDomProperties(string elementId)
        {
            return Client.GetDomProperties(elementId).Execute< Dictionary<string, object>>();
        }

        public string GetCurrentUrl()
        {
            return Client.GetCurrentUrl().Execute();
        }

        public string GetReadyStatus(string sessionToken)
        {
            string retVal = null;
 
            if (lastSessionToken != sessionToken)
            {
                lastErrorMessage.Clear();
                supressedMessage = false;
                lastSessionToken = sessionToken;
            }
 
            dynamic obj = Client.GetReadyStatus().Execute();
            if ("null" != obj && "NOTOK" != obj)
                retVal = obj ?? @"""loading""";
            else
                retVal = "complete";
 
            string currentErrorMessage = $"Document.Ready status [{retVal}]";
 
            if (false == lastErrorMessage.Contains(currentErrorMessage))
            {
                lastErrorMessage.Add(currentErrorMessage);
                GPAL.PublishSimpleEvent(GPALEventType.INFO, currentErrorMessage, null, GPALObjectType.None);
                supressedMessage = false;
            }
            else if (false == supressedMessage)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Supressing repeat messages", currentErrorMessage, GPALObjectType.Other);
                supressedMessage = true;
            }
 
            return retVal;
        }

        public GPALElement GetShadowRoot(string elementId)
        {
            return Client.GetShadowRoot(elementId).Execute<GPALElement>();
        }

        public TabTuple GoTo(GPALUrl URL)
        {
            if (null == URL)
                URL = new GPALUrl("google.com");
 
            bool areRobotsAllowed = true;
 
            URL.ForUrl(GetFullUrl(URL.Url, browser, out areRobotsAllowed));
 
            if (null != browser)
                ((Browser)browser)._areRobotsAllowed = (bool) areRobotsAllowed;
 
            TabTuple retVal = Client.GoTo(URL.Url).Execute<TabTuple>();
            browser.ServerResponseCode = ((RESTClient)Client).StatusCode;
 
            return retVal;
        }

        public TabTuple GotoTab(dynamic URLorTabId)
        {
            if (URLorTabId is GPALUrl v)
            {
                TabTuple tt = Client.GoToTab(v).Execute<TabTuple>();
                browser.ServerResponseCode = ((RESTClient)Client).StatusCode;
                return tt;
            }
            else if (URLorTabId is TabTuple tuple)
            {
                TabTuple tt = Client.GoToTab(tuple.TabId).Execute<TabTuple>();
                browser.ServerResponseCode = ((RESTClient)Client).StatusCode;
                return tt;
            }
            else
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"URLorTabId is unknown type [{URLorTabId.GetType()}]", this, GPALObjectType.Other);
                return new TabTuple();
            }
        }

        public void SwitchToElement(string elementId)
        {
            Client.SwitchToElement(elementId).Execute();
        }

        public void InFrame(string elementId)
        {
            Client.SwitchToElement(elementId).Execute();
        }

        public void Minimize()
        {
            Client.Minimize().Execute();
        }

        public void Maximize()
        {
            Client.Maximize().Execute();
        }

        public void MoveTo(string elementId)
        {
            Client.ScrollIntoView(elementId).Execute();
            Client.Hover(elementId).Execute();
        }

        public void Normal()
        {
            Client.Normal().Execute();
        }

        public TabTuple NewTab(GPALUrl URL = null)
        {
            if (null == URL)
                URL = new GPALUrl();
 
            URL.ForUrl(GetFullUrl(URL.Url, browser, out ((Browser)browser)._areRobotsAllowed));
 
            TabTuple retVal = Client.NewTab(URL.Url).Execute<TabTuple>();
            browser.ServerResponseCode = ((RESTClient)Client).StatusCode;
 
            return retVal;
        }

        public TabTuple NextTab()
        {
            return Client.NextTab().Execute<TabTuple>();
        }

        public int TabCount()
        {
            return Client.TabCount().Execute<int>();
        }

        public WindowTuple NextWindow()
        {
            return Client.NextWindow().Execute<WindowTuple>();
        }

        public void OverrideReferrer(string referrer)
        {
            Client.OverrideReferrer(referrer).Execute();
        }

        public void SetUserAgent(string userAgent)
        {
            Client.SetUserAgent(userAgent).Execute();
        }

        public void PageDown()
        {
            Client.PageDown().Execute();
        }

        public void PageEnd()
        {
            Client.PageEnd().Execute();
        }

        public void PageTop()
        {
            Client.PageTop().Execute();
        }

        public void PageUp()
        {
            Client.PageUp().Execute();
        }

        public void PressModifierKey(ModifierKeys modifierKeys)
        {
            Client.PressModifierKey(modifierKeys).Execute();
        }

        public TabTuple PreviousTab()
        {
            return Client.PreviousTab().Execute<TabTuple>();
        }
 

        public WindowTuple PreviousWindow()
        {
            return Client.PreviousWindow().Execute<WindowTuple>();
        }

        public void Refresh()
        {
            Client.Refresh().Execute();
        }

        public void ReleaseModifierKey(ModifierKeys modifierKeys)
        {
            Client.ReleaseModifierKey(modifierKeys).Execute();
        }

        public void Restore()
        {
            Client.Restore().Execute();
        }

        public void ScrollWindowByHorizontal(int pixels)
        {
            Client.ScrollWindowByHorizontal(pixels).Execute();
        }

        public void ScrollWindowByVertical(int pixels)
        {
            Client.ScrollWindowByVertical(pixels).Execute();
        }

        public void StealthOverrideReferrer()
        {
            Client.StealthOverrideReferrer().Execute();
        }

        public void WithDownloadFile(string downloadPath)
        {
            Client.SetDownloadFilename(Path.GetFileName(downloadPath)).Execute();
        }

        public int WindowScreenLeft()
        {
            return Client.WindowScreenLeft().Execute<int>();
        }

        public int WindowScreenTop()
        {
            return Client.WindowScreenTop().Execute<int>();
        }

        public void Focus(string elementId)
        {
            Client.Focus(elementId).Execute();
        }

        public void FireChangeEvent(string elementId)
        {
            Client.FireChangeEvent(elementId).Execute();
        }

        public Rectangle GetWindowRectangle()
        {
            Rectangle rect = new Rectangle();
            var result =  Client.GetWindowRectangle().Execute<Dictionary<string, object>>();
            Dictionary<string, object> obj = null;
 
            if (null != result)
            {
                obj = (Dictionary<string, object>)result;
 
                var left = obj.ContainsKey("Left") ? Convert.ToInt32(obj["Left"]) : obj.ContainsKey("left") ? Convert.ToInt32(obj["left"]) : obj.ContainsKey("X") ? Convert.ToInt32(obj["X"]) : obj.ContainsKey("x") ? Convert.ToInt32(obj["x"]) : 0;
                var top = obj.ContainsKey("Top") ? Convert.ToInt32(obj["Top"]) : obj.ContainsKey("top") ? Convert.ToInt32(obj["top"]) : obj.ContainsKey("Y") ? Convert.ToInt32(obj["Y"]) : obj.ContainsKey("y") ? Convert.ToInt32(obj["y"]) : 0;
                var width = obj.ContainsKey("Width") ? Convert.ToInt32(obj["Width"]) : obj.ContainsKey("width") ? Convert.ToInt32(obj["width"]) : 0;
                var height = obj.ContainsKey("Height") ? Convert.ToInt32(obj["Height"]) : obj.ContainsKey("height") ? Convert.ToInt32(obj["height"]) : 0;
 
                rect = new Rectangle(left, top, width, height);
            }
 
            return rect;
        }

        public void ScrollIntoView(string elementId)
        {
            Client.ScrollIntoView(elementId).Execute();
        }

        public void ScrollElement(string elementId, int hPixels, int vPixels)
        {
            Client.ScrollElement(elementId).WithHPixels(hPixels).WithVPixels(vPixels).Execute();
        }

        public string GetElementAttributeHash(string elementId)
        {
            return Client.GetElementAttributeHash(elementId).Execute();
        }

        public List<GPALElement> GetOptions(string elementId)
        {
            List<GPALElement> tmpElements = Client.GetOptions(elementId).Execute<List<GPALElement>>();
            SwitchToDefaultContent();
            return tmpElements;
        }

        public string GetPageSource()
        {
            return Client.GetPageSource().Execute();
        }

        public GPALElement GetParentNode(string elementId)
        {
            return Client.GetParentNode(elementId).Execute<List<GPALElement>>()[0];
        }

        public string GetAttribute(string elementId, string attribute)
        {
            return Client.GetAttribute(elementId).WithAttribute(attribute).Execute();
        }

        public void SetAttribute(string elementId, string attribute, string value)
        {
            Client.SetAttribute(elementId).WithAttribute(attribute).WithValue(value).Execute();
        }
 
        public void SetRange(string elementId, int rangeValue)
        {
            Client.SetRange(elementId).WithRangeValue(rangeValue).Execute();
        }

        public void SetValueFromElement(string srcSelector, string destElementId)
        {
            Client.WithElementId(srcSelector).SetValueFrom(destElementId).Execute();
        }

        public void LeftClick(string elementId)
        {
            Client.LeftClick(elementId).Execute();
            Thread.Sleep(500);
        }

        public void LeftDoubleClick(string elementId)
        {
            Client.LeftDoubleClick(elementId).Execute();
        }

        public void MiddleClick(string elementId)
        {
            Client.MiddleClick(elementId).Execute();
            Thread.Sleep(500);
        }

        public void RightClick(string elementId)
        {
            Client.RightClick(elementId).Execute();
        }
 
        public void SelectClick(string elementId, dynamic indexOrValue)
        {
            if (indexOrValue is int idx)
                Client.SelectClick(elementId).WithSelectIndex(idx).Execute();
            else
                Client.SelectClick(elementId).WithSelectValue(indexOrValue).Execute();
        }

        public bool IsVisibleInViewport(string elementId)
        {
            return Client.IsVisibleInViewport(elementId).Execute<bool>();
        }

        public bool IsEndOfPage()
        {
            return Client.IsEndOfPage().Execute<bool>();
        }

        public void Hover(string elementId)
        {
            Client.Hover(elementId).Execute();
        }

        public void DragAndDrop(string elementId, int deltaX, int deltaY, int offsetX = 0, int offsetY = 0)
        {
            Client.DragAndDrop(elementId).WithDeltaX(deltaX).WithDeltaY(deltaY).WithOffsetX(offsetX).WithOffsetY(offsetY).Execute();
        }

        public int WindowInnerHeight()
        {
            return Client.WindowInnerHeight().Execute<int>();
        }

        public int WindowInnerWidth()
        {
            return Client.WindowInnerWidth().Execute<int>();
        }

        public int WindowOuterHeight()
        {
            return Client.WindowOuterHeight().Execute<int>();
        }

        public int WindowOuterWidth()
        {
            return Client.WindowOuterWidth().Execute<int>();
        }

        public string Fetch(string url, string method = null, string body = null, string contentType = null, string[] headers = null, bool asBytes = false)
        {
            string retVal = Client
                .Fetch(url)
                .WithVerb(method)
                .WithBody(body)
                .WithContentType(contentType)
                .WithHeaders(headers)
                .WithBytes(asBytes)
                .Execute();
 
            // the extension answers NOTOK when the page could not make the request, and the caller reads null as
            // the failure, the same as the other engines
            return "NOTOK" == retVal || "null" == retVal ? null : retVal;
        }

        public int WindowPageOffsetX()
        {
            return Client.WindowPageOffsetX().Execute<int>();
        }

        public int WindowPageOffsetY()
        {
            return Client.WindowPageOffsetY().Execute<int>();
        }

        public bool ElementFromPoint(string elementId, int x, int y)
        {
            return true == Client.ElementFromPoint(elementId, x, y).Execute<bool>();
        }
 
        public Rectangle GetBoundingClientRect(string elementId)
        {
            Rectangle rect = new Rectangle(0,0,0,0);
            var result = Client.GetBoundingClientRect(elementId).Execute<Dictionary<string, object>>();
            Dictionary<string, object> obj = null;
 
            if (null != result)
            {
                obj = (Dictionary<string, object>)result;
 
                var left = obj.ContainsKey("Left") ? Convert.ToInt32(obj["Left"]) : obj.ContainsKey("left") ? Convert.ToInt32(obj["left"]) : obj.ContainsKey("X") ? Convert.ToInt32(obj["X"]) : obj.ContainsKey("x") ? Convert.ToInt32(obj["x"]) : 0;
                var top = obj.ContainsKey("Top") ? Convert.ToInt32(obj["Top"]) : obj.ContainsKey("top") ? Convert.ToInt32(obj["top"]) : obj.ContainsKey("Y") ? Convert.ToInt32(obj["Y"]) : obj.ContainsKey("y") ? Convert.ToInt32(obj["y"]) : 0;
                var width = obj.ContainsKey("Width") ? Convert.ToInt32(obj["Width"]) : obj.ContainsKey("width") ? Convert.ToInt32(obj["width"]) : 0;
                var height = obj.ContainsKey("Height") ? Convert.ToInt32(obj["Height"]) : obj.ContainsKey("height") ? Convert.ToInt32(obj["height"]) : 0;
 
                rect = new Rectangle(left, top, width, height);
            }
 
            return rect;
        }

        public void HideElement(string elementId)
        {
            Client.HideElement(elementId).Execute();
        }

        public void InjectScript(string script)
        {
            string answered = Client.InjectScript(script).Execute();
 
            // the extension answers with the id it registered under, or with an error. thrown away, a script that
            // never registered looks exactly like one that did until the page fails to behave, which is a long way
            // from here. the usual cause is Allow user scripts still being off for the extension
            if (true == answered?.Contains("error"))
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"The extension did not register the script [{answered}]", this, GPALObjectType.Other);
        }

        public string ExecuteJavaScript(string script)
        {
            return Client.ExecuteJavaScript(script).Execute();
        }

        public void ClearInjectedScripts()
        {
            Client.ClearInjectedScripts().Execute();
        }

        public bool IsClickAble(string elementId)
        {
            return Client.IsClickable(elementId).Execute<bool>();
        }

        public bool IsDisplayed(string elementId)
        {
            return Client.IsDisplayed(elementId).Execute<bool>();
        }

        public bool IsEnabled(string elementId)
        {
            return Client.IsEnabled(elementId).Execute<bool>();
        }

        public void ClearReferrer()
        {
            Client.ClearReferrer().Execute();
        }

        public GPALElement Evaluate(string xpath)
        {
            return Client.Evaluate(xpath).Execute<GPALElement>();
        }

        public List<GPALElement> EvaluateAll(string xpath)
        {
            return Client.EvaluateAll(xpath).Execute<List<GPALElement>>();
        }

        public GPALElement EvaluatePersistent(string xpath)
        {
            return Client.EvaluatePersistent(xpath).Execute<GPALElement>();
        }

        public List<GPALElement> EvaluateAllPersistent(string xpath)
        {
            return Client.EvaluateAllPersistent(xpath).Execute<List<GPALElement>>();
        }

        public string GetUserAgent()
        {
            return Client.GetUserAgent().Execute();
        }

        public string GetLanguages()
        {
            return Client.GetLanguages().Execute();
        }

        public void CaptureCalls(bool capture = true, string urlFragment = null, bool clear = false)
        {
            IAllowRESTCaptureOptions capturing = Client.CaptureCalls(capture);
 
            if (false == string.IsNullOrWhiteSpace(urlFragment)) capturing.WithCallFilter(urlFragment);
            if (true == clear) capturing.WithClearCalls(true);
 
            capturing.Execute();
        }

        public string GetCapturedCalls()
        {
            return Client.GetCapturedCalls().Execute();
        }

        public void FillInAppend(string elementId, string text)
        {
            Client.FillInAppend(elementId).WithText(text).Execute();
        }

        public void FillInInsert(string elementId, string text)
        {
            Client.FillInInsert(elementId).WithText(text).Execute();
        }

        public void FillInOverwrite(string elementId, string text)
        {
            Client.FillInOverwrite(elementId).WithText(text).Execute();
        }

        public void LeftClickAndDownload(string elementId, string filenameAndPath)
        {
            Client.LeftClickAndDownload(elementId).WithDownloadFile(Path.GetFileName(filenameAndPath)).Execute();
        }

        public void LeftClickAndUpload(string elementId, string filenameAndPath)
        {
            Client.LeftClickAndUpload(elementId).WithUploadFile(filenameAndPath).Execute();
        }

        public void LeftClickAndUpload(string elementId, GPALFile filenamesAndPaths)
        {
            Client.LeftClickAndUpload(elementId).WithUploadFiles(filenamesAndPaths).Execute();
        }
 

        public GPALElement QuerySelector(string css)
        {
            var elem = Client.QuerySelector(css).Execute<GPALElement>();
            return elem;
        }

        public List<GPALElement> QuerySelectors(string css)
        {
            return Client.QuerySelectors(css).Execute<List<GPALElement>>();
        }

        public GPALElement QueryPersistentSelector(string css)
        {
            var persist = Client.QueryPersistentSelector(css).Execute<GPALElement>();
 
            return persist;
        }

        public List<GPALElement> QueryPersistentSelectors(string css)
        {
            return Client.QueryPersistentSelectors(css).Execute<List<GPALElement>>();
        }

        public void ScrollWindow(int hPixels, int vPixels)
        {
            // Note: Requires RESTClient to implement ScrollWindow, or use ExecuteJavaScript as fallback
            Client.ScrollWindow(hPixels, vPixels).Execute();
        }

        public void SendKey(byte vkcode)
        {
            Client.SendKey(vkcode).Execute();
        }

        public void SendString(string text, int delayMs = 0)
        {
            Client.SendString(text, delayMs).Execute();
        }

        public void SubmitForm(string elementId)
        {
            Client.SubmitForm(elementId).Execute();
        }

        public void SwitchToDefaultContent()
        {
            Client.SwitchToDefaultContent().Execute();
        }

        public void SwitchToShadowRoot(string elementId)
        {
            Client.SwitchToShadowRoot(elementId).Execute();
        }
 
        #region Window Handling
        public WindowTuple OpenWindow(GPALUrl URL = null)
        {
            if (null == URL)
                URL = new GPALUrl();
 
            URL.ForUrl(GetFullUrl(URL.Url, browser, out ((Browser)browser)._areRobotsAllowed));
 
            WindowTuple retVal = Client.OpenWindow(URL.Url).Execute<WindowTuple>();
            browser.ServerResponseCode = ((RESTClient)Client).StatusCode;
 
            return retVal;
        }
 
        public WindowTuple GoToWindow(dynamic urlOrId)
        {
            if (urlOrId is string v)
                return Client.GoToWindow(v).Execute<WindowTuple>();
            else
                return Client.GoToWindow(urlOrId.ToString()).Execute<WindowTuple>();
        }
        public string CloseBrowser()
        {
            return Client.CloseBrowser().Execute();
        }
 
        public WindowTuple CloseWindow(dynamic urlOrId)
        {
            if (urlOrId is string v)
                return Client.CloseWindow(v).Execute<WindowTuple>();
            else
                return Client.CloseWindow(urlOrId.ToString()).Execute<WindowTuple>();
        }
 
        public WindowTuple GetCurrentWindow()
        {
            return Client.GetCurrentWindow().Execute<WindowTuple>();
        }
        #endregion Window Handling
        #region GPAL Studio
        public string GetBrowserSettings()
        {
            return Client.GetBrowserSettings().Execute();
        }

        public string GetGpalSettings()
        {
            return Client.GetGpalSettings().Execute();
        }

        public string GetSettings()
        {
            return Client.GetSettings().Execute();
        }

        public string GetWorkflow()
        {
            return Client.GetWorkflow().Execute();
        }
        #endregion GPAL Studio
        #endregion Commands
        #region Helpers
        static void PublishBrowserOutput(BrowserSettings browserSettings, DataReceivedEventArgs data)
        {
            if (null != data.Data)
                GPAL.PublishSimpleEvent(GPALEventType.DEEPDEBUG, $"[{browserSettings.BrowserType}] {data.Data}", browserSettings, GPALObjectType.Other);
        }
        public Process LaunchBrowser(Browser browser, GPALUrl URL, List<string> additionalArguments = null)
        {
            BrowserSettings browserSettings = browser.BrowserSettings;
            bool created = false;
            int secondsToWait = 30;
            TimeSpan timeout = TimeSpan.FromSeconds(secondsToWait);
 
            if (null == browserSettings)
            {
                // this should only be called when we are using Oauth credentials, to log someone in? we could require a browser passed into credentials, but why?
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, "BrowserSettings parameter is null. Using chrome puppeteer port.", null, GPALObjectType.Other);
                browser = (Browser)GPAL.Browser.ToGPALObject();
                browserSettings = ((Browser)browser).BrowserSettings;
            }
 
            if (null != browserSettings.ExistingBrowserPort)
            {
                foreach (Browser b in GPAL.Browsers)
                    if (b != browser && b.BrowserSettings.ExistingBrowserPort == browserSettings.ExistingBrowserPort)
                    {
                        browser.PuppeteerClient = b.PuppeteerClient;
                        browser.PuppeteerCommunicator = b.PuppeteerCommunicator;
 
                        return b.Process;
                    }
            }
 
            if (null == URL || null == URL?.Url)
                URL = "https://www.google.com";
 
            URL.ForUrl(GetFullUrl(URL?.Url, browser, out ((Browser)browser)._areRobotsAllowed, true));  // on launch we now open to google.com, don't do a robots.txt check on launch
 
            if (false == ((Browser)browser).AreRobotsAllowed && true == ((Browser)browser).BrowserSettings.ObeyRobotsTxt)
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Visiting [{URL?.Url}] is disallowed by robots.txt and your request to honor it. Not going to URL. Workflow will fail.", browser, GPALObjectType.Browser);
                URL.ForUrl("https://google.com");
            }
 
            bool headless = browserSettings.UseHeadless;
            var chromeArguments = additionalArguments ?? new List<string>();
            string browserPath;
 
            // Get browser executable path
            switch (browserSettings.BrowserType)
            {
                case BrowserType.Chrome:
                    browserPath = Puppeteer.GetChromePath();
                    break;
                case BrowserType.Edge:
                    browserPath = Puppeteer.GetEdgePath();
                    break;
                case BrowserType.FireFox:
                    browserPath = Puppeteer.GetFirefoxPath();
                    break;
                default:
                    Exception ex = new Exception("Unsupported Browser Type");
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unsupported browser type [{browserSettings.BrowserType}]", browserSettings, GPALObjectType.Other, ex);
                    return null;
            }
 
            if (string.IsNullOrEmpty(browserPath) || false == File.Exists(browserPath))
            {
                string msg = $"[{browserSettings.BrowserType}] executable not found at [{browserPath}]";
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, msg, browserSettings, GPALObjectType.Other);
                throw new NotFoundException(msg);
            }
 
            // Common arguments for Chrome/Edge
            if (browserSettings.BrowserType == BrowserType.Chrome || browserSettings.BrowserType == BrowserType.Edge)
            {
                string userAgent = null;
 
                // Profile settings
                if (false == string.IsNullOrEmpty(browserSettings.ProfileUserName) || false == string.IsNullOrEmpty(browserSettings.ProfileName))
                {
                    browserSettings.ProfileDataDirectory = BrowserHelper.GetBrowserProfileDirectory(browserSettings);
                    if (false == string.IsNullOrEmpty(browserSettings.ProfileName))
                    {
                        string profileDir = BrowserHelper.FindProfileDirectory(browserSettings.ProfileDataDirectory, browserSettings.ProfileName) ?? browserSettings.ProfileName;
                        chromeArguments.Add($"--profile-directory={profileDir}");
                    }
                }
                else if (true == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory))
                {
                    browserSettings.ProfileDataDirectory = ChromeProfileManager.CreateTempUserProfile(
                        browserSettings.DownloadLocation ?? FileHelper.GetDefaultDownloadDirectory(browser),
                        browserSettings.PromptForDownload,
                        browserSettings.OpenPDFExternally,
                        browserSettings.LoadImages,
                        browserSettings.UseOttoMagic
                    );
                    browserSettings.TempProfileCreated = true;
                }
 
                // prompting and the pdf viewer are profile preferences. the extension cannot set either one and
                // there is no CDP here, so a supplied profile has to be written to for the workflow's settings to
                // mean anything, and Browser.Close puts it back once the browser is gone. selenium and puppeteer
                // get this in BrowserHelper while their options are built, and firefox gets it from CreateUserJs
                // below, so chromium under the extension was the one path running on whatever the profile said
                if (false == browserSettings.TempProfileCreated && false == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory))
                    browserSettings.PreviousDownloadPreferences = ChromeProfileManager.ApplyDownloadPreferences(
                        browserSettings.ProfileDataDirectory,
                        browserSettings.DownloadLocation,
                        browserSettings.PromptForDownload,
                        browserSettings.OpenPDFExternally);
 
                // a profile can only be owned by one browser. launching onto one that is already open hands the
                // command line to that browser and exits, so the extension never loads and GPALRestAPI never
                // announces a port. selenium has refused this since it was written; this path did not, which is
                // why an OttoMagic run on a held profile failed later saying nothing about the profile
                if (false == browserSettings.TempProfileCreated)
                    BrowserHelper.RefuseIfProfileIsOpen(browserSettings);
 
                // Debugging
                if (true == browserSettings.UsePuppeteer)
                {
                    if (true == browserSettings.DebugPipe)
                        chromeArguments.Add("--remote-debugging-pipe --enable-unsafe-extension-debugging --remote-debugging-port=0");
                    else
                    {
                        // asked for rather than assumed: a second browser starting alongside this one has to be
                        // given a different port, and 0xdead for everybody is how they end up in one session
                        if (null == browserSettings.DebugPort)
                            browserSettings.DebugPort = BrowserHelper.FindFreePort();
 
                        browserSettings.PuppeteerUrl = $"http://localhost:{browserSettings.DebugPort}";
                        chromeArguments.Add($"--remote-debugging-port={browserSettings.DebugPort.Value}");
                    }
 
                    // *******************************************************************
                    // NOTE: CAVEAT: suppose a time comes to run with some sort of extension, this isn't causing any harm, just annoying...
                    // not keen on disabling extensions, they make the bot look alive, but somehow we score low on recaptcha with ottomagic, no idea why, puppeteer is awesome
                    // *******************************************************************
                    // chromeArguments.Add("--disable-extensions");    // no ottomagic
                    // *******************************************************************
                    // *******************************************************************
                }
 
                // set windowsize if provided or not - new
                if (false == browserSettings.FullScreen &&
                    false == browserSettings.Maximize &&
                    false == browserSettings.Minimize)
                {
                    if (false == browserSettings.WindowSize.IsEmpty)
                    {
                        // explicit WindowSize provided - must set screen-info in headless=new
                        // otherwise CDP interactions fail with "Browser window not found"
                        chromeArguments.Add($"--window-size={browserSettings.WindowSize.Width},{browserSettings.WindowSize.Height}");
 
                        // the rectangle carries where as well as how big, and a workflow running several browsers
                        // wants them beside each other rather than stacked. left at the origin it is not asked for
                        if (0 != browserSettings.WindowSize.X || 0 != browserSettings.WindowSize.Y)
                            chromeArguments.Add($"--window-position={browserSettings.WindowSize.X},{browserSettings.WindowSize.Y}");
                    }
                    else if (browserSettings.UseHeadless)
                        chromeArguments.Add("--start-fullscreen");
                }
                else if (true == browserSettings.FullScreen)
                    chromeArguments.Add("--start-fullscreen");
                else if (true == browserSettings.Maximize)
                    chromeArguments.Add("--start-maximized");
 
                // Anti-bot detection
                //if (true == myBrowserSettings.UsePuppeteer || AutomationEngine.PuppeteerPortHW == myBrowserSettings.AutomationEngine)
                //{
                // chromeArguments.Add("--disable-blink-features=AutomationControlled");
                chromeArguments.Add("--excludeSwitches=enable-automation");
                chromeArguments.Add("--useAutomationExtension=false");
                //}
 
                // Core arguments to suppress popups and stabilize CDP
                chromeArguments.Add("--no-first-run");
                chromeArguments.Add("--no-default-browser-check");
                //chromeArguments.Add("--no-default-browser-check");
                //chromeArguments.Add("--disable-default-apps");
                //chromeArguments.Add("--disable-features=TranslateUI,FirstRun,ChromeWhatsNewUI,InterestCohort");
                //chromeArguments.Add("--disable-background-networking");
                //chromeArguments.Add("--disable-background-timer-throttling");
                //chromeArguments.Add("--disable-renderer-backgrounding");
                //chromeArguments.Add("--disable-backgrounding-occluded-windows");
                //chromeArguments.Add("--no-service-autorun");
                //chromeArguments.Add("--disable-client-side-phishing-detection");
                //chromeArguments.Add("--disable-sync");
                chromeArguments.Add("--disable-dev-shm-usage");
 
                // avoid chrome overlays on the rendered area
                chromeArguments.Add("--disable-infobars");
                chromeArguments.Add("--disable-session-crashed-bubble");
                chromeArguments.Add("--hide-crash-restore-bubble");
                chromeArguments.Add("--noerrdialogs");
                // one --disable-features switch only, chromium takes the last one rather than merging them.
                // edge runs its own shopping content scripts on retail pages, and their constant dom mutation is
                // what has DOM.getDocument hanging there while the same page is fine under chrome
                string disableFeatures = "TranslateUI,Translate";
 
                if (BrowserType.Edge == browserSettings.BrowserType)
                    disableFeatures += ",msShoppingTrigger,msShopping,msEdgeShoppingUI,msEdgeShoppingList";
 
                chromeArguments.Add($"--disable-features={disableFeatures}");
                chromeArguments.Add("--load-media-router-component-extension");
                chromeArguments.Add("--enable-media-router");
                chromeArguments.Add("--force-discovery");
                chromeArguments.Add("--allow-local-network-access");
    
                if (false == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory))
                    chromeArguments.Add($@"--user-data-dir=""{browserSettings.ProfileDataDirectory.Replace("\\", "/")}""");
 
                // Headless mode
                if (headless)
                {
                    chromeArguments.Add("--headless=new");
                    chromeArguments.Add("--disable-notifications");
                    chromeArguments.Add("--disable-gpu");
                    if (AutomationEngine.PuppeteerPipe == browserSettings.AutomationEngine || AutomationEngine.PuppeteerPipeHW == browserSettings.AutomationEngine)
                        chromeArguments.Add("--no-sandbox"); // REQUIRED FOR PIPE IN HEADLESS + TEMP PROFILE
 
                    // Fetch user-agent
                    userAgent = GetUserAgent(browserSettings, chromeArguments);
 
                    userAgent = userAgent?.Trim('"');
 
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Headless [{browserSettings.BrowserType}] using UserAgent [{userAgent}]", browserSettings, GPALObjectType.Other);
 
                    chromeArguments.Add($"--user-agent=\"{userAgent}\"");
                }
                else
                {
                    if (false == browserSettings.BlockPopUps)
                        chromeArguments.Add("--disable-popup-blocking");
                }
 
                // Dark mode
                if (browserSettings.StealthType.HasFlag(StealthType.DarkMode))
                    chromeArguments.Add("--force-dark-mode");
 
                // Safe browsing - ads latency to each browse
                //chromeArguments.Add("--safebrowsing.enabled");
 
                // full accessibility for using UIAutomation (it is a fallback for method: GetWindowRectangle to get the content dimensions
                // no longer needed, we don't use uiautomation to calculate browser coords
                // this creates extra work = slower browser response
                // chromeArguments.Add("--force-renderer-accessibility");
            }
            else if (browserSettings.BrowserType == BrowserType.FireFox)
            {
                // Firefox arguments
                var firefoxArguments = new List<string>();
 
                if (headless)
                {
                    firefoxArguments.Add("-headless");
                    firefoxArguments.Add($"-width={Puppeteer.GetScreenWidth()}");
                    firefoxArguments.Add($"-height={Puppeteer.GetScreenHeight()}");
                    firefoxArguments.Add("--disable-notifications");
                }
                // nothing goes here. a headful firefox on OttoMagic is driven through the extension's rest api,
                // on the port GPALRestAPI announces over the handshake, so it has no use for a debugging port.
                // note firefox has no --websocket-port: it drops the flag and takes the port number for a url
 
                // Call firefoxhelper.CreateUserJs to load set these preferences
                // firefox has a unique requirement for loading a browser extension on demand
                // we have to create an .xpi file and get the extension id, then save it to a profile dir
                // then specify that profile dir
                if (true == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory))
                    browserSettings.ProfileDataDirectory = FirefoxProfileManager.CreateTempProfileDirectory();
 
                // same rule as chromium: one browser to a profile. a firefox already on this one takes the launch
                // and leaves the run waiting for a browser that was never started
                if (false == browserSettings.TempProfileCreated)
                    BrowserHelper.RefuseIfProfileIsOpen(browserSettings);
 
                browserSettings.PreviousFirefoxPreferences = FirefoxProfileManager.CreateUserJs(browserSettings.ProfileDataDirectory, browserSettings);
 
                if (false == string.IsNullOrEmpty(browserSettings.ProfileUserName) || false == string.IsNullOrEmpty(browserSettings.ProfileName))
                    firefoxArguments.Add($"-profile \"{(browserSettings.ProfileName ?? browserSettings.ProfileUserName).TrimEnd('\\')}\"");
                else if (false == string.IsNullOrEmpty(browserSettings.ProfileDataDirectory))
                    firefoxArguments.Add($"-profile \"{browserSettings.ProfileDataDirectory.TrimEnd('\\')}\"");
 
                chromeArguments = firefoxArguments; // Reuse chromeArguments for Process.Start
            }
 
            bool holdingHandshakeGate = false;   // set when this launch owns the OttoMagic port handshake
            UdpClient udpHandshake = null;       // out here so a launch that throws before the handshake still closes it
 
            // Launch browser
            try
            {
                // we will do a goto. for various reason this might not work and for various reasons this might work better
 
                // NOTE: we can't go to the real url on lauch, we have to apply stealth first :(
                if (null != URL && false == string.IsNullOrEmpty(URL.Url))
                    chromeArguments.Add(URL?.Url);
 
                if (AutomationEngine.PuppeteerPipe == browserSettings.AutomationEngine || AutomationEngine.PuppeteerPipeHW == browserSettings.AutomationEngine)
                    browserSettings.DebugPipe = true;
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Running workflow for Automation Engine [{browserSettings.AutomationEngine}]", null, GPALObjectType.None);
 
                if (true == browserSettings.UseOttoMagic)
                {
                    // wait for any launch already in flight to have read its port before we take the socket.
                    // An abandoned gate means the process holding it died, and we are the owner now
                    try { holdingHandshakeGate = ottoHandshakeGate.WaitOne(OttoHandshakeGateTimeoutMs); }
                    catch (AbandonedMutexException) { holdingHandshakeGate = true; }
 
                    if (false == holdingHandshakeGate)
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Waited [{OttoHandshakeGateTimeoutMs}] ms for another OttoMagic browser to finish its port handshake. Launching anyway, this browser may not get a port of its own.", browserSettings, GPALObjectType.Browser);
 
                    udpHandshake = new UdpClient(47623);
                    udpHandshake.Client.ReceiveTimeout = 30000;
 
                    KeepFromChildren(udpHandshake);
                }
 
                int firefoxLauncherPid = 0;
 
                if (true == browserSettings.DebugPipe)
                {
                    var sa = new Native.SECURITY_ATTRIBUTES
                    {
                        nLength = Marshal.SizeOf<Native.SECURITY_ATTRIBUTES>(),
                        bInheritHandle = 1,
                        lpSecurityDescriptor = IntPtr.Zero
                    };
 
                    IntPtr hProcess = IntPtr.Zero;
                    IntPtr reservedPtr = IntPtr.Zero;
                    AnonymousPipeServerStream stdinPipe = null, stdoutPipe = null, stderrPipe = null;
 
                    try
                    {
                        // Create anonymous pipes. bInheritHandle is set to true via HandleInheritability.Inheritable
                        stdinPipe = new AnonymousPipeServerStream(PipeDirection.Out, HandleInheritability.Inheritable, 1048576);
                        stdoutPipe = new AnonymousPipeServerStream(PipeDirection.In, HandleInheritability.Inheritable, 1048576);
                        stderrPipe = new AnonymousPipeServerStream(PipeDirection.In, HandleInheritability.Inheritable, 1048576);
 
                        // Build command line
                        var chromeCommandLine = new StringBuilder();
                        chromeCommandLine.Append($@"""{browserPath}""");
                        chromeCommandLine.Append(" ");
                        chromeCommandLine.Append(string.Join(" ", chromeArguments));
 
                        // Create process
                        var startupInfo = new Native.STARTUPINFO
                        {
                            cb = (uint)Marshal.SizeOf<Native.STARTUPINFO>(),
                            // named outright on the one launch we build ourselves, rather than left to inherit
                            // from the launching thread. null is the desktop the caller is on
                            lpDesktop = true == browserSettings.HiddenDesktop ? HiddenDesktop.StartupNameFor(browserSettings.HiddenDesktopName) : null,
                            dwFlags = Native.STARTF_USESHOWWINDOW | Native.STARTF_USESTDHANDLES, // Use standard handles flag
                            wShowWindow = (ushort)(headless ? Native.SW_HIDE : Native.SW_SHOWNORMAL),
                            hStdInput = stdinPipe.ClientSafePipeHandle.DangerousGetHandle(), // Assign to standard input
                            hStdOutput = stdoutPipe.ClientSafePipeHandle.DangerousGetHandle(), // Assign to standard output
                            hStdError = stderrPipe.ClientSafePipeHandle.DangerousGetHandle(), // Assign to standard error
                            //cbReserved2 = 0, // Reset unused fields
                            //lpReserved2 = IntPtr.Zero
                        };
 
                        // Call CreateProcess with bInheritHandles set to true
                        created = Native.CreateProcess(
                            null,
                            chromeCommandLine.ToString(),
                            IntPtr.Zero,
                            IntPtr.Zero,
                            true, // Inherit handles
                            0,
                            IntPtr.Zero,
                            browserSettings.ProfileDataDirectory,
                            ref startupInfo,
                            out var processInfo2);
 
                        if (!created)
                            throw new Win32Exception(Marshal.GetLastWin32Error(), "PuppeteerPipe CreateProcess failed.");
 
                        hProcess = processInfo2.hProcess;
 
                        // Assign streams
                        browserSettings.InboundPipe = stdinPipe;
                        browserSettings.OutboundPipe = stdoutPipe;
                        browserSettings.ErrorPipe = stderrPipe;
 
                        // Capture stderr
                        string stderr = "";
                        var errorReader = new StreamReader(stderrPipe, Encoding.UTF8, true, 4096, false);
                        _ = Task.Run(async () =>
                        {
                            try
                            {
                                char[] buffer = new char[4096];
                                int read;
                                while ((read = await errorReader.ReadAsync(buffer, 0, buffer.Length)) > 0)
                                {
                                    stderr += new string(buffer, 0, read);
                                }
                                if (Native.GetExitCodeProcess(hProcess, out uint finalExitCode) && finalExitCode != 0)
                                {
                                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Chrome exited with code: [{finalExitCode}], Stderr: [{stderr}]", browserSettings, GPALObjectType.Other);
                                }
                            }
                            catch (GPALException)
                            {
                                throw;
                            }
                            catch (Exception ex)
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Error reading stderr", browserSettings, GPALObjectType.Other, ex);
                            }
                            finally
                            {
                                errorReader.Dispose();
                            }
                        });
 
                        try
                        {
                            // Wait/verify process
                            if (!Native.GetExitCodeProcess(hProcess, out uint exitCode))
                                throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to get process exit code.");
                            if (exitCode != Native.STILL_ACTIVE)
                                throw new GPALException($"Chrome exited. Exit code: {exitCode}, Stderr: {stderr}, Check {Path.Combine(browserSettings.ProfileDataDirectory, "chrome_log.txt")}");
 
                            browserSettings.Process = Process.GetProcessById((int)processInfo2.dwProcessId);
                            browserSettings.Process.EnableRaisingEvents = true;
                        }
                        catch (ArgumentException ex)
                        {
                            if (Native.GetExitCodeProcess(hProcess, out uint exitCode) && exitCode != Native.STILL_ACTIVE)
                                throw new GPALException($"Chrome exited early. Exit code: {exitCode}, Stderr: {stderr}");
                            throw new GPALException($"Failed to get Chrome process by ID: {ex.Message}, Stderr: {stderr}");
                        }
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to launch Chrome with pipes", browserSettings, GPALObjectType.Other, ex);
                        throw;
                    }
                    finally
                    {
                        if (reservedPtr != IntPtr.Zero)
                            Marshal.FreeHGlobal(reservedPtr);
                        if (hProcess != IntPtr.Zero)
                            Native.CloseHandle(hProcess);
                        if (!created)
                        {
                            stdinPipe?.Dispose();
                            stdoutPipe?.Dispose();
                            stderrPipe?.Dispose();
                        }
                    }
                }
                else if (true == browserSettings.HiddenDesktop)
                {
                    // built by hand because ProcessStartInfo has nowhere to put lpDesktop, and a child started
                    // without one named takes the desktop of the parent process rather than of the thread that
                    // started it. Naming it is the only thing that puts the browser on another desktop.
                    // The cost is the browser's own stdout and stderr, which only ever reach DEEPDEBUG
                    Native.STARTUPINFO hiddenStartup = new Native.STARTUPINFO
                    {
                        cb = (uint)Marshal.SizeOf<Native.STARTUPINFO>(),
                        lpDesktop = HiddenDesktop.StartupNameFor(browserSettings.HiddenDesktopName)
                    };
 
                    string hiddenCommandLine = $@"""{browserPath}"" {string.Join(" ", chromeArguments)}";
 
                    if (false == Native.CreateProcess(null, hiddenCommandLine, IntPtr.Zero, IntPtr.Zero, false, 0, IntPtr.Zero, null, ref hiddenStartup, out var hiddenProcess))
                        throw new Win32Exception(Marshal.GetLastWin32Error(), $"Could not start [{browserSettings.BrowserType}] on [{browserSettings.HiddenDesktopName}]");
 
                    Native.CloseHandle(hiddenProcess.hProcess);
                    Native.CloseHandle(hiddenProcess.hThread);
 
                    browserSettings.Process = Process.GetProcessById((int)hiddenProcess.dwProcessId);
 
                    // Capture the launcher PID immediately before it exits (Firefox only).
                    if (BrowserType.FireFox == browserSettings.BrowserType)
                        firefoxLauncherPid = browserSettings.Process?.Id ?? 0;
                }
                else
                {
                    browserSettings.Process = Process.Start(new ProcessStartInfo
                    {
                        FileName = browserPath,
                        Arguments = string.Join(" ", chromeArguments),
                        UseShellExecute = false,
                        // chrome and edge announce "DevTools listening on ws://..." and a stream of their own
                        // internal ERROR lines on stderr. left alone the child inherits this console and writes
                        // them straight into the middle of the workflow's output
                        RedirectStandardOutput = true,
                        RedirectStandardError = true
                    });
 
                    // the streams have to be drained or the browser stalls once the pipe buffer fills. none of it
                    // is protocol here, the devtools connection is over the port, so it goes no further than
                    // DEEPDEBUG for anyone who wants to read it
                    browserSettings.Process.OutputDataReceived += (sender, data) => PublishBrowserOutput(browserSettings, data);
                    browserSettings.Process.ErrorDataReceived += (sender, data) => PublishBrowserOutput(browserSettings, data);
                    browserSettings.Process.BeginOutputReadLine();
                    browserSettings.Process.BeginErrorReadLine();
 
                    // Capture the launcher PID immediately before it exits (Firefox only).
                    if (BrowserType.FireFox == browserSettings.BrowserType)
                        firefoxLauncherPid = browserSettings.Process?.Id ?? 0;
 
                    DateTime deadline = DateTime.UtcNow + timeout;
 
                    // we have no window to check to ensure the browser launched in headless
                    // so check the process flag
                    // for windowed, wait for the window to open
                    if (true == headless)
                    {
                        if (true == browserSettings.Process.HasExited)
                        {
                            string msg = $"[{browserSettings.BrowserType}] did not start. Exiting.";
                            GPAL.PublishSimpleEvent(GPALEventType.ERROR, msg);
                            throw new TimeoutException(msg);
                        }
                    }
                    else
                    {
                        while (browserSettings.Process.MainWindowHandle == IntPtr.Zero && !browserSettings.Process.HasExited)
                        {
                            if (DateTime.UtcNow > deadline)
                            {
                                string msg = $"[{browserSettings.BrowserType}] did not start in [{secondsToWait}] seconds. Exiting.";
                                GPAL.PublishSimpleEvent(GPALEventType.ERROR, msg);
                                throw new TimeoutException(msg);
                            }
 
                            Thread.Sleep(100); // prevents busy-wait CPU spike
                        }
 
                        // the window exists now, so it goes where the workflow asked. this is the only placement
                        // that works for every browser: the arguments above are chromium's, and firefox has no
                        // command line way to be given a position at all. the arguments still earn their keep by
                        // opening the window at the right size, so there is no jump to watch
                        if (false == browserSettings.WindowSize.IsEmpty &&
                            false == browserSettings.FullScreen &&
                            false == browserSettings.Maximize &&
                            false == browserSettings.Minimize)
                        {
                            browserSettings.Process.Refresh();
                            WindowHelper.ResizeWindow(browserSettings.Process, browserSettings.WindowSize);
                        }
                    }
                }
 
                if (true == browserSettings.UseOttoMagic)
                {
                    try
                    {
                        var remoteEp = new IPEndPoint(IPAddress.Any, 0);
                        int restPort = 0;
                        int firstHeard = 0;
                        int launchedPid = browserSettings.Process?.Id ?? 0;
 
                        // every browser carrying the extension announces a port, whatever engine is driving it, so
                        // what arrives here is not necessarily this browser's. each announcement says which host
                        // sent it, and a host is started by the browser hosting the extension, so an announcement
                        // belongs to this launch when its host traces back to the process this launch started.
                        // the others are somebody else's and are left for whoever is waiting on them
                        while (0 == restPort)
                        {
                            byte[] data;
 
                            // nothing more is coming. an announcement that could not be traced is better than no
                            // browser at all, so one that was heard is taken now, with the reason it is a guess
                            try { data = udpHandshake.Receive(ref remoteEp); }
                            catch (SocketException) when (0 != firstHeard)
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"No GPALRestAPI traced back to this browser, so port [{firstHeard}] is being used on the strength of it being the only one announced. An older GPALRestAPI cannot say which browser it belongs to", browserSettings, GPALObjectType.Browser);
 
                                restPort = firstHeard;
                                break;
                            }
 
                            string[] said = Encoding.UTF8.GetString(data).Trim().Split(' ');
 
                            // a browser that never heard the port has no way to reach GPALRestAPI, and everything
                            // after this would be asking a url that was never worked out. it stops here instead,
                            // rather than carrying on and failing somewhere that does not say why
                            if (false == int.TryParse(said[0], out int announced))
                                throw new InvalidOperationException($"GPALRestAPI announced [{string.Join(" ", said)}], which is not a port number");
 
                            // an older GPALRestAPI sends the port on its own and cannot be traced to a browser.
                            // it is kept as the fallback rather than used straight away, so an announcement that
                            // can be confirmed still wins if one turns up
                            if (2 > said.Length || false == int.TryParse(said[1], out int hostPid))
                            {
                                if (0 == firstHeard)
                                    firstHeard = announced;
 
                                continue;
                            }
 
                            if (0 == launchedPid || true == BrowserHelper.CameFromProcess(hostPid, launchedPid))
                                restPort = announced;
                            else
                            {
                                if (0 == firstHeard)
                                    firstHeard = announced;
 
                                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Port [{announced}] was announced by another browser's GPALRestAPI [{hostPid}], so this launch is still waiting for its own", browserSettings, GPALObjectType.Browser);
                            }
                        }
 
                        browserSettings.RestApiBaseUrl = $"http://localhost:{restPort}/";
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"GPALRestAPI ready on port [{restPort}].", browserSettings, GPALObjectType.Browser);
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"GPALRestAPI never announced a port, so this browser cannot reach it", browserSettings, GPALObjectType.Browser, ex);
                        throw;
                    }
                    finally
                    {
                        udpHandshake?.Close();
 
                        // the port is read and the socket is free, so the next launch can go. Everything below here
                        // is this browser's own business and does not touch the handshake
                        if (true == holdingHandshakeGate)
                        {
                            ottoHandshakeGate.ReleaseMutex();
                            holdingHandshakeGate = false;
                        }
                    }
 
                    // Firefox launches via a wrapper that exits immediately; the real process is a child.
                    // Windows preserves ParentProcessId even after the parent exits, so we can query
                    // for the direct firefox.exe child of the launcher PID we captured at start.
                    if (firefoxLauncherPid > 0 &&
                        (browserSettings.Process == null || browserSettings.Process.HasExited))
                    {
                        try
                        {
                            using (var searcher = new ManagementObjectSearcher(
                                $"SELECT ProcessId FROM Win32_Process WHERE ParentProcessId = {firefoxLauncherPid} AND Name = 'firefox.exe'"))
                            {
                                foreach (ManagementObject obj in searcher.Get())
                                {
                                    int childPid = Convert.ToInt32(obj["ProcessId"]);
                                    try { browserSettings.Process = System.Diagnostics.Process.GetProcessById(childPid); break; }
                                    catch { }
                                }
                            }
                        }
                        catch { }
                    }
                }
                else if (true == browserSettings.UsePuppeteer)
                {
                    _ = browserSettings.PuppeteerCommunicator;    // initialize before we start using
                }
 
                // Handle maximize/minimize
                if (browserSettings.Maximize)
                {
                    try
                    {
                        if (true == browserSettings.UseOttoMagic)
                            // Use maximize window via extension - the browser we just launched, not whoever's helper was called
                            browser.MagicHelper.Client.Maximize().Execute();
                        else if (true == browserSettings.UsePuppeteer)
                            browserSettings.PuppeteerCommunicator.Maximize().GetAwaiter().GetResult();
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to maximize browser window", browserSettings, GPALObjectType.Other, ex);
                    }
                }
                else if (browserSettings.Minimize)
                {
                    try
                    {
                        if (true == browserSettings.UseOttoMagic)
                            browser.MagicHelper.Client.Minimize().Execute();
                        else if (true == browserSettings.UsePuppeteer)
                            browserSettings.PuppeteerCommunicator.Minimize().GetAwaiter().GetResult();
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to minimize browser window", browserSettings, GPALObjectType.Other, ex);
                    }
                }
                else if (browserSettings.FullScreen)
                {
                    try
                    {
                        if (true == browserSettings.UseOttoMagic)
                            browser.MagicHelper.Client.FullScreen().Execute();
                        else if (true == browserSettings.UsePuppeteer)
                            browserSettings.PuppeteerCommunicator.FullScreen().GetAwaiter().GetResult();
                    }
                    catch (GPALException)
                    {
                        throw;
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to full screen browser window", browserSettings, GPALObjectType.Other, ex);
                    }
                }
 
                // which exe, which process and which port, together on one line. a workflow driving a browser
                // it did not start looks identical to one driving its own until these three are compared
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Browser [{browserSettings.BrowserType}] is process [{browserSettings.Process?.Id}] from [{browserPath}] on port [{browserSettings.DebugPort}] at [{browserSettings.PuppeteerUrl}]", browserSettings, GPALObjectType.Other);
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Browser [{browserSettings.BrowserType}] launched.", browserSettings, GPALObjectType.Other);
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                // said, then handed to whoever asked for the browser. it does not end the application here: a
                // scheduler running ten workflows loses the other nine, and a form loses itself, over one browser
                // that would not start. Nothing catches it, the uncaught handler still cleans up and exits
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to launch browser [{browserSettings.BrowserType}] for URL [{URL}]", browserSettings, GPALObjectType.Other, ex);
                throw;
            }
            finally
            {
                // a launch that failed before it ever read a port still has to let the next one in, and it has to
                // give up the socket as well. Holding 47623 past a failed launch is not a slow browser, it is the
                // next OttoMagic launch in this application failing the moment it asks for the socket
                udpHandshake?.Close();
 
                if (true == holdingHandshakeGate)
                    ottoHandshakeGate.ReleaseMutex();
            }
 
             return browserSettings.Process;
        }

        public bool TopBrowser(Process process) => WindowHelper.TopBrowser(process);
        // create legit cookies some antibot sites use, captcha, recaptcha, others
        /*
        [Obsolete]
        internal bool LetsDoSomeSurfing(BrowserSettings browserSettings)
        {
            bool retVal = false;
            if (browserSettings.StealthType.HasFlag(StealthType.GenerateRealCookies))
            {
                //browserSettings.Browser.WithWaitOnIdleConnection(true);   // taking really long...
                for (int i = 0; i < 10; i++)
                {
                    string targetUrl = GetTargetUrl();
                    int sleepTime = 2000 + new Random().Next(1500);
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Generating real cookie with visit to [{targetUrl}]. Waiting [{sleepTime}] ms before next visit.", browserSettings.Browser, GPALObjectType.Browser);
                    browserSettings.Browser.GoTo(targetUrl);
                    Thread.Sleep(sleepTime);
                    _ = browserSettings.Browser.PageDown;
                    _ = browserSettings.Browser.PageDown;
                    _ = browserSettings.Browser.PageDown;
                    _ = browserSettings.Browser.PageDown;
                }
                browserSettings.Browser.WithWaitOnIdleConnection(false);
                retVal = true;
            }
            return retVal;
        }
        */

        public static string GetRandomStarWarsQuery()
        {
            IRESTClient restClient = GPAL.RESTClient.WithAPIBase("https://swapi.info/api/").ToGPALObject();
 
            var endpoints = new[]
            {
                ("people", 83, "name"),    // e.g., "Yoda"
                ("planets", 60, "name"),   // e.g., "Alderaan"
                ("films", 6, "title"),     // e.g., "A New Hope"
                ("species", 37, "name"),   // e.g., "Wookiee"
                ("vehicles", 39, "name"),  // e.g., "X-wing"
                ("starships", 36, "name")  // e.g., "Millennium Falcon"
            };
 
            string FetchQuery(string endpoint, int maxId, string field, int retries = 3)
            {
                for (int attempt = 0; attempt < retries; attempt++)
                {
                    int id = new Random().Next(1, maxId + 1);
                    var response = restClient.WithEndpoint($"{endpoint}/{id}").Execute<SwapiResponse>(false);
                    if (response != null)
                    {
                        return field == "name" ? response.name : response.title;
                    }
                    Thread.Sleep(100 * (int)Math.Pow(2, attempt)); // Backoff: 100ms, 200ms, 400ms
                }
                return null; // Let caller handle failure
            }
 
            // Select random primary endpoint
            var (endpoint, maxId, field) = endpoints[new Random().Next(endpoints.Length)];
            string query = FetchQuery(endpoint, maxId, field);
 
            // Fallback to another endpoint if null
            if (query == null)
            {
                var fallbackEndpoint = endpoints[new Random().Next(endpoints.Length)];
                query = FetchQuery(fallbackEndpoint.Item1, fallbackEndpoint.Item2, fallbackEndpoint.Item3);
                if (query == null) // Last resort, pick another
                {
                    fallbackEndpoint = endpoints[new Random().Next(endpoints.Length)];
                    query = FetchQuery(fallbackEndpoint.Item1, fallbackEndpoint.Item2, fallbackEndpoint.Item3) ?? "star wars"; // Absolute last fallback
                }
            }
 
            if (new Random().NextDouble() > 0.4) // 60% chance to add second term
            {
                var (otherEndpoint, otherMaxId, otherField) = endpoints[new Random().Next(endpoints.Length)];
                string otherQuery = FetchQuery(otherEndpoint, otherMaxId, otherField);
                if (otherQuery != null)
                {
                    query += $" {otherQuery}"; // e.g., "Yoda Millennium Falcon"
                }
                else
                {
                    var fallbackEndpoint = endpoints[new Random().Next(endpoints.Length)];
                    otherQuery = FetchQuery(fallbackEndpoint.Item1, fallbackEndpoint.Item2, fallbackEndpoint.Item3);
                    query += otherQuery != null ? $" {otherQuery}" : ""; // Skip if still null
                }
            }
 
            return query;
        }
 
        // visit various google sites to generate legit cookies using random search terms
        private string GetTargetUrl()
        {
            var urls = new[]
            {
                $"https://www.google.com/search?q={Uri.EscapeDataString(GetRandomStarWarsQuery())}", // Search
                $"https://www.youtube.com/results?search_query={Uri.EscapeDataString(GetRandomStarWarsQuery())}", // Search
                $"https://maps.google.com/?q={Uri.EscapeDataString(GetRandomStarWarsQuery())}", // Search
                "https://news.google.com/" // Static
            };
            return urls[new Random().Next(urls.Length)];
        }
        private class SwapiResponse
        {
            public string name { get; set; }
            public string title { get; set; }
        }
 
        public static string GetFullUrl(string url, IBrowser browser, out bool areRobotsAllowed, bool noRobotsCheck = false)
        {
            List<string> robotsTxt = new List<string>();
            string normalized = url;
            areRobotsAllowed = true;
 
            try
            {
                // Default to HTTPS only if no scheme supplied
                if (!Uri.TryCreate(url, UriKind.Absolute, out _))
                {
                    normalized = "https://" + url;
                }
 
                // files the browser merely opens or downloads (sitemap.xml, pdfs, images...) are not pages, skip the checks
                if (false == noRobotsCheck && true == ((Browser)browser).BrowserSettings.ObeyRobotsTxt && false == UrlHelper.IsNonHtmlResource(normalized))
                    areRobotsAllowed = UrlHelper.CheckRobotsTxt(normalized, browser, robotsTxt);
 
                if (true == url.StartsWith("https://"))
                    return url;
 
                if (string.IsNullOrWhiteSpace(url) || UrlHelper.AreEquivalent(url, "https://google.com"))
                    return "https://google.com";
 
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Error getting full url", browser, GPALObjectType.Other, ex);
            }
 
            return normalized;
        }

        public static int CalculateHash(Dictionary<string, object> attributes)
        {
            // 1. Collect and format attributes into a list of strings.
            // We use LINQ's OrderBy to sort the dictionary entries by key
            // before converting them to strings, ensuring consistent order.
            var attrList = attributes.OrderBy(kvp => kvp.Key)
                                     .Select(kvp => $"{kvp.Key}={kvp.Value}")
                                     .ToList();
 
            // 2. Join the sorted attributes into a single string.
            string str = string.Join("|", attrList);
 
            // 3. Implement the custom hashing algorithm.
            int hash = 0;
            foreach (char c in str)
            {
                // The bitwise operations from the JS code are translated to C#
                hash = ((hash << 5) - hash + c) | 0;
            }
 
            return hash;
        }

        public static string GetUserAgentString(BrowserType browserType)
        {
            // construct headless user-agent
            string chromeVersion;
            string edgeVersion;
            string firefoxVersion;
            string retval = null;
 
            if (BrowserType.Chrome == browserType)
            {
                DriverHelper.GetChromeVersion(out chromeVersion);
                // a dotless or empty version means detection failed; slicing to IndexOf(-1) would throw
                if (null != chromeVersion && 0 < chromeVersion.IndexOf('.'))
                    chromeVersion = chromeVersion.AsSpan().Slice(0, chromeVersion.IndexOf('.')).ToString(); // NOTE: fastest way to get the first token
                retval = String.Format(GPAL.GPALSettings.ChromeUserAgentTemplate, chromeVersion);
            }
            else if (BrowserType.Edge == browserType)
            {
                DriverHelper.GetEdgeVersion(out edgeVersion);
                // a dotless or empty version means detection failed; slicing to IndexOf(-1) would throw
                if (null != edgeVersion && 0 < edgeVersion.IndexOf('.'))
                    edgeVersion = edgeVersion.AsSpan().Slice(0, edgeVersion.IndexOf('.')).ToString(); // NOTE: fastest way to get the first token
                retval = String.Format(GPAL.GPALSettings.EdgeUserAgentTemplate, edgeVersion);
            }
            else if (BrowserType.FireFox == browserType)
            {
                DriverHelper.GetFirefoxVersion(out firefoxVersion);
                // a dotless or empty version means detection failed; slicing to IndexOf(-1) would throw
                if (null != firefoxVersion && 0 < firefoxVersion.IndexOf('.'))
                    firefoxVersion = firefoxVersion.AsSpan().Slice(0, firefoxVersion.IndexOf('.')).ToString(); // NOTE: fastest way to get the first token
                retval = String.Format(GPAL.GPALSettings.FirefoxUserAgentTemplate, firefoxVersion);
            }
 
            return retval;
        }

        private static void KeepFromChildren(UdpClient client)
        {
            if (false == Native.SetHandleInformation(client.Client.Handle, Native.HANDLE_FLAG_INHERIT, 0))
                GPAL.PublishSimpleEvent(GPALEventType.WARNING,
                    $"Could not take the inherit flag off the handshake socket, so the next OttoMagic launch may find [47623] still held",
                    null, GPALObjectType.Browser, new Win32Exception(Marshal.GetLastWin32Error()));
        }
 
        internal string GetUserAgent(BrowserSettings browserSettings, List<string> chromeArguments)
        {
            string userAgent = null;
            string browserPath = null;
 
            // starting a whole browser to ask it its own name is the accurate answer and the expensive one, so it
            // happens only when the workflow asked for it. Otherwise what is already known is good enough, and it
            // is the same string GPAL's own requests present
            if (false == browserSettings.UserAgentFromBrowser)
                return BrowserHelper.ResolveUserAgent(browserSettings);
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, "Getting user agent for headless use.", chromeArguments, GPALObjectType.Other);
 
            // Get browser executable path
            switch (browserSettings.BrowserType)
            {
                case BrowserType.Chrome:
                    browserPath = Puppeteer.GetChromePath();
                    break;
                case BrowserType.Edge:
                    browserPath = Puppeteer.GetEdgePath();
                    break;
                case BrowserType.FireFox:
                    browserPath = Puppeteer.GetFirefoxPath();
                    break;
                default:
                    Exception ex = new Exception("Unsupported Browser Type");
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unsupported browser type [{browserSettings.BrowserType}]", browser, GPALObjectType.Browser, ex);
                    return GetUserAgentString(browserSettings.BrowserType);
            }
 
            UdpClient probeHandshake = null;
            bool holdingProbeGate = false;
 
            try
            {
                List<string> ourArgs = new List<string>(chromeArguments);
 
                // same reason as the launch above: every browser gets its own port or they share a session
                if (null == browserSettings.DebugPort)
                    browserSettings.DebugPort = BrowserHelper.FindFreePort();
 
                if (BrowserType.FireFox != browserSettings.BrowserType)
                    ourArgs.Add($"--remote-debugging-port={browserSettings.DebugPort.Value}");
                else
                {
                    // firefox is asked over the rest api, so listen before starting it: the browser announces the
                    // port it picked the way every OttoMagic browser does, and nothing else is listening this early
                    // in a launch. Without it the question goes to whichever browser happens to already be up
                    try { holdingProbeGate = ottoHandshakeGate.WaitOne(OttoHandshakeGateTimeoutMs); }
                    catch (AbandonedMutexException) { holdingProbeGate = true; }
 
                    probeHandshake = new UdpClient(47623);
                    probeHandshake.Client.ReceiveTimeout = 30000;
 
                    KeepFromChildren(probeHandshake);
                }
 
                browserSettings.Process = Process.Start(new ProcessStartInfo
                {
                    FileName = browserPath,
                    Arguments = string.Join(" ", ourArgs),
                    UseShellExecute = false
                });
 
                //                Thread.Sleep(3000);
 
                // NOTE: always use puppeteer to get the useragent, it's very reliable, except ff which doesn't use puppeteer
                if (BrowserType.FireFox == browserSettings.BrowserType)
                {
                    var probeEndPoint = new IPEndPoint(IPAddress.Any, 0);
                    string probePortText = Encoding.UTF8.GetString(probeHandshake.Receive(ref probeEndPoint)).Trim();
 
                    // ask the browser we just launched, on the port it just announced
                    userAgent = true == int.TryParse(probePortText, out int probePort)
                        ? ((IRESTClient)GPAL.RESTClient.WithAPIBase($"http://localhost:{probePort}/").ToGPALObject()).GetUserAgent().Execute()
                        : Client.GetUserAgent().Execute();
                }
                else
                {
                    AutomationEngine aeSave = browserSettings.AutomationEngine;
                    browserSettings.AutomationEngine = AutomationEngine.PuppeteerPort;
                    _ = browserSettings.PuppeteerCommunicator;
                    userAgent = browserSettings.PuppeteerClient.GetUserAgent().Execute();
                    browserSettings.AutomationEngine = aeSave;
                }
                userAgent = userAgent.Replace("Headless", "");
 
                //if (true == browserSettings.UsePuppeteer)
                {
                    var parameters = new Dictionary<string, object>();
                    // gracefully close the browser, release resources, shutdown chrome
                    //browserSettings.PuppeteerCommunicator.SendCommand(DevToolsMethods.BrowserClose, parameters, null).GetAwaiter().GetResult();
                    browserSettings.PuppeteerCommunicator._readerReadyTcs?.SetCanceled();
                    if (false == browserSettings.PuppeteerCommunicator._usePipes)
                        browserSettings.PuppeteerCommunicator.CloseOutputWebSocketAsync().GetAwaiter();
                }
 
                // NOTE: KLUDGE: might not be required but we are, at least for puppeteer getting an error relaunching chrome after getting the useragent
                // - we added puppeteer browserclose rather than a hard kill
                // Lock file can not be created! Error code: 32
                // Failed to create a ProcessSingleton for your profile directory. This means that running multiple instances would start multiple browser processes rather than opening a new window in the existing process. Aborting now to avoid profile corruption.
 
                // kill the opened chrome and any children used to get the agent string
                try
                {
                    var psi = new ProcessStartInfo
                    {
                        FileName = "taskkill",
                        Arguments = $"/PID {browserSettings.Process.Id} /F /T",
                        CreateNoWindow = true,
                        UseShellExecute = false
                    };
                    Process.Start(psi).WaitForExit();
                }
                catch { }
 
                Thread.Sleep(1_000);
 
                if (null != browserSettings.Process && false == browserSettings.Process.HasExited)
                {
                    try
                    {
                        browserSettings.Process.Kill();
                        browserSettings.Process.Close();
                    }
                    catch { }
                }
 
                Thread.Sleep(1_000);
 
                // GPALRestAPI is not killed here: it shuts itself down when the extension's port closes, and killing
                // it by name would take down every other browser's rest api too
                browserSettings.PuppeteerClient = null;
                browserSettings.PuppeteerCommunicator = null;
                browserSettings.Process = null;
 
            }
            catch (GPALException)
            {
                throw;
            }
            catch (Exception ex)
            {
                userAgent = GetUserAgentString(browserSettings.BrowserType);
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to fetch user-agent using [{userAgent}].", browserSettings, GPALObjectType.Other, ex);
            }
            finally
            {
                probeHandshake?.Close();
 
                // the throwaway browser is gone, so the socket and the gate go back before the real launch asks for them
                if (true == holdingProbeGate)
                    ottoHandshakeGate.ReleaseMutex();
            }
 
            return false == string.IsNullOrEmpty(userAgent) ? userAgent : GetUserAgentString(browserSettings.BrowserType);
        }
        #endregion Helpers
    }
 
    public class WindowHelper
    {
        [DllImport("user32.dll")]
        [return: MarshalAs(UnmanagedType.Bool)]
        private static extern bool SetForegroundWindow(IntPtr hWnd);
 
        [DllImport("user32.dll")]
        [return: MarshalAs(UnmanagedType.Bool)]
        private static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
 
        [DllImport("user32.dll")]
        [return: MarshalAs(UnmanagedType.Bool)]
        private static extern bool IsIconic(IntPtr hWnd);
 
        [DllImport("user32.dll")]
        private static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam);
 
        [DllImport("user32.dll")]
        private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint lpdwProcessId);
 
        [DllImport("user32.dll")]
        private static extern bool IsWindowVisible(IntPtr hWnd);
 
        [DllImport("user32.dll")]
        private static extern int GetWindowTextLength(IntPtr hWnd);
 
        private delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
        private const int SW_RESTORE = 9;

        public static bool TopBrowser(Process process, int processId = 0)
        {
            if ((null == process || true == process?.HasExited) && 0 == processId)
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, "Unable to top browser. Browser has exited.", process, GPALObjectType.Other);
                return false;
            }
 
            IntPtr windowHandle = IntPtr.Zero;
 
            // 1. If a process ID is explicitly passed, scan the OS for its visible window
            if (processId > 0)
            {
                windowHandle = FindWindowByProcessId(processId);
            }
 
            // 2. Fallback to your original logic if the PID scan failed or wasn't requested
            if (IntPtr.Zero == windowHandle)
            {
                // Fixed: Removed process.Handle fallback because process handles cannot be passed to ShowWindow/SetForegroundWindow
                windowHandle = process.MainWindowHandle;
            }
 
            if (IntPtr.Zero == windowHandle)
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, "Unable to top browser. window handle is IntPtr.Zero", process, GPALObjectType.Other);
                return false;
            }
 
            // only a minimized window needs restoring. SW_RESTORE on a maximized one un-maximizes it back to
            // whatever size it was before, so topping the browser was quietly undoing .Maximize - it maximized,
            // then the first navigation topped it and it came back to its startup size
            if (true == IsIconic(windowHandle))
                ShowWindow(windowHandle, SW_RESTORE);
 
            bool success = SetForegroundWindow(windowHandle);
            if (false == success)
            {
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, "Unable to top browser. SetForegroundWindow returned false.", process, GPALObjectType.Other);
            }
 
            return success;
        }
 
        private static IntPtr FindWindowByProcessId(int processId)
        {
            IntPtr foundHandle = IntPtr.Zero;
 
            // The visible browser window may belong to the target process OR a descendant of it. Firefox is the
            // classic case: geckodriver (the pid we are handed) spawns firefox.exe as a child, and the window
            // belongs to that child - a direct pid match finds nothing. So match any visible top-level window
            // whose owning pid is the target or a descendant. Prefer a titled window (the main browser window
            // has a title; content/helper windows generally do not) so we do not grab a stray popup.
            HashSet<int> targetPids = GetProcessAndDescendantIds(processId);
 
            EnumWindows(delegate (IntPtr hWnd, IntPtr lParam)
            {
                if (IsWindowVisible(hWnd))
                {
                    GetWindowThreadProcessId(hWnd, out uint windowPid);
                    if (targetPids.Contains((int)windowPid))
                    {
                        foundHandle = hWnd;   // remember the last match, but keep looking for a titled one
                        if (0 < GetWindowTextLength(hWnd))
                            return false;     // titled window - this is the main browser window, stop
                    }
                }
                return true; // Keep looking
            }, IntPtr.Zero);
 
            return foundHandle;
        }

        private static HashSet<int> GetProcessAndDescendantIds(int rootPid)
        {
            var result = new HashSet<int> { rootPid };
 
            try
            {
                var frontier = new Queue<int>();
                frontier.Enqueue(rootPid);
 
                while (0 < frontier.Count)
                {
                    int parent = frontier.Dequeue();
                    using (var searcher = new ManagementObjectSearcher(
                        $"SELECT ProcessId FROM Win32_Process WHERE ParentProcessId = {parent}"))
                    {
                        foreach (ManagementObject obj in searcher.Get())
                        {
                            int childPid = Convert.ToInt32(obj["ProcessId"]);
                            if (result.Add(childPid))
                                frontier.Enqueue(childPid);
                        }
                    }
                }
            }
            catch { /* WMI unavailable - fall back to just the root pid */ }
 
            return result;
        }
 
 
        // Define the Windows API function
        [DllImport("user32.dll", SetLastError = true)]
        private static extern bool SetWindowPos(
            IntPtr hWnd,
            IntPtr hWndInsertAfter,
            int X,
            int Y,
            int cx,
            int cy,
            uint uFlags);
 
        // Define the Windows API constant for getting the window's current position and size
        [DllImport("user32.dll")]
        private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
 
        // Define the RECT structure for GetWindowRect
        [StructLayout(LayoutKind.Sequential)]
        private struct RECT
        {
            public int Left;
            public int Top;
            public int Right;
            public int Bottom;
        }
 
        private const uint SWP_NOMOVE = 0x0002;
        private const uint SWP_NOSIZE = 0x0001;
        private const uint SWP_NOZORDER = 0x0004;

        public static void ResizeWindow(
            Process process,
            Rectangle inRect)
        {
            // Check if the process has a main window handle
            if (process.MainWindowHandle == IntPtr.Zero)
            {
                return;
            }
 
            int maxWwidth = Puppeteer.GetScreenWidthInt();
            int maxHeight = Puppeteer.GetScreenHeightInt();
 
            if (inRect.Width > maxWwidth)
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Desired browser width [{inRect.Width}] is greater than the screen width [{maxWwidth}]");
 
            if (inRect.Height > maxHeight)
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Desired browser height [{inRect.Height}] is greater than the screen height [{maxHeight}]");
 
            // Determine the flags for SetWindowPos based on null parameters
            uint uFlags = SWP_NOZORDER;
 
            // Call the Windows API function
            SetWindowPos(
                process.MainWindowHandle,
                IntPtr.Zero, // No change to z-order
                inRect.Left,
                inRect.Top,
                inRect.Width,
                inRect.Height,
                uFlags);
        }
    }
}