ElementHelper.cs

// =============================================================================
// GPAL - Generally Positive Automation Library
// Copyright © 2026 Software Decisions, Inc. All rights reserved.
//
// This file is part of GPAL.
// Licensed under the Business Source License 1.1
//
// Primary development, architecture, and vision by Michael B. Vederman,
// CEO of Software Decisions, Inc., Texas.
//
// Internal development maintained privately.
// Public releases appear on GitHub: https://github.com/SoftwareDecisionsInc/GPAL.
//
// See LICENSE for full terms, including Additional Use Grant.
// =============================================================================
 
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Drawing;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using System.Threading;
using System.Windows;
using System.Windows.Automation;
using System.Windows.Forms;
using OpenQA.Selenium;
using OpenQA.Selenium.Interactions;
using OpenQA.Selenium.Support.UI;
using static GenerallyPositive.Enums;
using static GenerallyPositive.SelectorSettings;
using Condition = System.Windows.Automation.Condition;
using Desktops = GenerallyPositive.Browser.HiddenDesktop;
 
namespace GenerallyPositive.Application
{
 
    internal class ElementHelper
    {
        delegate ReadOnlyCollection<GPALAutomationElement> FindElementsByDelegate(Application application, SelectorPathEntry selectorPath, bool searchForSelector = true, AutomationElement element = null);
        // virtual keycodes: https://docs.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes?redirectedfrom=MSDN
        // scan codes: https://en.wikipedia.org/wiki/Scancode
        static byte ScanCode {get; } = 0;
 
        // to avoid repeat messages for the same selector when waiting for a selector
        static List<string> listOfMessages = new List<string>();
        static bool supressedMessage = false;
        static Selector lastSelector = null;

        public static ReadOnlyCollection<GPALAutomationElement> FindElements(Application application, UnitOfWork currentUOW, Selector selector, out bool matchedAll, out List<GPALAutomationElement> matchedElements, AutomationElement automationElement = null, bool callIfHandler = true)
        {
            int foundCount = 0;
 
            string elementStringToMatch = null;
            ReadOnlyCollection<GPALAutomationElement> retElems = null;
            List<GPALAutomationElement> matchedElems = new List<GPALAutomationElement>();
            MatchCollection mc = null;
            ReadOnlyCollection<GPALAutomationElement> allFoundElems = null;
            List<GPALAutomationElement> totalAllFoundElems = new List<GPALAutomationElement>();
            List<GPALAutomationElement> iterateOverTheseElements = null;
            bool tryNextSelector = false;
            int criteriaCnt = 0;
            string message = string.Empty;
            int retries = 1;
            bool retry = false;
 
            matchedAll = false;
            matchedElements = new List<GPALAutomationElement>();
 
            currentUOW.CurrentSelector = selector;
 
            if (false == lastSelector?.Name.Equals(selector.Name))
            {
                listOfMessages.Clear();
                supressedMessage = false;
                lastSelector = selector;
            }
            else if (null == lastSelector)
                lastSelector = selector;
 
            // !!!!!!!!!!!!!!! MUST MATCH THE ENUM ORDER !!!!!!!!!!!!!!!!!!
            // public enum SelectorPathType { NotSet = 0, Css = 1, Image = 2, Text = 4, Value = 8, Xpath = 16, Name = 32, AutomationID = 64, ClassName = 128 };
            List<FindElementsByDelegate> functionList = new List<FindElementsByDelegate> { null /* NotSet=0 */, FindElementsByCSS, FindElementsByImage, FindElementsByText, FindElementsByValue, FindElementsByXPath, FindElementsByName, FindElementsByAutomationID, FindElementsByClassName, FindElementsByPlaceholder };
            // iterate selectors, finding elements
            foreach (SelectorSettings.SelectorPathEntry selectorPathEntry in selector.SelectorPaths)
            {
            tryAgain:   // NOTE: CAVEAT: goto label - don't like it, but it has to happen sometimes. we cannot have any inner loop as that will break retrying the next selector
                int textLength = selectorPathEntry.SelectorPath.Length - 1;
                string abbreviatedText = selectorPathEntry.SelectorPath.Substring(0, Math.Min(7, textLength)) + "..." + selectorPathEntry.SelectorPath.Substring(Math.Max(0, textLength - 7));
                if (18 > selectorPathEntry.SelectorPath.Length)
                    abbreviatedText = selectorPathEntry.SelectorPath;
 
                int idx = GetIndexOfEnum<SelectorPathType>((int)selectorPathEntry.SelectorPathType);
                tryNextSelector = false;
                string errorMessage = $"Error finding elements for selector [{selector.Name}][{abbreviatedText}]";
 
                try
                {
                    message = $"Searching for elements for selector [{selector.Name}] using [{selectorPathEntry.SelectorPathType}] [{abbreviatedText}]";
 
                    if (false == listOfMessages.Contains(message))
                    {
                        listOfMessages.Add(message);
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, message, application, GPALObjectType.Application);
                    }
 
                    allFoundElems = functionList[idx](application, selectorPathEntry, selector.SearchForSelector, automationElement);
 
                    // if we get null back, we will retry once just to make sure
                    if (0 < (allFoundElems?.Count ?? 0))   // did not find any elements
                        totalAllFoundElems.AddRange(allFoundElems.ToList());
 
                    #region <Element Matching>
                    if (0 < selector.MatchCriteria.Count)
                    {
                        foreach (SelectorSettings.MatchCriteriaEntry matchCriteriaEntry in selector.MatchCriteria)
                        {
                            if (MatchType.Custom == matchCriteriaEntry.MatchType)
                            {
                                // match function returns true or false for matchedAll
                                CallIfStatus handled = matchCriteriaEntry.AppMatchingFunction(allFoundElems.ToList(), out matchedElems, out matchedAll, selector, matchCriteriaEntry.ExactMatch);
 
                                if (CallIfStatus.Terminate == handled)
                                {
                                    string msg = $"Custom match handler [{matchCriteriaEntry.AppMatchingFunction.GetInvocationList()[0].Method.Name}] requested program termination for selector {selector.Name} path [{selectorPathEntry.SelectorPath}].";
                                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, msg, application, GPALObjectType.Application);
                                    throw new GPALException(msg);
                                }
 
                                // always return matched elements, it's either all or just those matched
                                matchedElements.AddRange(matchedElems);
                                matchedElements = matchedElements.Distinct().ToList();  // dedupe   // what order these will be in over iterations is anyone's guess, but they will be on one page
 
                                if (CallIfStatus.Handled == handled)   // stop matching
                                    break;
                                else if (CallIfStatus.TryNext == handled)   // custom algorithm didn't like the result set and wants us to try the next selectorpath
                                {
                                    tryNextSelector = true;
                                    break;
                                }
                            }
                        }
 
                        if (true == tryNextSelector)
                            continue;
 
                        // if our custom match matched items, then apply further matching to these items
                        // else apply matching to all found elements
                        if (0 == matchedElements.Count)
                            iterateOverTheseElements = new List<GPALAutomationElement>(allFoundElems);
                        else
                            iterateOverTheseElements = new List<GPALAutomationElement>(matchedElements);
 
                        // we are done with idx, we can reuse, used as index into findelement method array
                        for (idx = 0; idx < iterateOverTheseElements.Count; idx++)
                        {
                            GPALAutomationElement automationElement2 = iterateOverTheseElements[idx];
                            criteriaCnt = 0;
 
                            for (int matchIdx = 0; matchIdx < selector.MatchCriteria.Count; matchIdx++)
                            {
                                SelectorSettings.MatchCriteriaEntry matchCriteriaEntry = selector.MatchCriteria[matchIdx];
 
                                if (MatchType.None != matchCriteriaEntry.MatchType)
                                {
                                    criteriaCnt++;  // keep count of how many match criteria we have, ALL must match on the element, we should have as many matched elements as criteria for ALL to match
 
                                    // Mask out the REGEX 1 bit to get to the actual type
                                    switch ((MatchType)((int)matchCriteriaEntry.MatchType & 0XFE))
                                    {
                                        case MatchType.Text:
                                            elementStringToMatch = automationElement2.Ae.GetText();
                                            break;
 
                                        // none has a meaning on a desktop control, so they read as the
                                        // element's text. placeholder had no case and compared against
                                        // whatever the criterion before it had read, or null
                                        case MatchType.Href:
                                        case MatchType.Src:
                                        case MatchType.Placeholder:
                                            elementStringToMatch = automationElement2.Ae.GetText();
                                            break;
 
                                        // the short one on purpose. A value match is exact and the length of
                                        // what is being matched is known, so a provider truncating a long value
                                        // cannot change the answer. MatchText is the one that has to see a whole
                                        // document, and a control offering no value still answers with its text
                                        case MatchType.Value:
                                            elementStringToMatch = automationElement2.Ae.GetValue() ?? automationElement2.Ae.GetText();
                                            break;
                                    }
 
                                    if (1 == ((int)MatchType.Regex & (int)matchCriteriaEntry.MatchType))
                                    {
 
                                        mc = Regex.Matches(elementStringToMatch, matchCriteriaEntry.StringToMatch);
                                        if (0 < mc.Count)
                                            matchedElems.Add(automationElement2);
                                    }
                                    else
                                    {
                                        if (false == matchCriteriaEntry.ExactMatch && elementStringToMatch.Contains(matchCriteriaEntry.StringToMatch))
                                            matchedElems.Add(automationElement2);
                                        else if (true == matchCriteriaEntry.ExactMatch && elementStringToMatch.Equals(matchCriteriaEntry.StringToMatch))
                                            matchedElems.Add(automationElement2);
                                    }
                                }
                            }
                            if (criteriaCnt == matchedElems.Count)  // we have one element that must be added once for every match, if not, it doesn't match all criteria, so try the next element
                                matchedElements.AddRange(matchedElems.Distinct());     // add this match to the out return list (dedupe it first)
                        }
                        matchedElements = matchedElements.Distinct().ToList();  // dedupe   // what order these will be in over iterations is anyone's guess, but they will be on one page
                        matchedAll = (allFoundElems.Count == foundCount);
                    }
                    else // no match criteria - all found
                    {
                        matchedAll = true;
                        matchedElems = allFoundElems?.ToList() ?? new List<GPALAutomationElement>();
                        matchedElements = matchedElems;     // out return list
                    }
                    #endregion <Element Matching>
 
                }
                catch (Exception ex) // element not found for some reason
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Exception thrown trying to find elements for selector [{selector.Name}]. Continuing.", currentUOW, GPALObjectType.UnitOfWork, ex);
                    continue;
                }
 
                // return only WithAllThatMatch row count of those that matched (or from all)
                // matchedElems contains either All elememts or just the matched
                if (int.MaxValue != currentUOW.WithAllThatMatch && 0 < matchedElements?.Count)  // just WithAllThatMatch row count of elements
                {
                    int elemIdx = 0;
                    List<GPALAutomationElement> tmpList = new List<GPALAutomationElement>();
 
                    // ensure we copy over only as many results as requested, or how many results we have
                    while (elemIdx < Math.Min((int)currentUOW.WithAllThatMatch, (int)matchedElements?.Count))
                        tmpList.Add(matchedElements[elemIdx++]);
 
                    matchedElements = tmpList;
                }
 
                selectorPathEntry.AppSelectorFoundResults = allFoundElems?.ToList() ?? new List<GPALAutomationElement>();
                selectorPathEntry.AppSelectorMatchedResults = matchedElems;
 
                retElems = new ReadOnlyCollection<GPALAutomationElement>(totalAllFoundElems.Distinct().ToList());    // always return everything found
 
                foreach (GPALAutomationElement element in retElems)
                {
                    element.Application = application;
                    element.Selector = selector;
                }
 
                foreach (GPALAutomationElement element in matchedElements)
                {
                    element.Application = application;
                    element.Selector = selector;
                }
 
                if (true == retry && 0 < retries--)
                {
                    retry = false;
                    goto tryAgain;  // NOTE: CAVEAT: goto label
                }
 
                selector.ElementsFoundAndMatchedCount = matchedElements.Count();
 
                message = $"[{retElems?.Count ?? 0}] elements found for selector [{selector.Name}]";
 
                if (false == listOfMessages.Contains(message))
                {
                    listOfMessages.Add(message);
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, message, application, GPALObjectType.Application);
 
                    if (0 < retElems?.Count && (true == selector.AnyMatchCriteria() || Int32.MaxValue != currentUOW.WithAllThatMatch))
                    {
                        var matchCount = selector.MatchCriteria.Count > 0 ? selector.MatchCriteria.Count.ToString() : $"WithAllThatMatch({currentUOW.WithAllThatMatch})";
 
                        if ((true == matchCount.Contains("WithAllThatMatch") && 1 < currentUOW.WithAllThatMatch) || 0 < selector.MatchCriteria.Count)
                            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{matchedElements.Count}] elements matched for selector [{selector.Name}] with [{matchCount}] match criteria", application, GPALObjectType.Application);
                    }
                }
                else if (false == supressedMessage)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, "Supressing repeat messages", application, GPALObjectType.Application);
                    supressedMessage = true;
                }
 
                // CAVEAT:if one path matches all elements found, are we done? maybe the next path finds different elements we want to add?
                // TODO: do we have a fluent switch 'TryAllPaths' or 'StopOnSuccess'
                // For complex evaluations, roll your own matching function, 
                if (true == matchedAll || retElems?.Count > 0) // for now, if a selector path matches one match criteria, we consider ourselves done with matching
                    break;
 
            }   // foreach selectorpath
 
            // false == callIfHandler will be the waitfor call
            if (true == callIfHandler)
                CallIfHandlers(application, currentUOW, allFoundElems, matchedElements, selector, matchedAll);
 
            // a selector whose every path threw leaves both of these null, and the catch above says "Continuing",
            // so counting them unguarded turned a handled search failure into a crash. Counted the way the two
            // reads further up already do it, and the way the browser twin does
            selector.ElementsFoundAndMatchedCount = true == selector.AnyMatchCriteria()
                ? (matchedElements?.Count() ?? 0)
                : (retElems?.Count ?? 0);
 
            return retElems;
        }
 
        // TODO: what do these values mean outside of the bubble handlers?
        // currentUOW.callFoundHandled
        // currentUOW.callNotFoundHandled
        private static void CallIfHandlers(Application application, UnitOfWork currentUOW, ReadOnlyCollection<GPALAutomationElement> foundElements, List<GPALAutomationElement> matchedElements, Selector selector, bool matchedAll)
        {
            string str = null;
            string currentMethodName = null;
            UnitOfWork safeUOW;
 
            // TODO: do we care about matchedall?
 
            // bubble thru handlers based upon handler return value
            // 0 = not handled, bubble to next handler
            // 1 = handled and continue with the program
            // -1 = unexpected error and terminate program
            if (0 < (foundElements?.Count ?? 0))
            {
                if (0 < selector.AppCallIfFound?.Count)
                {
                    foreach (Application.CallIfDelegate func in selector.AppCallIfFound)
                    {
                        bool safeActionCalled = application.CurrentUOW.ActionCalled;
                        safeUOW = application.CurrentUOW;
                        application.CurrentUOW.ActionCalled = true; // this will allow the callif method to immediately use application.WaitFor
 
                        currentMethodName = func.Method.Name;
                        str = $"Application Selector CallIfFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
 
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking Selector CallIfFound [{func.Method.Name}]", application, GPALObjectType.Application);
 
                        selector.AppCallIfFoundHandled = func(application, foundElements?.Cast<IGPALAutomationElement>().ToList(), matchedElements?.Cast<IGPALAutomationElement>().ToList(), selector, matchedAll);
 
                        application.CurrentUOW = safeUOW;
                        application.CurrentUOW.ActionCalled = safeActionCalled;
 
                        if (CallIfStatus.NotHandled == selector.AppCallIfFoundHandled)
                            continue;
                        else
                            break;
                    }
 
                    if (CallIfStatus.Terminate == selector.AppCallIfFoundHandled)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, str, application, GPALObjectType.Application);
                        throw new GPALException(str);
                    }
                }
 
                // selector.AppCallIfFoundHandled - if handled by selector handler, don't call UOW handler
                if (0 == selector.AppCallIfFoundHandled && 0 < currentUOW.AppCallIfFound.Count)
                {
                    foreach (Application.CallIfDelegate func in currentUOW.AppCallIfFound)
                    {
                        safeUOW = application.CurrentUOW;
 
                        currentMethodName = func.Method.Name;
                        str = $"Application UOW Selector CallIfFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
 
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking UOW CallIfFound [{func.Method.Name}]", application, GPALObjectType.Application);
 
                        currentUOW.CallIfNotFoundHandled = func(application, foundElements?.Cast<IGPALAutomationElement>().ToList(), matchedElements?.Cast<IGPALAutomationElement>().ToList(), selector, matchedAll);
 
                        application.CurrentUOW = safeUOW;
 
                        if (CallIfStatus.NotHandled == currentUOW.CallIfNotFoundHandled)
                            continue;
                        else
                            break;
                    }
 
                    if (CallIfStatus.Terminate == currentUOW.AppCallIfFoundHandled)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, str, application, GPALObjectType.Application);
                        throw new GPALException(str);
                    }
                }
 
                // currentUOW.AppCallIfFoundHandled - if handled by UOW handler, don't call global handler
                if (0 == currentUOW.AppCallIfFoundHandled && 0 < GPAL.GPALSettings.AppCallIfFoundList.Count)
                {
                    foreach (Application.CallIfDelegate func in GPAL.GPALSettings.AppCallIfFoundList)
                    {
                        safeUOW = application.CurrentUOW;
 
                        currentMethodName = func.Method.Name;
                        str = $"Gloabl CallIfFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
 
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking Global CallIfFound [{func.Method.Name}]", application, GPALObjectType.Application);
 
                        GPAL.GPALSettings.AppCallIfFoundHandled = func(application, foundElements?.Cast<IGPALAutomationElement>().ToList(), matchedElements?.Cast<IGPALAutomationElement>().ToList(), selector, matchedAll);
 
                        application.CurrentUOW = safeUOW;
 
                        if (CallIfStatus.NotHandled == GPAL.GPALSettings.AppCallIfFoundHandled)
                            continue;
                        else
                            break;
                    }
 
                    if (CallIfStatus.Terminate == GPAL.GPALSettings.AppCallIfFoundHandled)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, str, application, GPALObjectType.Browser);
                        throw new GPALException(str);
                    }
                }
            }
            else
            {
                if (0 < selector.AppCallIfNotFound.Count)
                {
                    foreach (Application.CallIfDelegate func in currentUOW.AppCallIfNotFound)
                    {
                        safeUOW = application.CurrentUOW;
 
                        currentMethodName = func.Method.Name;
                        str = $"Application Selector CallIfNotFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
 
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking Selector CallIfNotFound [{func.Method.Name}]", application, GPALObjectType.Application);
 
                        selector.AppCallIfNotFoundHandled = func(application, foundElements?.Cast<IGPALAutomationElement>().ToList(), matchedElements?.Cast<IGPALAutomationElement>().ToList(), selector, matchedAll);
 
                        application.CurrentUOW = safeUOW;
 
                        if (CallIfStatus.NotHandled == selector.AppCallIfNotFoundHandled)
                            continue;
                        else
                            break;
                    }
 
                    if (CallIfStatus.Terminate == currentUOW.AppCallIfNotFoundHandled)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, str, application, GPALObjectType.Application);
                        throw new GPALException(str);
                    }
                }
 
                if (0 == selector.AppCallIfNotFoundHandled && 0 < currentUOW.AppCallIfNotFound.Count)
                {
                    foreach (Application.CallIfDelegate func in currentUOW.AppCallIfNotFound)
                    {
                        safeUOW = application.CurrentUOW;
 
                        currentMethodName = func.Method.Name;
                        str = $"Application UOW Selector CallIfNotFound handler [{currentMethodName}] requested program termination on selector ]{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
 
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking UOW CallIfNotFound [{func.Method.Name}]", application, GPALObjectType.Application);
 
                        currentUOW.AppCallIfNotFoundHandled = func(application, foundElements?.Cast<IGPALAutomationElement>().ToList(), matchedElements?.Cast<IGPALAutomationElement>().ToList(), selector, matchedAll);
 
                        application.CurrentUOW = safeUOW;
 
                        if (CallIfStatus.NotHandled == currentUOW.AppCallIfNotFoundHandled)
                            continue;
                        else
                            break;
                    }
 
                    if (CallIfStatus.Terminate == currentUOW.AppCallIfNotFoundHandled)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, str, application, GPALObjectType.Application);
                        throw new GPALException(str);
                    }
                }
 
                // currentUOW.AppCallIfNotFoundHandled - if handled by UOW handler, don't call global handler
                if (0 == currentUOW.AppCallIfNotFoundHandled && 0 < GPAL.GPALSettings.AppCallIfNotFoundList.Count)
                {
                    foreach (Application.CallIfDelegate func in GPAL.GPALSettings.AppCallIfNotFoundList)
                    {
                        safeUOW = application.CurrentUOW;
 
                        currentMethodName = func.Method.Name;
                        str = $"Gloabl CallIfNotFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
 
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking Global CallIfNotFound [{func.Method.Name}]", application, GPALObjectType.Application);
 
                        GPAL.GPALSettings.AppCallIfNotFoundHandled = func(application, foundElements?.Cast<IGPALAutomationElement>().ToList(), matchedElements?.Cast<IGPALAutomationElement>().ToList(), selector, matchedAll);
 
                        application.CurrentUOW = safeUOW;
 
                        if (CallIfStatus.NotHandled == GPAL.GPALSettings.AppCallIfNotFoundHandled)
                            continue;
                        else
                            break;
                    }
 
                    if (CallIfStatus.Terminate == GPAL.GPALSettings.AppCallIfNotFoundHandled)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, str, application, GPALObjectType.Browser);
                        throw new GPALException(str);
                    }
                }
 
                if (true == selector.StopOnNotFound || true == GPAL.StopOnNotFound)
                {
                    str = $@"{(true == selector.StopOnNotFound ? "selector." : "GPAL.")}StopOnNotFound set, terminating on Not Found element for [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, str, application, GPALObjectType.Application);
                    throw new GPALException(str);
                }
            }
        }
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsByImage(Application application, SelectorPathEntry selectorPath, bool searchForSelector = true, AutomationElement element = null)
        {
            ReadOnlyCollection<GPALAutomationElement> elems = null;
 
            Rectangle foundImageBounds = new Rectangle();
            List<GPALAutomationElement> retElement = new List<GPALAutomationElement>(); ;
 
            if (null == selectorPath.Image)
            {
                try
                {
                    selectorPath.Image = Image.FromFile(selectorPath.SelectorPath);
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to load image for [{selectorPath.SelectorPath}]", application, GPALObjectType.Application, ex);
                    return null;
                }
            }
 
            Image template = selectorPath.Image;
            foundImageBounds = ((ImageHelper)GPAL.ImageHelper).FindImage(selectorPath.SelectorPath, application); // first check, is image we seek already on-screen?
 
            // not on screen: from the top of the application's scrolling view, page down through it the way the
            // browser pages down a document, until the image shows up or the view is at its end. ScrollPattern moves
            // the view without a keystroke, so this works on a hidden desktop too
            // with no matcher registered nothing can be found, so the view is not paged through for nothing
            if (true == searchForSelector && true == foundImageBounds.IsEmpty && null != GPAL.ImageMatcher)
            {
                ScrollPattern view = application.ScrollableView();
 
                if (null != view)
                {
                    view.SetScrollPercent(ScrollPattern.NoScroll, 0);
                    Thread.Sleep(250);
                    foundImageBounds = ((ImageHelper)GPAL.ImageHelper).FindImage(selectorPath.SelectorPath, application);
 
                    // a page step that does not move the view is the end too, so a control that will not scroll
                    // cannot hold the search forever
                    double was = -1;
 
                    while (true == foundImageBounds.IsEmpty && 100 > Math.Round(view.Current.VerticalScrollPercent) && was != view.Current.VerticalScrollPercent)
                    {
                        was = view.Current.VerticalScrollPercent;
                        view.ScrollVertical(ScrollAmount.LargeIncrement);
                        Thread.Sleep(250);
                        foundImageBounds = ((ImageHelper)GPAL.ImageHelper).FindImage(selectorPath.SelectorPath, application);
                    }
                }
            }
 
            if (false == foundImageBounds.IsEmpty)
            {
                // create a psuedo automationelement to use in the subsequent methods
                retElement.Add(new GPALAutomationElement(null, new GPALElement(foundImageBounds.Location, foundImageBounds.Size, selectorPath.SelectorPathType.ToString(), null, "GPALElement")));
                elems = new ReadOnlyCollection<GPALAutomationElement>(retElement);
 
                // an image match is one more way of finding a control. unless the workflow asked for hardware, the
                // control under the match is handed back instead, so what follows is an ordinary UIAutomation
                // action. a hidden desktop is left alone, since the match was made against the visible screen
                Selector selector = application.CurrentUOW?.CurrentSelector;
 
                if (false == application.applicationSettings.HiddenDesktop
                    && false == application.applicationSettings.UseHardware
                    && Enums.InteractionType.Hardware != selector?.InteractionType)
                {
                    GPALAutomationElement under = ControlUnderImage(application, foundImageBounds, selector);
 
                    if (null != under)
                    {
                        under.ImageMatch = retElement[0];
                        elems = new ReadOnlyCollection<GPALAutomationElement>(new List<GPALAutomationElement> { under });
                    }
                }
            }
 
            return elems;
        }
        private static GPALAutomationElement ControlUnderImage(Application application, Rectangle match, Selector selector)
        {
            GPALAutomationElement retVal = null;
 
            // both zero means nothing was asked for, the middle of the match, same rule the clicks use
            int aimX = match.X + ((0 == (selector?.OffsetX ?? 0) && 0 == (selector?.OffsetY ?? 0)) ? match.Width / 2 : selector.OffsetX);
            int aimY = match.Y + ((0 == (selector?.OffsetX ?? 0) && 0 == (selector?.OffsetY ?? 0)) ? match.Height / 2 : selector.OffsetY);
 
            try
            {
                AutomationElement ae = AutomationElement.FromPoint(new System.Windows.Point(aimX, aimY));
 
                if (null != ae)
                {
                    // walked in the raw view, since WinForms hides whole controls from the control view. stops at a
                    // top level window, whose parent is the desktop
                    AutomationElement step = ae;
                    AutomationElement actionable = null;
 
                    while (null == actionable && null != step && false == Automation.Compare(step, AutomationElement.RootElement))
                    {
                        if (true == IsActionable(step))
                            actionable = step;
                        else
                            step = TreeWalker.RawViewWalker.GetParent(step);
                    }
 
                    retVal = new GPALAutomationElement(actionable ?? ae);
                }
            }
            catch (Exception ex)
            {
                // a control that goes away while it is being read leaves only the point
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Could not read the control under the image match for [{selector?.Name}]", application, GPALObjectType.Application, ex);
            }
 
            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, null != retVal
                ? $"Image match for [{selector?.Name}] is control [{retVal.Ae.Current.ControlType.ProgrammaticName}][{retVal.Ae.Current.Name}], clicked at the matched point"
                : $"Image match for [{selector?.Name}] has no control of its own under it, so only the point is used", application, GPALObjectType.Application);
 
            return retVal;
        }
        private static bool IsActionable(AutomationElement ae)
        {
            ControlType type = ae.Current.ControlType;
 
            return true == (bool)ae.GetCurrentPropertyValue(AutomationElement.IsInvokePatternAvailableProperty)
                || true == (bool)ae.GetCurrentPropertyValue(AutomationElement.IsTogglePatternAvailableProperty)
                || true == (bool)ae.GetCurrentPropertyValue(AutomationElement.IsValuePatternAvailableProperty)
                || true == (bool)ae.GetCurrentPropertyValue(AutomationElement.IsSelectionItemPatternAvailableProperty)
                || true == (bool)ae.GetCurrentPropertyValue(AutomationElement.IsExpandCollapsePatternAvailableProperty)
                || ControlType.Button == type || ControlType.MenuItem == type || ControlType.CheckBox == type
                || ControlType.RadioButton == type || ControlType.Edit == type || ControlType.Hyperlink == type
                || ControlType.ComboBox == type || ControlType.ListItem == type || ControlType.TabItem == type
                || ControlType.TreeItem == type || ControlType.SplitButton == type || ControlType.DataItem == type;
        }
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsByCSS(Application application, SelectorPathEntry selectorPath, bool searchForElement = true, AutomationElement element = null)
        {
            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Find elements by CSS is not supported for applications.", application, GPALObjectType.Application);
            return null;
        }
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsByPlaceholder(Application application, SelectorPathEntry selectorPath, bool searchForElement = true, AutomationElement element = null)
        {
            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Find elements by placeholder is not supported for applications.", application, GPALObjectType.Application);
            return null;
        }

        private static bool RootedAtDesktop(string selectorPath)
        {
            string[] tokens = (selectorPath ?? string.Empty).Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
 
            return 1 < tokens.Length
                && "desktop" == tokens[0].ToLowerInvariant()
                && false == tokens[1].ToLowerInvariant().StartsWith("window");
        }
 
        [DllImport("user32.dll", SetLastError = true)]
        private static extern IntPtr GetMenu(IntPtr window);
 
        [DllImport("user32.dll", SetLastError = true)]
        private static extern IntPtr GetSubMenu(IntPtr menu, int position);
 
        [DllImport("user32.dll", SetLastError = true)]
        private static extern int GetMenuItemCount(IntPtr menu);
 
        [DllImport("user32.dll", SetLastError = true)]
        private static extern int GetMenuItemID(IntPtr menu, int position);
 
        [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
        private static extern int GetMenuString(IntPtr menu, int item, System.Text.StringBuilder text, int count, uint flags);
 
        [DllImport("user32.dll", SetLastError = true)]
        private static extern bool PostMessage(IntPtr window, int message, IntPtr wParam, IntPtr lParam);
 
        private const uint MF_BYPOSITION = 0x0400;
        private const int WM_COMMAND = 0x0111;

        private static List<string> MenuNamesIn(string selectorPath)
        {
            List<string> retVal = new List<string>();
 
            foreach (Match step in Regex.Matches(selectorPath ?? string.Empty, @"MenuItem[^/]*?\‍[@Name='([^']*)'\‍]"))
                retVal.Add(step.Groups[1].Value);
 
            return retVal;
        }

        private static string MenuItemName(IntPtr menu, int position)
        {
            System.Text.StringBuilder text = new System.Text.StringBuilder(256);
 
            GetMenuString(menu, position, text, text.Capacity, MF_BYPOSITION);
 
            string name = text.ToString().Replace("&", string.Empty);
            int shortcut = name.IndexOf('\t');
 
            return (0 <= shortcut ? name.Substring(0, shortcut) : name).Trim();
        }

        private static int CommandForMenuPath(IntPtr menu, List<string> names, int index)
        {
            int retVal = 0;
 
            if (IntPtr.Zero != menu && index < names.Count)
                for (int position = 0; position < GetMenuItemCount(menu) && 0 == retVal; position++)
                {
                    if (false == string.Equals(MenuItemName(menu, position), names[index], StringComparison.OrdinalIgnoreCase))
                        continue;
 
                    IntPtr inside = GetSubMenu(menu, position);
 
                    retVal = index == names.Count - 1 && IntPtr.Zero == inside
                           ? GetMenuItemID(menu, position)
                           : CommandForMenuPath(inside, names, index + 1);
                }
 
            return retVal;
        }

        private static GPALAutomationElement MenuItemFor(Application application, string selectorPath)
        {
            GPALAutomationElement retVal = null;
            List<string> names = MenuNamesIn(selectorPath);
            AutomationElement root = application.RootAutomationElement;
 
            if (0 < names.Count && null != root)
            {
                IntPtr window = new IntPtr(root.Current.NativeWindowHandle);
                int command = CommandForMenuPath(GetMenu(window), names, 0);
 
                if (0 != command)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{string.Join(" then ", names)}] was not there to be walked, so it was read from the window's own menu, where it sends command [{command}].", application, GPALObjectType.Application);
 
                    retVal = new GPALAutomationElement { MenuCommand = command, MenuWindow = window };
                }
            }
 
            return retVal;
        }

        private static List<AutomationElement> StartingPoints(Application application, string selectorPath)
        {
            List<AutomationElement> retVal = new List<AutomationElement>();
 
            if (false == RootedAtDesktop(selectorPath))
                retVal.Add(application.RootAutomationElement);
            else if (false == application.applicationSettings.HiddenDesktop)
                retVal.Add(AutomationElement.RootElement);
            else
            {
                // the application's own window first, since that is what most paths are after and it saves
                // walking the rest
                retVal.Add(application.RootAutomationElement);
 
                foreach (IntPtr window in Desktops.WindowsOn(application.applicationSettings.HiddenDesktopName))
                {
                    AutomationElement onDesktop = AutomationElement.FromHandle(window);
 
                    if (null != onDesktop && false == onDesktop.Equals(application.RootAutomationElement))
                        retVal.Add(onDesktop);
                }
            }
 
            retVal.RemoveAll(each => null == each);
 
            return retVal;
        }
 
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsByXPath(
            Application application,
            SelectorPathEntry selectorPath,
            bool searchForElement = true,
            AutomationElement element = null)
        {
            if (string.IsNullOrWhiteSpace(selectorPath?.SelectorPath))
                return new ReadOnlyCollection<GPALAutomationElement>(Array.Empty<GPALAutomationElement>());
 
            List<AutomationElement> roots = null != element
                ? new List<AutomationElement> { element }
                : StartingPoints(application, selectorPath.SelectorPath);
 
            AutomationElement current = 0 < roots.Count ? roots[0] : null;
 
            if (current == null)
                return new ReadOnlyCollection<GPALAutomationElement>(Array.Empty<GPALAutomationElement>());
 
            string[] tokens = selectorPath.SelectorPath.Split('/');
 
            // the last step of a path is the element being asked for, not a step on the way to it
            int lastToken = Array.FindLastIndex(tokens, t => false == string.IsNullOrWhiteSpace(t));
 
            WalkFailure failure = new WalkFailure();
            List<AutomationElement> found = new List<AutomationElement>();
 
            foreach (AutomationElement root in roots)
            {
                found = WalkFrom(application, root, tokens, 0, lastToken, true, failure);
 
                if (0 < found.Count)
                    break;
            }
 
            // and a menu that would not open is a path that cannot be walked however the tree is searched, so
            // the window's own menu is asked instead. That is the only way in on a desktop that is not
            // receiving input, where opening a menu at all is the mouse or the keyboard
            GPALAutomationElement throughTheMenu = 0 == found.Count ? MenuItemFor(application, selectorPath.SelectorPath) : null;
 
            if (0 == found.Count && null == throughTheMenu)
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Unable to locate element for Selector [{selectorPath.SelectorPath}], stopped at token [{failure.Token}]. {WhatIsThere(failure.Under ?? current)}", application, GPALObjectType.Application);
 
            List<GPALAutomationElement> retVal = new List<GPALAutomationElement>();
 
            foreach (AutomationElement each in found)
                retVal.Add(new GPALAutomationElement(each));
 
            if (null != throughTheMenu)
                retVal.Add(throughTheMenu);
 
            return new ReadOnlyCollection<GPALAutomationElement>(retVal);
        }

        private class WalkFailure
        {
            internal int Depth = -1;
            internal string Token;
            internal AutomationElement Under;
        }

        private static List<AutomationElement> WalkFrom(Application application, AutomationElement current, string[] tokens, int index, int lastToken, bool firstToken, WalkFailure failure)
        {
            List<AutomationElement> result = new List<AutomationElement>();
 
            // an empty step from a leading slash, and Desktop, which names no control type and carries no
            // attributes, are steps with nothing to match. They are passed over rather than failed on
            int next = index;
            bool descendant = false;
            Condition condition = null;
 
            while (next < tokens.Length && null == (condition = ConditionFor(tokens[next])))
            {
                // an empty step that follows another step is the second slash of a descendant axis, as in
                // Window//Button, which asks for the next step anywhere below instead of directly under. The
                // empty step a leading slash makes is the first of all, with nothing above it to descend from
                if (true == string.IsNullOrWhiteSpace(tokens[next]) && 0 < next)
                    descendant = true;
 
                next++;
            }
 
            if (next >= tokens.Length)
                result.Add(current);
            else
            {
                List<AutomationElement> candidates = new List<AutomationElement>();
 
                // RootAutomationElement is the window itself, which is what the recorder roots its path at, so
                // the first step is matched against the current element before its children
                if (true == firstToken)
                {
                    AutomationElement self = current.FindFirst(TreeScope.Element, condition);
 
                    if (null != self)
                        candidates.Add(self);
                }
 
                if (0 == candidates.Count)
                    foreach (AutomationElement child in current.FindAll(true == descendant ? TreeScope.Descendants : TreeScope.Children, condition))
                        candidates.Add(child);
 
                // and when that finds nothing, the same step against the raw view. FindAll searches the
                // control view, which holds what a framework chose to publish, and WinForms leaves whole
                // controls out of it: a spinner on a tab page is missing from the control view with only its
                // two buttons hoisted up in its place, while the raw view has the spinner, the edit box inside
                // it and the tab page they sit on. Looking here second rather than first means every path that
                // already worked still walks the tree it always walked
                if (0 == candidates.Count)
                    candidates = RawMatches(current, condition, descendant);
 
                // a step written as Pane[2] names the second match and no others, so the rest stop being
                // candidates. A position past the end of the matches names nothing, which fails the step
                int position = IndexFor(tokens[next]);
 
                if (0 < position)
                    candidates = position <= candidates.Count
                        ? new List<AutomationElement> { candidates[position - 1] }
                        : new List<AutomationElement>();
 
                if (0 == candidates.Count && next > failure.Depth)
                {
                    failure.Depth = next;
                    failure.Token = tokens[next];
                    failure.Under = current;
                }
 
                foreach (AutomationElement candidate in candidates)
                {
                    // Expand a menu item to expose what is inside it, but never the one the path is asking for:
                    // expanding falls through to InvokePattern, which activates it, and finding an element is
                    // not supposed to click it
                    if (candidate.Current.ControlType == ControlType.MenuItem && next < lastToken)
                        ExpandMenuItem(application, candidate);
 
                    List<AutomationElement> reached = WalkFrom(application, candidate, tokens, next + 1, lastToken, false, failure);
 
                    if (0 == reached.Count)
                        continue;
 
                    result.AddRange(reached);
 
                    // a step before the last one is a way through, and one way through is all that is wanted.
                    // The last step is the element being asked for, so every match of it is kept
                    if (next < lastToken)
                        break;
                }
            }
 
            return result;
        }

        internal static List<AutomationElement> RawMatches(AutomationElement current, Condition condition, bool descendant)
        {
            List<AutomationElement> retVal = new List<AutomationElement>();
            TreeWalker walker = TreeWalker.RawViewWalker;
 
            for (AutomationElement child = walker.GetFirstChild(current); null != child; child = walker.GetNextSibling(child))
            {
                if (true == Matches(child, condition))
                    retVal.Add(child);
 
                if (true == descendant)
                    retVal.AddRange(RawMatches(child, condition, true));
            }
 
            return retVal;
        }

        private static bool Matches(AutomationElement element, Condition condition)
        {
            bool retVal = false;
 
            if (ReferenceEquals(Condition.TrueCondition, condition))
                retVal = true;
            else if (condition is PropertyCondition property)
            {
                object wanted = property.Value;
                object actual = element.GetCurrentPropertyValue(property.Property);
 
                // a control type condition holds the type's number rather than the type, while the element
                // answers with the type itself, so those two are compared on the number they both carry
                retVal = actual is ControlType controlType && wanted is int wantedType
                    ? controlType.Id == wantedType
                    : Equals(actual, wanted);
            }
            else if (condition is AndCondition all)
            {
                retVal = true;
 
                foreach (Condition each in all.GetConditions())
                    if (false == Matches(element, each))
                    {
                        retVal = false;
                        break;
                    }
            }
            else if (condition is OrCondition any)
            {
                foreach (Condition each in any.GetConditions())
                    if (true == Matches(element, each))
                    {
                        retVal = true;
                        break;
                    }
            }
            else if (condition is NotCondition other)
                retVal = false == Matches(element, other.Condition);
 
            return retVal;
        }

        private static int IndexFor(string token)
        {
            int result = 0;
            int start = (token ?? string.Empty).IndexOf('[');
 
            // a step can carry a position and attributes together, as in Pane[2][@ClassName='x'], so the
            // brackets are read in turn until one of them holds a number
            while (0 < start && 0 == result)
            {
                int end = token.IndexOf(']', start);
 
                if (0 > end)
                    break;
 
                if (true == int.TryParse(token.Substring(start + 1, end - start - 1), out int position) && 0 < position)
                    result = position;
 
                start = token.IndexOf('[', end);
            }
 
            return result;
        }

        private static Condition ConditionFor(string token)
        {
            Condition result = null;
 
            if (false == string.IsNullOrWhiteSpace(token))
            {
                string controlTypeStr = token;
                var conditions = new List<Condition>();
 
                int attrStart = token.IndexOf('[');
                if (attrStart > 0)
                {
                    controlTypeStr = token.Substring(0, attrStart);
 
                    // Parse attributes [@Name='xxx'][@ClassName='yyy'] etc.
                    string remaining = token.Substring(attrStart);
                    while (remaining.Contains("[@"))
                    {
                        int start = remaining.IndexOf("[@");
                        int end = remaining.IndexOf(']', start);
                        if (end == -1) break;
 
                        string attrSegment = remaining.Substring(start + 2, end - start - 2);
 
                        var parts = attrSegment.Split(new char[] { '=' }, 2);
 
                        if (parts.Length == 2)
                        {
                            string attrName = parts[0].Trim();
                            string value = parts[1].Trim().Trim('\'', '"');
 
                            Condition cond = attrName.ToLowerInvariant() switch
                            {
                                "name" => new PropertyCondition(AutomationElement.NameProperty, value),
                                "automationid" => new PropertyCondition(AutomationElement.AutomationIdProperty, value),
                                "classname" => new PropertyCondition(AutomationElement.ClassNameProperty, value),
                                _ => null
                            };
 
                            if (cond != null)
                                conditions.Add(cond);
                        }
 
                        remaining = remaining.Substring(end + 1);
                    }
                }
 
                // Add ControlType condition
                ControlType controlType = GetControlType(controlTypeStr);
                if (controlType != null)
                    conditions.Add(new PropertyCondition(AutomationElement.ControlTypeProperty, controlType));
 
                if (1 == conditions.Count)
                    result = conditions[0];
                else if (1 < conditions.Count)
                    result = new AndCondition(conditions.ToArray());
            }
 
            return result;
        }

        private static ControlType GetControlType(string programmaticName)
        {
            if (string.IsNullOrWhiteSpace(programmaticName))
                return null;
 
            // Force initialization of the static constructor (important quirk!)
            _ = ControlType.Button.ToString();
 
            // Try direct lookup by name via the known AutomationControlType enum + offset
            if (Enum.TryParse<System.Windows.Automation.Peers.AutomationControlType>(programmaticName, true, out var enumValue))
            {
                int id = (int)enumValue + 50000;   // Official offset for ControlType IDs
                return ControlType.LookupById(id);
            }
 
            // Fallback: try common aliases
            return programmaticName.ToLowerInvariant() switch
            {
                "window" => ControlType.Window,
                "pane" => ControlType.Pane,
                "button" => ControlType.Button,
                "edit" => ControlType.Edit,        // TextBox
                "textbox" => ControlType.Edit,
                "combobox" => ControlType.ComboBox,
                "list" => ControlType.List,
                "listitem" => ControlType.ListItem,
                "menuitem" => ControlType.MenuItem,
                "tab" => ControlType.Tab,
                "tabitem" => ControlType.TabItem,
                "tree" => ControlType.Tree,
                "treeitem" => ControlType.TreeItem,
                "document" => ControlType.Document,
                "group" => ControlType.Group,
                "checkbox" => ControlType.CheckBox,
                "radiobutton" => ControlType.RadioButton,
                "hyperlink" => ControlType.Hyperlink,
                "image" => ControlType.Image,
                "spinner" => ControlType.Spinner,
                "slider" => ControlType.Slider,
                "statusbar" => ControlType.StatusBar,
                "toolbar" => ControlType.ToolBar,
                "datagrid" => ControlType.DataGrid,
                "dataitem" => ControlType.DataItem,
                _ => null
            };
        }
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsByAutomationID(Application application, SelectorPathEntry selectorPath, bool searchForElement = true, AutomationElement element = null)
        {
            return FindElementsBy(application, selectorPath, ConditionPropertyType.AutomationID, searchForElement, element);
        }
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsByText(Application application, SelectorPathEntry selectorPath, bool searchForElement = true, AutomationElement element = null)
        {
            return FindElementsBy(application, selectorPath, ConditionPropertyType.Value, searchForElement, element);   // TODO: CAVEAT: KLUDGE: is this correct? Value?
        }
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsByValue(Application application, SelectorPathEntry selectorPath, bool searchForElement = true, AutomationElement element = null)
        {
            return FindElementsBy(application, selectorPath, ConditionPropertyType.Value, searchForElement, element);
        }
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsByName(Application application, SelectorPathEntry selectorPath, bool searchForElement = true, AutomationElement element = null)
        {
            return FindElementsBy(application, selectorPath, ConditionPropertyType.Name, searchForElement, element);
        }
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsByClassName(Application application, SelectorPathEntry selectorPath, bool searchForElement = true, AutomationElement element = null)
        {
            return FindElementsBy(application, selectorPath, ConditionPropertyType.ClassName, searchForElement, element);
        }
        private static ReadOnlyCollection<GPALAutomationElement> FindElementsBy(Application application, SelectorPathEntry selectorPath, ConditionPropertyType conditionPropertyType, bool searchForElement = true, AutomationElement element = null)
        {
            ReadOnlyCollection<GPALAutomationElement> elem;
 
            if (null != element)
                elem = element.FindElements(selectorPath.SelectorPath, conditionPropertyType);
            else
                elem = application.RootAutomationElement.FindElements(selectorPath.SelectorPath, conditionPropertyType);
 
            return elem;
        }
        internal static void PublishToEventHandler(Application application, Selector selector, List<GPALAutomationElement> elements)
        {
            string msg = "";
            if (true == selector.DeleteMe)
                msg = $"Ignoring selector {selector.Name} marked for deletion. Continuing.";
            else if (null == elements || 0 == elements.Count)
                msg = $"No elements found for selector {selector.Name}. Continuing.";
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, msg, application, GPALObjectType.Application);
        }
        public static List<GPALAutomationElement> WaitFor(Application application, int timeoutInTicks, out bool matchedAll)
        {
            bool matchedAll2 = true;
            List<GPALAutomationElement> foundElements = null;
            List<GPALAutomationElement> tmpFoundElements = null;
            List<GPALAutomationElement> matchedElements;
 
            foundElements = new List<GPALAutomationElement>();
 
            foreach (Selector selector in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != selector.SelectorType)
                    continue;   // we don't look up data literals
 
                tmpFoundElements = WaitFor(application, selector, timeoutInTicks, out matchedAll2, out matchedElements);
 
                tmpFoundElements = matchedElements;
 
                // FindElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
                if (true == selector.DeleteMe || null == tmpFoundElements)
                {
                    PublishToEventHandler(application, selector, tmpFoundElements);
                    continue;
                }
 
                int rowCount = 0;
 
                foreach (GPALAutomationElement element in tmpFoundElements)
                {
                    foundElements.Add(element);
                    if (++rowCount >= application.CurrentUOW.WithAllThatMatch)
                        break;
                }
            }
 
            matchedAll = matchedAll2;
            return foundElements;
        }

        private static List<GPALAutomationElement> WaitFor(Application application, Selector selector, int timeoutInTicks, out bool matchedAll, out List<GPALAutomationElement> matchedElements, AutomationElement element = null)
        {
            bool matchedAll2 = true, matchedAll3 = true;
            List<GPALAutomationElement> matchedElements2;
            ReadOnlyCollection<GPALAutomationElement> elements = FindElements(application, application.CurrentUOW, selector, out matchedAll2, out matchedElements2, element);
 
            if (WaitTime.Forever == timeoutInTicks)
                timeoutInTicks = Int32.MaxValue;    // forever
 
            // keep looking while we have time left and nothing has turned up. FindElements returns an empty
            // collection (never null) when it finds nothing, so this has to test the count, not the reference.
            for (; 0 < timeoutInTicks && 0 == (elements?.Count ?? 0); timeoutInTicks -= 1000)
            {
                elements = FindElements(application, application.CurrentUOW, selector, out matchedAll3, out matchedElements2, element);
                if (0 == elements.Count)
                    Thread.Sleep(Math.Min(1000, timeoutInTicks));
                else
                    break;
            }
 
            matchedAll = (matchedAll2 | matchedAll3);   // one of those found it or not :)
            matchedElements = matchedElements2;         // always use matched, it's either all or those matched
 
            return elements?.ToList<GPALAutomationElement>();
        }

        public static void FillInFrom(Application application, string useText, WriteMode writeMode)
        {
            List<GPALAutomationElement> matchedElements;
 
            foreach (Selector selector in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != selector.SelectorType)
                    continue;   // we don't look up data literals
 
                ReadOnlyCollection<GPALAutomationElement> elems = ElementHelper.FindElements(application, application.CurrentUOW, selector, out bool matchedAll, out matchedElements);
 
                // FindElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
                if (true == selector.DeleteMe || null == elems)
                {
                    PublishToEventHandler(application, selector, elems?.ToList());
                    continue;
                }
 
                if (null != matchedElements)
                    elems = new ReadOnlyCollection<GPALAutomationElement>(matchedElements);
 
                int rowCount = 0;
 
                foreach (GPALAutomationElement elem in elems)
                {
                    if (null != elem.Ae)
                    {
                        bool isPassword = elem.Ae.Current.IsPassword;
 
                        // an empty string is a workflow clearing the field and a one or two character string has
                        // no middle to leave out, so those are said in full. Indexing unconditionally threw
                        // IndexOutOfRange on the empty string, which made clearing a field a crash
                        string abbreviatedText = 2 < (useText?.Length ?? 0)
                            ? useText[0] + "..." + useText[useText.Length - 1]
                            : useText;
 
                        if (false == isPassword)
                        {
                            if (GPALEventType.DEBUG == (GPALEventType.DEBUG & GPAL.GPALSettings.DebugEvents) || GPALEventType.DEBUG == (GPALEventType.DEBUG & GPAL.GPALSettings.ConsoleEvents))
                                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{writeMode}] text [{useText}] in element [{elem.Ae.Current.LocalizedControlType}][{selector.Name}]", application, GPALObjectType.Application);
                            else
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{writeMode}] text [{abbreviatedText}] in element [{elem.Ae.Current.LocalizedControlType}][{selector.Name}]", application, GPALObjectType.Application);
                        }
                        else
                            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{writeMode}] XXXpasswordXXX in element [{elem.Ae.Current.LocalizedControlType}][{selector.Name}]", application, GPALObjectType.Application);
 
                        FillInFrom(application, elem.Ae, new UnitOfWork.ElementNode() { OffsetX = selector.OffsetX, OffsetY = selector.OffsetY }, useText, writeMode);
                        if (++rowCount >= HowManyToActOn(application))
                            break;
                    }
                }
 
                UnitOfWork safeUOW = application.CurrentUOW;
                CallIfStatus handled = 0;
 
                // CAVEAT: there is no concept of 'handled (1)' vs 'not handled (0)' but definitely can request to exit
                IGPALGrid<string> tokens = GPAL.Grid.ToGPALObject();
                tokens.AddRow(new List<string> { useText });
                if (null != application.CurrentUOW.AppCallAfterFillIn)
                    handled = application.CurrentUOW.AppCallAfterFillIn(application, tokens, 1);
 
                // CAVEAT: kludge - the call after handler can and probably will set a new current unit of work, but we need our old current unit
                // the call after handler UOW is no longer in scope, so restore our UOW
                application.CurrentUOW = safeUOW;
 
                if (CallIfStatus.Terminate == handled)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"CallAfterFillIn handler [{application.CurrentUOW.AppCallAfterFillIn.GetInvocationList()[0].Method.Name}] requested program termination.", application, GPALObjectType.Application);
                    throw new GPALException($"CallAfterFillIn handler [{application.CurrentUOW.AppCallAfterFillIn.GetInvocationList()[0].Method.Name}] requested program termination.");
                }
            }
        }
        internal static void Scroll(Application application, object amount, ScrollTypes incrementsPercent, ScrollTypes horzVert)
        {
            List<GPALAutomationElement> matchedElements;
 
            foreach (Selector selector in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != selector.SelectorType)
                    continue;   // we don't look up data literals
 
                ReadOnlyCollection<GPALAutomationElement> elems = ElementHelper.FindElements(application, application.CurrentUOW, selector, out bool matchedAll, out matchedElements);
 
                // FindElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
                if (true == selector.DeleteMe || null == elems)
                {
                    PublishToEventHandler(application, selector, elems?.ToList());
                    continue;
                }
 
                if (null != matchedElements)
                    elems = new ReadOnlyCollection<GPALAutomationElement>(matchedElements);
 
                if (0 == elems.Count)
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selector [{selector.Name}] matched no elements, so there was nothing to scroll.", application, GPALObjectType.Application);
 
                int rowCount = 0;
 
                foreach (GPALAutomationElement elem in elems)
                {
                    if (null != elem.Ae)
                    {
                        Scroll(application, elem.Ae, selector.InteractionType, amount, incrementsPercent, horzVert);
                        if (++rowCount >= HowManyToActOn(application))
                            break;
                    }
                }
            }
        }
        private static void SetRangeValue(Application application, RangeValuePattern rangeValue, AutomationElement ae, object amount, ScrollTypes incrementsPercent)
        {
            var now = rangeValue.Current;
 
            if (true == now.IsReadOnly)
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{ae.Current.ControlType.LocalizedControlType}] reports its range as read only, so it cannot be scrolled.", application, GPALObjectType.Application);
            else
            {
                double span = now.Maximum - now.Minimum;
                double wanted;
 
                if (ScrollTypes.Percent == incrementsPercent)
                    wanted = now.Minimum + (((double)(int)amount / 100.0) * span);
                else
                {
                    // the enum names the size and the direction; the control names how far each size goes. A
                    // control that does not say is documented to answer NaN, so a hundredth and a tenth of its
                    // own range stand in, which is what those steps mean everywhere else
                    ScrollAmount step = (ScrollAmount)amount;
 
                    bool large = ScrollAmount.LargeIncrement == step || ScrollAmount.LargeDecrement == step;
                    bool back = ScrollAmount.SmallDecrement == step || ScrollAmount.LargeDecrement == step;
 
                    double by = large ? now.LargeChange : now.SmallChange;
 
                    if (double.IsNaN(by) || 0 == by)
                        by = span * (large ? 0.1 : 0.01);
 
                    wanted = now.Value + (back ? -by : by);
                }
 
                // a value off the end of its own range is refused by the control, and the endpoint is what was
                // being asked for anyway
                wanted = wanted < now.Minimum ? now.Minimum : (wanted > now.Maximum ? now.Maximum : wanted);
 
                try
                {
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"RangeValuePattern on [{ae.Current.ControlType.LocalizedControlType}]: range [{now.Minimum}] to [{now.Maximum}], value [{now.Value}], setting [{wanted}].", application, GPALObjectType.Application);
 
                    rangeValue.SetValue(wanted);
 
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"RangeValuePattern on [{ae.Current.ControlType.LocalizedControlType}]: value is now [{rangeValue.Current.Value}].", application, GPALObjectType.Application);
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to set a value of [{wanted}] on control [{ae.Current.ControlType.LocalizedControlType}].", application, GPALObjectType.Application, ex);
                }
            }
        }

        private static void ScrollToPercent(Application application, AutomationElement ae, ScrollPattern scrollPattern, int percent, ScrollTypes horzVert)
        {
            bool vertical = ScrollTypes.Vertical == horzVert;
            var info = scrollPattern.Current;
 
            if (false == (vertical ? info.VerticallyScrollable : info.HorizontallyScrollable))
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{ae.Current.ControlType.LocalizedControlType}] is not scrollable {(vertical ? "vertically" : "horizontally")}.", application, GPALObjectType.Application);
            else
            {
                double page = vertical ? info.VerticalViewSize : info.HorizontalViewSize;
                double at = vertical ? info.VerticalScrollPercent : info.HorizontalScrollPercent;
                double was = -1;
                int steps = 0;
 
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"ScrollPattern on [{ae.Current.ControlType.LocalizedControlType}]: at [{at}], one view is [{page}] percent, walking to [{percent}].", application, GPALObjectType.Application);
 
                // whole views first
                while (page <= Math.Abs(percent - at) && at != was && steps < STEP_LIMIT)
                {
                    was = at;
                    at = OneStep(scrollPattern, vertical, percent > at, true);
                    steps++;
                }
 
                double off = Math.Abs(percent - at);
                was = -1;
 
                // then lines, stopping the moment a step no longer gets closer
                while (0 < off && at != was && steps < STEP_LIMIT)
                {
                    was = at;
                    at = OneStep(scrollPattern, vertical, percent > at, false);
                    steps++;
 
                    double closer = Math.Abs(percent - at);
 
                    if (closer >= off)
                        break;
 
                    off = closer;
                }
 
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"ScrollPattern on [{ae.Current.ControlType.LocalizedControlType}]: landed at [{at}] after [{steps}] step(s).", application, GPALObjectType.Application);
            }
        }

        private static double OneStep(ScrollPattern scrollPattern, bool vertical, bool forward, bool large)
        {
            ScrollAmount step = large
                ? (forward ? ScrollAmount.LargeIncrement : ScrollAmount.LargeDecrement)
                : (forward ? ScrollAmount.SmallIncrement : ScrollAmount.SmallDecrement);
 
            scrollPattern.Scroll(vertical ? ScrollAmount.NoAmount : step, vertical ? step : ScrollAmount.NoAmount);
 
            return vertical ? scrollPattern.Current.VerticalScrollPercent : scrollPattern.Current.HorizontalScrollPercent;
        }

        private const int STEP_LIMIT = 500;

        internal static bool HardwareReaches(Application application)
        {
            bool retVal = true;
 
            if (true == application.applicationSettings.HiddenDesktop)
            {
                // once per application. A workflow driving a hidden desktop turns hardware down on nearly every
                // action, and saying so each time buries the log
                if (false == application.applicationSettings.HiddenDesktopHardwareReported)
                {
                    application.applicationSettings.HiddenDesktopHardwareReported = true;
 
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Hardware input cannot reach [{application.applicationSettings.HiddenDesktopName}] while it is hidden, so UIAutomation is used instead. Peek at the desktop to use hardware there.", application, GPALObjectType.Application);
                }
 
                retVal = false;
            }
 
            return retVal;
        }
 
        internal static void Scroll(Application application, AutomationElement ae, InteractionType interactionType, object amount, ScrollTypes incrementsPercent, ScrollTypes horzVert)
        {
            //UIAutomation only
            if ((Enums.InteractionType.Hardware == interactionType || true == application.applicationSettings.UseHardware) && true == HardwareReaches(application))
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Hardware scrolling is not supported at this time. Find the scroll controls and click them instead.", application, GPALObjectType.Application);
            else
            {
                // the element is a scrollbar, so it only goes one direction horizontal or vertical
                if (true == ae.TryGetCurrentPattern(RangeValuePattern.Pattern, out object rangeValuePattern))
                {
                    SetRangeValue(application, (RangeValuePattern)rangeValuePattern, ae, amount, incrementsPercent);
                }
                else if (true == ae.TryGetCurrentPattern(ScrollPattern.Pattern, out object scrollPattern))
                {
                    if (ScrollTypes.Percent == incrementsPercent)
                        ScrollToPercent(application, ae, (ScrollPattern)scrollPattern, (int)amount, horzVert);
                    else
                    {
                        ScrollAmount hAmount = ScrollTypes.Horizontal == horzVert ? (ScrollAmount)amount : ScrollAmount.NoAmount;
                        ScrollAmount vAmount = ScrollTypes.Vertical == horzVert ? (ScrollAmount)amount : ScrollAmount.NoAmount;
 
                        try
                        {
                            var before = ((ScrollPattern)scrollPattern).Current;
 
                            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"ScrollPattern on [{ae.Current.ControlType.LocalizedControlType}]: scrollable h [{before.HorizontallyScrollable}] v [{before.VerticallyScrollable}], at h [{before.HorizontalScrollPercent}] v [{before.VerticalScrollPercent}], applying h [{hAmount}] v [{vAmount}].", application, GPALObjectType.Application);
 
                            ((ScrollPattern)scrollPattern).Scroll(hAmount, vAmount);
 
                            var after = ((ScrollPattern)scrollPattern).Current;
 
                            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"ScrollPattern on [{ae.Current.ControlType.LocalizedControlType}]: now at h [{after.HorizontalScrollPercent}] v [{after.VerticalScrollPercent}].", application, GPALObjectType.Application);
                        }
                        catch (Exception ex)
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to Scroll on control [{ae.Current.ControlType.LocalizedControlType}].", application, GPALObjectType.Application, ex); ;
                        }
                    }
                }
                else
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{ae.Current.ControlType.LocalizedControlType}] offers neither RangeValuePattern nor ScrollPattern, so UIAutomation cannot scroll it. Point the selector at its ScrollBar, which does.", application, GPALObjectType.Application);
 
            }
        }
        internal static void FillInFrom(Application application, AutomationElement ae, UnitOfWork.ElementNode elementNode, string useText, WriteMode writeMode)
        {
            bool taken = false;
 
            if ((Enums.InteractionType.Hardware == elementNode.InteractionType || true == application.applicationSettings.UseHardware) && true == HardwareReaches(application))
            {
                HardwareFillInFrom(application, new GPALAutomationElement(ae), elementNode.OffsetX, elementNode.OffsetY, useText, writeMode);
 
                taken = true;
            }
 
            if (false == taken)
            {
                // LegacyIAccessiblePattern is only accessible via the UIAComWrapperX nuget package
                /*
                if (true == ae.TryGetCurrentPattern(System.Windows.Automation.LegacyIAccessiblePattern.Pattern, out object legacyPattern))
                {
                    // Set focus for input functionality and begin.
                    try
                    {
                        ae.SetFocus();
                    }
                    catch (Exception ex)
                    {
                        GPAL.GPALEventArgs args = new GPAL.GPALEventArgs()
                        {
                            GPALObject = application,
                            GPALObjectType = GPALObjectType.Application,
                            CurrentUOW = application.CurrentUOW,
                            AutomationElement = ae,
                            Message = $"Unable to set focus on Control {ae.Current.ControlType.LocalizedControlType}.",
                            ExceptionRaised = ex,
                            ScreenShot = ImageHelper.CaptureScreen(),
                            DateTimeStamp = DateTime.Now
                        };
 
                        GPAL.ExceptionHandler?.Invoke(application, args);
                    }
                    if (WriteMode.Overwrite == writeMode)
                        ((LegacyIAccessiblePattern)legacyPattern).SetValue(useText);
                    else if (WriteMode.Append == writeMode)
                    {
                        string content = ((LegacyIAccessiblePattern)legacyPattern).Current.Value;
                        content += useText;
                        ((LegacyIAccessiblePattern)legacyPattern).SetValue(content);
                    }
                    else if (WriteMode.Insert == writeMode)
                    {
                        string content = ((LegacyIAccessiblePattern)legacyPattern).Current.Value;
                        useText += content;
                        ((LegacyIAccessiblePattern)legacyPattern).SetValue(content);
                    }
                }
                else
                */
                // a control offering a value does not have to let it be written. A read only value answers the
                // offer the same way and throws when it is taken up, which a multi line edit does, so the write
                // is attempted rather than trusted and what follows is the next way in
                if (true == ae.TryGetCurrentPattern(ValuePattern.Pattern, out object valuePattern))
                {
                    // Set focus for input functionality and begin.
                    try
                    {
                        ae.SetFocus();
                    }
                    catch (Exception)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Unable to set focus on control [{ae.Current.ControlType.LocalizedControlType}], so the value is written to it without the focus.", application, GPALObjectType.Application);
                    }
 
                    try
                    {
                        string content = WriteMode.Overwrite == writeMode ? string.Empty : ((ValuePattern)valuePattern).Current.Value;
 
                        ((ValuePattern)valuePattern).SetValue(WriteMode.Append == writeMode ? content + useText
                                                            : WriteMode.Insert == writeMode ? useText + content
                                                            : useText);
 
                        taken = true;
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"ValuePattern on [{ae.Current.ControlType.ProgrammaticName}] would not take the text, so the next way of writing it is tried.", application, GPALObjectType.Application, ex);
                    }
                }
 
                // and a control with no value to write, or one that would not take what it was given, is still
                // a window, and a window holds its text whether or not it publishes one
                if (false == taken)
                    taken = SetNativeText(application, ae, useText, writeMode);
 
                if (false == taken)
                {
                    // typing is recovery here rather than what the workflow asked for, so it is what
                    // NoFallbackRecoveryActions turns off
                    if (false == GPAL.NoFallbackRecoveryActions && true == HardwareReaches(application))
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{ae.Current.ControlType.ProgrammaticName}] offers no way to write text into it, so it is typed into instead.", application, GPALObjectType.Application);
 
                        HardwareFillInFrom(application, new GPALAutomationElement(ae), elementNode.OffsetX, elementNode.OffsetY, useText, writeMode);
                    }
                    else
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{ae.Current.ControlType.ProgrammaticName}] offers no way to write text into it, so it was left as it was.", application, GPALObjectType.Application);
                }
            }
        }
        public static void HardwareFillInFrom(Application application, GPALAutomationElement element, int offsetX, int offsetY, string text, WriteMode writeMode)
        {
            ApplicationHelper.TopProcess(application.Process);
 
            HardwareFocus(application, element, offsetX, offsetY);
 
            if (WriteMode.Overwrite == writeMode)
            {
                // clear the input, send CTRL-a (select all), then delete *altho we can prolly just typeover*
                HardwareHelper.SendChar('a', ModifierKeys.Control, true);
                Thread.Sleep(150);
                //                HardwareHelper.SendKey(GPAL.VK_DELETE);
            }
            else if (WriteMode.Append == writeMode)   // CTRL-End goto end of document
            {
                HardwareHelper.SendKey(GPAL.VK_END, ModifierKeys.Control);
                Thread.Sleep(150);
            }
            else if (WriteMode.Insert == writeMode)   // CTRL-End goto start of document
            {
                HardwareHelper.SendKey(GPAL.VK_HOME, ModifierKeys.Control);
                Thread.Sleep(150);
            }
 
            HardwareHelper.SendString(text, GPAL.TypingDelay);
 
            SettleInput(application);
        }

        internal static void SettleInput(Application application)
        {
            try
            {
                application?.Process?.WaitForInputIdle(INPUT_SETTLE_MS);
            }
            catch (Exception)
            {
                // a process with no message loop has no input queue to empty, and one that has gone has no
                // input at all. Asking is how that is found out, and neither is a failure
            }
        }
 
        // how long to wait for an application to take what was sent it before carrying on regardless. Reached
        // only by a process that never reports itself idle, never by one that simply took the keystrokes
        private const int INPUT_SETTLE_MS = 2_000;

        public static void Focus(Application application)
        {
            List<GPALAutomationElement> matchedElements;
 
            foreach (Selector selector in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != selector.SelectorType)
                    continue;   // we don't look up data literals
 
                ReadOnlyCollection<GPALAutomationElement> elems = ElementHelper.FindElements(application, application.CurrentUOW, selector, out bool matchedAll, out matchedElements);
 
                // FindElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
                if (true == selector.DeleteMe || null == elems)
                {
                    PublishToEventHandler(application, selector, elems?.ToList());
                    continue;
                }
 
                if (null != matchedElements)
                    elems = new ReadOnlyCollection<GPALAutomationElement>(matchedElements);
 
                int rowCount = 0;
 
                foreach (GPALAutomationElement elem in elems)
                {
                    if (null != elem.Ae)
                    {
                        if ((Enums.InteractionType.Hardware == selector.InteractionType || true == application.applicationSettings.UseHardware) && true == HardwareReaches(application))
                        {
                            int offsetX, offsetY;
                            offsetX = selector.SelectorSettings.OffsetX;
                            offsetY = selector.SelectorSettings.OffsetY;
 
                            HardwareFocus(application, elem, offsetX, offsetY);   // hardwre focus will try to click in text inputs
                        }
                        else
                        {
 
                            try
                            {
                                if (null != elem.Ae)
                                    elem.Ae.SetFocus();
                                else
                                    HardwareFocus(application, elem, 0, 0);
                            }
                            catch (Exception ex)
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Focus threw on selector [{selector.Name}], so the focus did not move.", application, GPALObjectType.Application, ex);
                            }
                        }
 
                        if (++rowCount >= HowManyToActOn(application))
                            break;
                    }
                }
            }
        }
        public static void Hover(Application application, UnitOfWork currentUOW)
        {
            List<GPALAutomationElement> matchedElements;
 
            foreach (Selector selector in currentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != selector.SelectorType)
                    continue;   // we don't look up data literals
 
                ReadOnlyCollection<GPALAutomationElement> elems = ElementHelper.FindElements(application, currentUOW, selector, out bool matchedAll, out matchedElements);
 
                // FindElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
                if (true == selector.DeleteMe || null == elems)
                {
                    PublishToEventHandler(application, selector, elems?.ToList());
                    continue;
                }
 
                if (null != matchedElements)
                    elems = new ReadOnlyCollection<GPALAutomationElement>(matchedElements);
 
                int rowCount = 0;
 
                foreach (GPALAutomationElement elem in elems)
                {
                    if (null != elem.Ae)
                    {
                        if (0 == elem.Ae.Current.BoundingRectangle.Width && 0 == elem.Ae.Current.BoundingRectangle.Height)
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Element has Zero Width and Height, skipping", application, GPALObjectType.Application);
                            continue;
                        }
 
                        // both ways of moving to an element are the mouse, because a hover is a mouse position and
                        // there is no pattern for one. So a hidden desktop cannot be hovered over at all, and
                        // saying so beats moving the real pointer across the user's screen
                        if (false == HardwareReaches(application))
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selector [{selector.Name}] cannot be moved to on a hidden desktop, since a hover is a mouse position and UIAutomation has no equivalent. Peek at the desktop to hover, or click, which does work there.", application, GPALObjectType.Application);
                        }
                        else if (Enums.InteractionType.Hardware == selector.InteractionType || true == application.applicationSettings.UseHardware)
                        {
                            // same as hardware moveto
                            HardwareMoveTo(application, elem.ImageMatch ?? elem, selector.OffsetX, selector.OffsetY);   // an image match moves to where the picture is
                        }
                        else if (0 != elem.Ae.Current.BoundingRectangle.Width && 0 != elem.Ae.Current.BoundingRectangle.Height)
                        {
                            try
                            {
                                elem.Ae.SetFocus();
                                HardwareMoveTo(application, elem.ImageMatch ?? elem, selector.OffsetX, selector.OffsetY);   // an image match moves to where the picture is
                            }
                            catch (Exception)
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Focusing [{selector.Name}] threw, so it is moved to without the focus.", application, GPALObjectType.Application);
                                HardwareMoveTo(application, elem.ImageMatch ?? elem, selector.OffsetX, selector.OffsetY);   // an image match moves to where the picture is
                            }
                        }
 
                        if (++rowCount >= HowManyToActOn(application))
                            break;
                    }
                }
            }
        }
        public static void MoveTo(Application application)
        {
            List<GPALAutomationElement> matchedElements;
 
            foreach (Selector selector in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != selector.SelectorType)
                    continue;   // we don't look up data literals
 
                ReadOnlyCollection<GPALAutomationElement> elems = ElementHelper.FindElements(application, application.CurrentUOW, selector, out bool matchedAll, out matchedElements);
 
                // FindElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
                if (true == selector.DeleteMe || null == elems)
                {
                    PublishToEventHandler(application, selector, elems?.ToList());
                    continue;
                }
 
                if (null != matchedElements)
                    elems = new ReadOnlyCollection<GPALAutomationElement>(matchedElements);
 
                int rowCount = 0;
 
                // moving to an element is the mouse and nothing else, since UIAutomation has no way to put a
                // pointer anywhere, so this is one of the two things a hidden desktop cannot be asked for
                if (false == HardwareReaches(application))
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selector [{selector.Name}] cannot be moved to on a hidden desktop, since moving the pointer is the mouse and UIAutomation has no equivalent. Peek at the desktop to move there, or click, which does work.", application, GPALObjectType.Application);
                else
                    foreach (GPALAutomationElement elem in elems)
                    {
                        if (null != elem.Gae || null != elem.Ae)
                            HardwareMoveTo(application, elem.ImageMatch ?? elem, selector.OffsetX, selector.OffsetY);   // an image match moves to where the picture is
 
                        if (++rowCount >= HowManyToActOn(application))
                            break;
                    }
            }
        }

        internal static string WhatIsThere(AutomationElement parent)
        {
            var names = new List<string>();
            string retVal;
 
            try
            {
                AutomationElement at = parent ?? AutomationElement.RootElement;
 
                foreach (AutomationElement child in at.FindAll(TreeScope.Children, System.Windows.Automation.Condition.TrueCondition))
                {
                    string controlType = child.Current.ControlType.ProgrammaticName.Replace("ControlType.", "");
                    string automationId = child.Current.AutomationId;
 
                    string className = child.Current.ClassName;
 
                    names.Add(string.IsNullOrEmpty(automationId)
                        ? $"{controlType}[@ClassName='{className}'] '{child.Current.Name}'"
                        : $"{controlType}[@AutomationId='{automationId}'][@ClassName='{className}'] '{child.Current.Name}'");
                }
 
                retVal = 0 == names.Count
                    ? "It has no children."
                    : $"What is there: {string.Join(" | ", names)}";
            }
            catch (Exception ex)
            {
                retVal = $"Its children could not be read: {ex.Message}";
            }
 
            return retVal;
        }

        private static void MoveTo(AutomationElement element, int offsetX, int offsetY, string appName)
        {
            try
            {
                HardwareMoveTo(null, new GPALAutomationElement(element), offsetX, offsetY);
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"(ElementHelper): unable to move the mouse to the element in Application [{appName}].", element, GPALObjectType.Other, ex);
            }
        }
        private static void HardwareFocus(Application application, GPALAutomationElement element, int OffsetX, int OffsetY)
        {
            if (null != element.Ae && ControlType.Text == element.Ae.Current.ControlType)
                HardwareClick(application, element, ClickType.LeftClick, ModifierKeys.NONE, OffsetX, OffsetY);
            else
                HardwareMoveTo(application, element, OffsetX, OffsetY);
        }
        private static System.Drawing.Point HardwareMoveTo(Application application, GPALAutomationElement element, int OffsetX, int OffsetY)
        {
            Rectangle rect2 = new Rectangle(), iframeRect = new Rectangle(0, 0, 0, 0);
 
            // the size as well as the corner. Without it the rectangle is a point, the middle of it is the
            // corner, and every aim inside the element lands on its top left border
            if (null != element.Gae)
            {
                rect2.Location = element.Gae.Location;
                rect2.Size = element.Gae.Size;
            }
            else
            {
                rect2.Location = new System.Drawing.Point((int) element.Ae.Current.BoundingRectangle.Location.X, (int) element.Ae.Current.BoundingRectangle.Location.Y);
                rect2.Size = new System.Drawing.Size((int) element.Ae.Current.BoundingRectangle.Width, (int) element.Ae.Current.BoundingRectangle.Height);
 
                // TODO: does this mean anything in an application?
                //if (true == application.CurrentUOW.CurrentSelector?.SearchForSelector)
                //{
                //    // TODO: CAVEAT: is this good? we have to try to scroll the element on screen, but what happens if it can't find it?
                //    if (false == IsVisibleInViewport(element))
                //    {
                //        SendKey(GPAL.VK_HOME);   // home - go to document top, then scroll down
 
                //        while (false == IsVisibleInViewport(element) && false == IsEndOfPage(element))
                //        {
                //            SendKey(GPAL.VK_NEXT);   // pagedown
                //            Thread.Sleep(125);
                //        }
                //        rect2 = GetAbsCoordinates(application.BrowserSettings.BrowserType, element);
                //    }
                //}
            }
 
            // where inside the element to aim. A zero offset means the middle, give or take, the same as it
            // does on the browser side: the corner of a control is its border, and on a list row it is the
            // blank margin beside the icon. The jitter keeps repeated clicks off one pixel without wandering
            // out of the control
            int offsetX = OffsetX;
            int offsetY = OffsetY;
 
            if (0 == offsetX && 0 == offsetY)
            {
                int spreadX = Math.Max(0, Math.Min(2, (rect2.Width / 2) - 1));
                int spreadY = Math.Max(0, Math.Min(2, (rect2.Height / 2) - 1));
 
                offsetX = rect2.Width / 2 + Jitter.Next(-spreadX, spreadX + 1);
                offsetY = rect2.Height / 2 + Jitter.Next(-spreadY, spreadY + 1);
            }
 
            System.Drawing.Point aimedAt = new System.Drawing.Point(rect2.X + offsetX, rect2.Y + offsetY);
 
            // a control with no size gives nothing to aim inside, and its corner is as likely to be on the
            // thing beside it. UIAutomation's own answer for where a control can be clicked is the fallback,
            // and it is only a fallback: taken ahead of the rectangle it puts the pointer somewhere other than
            // where the workflow's offset asked for
            if ((0 == rect2.Width || 0 == rect2.Height)
                && null != element.Ae
                && true == element.Ae.TryGetClickablePoint(out System.Windows.Point clickablePoint))
                aimedAt = new System.Drawing.Point((int)clickablePoint.X, (int)clickablePoint.Y);
 
            if (true == GPAL.SimulateMouse || true == application.CurrentUOW?.CurrentSelector?.SimulateMouse)
                HardwareHelper.MoveMouse(aimedAt.X, aimedAt.Y, 10, 10);    // source didn't define values for rx, ry
            else
                HardwareHelper.MoveMouse(aimedAt.X, aimedAt.Y);
 
            // the point the pointer is now on, so a caller clicks where it moved rather than working the
            // location out a second time and landing somewhere else
            return aimedAt;
        }
 
        private static Random _jitter;
        private static Random Jitter => _jitter ?? (_jitter = new Random(Guid.NewGuid().GetHashCode()));
        public static void LeftClick(Application application, ModifierKeys modifierKeys = ModifierKeys.NONE)
        {
            List<GPALAutomationElement> matchedElements;
 
            foreach (Selector selector in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != selector.SelectorType)
                    continue;   // we don't look up data literals
 
                ReadOnlyCollection<GPALAutomationElement> elems = ElementHelper.FindElements(application, application.CurrentUOW, selector, out bool matchedAll, out matchedElements);
 
                // FindElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
                if (true == selector.DeleteMe || null == elems)
                {
                    PublishToEventHandler(application, selector, elems?.ToList());
                    continue;
                }
 
                if (null != matchedElements)
                    elems = new ReadOnlyCollection<GPALAutomationElement>(matchedElements);    // either All or just those that matched
 
                int rowCount = 0;
 
                foreach (GPALAutomationElement elem in elems)
                {
                    Click(application, selector, elem, ClickType.LeftClick, modifierKeys);
 
                    if (++rowCount >= HowManyToActOn(application))
                        break;
                }
            }
        }
        public static bool GenericClick(Application application, ClickType clickType)
        {
            List<GPALAutomationElement> matchedElements;
            bool matchedAll = false;
            bool clickPerformed = false;
 
            foreach (Selector sel in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != sel.SelectorType)
                    continue;   // we don't look up data literals
 
                ElementHelper.FindElements(application, application.CurrentUOW, sel, out matchedAll, out matchedElements);
 
                if (0 == (matchedElements?.Count ?? 0))
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selector [{sel.Name}] matched no elements, so there was nothing to [{clickType}].", application, GPALObjectType.Application);
 
                if (null != matchedElements)
                {
                    int rowCount = 0;
 
                    foreach (GPALAutomationElement elem in matchedElements)
                    {
                        Click(application, sel, elem, clickType, ModifierKeys.NONE);
                        clickPerformed = true;
 
                        if (++rowCount >= HowManyToActOn(application))
                            break;
                    }
                }
            }
            return clickPerformed;
        }
        public static void Click(Application application, Selector selector, GPALAutomationElement element, ClickType clickType, ModifierKeys modifierKeys)
        {
            int offsetX, offsetY;
 
            // a control found under an image is clicked where the picture is, not in its own middle, which on a
            // large control is somewhere else entirely. on a desktop a point is clicked with the mouse
            if (null != element.ImageMatch)
                element = element.ImageMatch;
            offsetX = selector.SelectorSettings.OffsetX;
            offsetY = selector.SelectorSettings.OffsetY;
 
            // an item reached through the window's own menu is clicked by sending the command it sends, which
            // is what the menu would have sent had anyone been able to open it
            if (0 != element.MenuCommand)
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Selector [{selector.Name}] is a menu item, so command [{element.MenuCommand}] was sent to the window it belongs to.", application, GPALObjectType.Application);
 
                PostMessage(element.MenuWindow, WM_COMMAND, new IntPtr(element.MenuCommand), IntPtr.Zero);
            }
            // a right click is the mouse or nothing: UIAutomation has no pattern for a context menu, so it is
            // the one click a hidden desktop cannot be given
            else if (ClickType.RightClick == clickType && false == HardwareReaches(application))
            {
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selector [{selector.Name}] cannot be right clicked on a hidden desktop, since UIAutomation offers no pattern for one. Peek at the desktop to right click there.", application, GPALObjectType.Application);
            }
            else if ((Enums.InteractionType.Hardware == selector.InteractionType || null != element.Gae || true == application.applicationSettings.UseHardware || ClickType.RightClick == clickType)
                     && true == HardwareReaches(application))
                HardwareClick(application, element, clickType, modifierKeys, offsetX, offsetY);
            else if (null != element.Ae)
            {
                UIAutomationClick(application, selector, element, clickType, modifierKeys);
            }
            // an image match is a place on the screen with no control behind it, so the mouse is its only way in,
            // and on a hidden desktop the mouse cannot get there
            else
                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selector [{selector.Name}] is an image match, which only the mouse can click, and the mouse cannot reach [{application.applicationSettings.HiddenDesktopName}]. Peek at the desktop to click there.", application, GPALObjectType.Application);
        }

        public static void UIAutomationClick(Application application, Selector selector, GPALAutomationElement element, ClickType clickType, ModifierKeys modifierKeys)
        {
            object objPattern;
            bool invoked = false;
            string control = element.Ae.Current.LocalizedControlType;
 
            if (ModifierKeys.NONE != modifierKeys)
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"A click holding [{Browser.Browser.GetModifierKeys(modifierKeys)}] has no UIAutomation equivalent, so [{control}] is clicked with the mouse.", application, GPALObjectType.Application);
            else if (ClickType.LeftClick == clickType && true == element.Ae.TryGetCurrentPattern(SelectionItemPattern.Pattern, out objPattern))
            {
                // a left click on a selectable item selects it. Invoke would be its default action, which for
                // an item is what a double click does, so a single click must not go there
                try
                {
                    ((SelectionItemPattern)objPattern).Select();
                    invoked = true;
 
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Element [{NameOf(element)}] [{clickType}] clicked through SelectionItemPattern.", application, GPALObjectType.Application);
                }
                catch (Exception)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"SelectionItemPattern threw on control [{control}], falling back to the mouse.", application, GPALObjectType.Application);
                }
            }
            else if ((ClickType.LeftClick == clickType || ClickType.LeftDoubleClick == clickType)
                     && true == element.Ae.TryGetCurrentPattern(InvokePattern.Pattern, out objPattern))
            {
                // the default action: a left click on a button, and a double click on anything that has one,
                // which is how an item is activated
                try
                {
                    ((InvokePattern)objPattern).Invoke();
                    invoked = true;
 
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Element [{NameOf(element)}] [{clickType}] clicked through InvokePattern.", application, GPALObjectType.Application);
                }
                catch (Exception)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"InvokePattern threw on control [{control}], falling back to the mouse.", application, GPALObjectType.Application);
                }
            }
            else
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Control [{control}] offers no pattern for a [{clickType}], so the mouse is how it is clicked.", application, GPALObjectType.Application);
 
            // a control with no InvokePattern, and any click that is not a plain left one, are clicked with
            // the mouse
            if (false == invoked)
                HardwareClick(application, element, clickType, modifierKeys, selector?.OffsetX ?? 5, selector?.OffsetY ?? 5); // CAVEAT: if .ForSelector is not provided, default to 5
        }

        private static string NameOf(GPALAutomationElement element)
        {
            string retVal = null == element || null == element.Ae ? "nothing" : element.Ae.Current.Name;
 
            if (true == string.IsNullOrWhiteSpace(retVal) && null != element && null != element.Ae)
                retVal = element.Ae.Current.ControlType.ProgrammaticName;
 
            return retVal;
        }

        public static void HardwareClick(Application application, GPALAutomationElement element, ClickType clickType, ModifierKeys modifierKeys, int OffsetX = 0, int OffsetY = 0)
        {
            try
            {
                if (ModifierKeys.NONE != modifierKeys)
                    HardwareHelper.PressModifierKey(modifierKeys);
 
                // the move decides where inside the element to aim and answers with the point it moved to, so
                // the click lands where the pointer already is. Working it out a second time here is what had
                // the pointer jump to the corner and then to somewhere else for one click
                System.Drawing.Point clickAt = HardwareMoveTo(application, element, OffsetX, OffsetY);
 
                // said here, between the modifier going down and coming up, because that is where the click
                // happens. Said at all because a press and a release with nothing logged between them read as
                // a modifier held around nothing
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Element [{NameOf(element)}] [{clickType}] clicked with modifiers [{Browser.Browser.GetModifierKeys(modifierKeys)}] at [{clickAt.X},{clickAt.Y}] with the mouse.", application, GPALObjectType.Application);
 
                HardwareHelper.HardwareClick(clickAt.X, clickAt.Y, clickType);
            }
            catch
            {
 
            }
            finally
            {
                if (ModifierKeys.NONE != modifierKeys)
                    HardwareHelper.ReleaseModifierKeys(modifierKeys);
            }
        }
        public static void DragAndDrop(Application application, ModifierKeys modifierKeys = ModifierKeys.NONE)
        {
            var selectors = new List<Selector>();
 
            foreach (Selector selector in application.CurrentUOW.WithSelectorList)
                if (SelectorType.Selector == selector.SelectorType)
                    selectors.Add(selector);
 
            // two selectors name the two ends of the drag: grab the first, let go over the second. The distance
            // is worked out from where the two elements actually are, so the workflow never counts pixels and
            // the drag survives a resized window, a re-sorted list or a different screen
            if (2 == selectors.Count)
                DragToTarget(application, selectors[0], selectors[1], modifierKeys);
            else
                DragByDelta(application, modifierKeys);
        }

        private static void DragToTarget(Application application, Selector source, Selector target, ModifierKeys modifierKeys)
        {
            Rectangle from = FirstRectangle(application, source);
            Rectangle to = FirstRectangle(application, target);
 
            if (true == from.IsEmpty || true == to.IsEmpty)
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"A drag needs both ends. Selector [{source.Name}] {(true == from.IsEmpty ? "matched nothing" : "matched")} and selector [{target.Name}] {(true == to.IsEmpty ? "matched nothing" : "matched")}.", application, GPALObjectType.Application);
            else
            {
                // the middle of the element unless the selector says where. A drag that starts at the corner
                // starts on whatever sits there, which on a list row is the blank margin beside the icon, and
                // pressing empty space and moving is a rubber band selection rather than a drag of the row
                int grabX = from.X + (0 == source.OffsetX ? from.Width / 2 : source.OffsetX);
                int grabY = from.Y + (0 == source.OffsetY ? from.Height / 2 : source.OffsetY);
 
                int dropX = to.X + (0 == target.OffsetX ? to.Width / 2 : target.OffsetX) + target.DeltaX;
                int dropY = to.Y + (0 == target.OffsetY ? to.Height / 2 : target.OffsetY) + target.DeltaY;
 
                // a held key is machine wide, so it is released in a finally. Anything that throws mid drag
                // would otherwise leave Control or Shift down for every window the user touches afterwards
                try
                {
                    HardwareHelper.ReleaseStrayModifiers(modifierKeys);
 
                    if (ModifierKeys.NONE != modifierKeys)
                        HardwareHelper.PressModifierKey(modifierKeys);
 
                    HardwareHelper.MouseDown(grabX, grabY, ClickType.LeftClick);
                    HardwareHelper.DragTravel(dropX, dropY, true == GPAL.SimulateMouse || true == source.SimulateMouse);
                    HardwareHelper.MouseUp(dropX, dropY, ClickType.LeftClick);
                }
                finally
                {
                    if (ModifierKeys.NONE != modifierKeys)
                        HardwareHelper.ReleaseModifierKeys(modifierKeys);
                }
 
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Dragged [{source.Name}] from [{grabX},{grabY}] onto [{target.Name}] at [{dropX},{dropY}]", application, GPALObjectType.Application);
            }
        }

        private static Rectangle FirstRectangle(Application application, Selector selector)
        {
            Rectangle retVal = Rectangle.Empty;
 
            ReadOnlyCollection<GPALAutomationElement> elems = ElementHelper.FindElements(application, application.CurrentUOW, selector, out bool matchedAll, out List<GPALAutomationElement> matchedElements);
 
            if (null != matchedElements && 0 < matchedElements.Count)
                elems = new ReadOnlyCollection<GPALAutomationElement>(matchedElements);
 
            if (false == selector.DeleteMe && null != elems && 0 < elems.Count)
            {
                // a control found under an image is dragged from, or dropped on, where the picture is
                GPALAutomationElement elem = elems[0].ImageMatch ?? elems[0];
 
                if (null != elem.Gae)
                    retVal = new Rectangle(elem.Gae.Location, elem.Gae.Size);
                else
                    retVal = new Rectangle(
                        new System.Drawing.Point((int)elem.Ae.Current.BoundingRectangle.X, (int)elem.Ae.Current.BoundingRectangle.Y),
                        new System.Drawing.Size((int)elem.Ae.Current.BoundingRectangle.Width, (int)elem.Ae.Current.BoundingRectangle.Height));
            }
            else
                PublishToEventHandler(application, selector, elems?.ToList());
 
            return retVal;
        }

        private static void DragByDelta(Application application, ModifierKeys modifierKeys)
        {
            List<GPALAutomationElement> matchedElements;
 
            foreach (Selector selector in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != selector.SelectorType)
                    continue;
 
                ReadOnlyCollection<GPALAutomationElement> elems = ElementHelper.FindElements(application, application.CurrentUOW, selector, out bool matchedAll, out matchedElements);
 
                if (true == selector.DeleteMe || null == elems)
                {
                    PublishToEventHandler(application, selector, elems?.ToList());
                    continue;
                }
 
                if (null != matchedElements)
                    elems = new ReadOnlyCollection<GPALAutomationElement>(matchedElements);
 
                int rowCount = 0;
 
                foreach (GPALAutomationElement found in elems)
                {
                    Rectangle rect;
 
                    // a control found under an image is dragged from where the picture is
                    GPALAutomationElement elem = found.ImageMatch ?? found;
 
                    if (null != elem.Gae)
                        rect = new Rectangle(elem.Gae.Location, elem.Gae.Size);
                    else
                        rect = new Rectangle(
                            new System.Drawing.Point((int)elem.Ae.Current.BoundingRectangle.X, (int)elem.Ae.Current.BoundingRectangle.Y),
                            new System.Drawing.Size((int)elem.Ae.Current.BoundingRectangle.Width, (int)elem.Ae.Current.BoundingRectangle.Height));
 
                    // Mouse down at element origin + OffsetX/Y, travel to that point + DeltaX/Y, mouse up there.
                    // The travel is the drag. Windows starts one only when the pointer moves past its drag
                    // threshold while the button is held, and only if it sees the movement: going straight from
                    // down to up is a click wherever it lands, which is why Explorer went through the motions
                    // and never picked the file up. This is what the browser side has always done
                    // the middle of the element unless the selector says where, for the same reason the two
                    // selector drag does it: a press on the corner lands on whatever is there, not on this
                    int grabX = rect.X + (0 == selector.OffsetX ? rect.Width / 2 : selector.OffsetX);
                    int grabY = rect.Y + (0 == selector.OffsetY ? rect.Height / 2 : selector.OffsetY);
 
                    int endX = grabX + selector.DeltaX;
                    int endY = grabY + selector.DeltaY;
 
                    // a held key is machine wide, so it is released in a finally whatever happens in between
                    try
                    {
                        HardwareHelper.ReleaseStrayModifiers(modifierKeys);
 
                        if (ModifierKeys.NONE != modifierKeys)
                            HardwareHelper.PressModifierKey(modifierKeys);
 
                        HardwareHelper.MouseDown(grabX, grabY, ClickType.LeftClick);
                        HardwareHelper.DragTravel(endX, endY, true == GPAL.SimulateMouse || true == selector.SimulateMouse);
                        HardwareHelper.MouseUp(endX, endY, ClickType.LeftClick);
                    }
                    finally
                    {
                        if (ModifierKeys.NONE != modifierKeys)
                            HardwareHelper.ReleaseModifierKeys(modifierKeys);
                    }
 
                    if (++rowCount >= HowManyToActOn(application))
                        break;
                }
            }
        }

        public static void ExpandMenuItem(Application application, AutomationElement automationElement)
        {
            if (true == GPAL.SimulateMouse || true == application.CurrentUOW.CurrentSelector?.SimulateMouse)
            {
                HardwareHelper.MoveMouse((int)automationElement.Current.BoundingRectangle.X, (int)automationElement.Current.BoundingRectangle.Y, 10, 10);    // source didn't define values for rx, ry            }
                HardwareHelper.LeftClick((int)automationElement.Current.BoundingRectangle.X, (int)automationElement.Current.BoundingRectangle.Y);
                Thread.Sleep(250); // wait for display before trying to find elements
            }
            else
            {
                try
                {
                    ExpandCollapsePattern pattern = automationElement.GetCurrentPattern(ExpandCollapsePattern.Pattern) as ExpandCollapsePattern;
                    pattern.Expand();
                }
                catch (Exception)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Unable to expand menuitem for Control [{automationElement.Current.LocalizedControlType}], trying InvokePattern.", application, GPALObjectType.Application);
                    try
                    {
                        InvokePattern pattern = automationElement.GetCurrentPattern(InvokePattern.Pattern) as InvokePattern;
                        pattern.Invoke();
                    }
                    catch (Exception ex2)
                    {
                        // a menu that answers neither pattern opens with the mouse or does not open at all. An
                        // item on a menu that stays shut is reached by reading the window's own menu instead,
                        // which is what the search does next, so this gives up quietly rather than throwing on
                        // the way out and taking that with it.
                        //
                        // Nothing is asked of the element here. Invoking a menu item can leave the element it
                        // was invoked on behind, and asking a gone element for its rectangle throws, which is
                        // the failure this whole branch exists to avoid
                        if (false == HardwareReaches(application))
                            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, "A menu would not open through UIAutomation and the mouse cannot open it either, so what is on it is read from the window's own menu.", application, GPALObjectType.Application, ex2);
                        else
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Unable to expand menuitem for Control [{automationElement.Current.LocalizedControlType}], using hardware fallback.", application, GPALObjectType.Application);
 
                            Random random = new Random(10007);
 
                            int offsetX = random.Next(1, (int)automationElement.Current.BoundingRectangle.Width - 2);
                            int offsetY = random.Next(1, (int)automationElement.Current.BoundingRectangle.Height - 2);
                            ElementHelper.HardwareClick(null, new GPALAutomationElement(automationElement), Enums.ClickType.LeftClick, Enums.ModifierKeys.NONE, offsetX, offsetY);
                        }
                    }
                }
            }
        }
        public static void SwitchToTab(Application application)
        {
            List<GPALAutomationElement> matchedElements;
            bool matchedAll = false;
 
            foreach (Selector sel in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != sel.SelectorType)
                    continue;   // we don't look up data literals
 
                ElementHelper.FindElements(application, application.CurrentUOW, sel, out matchedAll, out matchedElements);
 
                if (null != matchedElements)
                {
                    int rowCount = 0;
 
                    // what makes something switchable is that it can be selected, not what it is called.
                    // LocalizedControlType is a display string in the user's language, "tab item" on an
                    // English Windows and something else everywhere else, so it was never "TabControl" and
                    // this loop never did anything
                    foreach (GPALAutomationElement elem in matchedElements)
                    {
                        if (null == elem.Ae)
                            continue;
 
                        if (true == elem.Ae.TryGetCurrentPattern(SelectionItemPattern.Pattern, out object selectionPattern))
                        {
                            try
                            {
                                // which tab, whether it was already the one showing, and that the pattern is
                                // what switched it. Selecting through the pattern moves nothing on screen, so
                                // without this a switch that worked and a switch that did nothing look alike
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Switching to [{elem.Ae.Current.Name}] through SelectionItemPattern, which reads Selected [{elem.Selected}] before the switch.", application, GPALObjectType.Application);
 
                                ((SelectionItemPattern)selectionPattern).Select();
 
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{elem.Ae.Current.Name}] reads Selected [{elem.Selected}] after the switch.", application, GPALObjectType.Application);
                            }
                            catch (Exception ex)
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Unable to use SelectionPattern to select the tab [{elem.Ae.Current.Name}], falling back on hardware click.", application, GPALObjectType.Application, ex);
 
                                if (true == MsaaHelper.Select(application, elem.Ae))
                                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Switched to [{elem.Ae.Current.Name}] through MSAA.", application, GPALObjectType.Application);
                                else if (true == HardwareReaches(application))
                                    HardwareClick(application, elem, ClickType.LeftClick, ModifierKeys.NONE, sel.OffsetX, sel.OffsetY);
                                else
                                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"The tab [{elem.Ae.Current.Name}] was not switched to. Neither SelectionPattern nor MSAA would select it and the mouse cannot reach it here.", application, GPALObjectType.Application);
                            }
                        }
                        else if (true == MsaaHelper.Select(application, elem.Ae))
                        {
                            // the tab reads as selected afterwards or this arm was not taken, so the switch is
                            // known to have happened rather than assumed. That is what makes it safe to try
                            // before the mouse: a refusal costs a read and falls through
                            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Switched to [{elem.Ae.Current.Name}] through MSAA.", application, GPALObjectType.Application);
                        }
                        else if (true == HardwareReaches(application))
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{elem.Ae.Current.Name}] publishes no selection through UIAutomation or MSAA, so the tab is switched to with the mouse.", application, GPALObjectType.Application);
 
                            HardwareClick(application, elem, ClickType.LeftClick, ModifierKeys.NONE, sel.OffsetX, sel.OffsetY);
                        }
                        else
                        {
                            // no selection to take and no mouse to click with is no way in at all. Said out
                            // loud because the click used to be made anyway, land nowhere, and leave the
                            // workflow reading the tab it started on
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"The tab [{elem.Ae.Current.Name}] was not switched to. It publishes no selection through UIAutomation or MSAA, so switching it is the mouse, and the mouse cannot reach [{application.applicationSettings.HiddenDesktopName}] while it is hidden.", application, GPALObjectType.Application);
                        }
 
                        if (++rowCount >= HowManyToActOn(application))
                            break;
                    }
                }
            }
 
        }
        private static int HowManyToActOn(Application application)
        {
            return int.MaxValue == application.CurrentUOW.WithAllThatMatch ? 1 : application.CurrentUOW.WithAllThatMatch;
        }

        private static AutomationElement WritableFor(Application application, AutomationElement element)
        {
            AutomationElement retVal = element;
 
            if (false == element.TryGetCurrentPattern(RangeValuePattern.Pattern, out object unused)
                && false == element.TryGetCurrentPattern(ValuePattern.Pattern, out unused))
            {
                AutomationElement inside = element.FindFirst(TreeScope.Descendants, new OrCondition(
                    new PropertyCondition(AutomationElement.IsRangeValuePatternAvailableProperty, true),
                    new PropertyCondition(AutomationElement.IsValuePatternAvailableProperty, true)));
 
                if (null != inside)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{element.Current.ControlType.ProgrammaticName}] carries no value of its own, so the [{inside.Current.ControlType.ProgrammaticName}] inside it is what takes one.", application, GPALObjectType.Application);
 
                    retVal = inside;
                }
            }
 
            return retVal;
        }
 
        // a control's own windows. UIAutomation builds its tree from what a framework chooses to publish, and
        // WinForms publishes a spinner as two buttons and nothing else, so the box holding the value is reached
        // through Windows itself
        [DllImport("user32.dll")]
        private static extern bool EnumChildWindows(IntPtr parent, EnumChildProc callback, IntPtr lParam);
 
        private delegate bool EnumChildProc(IntPtr handle, IntPtr lParam);
 
        [DllImport("user32.dll")]
        private static extern IntPtr SendMessage(IntPtr handle, int message, IntPtr wParam, IntPtr lParam);
 
        [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
        private static extern IntPtr SendMessage(IntPtr handle, int message, IntPtr wParam, string lParam);
 
        [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
        private static extern int SendMessage(IntPtr handle, int message, int wParam, System.Text.StringBuilder lParam);
 
        private const int WM_KILLFOCUS = 0x0008;
        private const int WM_SETTEXT = 0x000C;
        private const int WM_GETTEXT = 0x000D;
        private const int WM_GETTEXTLENGTH = 0x000E;

        private static string NativeTextOf(IntPtr handle)
        {
            int length = SendMessage(handle, WM_GETTEXTLENGTH, 0, null);
            System.Text.StringBuilder text = new System.Text.StringBuilder(length + 1);
 
            SendMessage(handle, WM_GETTEXT, text.Capacity, text);
 
            return text.ToString();
        }

        private static bool SetNativeText(Application application, AutomationElement element, string useText, WriteMode writeMode)
        {
            bool retVal = false;
            IntPtr handle = new IntPtr(element.Current.NativeWindowHandle);
 
            // WM_SETTEXT replaces everything the control holds, so it can only write text that was going to
            // replace everything anyway. An Append or an Insert has to keep what is there, and keeping it
            // means reading it, adding to it and writing the whole lot back: if that write lands short, and a
            // window is free to take one character of it and answer TRUE, what the workflow asked to keep is
            // gone, and putting it back goes through the same message that just lost it. Adding to a control
            // is typed in, where the caret is moved and nothing existing is rewritten
            if (WriteMode.Overwrite != writeMode)
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{element.Current.ControlType.ProgrammaticName}] is added to rather than replaced, which its window cannot be asked to do, so it is typed into instead.", application, GPALObjectType.Application);
            else if (IntPtr.Zero != handle)
            {
                // and the write is checked rather than trusted, the same way the value above it is. A window
                // that answers WM_SETTEXT with TRUE has not promised to have stored what it was given: one
                // that reads the message as bytes takes a string of wide characters as its first character and
                // the zero beside it, answers TRUE, and holds one letter. Reading it back is the only thing
                // that tells the difference, and nothing is lost by finding out, because an overwrite was
                // taking the old text away regardless
                bool answered = IntPtr.Zero != SendMessage(handle, WM_SETTEXT, IntPtr.Zero, useText);
 
                retVal = true == answered
                      && (true == string.IsNullOrEmpty(useText) || true == NativeTextOf(handle).Contains(useText));
 
                if (true == retVal)
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{element.Current.ControlType.ProgrammaticName}] publishes no value that can be written, so the text was written to the window holding it.", application, GPALObjectType.Application);
                else if (true == answered)
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{element.Current.ControlType.ProgrammaticName}] said it took the text written to its window but is not holding it, so it is typed into instead.", application, GPALObjectType.Application);
            }
 
            return retVal;
        }

        private static AutomationElement NativeWritableFor(AutomationElement element)
        {
            AutomationElement retVal = null;
            IntPtr handle = new IntPtr(element.Current.NativeWindowHandle);
 
            if (IntPtr.Zero != handle)
            {
                List<IntPtr> inside = new List<IntPtr>();
 
                EnumChildWindows(handle, (child, unused) => { inside.Add(child); return true; }, IntPtr.Zero);
 
                foreach (IntPtr child in inside)
                {
                    AutomationElement fromWindow = AutomationElement.FromHandle(child);
 
                    // ValuePattern and nothing else. The same windows answer to RangeValuePattern as well, but
                    // the range it reports is not the control's: it reads a value of zero and calls the real
                    // maximum out of bounds, so setting through it throws on any number worth setting
                    if (true == fromWindow.TryGetCurrentPattern(ValuePattern.Pattern, out object unused))
                    {
                        retVal = fromWindow;
                        break;
                    }
                }
            }
 
            return retVal;
        }

        private static void Commit(AutomationElement element)
        {
            IntPtr handle = new IntPtr(element.Current.NativeWindowHandle);
 
            if (IntPtr.Zero != handle)
                SendMessage(handle, WM_KILLFOCUS, IntPtr.Zero, IntPtr.Zero);
        }

        private static bool SetNativeValue(Application application, AutomationElement element, string value)
        {
            bool retVal = false;
            AutomationElement writable = NativeWritableFor(element);
 
            if (null != writable && true == writable.TryGetCurrentPattern(ValuePattern.Pattern, out object valuePattern))
            {
                ((ValuePattern)valuePattern).SetValue(value);
 
                Commit(writable);
 
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{element.Current.ControlType.ProgrammaticName}] keeps its value in a window UIAutomation does not list, so [{value}] was written there and the control was told to read it.", application, GPALObjectType.Application);
 
                retVal = true;
            }
 
            return retVal;
        }
 
        public static void SetRange(Application application, double rangeValue)
        {
            List<GPALAutomationElement> matchedElements;
            bool matchedAll = false;
 
            foreach (Selector sel in application.CurrentUOW.WithSelectorList)
            {
                if (SelectorType.Selector != sel.SelectorType)
                    continue;
 
                ElementHelper.FindElements(application, application.CurrentUOW, sel, out matchedAll, out matchedElements);
 
                if (null != matchedElements)
                {
                    foreach (GPALAutomationElement elem in matchedElements)
                    {
                        if (null == elem.Ae) continue;
 
                        // a spinner is not one control, it is an edit with two buttons beside it, and which of
                        // them carries the value is the framework's business rather than the workflow's. So the
                        // element named is asked first and what is inside it second, which means a workflow can
                        // say "set the spinner" and mean it
                        AutomationElement writable = WritableFor(application, elem.Ae);
                        bool taken = false;
 
                        // each way of setting it is tried until one works. A control that offers a pattern
                        // does not have to honour it: a WinForms spinner offers RangeValuePattern and then
                        // reports a range that is not its own, refusing anything inside the range it really
                        // has, so an offer that comes to nothing has to be followed by the next way in
                        if (true == writable.TryGetCurrentPattern(RangeValuePattern.Pattern, out object rangeValuePattern))
                        {
                            try
                            {
                                ((RangeValuePattern)rangeValuePattern).SetValue(rangeValue);
 
                                taken = true;
                            }
                            catch (Exception ex)
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"RangeValuePattern on [{writable.Current.ControlType.ProgrammaticName}] refused [{rangeValue}], so the next way of setting it is tried.", application, GPALObjectType.Application, ex);
                            }
                        }
 
                        if (false == taken && true == writable.TryGetCurrentPattern(ValuePattern.Pattern, out object valuePattern))
                        {
                            try
                            {
                                ((ValuePattern)valuePattern).SetValue(rangeValue.ToString());
 
                                // the box the value went into may be a spinner's, whether the workflow named
                                // the spinner or the box itself, and either way it is not the value until the
                                // control has read it
                                Commit(writable);
 
                                taken = true;
                            }
                            catch (Exception ex)
                            {
                                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"ValuePattern on [{writable.Current.ControlType.ProgrammaticName}] refused [{rangeValue}], so the next way of setting it is tried.", application, GPALObjectType.Application, ex);
                            }
                        }
 
                        // and failing both, the control still has the value somewhere. A WinForms spinner is
                        // older than UIAutomation and publishes nothing that holds it, while the edit box it
                        // keeps it in sits there as a window Windows will hand over on request
                        if (false == taken)
                            taken = SetNativeValue(application, elem.Ae, rangeValue.ToString());
 
                        // and when even that finds nothing, the way a person does it
                        if (false == taken)
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{elem.Ae.Current.ControlType.ProgrammaticName}] offers no value to set through UIAutomation, so it is typed into instead.", application, GPALObjectType.Application);
 
                            HardwareFillInFrom(application, elem, sel.OffsetX, sel.OffsetY, rangeValue.ToString(), WriteMode.Overwrite);
                        }
                    }
                }
            }
        }
    }
}