GPAL - Generally Positive Automation Library v1.0
GPAL The Fluent Automation LIbrary
Loading...
Searching...
No Matches
ElementHelper.cs
1// =============================================================================
2// GPAL - Generally Positive Automation Library
3// Copyright © 2026 Software Decisions, Inc. All rights reserved.
4//
5// This file is part of GPAL.
6// Licensed under the Business Source License 1.1
7//
8// Primary development, architecture, and vision by Michael B. Vederman,
9// CEO of Software Decisions, Inc., Texas.
10//
11// Internal development maintained privately.
12// Public releases appear on GitHub: https://github.com/SoftwareDecisionsInc/GPAL.
13//
14// See LICENSE for full terms, including Additional Use Grant.
15// =============================================================================
16
17using System;
18using System.Collections.Generic;
19using System.Collections.ObjectModel;
20using System.Diagnostics;
21using System.Drawing;
22using System.IO;
23using System.Linq;
24using System.Net;
25using System.Net.WebSockets;
26using System.Reflection;
27using System.Runtime.InteropServices;
28using System.Text;
29using System.Text.Json;
30using System.Text.RegularExpressions;
31using System.Threading;
32using System.Threading.Tasks;
33using System.Windows.Controls.Primitives;
34using System.Windows.Forms;
35using System.Windows.Threading;
36//using DocumentFormat.OpenXml.Bibliography;
37//using DocumentFormat.OpenXml.Drawing;
38using Microsoft.Win32;
39using OpenQA.Selenium;
40using OpenQA.Selenium.Interactions;
41using OpenQA.Selenium.Support.UI;
42using Org.BouncyCastle.Security;
43using static GenerallyPositive.Browser.Puppeteer;
44using static GenerallyPositive.Enums;
45using static GenerallyPositive.SelectorSettings;
46using Keys = OpenQA.Selenium.Keys;
47
48namespace GenerallyPositive.Browser
49{
50
51 internal class ElementHelper
52 {
53 public delegate ReadOnlyCollection<GPALElement> FindWebElementsByDelegate(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false);
54
55 // to avoid repeat messages for the same selector when waiting for a selector
56 static List<string> listOfMessages = new List<string>();
57 static bool supressedMessage = false;
58 static Selector lastSelector = null;
59
60 // how hard ScrollUntilVisible works before it gives up and lets the workflow carry on
61 const int VisibilitySettleChecks = 4; // rechecks after a scroll, to ride out a header sliding in
62 const int VisibilitySettleMs = 100; // wait between those rechecks
63 const int MaxScrollPages = 40; // screens to page through before giving up on finding it
64
65 // grok refactor
66 public static ReadOnlyCollection<GPALElement> FindWebElements(
67 Browser browser,
68 UnitOfWork currentUOW,
69 Selector selector,
70 out bool matchedAll,
71 out List<GPALElement> matchedElements,
72 GPALElement webElement = null,
73 bool callIfHandler = true,
74 bool isPersistent = false)
75 {
76 int retries = 1;
77 bool retry = false;
78 int withAllThatMatch = int.MaxValue;
79
80 // next page button does not go thru .withselector to add browser. i suppose they could all just use this spot.
81 if (null == selector.Browser)
82 selector.Browser = browser;
83
84 dynamic ourRetryElement = webElement;
85 ReadOnlyCollection<GPALElement> retElems = new ReadOnlyCollection<GPALElement>(new List<GPALElement>());
86 string message = string.Empty;
87
88 currentUOW.CurrentSelector = selector;
89
90 if (false == lastSelector?.Name.Equals(selector.Name))
91 {
92 listOfMessages.Clear();
93 supressedMessage = false;
94 lastSelector = selector;
95 }
96 else if (null == lastSelector)
97 lastSelector = selector;
98
99 if (browser.persistentUOW != currentUOW)
100 browser.CurrentUOW.CurrentSelector = selector;
101
102 matchedAll = false;
103 matchedElements = new List<GPALElement>();
104
105 List<FindWebElementsByDelegate> functionList = new List<FindWebElementsByDelegate>
106 {
107 null, FindWebElementsByCss, FindWebElementsByImage, FindWebElementsByText,
108 FindWebElementsByValue, FindWebElementsByXPath, FindWebElementsByName,
109 null, FindWebElementsByClassName, FindWebElementsByPlaceholder, FindWebElementsByHRef
110 };
111
112 foreach (var selectorPathEntry in selector.SelectorPaths)
113 {
114 tryAgain: // NOTE: CAVEAT: goto label - don't like it, but it has to happen sometimes. we cannot have any inner loop as that will break retrying the next selector
115 int textLength = selectorPathEntry.SelectorPath.Length - 1;
116 string abbreviatedText = selectorPathEntry.SelectorPath.Substring(0, Math.Min(7, textLength)) + "..." + selectorPathEntry.SelectorPath.Substring(Math.Max(0, textLength - 7));
117 if (18 > selectorPathEntry.SelectorPath.Length)
118 abbreviatedText = selectorPathEntry.SelectorPath;
119
120 int idx = GetIndexOfEnum<SelectorPathType>((int)selectorPathEntry.SelectorPathType);
121 bool tryNextSelector = false;
122 ReadOnlyCollection<GPALElement> allFoundElems = new ReadOnlyCollection<GPALElement>(new List<GPALElement>());
123 string errorMessage = $"Error finding elements for selector [{selector.Name}][{abbreviatedText}]";
124 int selectorPathIdx = 0;
125
126 try
127 {
128 message = $"Searching for elements for selector [{selector.Name}] using [{selectorPathEntry.SelectorPathType}] [{abbreviatedText}]";
129
130 if (false == listOfMessages.Contains(message))
131 {
132 listOfMessages.Add(message);
133 GPAL.PublishSimpleEvent(GPALEventType.INFO, message, browser, GPALObjectType.Browser);
134 }
135
136 var tmpAllFoundElems = FindElements(browser, currentUOW, selectorPathEntry, selector, webElement, idx, functionList, isPersistent);
137
138 // several finders hand back null rather than an empty collection - an image that did not match,
139 // or an otto xpath with no hits - and that null would replace the empty collection this method
140 // was initialized with, so both the return value and the out param would go null on the most
141 // ordinary outcome there is. nothing found is an empty collection, never null.
142 retElems = allFoundElems = tmpAllFoundElems ?? new ReadOnlyCollection<GPALElement>(new List<GPALElement>());
143 matchedElements = allFoundElems.ToList();
144
145 if (0 < retElems?.Count)
146 {
147 // NOTE: puppeteer and ottomagic give us gpalelements with individual Css on each element, not so for selenium
148 if (true == browser.UseSelenium)
149 foreach (GPALElement gPALElement in allFoundElems)
150 gPALElement.Css = GenerateCssSelector(browser, gPALElement.WebElement);
151
152 var matchingResult = ApplyCustomMatchCriteria(selector, allFoundElems, browser, matchedElements, out tryNextSelector);
153 matchedAll = matchingResult;
154
155 if (tryNextSelector)
156 {
157 retElems = new ReadOnlyCollection<GPALElement>(new List<GPALElement>());
158 matchedElements.Clear();
159 continue;
160 }
161
162 List<GPALElement> simpleMatches = matchedElements;
163 // simple matching applies on top of custom matching, so pass in matchedELements which will be allfound if no custom matching
164 if (selector.MatchCriteria.Count > 0)
165 matchedAll = ApplySimpleMatching(selector, matchedElements, matchedElements, out simpleMatches);
166 else
167 matchedAll = true;
168
169 matchedElements = simpleMatches; // after simple matching, this will contain only matches
170
171 selectorPathEntry.WebSelectorFoundResults = allFoundElems?.ToList();
172 selectorPathEntry.WebSelectorMatchedResults = simpleMatches;
173
174 // if we use .WithAllThatMatch(number) we want up to that number, either in total or per page
175 // int.MaxValue means all rows, so the most specific scope that asked for a limit wins:
176 // unit of work, then browser, then global. if none did, we take them all.
177 if (int.MaxValue != currentUOW.WithAllThatMatch && 0 < currentUOW.WithAllThatMatch)
178 withAllThatMatch = currentUOW.WithAllThatMatch;
179 else if (int.MaxValue != browser.BrowserSettings.WithAllThatMatch && 0 < browser.BrowserSettings.WithAllThatMatch)
180 withAllThatMatch = browser.BrowserSettings.WithAllThatMatch;
181 else if (0 < GPAL.GPALSettings.WithAllThatMatch)
182 withAllThatMatch = GPAL.GPALSettings.WithAllThatMatch; // int.MaxValue if it was never set
183
184 if (int.MaxValue != withAllThatMatch && 0 < simpleMatches?.Count)
185 {
186 List<GPALElement> tmpList = new List<GPALElement>();
187
188 // ensure we copy over only as many results as requested, or how many results we have
189 tmpList.AddRange(simpleMatches.Take(Math.Min(withAllThatMatch, matchedElements?.Count ?? 0)));
190
191 matchedElements = tmpList;
192 }
193
194 foreach (GPALElement element in retElems)
195 {
196 element.Browser = browser;
197 element.Selector = selector;
198 }
199 }
200 }
201 catch (StaleElementReferenceException sereex)
202 {
203 message = errorMessage + $" StaleElementReferenceException: [{sereex.Message}]";
204 retry = ResolveWebElementIssue(browser, currentUOW, ref ourRetryElement, sereex, ref retries, message);
205 }
206 catch (ElementClickInterceptedException ecieex)
207 {
208 message = errorMessage + $" ElementClickInterceptedException: [{ecieex.Message}]";
209 retry = ResolveWebElementIssue(browser, currentUOW, ref ourRetryElement, ecieex, ref retries, message);
210 }
211 catch (ElementNotInteractableException enieex)
212 {
213 message = errorMessage + $" ElementNotInteractableException: [{enieex.Message}]";
214 retry = ResolveWebElementIssue(browser, currentUOW, ref ourRetryElement, enieex, ref retries, message);
215 }
216 catch (GPALException)
217 {
218 throw;
219 }
220 catch (Exception ex)
221 {
222 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, errorMessage, browser, GPALObjectType.Browser, ex);
223 break;
224 }
225
226 if (true == retry && 0 < retries--)
227 {
228 retry = false;
229 goto tryAgain; // NOTE: CAVEAT: goto label
230 }
231
232 selector.ElementsFoundAndMatchedCount = (int)(null != matchedElements ? matchedElements?.Count() : 0);
233
234 message = $"[{retElems?.Count ?? 0}] elements found for selector [{selector.Name}]";
235
236 if (false == listOfMessages.Contains(message))
237 {
238 listOfMessages.Add(message);
239 GPAL.PublishSimpleEvent(GPALEventType.INFO, message, browser, GPALObjectType.Browser);
240
241 if (0 < retElems?.Count && (true == selector.AnyMatchCriteria() || Int32.MaxValue != withAllThatMatch))
242 {
243 var matchCount = selector.MatchCriteria.Count > 0 ? selector.MatchCriteria.Count.ToString() : $"WithAllThatMatch({withAllThatMatch})";
244
245 if ((true == matchCount.Contains("WithAllThatMatch") && 0 < withAllThatMatch) || 0 < selector.MatchCriteria.Count)
246 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{matchedElements?.Count}] elements matched for selector [{selector.Name}] with [{matchCount}] match criteria", browser, GPALObjectType.Browser);
247 }
248 }
249 else if (false == supressedMessage)
250 {
251 GPAL.PublishSimpleEvent(GPALEventType.INFO, "Supressing repeat messages", browser, GPALObjectType.Browser);
252 supressedMessage = true;
253 }
254
255 if (matchedAll || retElems?.Count > 0)
256 break;
257 else if (++selectorPathIdx == selector.SelectorPaths.Count)
258 GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"No more selector paths. No elements found.", browser, GPALObjectType.Browser);
259 }
260
261 // false == callIfHandler will be the waitfor call
262 if (true == callIfHandler)
263 CallIfHandlers(browser, currentUOW, retElems, matchedElements, selector, matchedAll);
264
265 return retElems;
266 }
267
268 // Helper to handle shadow DOM or normal search with retries
283 public static ReadOnlyCollection<GPALElement> FindElements(
284 Browser browser,
285 UnitOfWork currentUOW,
286 SelectorSettings.SelectorPathEntry selectorPathEntry,
287 Selector selector,
288 GPALElement gPalElement,
289 int idx,
290 List<FindWebElementsByDelegate> functionList,
291 bool isPersistent)
292 {
293 if (0 < browser.CurrentUOW.ContextPath?.Count && null == gPalElement)
294 {
295
296 if (true == browser.UseOttoMagic)
297 browser.MagicHelper.SwitchToDefaultContent();
298 else if (true == browser.UsePuppeteer)
299 browser.PuppeteerClient.SwitchToDefaultContent().Execute();
300 else if (true == browser.UseSelenium)
301 browser.BrowserDriver.SwitchTo().DefaultContent();
302
303 IEnumerable<UnitOfWork.WebElementWithType> stepsToProcess;
304
305 // puppeteer or selenium have to be iframe aware: process the full original path
306 stepsToProcess = browser.CurrentUOW.ContextPath;
307
308 // Now walk the (possibly filtered) steps
309 foreach (UnitOfWork.WebElementWithType contextType in stepsToProcess)
310 {
311 if (ElementType.IFrame == contextType.elementType)
312 {
313 if (true == browser.UseOttoMagic)
314 browser.MagicHelper.SwitchToElement(contextType.gPalElement.Css);
315 else if (true == browser.UsePuppeteer)
316 browser.PuppeteerClient.SwitchToFrame(contextType.gPalElement.Css).Execute();
317 else
318 browser.BrowserDriver.SwitchTo().Frame((IWebElement)contextType.gPalElement.WebElement);
319
320 // a search context belongs to one document, so anything scoped in the frame we just left
321 // is stale now - a shadow root from the parent throws "no such shadow root" if we keep
322 // using it here. whatever comes after this step re-establishes the scope.
323 gPalElement = null;
324 }
325 else if (ElementType.ShadowRoot == contextType.elementType)
326 {
327 if (true == browser.UseOttoMagic)
328 browser.MagicHelper.SwitchToShadowRoot(contextType.gPalElement.Css);
329 else if (true == browser.UsePuppeteer)
330 browser.PuppeteerClient.SwitchToShadowRoot(contextType.gPalElement.Css).Execute();
331
332 // scope to THIS step's shadow root, the same one otto and puppeteer just switched into.
333 // selenium has no switch call - handing the search its ShadowRoot context is the switch -
334 // so it has to be this step's root and not the last one added to the unit of work, or a
335 // path with more than one shadow root scopes every step to the innermost one.
336 gPalElement = contextType.gPalElement;
337 }
338 else if (ElementType.Element == contextType.elementType)
339 {
340 // the one this pass is about when .InElement matched many, the only one otherwise
341 gPalElement = contextType.CurrentElement; // this is what gets passed down
342 }
343 }
344 }
345
346 int retries = 1;
347 while (retries-- > 0)
348 {
349 try
350 {
351 return functionList[idx](browser, selectorPathEntry, selector, gPalElement, isPersistent);
352 }
353 catch (GPALException)
354 {
355 throw;
356 }
357 catch (Exception ex)
358 {
359 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Error finding elements for selector [{selector.Name}]. Trying next selector path if defined.", browser, GPALObjectType.Browser, ex);
360 break; ; // for retry handler
361 }
362 }
363
364 return new ReadOnlyCollection<GPALElement>(new List<GPALElement>());
365 }
366 // Helper to apply custom matching criteria
380 private static bool ApplyCustomMatchCriteria(
381 Selector selector,
382 ReadOnlyCollection<GPALElement> allFoundElems,
383 Browser browser,
384 List<GPALElement> matchedElements,
385 out bool tryNextSelector,
386 bool AndOrOr = true) // true = AND, false = OR
387 {
388 tryNextSelector = false;
389
390 var universe = allFoundElems?.ToList();
391
392 // initial working set
393 var workingSet = matchedElements.Count > 0
394 ? new List<GPALElement>(matchedElements)
395 : new List<GPALElement>(universe);
396
397 bool matchedAll = true; // default to true, if no custom match is applied to tell us otherwise
398
399 foreach (var matchCriteriaEntry in selector.MatchCriteria)
400 {
401 if (matchCriteriaEntry.MatchType != MatchType.Custom)
402 continue;
403
404 // Determine the source set
405 var source = AndOrOr ? workingSet : universe;
406
407 GPAL.PublishSimpleEvent(GPALEventType.DEBUG,
408 $"Applying custom match [{matchCriteriaEntry.WebMatchingFunction.GetInvocationList()[0].Method.Name}] " +
409 $"to {(AndOrOr ? "working set" : "full universe")} ({source.Count} elements)",
410 browser,
411 GPALObjectType.Browser);
412
413 CallIfStatus handled = matchCriteriaEntry.WebMatchingFunction(
414 source.Cast<IGPALElement>().ToList(),
415 out List<IGPALElement> tempMatchedElemsRaw,
416 out matchedAll,
417 selector,
418 matchCriteriaEntry.ExactMatch);
419 List<GPALElement> tempMatchedElems = tempMatchedElemsRaw?.Cast<GPALElement>().ToList();
420
421 // Deduplicate
422 var matchedThisRound = new List<GPALElement>();
423 var roundSet = new HashSet<GPALElement>();
424
425 foreach (var elem in tempMatchedElems ?? new List<GPALElement>())
426 {
427 if (roundSet.Add(elem))
428 {
429 matchedThisRound.Add(elem);
430
431 GPAL.PublishSimpleEvent(GPALEventType.DEBUG,
432 $"Element matched custom criteria [{matchCriteriaEntry.WebMatchingFunction.GetInvocationList()[0].Method.Name}] " +
433 $"TagName: [{elem.TagName}], Text: [{elem.Text}]",
434 browser,
435 GPALObjectType.Browser);
436 }
437 }
438
439 if (CallIfStatus.Terminate == handled)
440 {
441 GPAL.PublishSimpleEvent(
442 GPALEventType.EXCEPTION,
443 $"Custom match handler [{matchCriteriaEntry.WebMatchingFunction.GetInvocationList()[0].Method.Name}] requested program termination for selector [{selector.Name}].",
444 browser,
445 GPALObjectType.Browser);
446
447 throw new GPALException(
448 $"Custom match handler [{matchCriteriaEntry.WebMatchingFunction.GetInvocationList()[0].Method.Name}] requested program termination for selector [{selector.Name}].");
449 }
450
451 if (AndOrOr)
452 {
453 // AND = shrink working set
454 workingSet = matchedThisRound;
455 }
456 else
457 {
458 // OR = union into working set
459 var unionSet = new HashSet<GPALElement>(workingSet);
460 foreach (var elem in matchedThisRound)
461 {
462 if (unionSet.Add(elem))
463 {
464 workingSet.Add(elem);
465
466 GPAL.PublishSimpleEvent(GPALEventType.DEBUG,
467 $"Element added to OR working set from custom match: TagName: [{elem.TagName}], Text: [{elem.Text}]",
468 browser,
469 GPALObjectType.Browser);
470 }
471 }
472 }
473
474 if (CallIfStatus.Handled == handled)
475 break;
476
477 if (CallIfStatus.TryNext == handled)
478 {
479 tryNextSelector = true;
480 break;
481 }
482 }
483
484 matchedElements.Clear();
485 matchedElements.AddRange(workingSet);
486
487 return matchedAll;
488 }
489
490 // Helper to apply simple matching criteria to elements after custom matching
501 private static bool ApplySimpleMatching(
502 Selector selector,
503 List<GPALElement> matchedElements,
504 List<GPALElement> allFoundElems,
505 out List<GPALElement> matchedElementsOut,
506 bool AndOrOr = true) // true = AND, false = OR
507 {
508 matchedElementsOut = new List<GPALElement>();
509 var universe = allFoundElems?.ToList();
510
511 // initial working set
512 var workingSet = matchedElements.Count > 0
513 ? new List<GPALElement>(matchedElements)
514 : new List<GPALElement>(universe);
515
516 foreach (var matchCriteriaEntry in selector.MatchCriteria)
517 {
518 if (matchCriteriaEntry.MatchType == MatchType.None || matchCriteriaEntry.MatchType == MatchType.Custom)
519 continue;
520
521 // source for this round
522 var source = AndOrOr ? workingSet : universe;
523 var matchedThisRound = new List<GPALElement>();
524 var roundSet = new HashSet<GPALElement>();
525
526 foreach (var element in source)
527 {
528 string elementStringToMatch = null;
529
530 if ("GPALElement" != element.TagName)
531 {
532 switch ((MatchType)((int)matchCriteriaEntry.MatchType & 0xFE))
533 {
534 case MatchType.Text:
535 elementStringToMatch = element.Text;
536 if (string.IsNullOrEmpty(elementStringToMatch))
537 elementStringToMatch = element.GetAttribute("title");
538 if (string.IsNullOrEmpty(elementStringToMatch))
539 elementStringToMatch = element.GetAttribute("value");
540 break;
541 case MatchType.Href:
542 elementStringToMatch = element.GetAttribute("href");
543 break;
544 case MatchType.Src:
545 elementStringToMatch = element.GetAttribute("src");
546 break;
547 case MatchType.Placeholder:
548 elementStringToMatch = element.GetAttribute("placeholder");
549 break;
550 case MatchType.Value:
551 elementStringToMatch = element.GetAttribute("value");
552 break;
553 case MatchType.Attribute:
554 elementStringToMatch = element.GetAttribute(selector.AttributeName);
555 break;
556 }
557
558 if (!string.IsNullOrEmpty(elementStringToMatch))
559 {
560 GPAL.PublishSimpleEvent(GPALEventType.DEBUG,
561 $"String to match [{matchCriteriaEntry.StringToMatch}] element text [{elementStringToMatch}]");
562
563 bool isMatch = false;
564
565 if ((int)(matchCriteriaEntry.MatchType & MatchType.Regex) == 1)
566 {
567 var mc = Regex.Matches(elementStringToMatch, matchCriteriaEntry.StringToMatch);
568 if (mc.Count > 0)
569 isMatch = true;
570 }
571 else
572 {
573 if (false == matchCriteriaEntry.ExactMatch && elementStringToMatch.Contains(matchCriteriaEntry.StringToMatch))
574 isMatch = true;
575 else if (true == matchCriteriaEntry.ExactMatch && elementStringToMatch.Equals(matchCriteriaEntry.StringToMatch))
576 isMatch = true;
577 }
578
579 if (isMatch && roundSet.Add(element))
580 {
581 GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Adding [{element.TagName}][{element.Text}]");
582 matchedThisRound.Add(element);
583 }
584 }
585 }
586 else
587 {
588 if (roundSet.Add(element))
589 matchedThisRound.Add(element);
590 }
591 }
592
593 if (AndOrOr)
594 {
595 // AND = narrow
596 workingSet = matchedThisRound;
597 }
598 else
599 {
600 // OR = union
601 var unionSet = new HashSet<GPALElement>(workingSet);
602 foreach (var elem in matchedThisRound)
603 {
604 if (unionSet.Add(elem))
605 {
606 workingSet.Add(elem);
607 }
608 }
609 }
610 }
611
612 matchedElementsOut = workingSet;
613 return universe.Count == matchedElementsOut.Count;
614 }
615
616 // TODO: what do these values mean outside of the bubble handlers?
617 // currentUOW.callFoundHandled
618 // currentUOW.callNotFoundHandled
631 private static void CallIfHandlers(Browser browser, UnitOfWork currentUOW, ReadOnlyCollection<GPALElement> foundElements, List<GPALElement> matchedElements, Selector selector, bool matchedAll)
632 {
633 string str = null;
634 string currentMethodName = null;
635 int selectorsFoundOrMatched = selector.AnyMatchCriteria() ? matchedElements?.Count ?? 0 : foundElements?.Count ?? 0;
636 UnitOfWork safeUOW;
637
638 // TODO: do we care about matchedall?
639
640 // bubble thru handlers based upon handler return value
641 // 0 = not handled, bubble to next handler
642 // 1 = handled and continue with the program
643 // -1 = unexpected error and terminate program
644 if (0 < selectorsFoundOrMatched)
645 {
646 if (0 != (selector.SelectorSettings.CallIfFound?.Count ?? 0))
647 {
648 foreach (Browser.CallIfDelegate func in selector.SelectorSettings.CallIfFound)
649 {
650 if (true == selector.RemovedMethods.Contains(func))
651 continue; // soft deleted by RemoveCallIfHandlerEverywhere
652
653 bool safeActionCalled = browser.CurrentUOW.ActionCalled;
654 safeUOW = browser.CurrentUOW;
655 browser.CurrentUOW.ActionCalled = true; // this will allow the callif method to immediately use browser.WaitFor
656
657 currentMethodName = func.Method.Name;
658 str = $"Browser Selector CallIfFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
659
660 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking Selector CallIfFound [{func.Method.Name}]", browser, GPALObjectType.Browser);
661
662 selector.CallIfFoundHandled = func(browser, foundElements?.Cast<IGPALElement>().ToList(), matchedElements?.Cast<IGPALElement>().ToList(), selector, matchedAll);
663
664 browser.CurrentUOW = safeUOW;
665 browser.CurrentUOW.ActionCalled = safeActionCalled;
666
667 if (CallIfStatus.NotHandled == selector.CallIfFoundHandled)
668 continue;
669 else
670 break;
671 }
672
673 if (CallIfStatus.Terminate == selector.CallIfFoundHandled)
674 {
675 GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
676 throw new GPALException(str);
677 }
678 }
679
680 // selector.CallIfFoundHandled - if handled by selector handler, don't call UOW handler
681 if (CallIfStatus.NotHandled == selector.CallIfFoundHandled && 0 < (currentUOW.CallIfFound?.Count ?? 0))
682 {
683 foreach (Browser.CallIfDelegate func in currentUOW.CallIfFound)
684 {
685 if (true == selector.RemovedMethods.Contains(func))
686 continue; // soft deleted by RemoveCallIfHandlerEverywhere
687
688 safeUOW = browser.CurrentUOW;
689
690 currentMethodName = func.Method.Name;
691 str = $"Browser UOW Selector CallIfFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
692
693 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking UOW CallIfFound [{func.Method.Name}]", browser, GPALObjectType.Browser);
694
695 currentUOW.CallIfFoundHandled = func(browser, foundElements?.Cast<IGPALElement>().ToList(), matchedElements?.Cast<IGPALElement>().ToList(), selector, matchedAll);
696
697 browser.CurrentUOW = safeUOW;
698
699 if (CallIfStatus.NotHandled == currentUOW.CallIfFoundHandled)
700 continue;
701 else
702 break;
703 }
704
705 if (CallIfStatus.Terminate == currentUOW.CallIfFoundHandled)
706 {
707 GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
708 throw new GPALException(str);
709 }
710 }
711
712 // currentUOW.CallIfFoundHandled - if handled by UOW handler, don't call global handler
713 if (CallIfStatus.NotHandled == selector.CallIfFoundHandled && 0 == currentUOW.CallIfFoundHandled && 0 != (GPAL.GPALSettings.CallIfFoundList?.Count ?? 0))
714 {
715 foreach (Browser.CallIfDelegate func in GPAL.GPALSettings.CallIfFoundList)
716 {
717 if (true == selector.RemovedMethods.Contains(func))
718 continue; // soft deleted by RemoveCallIfHandlerEverywhere
719
720 safeUOW = browser.CurrentUOW;
721
722 currentMethodName = func.Method.Name;
723 str = $"Gloabl CallIfFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
724
725 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking Global CallIfFound [{func.Method.Name}]", browser, GPALObjectType.Browser);
726
727 GPAL.GPALSettings.CallIfFoundHandled = func(browser, foundElements?.Cast<IGPALElement>().ToList(), matchedElements?.Cast<IGPALElement>().ToList(), selector, matchedAll);
728
729 browser.CurrentUOW = safeUOW;
730
731 if (CallIfStatus.NotHandled == GPAL.GPALSettings.CallIfFoundHandled)
732 continue;
733 else
734 break;
735 }
736
737 if (CallIfStatus.Terminate == (GPAL.GPALSettings.CallIfFoundHandled))
738 {
739 GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
740 throw new GPALException(str);
741 }
742 }
743 }
744 else
745 {
746 if (0 < selector.SelectorSettings.CallIfNotFound.Count)
747 {
748 foreach (Browser.CallIfDelegate func in selector.SelectorSettings.CallIfNotFound)
749 {
750 if (true == selector.RemovedMethods.Contains(func))
751 continue; // soft deleted by RemoveCallIfHandlerEverywhere
752
753 safeUOW = browser.CurrentUOW;
754
755 currentMethodName = func.Method.Name;
756 str = $"Browser Selector CallIfNotFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}[";
757
758 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking Selector CallIfNotFound [{func.Method.Name}]", browser, GPALObjectType.Browser);
759
760 selector.CallIfNotFoundHandled = func(browser, foundElements?.Cast<IGPALElement>().ToList(), matchedElements?.Cast<IGPALElement>().ToList(), selector, matchedAll);
761
762 browser.CurrentUOW = safeUOW;
763
764 if (CallIfStatus.NotHandled == selector.CallIfNotFoundHandled)
765 continue;
766 else
767 break;
768 }
769
770 if (CallIfStatus.Terminate == selector.CallIfNotFoundHandled)
771 {
772 GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
773 throw new GPALException(str);
774 }
775 }
776
777 if (0 == selector.CallIfNotFoundHandled && 0 < currentUOW.CallIfNotFound.Count)
778 {
779 foreach (Browser.CallIfDelegate func in currentUOW.CallIfNotFound)
780 {
781 if (true == selector.RemovedMethods.Contains(func))
782 continue; // soft deleted by RemoveCallIfHandlerEverywhere
783
784 safeUOW = browser.CurrentUOW;
785
786 currentMethodName = func.Method.Name;
787 str = $"Browser UOW Selector CallIfNotFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
788
789 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking UOW CallIfNotFound [{func.Method.Name}]", browser, GPALObjectType.Browser);
790
791 currentUOW.CallIfNotFoundHandled = func(browser, foundElements?.Cast<IGPALElement>().ToList(), matchedElements?.Cast<IGPALElement>().ToList(), selector, matchedAll);
792
793 browser.CurrentUOW = safeUOW;
794
795 if (CallIfStatus.NotHandled == currentUOW.CallIfNotFoundHandled)
796 continue;
797 else
798 break;
799 }
800
801 if (CallIfStatus.Terminate == currentUOW.CallIfNotFoundHandled)
802 {
803 GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
804 throw new GPALException(str);
805 }
806 }
807 // currentUOW.CallIfNotFoundHandled - if handled by UOW handler, don't call global handler
808 if (0 == selector.CallIfNotFoundHandled && 0 == currentUOW.CallIfNotFoundHandled && 0 < GPAL.GPALSettings.CallIfNotFoundList.Count)
809 {
810 foreach (Browser.CallIfDelegate func in GPAL.GPALSettings.CallIfNotFoundList)
811 {
812 if (true == selector.RemovedMethods.Contains(func))
813 continue; // soft deleted by RemoveCallIfHandlerEverywhere
814
815 safeUOW = browser.CurrentUOW;
816
817 currentMethodName = func.Method.Name;
818 str = $"Gloabl CallIfNotFound handler [{currentMethodName}] requested program termination on selector [{selector.Name}] [{selector.SelectorPaths[0].SelectorPath}]";
819
820 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking Global CallIfNotFound [{func.Method.Name}]", browser, GPALObjectType.Browser);
821
822 GPAL.GPALSettings.CallIfNotFoundHandled = func(browser, foundElements?.Cast<IGPALElement>().ToList(), matchedElements?.Cast<IGPALElement>().ToList(), selector, matchedAll);
823
824 browser.CurrentUOW = safeUOW;
825
826 if (CallIfStatus.NotHandled == GPAL.GPALSettings.CallIfNotFoundHandled)
827 continue;
828 else
829 break;
830 }
831
832 if (CallIfStatus.Terminate == GPAL.GPALSettings.CallIfNotFoundHandled)
833 {
834 GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
835 throw new GPALException(str);
836 }
837 }
838
839 if (true == selector.StopOnNotFound || true == GPAL.StopOnNotFound)
840 {
841 str = $@"{(true == selector.StopOnNotFound ? "selector." : "GPAL.")}StopOnNotFound set, terminating on Not Found element for [{selector.Name}][{selector.SelectorPaths[0].SelectorPath}]";
842 GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
843 throw new GPALException(str);
844 }
845 }
846 }
857 private static ReadOnlyCollection<GPALElement> FindWebElementsByCss(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false)
858 {
859 bool searchForElement = selector.SearchForSelector;
860 ReadOnlyCollection<IWebElement> elems;
861 ReadOnlyCollection<GPALElement> GPALelems = null;
862
863 if (browser.UseOttoMagic)
864 {
865 List<GPALElement> result = null;
866
867 if (false == webElement?.IsShadowRoot)
868 browser.MagicHelper.SwitchToElement(webElement.Css);
869
870 if (true == isPersistent)
871 result = browser.MagicHelper
872 .QueryPersistentSelectors(selectorPath.SelectorPath);
873 else
874 result = browser.MagicHelper
875 .QuerySelectors(selectorPath.SelectorPath);
876
877 if (null != result)
878 GPALelems = new ReadOnlyCollection<GPALElement>(result);
879 }
880 else if (true == browser.UsePuppeteer)
881 {
882 if (false == webElement?.IsShadowRoot)
883 browser.PuppeteerClient.InElement(webElement.Css).Execute();
884
885 var result2 = Task.Run(() => browser.PuppeteerCommunicator.EvaluateSelector(selectorPath.SelectorPath));
886 if (null != result2)
887 GPALelems = new ReadOnlyCollection<GPALElement>(result2.Result);
888 }
889 else
890 {
891 string css = selectorPath.SelectorPath;
892
893 elems = webElement?.WebElement != null
894 ? webElement.WebElement.FindElements(By.CssSelector(css))
895 // webelement is the shadowroot
896 //: browser.CurrentUOW.ShadowRoot?.WebElement != null
897 // ? browser.CurrentUOW.ShadowRoot.WebElement.FindElements(By.CssSelector(css))
898 : browser.BrowserDriver.FindElements(By.CssSelector(css));
899
900 // the search above ran against the closed root's host, which is all webdriver will ever give us.
901 // finding nothing means the target is inside the closed root, so hand the host back - the same
902 // contract otto uses - and a workflow can still click relative to its corner. persistent
903 // selectors are excluded: they are polled speculatively, and a host that always matches would
904 // fire their CallIf handlers on every sweep.
905 if (0 == elems.Count && true == webElement?.IsClosedShadowRoot && false == isPersistent)
906 return new ReadOnlyCollection<GPALElement>(new List<GPALElement>() { webElement });
907
908 GPALelems = ToGPALElements(browser, selector, elems);
909 }
910
911 return GPALelems;
912 }
924 private static ReadOnlyCollection<GPALElement> FindWebElementsByImage(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false)
925 {
926 bool searchForElement = selector.SearchForSelector;
927
928 // only hardware clicks a screen position, so only hardware needs the screen. everything else matches
929 // against the page itself, the way headless always has: the engine's own capture, in page coordinates,
930 // where nothing outside the page can match and nothing has to be converted
931 bool inPage = true == browser.BrowserSettings.UseHeadless || true == browser.BrowserSettings.HiddenDesktop
932 || false == UsesScreenCoordinates(browser, selector.InteractionType);
933
934 ReadOnlyCollection<GPALElement> elems = null;
935
936 Rectangle foundImageBounds = new Rectangle();
937 List<GPALElement> retWebElement = new List<GPALElement>(); ;
938
939 if (null == selectorPath.Image)
940 {
941 try
942 {
943 selectorPath.Image = Image.FromFile(selectorPath.SelectorPath);
944 }
945 catch (GPALException)
946 {
947 throw;
948 }
949 catch (Exception ex)
950 {
951 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to load image for [{selector.Name}].", browser, GPALObjectType.Browser, ex);
952 return null;
953 }
954 }
955
956 // read by CaptureScreen for the length of this search, and put back as soon as it is over so a
957 // screenshot the workflow takes afterwards is not affected
958 browser.BrowserSettings.ImageSearchInPage = inPage;
959
960 foundImageBounds = ((ImageHelper)GPAL.ImageHelper).FindImage(selectorPath.SelectorPath, browser); // first check, is image we seek already on-screen?
961
962 // page down the browser page looking for the image (just send page down key). With no matcher
963 // registered nothing can be found, so the page is not walked for nothing
964 // no click on the body first to take focus from the address bar. every engine pages through its own
965 // protocol, selenium's with javascript, so focus does not matter, and selenium clicks an element in its
966 // middle, which on a real page is whatever sits there, often a link
967 if (true == searchForElement && null != GPAL.ImageMatcher)
968 {
969 // Not found, start from top of page
970 if (true == foundImageBounds.IsEmpty)
971 _ = browser.PageTop;
972
973 while (true == foundImageBounds.IsEmpty && false == IsEndOfPage(browser, webElement))
974 {
975 _ = browser.PageDown;
976 Thread.Sleep(250);
977 foundImageBounds = ((ImageHelper)GPAL.ImageHelper).FindImage(selectorPath.SelectorPath, browser);
978 }
979 }
980
981 browser.BrowserSettings.ImageSearchInPage = false;
982
983 if (false == foundImageBounds.IsEmpty)
984 {
985 // create a psuedo webelement to use in the subsequent methods
986 // a match made against the page is viewport relative; one made against the screen for hardware is
987 // already absolute - record which, the downstream mouse paths need to know whose origin these are
988 // measured from
989 retWebElement.Add(new GPALElement(foundImageBounds.Location, foundImageBounds.Size, selectorPath.SelectorPathType.ToString(), browser.BrowserDriver, "GPALElement")
990 {
991 Browser = browser,
992 Selector = selector,
993 CoordinateSpace = true == inPage ? CoordinateSpace.Viewport : CoordinateSpace.Screen
994 });
995 elems = new ReadOnlyCollection<GPALElement>(retWebElement);
996
997 // an image match is one more way of finding an element. unless the workflow asked for hardware, or
998 // there is no page to ask, the page is asked which element is under the match and that is handed
999 // back instead, so what follows is an ordinary element action on any engine. OttoMagic asks through
1000 // execute-javascript, which needs Allow user scripts on. with no element under it the point is kept,
1001 // and clicked by the engine at that coordinate, or by the mouse when hardware was asked for
1002 if (true == inPage)
1003 {
1004 ReadOnlyCollection<GPALElement> underMatch = ElementUnderImage(browser, retWebElement[0], selector);
1005
1006 if (0 < underMatch?.Count)
1007 elems = underMatch;
1008 }
1009 // TODO: highlight the found element briefly with a border? .WithHighlightFoundImage
1010 }
1011
1012 return elems;
1013 }
1024 private static ReadOnlyCollection<GPALElement> FindWebElementsByXPath(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false)
1025 {
1026 bool searchForElement = selector.SearchForSelector;
1027
1028 ReadOnlyCollection<IWebElement> elems;
1029 ReadOnlyCollection<GPALElement> GPALelems = null;
1030
1031 if (browser.UseOttoMagic)
1032 {
1033 List<GPALElement> result = null;
1034
1035 if (false == webElement?.IsShadowRoot)
1036 browser.MagicHelper.SwitchToElement(webElement.Css);
1037
1038 if (true == isPersistent)
1039 result = browser.MagicHelper
1040 .EvaluateAllPersistent(selectorPath.SelectorPath);
1041 else
1042 result = browser.MagicHelper
1043 .EvaluateAll(selectorPath.SelectorPath);
1044
1045 if (null != result)
1046 GPALelems = new ReadOnlyCollection<GPALElement>(result);
1047 }
1048 else if (true == browser.UsePuppeteer)
1049 {
1050 if (false == webElement?.IsShadowRoot)
1051 browser.PuppeteerClient.InElement(webElement.Css).Execute();
1052
1053 var result2 = Task.Run(() => browser.PuppeteerCommunicator.EvaluateSelector(selectorPath.SelectorPath));
1054
1055 if (null != result2)
1056 GPALelems = new ReadOnlyCollection<GPALElement>(result2.Result);
1057 }
1058 else
1059 {
1060 string xpath = selectorPath.SelectorPath;
1061
1062 elems = webElement?.WebElement != null
1063 ? webElement.WebElement.FindElements(By.XPath(xpath))
1064 : browser.BrowserDriver.FindElements(By.XPath(xpath));
1065
1066 // closed shadow root - see the CSS search, the host is all we can offer, and never for a
1067 // persistent selector
1068 if (0 == elems.Count && true == webElement?.IsClosedShadowRoot && false == isPersistent)
1069 return new ReadOnlyCollection<GPALElement>(new List<GPALElement>() { webElement });
1070
1071 GPALelems = ToGPALElements(browser, selector, elems);
1072 }
1073
1074 return GPALelems;
1075 }
1092 internal static ReadOnlyCollection<GPALElement> ToGPALElements(Browser browser, Selector selector, ReadOnlyCollection<IWebElement> elems)
1093 {
1094 List<GPALElement> gPALElements = new List<GPALElement>();
1095
1096 if (0 < elems.Count)
1097 {
1098 string script = @"
1099 return arguments[0].map(function (element) {
1100 var rect = element.getBoundingClientRect();
1101 var style = window.getComputedStyle(element);
1102
1103 return {
1104 tag: element.tagName ? element.tagName.toLowerCase() : null,
1105 text: element.innerText,
1106 enabled: false === element.disabled || undefined === element.disabled,
1107 selected: true === element.checked || true === element.selected,
1108 indeterminate: true === element.indeterminate,
1109 displayed: (0 < element.getClientRects().length) && 'hidden' !== style.visibility && 'none' !== style.display,
1110 x: Math.round(rect.left + window.pageXOffset),
1111 y: Math.round(rect.top + window.pageYOffset),
1112 w: Math.round(rect.width),
1113 h: Math.round(rect.height),
1114 href: element.getAttribute('href'),
1115 src: element.getAttribute('src'),
1116 value: undefined !== element.value ? element.value : element.getAttribute('value'),
1117 placeholder: element.getAttribute('placeholder'),
1118 type: element.getAttribute('type')
1119 };
1120 });
1121 ";
1122
1123 string pageUrl = browser.BrowserDriver.Url;
1124 object answered = ((IJavaScriptExecutor)browser.BrowserDriver).ExecuteScript(script, elems.ToList());
1125 List<object> properties = (answered as IEnumerable<object>)?.ToList();
1126
1127 for (int idx = 0; idx < elems.Count; idx++)
1128 {
1129 // a page that could not answer for its own elements leaves the old path as the way out, one
1130 // element at a time, rather than handing back elements with nothing in them
1131 GPALElement gPALElement = properties?.Count == elems.Count && properties[idx] is Dictionary<string, object> bag
1132 ? new GPALElement(elems[idx], selector.AttributeName, bag, pageUrl)
1133 : new GPALElement(elems[idx], selector.AttributeName);
1134
1135 gPALElement.Browser = browser;
1136 gPALElement.Selector = selector;
1137
1138 gPALElements.Add(gPALElement);
1139 }
1140 }
1141
1142 return new ReadOnlyCollection<GPALElement>(gPALElements);
1143 }
1155 private static ReadOnlyCollection<GPALElement> FindWebElementsByText(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false)
1156 {
1157 bool searchForElement = selector.SearchForSelector;
1158 ReadOnlyCollection<IWebElement> elems;
1159 ReadOnlyCollection<GPALElement> GPALelems = null;
1160 string xpathSelector = $"//*[contains(text(), '{selectorPath.SelectorPath}')]"; // Same XPath as Selenium
1161
1162 if (browser.UseOttoMagic)
1163 {
1164 List<GPALElement> result = null;
1165
1166 if (false == webElement?.IsShadowRoot)
1167 browser.MagicHelper.SwitchToElement(webElement.Css);
1168
1169 if (true == isPersistent)
1170 result = browser.MagicHelper
1171 .EvaluateAllPersistent(xpathSelector);
1172 else
1173 result = browser.MagicHelper
1174 .EvaluateAll(xpathSelector);
1175
1176 if (null != result)
1177 GPALelems = new ReadOnlyCollection<GPALElement>(result);
1178 }
1179 else if (true == browser.UsePuppeteer)
1180 {
1181 if (false == webElement?.IsShadowRoot)
1182 browser.PuppeteerClient.InElement(webElement.Css).Execute();
1183
1184 var result2 = Task.Run(() => browser.PuppeteerCommunicator.EvaluateSelector(xpathSelector));
1185
1186 if (null != result2)
1187 GPALelems = new ReadOnlyCollection<GPALElement>(result2.Result);
1188 }
1189 else
1190 {
1191 if (webElement != null)
1192 xpathSelector = $".{xpathSelector}"; // Make relative XPath for webElement scope
1193
1194 elems = webElement?.WebElement != null
1195 ? webElement.WebElement.FindElements(By.XPath(xpathSelector))
1196 : browser.BrowserDriver.FindElements(By.XPath(xpathSelector));
1197
1198 GPALelems = ToGPALElements(browser, selector, elems);
1199 }
1200
1201
1202 return GPALelems;
1203 }
1214 private static ReadOnlyCollection<GPALElement> FindWebElementsByHRef(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false)
1215 {
1216 bool searchForElement = selector.SearchForSelector;
1217 ReadOnlyCollection<IWebElement> elems;
1218 ReadOnlyCollection<GPALElement> GPALelems = null;
1219 string cssSelector = $"a[href*='{selectorPath.SelectorPath}']"; // Matches <a> elements with href containing the substring
1220
1221 if (browser.UseOttoMagic)
1222 {
1223 List<GPALElement> result = null;
1224
1225 if (false == webElement?.IsShadowRoot)
1226 browser.MagicHelper.SwitchToElement(webElement.Css);
1227
1228 if (true == isPersistent)
1229 result = browser.MagicHelper
1230 .QueryPersistentSelectors(cssSelector);
1231 else
1232 result = browser.MagicHelper
1233 .QuerySelectors(cssSelector);
1234
1235 if (null != result)
1236 GPALelems = new ReadOnlyCollection<GPALElement>(result);
1237 }
1238 else if (true == browser.UsePuppeteer)
1239 {
1240 if (false == webElement?.IsShadowRoot)
1241 browser.PuppeteerClient.InElement(webElement.Css).Execute();
1242
1243 var result2 = Task.Run(() => browser.PuppeteerCommunicator.EvaluateSelector(cssSelector));
1244
1245 if (null != result2)
1246 GPALelems = new ReadOnlyCollection<GPALElement>(result2.Result);
1247 }
1248 else
1249 {
1250 elems = webElement?.WebElement != null
1251 ? webElement.WebElement.FindElements(By.PartialLinkText(selectorPath.SelectorPath))
1252 : browser.BrowserDriver.FindElements(By.PartialLinkText(selectorPath.SelectorPath));
1253
1254 GPALelems = new ReadOnlyCollection<GPALElement>(
1255 elems.Select(e => new GPALElement(e, selector.AttributeName) { Browser = browser, Selector = selector }).ToList()
1256 );
1257 }
1258
1259 return GPALelems;
1260 }
1271 private static ReadOnlyCollection<GPALElement> FindWebElementsByClassName(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false)
1272 {
1273 bool searchForElement = selector.SearchForSelector;
1274 ReadOnlyCollection<IWebElement> elems;
1275 ReadOnlyCollection<GPALElement> GPALelems = null;
1276 string cssSelector = $".{selectorPath.SelectorPath}";
1277
1278 if (browser.UseOttoMagic)
1279 {
1280 List<GPALElement> result = null;
1281
1282 if (false == webElement?.IsShadowRoot)
1283 browser.MagicHelper.SwitchToElement(webElement.Css);
1284
1285 if (true == isPersistent)
1286 result = browser.MagicHelper
1287 .QueryPersistentSelectors(cssSelector);
1288 else
1289 result = browser.MagicHelper
1290 .QuerySelectors(cssSelector);
1291
1292 if (null != result)
1293 GPALelems = new ReadOnlyCollection<GPALElement>(result);
1294 }
1295 else if (true == browser.UsePuppeteer)
1296 {
1297 if (false == webElement?.IsShadowRoot)
1298 browser.PuppeteerClient.InElement(webElement.Css).Execute();
1299
1300 var result2 = Task.Run(() => browser.PuppeteerCommunicator.EvaluateSelector(cssSelector));
1301 if (null != result2)
1302 GPALelems = new ReadOnlyCollection<GPALElement>(result2.Result);
1303 }
1304 else
1305 {
1306 elems = webElement?.WebElement != null
1307 ? webElement.WebElement.FindElements(By.ClassName(selectorPath.SelectorPath))
1308 : browser.BrowserDriver.FindElements(By.ClassName(selectorPath.SelectorPath));
1309
1310 GPALelems = new ReadOnlyCollection<GPALElement>(
1311 elems.Select(e => new GPALElement(e, selector.AttributeName) { Browser = browser, Selector = selector }).ToList()
1312 );
1313 }
1314
1315 return GPALelems;
1316 }
1327 private static ReadOnlyCollection<GPALElement> FindWebElementsByName(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false)
1328 {
1329 bool searchForElement = selector.SearchForSelector;
1330 ReadOnlyCollection<IWebElement> elems;
1331 ReadOnlyCollection<GPALElement> GPALelems = null;
1332 string cssSelector = $"[name='{selectorPath.SelectorPath}']";
1333
1334 if (browser.UseOttoMagic)
1335 {
1336 List<GPALElement> result = null;
1337
1338 if (false == webElement?.IsShadowRoot)
1339 browser.MagicHelper.SwitchToElement(webElement.Css);
1340
1341 if (true == isPersistent)
1342 result = browser.MagicHelper
1343 .QueryPersistentSelectors(cssSelector);
1344 else
1345 result = browser.MagicHelper
1346 .QuerySelectors(cssSelector);
1347
1348 if (null != result)
1349 GPALelems = new ReadOnlyCollection<GPALElement>(result);
1350 }
1351 else if (true == browser.UsePuppeteer)
1352 {
1353 if (false == webElement?.IsShadowRoot)
1354 browser.PuppeteerClient.InElement(webElement.Css).Execute();
1355
1356 var result2 = Task.Run(() => browser.PuppeteerCommunicator.EvaluateSelector(cssSelector));
1357
1358 if (null != result2)
1359 GPALelems = new ReadOnlyCollection<GPALElement>(result2.Result);
1360 }
1361 else
1362 {
1363 elems = webElement?.WebElement != null
1364 ? webElement.WebElement.FindElements(By.Name(selectorPath.SelectorPath))
1365 : browser.BrowserDriver.FindElements(By.Name(selectorPath.SelectorPath));
1366
1367 GPALelems = new ReadOnlyCollection<GPALElement>(
1368 elems.Select(e => new GPALElement(e, selector.AttributeName) { Browser = browser, Selector = selector }).ToList()
1369 );
1370 }
1371
1372 return GPALelems;
1373 }
1385 private static ReadOnlyCollection<GPALElement> FindWebElementsByValue(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false)
1386 {
1387 bool searchForElement = selector.SearchForSelector;
1388 ReadOnlyCollection<IWebElement> elems;
1389 ReadOnlyCollection<GPALElement> GPALelems = null;
1390 string cssSelector = $"[value='{selectorPath.SelectorPath}']";
1391
1392 if (browser.UseOttoMagic)
1393 {
1394 List<GPALElement> result = null;
1395
1396 if (false == webElement?.IsShadowRoot)
1397 browser.MagicHelper.SwitchToElement(webElement.Css);
1398
1399 if (true == isPersistent)
1400 result = browser.MagicHelper
1401 .QueryPersistentSelectors(cssSelector);
1402 else
1403 result = browser.MagicHelper
1404 .QuerySelectors(cssSelector);
1405
1406 if (null != result)
1407 GPALelems = new ReadOnlyCollection<GPALElement>(result);
1408 }
1409 else if (true == browser.UsePuppeteer)
1410 {
1411 if (false == webElement?.IsShadowRoot)
1412 browser.PuppeteerClient.InElement(webElement.Css).Execute();
1413
1414 var result2 = Task.Run(() => browser.PuppeteerCommunicator.EvaluateSelector(cssSelector));
1415 if (null != result2)
1416 GPALelems = new ReadOnlyCollection<GPALElement>(result2.Result);
1417 }
1418 else
1419 {
1420 elems = webElement?.WebElement != null
1421 ? webElement.WebElement.FindElements(By.CssSelector($"[value='{selectorPath.SelectorPath}']"))
1422 : browser.BrowserDriver.FindElements(By.CssSelector($"[value='{selectorPath.SelectorPath}']"));
1423
1424 GPALelems = new ReadOnlyCollection<GPALElement>(
1425 elems.Select(e => new GPALElement(e, selector.AttributeName) { Browser = browser, Selector = selector }).ToList()
1426 );
1427 }
1428
1429 return GPALelems;
1430 }
1441 private static ReadOnlyCollection<GPALElement> FindWebElementsByPlaceholder(Browser browser, SelectorPathEntry selectorPath, Selector selector, GPALElement webElement = null, bool isPersistent = false)
1442 {
1443 bool searchForElement = selector.SearchForSelector;
1444 ReadOnlyCollection<IWebElement> elems;
1445 ReadOnlyCollection<GPALElement> GPALelems = null;
1446 // backslashes and quotes escaped, since placeholders are prose and "What's on your mind?" would end the css string early
1447 string cssSelector = $"[placeholder='{selectorPath.SelectorPath.Replace("\\", "\\\\").Replace("'", "\\'")}']"; // Any element with placeholder
1448
1449 if (browser.UseOttoMagic)
1450 {
1451 List<GPALElement> result = null;
1452
1453 if (false == webElement?.IsShadowRoot)
1454 browser.MagicHelper.SwitchToElement(webElement.Css);
1455
1456 if (true == isPersistent)
1457 result = browser.MagicHelper
1458 .QueryPersistentSelectors(cssSelector);
1459 else
1460 result = browser.MagicHelper
1461 .QuerySelectors(cssSelector);
1462
1463 if (null != result)
1464 GPALelems = new ReadOnlyCollection<GPALElement>(result);
1465 }
1466 else if (true == browser.UsePuppeteer)
1467 {
1468 if (false == webElement?.IsShadowRoot)
1469 browser.PuppeteerClient.InElement(webElement.Css).Execute();
1470
1471 var result2 = Task.Run(() => browser.PuppeteerCommunicator.EvaluateSelector(cssSelector));
1472 if (null != result2)
1473 GPALelems = new ReadOnlyCollection<GPALElement>(result2.Result);
1474 }
1475 else
1476 {
1477 elems = webElement?.WebElement != null
1478 ? webElement.WebElement.FindElements(By.CssSelector(cssSelector))
1479 : browser.BrowserDriver.FindElements(By.CssSelector(cssSelector));
1480
1481 GPALelems = new ReadOnlyCollection<GPALElement>(
1482 elems.Select(e => new GPALElement(e, selector.AttributeName) { Browser = browser, Selector = selector }).ToList()
1483 );
1484 }
1485
1486 return GPALelems;
1487 }
1488
1498 public static void PublishToEventHandler(Browser browser, Selector selector, List<GPALElement> webElements)
1499 {
1500 string msg = "";
1501 if (true == selector.DeleteMe)
1502 msg = $"Ignoring selector [{selector.Name}] marked for deletion. Continuing.";
1503 else if (null == webElements || 0 == webElements.Count)
1504 msg = $"No elements found for selector [{selector.Name}]. Continuing.";
1505
1506 GPAL.PublishSimpleEvent(GPALEventType.WARNING, msg, browser, GPALObjectType.Browser);
1507 }
1518 public static List<GPALElement> WaitFor(Browser browser, int timeoutInTicks, out bool matchedAll)
1519 {
1520 bool matchedAll2 = true;
1521 List<GPALElement> foundElements = null;
1522 List<GPALElement> matchedElements;
1523
1524 foundElements = new List<GPALElement>();
1525
1526 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
1527 {
1528 if (SelectorType.Selector != selector.SelectorType)
1529 continue; // we don't look up data literals
1530
1531 WaitFor(browser, selector, timeoutInTicks, out matchedAll2, out matchedElements);
1532
1533
1534 // FindWebElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
1535 if (true == selector.DeleteMe || null == matchedElements)
1536 {
1537 PublishToEventHandler(browser, selector, matchedElements);
1538 continue;
1539 }
1540
1541 int rowCount = 0;
1542
1543 foreach (GPALElement webElement in matchedElements)
1544 {
1545 foundElements.Add(webElement);
1546
1547 if (++rowCount >= browser.CurrentUOW.WithAllThatMatch)
1548 break;
1549 }
1550 }
1551
1552 matchedAll = matchedAll2;
1553 return foundElements;
1554 }
1568 private static List<GPALElement> WaitFor(Browser browser, Selector selector, int timeoutInTicks, out bool matchedAll, out List<GPALElement> matchedElements, GPALElement webElement = null)
1569 {
1570 bool matchedAll2 = true;
1571 List<GPALElement> matchedElements2;
1572 ReadOnlyCollection<GPALElement> elements;
1573 int orgTimeoutInTicks = timeoutInTicks;
1574
1575 if (WaitTime.Forever == timeoutInTicks)
1576 timeoutInTicks = Int32.MaxValue; // forever
1577
1578 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Waiting up to [{timeoutInTicks}] ms for elements matching selector [{selector.Name}][{selector.SelectorPath}]", browser, GPALObjectType.Browser);
1579
1580 var sw = Stopwatch.StartNew();
1581 do
1582 {
1583 elements = FindWebElements(browser, browser.CurrentUOW, selector,
1584 out matchedAll2, out matchedElements2, webElement, false);
1585
1586 // we might've run out of time already and we could generate a negative sleep time
1587 if (0 == matchedElements2?.Count && WaitTime.Immediate != timeoutInTicks && sw.ElapsedMilliseconds < timeoutInTicks)
1588 Thread.Sleep((int)Math.Min(1000, timeoutInTicks - sw.ElapsedMilliseconds));
1589
1590 } while (sw.ElapsedMilliseconds < timeoutInTicks &&
1591 (matchedElements2 == null || matchedElements2.Count == 0));
1592
1593 matchedAll = matchedAll2; // one of those found it or not :)
1594 matchedElements = matchedElements2; // always use matched, it's either all or those matched
1595
1596 // element found, did we set a waitfor state as well?
1597 if (ElementState.NotSet != browser.CurrentUOW.WaitForElementState)
1598 foreach (GPALElement gPALElement in matchedElements2) // will be all if no match set
1599 matchedAll &= WaitFor(browser, gPALElement, browser.CurrentUOW.WaitForElementState, orgTimeoutInTicks);
1600
1601 return elements?.ToList<GPALElement>();
1602 }
1603
1612 public static bool WaitFor(Browser browser, GPALElement element, ElementState state, int timeoutInTicks)
1613 {
1614 const int defaultStateWaitInTicks = 3000;
1615
1616 if (WaitTime.Forever == timeoutInTicks)
1617 {
1618 timeoutInTicks = Int32.MaxValue;
1619 }
1620 else if (timeoutInTicks < 0)
1621 {
1622 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"WaitFor(ElementState.[{state}]) needs a positive wait time but got [{timeoutInTicks}] ms (no .WaitFor(timeInMs) was set for this unit of work) - using the default of [{defaultStateWaitInTicks}] ms instead.", element, GPALObjectType.Other);
1623 timeoutInTicks = defaultStateWaitInTicks;
1624 }
1625
1626 int retries = 2;
1627 bool retry = true;
1628 dynamic retryElement = element;
1629 int tmpTimeoutInTicks = timeoutInTicks;
1630
1631 do
1632 {
1633 try
1634 {
1635 int sleepTime = Math.Min(1000, tmpTimeoutInTicks);
1636
1637 Thread.Sleep(sleepTime);
1638
1639 if (IsElementInState(browser, element, state))
1640 return true;
1641
1642 tmpTimeoutInTicks -= sleepTime;
1643 }
1644 catch (StaleElementReferenceException sereex)
1645 {
1646 string message = $"StaleElementReferenceException while waiting for state [{state}]: [{sereex.Message}]";
1647 retries--;
1648 retry = ResolveWebElementIssue(browser, browser.CurrentUOW, ref retryElement, sereex, ref retries, message);
1649 if (retry)
1650 element = retryElement;
1651 }
1652 catch (GPALException)
1653 {
1654 throw;
1655 }
1656 catch (Exception ex)
1657 {
1658 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Error while waiting for state [{state}]", element, GPALObjectType.Other, ex);
1659 retry = false;
1660 }
1661 } while (tmpTimeoutInTicks > 0 && retry);
1662
1663 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Element not in state [{state}] within [{timeoutInTicks}] ms.", element, GPALObjectType.Other);
1664 return false;
1665 }
1666
1674 private static bool IsElementInState(Browser browser, GPALElement element, ElementState state)
1675 {
1676 try
1677 {
1678 switch (state)
1679 {
1680 case ElementState.Visible:
1681 return element.Displayed;
1682 case ElementState.Hidden:
1683 return !element.Displayed;
1684 case ElementState.Enabled:
1685 return element.Enabled;
1686 case ElementState.Disabled:
1687 return !element.Enabled;
1688 case ElementState.Checked:
1689 return element.Selected || element.GetAttribute("checked") == "true";
1690 case ElementState.Unchecked:
1691 return !element.Selected && element.GetAttribute("checked") != "true";
1692 case ElementState.Selected:
1693 return element.Selected;
1694 case ElementState.Unselected:
1695 return !element.Selected;
1696 case ElementState.Clickable:
1697 return IsElementClickable(element);
1698 case ElementState.Editable:
1699 return IsElementEditable(element);
1700 default:
1701 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Unsupported element state [{state}].", element, GPALObjectType.Other);
1702 return false;
1703 }
1704 }
1705 catch (GPALException)
1706 {
1707 throw;
1708 }
1709 catch (Exception ex)
1710 {
1711 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Error checking state [{state}]", element, GPALObjectType.Other, ex);
1712 return false;
1713 }
1714 }
1722 public static bool FillInFrom(Browser browser, string useText, WriteMode writeMode, GPALElement element = null)
1723 {
1724 bool retVal = false; // did we fill in anything? useful for SendString to try this before sending to <body>
1725 ReadOnlyCollection<GPALElement> elems = null;
1726 List<GPALElement> matchedElements = null;
1727
1728 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
1729 {
1730 if (SelectorType.Selector != selector.SelectorType)
1731 continue; // we don't look up data literals
1732
1733 if (null == element)
1734 elems = ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out bool matchedAll, out matchedElements);
1735
1736 // FindWebElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
1737 if (true == selector.DeleteMe || null == elems)
1738 {
1739 PublishToEventHandler(browser, selector, elems?.ToList());
1740 continue;
1741 }
1742
1743 if (null != element)
1744 {
1745 var tmpElems = new List<GPALElement>();
1746 tmpElems.Add(element);
1747 elems = new ReadOnlyCollection<GPALElement>(tmpElems);
1748 }
1749 else if (null != matchedElements)
1750 elems = new ReadOnlyCollection<GPALElement>(matchedElements);
1751
1752 if (true == elems.Any())
1753 handleFillIn(elems);
1754
1755 if (true == retVal && null != browser.CurrentUOW.CallAfterFillIn && true == elems.Any())
1756 {
1757 UnitOfWork safeUOW = browser.CurrentUOW;
1758 CallIfStatus handled = 0;
1759
1760 // CAVEAT: there is no concept of 'handled (1)' vs 'not handled (0)' but definitely can request to exit
1761 // call CallAfterFillIn, if defined and pass in the tokens we are currently using (which is just a string in this method)
1762 IGPALGrid<string> tokens = GPAL.GridForType<string>();
1763 tokens.AddRow(new List<string> { useText });
1764
1765 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Invoking CallAfterFillIn [{browser.CurrentUOW.CallAfterFillIn.Method.Name}]", browser, GPALObjectType.Browser);
1766 handled = browser.CurrentUOW.CallAfterFillIn(browser, tokens, 1);
1767
1768 // CAVEAT: kludge - the call after handler can and probably will set a new current unit of work, but we need our old current unit
1769 // the call after handler UOW is no longer in scope, so restore our UOW
1770 browser.CurrentUOW = safeUOW;
1771
1772 // CAVEAT: elements may go stale because of callafterfillin handler actions
1773 // test if elements still valid by testing the first one
1774 //try
1775 //{
1776 // ((UnitOfWork.ElementNode)(browser.CurrentUOW.ElementGrid[0])[0]).GPALElement.GetAttribute("tag");
1777 //}
1778 //catch // failed, invalidate elementgrid
1779 //{
1780 // browser.CurrentUOW.ElementGrid.Clear();
1781 //}
1782
1783 if (CallIfStatus.Terminate == handled)
1784 {
1785 string str = $"CallAfterFillIn handler [{browser.CurrentUOW.AppCallAfterFillIn.GetInvocationList()[0].Method.Name}] requested program termination.";
1786 GPAL.PublishSimpleEvent(GPALEventType.INFO, str, browser, GPALObjectType.Browser);
1787 throw new GPALException($"" + str);
1788 }
1789 }
1790
1791 void handleFillIn(ReadOnlyCollection<GPALElement> elems)
1792 {
1793 int rowCount = 0;
1794
1795 foreach (GPALElement elem in elems)
1796 {
1797 string type = elem.GetAttribute("type");
1798 string abbreviatedText = useText[0] + "..." + useText[useText.Length - 1];
1799
1800 if (null == type || false == type.Equals("password"))
1801 {
1802 if (GPALEventType.DEBUG == (GPALEventType.DEBUG & GPAL.GPALSettings.DebugEvents) || GPALEventType.DEBUG == (GPALEventType.DEBUG & GPAL.GPALSettings.ConsoleEvents))
1803 GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{writeMode}] text [{useText}] in element [{elem.TagName}][{selector.Name}]", browser, GPALObjectType.Browser);
1804 else
1805 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{writeMode}] text [{abbreviatedText}] in element [{elem.TagName}][{selector.Name}]", browser, GPALObjectType.Browser);
1806 }
1807 else
1808 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{writeMode}] XXXpasswordXXX in element [{elem.TagName}][{selector.Name}]", browser, GPALObjectType.Browser);
1809
1810 if (null != elem)
1811 {
1812 retVal = true;
1813
1814 // an image match with no element under it goes to the mouse only when hardware was asked
1815 // for, like any other element
1816 if (false == browser.BrowserSettings.UseHeadless && false == browser.BrowserSettings.HiddenDesktop
1817 && (("GPALElement" == elem.TagName && InteractionType.Hardware == selector.InteractionType)
1818 || true == browser.BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware))
1819 HardwareFillInFrom(browser, elem, selector.OffsetX, selector.OffsetY, useText, writeMode);
1820 else if ("GPALElement" == elem.TagName)
1821 // image match with no element under it - focus the coordinate and type
1822 FillInFrom(browser, ToViewportPoint(browser, elem, selector), useText, writeMode);
1823 else if (browser.UseOttoMagic)
1824 {
1825 if (false == browser.BrowserSettings.UseHeadless && InteractionType.Hardware == selector.InteractionType)
1826 ElementHelper.HardwareFillInFrom(browser, elem, selector.OffsetX, selector.OffsetY, useText, writeMode);
1827 else
1828 switch (writeMode)
1829 {
1830 case WriteMode.Append:
1831 browser.MagicHelper.FillInAppend(elem.Css, useText);
1832 break;
1833 case WriteMode.Insert:
1834 browser.MagicHelper.FillInInsert(elem.Css, useText);
1835 break;
1836 case WriteMode.Overwrite:
1837 browser.MagicHelper.FillInOverwrite(elem.Css, useText);
1838 break;
1839 }
1840
1841 // browser.MagicHelper.FireChangeEvent(elem.Css);
1842
1843 if (++rowCount >= browser.CurrentUOW.WithAllThatMatch)
1844 break;
1845 }
1846 else if (true == browser.UsePuppeteer)
1847 {
1848 if (false == browser.BrowserSettings.UseHeadless && InteractionType.Hardware == selector.InteractionType)
1849 ElementHelper.HardwareFillInFrom(browser, elem, selector.OffsetX, selector.OffsetY, useText, writeMode);
1850 else
1851 switch (writeMode)
1852 {
1853 case WriteMode.Append:
1854 browser.PuppeteerClient.FillInAppend(elem).WithText(useText).Execute();
1855 break;
1856 case WriteMode.Insert:
1857 browser.PuppeteerClient.FillInInsert(elem).WithText(useText).Execute();
1858 break;
1859 case WriteMode.Overwrite:
1860 browser.PuppeteerClient.FillInOverwrite(elem).WithText(useText).Execute();
1861 break;
1862 }
1863
1864 // browser.PuppeteerClient.FireChangeEvent(elem.ElementHandle).Execute();
1865
1866 if (++rowCount >= browser.CurrentUOW.WithAllThatMatch)
1867 break;
1868 }
1869 else if (true == browser.UseSelenium)
1870 {
1871 if (false == browser.BrowserSettings.UseHeadless && InteractionType.Hardware == selector.InteractionType)
1872 ElementHelper.HardwareFillInFrom(browser, elem, selector.OffsetX, selector.OffsetY, useText, writeMode);
1873 else
1874 SeleniumFillInFrom(browser, elem, useText, writeMode);
1875 }
1876 }
1877 }
1878 }
1879 }
1880
1881 return retVal;
1882 }
1890 public static void SeleniumSendKeys(Browser browser, GPALElement element, byte keyCode)
1891 {
1892 // Map the byte keycode to Selenium Keys constants
1893 string keyToSend;
1894
1895 switch (keyCode)
1896 {
1897 case 0x12: // VK_ALT
1898 keyToSend = Keys.Alt;
1899 break;
1900 case 0x2e: // VK_DELETE
1901 keyToSend = Keys.Delete;
1902 break;
1903 case 0x79: // VK_F10
1904 keyToSend = Keys.F10;
1905 break;
1906 case 0x21: // VK_PRIOR
1907 keyToSend = Keys.PageUp;
1908 break;
1909 case 0x22: // VK_NEXT
1910 keyToSend = Keys.PageDown;
1911 break;
1912 case 0x23: // VK_END
1913 keyToSend = Keys.End;
1914 break;
1915 case 0x24: // VK_HOME
1916 keyToSend = Keys.Home;
1917 break;
1918 case 0x28: // VK_DOWN
1919 keyToSend = Keys.ArrowDown;
1920 break;
1921 case 0x09: // VK_TAB
1922 keyToSend = Keys.Tab;
1923 break;
1924 case 0x0D: // VK_RETURN
1925 keyToSend = Keys.Return;
1926 break;
1927 case 0x27: // VK_SHIFT_RIGHT
1928 keyToSend = Keys.Shift;
1929 break;
1930 case 0xA0: // VK_SHIFT_LEFT
1931 keyToSend = Keys.LeftShift;
1932 break;
1933 case 0xA2: // VK_CONTROL_LEFT
1934 keyToSend = Keys.LeftControl;
1935 break;
1936 case 0xA3: // VK_CONTROL_RIGHT
1937 keyToSend = Keys.Control;
1938 break;
1939 case 0x20: // VK_SPACE
1940 keyToSend = Keys.Space;
1941 break;
1942 case 0x08: // VK_BACK
1943 keyToSend = Keys.Backspace;
1944 break;
1945 default:
1946 // If it's a character key, use fromCharCode method for alphanumeric keys
1947 keyToSend = ((char)keyCode).ToString();
1948 break;
1949 }
1950
1951 // Find the element to send keys to
1952 element.SendKeys(keyToSend);
1953 }
1954
1964 // the input types the browser renders as a widget rather than a text box. Typing into one means matching
1965 // that browser's segment order, separators and tab behavior, which differ between chrome, edge and
1966 // firefox and change with locale. Every one of them accepts the ISO value written to .value, which is what
1967 // OttoMagic and Puppeteer have always done, so Selenium does it that way too
1968 private static readonly HashSet<string> valueOnlyInputTypes = new HashSet<string>
1969 {
1970 "color", "date", "datetime-local", "month", "week", "time",
1971 };
1972 public static void SeleniumFillInFrom(Browser browser, GPALElement elem, string textToUse, WriteMode writeMode)
1973 {
1974 // the ISO value goes to .value untouched - GetKeystrokeValueForInput exists to reshape it for typing
1975 if (true == valueOnlyInputTypes.Contains(elem.Type))
1976 {
1977 JavaScriptFillInFrom(browser, elem, textToUse, writeMode);
1978 return;
1979 }
1980
1981 textToUse = ElementHelper.GetKeystrokeValueForInput(elem, textToUse);
1982
1983 try
1984 {
1985 if (WriteMode.Overwrite == writeMode)
1986 elem.Clear();
1987 else if (WriteMode.Append == writeMode)
1988 {
1989 textToUse = elem.Text + textToUse;
1990 elem.Clear();
1991 }
1992
1993 elem.SendKeys(textToUse); // fast path - whole string at once
1994 }
1995 catch (GPALException)
1996 {
1997 throw;
1998 }
1999 catch (Exception ex)
2000 {
2001 if (false == GPAL.NoFallbackRecoveryActions)
2002 {
2003 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, "Unable to fill in text. Falling back on javascript.", browser, GPALObjectType.Browser, ex);
2004 JavaScriptFillInFrom(browser, elem, textToUse, writeMode);
2005 }
2006 }
2007
2008 }
2009
2010
2020 public static void HardwareFillInFrom(Browser browser, GPALElement webElement, int offsetX, int offsetY, string text, WriteMode writeMode)
2021 {
2022 string inputType = webElement.Type?.ToLowerInvariant() ?? "";
2023
2024 // Special handling for native picker inputs
2025 bool isSpecialInput = new[] { "date", "time", "datetime-local", "month", "week", "color" }
2026 .Contains(inputType);
2027
2028 HardwareFocus(browser, webElement, offsetX, offsetY);
2029
2030 // If focusing the element could not be trusted (off-screen/odd coords, so no real hardware click
2031 // happened), do NOT type. Focus is likely still on whatever it was before - e.g. the browser's
2032 // address bar - and blasting keystrokes there is how a search term ends up in the URL bar.
2033 if (true == _lastPlacementQuestionable)
2034 {
2035 GPAL.PublishSimpleEvent(GPALEventType.WARNING,
2036 $"Skipping fill-in [{browser.CurrentUOW?.CurrentSelector?.Name}], questionable element placement.",
2037 browser, GPALObjectType.Browser);
2038 return;
2039 }
2040
2041 if (isSpecialInput)
2042 {
2043 // firefox does not consume Ctrl+A inside a segmented date or time widget, it hands it to the
2044 // document and the whole page ends up selected. the segments overwrite as they are typed anyway,
2045 // so the clear is only there for the browsers that want it
2046 if ("week" != inputType && BrowserType.FireFox != browser.BrowserSettings.BrowserType)
2047 {
2048 // Aggressive clear for special inputs (they can be picky)
2049 HardwareHelper.SendChar('a', ModifierKeys.Control, true);
2050 Thread.Sleep(120);
2051 HardwareHelper.SendKey(GPAL.VK_DELETE);
2052 Thread.Sleep(80);
2053 }
2054 }
2055 else if (WriteMode.Overwrite == writeMode)
2056 {
2057 HardwareHelper.SendChar('a', ModifierKeys.Control, true);
2058 Thread.Sleep(150);
2059 //HardwareHelper.SendKey(GPAL.VK_DELETE);
2060 }
2061 else if (WriteMode.Append == writeMode)
2062 {
2063 HardwareHelper.SendKey(GPAL.VK_END, ModifierKeys.Control);
2064 Thread.Sleep(150);
2065 }
2066 else if (WriteMode.Insert == writeMode)
2067 {
2068 HardwareHelper.SendKey(GPAL.VK_HOME, ModifierKeys.Control);
2069 Thread.Sleep(150);
2070 }
2071
2072 string textToType = GetKeystrokeValueForInput(webElement, text);
2073 switch (inputType)
2074 {
2075 case "color":
2076 // the picker takes the six hex digits, not the css value, and anything else falls back to black
2077 HardwareFillInColor(textToType.TrimStart('#'), browser.BrowserSettings.BrowserType);
2078 break;
2079 case "datetime-local":
2080 HardwareFillInDateTimeLocal(textToType, browser.BrowserSettings.BrowserType);
2081 break;
2082 case "week":
2083 HardwareFillInWeek(textToType);
2084 break;
2085 case "month":
2086 HardwareFillInMonth(textToType);
2087 break;
2088 default:
2089 HardwareHelper.SendString(textToType);
2090 break;
2091 }
2092 }
2093
2098 public static void HardwareFillInMonth(string isoValue)
2099 {
2100 var parts = isoValue.Split('/');
2101 string year = parts[0];
2102 string month = parts[1];
2103 HardwareHelper.SendString(month); // "12"
2104 // TAB to move to year field
2105 HardwareHelper.SendKey(GPAL.VK_TAB);
2106 HardwareHelper.SendString(year); // "2026"
2107 }
2108
2113 public static void HardwareFillInWeek(string isoValue)
2114 {
2115 if (isoValue.Contains("-W"))
2116 {
2117 var parts = isoValue.Split('-');
2118 string year = parts[0];
2119 string week = parts[1].TrimStart('W');
2120 HardwareHelper.SendString($"{week}{year}"); // "31 2026"
2121 }
2122 }
2123
2128 public static void HardwareFillInDateTimeLocal(string isoValue, BrowserType browserType)
2129 {
2130 // Convert ISO "2026-06-18T18:32" → US human format "06/18/2026 06:32 PM"
2131 if (!DateTime.TryParseExact(isoValue,
2132 "MM/dd/yyyy HH:mm",
2133 null,
2134 System.Globalization.DateTimeStyles.None,
2135 out DateTime dt))
2136 {
2137 dt = DateTime.Now; // fallback
2138 }
2139
2140 string datePart = dt.ToString("MM/dd/yyyy"); // US date
2141 string timePart = dt.ToString("hh:mm"); // 06:32 pm (lowercase a/p)
2142 string amPm = dt.ToString("tt");
2143
2144 // Type date
2145 HardwareHelper.SendString(datePart);
2146
2147 // firefox advances to the hour on its own once the year is typed, so tabbing here moves off the hour
2148 // and the time lands a segment late. the chromium widget still wants the tab
2149 if (BrowserType.FireFox != browserType)
2150 {
2151 // TAB to move to time field
2152 HardwareHelper.SendKey(GPAL.VK_TAB);
2153 Thread.Sleep(120);
2154 }
2155
2156 // Type time in 12-hour format
2157 HardwareHelper.SendString(timePart);
2158 Thread.Sleep(120);
2159
2160 // am/pm
2161 HardwareHelper.SendString(amPm);
2162 }
2163
2170 public static void HardwareFillInColor(string hexValue, BrowserType browserType)
2171 {
2172 if (hexValue.Length != 6)
2173 {
2174 // fallback or error
2175 hexValue = "000000";
2176 }
2177
2178 // Convert #RRGGBB → R, G, B decimals
2179 int r = Convert.ToInt32(hexValue.Substring(0, 2), 16);
2180 int g = Convert.ToInt32(hexValue.Substring(2, 2), 16);
2181 int b = Convert.ToInt32(hexValue.Substring(4, 2), 16);
2182
2183 // Navigate into the RGB fields. Three tabs reaches the first one in the chromium picker. Firefox lists
2184 // hue, saturation and luminance ahead of red, green and blue, so it takes four more to clear them
2185 int tabsToFirstField = BrowserType.FireFox == browserType ? 7 : 3;
2186
2187 for (int tab = 0; tab < tabsToFirstField; tab++)
2188 HardwareHelper.SendKey(GPAL.VK_TAB);
2189
2190 // Enter R
2191 HardwareHelper.SendString(r.ToString());
2192 HardwareHelper.SendKey(GPAL.VK_TAB);
2193
2194 // Enter G
2195 HardwareHelper.SendString(g.ToString());
2196 HardwareHelper.SendKey(GPAL.VK_TAB);
2197
2198 // Enter B
2199 HardwareHelper.SendString(b.ToString());
2200
2201 // Optional: Tab out + Enter to confirm / close picker
2202 //HardwareHelper.SendKey(GPAL.VK_TAB);
2203 HardwareHelper.SendKey(GPAL.VK_RETURN); // or VK_ENTER
2204 }
2211 public static string GetKeystrokeValueForInput(GPALElement webElement, string isoValue)
2212 {
2213 if (string.IsNullOrEmpty(isoValue) || webElement?.TagName?.ToLowerInvariant() != "input")
2214 return isoValue;
2215
2216 string type = webElement.Type?.ToLowerInvariant() ?? "";
2217
2218 try
2219 {
2220 switch (type)
2221 {
2222 case "date":
2223 // ISO: 2026-06-18 US typing: 06/18/2026
2224 if (DateTime.TryParseExact(isoValue, "yyyy-MM-dd", null, System.Globalization.DateTimeStyles.None, out DateTime dt))
2225 return dt.ToString("MM/dd/yyyy");
2226 return isoValue;
2227
2228 case "datetime-local":
2229 // ISO: 2026-06-18T18:32 US: 06/18/2026 18:32
2230 if (DateTime.TryParseExact(isoValue.Replace("T", " "), "yyyy-MM-dd HH:mm", null, System.Globalization.DateTimeStyles.None, out dt))
2231 return dt.ToString("MM/dd/yyyy HH:mm");
2232 return isoValue.Replace("T", " ");
2233
2234 case "month":
2235 // ISO: 2026-12 12/2026 (most browsers accept this when typing)
2236 return isoValue.Replace("-", "/");
2237
2238 case "week":
2239 // 2026-W31 keep as-is or try "31 2026" — Wxx format is usually required
2240 // Many browsers expect the ISO week format even when typing
2241 return isoValue; // safest starting point
2242
2243 case "time":
2244 // the value is HH:mm on its own, and the widget types a 12 hour clock with a meridiem
2245 // segment, so am/pm is worked out from the 24 hour value rather than supplied
2246 if (DateTime.TryParseExact(isoValue, new[] { "HH:mm", "H:mm", "HH:mm:ss" }, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None, out dt))
2247 return dt.ToString("hh:mm tt", System.Globalization.CultureInfo.InvariantCulture);
2248 return isoValue;
2249
2250 default:
2251 return isoValue;
2252 }
2253 }
2254 catch
2255 {
2256 return isoValue; // fallback
2257 }
2258 }
2259
2266 public static void JavaScriptFillInFrom(Browser browser, GPALElement webElement, string text, WriteMode writeMode)
2267 {
2268 IWebDriver browserDriver = browser.BrowserDriver;
2269
2270 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].focus();", webElement.WebElement);
2271
2272 // .value, not setAttribute('value'). The attribute is the field's default value: once anything has been
2273 // typed into the field the browser submits the property, so writing the attribute changes what the page
2274 // looks like and not what it sends
2275 if (WriteMode.Overwrite == writeMode)
2276 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].value = arguments[1]", webElement.WebElement, text);
2277 else if (WriteMode.Append == writeMode)
2278 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].value = arguments[0].value + arguments[1]", webElement.WebElement, text);
2279 else if (WriteMode.Insert == writeMode)
2280 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].value = arguments[1] + arguments[0].value", webElement.WebElement, text);
2281
2282 // typing raises these, so a page that keeps its own copy of the field only hears about the new text if
2283 // we raise them too. Without it the page keeps acting on whatever was in the box before
2284 // todo: make firing this an optional param
2285 BrowserHelper.ExecuteJavaScript(browser,
2286 "arguments[0].dispatchEvent(new Event('input', { bubbles: true })); arguments[0].dispatchEvent(new Event('change', { bubbles: true }));",
2287 webElement.WebElement);
2288 }
2295 public static async void PuppeteerFillInFrom(Browser browser, GPALElement webElement, string text, WriteMode writeMode)
2296 {
2297 BrowserSettings browserSettings = browser.BrowserSettings;
2298
2299 PuppeteerCommunicator puppeteerCommunicator = browserSettings.PuppeteerCommunicator;
2300
2301 // https://stackoverflow.com/questions/11337353/correct-way-to-focus-an-element-in-selenium-webdriver-using-java
2302 // puppeteerCommunicator.Focus(webElement.Css);
2303 switch (writeMode)
2304 {
2305 case WriteMode.Append:
2306 browser.PuppeteerClient.FillInAppend(webElement).WithText(text).Execute();
2307 break;
2308 case WriteMode.Insert:
2309 browser.PuppeteerClient.FillInInsert(webElement).WithText(text).Execute();
2310 break;
2311 case WriteMode.Overwrite:
2312 browser.PuppeteerClient.FillInOverwrite(webElement).WithText(text).Execute();
2313 break;
2314 }
2315 VkCodeToDomKeyConverter.TryConvertAsciiToVkCode('\t', out byte vkCode);
2316 VkCodeToDomKeyConverter.TryConvertVkCodeToDomKey(vkCode, out string key, out string code);
2317 await puppeteerCommunicator.SendKey(key, code, vkCode); // tab out, force change event to fire - todo: make firing this an optional param
2318 }
2319
2327 public static void JavaScriptSendKey(Browser browser, GPALElement webElement, byte keycode)
2328 {
2329 IWebDriver browserDriver = browser.BrowserDriver;
2330 if (true == VkCodeToDomKeyConverter.TryConvertVkCodeToDomKey(keycode, out string key, out string code))
2331 {
2332 string sendKeyScript = @"
2333 function isElementEditable(target) {
2334 if (!target || !(target instanceof Element)) return false;
2335 if (target.disabled || target.readOnly) return false;
2336
2337 const tag = target.tagName.toUpperCase();
2338 if (tag === 'TEXTAREA') return true;
2339 if (tag === 'SELECT') return true;
2340 if (tag === 'INPUT') {
2341 const editableTypes = new Set(['text','email','password','search','number','url','tel',
2342 'date','datetime-local','month','time','week','color']);
2343 return editableTypes.has(target.type.toLowerCase());
2344 }
2345 if (target.isContentEditable === true ||
2346 target.getAttribute('contenteditable') === 'true' ||
2347 target.ownerDocument?.designMode === 'on') {
2348 return true;
2349 }
2350 const style = window.getComputedStyle(target);
2351 if (style.userModify === 'read-write' || style.webkitUserModify === 'read-write' ||
2352 style.cursor === 'text') {
2353 return true;
2354 }
2355 return false;
2356 }
2357
2358 // === Parameters passed from C# ===
2359 const key = arguments[0]; // e.g. 'a', 'Enter', 'Tab'
2360 const code = arguments[1]; // e.g. 'KeyA', 'Enter'
2361 const keyCode = arguments[2]; // numeric keyCode if needed (mostly legacy)
2362
2363 let target = document.activeElement;
2364 if (!target || target === document.body) {
2365 target = document; // fallback
2366 }
2367
2368 const eventInit = {
2369 key: key,
2370 code: code,
2371 keyCode: keyCode,
2372 bubbles: true,
2373 cancelable: true,
2374 composed: true
2375 };
2376
2377 try {
2378 // 1. Key events (for listeners)
2379 target.dispatchEvent(new KeyboardEvent('keydown', eventInit));
2380
2381 if (key.length === 1) { // printable char
2382 target.dispatchEvent(new KeyboardEvent('keypress', eventInit));
2383 }
2384
2385 // 2. Actually insert text (this is what most sites need)
2386 const isEditable = isElementEditable(target);
2387 if (isEditable) {
2388 if (target.tagName === 'INPUT' || target.tagName === 'TEXTAREA') {
2389 const start = target.selectionStart ?? target.value.length;
2390 const end = target.selectionEnd ?? target.value.length;
2391 target.value = target.value.slice(0, start) + key + target.value.slice(end);
2392 target.selectionStart = target.selectionEnd = start + key.length;
2393 }
2394 else if (target.isContentEditable) {
2395 // Simpler fallback for contenteditable
2396 document.execCommand('insertText', false, key); // still works in many places
2397 }
2398
2399 // Fire input event (React, Vue, Angular, etc. listen to this)
2400 target.dispatchEvent(new InputEvent('input', {
2401 bubbles: true,
2402 cancelable: true,
2403 data: key,
2404 inputType: 'insertText',
2405 composed: true
2406 }));
2407
2408 // Also fire change for some frameworks
2409 target.dispatchEvent(new Event('change', { bubbles: true }));
2410 }
2411
2412 // 3. Keyup
2413 target.dispatchEvent(new KeyboardEvent('keyup', eventInit));
2414
2415 // Special case: Enter on button/submit
2416 if (key === 'Enter' && (target.tagName === 'BUTTON' ||
2417 (target.tagName === 'INPUT' && (target.type === 'submit' || target.type === 'button')))) {
2418 target.click();
2419 }
2420
2421 } catch (e) {
2422 console.error('SendKey script error:', e);
2423 }
2424 ";
2425
2426 // Call it like this:
2427 BrowserHelper.ExecuteJavaScript(browser, sendKeyScript, key, code, keycode.ToString()); // example for 'A'
2428 }
2429 }
2438 public static void Focus(Browser browser)
2439 {
2440 List<GPALElement> matchedElements;
2441
2442 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
2443 {
2444 if (SelectorType.Selector != selector.SelectorType)
2445 continue; // we don't look up data literals
2446
2447 ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out bool matchedAll, out matchedElements);
2448
2449 // FindWebElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
2450 if (true == selector.DeleteMe || null == matchedElements)
2451 {
2452 PublishToEventHandler(browser, selector, matchedElements?.ToList());
2453 continue;
2454 }
2455
2456 int rowCount = 0;
2457
2458 foreach (GPALElement elem in matchedElements)
2459 {
2460 if (null != elem)
2461 {
2462 if (false == browser.BrowserSettings.UseHeadless && (true == browser.BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware))
2463 {
2464 HardwareFocus(browser, elem, selector.OffsetX, selector.OffsetY);
2465 }
2466 else if (true == IsElementEditable(elem))
2467 {
2468 // click to focus and place the caret, readying the element for typing
2469 Click(browser, selector, elem, ClickType.LeftClick, ModifierKeys.NONE);
2470 }
2471 else if (true == browser.UseOttoMagic)
2472 {
2473 browser.MagicHelper.Focus(elem.Css);
2474 }
2475 else if (true == browser.UsePuppeteer)
2476 {
2477 browser.PuppeteerClient.Focus(elem.Css).Execute();
2478 }
2479 else if (true == browser.UseSelenium)
2480 {
2481 if (Enums.InteractionType.JavaScript == selector.InteractionType || true == browser.BrowserSettings.UseJavaScript) // selenium
2482 {
2483 JavaScriptFocus(browser, elem);
2484 }
2485 else if (elem.TagName.ToLower().Equals("input"))
2486 {
2487 elem.SendKeys(OpenQA.Selenium.Keys.Shift);
2488 elem.SendKeys("");
2489 }
2490 else
2491 {
2492 try
2493 {
2494 new Actions(browser.BrowserDriver).MoveToElement((IWebElement)elem.WebElement).Perform();
2495 }
2496 catch (GPALException)
2497 {
2498 throw;
2499 }
2500 catch (Exception ex)
2501 {
2502 // the recovery arm says what it did instead, so it is a warning. The other
2503 // arm recovers from nothing, because the workflow turned recovery off, so
2504 // Selenium throwing is the whole outcome and it is published as one
2505 if (false == GPAL.NoFallbackRecoveryActions)
2506 {
2507 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selenium threw on selector [{selector.Name}], falling back to javascript.", browser, GPALObjectType.Browser);
2508 ElementHelper.ScrollIntoView(browser, elem);
2509 }
2510 else
2511 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Selenium threw on selector [{selector.Name}], and recovery actions are off, so nothing was done about it.", browser, GPALObjectType.Browser, ex);
2512 }
2513 }
2514 }
2515
2516 if (++rowCount >= browser.CurrentUOW.WithAllThatMatch)
2517 break;
2518 }
2519 }
2520 }
2521 }
2528 private static void JavaScriptFocus(Browser browser, GPALElement webElement)
2529 {
2530 IWebDriver browserDriver = browser.BrowserDriver;
2531
2532 string forceFocusScript = $@"
2533 function forceFocus(el) {{
2534 // Remember original state
2535 const wasDisabled = el.disabled;
2536 const originalTabIndex = el.getAttribute('tabindex');
2537
2538 try {{
2539 // 1. Enable if disabled
2540 if (wasDisabled) el.disabled = false;
2541
2542 // 2. Make sure it can receive focus
2543 if (el.tabIndex === -1 || !el.hasAttribute('tabindex')) {{
2544 el.setAttribute('tabindex', '0');
2545 }}
2546
2547 // 3. Focus it
2548 el.focus({{ preventScroll: true }});
2549
2550 // 4. Small delay + aggressive fallback (helps with buttons, React, Shadow DOM, etc.)
2551 setTimeout(() => {{
2552 if (document.activeElement !== el) {{
2553 el.focus();
2554
2555 // Dispatch synthetic focus events
2556 el.dispatchEvent(new FocusEvent('focusin', {{ bubbles: true }}));
2557 el.dispatchEvent(new FocusEvent('focus', {{ bubbles: false }}));
2558 }}
2559 }}, 0);
2560
2561 // Final check
2562 return document.activeElement === el;
2563
2564 }} catch (err) {{
2565 return false;
2566 }} finally {{
2567 // Restore original state
2568 if (wasDisabled) el.disabled = true;
2569 if (originalTabIndex !== null) {{
2570 el.setAttribute('tabindex', originalTabIndex);
2571 }} else if (el.getAttribute('tabindex') === '0') {{
2572 el.removeAttribute('tabindex');
2573 }}
2574 }}
2575 }}
2576 forceFocus(arguments[0]);
2577 ";
2578 BrowserHelper.ExecuteJavaScript(browser, forceFocusScript, webElement.WebElement); // focus on element
2579 }
2584 public static void Hide(Browser browser)
2585 {
2586 List<GPALElement> matchedElements;
2587
2588 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
2589 {
2590 if (SelectorType.Selector != selector.SelectorType)
2591 continue; // we don't look up data literals
2592
2593 ReadOnlyCollection<GPALElement> elems = ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out bool matchedAll, out matchedElements);
2594
2595 // FindWebElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
2596 if (true == selector.DeleteMe || null == elems || 0 == elems.Count)
2597 {
2598 PublishToEventHandler(browser, selector, elems?.ToList());
2599 continue;
2600 }
2601
2602 if (null != matchedElements)
2603 elems = new ReadOnlyCollection<GPALElement>(matchedElements);
2604
2605 foreach (GPALElement elem in elems)
2606 Hide(browser, elem);
2607 }
2608 }
2615 public static void Hide(Browser browser, GPALElement element)
2616 {
2617 if (true == browser.UseOttoMagic)
2618 browser.MagicHelper.HideElement(element.Css);
2619 else if (true == browser.UsePuppeteer)
2620 browser.PuppeteerClient.HideElement(element.ElementBackendNodeId.ToString());
2621 else
2622 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].style.display = 'none';", element.WebElement);
2623
2624 }
2631 public static void SetAttribute(Browser browser, string attribute, string value)
2632 {
2633 List<GPALElement> matchedElements;
2634
2635 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
2636 {
2637 if (SelectorType.Selector != selector.SelectorType)
2638 continue; // we don't look up data literals
2639
2640 ReadOnlyCollection<GPALElement> elems = ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out bool matchedAll, out matchedElements);
2641
2642 // FindWebElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
2643 if (true == selector.DeleteMe || null == elems || 0 == elems.Count)
2644 {
2645 PublishToEventHandler(browser, selector, elems?.ToList());
2646 continue;
2647 }
2648
2649 if (null != matchedElements)
2650 elems = new ReadOnlyCollection<GPALElement>(matchedElements);
2651
2652 foreach (GPALElement elem in elems)
2653 SetAttribute(browser, elem, attribute, value);
2654 }
2655 }
2665 public static void SetAttribute(Browser browser, GPALElement element, string attribute, string value)
2666 {
2667 if (true == browser.UseOttoMagic)
2668 browser.MagicHelper.SetAttribute(element.Css, attribute, value);
2669 else if (true == browser.UsePuppeteer)
2670 browser.PuppeteerClient.SetAttribute(element.ElementBackendNodeId.ToString()).WithAttribute(attribute).WithValue(value).Execute();
2671 else
2672 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].setAttribute(arguments[1], arguments[2]);", element.WebElement, attribute, value);
2673 }
2680 public static void SetValueFromElement(Browser browser, Selector srcSelector)
2681 {
2682 List<GPALElement> matchedElements;
2683
2684 ReadOnlyCollection<GPALElement> srcElems = ElementHelper.FindWebElements(browser, browser.CurrentUOW, srcSelector, out bool srcMatchedAll, out List<GPALElement> srcMatchedElements);
2685
2686 if (null != srcMatchedElements)
2687 srcElems = new ReadOnlyCollection<GPALElement>(srcMatchedElements);
2688
2689 if (null == srcElems || 0 == srcElems.Count)
2690 return;
2691
2692 GPALElement srcElem = srcElems[0];
2693
2694 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
2695 {
2696 if (SelectorType.Selector != selector.SelectorType)
2697 continue; // we don't look up data literals
2698
2699 ReadOnlyCollection<GPALElement> elems = ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out bool matchedAll, out matchedElements);
2700
2701 // FindWebElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
2702 if (true == selector.DeleteMe || null == elems || 0 == elems.Count)
2703 {
2704 PublishToEventHandler(browser, selector, elems?.ToList());
2705 continue;
2706 }
2707
2708 if (null != matchedElements)
2709 elems = new ReadOnlyCollection<GPALElement>(matchedElements);
2710
2711 foreach (GPALElement elem in elems)
2712 SetValueFromElement(browser, elem, srcElem);
2713 }
2714 }
2723 public static void SetValueFromElement(Browser browser, GPALElement element, GPALElement srcElement)
2724 {
2725 if (true == browser.UseOttoMagic)
2726 browser.MagicHelper.SetValueFromElement(srcElement.Css, element.Css);
2727 else if (true == browser.UsePuppeteer)
2728 browser.PuppeteerClient.WithElementId(srcElement.Css).SetValueFrom(element.Css).Execute();
2729 else
2730 BrowserHelper.ExecuteJavaScript(browser,
2731 "arguments[0].value = arguments[1].value; arguments[0].dispatchEvent(new Event('input', {bubbles: true})); arguments[0].dispatchEvent(new Event('change', {bubbles: true}));",
2732 element.WebElement, srcElement.WebElement);
2733 }
2752 public static void MoveTo(Browser browser)
2753 {
2754 List<GPALElement> matchedElements;
2755 UnitOfWork currentUOW = browser.CurrentUOW;
2756
2757 foreach (Selector selector in currentUOW.WithSelectorList)
2758 {
2759 if (SelectorType.Selector != selector.SelectorType)
2760 continue; // we don't look up data literals
2761
2762 ReadOnlyCollection<GPALElement> elems = ElementHelper.FindWebElements(browser, currentUOW, selector, out bool matchedAll, out matchedElements);
2763
2764 // FindWebElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
2765 if (true == selector.DeleteMe || null == elems || 0 == elems.Count)
2766 {
2767 PublishToEventHandler(browser, selector, elems?.ToList());
2768 continue;
2769 }
2770
2771 if (null != matchedElements)
2772 elems = new ReadOnlyCollection<GPALElement>(matchedElements);
2773
2774 int rowCount = 0;
2775
2776 foreach (GPALElement found in elems)
2777 {
2778 if (null == found)
2779 continue;
2780
2781 // an element found under an image is hovered where the picture is, like a click. OttoMagic
2782 // keeps the element, since it can move to an element but not to a point
2783 GPALElement elem = null != found.ImageMatch && false == browser.UseOttoMagic ? found.ImageMatch : found;
2784
2785 ScrollIntoView(browser, elem);
2786
2787 if (0 == elem.Size.Width && 0 == elem.Size.Height)
2788 {
2789 GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Element has Zero Width and Height, skipping.", browser, GPALObjectType.Browser);
2790 continue;
2791 }
2792
2793 // an image match still standing here has no element under it, only a point. it goes to the
2794 // mouse when hardware was asked for, like any other element, and on OttoMagic, which has no
2795 // way to move to a point. selector level .WithHardware is honored here too, matching what
2796 // DragAndDrop already does.
2797 if (false == browser.BrowserSettings.UseHeadless
2798 && (("GPALElement" == elem.TagName && true == browser.UseOttoMagic)
2799 || true == browser.BrowserSettings.UseHardware
2800 || true == GPAL.GPALSettings.UseHardware
2801 || InteractionType.Hardware == selector.InteractionType))
2802 {
2803 HardwareMoveTo(browser, elem, selector);
2804 }
2805 else if ("GPALElement" == elem.TagName)
2806 {
2807 // headless image match - no cursor to move and no handle to hover, so drive the engine's
2808 // own mouse at the coordinate instead
2809 MoveTo(browser, elem, selector);
2810 }
2811 else if (true == browser.UseOttoMagic || true == browser.UsePuppeteer)
2812 {
2813 MoveTo(elem, selector.OffsetX, selector.OffsetY, browser);
2814 }
2815 else if (Enums.InteractionType.JavaScript == selector.InteractionType || true == browser.BrowserSettings.UseJavaScript || true == browser.BrowserSettings.UseHeadless)
2816 {
2817 if (null != currentUOW.ShadowRoot)
2818 JavaScriptHover(browser, currentUOW.ShadowRoot);
2819 else
2820 JavaScriptHover(browser, elem);
2821 }
2822 else // selenium with a real cursor
2823 {
2824 MoveTo(elem, selector.OffsetX, selector.OffsetY, browser);
2825 }
2826
2827 if (++rowCount >= browser.CurrentUOW.WithAllThatMatch)
2828 break;
2829 }
2830 }
2831 }
2832
2842 private static void MoveTo(Browser browser, GPALElement element, Selector selector)
2843 {
2844 Point point = ToViewportPoint(browser, element, selector);
2845
2846 if (true == browser.UsePuppeteer)
2847 browser.PuppeteerClient.MoveTo(point).Execute();
2848 else if (true == browser.UseSelenium)
2849 new Actions(browser.BrowserDriver).MoveToLocation(point.X, point.Y).Perform();
2850 else
2851 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Cannot hover the image match for selector [{selector.Name}] - OttoMagic has no coordinate mouse move and there is no hardware mouse available.", browser, GPALObjectType.Browser);
2852 }
2853 // per-thread so concurrent browsers do not share a generator, and seeded from a Guid because the
2854 // parameterless Random seeds off a low resolution clock - constructed back to back it repeats itself
2855 [ThreadStatic] private static Random _jitter;
2856 private static Random Jitter => _jitter ?? (_jitter = new Random(Guid.NewGuid().GetHashCode()));
2865 private static Point ImageCenterOffset(Size size)
2866 {
2867 // never jitter so far that a tiny match is missed
2868 int jitterX = Math.Max(0, Math.Min(2, (size.Width / 2) - 1));
2869 int jitterY = Math.Max(0, Math.Min(2, (size.Height / 2) - 1));
2870
2871 return new Point((size.Width / 2) + Jitter.Next(-jitterX, jitterX + 1),
2872 (size.Height / 2) + Jitter.Next(-jitterY, jitterY + 1));
2873 }
2882 internal static string CssForXPath(Browser browser, Selector selector)
2883 {
2884 string retVal = null;
2885 string xpath = selector.SelectorPaths[0].SelectorPath;
2886
2887 if (true == browser.UseOttoMagic)
2888 {
2889 ReadOnlyCollection<GPALElement> found = FindWebElementsByXPath(browser, selector.SelectorPaths[0], selector);
2890
2891 retVal = 0 < (found?.Count ?? 0) ? found[0].Css : null;
2892 }
2893 else
2894 {
2895 // the XPath goes in as a JSON string, so its quotes cannot end the script's own
2896 string css = BrowserHelper.ExecuteJavaScript(browser, $@"
2897 return (function (xpath) {{
2898 var el = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
2899 var path = [];
2900
2901 while (el && el.nodeType === Node.ELEMENT_NODE) {{
2902 if (el.id) {{
2903 path.unshift('#' + CSS.escape(el.id));
2904 break;
2905 }}
2906
2907 var part = el.nodeName.toLowerCase(), sibling = el, nth = 1;
2908
2909 while ((sibling = sibling.previousElementSibling) != null)
2910 if (sibling.nodeName === el.nodeName)
2911 nth++;
2912
2913 path.unshift(part + ':nth-of-type(' + nth + ')');
2914 el = el.parentElement;
2915 }}
2916
2917 return path.join(' > ');
2918 }})({Newtonsoft.Json.JsonConvert.SerializeObject(xpath)});");
2919
2920 retVal = false == string.IsNullOrEmpty(css) ? css : null;
2921 }
2922
2923 if (null == retVal)
2924 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"XPath selector [{selector.Name}] matched nothing on the page, so it was left out", browser, GPALObjectType.Browser);
2925
2926 return retVal;
2927 }
2928
2940 internal static string CssUnderImage(Browser browser, Selector selector)
2941 {
2942 string retVal = null;
2943 ReadOnlyCollection<GPALElement> found = new ReadOnlyCollection<GPALElement>(selector.WebSelectorFoundResults);
2944 GPALElement first = 0 < (found?.Count ?? 0) ? found[0] : null;
2945
2946 // headless and hidden desktop runs have already resolved the match to its element
2947 if (null != first && "GPALElement" == first.TagName)
2948 {
2949 ReadOnlyCollection<GPALElement> under = ElementUnderImage(browser, first, selector);
2950
2951 first = 0 < (under?.Count ?? 0) ? under[0] : null;
2952 }
2953
2954 if (false == string.IsNullOrEmpty(first?.Css))
2955 retVal = first.Css;
2956 else
2957 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Image selector [{selector.Name}] found no element of its own on the page, so it was left out", browser, GPALObjectType.Browser);
2958
2959 return retVal;
2960 }
2961
2972 internal static string UrlUnderImage(GPALElement match)
2973 {
2974 string retVal = null;
2975 Browser browser = match.Browser;
2976
2977 if (null == browser)
2978 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"The url behind image match [{match.Selector?.Name}] cannot be read, since the match carries no browser to ask", null, GPALObjectType.None);
2979 else
2980 {
2981 Point aim = ToViewportPoint(browser, match, match.Selector);
2982
2983 // the numbers go into the script rather than as arguments, since the engines pass arguments differently
2984 string url = RunPageScript(browser, $@"
2985 (function (x, y) {{
2986 var el = document.elementFromPoint(x, y);
2987
2988 if (!el)
2989 return '';
2990
2991 var link = el.closest('a[href], area[href]');
2992
2993 return link ? link.href : (el.currentSrc || el.src || el.data || '');
2994 }})({aim.X}, {aim.Y})");
2995
2996 if (false == string.IsNullOrEmpty(url))
2997 retVal = url;
2998 }
2999
3000 return retVal;
3001 }
3002
3013 private static string RunPageScript(Browser browser, string expression)
3014 {
3015 // trimmed, because a return followed by a line break returns nothing
3016 string retVal = BrowserHelper.ExecuteJavaScript(browser, true == browser.UseSelenium ? "return " + expression.Trim() : expression);
3017
3018 if (true == browser.UseOttoMagic && true == retVal?.TrimStart().StartsWith("{\"error\""))
3019 {
3020 if (false == _userScriptsWarned)
3021 {
3022 _userScriptsWarned = true;
3023 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"OttoMagic could not run a page script [{retVal}]. Turn on Allow user scripts for the extension to use image matches as elements.", browser, GPALObjectType.Browser);
3024 }
3025
3026 retVal = null;
3027 }
3028 else if ("fail" == retVal)
3029 retVal = null;
3030
3031 return retVal;
3032 }
3033
3034 // RunPageScript says once per run that OttoMagic cannot run scripts, rather than on every image match
3035 private static bool _userScriptsWarned;
3036
3053 private static ReadOnlyCollection<GPALElement> ElementUnderImage(Browser browser, GPALElement match, Selector selector)
3054 {
3055 ReadOnlyCollection<GPALElement> retVal = null;
3056 Point aim = ToViewportPoint(browser, match, selector);
3057
3058 // the numbers go into the script rather than as arguments, since the engines pass arguments differently
3059 string css = RunPageScript(browser, $@"
3060 (function (x, y) {{
3061 var el = document.elementFromPoint(x, y);
3062
3063 if (!el || 'IFRAME' === el.tagName || 'FRAME' === el.tagName || el.shadowRoot)
3064 return '';
3065
3066 var actionable = el.closest('a[href], area[href], button, input, select, textarea, label, summary, ' +
3067 '[role=button], [role=link], [role=menuitem], [role=menuitemcheckbox], [role=menuitemradio], ' +
3068 '[role=tab], [role=checkbox], [role=radio], [role=switch], [role=option], [role=treeitem], ' +
3069 '[onclick], [contenteditable]:not([contenteditable=false])');
3070
3071 if (actionable)
3072 el = actionable;
3073
3074 var path = [];
3075
3076 while (el && el.nodeType === Node.ELEMENT_NODE) {{
3077 if (el.id) {{
3078 path.unshift('#' + CSS.escape(el.id));
3079 break;
3080 }}
3081
3082 var part = el.nodeName.toLowerCase(), sibling = el, nth = 1;
3083
3084 while ((sibling = sibling.previousElementSibling) != null)
3085 if (sibling.nodeName === el.nodeName)
3086 nth++;
3087
3088 path.unshift(part + ':nth-of-type(' + nth + ')');
3089 el = el.parentElement;
3090 }}
3091
3092 return path.join(' > ');
3093 }})({aim.X}, {aim.Y})");
3094
3095 if (false == string.IsNullOrEmpty(css))
3096 {
3097 Selector cssSel = GPAL.Selector.WithCSS(css).ToGPALObject();
3098
3099 retVal = FindWebElementsByCss(browser, cssSel.SelectorPaths[0], cssSel);
3100
3101 if (0 < (retVal?.Count ?? 0))
3102 {
3103 retVal[0].Css = css;
3104 retVal[0].ImageMatch = match;
3105 }
3106 }
3107
3108 GPAL.PublishSimpleEvent(GPALEventType.DEBUG, 0 < retVal?.Count
3109 ? $"Image match for [{selector.Name}] is element [{css}], clicked at the matched point"
3110 : $"Image match for [{selector.Name}] has no element of its own under it, so only the point is used", browser, GPALObjectType.Browser);
3111
3112 return retVal;
3113 }
3114
3129 private static bool PointIsClear(Browser browser, GPALElement resolved, Selector selector, ref Point aim)
3130 {
3131 bool retVal = true;
3132
3133 string answer = RunPageScript(browser, $@"
3134 (function (css, x, y) {{
3135 var el = document.querySelector(css);
3136
3137 if (!el)
3138 return 'gone';
3139
3140 function hits(px, py) {{
3141 var hit = document.elementFromPoint(px, py);
3142 return !!hit && (hit === el || el.contains(hit));
3143 }}
3144
3145 function named(px, py) {{
3146 var hit = document.elementFromPoint(px, py);
3147
3148 if (!hit)
3149 return 'nothing';
3150
3151 var classes = 'string' === typeof hit.className ? hit.className.trim().split(/\s+/).filter(Boolean).slice(0, 3).join('.') : '';
3152
3153 return hit.tagName.toLowerCase() + (hit.id ? '#' + hit.id : '') + (classes ? '.' + classes : '');
3154 }}
3155
3156 if (hits(x, y))
3157 return 'clear|0|0|';
3158
3159 var covering = named(x, y);
3160 var before = el.getBoundingClientRect();
3161
3162 el.scrollIntoView({{ block: 'center', inline: 'nearest', behavior: 'instant' }});
3163
3164 var after = el.getBoundingClientRect();
3165 var dx = Math.round(after.left - before.left), dy = Math.round(after.top - before.top);
3166
3167 return (hits(x + dx, y + dy) ? 'centred' : 'covered') + '|' + dx + '|' + dy + '|' + covering;
3168 }})({Newtonsoft.Json.JsonConvert.SerializeObject(resolved.Css)}, {aim.X}, {aim.Y})");
3169
3170 // no answer, or the element gone from the page, leaves the point to be clicked as it is
3171 string[] parts = answer?.Split(new[] { '|' }, 4);
3172
3173 if (4 == parts?.Length && ("centred" == parts[0] || "covered" == parts[0]))
3174 {
3175 aim = new Point(aim.X + Convert.ToInt32(parts[1]), aim.Y + Convert.ToInt32(parts[2]));
3176
3177 if ("centred" == parts[0])
3178 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{parts[3]}] covered selector [{selector.Name}] where its picture was, so the element was moved to the middle of the window and is clicked there", browser, GPALObjectType.Browser);
3179 else
3180 {
3181 retVal = false;
3182
3183 GPAL.PublishSimpleEvent(GPALEventType.WARNING, true == GPAL.NoFallbackRecoveryActions
3184 ? $"[{parts[3]}] covers selector [{selector.Name}] even with it in the middle of the window, and recovery actions are off, so the point is clicked and [{parts[3]}] may take the click"
3185 : $"[{parts[3]}] covers selector [{selector.Name}] even with it in the middle of the window, so the element itself is clicked instead of its picture", browser, GPALObjectType.Browser);
3186 }
3187 }
3188
3189 return retVal;
3190 }
3191
3202 private static void ClickCoveredElement(Browser browser, GPALElement element, ClickType clickType, ModifierKeys modifierKeys)
3203 {
3204 if (true == browser.UseOttoMagic)
3205 {
3206 if (ModifierKeys.NONE != modifierKeys)
3207 browser.MagicHelper.PressModifierKey(modifierKeys);
3208
3209 switch (clickType)
3210 {
3211 case ClickType.LeftDoubleClick:
3212 browser.MagicHelper.LeftDoubleClick(element.Css);
3213 break;
3214 case ClickType.MiddleClick:
3215 browser.MagicHelper.MiddleClick(element.Css);
3216 break;
3217 case ClickType.RightClick:
3218 browser.MagicHelper.RightClick(element.Css);
3219 break;
3220 default:
3221 browser.MagicHelper.LeftClick(element.Css);
3222 break;
3223 }
3224
3225 if (ModifierKeys.NONE != modifierKeys)
3226 browser.MagicHelper.ReleaseModifierKey(modifierKeys);
3227 }
3228 else
3229 JavaScriptClick(browser, element, clickType, modifierKeys);
3230 }
3231
3241 private static Point ToViewportPoint(Browser browser, GPALElement element, Selector selector)
3242 {
3243 // both zero means nothing was asked for, same rule the hardware path uses
3244 Point aim = (0 == selector.OffsetX && 0 == selector.OffsetY)
3245 ? ImageCenterOffset(element.Size)
3246 : new Point(selector.OffsetX, selector.OffsetY);
3247
3248 int pointX = element.Location.X + aim.X;
3249 int pointY = element.Location.Y + aim.Y;
3250
3251 if (CoordinateSpace.Screen == element.CoordinateSpace)
3252 {
3253 Rectangle content = GetWindowRectangle(browser);
3254 pointX -= content.X;
3255 pointY -= content.Y;
3256 }
3257
3258 return new Point(pointX, pointY);
3259 }
3268 private static void Click(Browser browser, Point point, ClickType clickType, ModifierKeys modifierKeys)
3269 {
3270 // OttoMagic's trusted click is a point and a button, with no keys held. a modified click at a point,
3271 // on a hidden desktop or headless where the mouse cannot go, is not something it can make, so it says
3272 // so rather than making a plain click that reads as the one asked for
3273 if (true == browser.UseOttoMagic && ModifierKeys.NONE != modifierKeys
3274 && (ClickType.LeftClick == clickType || ClickType.MiddleClick == clickType))
3275 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{Browser.GetModifierKeys(modifierKeys)}] cannot be held for a [{clickType}] at a point on OttoMagic, whose trusted click carries no keys, so the click is made without them.", browser, GPALObjectType.Browser);
3276
3277 if (true == browser.UsePuppeteer)
3278 {
3279 switch (clickType)
3280 {
3281 case ClickType.LeftClick:
3282 browser.PuppeteerClient.LeftClick(point).WithModifiers(modifierKeys).Execute();
3283 break;
3284 case ClickType.LeftDoubleClick:
3285 browser.PuppeteerClient.LeftDoubleClick(point).WithModifiers(modifierKeys).Execute();
3286 break;
3287 case ClickType.MiddleClick:
3288 browser.PuppeteerClient.MiddleClick(point).WithModifiers(modifierKeys).Execute();
3289 break;
3290 case ClickType.RightClick:
3291 browser.PuppeteerClient.RightClick(point).WithModifiers(modifierKeys).Execute();
3292 break;
3293 }
3294 }
3295 else if (true == browser.UseSelenium && ClickType.MiddleClick == clickType && browser.BrowserDriver is OpenQA.Selenium.Chromium.ChromiumDriver chromium)
3296 {
3297 // selenium's Actions has no middle button. chrome and edge take the same input commands Puppeteer and
3298 // OttoMagic send, through the driver's own devtools channel, so the middle click is made there: move,
3299 // press with the middle button held, release. a middle click on a link opens it in a new tab
3300 int cdpModifiers = (int)(modifierKeys & (ModifierKeys.Alt | ModifierKeys.Control | ModifierKeys.Windows | ModifierKeys.Shift));
3301
3302 chromium.ExecuteCdpCommand("Input.dispatchMouseEvent", new Dictionary<string, object>
3303 {
3304 ["type"] = "mouseMoved", ["x"] = point.X, ["y"] = point.Y, ["button"] = "none"
3305 });
3306 chromium.ExecuteCdpCommand("Input.dispatchMouseEvent", new Dictionary<string, object>
3307 {
3308 ["type"] = "mousePressed", ["x"] = point.X, ["y"] = point.Y, ["button"] = "middle", ["buttons"] = 4, ["clickCount"] = 1, ["modifiers"] = cdpModifiers
3309 });
3310 chromium.ExecuteCdpCommand("Input.dispatchMouseEvent", new Dictionary<string, object>
3311 {
3312 ["type"] = "mouseReleased", ["x"] = point.X, ["y"] = point.Y, ["button"] = "middle", ["buttons"] = 0, ["clickCount"] = 1, ["modifiers"] = cdpModifiers
3313 });
3314 }
3315 else if (true == browser.UseSelenium)
3316 {
3317 Actions action = new Actions(browser.BrowserDriver).MoveToLocation(point.X, point.Y);
3318
3319 if (modifierKeys.HasFlag(ModifierKeys.Alt))
3320 action.KeyDown(OpenQA.Selenium.Keys.LeftAlt).KeyDown(OpenQA.Selenium.Keys.Alt);
3321 if (modifierKeys.HasFlag(ModifierKeys.Control))
3322 action.KeyDown(OpenQA.Selenium.Keys.LeftControl).KeyDown(OpenQA.Selenium.Keys.Control);
3323 if (modifierKeys.HasFlag(ModifierKeys.Shift))
3324 action.KeyDown(OpenQA.Selenium.Keys.LeftShift).KeyDown(OpenQA.Selenium.Keys.Shift);
3325 if (modifierKeys.HasFlag(ModifierKeys.Windows))
3326 action.KeyDown(OpenQA.Selenium.Keys.Meta);
3327
3328 switch (clickType)
3329 {
3330 case ClickType.LeftClick:
3331 action.Click();
3332 break;
3333 case ClickType.LeftDoubleClick:
3334 action.DoubleClick();
3335 break;
3336 case ClickType.RightClick:
3337 action.ContextClick();
3338 break;
3339 default:
3340 // selenium's Actions has no middle click to bind to
3341 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{clickType}] on an image match by coordinate is not supported by Selenium, performing a left click.", browser, GPALObjectType.Browser);
3342 action.Click();
3343 break;
3344 }
3345
3346 if (modifierKeys.HasFlag(ModifierKeys.Alt))
3347 action.KeyUp(OpenQA.Selenium.Keys.LeftAlt).KeyUp(OpenQA.Selenium.Keys.Alt);
3348 if (modifierKeys.HasFlag(ModifierKeys.Control))
3349 action.KeyUp(OpenQA.Selenium.Keys.LeftControl).KeyUp(OpenQA.Selenium.Keys.Control);
3350 if (modifierKeys.HasFlag(ModifierKeys.Shift))
3351 action.KeyUp(OpenQA.Selenium.Keys.LeftShift).KeyUp(OpenQA.Selenium.Keys.Shift);
3352 if (modifierKeys.HasFlag(ModifierKeys.Windows))
3353 action.KeyUp(OpenQA.Selenium.Keys.Meta);
3354
3355 ModifierKeys unsupported = modifierKeys & (ModifierKeys.Application | ModifierKeys.ScrollLock);
3356 if (ModifierKeys.NONE != unsupported)
3357 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{clickType}]: [{Browser.GetModifierKeys(unsupported)}] is not supported via Selenium/WebDriver Actions and was ignored.", browser, GPALObjectType.Browser);
3358
3359 action.Build().Perform();
3360 }
3361 else if (ClickType.LeftClick != clickType && ClickType.MiddleClick != clickType)
3362 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{clickType}] on an image is not supported by OttoMagic, which clicks a point with the left or middle button only.", browser, GPALObjectType.Browser);
3363 else if (true == (ClickType.MiddleClick == clickType
3364 ? browser.MagicHelper.TrustedMiddleClick(point.X, point.Y)
3365 : browser.MagicHelper.TrustedLeftClick(point.X, point.Y)))
3366 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[Trusted][{clickType}] clicked at [{point.X},{point.Y}].", browser, GPALObjectType.Browser);
3367 else
3368 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[Trusted][{clickType}] at [{point.X},{point.Y}] was not made.", browser, GPALObjectType.Browser);
3369 }
3379 private static void FillInFrom(Browser browser, Point point, string text, WriteMode writeMode)
3380 {
3381 // there is no handle to set a value on, so focus it the way a person would and type
3382 Click(browser, point, ClickType.LeftClick, ModifierKeys.NONE);
3383
3384 if (true == browser.UsePuppeteer)
3385 {
3386 if (WriteMode.Overwrite == writeMode)
3387 GPAL.PublishSimpleEvent(GPALEventType.WARNING, "[Overwrite] cannot clear an image match by coordinate, the text will be added to whatever is already there.", browser, GPALObjectType.Browser);
3388
3389 browser.PuppeteerClient.SendString(text).Execute();
3390 }
3391 else if (true == browser.UseSelenium)
3392 {
3393 Actions actions = new Actions(browser.BrowserDriver);
3394
3395 if (WriteMode.Overwrite == writeMode)
3396 actions.KeyDown(Keys.Control).SendKeys("a").KeyUp(Keys.Control);
3397
3398 actions.SendKeys(text).Perform();
3399 }
3400 else
3401 {
3402 // the trusted click focused it and the extension has no call that types into the focused element, so
3403 // the text is written into it through execute-javascript, which needs Allow user scripts on. the
3404 // native value setter is used rather than .value so frameworks that track the value see the change,
3405 // and input and change are fired the way typing fires them. shadow roots are followed down to what
3406 // actually has focus
3407 string answer = RunPageScript(browser, $@"
3408 (function (text, mode) {{
3409 var el = document.activeElement;
3410
3411 while (el && el.shadowRoot && el.shadowRoot.activeElement)
3412 el = el.shadowRoot.activeElement;
3413
3414 if (!el || el === document.body)
3415 return 'nothing has focus';
3416
3417 if (el.isContentEditable) {{
3418 if ('Overwrite' === mode)
3419 document.execCommand('selectAll', false, null);
3420 else if ('Append' === mode) {{
3421 var range = document.createRange();
3422 range.selectNodeContents(el);
3423 range.collapse(false);
3424 var selection = window.getSelection();
3425 selection.removeAllRanges();
3426 selection.addRange(range);
3427 }}
3428
3429 document.execCommand('insertText', false, text);
3430 return 'ok';
3431 }}
3432
3433 var descriptor = Object.getOwnPropertyDescriptor(Object.getPrototypeOf(el), 'value');
3434
3435 if (!descriptor || !descriptor.set)
3436 return 'the focused ' + el.tagName.toLowerCase() + ' takes no text';
3437
3438 var current = el.value || '';
3439 var next = 'Overwrite' === mode ? text : current + text;
3440
3441 if ('Insert' === mode && typeof el.selectionStart === 'number' && null !== el.selectionStart)
3442 next = current.slice(0, el.selectionStart) + text + current.slice(el.selectionEnd);
3443
3444 descriptor.set.call(el, next);
3445 el.dispatchEvent(new Event('input', {{ bubbles: true }}));
3446 el.dispatchEvent(new Event('change', {{ bubbles: true }}));
3447 return 'ok';
3448 }})({Newtonsoft.Json.JsonConvert.SerializeObject(text)}, {Newtonsoft.Json.JsonConvert.SerializeObject(writeMode.ToString())})");
3449
3450 if ("ok" != answer)
3451 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Cannot fill in the image match by coordinate. OttoMagic focused it with a trusted click, and could not write into it [{answer ?? "no answer"}].", browser, GPALObjectType.Browser);
3452 }
3453 }
3462 private static void MoveTo(GPALElement element, int offsetX, int offsetY, Browser browser)
3463 {
3464 try
3465 {
3466 if (true == browser.UseOttoMagic)
3467 {
3468 browser.MagicHelper.MoveTo(element.Css);
3469 }
3470 else if (true == browser.UsePuppeteer)
3471 {
3472 browser.PuppeteerClient.MoveTo(element.ElementHandle).Execute();
3473 }
3474 else
3475 new Actions(browser.BrowserDriver).MoveToElement(element.WebElement, offsetX, offsetY).Perform();
3476 }
3477 catch (GPALException)
3478 {
3479 throw;
3480 }
3481 catch (Exception ex)
3482 {
3483 if (false == GPAL.NoFallbackRecoveryActions)
3484 {
3485 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selenium threw during move to Selector [{browser.CurrentUOW.CurrentSelector?.Name}], falling back to javascript.", browser, GPALObjectType.Browser);
3486 ElementHelper.ScrollIntoView(browser, element);
3487 }
3488 else
3489 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Selenium threw during move to Selector [{browser.CurrentUOW.CurrentSelector?.Name}], and recovery actions are off, so nothing was done about it.", browser, GPALObjectType.Browser, ex);
3490
3491 }
3492
3493 }
3502 private static void HardwareFocus(Browser browser, GPALElement webElement, int OffsetX, int OffsetY)
3503 {
3504 // focusing means putting the element where a real mouse can reach it, so the scroll belongs here rather
3505 // than in every caller. below the fold the element's screen position is a large positive Y that passes
3506 // the coordinate check in HardwareClick and lands on whatever is down there instead
3507 ScrollIntoView(browser, webElement, Enums.InteractionType.Hardware);
3508
3509 // IsElementEditable only counts the text-ish inputs, and the picker inputs take typing just as much.
3510 // without the click they never get focus, and the Ctrl+A that follows selects the whole document
3511 bool isPickerInput = valueOnlyInputTypes.Contains(webElement.Type?.ToLowerInvariant() ?? "");
3512
3513 if (true == ElementHelper.IsElementEditable(webElement) || true == isPickerInput)
3514 {
3515 // Safe / desired: click to focus (includes text inputs, textareas, contenteditable, etc.)
3516 HardwareClick(browser, webElement, ClickType.LeftClick, ModifierKeys.NONE, browser.CurrentUOW.CurrentSelector);
3517 }
3518 else
3519 {
3520 // Only move — avoid activating menus, toggles, navigation, submits, file pickers
3521 HardwareMoveTo(browser, webElement, browser.CurrentUOW.CurrentSelector);
3522 }
3523 }
3532 private static Rectangle HardwareMoveTo(Browser browser, dynamic element, Selector selector)
3533 {
3534 Rectangle content = new Rectangle(), iframeRect = new Rectangle(0, 0, 0, 0);
3535 bool isImageMatch = "GPALElement" == (string)element.TagName;
3536
3537 // ScrollIntoView re-reads the position when it scrolls, and otto and selenium find elements in document
3538 // coordinates, which hold until the page moves. puppeteer finds them in viewport coordinates and the
3539 // scroll is taken off below, so it is always re-read into document coordinates here
3540 if (false == isImageMatch && true == browser.UsePuppeteer)
3541 RefreshLocation(browser, (GPALElement)element);
3542
3543 Rectangle rect2 = new Rectangle(element.Location, element.Size);
3544 int offsetX = selector.OffsetX;
3545 int offsetY = selector.OffsetY;
3546
3547 // selenium reports IWebElement.Location in document coordinates, so it does not move when the page
3548 // scrolls - an element scrolled into view still reports where it sat at the top of the document, and
3549 // the mouse lands a full scroll offset away. the scroll the element was measured against rides along
3550 // on the element, so take it off to get where it actually sits in the viewport. an engine that reports
3551 // viewport coordinates sends no scroll, so nothing comes off those.
3552 Point scroll = Point.Empty;
3553 if (false == isImageMatch)
3554 {
3555 int pagex = 0;
3556 int pagey = 0;
3557
3558 if (true == browser.UseOttoMagic)
3559 {
3560 pagex = browser.MagicHelper.WindowPageOffsetX();
3561 pagey = browser.MagicHelper.WindowPageOffsetY();
3562 scroll = new Point(pagex, pagey);
3563 }
3564 else if (true == browser.UsePuppeteer)
3565 {
3566 pagex = browser.PuppeteerClient.WindowPageOffsetX().Execute<int>();
3567 pagey = browser.PuppeteerClient.WindowPageOffsetY().Execute<int>();
3568 scroll = new Point(pagex, pagey);
3569 }
3570 else
3571 {
3572 scroll = GetPageOffset(browser);
3573 }
3574 }
3575
3576 // find the absolute screen position of the webelement
3577 // if it's a posuedo GPALElement, then it is an image match and those coords are already absolute
3578 // iframe or shadowdom nesting, get parent coords
3579 if (0 < browser.CurrentUOW.ContextPath?.Count)
3580 {
3581 foreach (UnitOfWork.WebElementWithType contextType in browser.CurrentUOW.ContextPath)
3582 {
3583 if (ElementType.IFrame == contextType.elementType)
3584 {
3585 ClientRectangle tmpRect = contextType.gPalElement.BoundingRect;
3586 iframeRect.X += (int)contextType.gPalElement.Location.X;
3587 iframeRect.Y += (int)contextType.gPalElement.Location.Y;
3588 }
3589 }
3590 }
3591
3592 // if the click/hover offset is not specified, pick a point inside the element
3593 if (0 == offsetX && 0 == offsetY)
3594 {
3595 if ("GPALElement" == (string)element.TagName)
3596 {
3597 // an image match points at one specific graphic, so aim at the middle of it
3598 Point center = ImageCenterOffset(rect2.Size);
3599 offsetX = center.X;
3600 offsetY = center.Y;
3601 }
3602 else
3603 {
3604 // the middle, give or take. a point anywhere in the element lands on padding, a border or a
3605 // child that swallows the click, and the middle is where every other engine and every person
3606 // aims. the deviation keeps the exact same pixel from being hit every time, and is small
3607 // enough on a narrow element that it cannot walk off the middle of it
3608 int spreadX = Math.Max(1, (int)rect2.Width / 10);
3609 int spreadY = Math.Max(1, (int)rect2.Height / 10);
3610
3611 offsetX = (int)rect2.Width / 2 + Jitter.Next(-spreadX, spreadX + 1);
3612 offsetY = (int)rect2.Height / 2 + Jitter.Next(-spreadY, spreadY + 1);
3613 }
3614 }
3615
3616 content = GetWindowRectangle(browser);
3617 _lastContentArea = content;
3618
3619 // an image-matched surrogate carries whichever origin its capture used. a screen capture is already
3620 // absolute and needs nothing; a visible-tab capture is viewport relative and has to be shifted by the
3621 // content area origin to become a screen coordinate. display scaling is assumed to be 100%.
3622 Point imageOrigin = Point.Empty;
3623
3624 if ("GPALElement" == (string)element.TagName && CoordinateSpace.Viewport == ((GPALElement)element).CoordinateSpace)
3625 imageOrigin = new Point(content.X, content.Y);
3626
3627 // === FINAL SCREEN COORDS
3628 int screenX = (int)(("GPALElement" == element.TagName)
3629 ? rect2.Left + offsetX + imageOrigin.X
3630 : rect2.Left + offsetX + iframeRect.X + content.X - scroll.X);
3631
3632 int screenY = (int)(("GPALElement" == element.TagName)
3633 ? rect2.Top + offsetY + imageOrigin.Y
3634 : rect2.Top + offsetY + iframeRect.Y + content.Y - scroll.Y);
3635
3636 // an element bigger than the window cannot be scrolled fully on screen, and its left edge - the point
3637 // computed above - sits outside it. a file input stretched across its button is the usual one. only
3638 // here is the point adjusted, and only into the element's own visible area, so it is still the element
3639 // being clicked. an ordinary element that is merely scrolled off gets scrolled to instead
3640 if ("GPALElement" != (string)element.TagName
3641 && true == IsBiggerThanViewport(new Size((int)rect2.Width, (int)rect2.Height), content)
3642 && false == content.Contains(screenX, screenY))
3643 {
3644 // from where the element starts, not from the point being clicked. screenX already has offsetX
3645 // added, so measuring the overlap from there describes a rectangle half an element away from the
3646 // one on the page, and the part it calls visible is not the element's
3647 Point origin = new Point(screenX - offsetX, screenY - offsetY);
3648 Rectangle onScreen = Rectangle.Intersect(new Rectangle(origin.X, origin.Y, (int)rect2.Width, (int)rect2.Height), content);
3649
3650 if (false == onScreen.IsEmpty)
3651 {
3652 // the middle of what is on screen is a guess, and being inside the element's rectangle is not
3653 // the same as being reachable: an oversized element is mostly not its styled control, and
3654 // something else is often on top. so ask the page who would receive each point and take the
3655 // first that answers with this element. the middle is tried first because it is the point we
3656 // wanted anyway, and on an ordinary element it answers yes and nothing else is tried
3657 bool verified = false;
3658 int tryY = onScreen.Y + onScreen.Height / 2; // vertically it fits, so the middle is the middle
3659 List<int> candidates = new List<int>();
3660
3661 foreach (int inset in hitTestInsets)
3662 if (inset < onScreen.Width)
3663 candidates.Add(onScreen.Right - inset);
3664
3665 foreach (float fraction in hitTestFractions)
3666 candidates.Add(onScreen.X + (int)(onScreen.Width * fraction));
3667
3668 foreach (int tryX in candidates)
3669 if (true == PointHitsElement(browser, (GPALElement)element, new Point(tryX, tryY)))
3670 {
3671 screenX = tryX;
3672 screenY = tryY;
3673 verified = true;
3674 break;
3675 }
3676
3677 if (false == verified)
3678 {
3679 // nothing in the element's visible region belongs to it, so this is a click that is going
3680 // to miss whatever we do. the middle is as good a guess as any and saying so beats a
3681 // silent miss
3682 screenX = onScreen.X + onScreen.Width / 2;
3683 screenY = onScreen.Y + onScreen.Height / 2;
3684 }
3685
3686 // said as an offset from the element, which is the form WithOffset takes, so a workflow that
3687 // finds this point works can keep it and stop needing the correction. it is the click that
3688 // reports it, not this - working out a point is not clicking one
3689 _lastClampedPoint = $"at [WithOffset({screenX - origin.X}, {screenY - origin.Y})]{(true == verified ? "" : ", which nothing on this element answered to")}, for window [{_lastOuterWindow.X}, {_lastOuterWindow.Y}, {_lastOuterWindow.Width}, {_lastOuterWindow.Height}], because the element measures [{(int)rect2.Width}x{(int)rect2.Height}], larger than the window";
3690 }
3691 }
3692
3693 // === MOVE MOUSE ===
3694 if (true == GPAL.SimulateMouse || true == selector.SimulateMouse)
3695 HardwareHelper.MoveMouse(screenX, screenY, 10, 10);
3696 else
3697 HardwareHelper.MoveMouse(screenX, screenY);
3698
3699 return new Rectangle(screenX, screenY, (int)rect2.Width, (int)rect2.Height);
3700 }
3708 internal static Point GetPageOffset(Browser browser)
3709 {
3710 Point pageOffset = Point.Empty;
3711
3712 if (true == browser.UseOttoMagic)
3713 pageOffset = new Point(browser.MagicHelper.WindowPageOffsetX(), browser.MagicHelper.WindowPageOffsetY());
3714 else if (true == browser.UsePuppeteer)
3715 pageOffset = new Point(browser.PuppeteerClient.WindowPageOffsetX().Execute<int>(), browser.PuppeteerClient.WindowPageOffsetY().Execute<int>());
3716 else if (BrowserHelper.ExecuteJavaScriptObj("return [window.top.pageXOffset, window.top.pageYOffset]", browser) is IList<object> offsets && 2 <= offsets.Count)
3717 pageOffset = new Point(Convert.ToInt32(offsets[0]), Convert.ToInt32(offsets[1]));
3718
3719 return pageOffset;
3720 }
3730 private static bool UsesScreenCoordinates(Browser browser, Enums.InteractionType? interactionType)
3731 {
3732 Enums.InteractionType? useType = interactionType ?? browser.CurrentUOW?.CurrentSelector?.InteractionType;
3733
3734 return true == browser.BrowserSettings.UseHardware
3735 || true == GPAL.GPALSettings.UseHardware
3736 || Enums.InteractionType.Hardware == useType;
3737 }
3748 internal static void ScrollIntoView(Browser browser, dynamic element, Enums.InteractionType? interactionType = null, bool forVisibility = false)
3749 {
3750 if ("GPALElement".Equals(((GPALElement)element).TagName))
3751 return; // image surrogate, it's already on-screen if matched
3752
3753 // ottomagic works entirely through javascript against the css, so where the element sits on screen
3754 // makes no difference to it. only the hw engines and a hardware selector click a real coordinate.
3755 // a scroll is the exception: the caller asked for that element to move, so it has to be somewhere
3756 // the caller can see it move
3757 if (true == browser.UseOttoMagic && false == forVisibility && false == UsesScreenCoordinates(browser, interactionType))
3758 return;
3759
3760 // an element already wholly on screen is left where it is. scrolling it anyway costs the scroll, the
3761 // settle wait, the visibility check after it and a re-read of where it went, on every click
3762 if (true == IsVisibleInViewport(browser, (GPALElement)element))
3763 return;
3764
3765 if (true == browser.UseOttoMagic)
3766 browser.MagicHelper.ScrollIntoView(((GPALElement)element).Css);
3767 else if (true == browser.UsePuppeteer)
3768 browser.PuppeteerClient.ScrollIntoView((GPALElement)element).Execute();
3769 else if (null != element) // NOTE: CAVEAT: BUG: this is hiding a bug in selenium getgrid scrollintoview of the element... (??? is this relevant any more?)
3770 {
3771 string debounceScript = @"
3772 // Best JS replacement for scrollIntoViewIfNeeded - do not scroll to if it's already on-screen
3773 function scrollIntoViewIfNeeded(el) {
3774 return new Promise(resolve => {
3775 const io = new IntersectionObserver(([entry]) => {
3776 io.disconnect();
3777 if (entry.intersectionRatio < 1) {
3778 el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'center' });
3779 }
3780 resolve();
3781 }, { threshold: 1 });
3782 io.observe(el);
3783 });
3784 }
3785
3786 scrollIntoViewIfNeeded(arguments[0]);
3787 ";
3788
3789 BrowserHelper.ExecuteJavaScript(browser, debounceScript, typeof(GPALElement) == element.GetType() ? ((GPALElement)element).WebElement : element);
3790 //BrowserHelper.ExecuteJavaScript(browser, "arguments[0].scrollIntoView()", element.WebElement);
3791 }
3792 Thread.Sleep(250); // give the dom time to catch up before we start asking about it, we apparently can ask for page offset faster than it ha it...
3793
3794 // every scroll above moves the document, which cannot move a fixed or sticky element, so any of them
3795 // can return having changed nothing while the element is still off-screen. page the window to it
3796 ScrollUntilVisible(browser, (GPALElement)element);
3797
3798 // the scroll moved the element, so the position read when it was found is stale for anything that
3799 // clicks a coordinate. puppeteer is re-read in HardwareMoveTo every time, since it finds elements in
3800 // viewport coordinates, so it is not read twice here
3801 if (false == browser.UsePuppeteer && true == UsesScreenCoordinates(browser, interactionType))
3802 RefreshLocation(browser, (GPALElement)element);
3803 }
3812 public static void LeftClick(Browser browser, ModifierKeys modifierKeys = ModifierKeys.NONE, ClickType clickType = ClickType.LeftClick, bool trusted = false)
3813 {
3814 List<GPALElement> matchedElements;
3815 List<GPALElement> clickableElements;
3816
3817 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
3818 {
3819 if (SelectorType.Selector != selector.SelectorType)
3820 continue; // we don't look up data literals
3821
3822 ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out bool matchedAll, out matchedElements);
3823
3824 // FindWebElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
3825 if (true == selector.DeleteMe || null == matchedElements)
3826 {
3827 PublishToEventHandler(browser, selector, matchedElements);
3828 continue;
3829 }
3830
3831 // NOTE: _ALWAYS_ used matched elements, it will be the same as found elements if no match criteris
3832 // if match critieria, we want to use it cause it will be matched elems or none
3833
3834 if (1 < matchedElements.Count)
3835 {
3836 clickableElements = new List<GPALElement>();
3837 foreach (GPALElement elem in matchedElements)
3838 if (true == elem.IsClickable(false)) // no output from isClickable
3839 clickableElements.Add(elem);
3840
3841 if (0 == clickableElements.Count)
3842 clickableElements = matchedElements;
3843 }
3844 else
3845 clickableElements = matchedElements;
3846
3847 int rowCount = 0;
3848
3849 foreach (GPALElement elem in clickableElements)
3850 {
3851 if (null != elem)
3852 {
3853 if (true == elem.TagName.ToLower().Equals("option"))
3854 SelectClick(browser, selector, elem, SelectClickType.LeftClick, modifierKeys);
3855 else
3856 Click(browser, selector, elem, clickType, modifierKeys, trusted);
3857
3858 if (++rowCount >= browser.CurrentUOW.WithAllThatMatch)
3859 break;
3860 }
3861 }
3862
3863 //Thread.Sleep(500);
3864 }
3865 }
3870 public static void MiddleClick(Browser browser)
3871 {
3872 LeftClick(browser, ModifierKeys.NONE, ClickType.MiddleClick);
3873 }
3874
3882 public static void ScrollElement(Browser browser, int amount, ScrollTypes scrollTypes)
3883 {
3884 List<GPALElement> matchedElements;
3885 bool matchedAll = false;
3886
3887 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
3888 {
3889 if (SelectorType.Selector != selector.SelectorType)
3890 continue; // we don't look up data literals
3891
3892 ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out matchedAll, out matchedElements);
3893
3894 if (null != matchedElements)
3895 {
3896 int rowCount = 0;
3897
3898 foreach (GPALElement elem in matchedElements)
3899 {
3900 if (null != elem)
3901 {
3902 int hAmount = ScrollTypes.Horizontal == scrollTypes ? amount : 0;
3903 int vAmount = ScrollTypes.Vertical == scrollTypes ? amount : 0;
3904
3905 // naming an element to scroll it says you want at what is inside it, and an element
3906 // off the bottom of the page scrolls its content where nobody can see it. So it is
3907 // brought on screen first, the same as a click does
3908 ScrollIntoView(browser, elem, null, true);
3909
3910 if (true == browser.UseOttoMagic)
3911 browser.MagicHelper.ScrollElement(elem.Css, hAmount, vAmount);
3912 else if (true == browser.UsePuppeteer)
3913 browser.PuppeteerClient.ScrollElement(elem.Css).WithHPixels(hAmount).WithVPixels(vAmount).Execute();
3914 else if (true == browser.UseSelenium)
3915 BrowserHelper.ExecuteJavaScript(browser, $"arguments[0].scrollBy(left: {hAmount}, top: {vAmount}, behavior: 'instant')", elem.WebElement);
3916
3917 if (++rowCount >= browser.CurrentUOW.WithAllThatMatch)
3918 break;
3919 }
3920 }
3921 }
3922 }
3923 }
3931 public static bool GenericClick(Browser browser, ClickType clickType)
3932 {
3933 List<GPALElement> matchedElements;
3934 bool matchedAll = false;
3935 bool clickPerformed = false;
3936
3937 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
3938 {
3939 if (SelectorType.Selector != selector.SelectorType)
3940 continue; // we don't look up data literals
3941
3942 ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out matchedAll, out matchedElements);
3943
3944 if (null != matchedElements)
3945 {
3946 int rowCount = 0;
3947 foreach (GPALElement elem in matchedElements)
3948 {
3949 if (null != elem)
3950 {
3951 if (true == elem.TagName.ToLower().Equals("option"))
3952 SelectClick(browser, selector, elem, SelectClickType.LeftClick, ModifierKeys.NONE);
3953 else
3954 Click(browser, selector, elem, clickType, ModifierKeys.NONE);
3955
3956 clickPerformed = true;
3957
3958 if (++rowCount >= browser.CurrentUOW.WithAllThatMatch)
3959 break;
3960 }
3961 }
3962 }
3963 }
3964 return clickPerformed;
3965 }
3973 public static bool GenericClick(Browser browser, SelectClickType selectClickType)
3974 {
3975 List<GPALElement> matchedElements;
3976 bool matchedAll = false;
3977 bool clickPerformed = false;
3978
3979 foreach (Selector selector in browser.CurrentUOW.WithSelectorList)
3980 {
3981 if (SelectorType.Selector != selector.SelectorType)
3982 continue; // we don't look up data literals
3983
3984 ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out matchedAll, out matchedElements);
3985
3986 if (null != matchedElements)
3987 {
3988 int rowCount = 0;
3989 foreach (GPALElement elem in matchedElements)
3990 {
3991 if (null != elem)
3992 {
3993 if (true == elem.TagName.ToLower().Equals("option"))
3994 SelectClick(browser, selector, elem, selectClickType, ModifierKeys.NONE);
3995 else
3996 Click(browser, selector, elem, ClickType.LeftClick, ModifierKeys.NONE);
3997
3998 clickPerformed = true;
3999
4000 if (++rowCount >= browser.CurrentUOW.WithAllThatMatch)
4001 break;
4002 }
4003 }
4004 }
4005 }
4006 return clickPerformed;
4007 }
4008
4009 // Click type note
4010 // NOTE: Special types for Select menus: RandomSelect will pick a random menu item, SequentialSelect will walk the list (w/ & w/0 wrapping)
4011 // written by grok with updates from mbv
4012 internal static class SelectClickHelper
4013 {
4014 // Static dictionary to track the current index for SequentialSelect and SequentialSelectWithWrap
4015 internal static readonly Dictionary<string, int> _sequentialSelectIndices = new Dictionary<string, int>();
4016 // Track the last parentNode identifier
4017 internal static string _lastParentNodeId = null;
4018
4026 internal static string GetElementAttributeHashCsharp(GPALElement element, Browser browser)
4027 {
4028 try
4029 {
4030 // Get all attributes from the Attributes dictionary
4031 var attrList = new List<string>();
4032
4033 foreach (var attrName in element.Attributes.Keys)
4034 {
4035 // Use GetAttribute to retrieve the actual value
4036 string value = element.GetAttribute(attrName);
4037 if (!string.IsNullOrEmpty(value))
4038 {
4039 attrList.Add($"{attrName}={value}");
4040 }
4041 }
4042
4043 // Sort attributes for consistent hashing
4044 attrList.Sort();
4045 string attrString = string.Join("|", attrList);
4046
4047 // Compute hash (same algorithm as JavaScript)
4048 int hash = 0;
4049 foreach (char c in attrString)
4050 {
4051 hash = ((hash << 5) - hash + c) | 0; // Matches JavaScript: hash = ((hash << 5) - hash + charCode) | 0
4052 }
4053
4054 return hash.ToString();
4055 }
4056 catch
4057 {
4058 return element.GetHashCode().ToString(); // Fallback to CitrixHashCode
4059 }
4060 }
4061
4069 internal static string GetElementAttributeHashJavaScript(GPALElement element, Browser browser)
4070 {
4071 try
4072 {
4073 if (false == browser.UseOttoMagic)
4074 {
4075 string jsScript = @"
4076 let attrs = arguments[0].attributes;
4077 let attrList = [];
4078 for (let attr of attrs) {
4079 attrList.push(attr.name + '=' + attr.value);
4080 }
4081 attrList.sort(); // Sort attributes for consistent hashing
4082 let str = attrList.join('|');
4083 let hash = 0;
4084 for (let i = 0; i < str.length; i++) {
4085 hash = ((hash << 5) - hash + str.charCodeAt(i)) | 0;
4086 }
4087 return hash.toString();
4088 ";
4089 return (string)BrowserHelper.ExecuteJavaScript(browser, jsScript, element.WebElement) ?? element.GetHashCode().ToString();
4090 } else
4091 {
4092 return browser.OttoMagicClient.GetElementAttributeHash(element.Css).Execute().Replace("\"","");
4093 }
4094 }
4095 catch
4096 {
4097 return element.GetHashCode().ToString(); // Fallback to GetHashCode
4098 }
4099 }
4100 }
4111
4112 public static void SelectClick(Browser browser, Selector selector, GPALElement webElement, SelectClickType clickType, ModifierKeys modifierKeys)
4113 {
4114 // refactored by grok to add random/sequential selects
4115 dynamic result = null; // GPALElement or WebElement
4116 dynamic parentElem = null; // GPALElement or WebElement
4117 SelectElement elem = null;
4118 string selectedValue = null;
4119 dynamic options = null;
4120 string elementId = null;
4121
4122 // NOTE: since we have select menu options like select a random entry or select next or select previous, we track the select menu internally and track the currently selected item
4123 // in theory this allows us to track multiple select menus but may be overkill
4124 if (true == browser.UseOttoMagic)
4125 result = browser.MagicHelper.GetParentNode(webElement.Css);
4126 else if (true == browser.UsePuppeteer)
4127 result = browser.PuppeteerClient.GetParentNode(webElement.ElementHandle).Execute<GPALElement>();
4128 else // selenium
4129 result = BrowserHelper.ExecuteJavaScriptObj("return arguments[0].parentNode;", browser, webElement.WebElement);
4130
4131 if (result != null)
4132 {
4133 if (true == browser.UseSelenium)
4134 {
4135 parentElem = result;
4136 elementId = parentElem.GetAttribute("id");
4137 }
4138 else
4139 {
4140 parentElem = (GPALElement)result;
4141 parentElem.Browser = browser;
4142 elementId = parentElem.GetAttribute("id");
4143 }
4144
4145 try
4146 {
4147 ScrollIntoView(browser, parentElem);
4148
4149 // Get a unique identifier based on attributes
4150 if (true == browser.UseSelenium)
4151 {
4152 elem = new SelectElement(parentElem);
4153
4154 // Get all options in the select element
4155 if (SelectClickType.LeftClick != clickType)
4156 {
4157 options = elem.Options;
4158 if (options.Count == 0)
4159 {
4160 GPAL.PublishSimpleEvent(GPALEventType.ERROR, "No options found in select element.", webElement, GPALObjectType.Other);
4161 return;
4162 }
4163 }
4164
4165 // Use C# attribute iteration for non-JavaScript interaction types
4166 if (string.IsNullOrEmpty(elementId))
4167 {
4168 //if (selector.InteractionType != Enums.InteractionType.JavaScript)
4169 // elementId = SelectClickHelper.GetElementAttributeHashCsharp(parentElem, browser);
4170 //else
4171 elementId = SelectClickHelper.GetElementAttributeHashJavaScript(parentElem, browser);
4172 }
4173 }
4174 else
4175 {
4176 if (SelectClickType.LeftClick != clickType)
4177 {
4178 Selector optionsSelector = GPAL.CssSelector($"{parentElem.Css} > option").WithSelectorName("options").ToGPALObject();
4179
4180 options = FindWebElementsByCss(browser, optionsSelector.SelectorPaths[0], optionsSelector); // CAVEAT: hardcoded value, but it's our selector
4181
4182 if (0 == options.Count)
4183 {
4184 Click(browser, selector, webElement, ClickType.LeftClick, modifierKeys);
4185 return;
4186 }
4187 }
4188
4189 if (string.IsNullOrEmpty(elementId))
4190 {
4191 if (true == browser.UseOttoMagic)
4192 elementId = browser.MagicHelper.GetElementAttributeHash(parentElem.Css);
4193 else if (true == browser.UsePuppeteer)
4194 elementId = browser.PuppeteerClient.GetElementAttributeHash(parentElem.Css).Execute();
4195 }
4196 }
4197
4198 // Check if we're dealing with a new select element
4199
4200 if (SelectClickHelper._lastParentNodeId != elementId &&
4201 (clickType == SelectClickType.SelectNext || clickType == SelectClickType.SelectNextWithWrap))
4202 {
4203 SelectClickHelper._sequentialSelectIndices[elementId] = -1; // Reset index
4204 }
4205
4206 SelectClickHelper._lastParentNodeId = elementId;
4207
4208 if (false == browser.BrowserSettings.UseHeadless && (true == browser.BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware))
4209 {
4210 switch (clickType)
4211 {
4212 case SelectClickType.SelectRandom:
4213 Random rand = new Random();
4214 int randomIndex = rand.Next(0, options.Count);
4215 // Store the new index and perform the selection
4216 SelectClickHelper._sequentialSelectIndices[elementId] = randomIndex;
4217 HardwareClick(browser, parentElem, ClickType.LeftClick, modifierKeys, browser.CurrentUOW.CurrentSelector);
4218 HardwareHelper.SendKey(GPAL.VK_HOME);
4219 for (int i = 0; i < randomIndex; i++)
4220 {
4221 Thread.Sleep(50);
4222 HardwareHelper.SendKey(GPAL.VK_DOWN);
4223 }
4224 HardwareHelper.SendKey(GPAL.VK_RETURN);
4225 break;
4226
4227 case SelectClickType.SelectNext:
4228 case SelectClickType.SelectNextWithWrap:
4229 case SelectClickType.SelectPrevious:
4230 case SelectClickType.SelectPreviousWithWrap:
4231
4232 // +1 = move down (next), -1 = move up (previous)
4233 int incOrDec = (clickType == SelectClickType.SelectPrevious ||
4234 clickType == SelectClickType.SelectPreviousWithWrap)
4235 ? -1 : 1;
4236
4237 bool shouldWrap = (clickType == SelectClickType.SelectNextWithWrap ||
4238 clickType == SelectClickType.SelectPreviousWithWrap);
4239
4240 // Initialize to -1 on first use -> first Next -> index 0, first Previous -> last item
4241 if (!SelectClickHelper._sequentialSelectIndices.TryGetValue(elementId, out int currentIndex))
4242 {
4243 currentIndex = -1;
4244 SelectClickHelper._sequentialSelectIndices[elementId] = currentIndex;
4245 }
4246
4247 int nextIndex;
4248
4249 if (shouldWrap)
4250 {
4251 // Proper wrapping that works when going backwards from 0
4252 nextIndex = (currentIndex + incOrDec % options.Count + options.Count) % options.Count;
4253 }
4254 else
4255 {
4256 // No wrap -> clamp to bounds
4257 nextIndex = currentIndex + incOrDec;
4258 if (nextIndex < 0)
4259 nextIndex = 0;
4260 else if (nextIndex >= options.Count)
4261 nextIndex = options.Count - 1;
4262 }
4263
4264 // === Hardware emulation starts here ===
4265 HardwareClick(browser, parentElem, ClickType.LeftClick, modifierKeys,
4266 browser.CurrentUOW.CurrentSelector);
4267
4268 // Always start from the top (HOME) — this matches real dropdown behavior
4269 HardwareHelper.SendKey(GPAL.VK_HOME);
4270 Thread.Sleep(50); // small delay after HOME
4271
4272 if (incOrDec > 0)
4273 {
4274 // Going forward -> just press DOWN `nextIndex` times
4275 for (int i = 0; i < nextIndex; i++)
4276 {
4277 Thread.Sleep(50);
4278 HardwareHelper.SendKey(GPAL.VK_DOWN);
4279 }
4280 }
4281 else
4282 {
4283 // Going backward -> go to end first, then go up the required steps
4284 // This is the most reliable way when using pure keyboard
4285 HardwareHelper.SendKey(GPAL.VK_END);
4286 Thread.Sleep(50);
4287
4288 int stepsUp = options.Count - 1 - nextIndex; // how many UP arrows needed from bottom
4289 for (int i = 0; i < stepsUp; i++)
4290 {
4291 Thread.Sleep(50);
4292 HardwareHelper.SendKey(GPAL.VK_UP);
4293 }
4294 }
4295
4296 // Confirm selection
4297 HardwareHelper.SendKey(GPAL.VK_RETURN);
4298
4299 // Store the new index for next time
4300 SelectClickHelper._sequentialSelectIndices[elementId] = nextIndex;
4301
4302 break;
4303
4304 default:
4305 HardwareClick(browser, parentElem, ClickType.LeftClick, modifierKeys, browser.CurrentUOW.CurrentSelector);
4306
4307 // ReadOnlyCollection<GPALElement> optionElems = ElementHelper.FindWebElements(browser, browser.CurrentUOW, selector, out bool matchedAll, out List<GPALElement> matchedElements);
4308 int index = Int32.Parse(webElement.GetAttribute("index")); // optionElems[0].GetAttribute("index"));
4309 // Store the new index and perform the selection
4310 SelectClickHelper._sequentialSelectIndices[elementId] = index;
4311 HardwareHelper.SendKey(GPAL.VK_HOME);
4312 for (int cnt = 0; cnt < index; cnt++)
4313 {
4314 Thread.Sleep(50);
4315 HardwareHelper.SendKey(GPAL.VK_DOWN);
4316 }
4317 HardwareHelper.SendKey(GPAL.VK_RETURN);
4318 break;
4319 }
4320 }
4321 else if (true == browser.UseOttoMagic)
4322 {
4323 switch (clickType)
4324 {
4325 case SelectClickType.SelectRandom:
4326 Random rand = new Random();
4327 int randomIndex = rand.Next(0, options.Count);
4328 selectedValue = options[randomIndex].GetAttribute("value");
4329 // Store the new index and perform the selection
4330 SelectClickHelper._sequentialSelectIndices[elementId] = randomIndex;
4331 browser.MagicHelper.SelectClick(parentElem.Css, selectedValue);
4332 break;
4333
4334 case SelectClickType.SelectNext:
4335 case SelectClickType.SelectNextWithWrap:
4336 case SelectClickType.SelectPrevious:
4337 case SelectClickType.SelectPreviousWithWrap:
4338
4339 // +1 for next, -1 for previous
4340 int incOrDec = (clickType == SelectClickType.SelectPrevious ||
4341 clickType == SelectClickType.SelectPreviousWithWrap)
4342 ? -1 : 1;
4343
4344 // true if we should wrap around
4345 bool shouldWrap = (clickType == SelectClickType.SelectNextWithWrap ||
4346 clickType == SelectClickType.SelectPreviousWithWrap);
4347
4348 // Initialize to -1 on first use (so first "next" goes to 0, first "prev" goes to last)
4349 if (!SelectClickHelper._sequentialSelectIndices.TryGetValue(elementId, out int currentIndex))
4350 {
4351 currentIndex = -1;
4352 SelectClickHelper._sequentialSelectIndices[elementId] = currentIndex;
4353 }
4354
4355 int nextIndex;
4356
4357 if (shouldWrap)
4358 {
4359 // Safe modular arithmetic that works with negative numbers
4360 nextIndex = (currentIndex + incOrDec % options.Count + options.Count) % options.Count;
4361 }
4362 else
4363 {
4364 // Manual clamp: stay within [0, options.Count-1]
4365 nextIndex = currentIndex + incOrDec;
4366 if (nextIndex < 0)
4367 nextIndex = 0;
4368 else if (nextIndex >= options.Count)
4369 nextIndex = options.Count - 1;
4370 }
4371
4372 // Store the new index and perform the selection
4373 SelectClickHelper._sequentialSelectIndices[elementId] = nextIndex;
4374 selectedValue = options[nextIndex].GetAttribute("value");
4375 browser.MagicHelper.SelectClick(parentElem.Css, selectedValue);
4376
4377 break;
4378
4379 default:
4380 int index = Int32.Parse(webElement.GetAttribute("index"));
4381 // Store the new index and perform the selection
4382 SelectClickHelper._sequentialSelectIndices[elementId] = index;
4383 browser.MagicHelper.SelectClick(parentElem.Css, webElement.Value);
4384 break;
4385 }
4386 }
4387 else if (true == browser.UsePuppeteer)
4388 {
4389 switch (clickType)
4390 {
4391 case SelectClickType.SelectRandom:
4392 Random rand = new Random();
4393 int randomIndex = rand.Next(0, options.Count);
4394 selectedValue = options[randomIndex].GetAttribute("value");
4395 // Store the new index and perform the selection
4396 SelectClickHelper._sequentialSelectIndices[elementId] = randomIndex;
4397 // note, some weird c# late binding cannot find selectclick in the interface, thus the cast
4398 ((PuppeteerClient)browser.PuppeteerClient).SelectClick((dynamic)parentElem.ElementHandle).WithSelectValue(selectedValue).Execute();
4399 break;
4400
4401 case SelectClickType.SelectNext:
4402 case SelectClickType.SelectNextWithWrap:
4403 case SelectClickType.SelectPrevious:
4404 case SelectClickType.SelectPreviousWithWrap:
4405
4406 // +1 for next, -1 for previous
4407 int incOrDec = (clickType == SelectClickType.SelectPrevious ||
4408 clickType == SelectClickType.SelectPreviousWithWrap)
4409 ? -1 : 1;
4410
4411 // true if we should wrap around
4412 bool shouldWrap = (clickType == SelectClickType.SelectNextWithWrap ||
4413 clickType == SelectClickType.SelectPreviousWithWrap);
4414
4415 // Initialize to -1 on first use (so first "next" goes to 0, first "prev" goes to last)
4416 if (!SelectClickHelper._sequentialSelectIndices.TryGetValue(elementId, out int currentIndex))
4417 {
4418 currentIndex = -1;
4419 SelectClickHelper._sequentialSelectIndices[elementId] = currentIndex;
4420 }
4421
4422 int nextIndex;
4423
4424 if (shouldWrap)
4425 {
4426 // Safe modular arithmetic that works with negative numbers
4427 nextIndex = (currentIndex + incOrDec % options.Count + options.Count) % options.Count;
4428 }
4429 else
4430 {
4431 // Manual clamp: stay within [0, options.Count-1]
4432 nextIndex = currentIndex + incOrDec;
4433 if (nextIndex < 0)
4434 nextIndex = 0;
4435 else if (nextIndex >= options.Count)
4436 nextIndex = options.Count - 1;
4437 }
4438
4439 // Store the new index and perform the selection
4440 SelectClickHelper._sequentialSelectIndices[elementId] = nextIndex;
4441 selectedValue = options[nextIndex].GetAttribute("value");
4442 // note, some weird c# late binding cannot find selectclick in the interface, thus the cast
4443 ((PuppeteerClient)browser.PuppeteerClient).SelectClick((dynamic)parentElem.ElementHandle).WithSelectValue(selectedValue).Execute();
4444
4445 break;
4446
4447 default:
4448 int index = Int32.Parse(webElement.GetAttribute("index"));
4449 SelectClickHelper._sequentialSelectIndices[elementId] = index;
4450 // Store the new index and perform the selection
4451 // note, some weird c# late binding cannot find selectclick in the interface, thus the cast
4452 ((PuppeteerClient)browser.PuppeteerClient).SelectClick((dynamic)parentElem.ElementHandle).WithSelectIndex(index).Execute();
4453 break;
4454 }
4455 }
4456 else if (true == browser.UseSelenium)
4457 {
4458
4459 if (Enums.InteractionType.JavaScript == selector.InteractionType || browser.BrowserSettings.UseJavaScript) // only selenium can have javascript option
4460 {
4461 switch (clickType)
4462 {
4463 case SelectClickType.SelectRandom:
4464 Random rand = new Random();
4465 int randomIndex = rand.Next(0, options.Count);
4466 // Store the new index and perform the selection
4467 SelectClickHelper._sequentialSelectIndices[elementId] = randomIndex;
4468 selectedValue = options[randomIndex].GetAttribute("value");
4469 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].value = arguments[1];", parentElem, selectedValue);
4470 break;
4471
4472 case SelectClickType.SelectNext:
4473 case SelectClickType.SelectNextWithWrap:
4474 case SelectClickType.SelectPrevious:
4475 case SelectClickType.SelectPreviousWithWrap:
4476
4477 // +1 for next, -1 for previous
4478 int incOrDec = (clickType == SelectClickType.SelectPrevious ||
4479 clickType == SelectClickType.SelectPreviousWithWrap)
4480 ? -1 : 1;
4481
4482 // true if we should wrap around
4483 bool shouldWrap = (clickType == SelectClickType.SelectNextWithWrap ||
4484 clickType == SelectClickType.SelectPreviousWithWrap);
4485
4486 // Initialize to -1 on first use (so first "next" goes to 0, first "prev" goes to last)
4487 if (!SelectClickHelper._sequentialSelectIndices.TryGetValue(elementId, out int currentIndex))
4488 {
4489 currentIndex = -1;
4490 SelectClickHelper._sequentialSelectIndices[elementId] = currentIndex;
4491 }
4492
4493 int nextIndex;
4494
4495 if (shouldWrap)
4496 {
4497 // Safe modular arithmetic that works with negative numbers
4498 nextIndex = (currentIndex + incOrDec % options.Count + options.Count) % options.Count;
4499 }
4500 else
4501 {
4502 // Manual clamp: stay within [0, options.Count-1]
4503 nextIndex = currentIndex + incOrDec;
4504 if (nextIndex < 0)
4505 nextIndex = 0;
4506 else if (nextIndex >= options.Count)
4507 nextIndex = options.Count - 1;
4508 }
4509
4510 // Store the new index and perform the selection
4511 SelectClickHelper._sequentialSelectIndices[elementId] = nextIndex;
4512 selectedValue = options[nextIndex].GetAttribute("value");
4513 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].value = arguments[1];", parentElem, selectedValue);
4514 break;
4515
4516 default:
4517 int index = Int32.Parse(webElement.WebElement.GetAttribute("index")); // optionElems[0].GetAttribute("index"));
4518 // Store the new index and perform the selection
4519 SelectClickHelper._sequentialSelectIndices[elementId] = index;
4520 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].value = arguments[1].value", parentElem, webElement.WebElement);
4521 break;
4522 }
4523 }
4524 else // use native selenium rather than javascript injection
4525 switch (clickType)
4526 {
4527 case SelectClickType.SelectRandom:
4528 Random rand = new Random();
4529 int randomIndex = rand.Next(0, options.Count);
4530 // Store the new index and perform the selection
4531 SelectClickHelper._sequentialSelectIndices[elementId] = randomIndex;
4532 elem.SelectByIndex(randomIndex);
4533 break;
4534
4535 case SelectClickType.SelectNext:
4536 case SelectClickType.SelectNextWithWrap:
4537 case SelectClickType.SelectPrevious:
4538 case SelectClickType.SelectPreviousWithWrap:
4539
4540 // Determine direction: +1 = next, -1 = previous
4541 int incOrDec = (clickType == SelectClickType.SelectPrevious ||
4542 clickType == SelectClickType.SelectPreviousWithWrap)
4543 ? -1 : 1;
4544
4545 // Determine if we should wrap around
4546 bool shouldWrap = (clickType == SelectClickType.SelectNextWithWrap ||
4547 clickType == SelectClickType.SelectPreviousWithWrap);
4548
4549 // Get current index, default to -1 (so first "next" -> 0, first "prev" -> last)
4550 if (!SelectClickHelper._sequentialSelectIndices.TryGetValue(elementId, out int currentIndex))
4551 {
4552 currentIndex = -1;
4553 SelectClickHelper._sequentialSelectIndices[elementId] = currentIndex;
4554 }
4555
4556 int nextIndex;
4557
4558 if (shouldWrap)
4559 {
4560 // Safe wrap-around that works with negative numbers
4561 nextIndex = (currentIndex + incOrDec % options.Count + options.Count) % options.Count;
4562 }
4563 else
4564 {
4565 // No wrap: clamp to valid range
4566 nextIndex = currentIndex + incOrDec;
4567 if (nextIndex < 0)
4568 nextIndex = 0;
4569 else if (nextIndex >= options.Count)
4570 nextIndex = options.Count - 1;
4571 }
4572
4573 // Perform the selection and store the new index
4574 elem.SelectByIndex(nextIndex);
4575 SelectClickHelper._sequentialSelectIndices[elementId] = nextIndex;
4576
4577 break;
4578
4579 default:
4580 int index = Int32.Parse(webElement.GetAttribute("index")); // optionElems[0].GetAttribute("index"));
4581 // Store the new index and perform the selection
4582 SelectClickHelper._sequentialSelectIndices[elementId] = index;
4583 elem.SelectByValue(webElement.GetAttribute("value"));
4584 break;
4585 }
4586 }
4587 }
4588 catch (GPALException)
4589 {
4590 throw;
4591 }
4592 catch (Exception ex)
4593 {
4594 if (!GPAL.NoFallbackRecoveryActions)
4595 {
4596 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to Select using [{selector.InteractionType}]. Falling back on JavaScript.", webElement, GPALObjectType.Other, ex);
4597 BrowserHelper.ExecuteJavaScript(browser, "arguments[0].value = arguments[1].value", true == browser.UseSelenium ? parentElem : ((GPALElement)parentElem).WebElement, webElement.WebElement);
4598 }
4599 else
4600 {
4601 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to Select using [{selector.InteractionType}].", webElement, GPALObjectType.Other, ex);
4602 }
4603 }
4604 selectedValue = null != options ? options[SelectClickHelper._sequentialSelectIndices[elementId]].GetAttribute("value") : webElement.GetAttribute("value");
4605 string selectedText = null != options ? options[SelectClickHelper._sequentialSelectIndices[elementId]].Text : webElement.Text;
4606 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{selector.InteractionType}] selected [{selectedText}] value [{selectedValue}] index [{SelectClickHelper._sequentialSelectIndices[elementId]}]");
4607 }
4608 else
4609 {
4610 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Unable to click Parent Select for Option using [{selector.InteractionType}]. Clicking element [{selector.Name}].", webElement, GPALObjectType.Other);
4611 webElement.Click(modifierKeys);
4612 }
4613 }
4614
4626 public static void Click(Browser browser, Selector selector, GPALElement webElement, ClickType clickType, ModifierKeys modifierKeys, bool trusted = false)
4627 {
4628 bool dontPublish = false;
4629
4630 // an element found under an image is clicked where the picture is, not in its own middle, which on
4631 // a large element is somewhere else entirely. the element is kept, to check the point is still its own
4632 GPALElement resolved = null;
4633
4634 if (null != webElement.ImageMatch)
4635 {
4636 resolved = webElement;
4637 webElement = webElement.ImageMatch;
4638 }
4639
4640 ScrollIntoView(browser, webElement, selector.InteractionType);
4641
4642
4643 //if (true == webElement.IsClickable()) // NOTE: no real utility to printing out if elemenent is clickable, we will try anyways, just slows us down a little
4644 {
4645 try
4646 {
4647 if (false == browser.BrowserSettings.UseHeadless && false == browser.BrowserSettings.HiddenDesktop
4648 && (Enums.InteractionType.Hardware == selector.InteractionType
4649 || true == browser.BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware)
4650 )
4651 {
4652 // the position is current without a requery: ScrollIntoView re-reads it when it scrolls,
4653 // and HardwareMoveTo re-reads puppeteer's every time
4654 HardwareClick(browser, webElement, clickType, modifierKeys, selector, true);
4655 dontPublish = true;
4656 }
4657 else if ("GPALElement" == webElement.TagName)
4658 {
4659 // image match with no hardware asked for, so the engine clicks the coordinate directly. when
4660 // the match resolved to an element, the point is checked first: something that arrived after
4661 // the picture was found, an ad slid in by the scroll, would take the click
4662 Point aim = ToViewportPoint(browser, webElement, selector);
4663
4664 if (null != resolved && false == PointIsClear(browser, resolved, selector, ref aim) && false == GPAL.NoFallbackRecoveryActions)
4665 ClickCoveredElement(browser, resolved, clickType, modifierKeys);
4666 else
4667 Click(browser, aim, clickType, modifierKeys);
4668
4669 // OttoMagic's point click says itself whether it was made, and a generic "clicked" after a
4670 // "was not made" reads as a click that happened
4671 dontPublish = browser.UseOttoMagic;
4672 }
4673 else if (true == browser.UseOttoMagic)
4674 {
4675 bool clicked = false;
4676
4677 // the extension clicks a viewport point for a trusted click, so it holds no modifiers and
4678 // takes the rect the page has now, after the scroll above moved it
4679 if (true == trusted && ClickType.LeftClick == clickType && ModifierKeys.NONE == modifierKeys)
4680 {
4681 Rectangle rect = browser.MagicHelper.GetBoundingClientRect(webElement.Css);
4682
4683 Point aim = (0 == selector.OffsetX && 0 == selector.OffsetY)
4684 ? new Point(rect.X + rect.Width / 2, rect.Y + rect.Height / 2)
4685 : new Point(rect.X + selector.OffsetX, rect.Y + selector.OffsetY);
4686
4687 clicked = 0 < rect.Width && 0 < rect.Height && true == browser.MagicHelper.TrustedLeftClick(aim.X, aim.Y);
4688
4689 if (false == clicked)
4690 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[Trusted][{clickType}] on selector [{selector.Name}] was not made, falling back to [{clickType}].", browser, GPALObjectType.Browser);
4691 }
4692
4693 if (false == clicked)
4694 {
4695 if (modifierKeys != ModifierKeys.NONE)
4696 browser.MagicHelper.PressModifierKey(modifierKeys);
4697
4698 switch (clickType)
4699 {
4700 case ClickType.LeftClick:
4701 browser.MagicHelper.LeftClick(webElement.Css);
4702 break;
4703 case ClickType.LeftDoubleClick:
4704 browser.MagicHelper.LeftDoubleClick(webElement.Css);
4705 break;
4706 case ClickType.MiddleClick:
4707 browser.MagicHelper.MiddleClick(webElement.Css);
4708 break;
4709 case ClickType.RightClick:
4710 browser.MagicHelper.RightClick(webElement.Css);
4711 break;
4712 }
4713
4714 if (modifierKeys != ModifierKeys.NONE)
4715 browser.MagicHelper.ReleaseModifierKey(modifierKeys);
4716 }
4717 }
4718 else if (Enums.InteractionType.JavaScript == selector.InteractionType || true == browser.BrowserSettings.UseJavaScript) // selunium & puppeteer javascript override
4719 {
4720 if (false == JavaScriptClick(browser, webElement, clickType, modifierKeys) && false == GPAL.NoFallbackRecoveryActions)
4721 {
4722 if (true == browser.UseSelenium)
4723 {
4724 GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"JavaScript click failed on selector [{selector.Name}], falling back to selenium click.", browser, GPALObjectType.Browser);
4725 SeleniumClick(browser, webElement, clickType, modifierKeys);
4726 }
4727 else
4728 {
4729 GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"JavaScript click failed on selector [{selector.Name}], falling back to Puppeteer click.", browser, GPALObjectType.Browser);
4730 PuppeteerClick(browser, webElement, clickType, modifierKeys);
4731 dontPublish = true;
4732 }
4733 }
4734 }
4735 else if (true == browser.UsePuppeteer)
4736 {
4737 PuppeteerClick(browser, webElement, clickType, modifierKeys);
4738 dontPublish = true;
4739 }
4740 else
4741 SeleniumClick(browser, webElement, clickType, modifierKeys);
4742
4743 if (false == dontPublish)
4744 GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Element [{webElement.TagName}] for selector [{selector.Name}] [{selector.InteractionType.ToString()}][{clickType.ToString()}] clicked with modifiers [{GetEnumNames(modifierKeys)}].", browser, GPALObjectType.Browser);
4745
4746 }
4747 catch (GPALException)
4748 {
4749 throw;
4750 }
4751 catch (Exception ex)
4752 {
4753 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to [{clickType.ToString()}] click element [{webElement.TagName}] for selector [{selector.Name}]", browser, GPALObjectType.Browser, ex);
4754 }
4755 }
4756 //else
4757 // GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Element [{webElement.TagName}] for selector [{selector.Name}] [{selector.InteractionType.ToString()}][{clickType.ToString()}] is NOT clickable.", browser, GPALObjectType.Browser);
4758
4759 Thread.Sleep(500); // CAVEAT: don't like this, but a click could be navigating and we might send the next request before the page is ready and even idle network or document.ready can fail
4760
4761 }
4769 public static void PuppeteerClick(Browser browser, GPALElement webElement, ClickType clickType, ModifierKeys modifierKeys = ModifierKeys.NONE)
4770 {
4771 switch (clickType)
4772 {
4773 case ClickType.LeftClick:
4774 browser.PuppeteerClient.LeftClick(webElement).WithModifiers(modifierKeys).Execute();
4775 break;
4776 case ClickType.LeftDoubleClick:
4777 browser.PuppeteerClient.LeftDoubleClick(webElement).WithModifiers(modifierKeys).Execute();
4778 break;
4779 case ClickType.MiddleClick:
4780 browser.PuppeteerClient.MiddleClick(webElement).WithModifiers(modifierKeys).Execute();
4781 break;
4782 case ClickType.RightClick:
4783 browser.PuppeteerClient.RightClick(webElement).WithModifiers(modifierKeys).Execute();
4784 break;
4785 }
4786 }
4795 public static void SeleniumClick(Browser browser, GPALElement webElement, ClickType clickType, ModifierKeys modifierKeys = ModifierKeys.NONE)
4796 {
4797 int retries = 2;
4798 bool retry = false;
4799 Actions action = new Actions(browser.BrowserDriver);
4800 string errorMessage = $"Failed to {clickType.ToString()} element with text [{webElement.Text}]";
4801
4802 do
4803 {
4804 try
4805 {
4806 switch (clickType)
4807 {
4808 case ClickType.LeftClick:
4809 if (ModifierKeys.NONE == modifierKeys)
4810 {
4811 webElement.iWebElement.Click();
4812 }
4813 else
4814 {
4815 if (modifierKeys.HasFlag(ModifierKeys.Alt))
4816 action.KeyDown(OpenQA.Selenium.Keys.LeftAlt).KeyDown(OpenQA.Selenium.Keys.Alt);
4817 if (modifierKeys.HasFlag(ModifierKeys.Control))
4818 action.KeyDown(OpenQA.Selenium.Keys.LeftControl).KeyDown(OpenQA.Selenium.Keys.Control);
4819 if (modifierKeys.HasFlag(ModifierKeys.Shift))
4820 action.KeyDown(OpenQA.Selenium.Keys.LeftShift).KeyDown(OpenQA.Selenium.Keys.Shift);
4821 if (modifierKeys.HasFlag(ModifierKeys.Windows))
4822 action.KeyDown(OpenQA.Selenium.Keys.Meta);
4823
4824 action.Click(webElement.iWebElement);
4825
4826 if (modifierKeys.HasFlag(ModifierKeys.Alt))
4827 action.KeyUp(OpenQA.Selenium.Keys.LeftAlt).KeyUp(OpenQA.Selenium.Keys.Alt);
4828 if (modifierKeys.HasFlag(ModifierKeys.Control))
4829 action.KeyUp(OpenQA.Selenium.Keys.LeftControl).KeyUp(OpenQA.Selenium.Keys.Control);
4830 if (modifierKeys.HasFlag(ModifierKeys.Shift))
4831 action.KeyUp(OpenQA.Selenium.Keys.LeftShift).KeyUp(OpenQA.Selenium.Keys.Shift);
4832 if (modifierKeys.HasFlag(ModifierKeys.Windows))
4833 action.KeyUp(OpenQA.Selenium.Keys.Meta);
4834
4835 ModifierKeys unsupported = modifierKeys & (ModifierKeys.Application | ModifierKeys.ScrollLock);
4836 if (ModifierKeys.NONE != unsupported)
4837 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"LeftClick: [{Browser.GetModifierKeys(unsupported)}] is not supported via Selenium/WebDriver Actions and was ignored.", browser, GPALObjectType.Browser);
4838
4839 action.Build().Perform();
4840 }
4841 break;
4842 case ClickType.LeftDoubleClick:
4843 action.DoubleClick(webElement.iWebElement).Perform();
4844 break;
4845 case ClickType.MiddleClick:
4846 {
4847 // a real middle click, button 1, the same thing ottomagic and puppeteer send.
4848 // this used to hold ctrl and left click instead, which opens a tab on a plain anchor
4849 // but does nothing on a page that handles its own clicks, and never fires auxclick
4850 PointerInputDevice mouse = new PointerInputDevice(PointerKind.Mouse, "default mouse");
4851 ActionSequence middleClick = new ActionSequence(mouse);
4852
4853 middleClick.AddAction(mouse.CreatePointerMove(webElement.iWebElement, 0, 0, TimeSpan.Zero));
4854 middleClick.AddAction(mouse.CreatePointerDown(MouseButton.Middle));
4855 middleClick.AddAction(mouse.CreatePointerUp(MouseButton.Middle));
4856
4857 ((IActionExecutor)browser.BrowserDriver).PerformActions(new List<ActionSequence> { middleClick });
4858
4859 if (ModifierKeys.NONE != modifierKeys)
4860 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Selenium cannot press [{Browser.GetModifierKeys(modifierKeys)}] with a middle click.", browser, GPALObjectType.Browser);
4861
4862 break;
4863 }
4864 case ClickType.RightClick:
4865 action.ContextClick(webElement.iWebElement).Perform();
4866 break;
4867 }
4868 retry = false;
4869 }
4870 catch (StaleElementReferenceException sereex)
4871 {
4872 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, errorMessage, webElement, GPALObjectType.Other, sereex);
4873 retry = ResolveWebElementIssue(browser, browser.CurrentUOW, ref webElement.iWebElement, sereex, ref retries, errorMessage);
4874 retries--;
4875 }
4876 catch (ElementClickInterceptedException ecieex)
4877 {
4878 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, errorMessage, webElement, GPALObjectType.Other, ecieex);
4879 if (false == GPAL.NoFallbackRecoveryActions)
4880 retry = JavaScriptClick(browser, webElement, clickType, modifierKeys);
4881 else
4882 retry = false;
4883 retries--;
4884 }
4885 catch (ElementNotInteractableException enieex)
4886 {
4887 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, errorMessage, webElement, GPALObjectType.Other, enieex);
4888 retry = ResolveWebElementIssue(browser, browser.CurrentUOW, ref webElement.iWebElement, enieex, ref retries, errorMessage);
4889 retries--;
4890 }
4891 catch (GPALException)
4892 {
4893 throw;
4894 }
4895 catch (Exception ex)
4896 {
4897 retries = 0;
4898 retry = false;
4899 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, errorMessage, webElement, GPALObjectType.Other, ex);
4900 }
4901 } while (retry && 0 < retries);
4902
4903 if (retries == 0 && false == GPAL.NoFallbackRecoveryActions)
4904 {
4905 GPAL.PublishSimpleEvent(GPALEventType.INFO, "Falling back on javascript click", webElement, GPALObjectType.Other);
4906 JavaScriptClick(browser, webElement, clickType, modifierKeys);
4907 }
4908 }
4909 // https://stackoverflow.com/questions/809057/how-do-i-programmatically-click-on-an-element-in-javascript
4910 // comment from above (not yet tested)
4911 // set third last parameter of MouseEvent to true (meaning that metaKey (CMD) button was held down when you clicked), this would not open the tab in a background tab
4917 public static bool JavaScriptClick(Browser browser, GPALElement webElement, ClickType clickType, ModifierKeys modifierKeys = ModifierKeys.NONE)
4918 {
4919 string clickString = clickType switch
4920 {
4921 ClickType.LeftClick => "click",
4922 ClickType.LeftDoubleClick => "dblclick",
4923 ClickType.RightClick => "contextmenu",
4924 ClickType.MiddleClick => "auxclick", // This is the real middle-click event
4925 _ => "click"
4926 };
4927 int button = (true == "contextmenu".Equals(clickString) ? 2 : 0);
4928 bool shiftPressed = 0 != (modifierKeys & ModifierKeys.Shift);
4929 bool controlPressed = 0 != (modifierKeys & ModifierKeys.Control);
4930 bool altPressed = 0 != (modifierKeys & ModifierKeys.Alt);
4931
4932 // Randomize click within 90% of the element's border
4933 var random = new Random();
4934 var margin = 0.05f; // 5% margin on each side (90% clickable area)
4935 var clickAreaWidth = (float)webElement.BoundingRect.Width * (1f - 2 * margin);
4936 var clickAreaHeight = (float)webElement.BoundingRect.Height * (1f - 2 * margin);
4937 var x = (float)webElement.BoundingRect.X + (float)webElement.BoundingRect.Width * margin + (float)random.NextDouble() * clickAreaWidth;
4938 var y = (float)webElement.BoundingRect.Y + (float)webElement.BoundingRect.Height * margin + (float)random.NextDouble() * clickAreaHeight;
4939
4940 if (true == browser.UseSelenium)
4941 {
4942 IWebDriver browserDriver = browser.BrowserDriver;
4943
4944 string clickScript = $@"
4945 var theEvent = document.createEvent(""MouseEvent"");
4946 theEvent.initMouseEvent(""{clickString}"", true, true, window, 0, 0, 0, {x}, {y}, {controlPressed.ToString().ToLower()}, {altPressed.ToString().ToLower()}, {shiftPressed.ToString().ToLower()}, false, 0, null);
4947 arguments[0].dispatchEvent(theEvent);
4948 ";
4949
4950 try
4951 {
4952 if (2 == button) // contextmenu will not fire off via javascript, but SHIFT+F10 will but not in headless
4953 {
4954 JavaScriptFocus(browser, webElement);
4955
4956 if (false == browser.BrowserSettings.UseHeadless)
4957 HardwareHelper.SendKey(GPAL.VK_F10, ModifierKeys.Shift);
4958 else // must use selenium in headless mode NOTE: Untested, bet this doesn't work... we only get here if this is a straight rightclick (like custom website)
4959 new Actions(browserDriver).ContextClick((IWebElement)webElement.WebElement).Perform();
4960 }
4961 else
4962 BrowserHelper.ExecuteJavaScript(browser, clickScript, webElement.WebElement); // click on element
4963 }
4964 catch (GPALException)
4965 {
4966 throw;
4967 }
4968 catch (Exception ex)
4969 {
4970 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to [{clickString}] element [{webElement.Text}]", webElement, GPALObjectType.Other, ex);
4971 return false;
4972 }
4973 }
4974 else if (true == browser.UsePuppeteer) // we should never be hear for ottomagic which is all javascript
4975 {
4976 string sessionId = browser.PuppeteerCommunicator.GetEffectiveSessionId();
4977
4978 string selector = webElement.Css;
4979
4980 string clickScript = null;
4981
4982 clickScript = $@"
4983 (() => {{
4984 try {{
4985 let el = null;
4986 const sel = `{selector.Replace("'", "\\'")}`;
4987
4988 const el = document.querySelector(sel) || document.evaluate(sel, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
4989 if (!el) return false;
4990
4991 const rect = el.getBoundingClientRect();
4992 const cx = rect.left + rect.width / 2 + window.scrollX;
4993 const cy = rect.top + rect.height / 2 + window.scrollY;
4994
4995 const opts = {{ bubbles: true, cancelable: true, clientX: cx, clientY: cy, view: window }};
4996
4997 // === LEFT CLICK (or double) ===
4998 if ({(clickType == ClickType.LeftClick || clickType == ClickType.LeftDoubleClick).ToString().ToLower()}) {{
4999 const count = {(clickType == ClickType.LeftDoubleClick ? 2 : 1)};
5000 for (let i = 0; i < count; i++) {{
5001 el.dispatchEvent(new MouseEvent('mousedown', {{...opts, button: 0, buttons: 1}}));
5002 el.dispatchEvent(new MouseEvent('mouseup', {{...opts, button: 0, buttons: 0}}));
5003 el.dispatchEvent(new MouseEvent('click', {{...opts, button: 0, detail: i+1}}));
5004 if (i === 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 80); // ~80ms between clicks
5005 }}
5006 return true;
5007 }}
5008
5009 // === MIDDLE CLICK (eBay pop-under) ===
5010 if ({(clickType == ClickType.MiddleClick).ToString().ToLower()}) {{
5011 el.dispatchEvent(new MouseEvent('mousedown', {{...opts, button: 1, buttons: 4}}));
5012 el.dispatchEvent(new MouseEvent('mouseup', {{...opts, button: 1, buttons: 0}}));
5013 el.dispatchEvent(new MouseEvent('click', {{...opts, button: 1}}));
5014 return true;
5015 }}
5016
5017 // === RIGHT CLICK (context menu) ===
5018 if ({(clickType == ClickType.RightClick).ToString().ToLower()}) {{
5019 el.dispatchEvent(new MouseEvent('mousedown', {{...opts, button: 2, buttons: 2}}));
5020 el.dispatchEvent(new MouseEvent('mouseup', {{...opts, button: 2, buttons: 0}}));
5021 el.dispatchEvent(new MouseEvent('contextmenu', {{...opts, button: 2}}));
5022 return true;
5023 }}
5024
5025 return false;
5026 }} catch(e) {{
5027 return false;
5028 }}
5029 }})();
5030 ";
5031
5032 browser.PuppeteerCommunicator.SendCommand<object>(DevToolsMethods.RuntimeEvaluate, new
5033 {
5034 expression = clickScript,
5035 returnByValue = true
5036 }, sessionId).GetAwaiter().GetResult();
5037
5038 //clickScript =
5039 // $@"
5040 // var theEvent = document.createEvent(""MouseEvent"");
5041 // theEvent.initMouseEvent(""{clickString}"", true, true, window, 0, 0, 0, {x}, {y}, {controlPressed.ToString().ToLower()}, {altPressed.ToString().ToLower()}, {shiftPressed.ToString().ToLower()}, false, 0, null);
5042 // let el = null;
5043 // const sel = `{selector.Replace("'", "\\'")}`;
5044
5045 // const el = document.querySelector(sel) || document.evaluate(sel, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
5046 // if (!el) return false;
5047 // el.dispatchEvent(theEvent);
5048 // ";
5049
5050
5051 //browser.PuppeteerCommunicator.SendCommand<object>(DevToolsMethods.RuntimeEvaluate, new
5052 //{
5053 // expression = clickScript,
5054 // returnByValue = true
5055 //}, sessionId).GetAwaiter().GetResult();
5056
5057 }
5058
5059 return true;
5060 }
5061
5062 // https://www.javascripttutorial.net/dom/css/check-if-an-element-is-visible-in-the-viewport/
5083 internal static void RefreshLocation(Browser browser, GPALElement element)
5084 {
5085 try
5086 {
5087 Rectangle live = new Rectangle(0, 0, 0, 0);
5088
5089 if (true == browser.UsePuppeteer && 0 != element.ElementBackendNodeId)
5090 {
5091 // the backend node id, not ElementHandle. that one is the objectId, and this endpoint parses
5092 // what it is given as an integer, so an objectId throws inside it and the whole call comes
5093 // back as nothing - which reads here as "could not be re-read" and leaves the stale position
5094 // in place, the one thing this method exists to stop
5095 live = browser.PuppeteerClient.GetBoundingClientRect(element.ElementBackendNodeId.ToString()).Execute<Rectangle>();
5096
5097 // Same rule as the OttoMagic branch below. A top level element is measured against the
5098 // page and HardwareMoveTo subtracts the scroll once, so it is added back here. An element
5099 // inside an iframe is measured against that iframe, whose recorded position already
5100 // carries the page scroll
5101 bool inPuppeteerFrame = browser.CurrentUOW?.ContextPath?.Any(context => ElementType.IFrame == context.elementType) ?? false;
5102
5103 if (0 < live.Width && 0 < live.Height && false == inPuppeteerFrame)
5104 live.Offset(browser.PuppeteerClient.WindowPageOffsetX().Execute<int>(),
5105 browser.PuppeteerClient.WindowPageOffsetY().Execute<int>());
5106 }
5107 else if (true == browser.UseOttoMagic)
5108 {
5109 live = browser.MagicHelper.GetBoundingClientRect(element.Css);
5110
5111 // getBoundingClientRect measures against whichever document the element is in. a top
5112 // level element is measured against the page, and the screen calculation takes the
5113 // scroll off once, so the scroll goes back on here to meet it. an element inside an
5114 // iframe is measured against that iframe, whose own position already carries the
5115 // page scroll, so putting it on here would count it twice
5116 bool inFrame = browser.CurrentUOW?.ContextPath?.Any(context => ElementType.IFrame == context.elementType) ?? false;
5117
5118 if (0 < live.Width && 0 < live.Height && false == inFrame)
5119 live.Offset(browser.MagicHelper.WindowPageOffsetX(), browser.MagicHelper.WindowPageOffsetY());
5120 }
5121 else if (null != element.WebElement)
5122 live = new Rectangle(element.WebElement.Location, element.WebElement.Size); // selenium reads these live, in document coordinates
5123
5124 if (0 < live.Width && 0 < live.Height)
5125 {
5126 element.Location = live.Location;
5127 element.Size = live.Size;
5128 element.BoundingRect = new ClientRectangle(element.Location, element.Size);
5129 }
5130 else
5131 // the position on file is where the element was when it was found. after a scroll that
5132 // is not where it is, and a hardware click aimed there lands on whatever moved into it
5133 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Could not re-read where [{element.TagName}][{element.Css}] is, so it is still where it was when it was found", browser, GPALObjectType.Browser);
5134 }
5135 catch (GPALException)
5136 {
5137 throw;
5138 }
5139 catch (Exception ex)
5140 {
5141 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Could not re-read where [{element.TagName}] is, using the reading from when it was found", browser, GPALObjectType.Browser, ex);
5142 }
5143 }
5144
5157 internal static bool PointHitsElement(Browser browser, GPALElement element, Point screenPoint)
5158 {
5159 bool retVal = false;
5160
5161 try
5162 {
5163 // elementFromPoint asks in viewport coordinates, so the browser's content area comes back off
5164 Rectangle content = GetWindowRectangle(browser);
5165 int x = screenPoint.X - content.X;
5166 int y = screenPoint.Y - content.Y;
5167
5168 if (true == browser.UsePuppeteer)
5169 retVal = browser.BrowserSettings.PuppeteerCommunicator.ElementFromPoint(element, x, y).GetAwaiter().GetResult();
5170 else if (true == browser.UseOttoMagic)
5171 retVal = browser.MagicHelper.ElementFromPoint(element.Css, x, y);
5172 else
5173 {
5174 var result = BrowserHelper.ExecuteJavaScriptObj(
5175 "var elem = arguments[0]; " +
5176 "var hit = document.elementFromPoint(arguments[1], arguments[2]); " +
5177 "return hit === elem || elem.contains(hit);", browser, element.WebElement, x, y);
5178
5179 retVal = null != result && true == Convert.ToBoolean(result);
5180 }
5181
5182 //GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Asked what is at viewport [{x}, {y}] (screen [{screenPoint.X}, {screenPoint.Y}], content origin [{content.X}, {content.Y}]) for [{element.TagName}][{element.Css}]: [{retVal}]", browser, GPALObjectType.Browser);
5183 }
5184 catch (GPALException)
5185 {
5186 throw;
5187 }
5188 catch (Exception ex)
5189 {
5190 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Could not ask what is at [{screenPoint.X}, {screenPoint.Y}] for [{element.TagName}]", browser, GPALObjectType.Browser, ex);
5191 }
5192
5193 return retVal;
5194 }
5195
5196 // how far in from the anchored edge to try, in pixels, before falling back to fractions of the region.
5197 // an element far wider than the window is anchored to something and overflowing away from it, and the
5198 // control that does anything sits at the anchored edge while the rest is empty box. a percentage of a
5199 // 3782px element says nothing about where a button a few dozen pixels wide is, and 2% of it is close
5200 // enough to the edge to land half off. a real control is at least this far in
5201 private static readonly int[] hitTestInsets = { 24, 40, 60, 12 };
5202
5203 // and failing all of those, across the region, so an element that is not anchored the way this assumes
5204 // still gets tried everywhere rather than only at one end
5205 private static readonly float[] hitTestFractions = { 0.75f, 0.50f, 0.25f, 0.10f };
5206
5207 public static bool IsVisibleInViewport(Browser browser, GPALElement element)
5208 {
5209 bool retval = false; // a check that cannot be answered is not a yes - callers use this to decide whether a coordinate click is safe
5210
5211 if (true == browser.UsePuppeteer)
5212 retval = browser.PuppeteerClient.IsVisibleInViewport(element).Execute<bool>();
5213 else if (true == browser.UseOttoMagic)
5214 retval = browser.MagicHelper.IsVisibleInViewport(element.Css);
5215 else
5216 {
5217 IWebDriver browserDriver = browser.BrowserDriver;
5218
5219 var result = BrowserHelper.ExecuteJavaScriptObj(
5220 "var elem = arguments[0]; " +
5221 "const rect = elem.getBoundingClientRect(); " +
5222 "return ( " +
5223 " rect.top >= 0 && " +
5224 " rect.left >= 0 && " +
5225 " rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && " +
5226 " rect.right <= (window.innerWidth || document.documentElement.clientWidth) " +
5227 ");", browser, element.WebElement);
5228
5229 if (null != result)
5230 retval = (bool)result;
5231 }
5232
5233 return retval;
5234 }
5241 public static bool IsEndOfPage(Browser browser, GPALElement element)
5242 {
5243 bool retval = true;
5244
5245 if (true == browser.UsePuppeteer)
5246 retval = browser.PuppeteerClient.IsEndOfPage().Execute<bool>();
5247 else if (true == browser.UseOttoMagic)
5248 retval = browser.MagicHelper.IsEndOfPage();
5249 else
5250 {
5251 IWebDriver browserDriver = browser.BrowserDriver;
5252 // documentElement.scrollHeight, not body.offsetHeight - body is shorter than the scrollable area
5253 // whenever html is the scroll container or body carries margins, which reports end of page on page one
5254 var result = BrowserHelper.ExecuteJavaScriptObj("if ((window.innerHeight + window.scrollY) >= document.documentElement.scrollHeight) { return true; } else { return false; }", browser, null);
5255
5256 if (null != result)
5257 retval = (bool)result;
5258 }
5259
5260 return retval;
5261 }
5271 private static bool SettledIsVisibleInViewport(Browser browser, GPALElement element)
5272 {
5273 bool visible = IsVisibleInViewport(browser, element);
5274 int checksLeft = VisibilitySettleChecks;
5275
5276 while (false == visible && 0 < checksLeft--)
5277 {
5278 Thread.Sleep(VisibilitySettleMs);
5279 visible = IsVisibleInViewport(browser, element);
5280 }
5281
5282 return visible;
5283 }
5300 private static bool IsBiggerThanViewport(Size elementSize, Rectangle viewport)
5301 {
5302 return elementSize.Width > viewport.Width || elementSize.Height > viewport.Height;
5303 }
5304 internal static bool ScrollUntilVisible(Browser browser, GPALElement element)
5305 {
5306 bool visible = SettledIsVisibleInViewport(browser, element);
5307
5308 // IsVisibleInViewport wants the whole element inside the viewport, so an element bigger than the
5309 // window can never answer yes and paging the document looking for one only wastes the time
5310 if (false == visible && true == IsBiggerThanViewport(element.Size, GetWindowRectangle(browser)))
5311 visible = true;
5312
5313 if (false == visible)
5314 {
5315 int pagesLeft = MaxScrollPages;
5316
5317 _ = browser.PageTop;
5318 visible = SettledIsVisibleInViewport(browser, element);
5319
5320 // page cap as well as end of page - a site whose scroll container we cannot measure would
5321 // otherwise walk the whole document a screen at a time on every failed click
5322 while (false == visible && 0 < pagesLeft-- && false == IsEndOfPage(browser, element))
5323 {
5324 _ = browser.PageDown;
5325 visible = SettledIsVisibleInViewport(browser, element);
5326 }
5327
5328 if (false == visible)
5329 {
5330 _ = browser.PageTop;
5331 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Could not scroll element [{element.TagName}][{element.Css}] into the viewport. Left the page at the top and continuing.", browser, GPALObjectType.Browser);
5332 }
5333 }
5334
5335 return visible;
5336 }
5337
5343 private static IntPtr GetWindowHandle(BrowserType browserType)
5344 {
5345 IntPtr hWnd = IntPtr.Zero;
5346 Process[] processes = null;
5347
5348 if (BrowserType.Chrome == browserType)
5349 processes = Process.GetProcessesByName("chrome");
5350 else if (BrowserType.Edge == browserType)
5351 processes = Process.GetProcessesByName("msedge");
5352 else if (BrowserType.FireFox == browserType)
5353 processes = Process.GetProcessesByName("firefox");
5354
5355 if (processes.Length > 0)
5356 {
5357 foreach (Process proc in processes)
5358 {
5359 if (proc.MainWindowHandle == IntPtr.Zero)
5360 continue;
5361 else
5362 {
5363 hWnd = proc.MainWindowHandle;
5364 break;
5365 }
5366 }
5367 }
5368 return hWnd;
5369 }
5370
5382 // Set true by HardwareClick when the element's screen placement was questionable (odd/off-screen coords,
5383 // so we could not do a real hardware click). HardwareFillInFrom reads it to refuse to type - otherwise a
5384 // failed click leaves focus wherever it was (e.g. the browser address bar) and keystrokes leak there.
5385 // ThreadStatic so concurrent browser threads do not clobber each other's last-click state.
5386 [ThreadStatic] private static bool _lastPlacementQuestionable;
5387
5388 // set by HardwareMoveTo when it had to clamp the point into the element's visible part, read by the click
5389 // that follows so the message says where it actually clicked. null when nothing had to be done
5390 [ThreadStatic] private static string _lastClampedPoint;
5391
5392 // the browser's content area as HardwareMoveTo last measured it, read by the click that follows it
5393 [ThreadStatic] private static Rectangle _lastContentArea;
5394
5395 // the outer window as GetWindowRectangle last saw it, position and size. that method answers with the
5396 // content area, which is what a click has to be computed against, but what a workflow sets with
5397 // WithWindowSize is the outer window - so that is what is worth quoting back to it, in the same
5398 // x, y, width, height order the Rectangle it takes is written in
5399 [ThreadStatic] private static Rectangle _lastOuterWindow;
5400
5401 // One mouse per machine, and SendInput acts on whatever has focus. Topping a window and clicking in
5402 // it must therefore be atomic across workflow threads, or one thread tops its window between another
5403 // thread topping and clicking
5404 private static readonly object hardwareMouse = new object();
5405
5406 public static void HardwareClick(Browser browser, dynamic element, ClickType clickType, ModifierKeys modifierKeys, Selector selector, bool isClickable = false)
5407 {
5408 Rectangle rect, iframeRect = new Rectangle(0, 0, 0, 0);
5409 bool clicked = false;
5410 _lastPlacementQuestionable = false;
5411 _lastClampedPoint = null;
5412
5413 // Held across topping and clicking, which are one operation
5414 lock (hardwareMouse)
5415 try
5416 {
5417 BrowserHelper.TopBrowser(browser, false);
5418
5419 if (ModifierKeys.NONE != modifierKeys)
5420 HardwareHelper.PressModifierKey(modifierKeys);
5421
5422 // element location calculations handled in hardwaremoveto, we use the rectangle computed there
5423 // will perform simulate mouse move
5424 rect = HardwareMoveTo(browser, element, selector);
5425
5426 // the point has to land inside the browser window on every side. checking only for positive
5427 // coordinates lets an element below the fold through with a large positive Y, and the click then
5428 // lands on the taskbar or whatever window is behind. an image match is already a verified screen
5429 // position, so it is the one thing allowed past
5430 // the content area HardwareMoveTo just measured. the window has not moved since, and measuring it
5431 // again is three more calls to the browser before the click
5432 Rectangle browserRect = _lastContentArea;
5433 bool insideBrowser = 0 < rect.Width && 0 < rect.Height
5434 && rect.X > browserRect.Left && rect.X < browserRect.Right
5435 && rect.Y > browserRect.Top && rect.Y < browserRect.Bottom;
5436
5437 if ("GPALElement" == element.TagName || true == insideBrowser)
5438 {
5439 // will do a jump to x,y
5440 HardwareHelper.HardwareClick(rect.X, rect.Y, clickType);
5441 clicked = true;
5442 }
5443 else
5444 {
5445 // The element's screen placement is not trustworthy (off-screen/odd coords) so we could not
5446 // perform a real hardware click. Flag it so a following hardware fill-in refuses to type.
5447 _lastPlacementQuestionable = true;
5448
5449 if (false == GPAL.NoFallbackRecoveryActions)
5450 {
5451 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Element [{element.TagName}] for selector [{selector?.Name}] computed to [{rect.X}, {rect.Y}, {rect.Width}, {rect.Height}], outside the browser window [{browserRect.Left}, {browserRect.Top}, {browserRect.Right}, {browserRect.Bottom}]. No hardware click, falling back on javascript click.", browser, GPALObjectType.Browser);
5452
5453 if (ModifierKeys.NONE != modifierKeys)
5454 HardwareHelper.ReleaseModifierKeys(modifierKeys);
5455
5456 if (true == browser.UseOttoMagic)
5457 browser.MagicHelper.LeftClick(element.Css);
5458 else if (true == browser.UsePuppeteer)
5459 browser.PuppeteerClient.LeftClick(element).Execute();
5460 else
5461 JavaScriptClick(browser, element, clickType, modifierKeys);
5462
5463 clicked = true;
5464 }
5465 else
5466 GPAL.PublishSimpleEvent(GPALEventType.WARNING, "Unable to get screen coordinates for webelement, continuing.", browser, GPALObjectType.Browser);
5467 }
5468 }
5469 catch
5470 {
5471 }
5472 finally
5473 {
5474 // a clamped point is worth saying out loud, and worth saying as a warning: the click landed, but
5475 // the workflow could name that offset itself and stop needing the correction
5476 GPAL.PublishSimpleEvent(GPALEventType.DEBUG, // null == _lastClampedPoint ? GPALEventType.INFO : GPALEventType.WARNING,
5477 $"Element [{element.TagName}] for selector [{selector.Name}] [{selector.InteractionType.ToString()}][{clickType.ToString()}]{(true == clicked ? "" : " [NOT]")} clicked{(null == _lastClampedPoint ? "" : " " + _lastClampedPoint)}.", browser, GPALObjectType.Browser);
5478
5479 if (ModifierKeys.NONE != modifierKeys)
5480 HardwareHelper.ReleaseModifierKeys(modifierKeys);
5481
5482 // BrowserHelper.TopBrowser(browser, true); // just move the mouse to a safe area off the browser area that can cause hover issues
5483 }
5484 }
5485
5492 public static void DragAndDrop(IBrowser browser, ModifierKeys modifierKeys = ModifierKeys.NONE)
5493 {
5494 var driver = browser.BrowserDriver;
5495 Rectangle iframeRect = new Rectangle(0, 0, 0, 0);
5496
5497 List<GPALElement> matchedElements;
5498
5499 foreach (Selector selector in ((Browser)browser).CurrentUOW.WithSelectorList)
5500 {
5501 if (SelectorType.Selector != selector.SelectorType)
5502 continue; // we don't look up data literals
5503
5504 ReadOnlyCollection<GPALElement> elems = ElementHelper.FindWebElements((Browser)browser, ((Browser)browser).CurrentUOW, selector, out bool matchedAll, out matchedElements);
5505
5506 // FindWebElements calls CallIF handlers which can handle an action and delete a node, check here before moving on
5507 if (true == selector.DeleteMe || null == elems || 0 == elems.Count)
5508 {
5509 PublishToEventHandler((Browser)browser, selector, elems?.ToList());
5510 continue;
5511 }
5512
5513 if (null != matchedElements)
5514 elems = new ReadOnlyCollection<GPALElement>(matchedElements);
5515
5516 int rowCount = 0;
5517
5518 foreach (GPALElement elem in elems)
5519 {
5520 if (true == ((Browser)browser).BrowserSettings.UseHardware || InteractionType.Hardware == selector.InteractionType)
5521 {
5522 // remote-desktop / image-matched pseudo-element - no DOM handle, fall back to hardware emulation
5523 // hold the mouse down, then walk the cursor smoothly (wind mouse) to the drop point before releasing -
5524 // a remote desktop session can drop the drag if the cursor jumps instantly to the target
5525 // Same lock as the click, held for the whole drag
5526 lock (hardwareMouse)
5527 try
5528 {
5529 BrowserHelper.TopBrowser((Browser)browser, false);
5530
5531 // element location calculations handled in hardwaremoveto (iframe nesting, browser chrome,
5532 // random in-element point when no offset), we use the rectangle computed there as the grab point
5533 Rectangle rect = HardwareMoveTo((Browser)browser, elem, selector);
5534
5535 if ("GPALElement" == elem.TagName || (0 <= rect.Width && 0 <= rect.Height && 0 < rect.X && 0 < rect.Y))
5536 {
5537 int endX = rect.X + selector.DeltaX;
5538 int endY = rect.Y + selector.DeltaY;
5539
5540 HardwareHelper.MouseDown(rect.X, rect.Y, ClickType.LeftClick);
5541 HardwareHelper.MoveMouse(endX, endY, 4, 4, 0.5);
5542 HardwareHelper.MouseUp(endX, endY, ClickType.LeftClick);
5543 }
5544 else
5545 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Odd screen coordinates [{rect.X}, {rect.Y}, {rect.Width}, {rect.Height}] for hardware drag and drop, skipping element.", browser, GPALObjectType.Browser);
5546 }
5547 finally
5548 {
5549 // BrowserHelper.TopBrowser((Browser)browser, true); // just move the mouse to a safe area off the browser area that can cause hover issues
5550 }
5551 }
5552 else if (true == browser.UseOttoMagic)
5553 {
5554 browser.MagicHelper.DragAndDrop(elem.Css, selector.DeltaX, selector.DeltaY, selector.OffsetX, selector.OffsetY);
5555 }
5556 else if (true == browser.UsePuppeteer)
5557 {
5558 browser.PuppeteerClient.DragAndDrop(elem.ElementHandle).WithDeltaX(selector.DeltaX).WithDeltaY(selector.DeltaY).WithOffsetX(selector.OffsetX).WithOffsetY(selector.OffsetY).Execute();
5559 }
5560 else if (true == browser.UseSelenium) // real WebElement, selenium has a handle to drag
5561 {
5562 // firefox's native drag service only starts from real OS input, so selenium's synthesized
5563 // pointer events never fire dragstart on an HTML5 draggable and the drag silently does nothing.
5564 // chrome and edge inject low enough that they do fire it, so only firefox needs the javascript
5565 // path - and only for HTML5 draggables, since mouse driven dragging (map panning, jquery ui)
5566 // is moved by plain mousedown/mousemove/mouseup and Actions drives that fine everywhere.
5567 if (BrowserType.FireFox == ((Browser)browser).BrowserSettings.BrowserType
5568 && "true" == elem.WebElement?.GetAttribute("draggable"))
5569 JavaScriptDragAndDrop((Browser)browser, elem, selector.DeltaX, selector.DeltaY, selector.OffsetX, selector.OffsetY);
5570 else
5571 new Actions(driver).ClickAndHold(elem.WebElement).MoveByOffset(selector.DeltaX, selector.DeltaY).Release().Perform();
5572 }
5573
5574 if (++rowCount >= ((Browser)browser).CurrentUOW.WithAllThatMatch)
5575 break;
5576 }
5577 }
5578 }
5579
5593 public static bool JavaScriptDragAndDrop(Browser browser, GPALElement element, int deltaX, int deltaY, int offsetX, int offsetY)
5594 {
5595 // this can happen with a gpalelement
5596 if (null == element?.WebElement)
5597 return false;
5598
5599 ScrollIntoView(browser, element);
5600
5601 // grab point is the element center plus the offset, matching the puppeteer drag and drop
5602 string javaScript = @"
5603 var src = arguments[0];
5604 var offsetX = arguments[1], offsetY = arguments[2];
5605 var deltaX = arguments[3], deltaY = arguments[4];
5606
5607 var rect = src.getBoundingClientRect();
5608 var startX = rect.left + rect.width / 2 + offsetX;
5609 var startY = rect.top + rect.height / 2 + offsetY;
5610 var endX = startX + deltaX;
5611 var endY = startY + deltaY;
5612
5613 var target = document.elementFromPoint(endX, endY);
5614
5615 if (!target)
5616 return 'nothing at drop point ' + endX + ',' + endY;
5617
5618 var dt = new DataTransfer();
5619
5620 function fire(type, el, x, y) {
5621 el.dispatchEvent(new DragEvent(type, {
5622 bubbles: true, cancelable: true, composed: true,
5623 clientX: x, clientY: y, dataTransfer: dt
5624 }));
5625 }
5626
5627 fire('dragstart', src, startX, startY);
5628 fire('dragenter', target, endX, endY);
5629 fire('dragover', target, endX, endY);
5630 fire('drop', target, endX, endY);
5631 fire('dragend', src, endX, endY);
5632
5633 return 'ok';
5634 ";
5635
5636 string result = BrowserHelper.ExecuteJavaScript(browser, javaScript, element.WebElement, offsetX, offsetY, deltaX, deltaY);
5637
5638 if ("ok" != result?.Trim())
5639 {
5640 GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"HTML5 drag and drop did not complete for selector [{element.Selector?.Name}] [{result}].", browser, GPALObjectType.Browser);
5641 return false;
5642 }
5643
5644 return true;
5645 }
5655 public static void JavaScriptHover(Browser browser, GPALElement element)
5656 {
5657 // this can happen with a gpalelement
5658 if (null == element.WebElement)
5659 return;
5660
5661 ScrollIntoView(browser, element);
5662
5663 // dispatch a mouseover event on the targetted element which is hover, not focus
5664 String javaScript = "var evObj = document.createEvent('MouseEvents');"
5665 + "evObj.initMouseEvent(\"mouseover\",true, false, window, 0, 0, 0, 0, 0, false, false, false, false, 0, null);"
5666 + "arguments[0].dispatchEvent(evObj);";
5667
5668 BrowserHelper.ExecuteJavaScript(browser, javaScript, element.WebElement);
5669 }
5670 // for some reason we are having trouble getting the msedge bounding rect
5677 private static Rectangle GetWindowRect(string windowTitle)
5678 {
5679 IntPtr hWnd = IntPtr.Zero;
5680 foreach (Process pList in Process.GetProcesses())
5681 {
5682 if (pList.MainWindowTitle.Contains(windowTitle))
5683 {
5684 hWnd = pList.MainWindowHandle;
5685 break;
5686 }
5687 }
5688
5689 if (hWnd != IntPtr.Zero)
5690 {
5691 RECT rect;
5692 GetWindowRect(hWnd, out rect);
5693 return new Rectangle(rect.Left, rect.Top, rect.Right - rect.Left, rect.Bottom - rect.Top);
5694 }
5695 else
5696 {
5697 return Rectangle.Empty;
5698 }
5699 }
5700
5707 [DllImport("user32.dll")]
5708 [return: MarshalAs(UnmanagedType.Bool)]
5709 private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
5710
5711 [StructLayout(LayoutKind.Sequential)]
5712 public struct RECT
5713 {
5714 public int Left;
5715 public int Top;
5716 public int Right;
5717 public int Bottom;
5718 }
5719
5733 private static void RestoreFrameContext(Browser browser)
5734 {
5735 if (null == browser.CurrentUOW?.ContextPath)
5736 return;
5737
5738 foreach (UnitOfWork.WebElementWithType contextType in browser.CurrentUOW.ContextPath)
5739 if (ElementType.IFrame == contextType.elementType && null != contextType.gPalElement?.WebElement)
5740 browser.BrowserDriver.SwitchTo().Frame((IWebElement)contextType.gPalElement.WebElement);
5741 }
5749 public static Rectangle GetWindowRectangle(Browser browser, out Rectangle outerWindow)
5750 {
5751 Rectangle retVal = GetWindowRectangle(browser);
5752
5753 // Set by the read above and ThreadStatic, so it belongs to the calling workflow
5754 outerWindow = _lastOuterWindow;
5755
5756 return retVal;
5757 }
5758
5759 public static Rectangle GetWindowRectangle(Browser browser)
5760 {
5761 // Primary: Use JS to get screen pos and sizes (reliable, avoids UIA flakiness)
5762 bool jsSuccess = false;
5763 int screenX = 0, screenY = 0, outerHeight = 0, innerHeight = 0, outerWidth = 0, innerWidth = 0;
5764
5765 try
5766 {
5767 bool withReturn = !browser.UsePuppeteer;
5768 string prefix = withReturn ? "return " : "";
5769 dynamic value = null;
5770
5771 if (true == browser.UseOttoMagic)
5772 {
5773 Rectangle rect = browser.MagicHelper.GetWindowRectangle();
5774 // { y: window.innerHeight, x: window.innerWidth, height: window.outerHeight, width: window.outerWidth }
5775 outerHeight = rect.Height;
5776 innerHeight = rect.Y;
5777 outerWidth = rect.Width;
5778 innerWidth = rect.X;
5779 screenX = browser.MagicHelper.WindowScreenLeft();
5780 screenY = browser.MagicHelper.WindowScreenTop();
5781 jsSuccess = true;
5782 }
5783 else if (true == browser.UsePuppeteer) // NOTE: not using javascript for now
5784 {
5785 outerHeight = browser.PuppeteerClient.WindowOuterHeight().Execute<int>();
5786 innerHeight = browser.PuppeteerClient.WindowInnerHeight().Execute<int>();
5787 outerWidth = browser.PuppeteerClient.WindowOuterWidth().Execute<int>();
5788 innerWidth = browser.PuppeteerClient.WindowInnerWidth().Execute<int>();
5789 screenX = browser.PuppeteerClient.WindowScreenLeft().Execute<int>();
5790 screenY = browser.PuppeteerClient.WindowScreenTop().Execute<int>();
5791 jsSuccess = true;
5792 }
5793 else
5794 {
5795 string jsScript = $"{prefix} ["
5796 + "window.screenLeft, "
5797 + "window.screenTop, "
5798 + "window.outerHeight, window.innerHeight, window.outerWidth, window.innerWidth]";
5799
5800 // these describe the browser window, not whatever frame we happen to be inside. selenium runs
5801 // script in the frame the driver is switched into, where innerHeight is the frame's own
5802 // viewport - that makes the computed chrome height enormous and drops the content area near
5803 // the bottom of the screen. measure against the top document and put the context back.
5804 bool inFrame = browser.CurrentUOW?.ContextPath?.Any(context => ElementType.IFrame == context.elementType) ?? false;
5805
5806 try
5807 {
5808 if (true == inFrame)
5809 browser.BrowserDriver.SwitchTo().DefaultContent();
5810
5811 value = BrowserHelper.ExecuteJavaScriptObj(jsScript, browser, null);
5812 }
5813 finally
5814 {
5815 if (true == inFrame)
5816 RestoreFrameContext(browser);
5817 }
5818 IList<object> values;
5819
5820 if (value is Newtonsoft.Json.Linq.JArray jArray)
5821 {
5822 values = jArray.Select(token => (object)token).ToList();
5823 }
5824 else if (value is IList<object> list)
5825 {
5826 values = list;
5827 }
5828 else
5829 {
5830 // Fallback if unexpected format (e.g., dict); assume defaults
5831 values = new List<object> { 0, 0, 0, 0, 0, 0 };
5832 }
5833
5834 screenX = values.Count > 0 ? Convert.ToInt32(values[0]) : 0;
5835 screenY = values.Count > 1 ? Convert.ToInt32(values[1]) : 0;
5836 outerHeight = values.Count > 2 ? Convert.ToInt32(values[2]) : 0;
5837 innerHeight = values.Count > 3 ? Convert.ToInt32(values[3]) : 0;
5838 outerWidth = values.Count > 4 ? Convert.ToInt32(values[4]) : 0;
5839 innerWidth = values.Count > 5 ? Convert.ToInt32(values[5]) : 0;
5840 jsSuccess = true;
5841 }
5842
5843 if (jsSuccess && innerWidth > 0 && innerHeight > 0)
5844 {
5845 _lastOuterWindow = new Rectangle(screenX, screenY, outerWidth, outerHeight);
5846
5847 int navHeight = outerHeight - innerHeight;
5848 int navWidth = outerWidth - innerWidth;
5849 // the window's border is the same width on the left, the right and the bottom, so half the
5850 // width the content area loses is the border beside it. the height it loses is the chrome
5851 // above it plus that same border below it, and only the chrome sits above the content, so
5852 // the border comes back off. charging the bottom border to the top is what drops every
5853 // hardware click by a border's width
5854 int border = navWidth / 2;
5855 int contentLeft = screenX + border;
5856 int contentTop = screenY + navHeight - border;
5857 return new Rectangle(contentLeft, contentTop, innerWidth, innerHeight);
5858 }
5859 }
5860 catch (GPALException)
5861 {
5862 throw;
5863 }
5864 catch (Exception ex)
5865 {
5866 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"JS execution failed for browser [{browser.BrowserSettings.BrowserType}]", browser, GPALObjectType.Browser, ex);
5867 }
5868
5869 // Reached when every read above returned zero. No fallback measurement is attempted
5870 GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Failed to get rectangle for [{browser.BrowserSettings.BrowserType}].", browser, GPALObjectType.Browser);
5871
5872 return Rectangle.Empty;
5873 }
5881 public static Rectangle GetWindowOuterRectangle(Browser browser)
5882 {
5883 int screenX = 0, screenY = 0, outerWidth = 0, outerHeight = 0;
5884
5885 try
5886 {
5887 if (true == browser.UseOttoMagic)
5888 {
5889 // { y: innerHeight, x: innerWidth, height: outerHeight, width: outerWidth }
5890 Rectangle rect = browser.MagicHelper.GetWindowRectangle();
5891 outerWidth = rect.Width;
5892 outerHeight = rect.Height;
5893 screenX = browser.MagicHelper.WindowScreenLeft();
5894 screenY = browser.MagicHelper.WindowScreenTop();
5895 }
5896 else if (true == browser.UsePuppeteer)
5897 {
5898 outerWidth = browser.PuppeteerClient.WindowOuterWidth().Execute<int>();
5899 outerHeight = browser.PuppeteerClient.WindowOuterHeight().Execute<int>();
5900 screenX = browser.PuppeteerClient.WindowScreenLeft().Execute<int>();
5901 screenY = browser.PuppeteerClient.WindowScreenTop().Execute<int>();
5902 }
5903 else
5904 {
5905 var value = BrowserHelper.ExecuteJavaScriptObj("return [window.screenLeft, window.screenTop, window.outerWidth, window.outerHeight]", browser, null);
5906 IList<object> values;
5907
5908 if (value is Newtonsoft.Json.Linq.JArray jArray)
5909 values = jArray.Select(token => (object)token).ToList();
5910 else if (value is IList<object> list)
5911 values = list;
5912 else
5913 values = new List<object> { 0, 0, 0, 0 };
5914
5915 screenX = values.Count > 0 ? Convert.ToInt32(values[0]) : 0;
5916 screenY = values.Count > 1 ? Convert.ToInt32(values[1]) : 0;
5917 outerWidth = values.Count > 2 ? Convert.ToInt32(values[2]) : 0;
5918 outerHeight = values.Count > 3 ? Convert.ToInt32(values[3]) : 0;
5919 }
5920
5921 return new Rectangle(screenX, screenY, outerWidth, outerHeight);
5922 }
5923 catch (GPALException)
5924 {
5925 throw;
5926 }
5927 catch (Exception ex)
5928 {
5929 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to get outer window rectangle for [{browser.BrowserSettings.BrowserType}]", browser, GPALObjectType.Browser, ex);
5930 return new Rectangle(0, 0, 0, 0);
5931 }
5932 }
5939 public static Dictionary<string, object> GetCssAttributes(Browser browser, IWebElement webElement)
5940 {
5941 var script = @"
5942 const el = arguments[0];
5943 const styles = window.getComputedStyle(el);
5944 var props = {};
5945
5946 // 2. Values that are ""noise"" - we never want them in the result
5947 const EMPTY_VALUES = new Set([
5948 '', 'none', 'normal', 'inherit', 'initial', 'unset',
5949 'auto', '0px', '0', 'transparent'
5950 ]);
5951
5952 // 3. The only properties we ever care about
5953 const RELEVANT = new Set([
5954 'display', 'visibility', 'opacity', 'pointer-events', 'overflow', 'clip-path',
5955 'position', 'top', 'left', 'right', 'bottom', 'z-index',
5956 'width', 'height', 'min-width', 'min-height', 'max-width', 'max-height',
5957 'margin', 'margin-top', 'margin-right', 'margin-bottom', 'margin-left',
5958 'padding', 'padding-top', 'padding-right', 'padding-bottom', 'padding-left',
5959 'border', 'border-width', 'border-style', 'border-color', 'box-sizing',
5960 'flex', 'flex-direction', 'flex-wrap', 'flex-grow', 'flex-shrink', 'flex-basis',
5961 'justify-content', 'align-items', 'align-self', 'order',
5962 'grid', 'grid-template', 'grid-area', 'grid-column', 'grid-row',
5963 'transform', 'transform-origin', 'transition', 'animation',
5964 'cursor', 'user-select', 'content', 'filter'
5965 ]);
5966
5967 // 4. Iterate only the properties that exist on the style object
5968 for (let i = 0; i < styles.length; i++) {
5969 const prop = styles[i]; // e.g. ""display""
5970 if (!RELEVANT.has(prop)) { continue; } // skip everything else
5971
5972 const value = styles.getPropertyValue(prop).trim(); // ""block"", ""0.5"", etc.
5973
5974 // Skip empty / default values
5975 if (!value || EMPTY_VALUES.has(value)) { continue; }
5976
5977 props[prop] = value;
5978 }
5979 return props;
5980 ";
5981
5982 var dict = BrowserHelper.ExecuteJavaScriptObj(script, browser, webElement) as Dictionary<string, object>;
5983
5984 return dict?.ToDictionary(k => k.Key, v => v.Value ?? "");
5985 }
5990 public static Dictionary<string, object> GetDomAttributes(
5991 Browser browser,
5992 IWebElement webElement)
5993 {
5994 const string script = @"
5995 const cleanText = (str) => (str || '').replace(/\s+/g, ' ').trim();
5996 const el = arguments[0];
5997 const attrs = el.attributes;
5998 const result = {};
5999 for (let i = 0; i < attrs.length; i++) {
6000 const a = attrs[i];
6001 result[a.name] = a.value; // value is always a string or empty
6002 }
6003 // Also add a few handy built-in properties that are not attributes
6004 result['id'] = el.id;
6005 result['className'] = el.className;
6006 result['tagName'] = el.tagName;
6007 result['innerText'] = cleanText(el.innerText);
6008 result['textContent'] = cleanText(el.textContent);
6009 return result;
6010 ";
6011
6012 var raw = BrowserHelper.ExecuteJavaScriptObj(script, browser, webElement)
6013 as Dictionary<string, object>;
6014
6015 // Normalise null -> empty string (mirrors your CSS method)
6016 return raw?.ToDictionary(
6017 kvp => kvp.Key,
6018 kvp => kvp.Value ?? ""
6019 ) ?? new Dictionary<string, object>();
6020 }
6025 public static Dictionary<string, object> GetDomProperties(
6026 Browser browser,
6027 IWebElement webElement)
6028 {
6029 const string script = @"
6030 const el = arguments[0];
6031 const props = {};
6032
6033 if (""textContent"" in el) props.textContent = el.textContent;
6034 if (""innerHTML"" in el) props.innerHTML = el.innerHTML;
6035 if (""innerText"" in el) props.innerText = el.innerText;
6036 if (""outerHTML"" in el) props.outerHTML = el.outerHTML;
6037 if (""className"" in el) props.className = el.className;
6038 if (""disabled"" in el) props.disabled = el.disabled;
6039
6040 return props;
6041 ";
6042
6043 var raw = BrowserHelper.ExecuteJavaScriptObj(script, browser, webElement)
6044 as Dictionary<string, object>;
6045
6046 // Normalize null -> "" (consistent with your CSS method)
6047 return raw?.ToDictionary(
6048 kvp => kvp.Key,
6049 kvp => kvp.Value ?? ""
6050 ) ?? new Dictionary<string, object>();
6051 }
6064 internal static bool ResolveWebElementIssue(Browser browser, UnitOfWork currentUOW, ref dynamic webOrGpalElement, Exception exception, ref int retries, string errorMessage)
6065 {
6066 // no fallback
6067 if (true == GPAL.NoFallbackRecoveryActions)
6068 return false;
6069
6070 bool shouldRetry = false;
6071 IWebElement iWebElement;
6072 GPALElement gPalElement = null;
6073
6074 if (webOrGpalElement is GPALElement)
6075 {
6076 gPalElement = webOrGpalElement;
6077 iWebElement = gPalElement.WebElement;
6078 }
6079 else
6080 {
6081 iWebElement = webOrGpalElement;
6082 gPalElement = new GPALElement() { WebElement = iWebElement };
6083 }
6084
6085 // Log the recovery attempt
6086 string recoveryMessage = $"Attempting recovery for [{exception.GetType().Name}]. Retries left: [{retries}]";
6087 GPAL.PublishSimpleEvent(GPALEventType.INFO, recoveryMessage, iWebElement, GPALObjectType.Other);
6088
6089 try
6090 {
6091 if (exception is StaleElementReferenceException)
6092 {
6093 // clear the cache, update GPALElement.IWebElement - tested, recovery works for stale cache
6094 if (null != currentUOW.CurrentSelector.WebSelectorMatchedResults || null != currentUOW.CurrentSelector.WebSelectorFoundResults)
6095 {
6096 currentUOW.CurrentSelector.WebSelectorMatchedResults = null;
6097 currentUOW.CurrentSelector.WebSelectorFoundResults = null;
6098 }
6099 else
6100 // Step 2: Refresh page (no locator available)
6101 _ = browser.Refresh;
6102
6103 if (null != webOrGpalElement)
6104 {
6105 var elems = FindWebElements(browser, currentUOW, currentUOW.CurrentSelector, out bool matchedAll, out List<GPALElement> matched, null, false);
6106
6107 if (0 < elems.Count())
6108 webOrGpalElement = elems[0].WebElement;
6109 }
6110 shouldRetry = true;
6111 }
6112 else if (exception is ElementClickInterceptedException)
6113 {
6114 // Step 2: Scroll element into view
6115 ElementHelper.MoveTo(gPalElement, currentUOW.CurrentSelector.OffsetX, currentUOW.CurrentSelector.OffsetY, browser);
6116
6117 // Step 3: Wait for element to be clickable (visible and enabled)
6118 if (false == (shouldRetry = IsElementClickable(iWebElement)))
6119 {
6120 // CAVEAT: this uses a heuristic, no guarantees
6121 // Step 4: Handle overlays (modals, popups)
6122 try
6123 {
6124 var overlays = browser.BrowserDriver.FindElements(By.XPath("//div[contains(@class, 'modal') or contains(@class, 'overlay') or contains(@class, 'popup')]"));
6125 if (overlays.Any())
6126 {
6127 foreach (var overlay in overlays)
6128 {
6129 if (overlay.Displayed)
6130 {
6131 var closeButton = browser.BrowserDriver.FindElements(By.XPath(".//button[contains(@class, 'close') or contains(text(), 'Close') or @aria-label='Close']"));
6132 if (closeButton.Any())
6133 {
6134 closeButton.First().Click();
6135 shouldRetry = IsElementClickable(iWebElement);
6136 }
6137 else
6138 {
6139 BrowserHelper.ExecuteJavaScriptObj("arguments[0].style.display='none';", browser, overlay);
6140 }
6141 }
6142 }
6143 }
6144 }
6145 catch (NoSuchElementException)
6146 {
6147 // No overlay found, proceed
6148 }
6149 }
6150 }
6151 else if (exception is ElementNotInteractableException)
6152 {
6153 // Step 2: Scroll element into view (replaces JavaScriptMoveTo)
6154 ElementHelper.MoveTo(gPalElement, currentUOW.CurrentSelector.OffsetX, currentUOW.CurrentSelector.OffsetY, browser);
6155
6156 // Step 3: Wait for element to be clickable (visible and enabled)
6157 // Step 4: Adjust for sticky elements (e.g., headers)
6158 if (false == IsElementClickable(iWebElement))
6159 {
6160 browser.BrowserSettings.SuppressJavaScriptFailures = true;
6161 browser.ExecuteJavaScriptStr("window.scrollBy(0, -100, behavior: 'instant');"); // Scroll up slightly if needed
6162 browser.BrowserSettings.SuppressJavaScriptFailures = false;
6163 }
6164
6165 shouldRetry = IsElementClickable(iWebElement);
6166 }
6167 }
6168 catch (GPALException)
6169 {
6170 throw;
6171 }
6172 catch (Exception ex)
6173 {
6174 // Log recovery failure
6175 GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Recovery failed", iWebElement, GPALObjectType.Other, ex);
6176 shouldRetry = false;
6177 }
6178
6179 return shouldRetry && retries > 0;
6180 }
6186 internal static bool IsElementClickable(IWebElement webElement)
6187 {
6188 return webElement.Displayed && webElement.Enabled;
6189 }
6190 internal static bool IsElementClickable(GPALElement element)
6191 {
6192 return element.Displayed && element.Enabled;
6193 }
6200 public static bool IsElementEditable(GPALElement elem)
6201 {
6202 try
6203 {
6204 // Get the tag name and type
6205 string tag = elem?.WebElement?.TagName.ToUpperInvariant() ?? elem?.TagName.ToUpperInvariant();
6206 string type = string.Empty;
6207 if (tag == "INPUT")
6208 {
6209 type = elem.GetAttribute("type")?.ToLower();
6210 }
6211
6212 // Check CSS cursor property
6213 string cursor = elem?.GetCssValue("cursor")?.ToLower();
6214 if (cursor == "text")
6215 return true;
6216
6217 // Check tag/type/contentEditable
6218 bool retval = (tag == "INPUT" &&
6219 (type == "text" || type == "email" || type == "password" || type == "search"))
6220 || tag == "TEXTAREA"
6221 || (tag == "DIV" && elem.GetAttribute("contenteditable")?.ToLower() == "true");
6222
6223 // Optional: check for editable classes
6224 if (!retval)
6225 {
6226 string classAttr = elem?.GetAttribute("class") ?? "";
6227 if (null != classAttr)
6228 retval = classAttr.Split(' ').Contains("Editable");
6229 }
6230
6231 return retval;
6232 }
6233 catch
6234 {
6235 // Fail-forward: if anything throws, consider editable as the developer thinks it is
6236 return true;
6237 }
6238 }
6254 public static void SendString(Browser browser, string textToSend)
6255 {
6256 int delayMs = GPAL.TypingDelay;
6257
6258 if (true == browser.BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware)
6259 HardwareHelper.SendString(textToSend, delayMs);
6260 else if (true == browser.BrowserSettings.UseOttoMagic)
6261 browser.MagicHelper.SendString(textToSend, delayMs);
6262 else if (true == browser.BrowserSettings.UsePuppeteer)
6263 browser.PuppeteerClient.SendString(textToSend, delayMs);
6264 else
6265 {
6266 foreach (SelectorSettings.SelectorPathEntry selectorPathEntry in browser.CurrentUOW.CurrentSelector?.SelectorPaths)
6267 if (null != selectorPathEntry.WebSelectorFoundResults)
6268 foreach (GPALElement webElement in selectorPathEntry.WebSelectorFoundResults)
6269 if (InteractionType.JavaScript == browser.CurrentUOW.CurrentSelector?.InteractionType || true == GPAL.GPALSettings.UseJavaScript)
6270 {
6271 if (0 == delayMs)
6272 BrowserHelper.ExecuteJavaScriptObj(@"
6273 function sendStringBlindly(text) {
6274 for (let char of text) {
6275 document.dispatchEvent(new KeyboardEvent('keydown', {{
6276 key: char,
6277 bubbles: true
6278 }}));
6279 document.dispatchEvent(new KeyboardEvent('keyup', {{
6280 key: char,
6281 bubbles: true
6282 }}));
6283 }
6284 }" +
6285 $"sendStringBlindly({textToSend});", browser);
6286 else
6287 {
6288 bool first = true;
6289 foreach (char c in textToSend)
6290 {
6291 if (false == first)
6292 Thread.Sleep(HardwareHelper.GetTypingDelay(delayMs));
6293 BrowserHelper.ExecuteJavaScriptObj(@"
6294 var char = arguments[0];
6295 document.dispatchEvent(new KeyboardEvent('keydown', { key: char, bubbles: true }));
6296 document.dispatchEvent(new KeyboardEvent('keyup', { key: char, bubbles: true }));",
6297 browser, c.ToString());
6298 first = false;
6299 }
6300 }
6301 }
6302 else // selenium - use the first element we have found or the body if no selector was set
6303 {
6304 IWebElement body = browser.BrowserSettings.BrowserDriver.FindElement(By.TagName("body"));
6305 // Use Actions to simulate typing each character with key down and key up
6306 Actions actions = new Actions(browser.BrowserSettings.BrowserDriver);
6307 bool first = true;
6308 foreach (char c in textToSend)
6309 {
6310 if (false == first && 0 < delayMs)
6311 actions.Pause(TimeSpan.FromMilliseconds(HardwareHelper.GetTypingDelay(delayMs)));
6312 actions
6313 .KeyDown(body, c.ToString())
6314 .KeyUp(body, c.ToString());
6315 first = false;
6316 }
6317 actions.Build().Perform();
6318 }
6319
6320 }
6321 }
6329 public static void SendKey(Browser browser, byte VKCode)
6330 {
6331 if (true == browser.BrowserSettings.UseHardware || true == GPAL.GPALSettings.UseHardware)
6332 {
6333 if (GPAL.VK_RETURN == VKCode)
6334 {
6335 var matched = browser.CurrentUOW.CurrentSelector?.WebSelectorMatchedResults?.FirstOrDefault();
6336 if (false == ElementHelper.IsElementEditable(matched))
6337 {
6338 matched.Click();
6339 return;
6340 }
6341 }
6342
6343 HardwareHelper.SendKey(VKCode);
6344 }
6345 else if (true == browser.BrowserSettings.UseOttoMagic)
6346 browser.MagicHelper.SendKey(VKCode);
6347 else if (true == browser.BrowserSettings.UsePuppeteer)
6348 browser.PuppeteerClient.SendKey(VKCode).Execute();
6349 else
6350 {
6351 {
6352 foreach (SelectorSettings.SelectorPathEntry selectorPathEntry in browser.CurrentUOW.CurrentSelector?.SelectorPaths)
6353 if (null != selectorPathEntry.WebSelectorFoundResults)
6354 foreach (GPALElement webElement in selectorPathEntry.WebSelectorFoundResults)
6355 if (InteractionType.JavaScript == browser.CurrentUOW.CurrentSelector?.InteractionType)
6356 ElementHelper.JavaScriptSendKey(browser, webElement, VKCode);
6357 else
6358 ElementHelper.SeleniumSendKeys(browser, webElement, VKCode);
6359 }
6360 }
6361 }
6362 public static string GetEnumNames<T>(T value) where T : struct, Enum
6363 {
6364 if (!typeof(T).IsDefined(typeof(FlagsAttribute), false))
6365 return value.ToString(); // or throw, or return ""
6366
6367 var flags = Enum.GetValues(typeof(T))
6368 .Cast<T>()
6369 .Where(flag => !Equals(flag, default(T)) && value.HasFlag(flag))
6370 .Select(flag => flag.ToString());
6371
6372 string retVal = string.Join(", ", flags);
6373
6374 return false == string.IsNullOrEmpty(retVal) ? retVal : "NONE";
6375 }
6383 public static string GenerateCssSelector(IBrowser browser, IWebElement webElement)
6384 {
6385 return BrowserHelper.ExecuteJavaScript(browser, @"
6386 function getCssPath(el) {
6387 if (!(el instanceof Element))
6388 return null;
6389
6390 var path = [];
6391
6392 while (el && el.nodeType === Node.ELEMENT_NODE) {
6393
6394 if (el.id) {
6395 path.unshift('#' + CSS.escape(el.id));
6396 break;
6397 }
6398
6399 var selector = el.nodeName.toLowerCase();
6400
6401 var sibling = el;
6402 var nth = 1;
6403
6404 while ((sibling = sibling.previousElementSibling) != null) {
6405 if (sibling.nodeName === el.nodeName)
6406 nth++;
6407 }
6408
6409 selector += ':nth-of-type(' + nth + ')';
6410
6411 path.unshift(selector);
6412
6413 el = el.parentElement;
6414 }
6415
6416 return path.join(' > ');
6417 }
6418 return getCssPath(arguments[0]);
6419 ", webElement);
6420 }
6421 }
6422}
6423
GenerallyPositive.Browser.Browser
Browser object that contains the fluent methods to create your Browser workflow. Instatiated using G...
Definition Browser.cs:68
GenerallyPositive.GPALElement
Pseudo element used in Applications and Browser workflows for image matching and unified automation....
Definition GPALElement.cs:143
GenerallyPositive.GPALElement.TagName
string TagName
HTML tag name of the element.
Definition GPALElement.cs:296
GenerallyPositive.GPALElement.Css
string Css
CSS selector used to locate this element.
Definition GPALElement.cs:448
GenerallyPositive.GPALElement.WebElement
dynamic WebElement
The wrapped Selenium IWebElement or internal dynamic element.
Definition GPALElement.cs:256
GenerallyPositive.GPAL
Everything starts here, all the GPAL controls and global settings using fluent syntax are here....
Definition GPAL.cs:49
GenerallyPositive.GPAL.PublishSimpleEvent
static void PublishSimpleEvent(GPALEventType gPALEventType, string msg, dynamic gPALObject=null, Enums.GPALObjectType gPALObjectType=GPALObjectType.None, Exception ex=null)
Publish a message to either the information channel or exception channel (if exception passed in) Pub...
Definition GPAL.cs:2568
GenerallyPositive.Selector
GPAL Selector used to locate Application and Browser elements. Instantiated with GPAL....
Definition Selector.cs:56
GenerallyPositive.Selector.DeltaX
int DeltaX
Drag-and-drop deltaX to move from the elemenet.X + OffsetX.
Definition Selector.cs:838
GenerallyPositive.Selector.DeltaY
int DeltaY
Drag-and-drop deltaY to move from the elemenet.Y + OffsetY.
Definition Selector.cs:853
GenerallyPositive.Selector.OffsetY
int OffsetY
Y offset fromm the top left corner to interact with the element, image match included....
Definition Selector.cs:823
GenerallyPositive.Selector.InteractionType
InteractionType InteractionType
Defined method to interact with this element. NOTE: This can be overridden in the workflow.
Definition Selector.cs:743
GenerallyPositive.Selector.OffsetX
int OffsetX
X offset from the top left corner to interact with the element, image match included....
Definition Selector.cs:807
GenerallyPositive.SelectorSettings
Settings for the current selector. Use this only for debugging.
Definition SelectorSettings.cs:35