ImageHelper.cs

// =============================================================================
// GPAL - Generally Positive Automation Library
// Copyright © 2026 Software Decisions, Inc. All rights reserved.
//
// This file is part of GPAL.
// Licensed under the Business Source License 1.1
//
// Primary development, architecture, and vision by Michael B. Vederman,
// CEO of Software Decisions, Inc., Texas.
//
// Internal development maintained privately.
// Public releases appear on GitHub: https://github.com/SoftwareDecisionsInc/GPAL.
//
// See LICENSE for full terms, including Additional Use Grant.
// =============================================================================
 
using System;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Windows.Forms;
using GenerallyPositive.Browser;
using OpenQA.Selenium;
 
namespace GenerallyPositive
{
    public interface IImageMatcher
    {
        Rectangle FindImage(Bitmap screen, string templatePath, int matchingPercentage);
    }

    public class ImageHelper : IImageHelper
    {
        public Browser.Browser Browser { get; set; }

        public static Bitmap ScreenShot { get; set; }
 
        private static string base64StringInternal = null;
 
 
        /************** CAPTURE SCREEN ******************/
        // https://stackoverflow.com/questions/4978157/how-to-search-for-an-image-on-screen-in-c
        // TODO: Capture screen for Applications

        public IAllowGraphicSettings CaptureScreen(object ApplicationOrBrowser = null)
        {
            // what a browser handed back last time belongs to that capture and to nothing after it. Left set,
            // it is what ToBase64String and SaveTo return forever, so a desktop capture that follows a browser
            // one hands back the browser picture
            base64StringInternal = null;
 
            bool headlessMode = false;   // assume headless as we might be called before we even have a browser created
            Bitmap image = new Bitmap(1920, 1080);
            bool isBrowser = typeof(Browser.Browser) == ApplicationOrBrowser?.GetType();
            if (true == isBrowser)
                // a browser on a hidden desktop is headful, but the screen is showing the desktop the user is on
                // and not that one, so copying the screen would quietly hand back a picture of their desktop. the
                // browser is asked for the image instead, which is the same route headless takes
                // an image search that will not click with the mouse wants the page, not the screen, windowed too
                headlessMode = ((Browser.Browser)ApplicationOrBrowser).BrowserSettings.UseHeadless
                            || ((Browser.Browser)ApplicationOrBrowser).BrowserSettings.HiddenDesktop
                            || ((Browser.Browser)ApplicationOrBrowser).BrowserSettings.ImageSearchInPage;
 
            try
            {
                image = new Bitmap(Screen.PrimaryScreen.Bounds.Width, Screen.PrimaryScreen.Bounds.Height, PixelFormat.Format32bppArgb);
            }
            catch
            {
                try
                {
                    image = new Bitmap(Screen.PrimaryScreen.Bounds.Width, Screen.PrimaryScreen.Bounds.Height, PixelFormat.Format32bppArgb);
                }
                catch
                {
                    // can't use publishevent, would be recursive
                    //Console.WriteLine("new Bitmap failed: Unable to take screenshot");
                    System.Diagnostics.Debug.WriteLine("new Bitmap failed: Unable to take screenshot");
                    return this;
                }
            }
 
            if (false == headlessMode)
            {
                var gfx = Graphics.FromImage(image);
                try
                {
                    gfx.CopyFromScreen(Screen.PrimaryScreen.Bounds.X, Screen.PrimaryScreen.Bounds.Y, 0, 0, Screen.PrimaryScreen.Bounds.Size, CopyPixelOperation.SourceCopy);
                }
                catch { }
            }
            else if (true == isBrowser)
            {
                Browser = (Browser.Browser)ApplicationOrBrowser;
 
                try
                {
                    byte[] imageBytes = null;
 
                    if (true == ((Browser.Browser)ApplicationOrBrowser).UseSelenium)
                    {
                        if (null != ((Browser.Browser)ApplicationOrBrowser).BrowserDriver)
                        {
                            Screenshot screenshot = ((ITakesScreenshot)((Browser.Browser)ApplicationOrBrowser).BrowserDriver).GetScreenshot();
                            // Convert the screenshot to a Bitmap
                            base64StringInternal = screenshot.AsBase64EncodedString;
                        }
                    }
                    else if (0 < GPAL.Browsers.Count())
                    {
                        foreach (Browser.Browser browser in GPAL.Browsers)
                            if (null != browser.BrowserSettings.Process)
                            {
                                if (true == ((Browser.Browser)ApplicationOrBrowser).UsePuppeteer)
                                {
                                    base64StringInternal = ((Browser.Browser)ApplicationOrBrowser).PuppeteerClient.CaptureVisibleTab().Execute();
                                }
                                else if (((Browser.Browser)ApplicationOrBrowser).UseOttoMagic)
                                {
                                    base64StringInternal = ((Browser.Browser)ApplicationOrBrowser).MagicHelper.CaptureVisibleTab();
                                }
 
                                break;
                            }
                    }
                    char[] ca = { ',' };
 
                    // remove the header info, we want the raw data
                    // "data:image/png;base64,iVBORw0 ... "
                    try
                    {
                        base64StringInternal = base64StringInternal.Trim()
                                            .Trim('"')                    // remove surrounding quotes if present
                                            .Split(ca, 2)                 // split on first comma only
                                            [1];                          // take everything after the comma
                    }
                    catch
                    {
                        base64StringInternal = base64StringInternal.Trim()
                                            .Trim('"');                    // remove surrounding quotes if present
                    }
 
                    // also trim any leftover whitespace (rare but safe)
                    base64StringInternal = base64StringInternal.Trim();
 
                    imageBytes = Convert.FromBase64String(base64StringInternal);
                    using (MemoryStream ms = new MemoryStream(imageBytes))
                        // Create an Image object from the MemoryStream
                        image = new Bitmap(ms);
                }
                catch (Exception ex)
                {
                    string outMessageHeader = $@"[{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")}][EXCEPTION][ImageHelper]: ";
 
                    // can't use publishevent, would be recursive: publishing an exception takes a screenshot, and
                    // this is the screenshot. written directly instead, and silent still means silent
                    if ("1" != Environment.GetEnvironmentVariable("GPAL_IS_SILENT"))
                    {
                        Console.WriteLine($"{outMessageHeader} [{Browser.AutomationEngine}][{Browser.BrowserType}][{(true == Browser.BrowserSettings.UseHeadless ? "Headless" : "Headful")}] Unable to take screenshot [{ex.Message}]");
                        System.Diagnostics.Debug.WriteLine($"{outMessageHeader} [{Browser.AutomationEngine}][{Browser.BrowserType}][{(true == Browser.BrowserSettings.UseHeadless ? "Headless" : "Headful")}] Unable to take screenshot [{ex.Message}]");
                    }
                }
            }
            ScreenShot = image;
 
            return this;
        }

        public IAllowGraphicSettings ToBase64String(out string base64String)
        {
            System.IO.MemoryStream ms = new MemoryStream();
            if (null != base64StringInternal)
                base64String = base64StringInternal;
            else if (null != ScreenShot)
            {
                ScreenShot.Save(ms, ImageFormat.Jpeg);
                byte[] byteImage = ms.ToArray();
                base64String = Convert.ToBase64String(byteImage);
            }
            else
                base64String = string.Empty;
 
            return this;
        }

        public IAllowGraphicSettings ToBitmap(out Bitmap bitmap)
        {
            bitmap = ScreenShot;
            return this;
        }

        [File(Description = "Save to file (overwrite)")]
        public IAllowGraphicSettings SaveTo(GPALFile file)
        {
            SaveScreenshot(file.Next, ScreenShot);
 
            return this;
        }

        private static void SaveScreenshot(string fileName, Image screenshot)
        {
            try
            {
                if (null != screenshot)
                {
                    string saveToDirectory = Path.GetDirectoryName(fileName);
 
                    // ensure save to directory exists
                    if (false == string.IsNullOrEmpty(saveToDirectory))
                        Directory.CreateDirectory(saveToDirectory);
 
                    // the extension says the format, so a .png really is a PNG. Encoding one way whatever the
                    // name says leaves a file that lies about what is in it
                    screenshot.Save(fileName, FormatForFile(fileName));
 
                    GPAL.PublishSimpleEvent(Enums.GPALEventType.INFO, $"Screenshot saved to [{fileName}]");
                }
                else
                    GPAL.PublishSimpleEvent(Enums.GPALEventType.WARNING, "No screenshot to save");
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(Enums.GPALEventType.EXCEPTION, $"No screenshot saved to [{fileName}]", null, Enums.GPALObjectType.None, ex);
            }
        }

        internal static System.Drawing.Imaging.ImageFormat FormatForFile(string filename)
        {
            System.Drawing.Imaging.ImageFormat retVal;
 
            switch (Path.GetExtension(filename ?? string.Empty).ToLower())
            {
                case ".jpg":
                case ".jpeg":
                    retVal = System.Drawing.Imaging.ImageFormat.Jpeg;
                    break;
                case ".bmp":
                    retVal = System.Drawing.Imaging.ImageFormat.Bmp;
                    break;
                case ".gif":
                    retVal = System.Drawing.Imaging.ImageFormat.Gif;
                    break;
                case ".tif":
                case ".tiff":
                    retVal = System.Drawing.Imaging.ImageFormat.Tiff;
                    break;
                default:
                    retVal = System.Drawing.Imaging.ImageFormat.Png;
                    break;
            }
 
            return retVal;
        }
 
        #region Helpers

        public static System.Drawing.Image LoadImageFromFile(string filename)
        {
            return System.Drawing.Image.FromFile(filename);
        }

        public static Image Base64ToImage(string base64String)
        {
            base64String = base64String.Trim().Trim('\"');
 
            // a data URI names the media type ahead of the payload, so only that header comes off. The pattern
            // is anchored to the front and stops at the comma: matching the payload too would take the image
            // with the header and leave nothing to decode
            string pattern = @"^data:[^;]+;base64,";
            base64String = System.Text.RegularExpressions.Regex.Replace(base64String, pattern, string.Empty);
 
            // Convert Base64 String to byte[]
            byte[] imageBytes = Convert.FromBase64String(base64String);
 
            // the stream holds the bytes already and is left open, because an Image reads its pixels from the
            // stream underneath it and throws once that is gone
            MemoryStream ms = new MemoryStream(imageBytes, 0, imageBytes.Length);
 
            Image image = Image.FromStream(ms, true);
 
            return image;
        }

        public Rectangle FindImage(string imagePath, object ApplicationOrBrowser = null)
        {
            Rectangle retVal = Rectangle.Empty;
 
            if (null == GPAL.ImageMatcher)
            {
                // said once, since a search pages down the whole document asking again at every step
                if (false == matcherMissingSaid)
                {
                    matcherMissingSaid = true;
                    GPAL.PublishSimpleEvent(Enums.GPALEventType.WARNING, $"Image matching needs the GPAL.Image package, added and registered with GPALImage.Register() at startup, so [{imagePath}] was not looked for");
                }
            }
            else
            {
                try
                {
                    // pass the browser through so a headless run matches against a visible-tab capture instead of the
                    // physical screen (which has no window on it to find). the coordinates that come back are then
                    // viewport relative rather than screen absolute - see GPALElement.CoordinateSpace.
                    CaptureScreen(ApplicationOrBrowser);
 
                    retVal = GPAL.ImageMatcher.FindImage(ScreenShot, imagePath, GPAL.MatchingPercentage);
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(Enums.GPALEventType.EXCEPTION, $"Error locating [{imagePath}] on-screen.", null, Enums.GPALObjectType.None, ex);
                }
            }
 
            return retVal;
        }
 
        // whether the missing matcher has been reported yet
        private static bool matcherMissingSaid;
        #endregion Helpers

        public IImageHelper ToGPALObject()
        {
            return this;
        }
    }
 
    // extension ToByteArray for CV
    internal static class ImageExtensions
    {
        public static byte[] ToByteArray(this System.Drawing.Image image, ImageFormat format)
        {
            using (MemoryStream ms = new MemoryStream())
            {
                image.Save(ms, format);
                return ms.ToArray();
            }
        }
    }
}