ImageHelper.cs

// =============================================================================
// GPAL - Generally Positive Automation Library
// Copyright © 2026 Software Decisions, Inc. All rights reserved.
//
// This file is part of GPAL.
// Licensed under the Business Source License 1.1
//
// Primary development, architecture, and vision by Michael B. Vederman,
// CEO of Software Decisions, Inc., Texas.
//
// Internal development maintained privately.
// Public releases appear on GitHub: https://github.com/SoftwareDecisionsInc/GPAL.
//
// See LICENSE for full terms, including Additional Use Grant.
// =============================================================================
 
using System;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Windows.Forms;
using GenerallyPositive.Browser;
using OpenCvSharp;
using OpenQA.Selenium;
 
namespace GenerallyPositive
{
    public class ImageHelper : IImageHelper
    {
        public Browser.Browser Browser { get; set; }

        public static Bitmap ScreenShot { get; set; }
 
        private static string base64StringInternal = null;
 
 
        /************** CAPTURE SCREEN ******************/
        // https://stackoverflow.com/questions/4978157/how-to-search-for-an-image-on-screen-in-c
        // TODO: Capture screen for Applications

        public IAllowGraphicSettings CaptureScreen(object ApplicationOrBrowser = null)
        {
            bool headlessMode = false;   // assume headless as we might be called before we even have a browser created
            Bitmap image = new Bitmap(1920, 1080);
            bool isBrowser = typeof(Browser.Browser) == ApplicationOrBrowser?.GetType();
            if (true == isBrowser)
                // a browser on a hidden desktop is headful, but the screen is showing the desktop the user is on
                // and not that one, so copying the screen would quietly hand back a picture of their desktop. the
                // browser is asked for the image instead, which is the same route headless takes
                headlessMode = ((Browser.Browser)ApplicationOrBrowser).BrowserSettings.UseHeadless
                            || ((Browser.Browser)ApplicationOrBrowser).BrowserSettings.HiddenDesktop;
 
            try
            {
                image = new Bitmap(Screen.PrimaryScreen.Bounds.Width, Screen.PrimaryScreen.Bounds.Height, PixelFormat.Format32bppArgb);
            }
            catch
            {
                try
                {
                    image = new Bitmap(Screen.PrimaryScreen.Bounds.Width, Screen.PrimaryScreen.Bounds.Height, PixelFormat.Format32bppArgb);
                }
                catch
                {
                    // can't use publishevent, would be recursive
                    //Console.WriteLine("new Bitmap failed: Unable to take screenshot");
                    System.Diagnostics.Debug.WriteLine("new Bitmap failed: Unable to take screenshot");
                    return this;
                }
            }
 
            if (false == headlessMode)
            {
                var gfx = Graphics.FromImage(image);
                try
                {
                    gfx.CopyFromScreen(Screen.PrimaryScreen.Bounds.X, Screen.PrimaryScreen.Bounds.Y, 0, 0, Screen.PrimaryScreen.Bounds.Size, CopyPixelOperation.SourceCopy);
                }
                catch { }
            }
            else if (true == isBrowser)
            {
                Browser = (Browser.Browser)ApplicationOrBrowser;
 
                try
                {
                    byte[] imageBytes = null;
 
                    if (true == ((Browser.Browser)ApplicationOrBrowser).UseSelenium)
                    {
                        if (null != ((Browser.Browser)ApplicationOrBrowser).BrowserDriver)
                        {
                            Screenshot screenshot = ((ITakesScreenshot)((Browser.Browser)ApplicationOrBrowser).BrowserDriver).GetScreenshot();
                            // Convert the screenshot to a Bitmap
                            base64StringInternal = screenshot.AsBase64EncodedString;
                        }
                    }
                    else if (0 < GPAL.Browsers.Count())
                    {
                        foreach (Browser.Browser browser in GPAL.Browsers)
                            if (null != browser.BrowserSettings.Process)
                            {
                                if (true == ((Browser.Browser)ApplicationOrBrowser).UsePuppeteer)
                                {
                                    base64StringInternal = ((Browser.Browser)ApplicationOrBrowser).PuppeteerClient.CaptureVisibleTab().Execute();
                                }
                                else if (((Browser.Browser)ApplicationOrBrowser).UseOttoMagic)
                                {
                                    base64StringInternal = ((Browser.Browser)ApplicationOrBrowser).MagicHelper.CaptureVisibleTab();
                                }
 
                                break;
                            }
                    }
                    char[] ca = { ',' };
 
                    // remove the header info, we want the raw data
                    // "data:image/png;base64,iVBORw0 ... "
                    try
                    {
                        base64StringInternal = base64StringInternal.Trim()
                                            .Trim('"')                    // remove surrounding quotes if present
                                            .Split(ca, 2)                 // split on first comma only
                                            [1];                          // take everything after the comma
                    }
                    catch
                    {
                        base64StringInternal = base64StringInternal.Trim()
                                            .Trim('"');                    // remove surrounding quotes if present
                    }
 
                    // also trim any leftover whitespace (rare but safe)
                    base64StringInternal = base64StringInternal.Trim();
 
                    imageBytes = Convert.FromBase64String(base64StringInternal);
                    using (MemoryStream ms = new MemoryStream(imageBytes))
                        // Create an Image object from the MemoryStream
                        image = new Bitmap(ms);
                }
                catch (Exception ex)
                {
                    string outMessageHeader = $@"[{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")}][EXCEPTION][ImageHelper]: ";
 
                    // can't use publishevent, would be recursive: publishing an exception takes a screenshot, and
                    // this is the screenshot. written directly instead, and silent still means silent
                    if ("1" != Environment.GetEnvironmentVariable("GPAL_IS_SILENT"))
                    {
                        Console.WriteLine($"{outMessageHeader} [{Browser.AutomationEngine}][{Browser.BrowserType}][{(true == Browser.BrowserSettings.UseHeadless ? "Headless" : "Headful")}] Unable to take screenshot [{ex.Message}]");
                        System.Diagnostics.Debug.WriteLine($"{outMessageHeader} [{Browser.AutomationEngine}][{Browser.BrowserType}][{(true == Browser.BrowserSettings.UseHeadless ? "Headless" : "Headful")}] Unable to take screenshot [{ex.Message}]");
                    }
                }
            }
            ScreenShot = image;
 
            return this;
        }

        public IAllowGraphicSettings ToBase64String(out string base64String)
        {
            System.IO.MemoryStream ms = new MemoryStream();
            if (null != base64StringInternal)
                base64String = base64StringInternal;
            else if (null != ScreenShot)
            {
                ScreenShot.Save(ms, ImageFormat.Jpeg);
                byte[] byteImage = ms.ToArray();
                base64String = Convert.ToBase64String(byteImage);
            }
            else
                base64String = string.Empty;
 
            return this;
        }

        public IAllowGraphicSettings ToBitmap(out Bitmap bitmap)
        {
            bitmap = ScreenShot;
            return this;
        }

        [File(Description = "Save to file (overwrite)")]
        public IAllowGraphicSettings SaveTo(GPALFile file)
        {
            ToBase64String(out string base64String);
            SaveScreenshot(file.Next, base64String);
 
            return this;
        }

        private static void SaveScreenshot(string fileName, object screenshotResponse)
        {
            // Implement logic to save the screenshot to a file
            // The screenshotResponse should contain the base64-encoded image data
 
            // Example: Convert base64 to bytes and save to a file
            try
            {
                var base64Data = screenshotResponse?.ToString();
                if (!string.IsNullOrEmpty(base64Data))
                {
                    var imageBytes = Convert.FromBase64String(base64Data);
                    string saveToDirectory = Path.GetDirectoryName(fileName);
                    // ensure save to directory exists
                    if (false == string.IsNullOrEmpty(saveToDirectory))
                        Directory.CreateDirectory(saveToDirectory);
                    File.WriteAllBytes(fileName, imageBytes);
 
                    GPAL.PublishSimpleEvent(Enums.GPALEventType.INFO, $"Screenshot saved to [{fileName}]");
                }
                else
                    GPAL.PublishSimpleEvent(Enums.GPALEventType.WARNING, "No screenshot to save");
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(Enums.GPALEventType.EXCEPTION, $"No screenshot saved to [{fileName}]", null, Enums.GPALObjectType.None, ex);
            }
        }
 
        #region Helpers

        public static System.Drawing.Image LoadImageFromFile(string filename)
        {
            return System.Drawing.Image.FromFile(filename);
        }

        public static Image Base64ToImage(string base64String)
        {
            base64String = base64String.Trim().Trim('\"');
 
            string pattern = @"data:[^;]+;base64,[A-Za-z0-9+/=]+";
            base64String = System.Text.RegularExpressions.Regex.Replace(base64String, pattern, string.Empty);
 
            // Convert Base64 String to byte[]
            byte[] imageBytes = Convert.FromBase64String(base64String);
            MemoryStream ms = new MemoryStream(imageBytes, 0, imageBytes.Length);
 
            // Convert byte[] to Image
            ms.Write(imageBytes, 0, imageBytes.Length);
            Image image = Image.FromStream(ms, true);
 
            return image;
        }

        public Rectangle FindImage(string imagePath, object ApplicationOrBrowser = null)
        {
            Rectangle screenRect = Screen.PrimaryScreen.Bounds;
            Rect match = new Rect();
 
            try
            {
                // pass the browser through so a headless run matches against a visible-tab capture instead of the
                // physical screen (which has no window on it to find). the coordinates that come back are then
                // viewport relative rather than screen absolute - see GPALElement.CoordinateSpace.
                CaptureScreen(ApplicationOrBrowser);
                Bitmap screenshot = ScreenShot;
 
                // Convert Bitmaps to Mats and convert to grayscale
                Mat sourceMat = OpenCvSharp.Extensions.BitmapConverter.ToMat(screenshot);
                Mat templateMat = new Mat(imagePath);
                Mat sourceGray = new Mat();
                Mat templateGray = new Mat();
 
                // Check for empty Mats
                if (sourceMat.Empty())
                {
                    GPAL.PublishSimpleEvent(Enums.GPALEventType.WARNING, "SourceMat is empty.", sourceMat, Enums.GPALObjectType.Other);
                    return new Rectangle();
                }
                if (templateMat.Empty())
                {
                    GPAL.PublishSimpleEvent(Enums.GPALEventType.WARNING, "TemplateMat is empty.", templateMat, Enums.GPALObjectType.Other);
                    return new Rectangle();
                }
 
                // Convert to grayscale for better matching
                Cv2.CvtColor(sourceMat, sourceGray, ColorConversionCodes.BGR2GRAY);
                Cv2.CvtColor(templateMat, templateGray, ColorConversionCodes.BGR2GRAY);
 
                // Perform template matching
                Mat result = new Mat();
                Cv2.MatchTemplate(sourceGray, templateGray, result, TemplateMatchModes.CCoeffNormed);
 
                // Find match locations
                Cv2.MinMaxLoc(result, out _, out double res, out _, out OpenCvSharp.Point maxLoc);
 
                // Check if match meets threshold
                if (res >= (GPAL.MatchingPercentage * 0.01))
                    match = new Rect(maxLoc.X, maxLoc.Y, templateMat.Width, templateMat.Height);
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(Enums.GPALEventType.EXCEPTION, $"Error locating [{imagePath}] on-screen.", null, Enums.GPALObjectType.None, ex);
            }
 
            return new Rectangle(match.X, match.Y, match.Width, match.Height);
        }
        #endregion Helpers

        public IImageHelper ToGPALObject()
        {
            return this;
        }
    }
 
    // extension ToByteArray for CV
    internal static class ImageExtensions
    {
        public static byte[] ToByteArray(this System.Drawing.Image image, ImageFormat format)
        {
            using (MemoryStream ms = new MemoryStream())
            {
                image.Save(ms, format);
                return ms.ToArray();
            }
        }
    }
}