FileHelper.cs

// =============================================================================
// GPAL - Generally Positive Automation Library
// Copyright © 2026 Software Decisions, Inc. All rights reserved.
//
// This file is part of GPAL.
// Licensed under the Business Source License 1.1
//
// Primary development, architecture, and vision by Michael B. Vederman,
// CEO of Software Decisions, Inc., Texas.
//
// Internal development maintained privately.
// Public releases appear on GitHub: https://github.com/SoftwareDecisionsInc/GPAL.
//
// See LICENSE for full terms, including Additional Use Grant.
// =============================================================================
 
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Diagnostics;
using System.Threading;
using System.Text;
using System.Text.Json;
using System.Xml;
using System.Xml.Serialization;
using static GenerallyPositive.Enums;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
using YamlDotNet.RepresentationModel;
using YamlDotNet.Core;
using Newtonsoft.Json;
using System.Collections;
using System.Net;
using OpenQA.Selenium;
using System.Drawing;
using System.Text.RegularExpressions;
using GenerallyPositive.Browser;
using ClosedXML.Excel;
using Newtonsoft.Json.Linq;
using System.Linq.Expressions;
using System.Windows.Forms.DataVisualization.Charting;
 
namespace GenerallyPositive
{
    public class FileHelper
    {
        // Suppress repeat log messages (e.g. the "no delimiter set" notice fires once per parsed line - identical
        // every time). We publish a given message once, then emit a single "Supressing repeat messages" notice.
        private static readonly HashSet<string> _loggedMessages = new HashSet<string>();
        private static bool _suppressNoticeShown = false;

        private static void PublishSuppressingRepeats(GPALEventType type, string message, object source, GPALObjectType objectType)
        {
            if (true == _loggedMessages.Add(message))
                GPAL.PublishSimpleEvent(type, message, source, objectType);
            else if (false == _suppressNoticeShown)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Supressing repeat messages", source, objectType);
                _suppressNoticeShown = true;
            }
        }

        internal static int GetGrid(GPALFileSettings fileSettings, out List<IGPALGrid<string>> tokens)
        {
            FileStream fileStream = null;
            StreamReader streamReader = null;
            string currentLine;
            int fileIdx = 0;
            List<IGPALGrid<string>> myTokens = new List<IGPALGrid<string>>();
            IGPALGrid<string> myGrid = GPAL.Grid.ToGPALObject();
 
            foreach (string filename in fileSettings.Filenames)
                try
                {
                    string directory = Path.GetDirectoryName(filename);
                    string filenamepart = Path.GetFileName(filename);
                    if (true == File.Exists(filename))
                    foreach (string filename2 in Directory.GetFiles(false == string.IsNullOrEmpty(directory) ? directory : @".\", filenamepart))
                    {
                        bool firstLineSkipped = false;
 
                            using (fileStream = System.IO.File.OpenRead(filename2))
                            {
                                using (streamReader = new StreamReader(fileStream))
                                {
                                    while (null != (currentLine = streamReader?.ReadLine()))    // get one line
                                    {
                                        if (false == firstLineSkipped && 0 < fileSettings.IgnoreFirstLineColumnNames.Count && fileSettings.IgnoreFirstLineColumnNames[0])    // CAVEAT: hardcoded value (but we always have one file now)
                                        {
                                            firstLineSkipped = true;
                                            continue;
                                        }
                                        List<string> currentRow = new List<string>();
                                        char delim;
 
                                        if (0 < fileSettings.Delimiter.Count && true == fileSettings.Delimiter[fileIdx].HasValue)
                                            delim = fileSettings.Delimiter[fileIdx].Value;
                                        else
                                        {
                                            PublishSuppressingRepeats(GPALEventType.INFO, $"[{filename}] has no delimiter set, using comma.", fileSettings, GPALObjectType.FileSettings);
                                            delim = ',';
                                        }
 
                                        foreach (string token in ConverterHelper.SplitRespectingQuotes(currentLine, delim))   // parse tokens and add to our grid, quote-aware so embedded delimiters/commas inside a quoted field don't split it
                                            currentRow.Add(token);
 
                                        myGrid.AddRow(currentRow);  // add row to our grid
                                    }
                                    myTokens.Add(myGrid);   // add grid to our Token list
                                    fileSettings.FileGrid.Add(myGrid);
                                }
                            }
                        fileIdx++;
                    }
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to tokenize File [{filename}]. Tokens created [{myGrid.Rows}", fileSettings, GPALObjectType.FileSettings, ex);
                }
                finally
                {
                    streamReader?.Close();
                    fileStream?.Close();
                }
            tokens = myTokens;
            return tokens.Count();
        }
        internal static string CleanUpFileDestination(string filename, out string newFilename, bool DeleteFileBeforeDownload)
        {
            int counter = 0;
            return CleanUpFileDestination(filename, out newFilename, DeleteFileBeforeDownload, Enums.NextFilePattern.Timestamp, ref counter);
        }
        internal static string CleanUpFileDestination(string filename, out string newFilename, bool DeleteFileBeforeDownload,
            Enums.NextFilePattern nextFilePattern, ref int patternCounter)
        {
            string fullFilePath;
            string currentDirectory = Path.GetDirectoryName(filename);
 
            newFilename = filename;
            if (true == string.IsNullOrEmpty(currentDirectory))
            {
                currentDirectory = ".\\"; // KnownFolders.GetPath(KnownFolder.Downloads);
                fullFilePath = currentDirectory + filename;
            }
            else
                fullFilePath = filename;
 
            if (true == DeleteFileBeforeDownload)
            {
                try
                {
                    if (true == File.Exists(fullFilePath))
                    {
                        System.IO.File.Delete(fullFilePath);
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Deleted file [{fullFilePath}].", null, GPALObjectType.None);
                    }
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Failed to delete file [{fullFilePath}], Exception caught.", null, GPALObjectType.None, ex);
                }
            }
            else if (true == System.IO.File.Exists(fullFilePath))
            {
                string baseName = Path.GetFileNameWithoutExtension(filename);
                string extension = Path.GetExtension(filename);
 
                if (Enums.NextFilePattern.Counter == nextFilePattern || Enums.NextFilePattern.CounterPadded == nextFilePattern)
                {
                    // Seed the counter once from the highest existing "<base>_<n><ext>" via a single directory
                    // listing rather than probing _0001, _0002 ... one File.Exists at a time. This is O(1) syscalls
                    // per call after seeding, and (unlike a contiguous binary probe) stays correct when the numbering
                    // has gaps from deleted files. Zero means "not seeded yet"; the caller persists it thereafter.
                    if (0 == patternCounter)
                        patternCounter = HighestCounterIndex(currentDirectory, baseName, extension);
 
                    // max index + 1 is guaranteed free; the loop only re-spins if another process races in.
                    do
                    {
                        newFilename = string.Concat(baseName, NextFileSuffix(nextFilePattern, ref patternCounter), extension);
                        fullFilePath = currentDirectory + "\\" + newFilename;
                    }
                    while (true == System.IO.File.Exists(fullFilePath));
                }
                else
                {
                    // timestamp/date patterns are already unique enough, taken as-is
                    newFilename = string.Concat(baseName, NextFileSuffix(nextFilePattern, ref patternCounter), extension);
                    fullFilePath = currentDirectory + "\\" + newFilename;
                }
            }
 
            return fullFilePath;
        }
        private static int HighestCounterIndex(string directory, string baseName, string extension)
        {
            int max = 0;
 
            try
            {
                string searchDir = string.IsNullOrEmpty(directory) ? "." : directory;
                // '*' matches the number (and only the number, thanks to the digit check below)
                string searchPattern = baseName + "_*" + extension;
 
                foreach (string path in Directory.EnumerateFiles(searchDir, searchPattern))
                {
                    string name = Path.GetFileNameWithoutExtension(path);   // <base>_<digits>
                    int underscore = name.LastIndexOf('_');
                    if (0 > underscore || underscore + 1 >= name.Length)
                        continue;
 
                    string digits = name.Substring(underscore + 1);
                    if (true == digits.All(char.IsDigit) && true == int.TryParse(digits, out int n) && n > max)
                        max = n;
                }
            }
            catch
            {
                // any listing failure just leaves max at 0; the File.Exists guard on the caller still keeps us correct
            }
 
            return max;
        }
        private static string NextFileSuffix(Enums.NextFilePattern pattern, ref int counter)
        {
            switch (pattern)
            {
                case Enums.NextFilePattern.DateTimeStamp:
                    return "_" + DateTime.Now.ToString("yyyyMMdd_HHmmss");
                case Enums.NextFilePattern.DateStamp:
                    return "_" + DateTime.Now.ToString("yyyyMMdd");
                case Enums.NextFilePattern.Counter:
                    return "_" + (++counter).ToString();
                case Enums.NextFilePattern.CounterPadded:
                    // D4 so up to 9999 files sort correctly; past that it just grows an extra digit
                    return "_" + (++counter).ToString("D4");
                case Enums.NextFilePattern.Timestamp:
                default:
                    return DateTime.Now.ToString("yyyyMMddHHmmssfff");
            }
        }
        public static string SaveToDelimited(UnitOfWork currentUOW, string filename, char delim, bool appendToFile, bool DeleteFileBeforeDownload)
        {
            List<Selector> selectors = currentUOW.WithSelectorList; // BUG: with savefrom grid, this is not the correct selectorlist, this may give bogus results
            int colCnt = 0;
            string fullFilePath = CleanUpFileDestination(filename, out string newFilename, DeleteFileBeforeDownload : false);
 
            if (0 == currentUOW.HeaderList.Count)
            {
                foreach (Selector selector in selectors)
                {
                    // CAVEAT: BUG: what if the fist selector name is null but after that are not?
                    if (null != selector.Name)
                        currentUOW.HeaderList.Add(selector.Name);
                }
            }
            else if (selectors.Count > currentUOW.HeaderList.Count) // not enough headers for the output, so add the missing headers
            {
                for (int idx = currentUOW.HeaderList.Count; idx < selectors.Count; idx++)
                    currentUOW.HeaderList.Add(selectors[idx].Name);
            }
            else if (selectors.Count < currentUOW.HeaderList.Count)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, "Too many headers, not all will be output. This may not be correct if you are saving your own grid.", currentUOW, GPALObjectType.UnitOfWork);
            }
 
            if (false == appendToFile && true == System.IO.File.Exists(fullFilePath))
            {
                System.IO.File.Delete(fullFilePath);
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Asking to save new to existing file [{fullFilePath}], deleting file.", currentUOW, GPALObjectType.UnitOfWork);
            }
            else if (true == appendToFile && false == System.IO.File.Exists(fullFilePath))    // asking to append to a file that does not exist, really doing a save, output headers
            {
                appendToFile = false;
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Asking to append to non-existent file [{fullFilePath}], creating new file with headers.", currentUOW, GPALObjectType.UnitOfWork);
            }
 
            StreamWriter csv;
            using (csv = System.IO.File.AppendText(fullFilePath))
            {
                StringBuilder tmpRow = new StringBuilder();
                if (false == appendToFile)
                    if (0 < currentUOW.HeaderList.Count)
                    {
                        foreach (string header in currentUOW.HeaderList)
                        {
                            colCnt++;
                            tmpRow.Append(header);
                            if (colCnt < currentUOW.ColCount)
                                tmpRow.Append(delim);
                        }
                        csv.WriteLine(tmpRow);
                    }
 
                foreach (List<string> row in currentUOW.RetGrid)
                {
                    colCnt = 0;
                    tmpRow.Clear();
                    foreach (string col in row)
                    {
                        colCnt++;
                        tmpRow.Append(col);
                        if (colCnt < currentUOW.ColCount)
                            tmpRow.Append(delim);
                    }
                    csv.WriteLine(tmpRow);
                }
            }
 
            csv.Close();
            csv.Dispose();
            return fullFilePath;
        }

        internal static bool DownloadToFile(GPALElement webElement, string filename, string downloadUrl = null)
        {
            bool downloaded = false;
            string url = downloadUrl ?? GetDownloadUrl(webElement);
 
            try
            {
                using (System.Net.Http.HttpClient client = new System.Net.Http.HttpClient())
                {
                    System.Net.Http.HttpRequestMessage request = new System.Net.Http.HttpRequestMessage(System.Net.Http.HttpMethod.Get, url);
 
                    // present ourselves as the browser would. plenty of sites refuse a request carrying no user
                    // agent at all, and being refused for that reason is not worth falling back to a whole browser
                    // over. Same thing CheckRobotsTxt does before it resorts to opening a tab.
                    // Chrome when nothing was said, since a GPALFile need not have a browser to take a type from
                    request.Headers.TryAddWithoutValidation("User-Agent",
                        true == string.IsNullOrWhiteSpace(GPAL.GPALSettings.UserAgent)
                            ? MagicHelper.GetUserAgentString(BrowserType.Chrome)
                            : GPAL.GPALSettings.UserAgent);
 
                    if (true == File.Exists(filename))
                        request.Headers.IfModifiedSince = File.GetLastWriteTimeUtc(filename);
 
                    System.Net.Http.HttpResponseMessage response = client.SendAsync(request).GetAwaiter().GetResult();
 
                    if (System.Net.HttpStatusCode.NotModified == response.StatusCode)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{url}] has not changed, using [{filename}].", null, GPALObjectType.None);
                        downloaded = true;
                    }
                    else if (false == response.IsSuccessStatusCode)
                    {
                        // not fatal on its own. a 401 or 403 usually means the file is behind a session we do not
                        // have, which is what the browser fallback is for
                        GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{url}] returned HTTP [{(int)response.StatusCode}].", null, GPALObjectType.None);
                    }
                    else
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Downloading [{url}] to [{filename}].", null, GPALObjectType.None);
                        File.WriteAllBytes(filename, response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult());
 
                        // stamp it with the server's own modified time, so next run's question is about the content
                        // rather than about when we happened to fetch it
                        if (null != response.Content.Headers.LastModified)
                            File.SetLastWriteTimeUtc(filename, response.Content.Headers.LastModified.Value.UtcDateTime);
 
                        // nothing to wait for. WriteAllBytes returned, so the bytes are on disk and the file is
                        // whole. watching its size settle is for a browser writing in the background, and here it
                        // only confirmed a write we had already finished
                        downloaded = true;
                    }
                }
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{url}] failed to save to [{filename}].", null, GPALObjectType.None, ex);
            }
 
            return downloaded;
        }
        internal static string GetDownloadUrl(GPALElement element)
        {
            // Check the type of element or its attributes to determine the appropriate URL for downloading
            string downloadUrl = null;
 
            // If the element is an <a> (anchor) tag, get the href attribute value
            if (element.TagName.ToLower() == "a")
            {
                downloadUrl = element.GetAttribute("href");
            }
            // If the element is an <img> (image) tag, get the src attribute value
            else if (element.TagName.ToLower() == "img")
            {
                downloadUrl = element.GetAttribute("src");
            }
            // If the element is a <script> tag, get the src attribute value
            else if (element.TagName.ToLower() == "script")
            {
                downloadUrl = element.GetAttribute("src");
            }
            // If the element is a <link> tag, get the href attribute value
            else if (element.TagName.ToLower() == "link")
            {
                downloadUrl = element.GetAttribute("href");
            }
            // If the element is an <object> tag, get the data attribute value
            else if (element.TagName.ToLower() == "object")
            {
                downloadUrl = element.GetAttribute("data");
            }
            // If the element is an <embed> tag, get the src attribute value
            else if (element.TagName.ToLower() == "embed")
            {
                downloadUrl = element.GetAttribute("src");
            }
            // If the element is a <frame> or <iframe> tag, get the src attribute value
            else if (element.TagName.ToLower() == "frame" || element.TagName.ToLower() == "iframe")
            {
                downloadUrl = element.GetAttribute("src");
            }
            // If the element is an <audio> tag, get the src attribute value
            else if (element.TagName.ToLower() == "audio")
            {
                downloadUrl = element.GetAttribute("src");
            }
            // If the element is a <video> tag, get the src attribute value
            else if (element.TagName.ToLower() == "video")
            {
                downloadUrl = element.GetAttribute("src");
            }
            // Add more conditions for other types of elements as needed...
 
            return downloadUrl;
        }
        internal static FileSystemWatcher watcher;
        internal static DateTime watchArmed;    // when the watcher went up, so the file a previous download left in the same directory is not taken for this one
        internal static readonly HashSet<string> watchedScratchFiles = new HashSet<string>(StringComparer.OrdinalIgnoreCase);   // the ones this download was seen to start, so an abandoned one can be told from anyone else's
        // NOTE: BUG: CAVEAT: this never worked, maybe cause of permissions on the 'special' download folder?
        internal static string SetupDownloadWatcher(string fullFilePathToSave, Browser.BrowserSettings browserSettings)
        {
            // chrome and edge are told to download straight into the destination directory before the click, so
            // that is where to watch. the two that cannot be told go to the browser's own download directory and
            // handleFile moves the file from there: ottomagic because the extension can name a download but not
            // place it, firefox because it has no cdp and only reads its download directory at startup
            bool downloadsWhereItLikes = true == browserSettings.UseOttoMagic
                                      || BrowserType.FireFox == browserSettings.BrowserType;
            string downloadDirectory = true == downloadsWhereItLikes
                                     ? FileHelper.GetDefaultDownloadDirectory(browserSettings.Browser)
                                     : Path.GetDirectoryName(fullFilePathToSave) ?? FileHelper.GetDefaultDownloadDirectory(browserSettings.Browser);
 
            string fileName = Path.GetFileName(fullFilePathToSave);
            string actualDownloadedFilename = null;
 
            watchArmed = DateTime.Now;
            watchedScratchFiles.Clear();
 
            watcher = new FileSystemWatcher(downloadDirectory);
            watcher.NotifyFilter = NotifyFilters.FileName | NotifyFilters.LastWrite | NotifyFilters.Size | NotifyFilters.Attributes;
            watcher.Filter = "*.*";
            watcher.EnableRaisingEvents = true;
 
            // Add Created (often fires for temp file start)
            watcher.Created += (sender, e) => handleFile(sender, e, downloadDirectory, fullFilePathToSave);
            watcher.Changed += (sender, e) => handleFile(sender, e, downloadDirectory, fullFilePathToSave);
            watcher.Renamed += (sender, e) => handleFile(sender, e, downloadDirectory, fullFilePathToSave);
 
            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Watching [{downloadDirectory}] for [{fileName}] w/timeout [{browserSettings.DownloadTimeoutInSec}] secs", null, GPALObjectType.None);
            System.Threading.Thread.Sleep(1_000);
 
            return downloadDirectory;
 
            void handleFile(object sender, FileSystemEventArgs e, string downloaddirectory, string fullFilePathToSave)
            {
                try
                {
                    // a browser downloads to a temporary name and renames it to the real one when it finishes, and
                    // the name it lands under is the site's, not the temporary one with its suffix taken off. the
                    // rename is the only event that carries where the file actually went
                    if (e is RenamedEventArgs renamed
                        && (true == renamed.OldName.EndsWith(".crdownload") || true == renamed.OldName.EndsWith(".part")))
                    {
                        actualDownloadedFilename = WaitForDownloadToFinish(Path.Combine(downloaddirectory, renamed.Name), browserSettings.DownloadTimeoutInSec);
                        browserSettings.FileDownloaded = false == string.IsNullOrEmpty(actualDownloadedFilename);
 
                        if (true == browserSettings.FileDownloaded)
                        {
                            File.Move(actualDownloadedFilename, fullFilePathToSave);
                            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{actualDownloadedFilename}] moved to [{fullFilePathToSave}]", null, GPALObjectType.None);
                        }
                        else
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"[{renamed.OldName}] was renamed to [{renamed.Name}] but it never settled", null, GPALObjectType.None);
 
                        // only once it is done. these events keep arriving and a later one can resolve what this
                        // one could not, so standing the watcher down on a miss is what makes a miss permanent
                        if (true == browserSettings.FileDownloaded)
                            ((FileSystemWatcher)sender).Dispose();
 
                        return;
                    }
 
                    if (e.Name.EndsWith(".crdownload") || e.Name.EndsWith(".part"))
                    {
                        string filename = e.Name.Replace(".crdownload", "").Replace(".part", "");
                        string fullSavedFilePath = Path.Combine(downloaddirectory, filename);  // Safer path building
 
                        string pattern = @"^downloads(?:\.htm|\s*\‍(\d+\‍)\.htm|\.htm\s*\‍(\d+\‍))$";
                        if (Regex.IsMatch(filename, pattern, RegexOptions.IgnoreCase))
                        {
                            return;
                        }
 
                        GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Download started for [{e.Name}] waiting on expected [{fullSavedFilePath}]", null, GPALObjectType.None);
                        watchedScratchFiles.Add(Path.Combine(downloaddirectory, e.Name));
 
                        // disable events temporarily to avoid duplicate calls during wait
                        // watcher.EnableRaisingEvents = false;
 
                        // first check and wait for the crdownload file to finish, which might finish before we even call, so it never finds it...
                        if (true == File.Exists(Path.Combine(downloaddirectory, e.Name)))
                            actualDownloadedFilename = WaitForDownloadToFinish(Path.Combine(downloaddirectory, e.Name), browserSettings.DownloadTimeoutInSec);
 
                        // the scratch file is never the download. WaitForDownloadToFinish hands its name back to
                        // say it watched one and it has gone, which is the browser renaming it to the real name,
                        // so drop it here and let the real name be found below. moving it is how a download that
                        // stalled with a steady size, or one still being written, ended up saved as the answer
                        if (true == IsTemporaryDownload(actualDownloadedFilename))
                            actualDownloadedFilename = null;
 
                        // the destination itself. when the filename was set for this download the browser renames
                        // the temporary file straight to it, so it is already where it was asked to go
                        if (true == File.Exists(fullFilePathToSave))
                            actualDownloadedFilename = WaitForDownloadToFinish(fullFilePathToSave, browserSettings.DownloadTimeoutInSec);
 
                        // and failing that the temporary name with its suffix taken off, for a browser that lands
                        // it there. only when nothing above answered - asking regardless threw away what we had
                        if (true == string.IsNullOrEmpty(actualDownloadedFilename))
                            actualDownloadedFilename = WaitForDownloadToFinish(fullSavedFilePath, browserSettings.DownloadTimeoutInSec);
 
                        browserSettings.FileDownloaded = !string.IsNullOrEmpty(actualDownloadedFilename);
 
                        if (true == browserSettings.FileDownloaded)
                        {
                            // the browser was told where to put it and did, so there is nothing to move and
                            // File.Move onto itself only throws
                            if (true == fullFilePathToSave.Equals(actualDownloadedFilename, StringComparison.OrdinalIgnoreCase))
                                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{fullFilePathToSave}] downloaded straight to where it was asked to go", null, GPALObjectType.None);
                            else
                            {
                                File.Move(actualDownloadedFilename, fullFilePathToSave);
                                if (File.Exists(actualDownloadedFilename))
                                    GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Unable to move [{actualDownloadedFilename}] to [{fullFilePathToSave}]", null, GPALObjectType.None);
                                else
                                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{actualDownloadedFilename}] moved to [{fullFilePathToSave}]", null, GPALObjectType.None);
                            }
                        }
                        else
                        {
                            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"WaitForDownloadToFinish returned null for [{fullSavedFilePath}] - check timeout or stability", null, GPALObjectType.None);
                        }
 
                        // as above: a browser still finalizing has not renamed anything yet, and the event that
                        // says it did is the one this would have thrown away
                        if (true == browserSettings.FileDownloaded)
                            ((FileSystemWatcher)sender).Dispose();
                    }
                }
                catch (Exception ex)
                {
                    // the watcher stays up and the flag stays as it was. these events arrive many times over one
                    // download and any of them can catch the directory mid-change, so this is one event going
                    // wrong rather than the download failing. shutting the watcher down here meant the download
                    // could never be reported, and answering false here would end the wait on the same mistake
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Error for [{fullFilePathToSave}]", null, GPALObjectType.None, ex);
                }
            }
        }

        internal const int DownloadSizeSampleMs = 500;   // long enough that a file still being written is seen to grow

        internal static bool IsTemporaryDownload(string path)
        {
            bool retVal = true == path?.EndsWith(".crdownload") || true == path?.EndsWith(".part");
 
            return retVal;
        }

        internal static string NewestDownloadSince(string directory, DateTime since)
        {
            string retVal = null;
            DateTime newest = DateTime.MinValue;
 
            foreach (string candidate in Directory.EnumerateFiles(directory))
                if (false == IsTemporaryDownload(candidate))
                {
                    // the later of the two, because either one on its own can read older than the file is. a
                    // fetched file is stamped with the server's Last-Modified and its write time can be years
                    // old, and windows hands a file recreated under a name it just used the original's creation
                    // time. both being stale at once is the one thing that does not happen
                    DateTime written = File.GetLastWriteTime(candidate);
                    DateTime created = File.GetCreationTime(candidate);
                    DateTime arrived = created > written ? created : written;
 
                    if (arrived >= since && arrived > newest)
                    {
                        newest = arrived;
                        retVal = candidate;
                    }
                }
 
            return retVal;
        }
 
        internal static string WaitForDownloadToFinish(string filePath, int downloadTimeoutInSec)
        {
            long size = 0, newSize = 0;
            string downloadedFilename;
            bool fileStarted = false;
 
            newSize = WaitForFileToDownload(filePath, downloadTimeoutInSec, size, out downloadedFilename);   // -1 return if timed out waiting for file to exist (not completed)
 
            // KLUDGE: file showed up but is zero bytes, this could be correct, but let's give an extra second and wait
            // NOTE: hardcoded value - not really in favor of hard sleeps during the workflow espcially since the download has started, but maybe it's slow for some reason
            // our file checks can run very quuckly
            if (0 == newSize)
                System.Threading.Thread.Sleep(1_000);
            else if (-1 == newSize)
            {
                if (false == filePath.EndsWith(".crdownload") && false == filePath.EndsWith(".part"))
                    return string.Empty;
                else
                    return filePath;
            }
 
            size = newSize;
 
            // NOTE: this is prolly really overkill, essentially checking multiple times for completion
            do
            {
                if (0 <= (newSize = WaitForFileToDownload(filePath, downloadTimeoutInSec, newSize, out downloadedFilename)))
                {
                    // NOTE: new scenario, race condition, if file starts download then chrome renames, we get -1 and think it is not downloaded
                    fileStarted = true;
 
                    // KLUDGE: file showed up but is zero bytes, this could be correct, but let's give an extra second and wait
                    // NOTE: hardcoded value - not really in favor of hard sleeps during the workflow espcially since the download has started, but maybe it's slow for some reason
                    // our file checks can run very quuckly
                    if (0 == newSize)
                        System.Threading.Thread.Sleep(1_000);
 
                    // a file being written has to be given time to grow between the two reads. taking both in the
                    // same instant made every download look finished the moment it was first seen, which is how a
                    // part-written file was taken for a complete one
                    System.Threading.Thread.Sleep(DownloadSizeSampleMs);
 
                    if (size == (newSize = WaitForFileToDownload(filePath, downloadTimeoutInSec, newSize, out downloadedFilename)))  // NOTE: ensure the filesize stays the same (prolly overkill)
                        break;
                    else
                        size = newSize;
                }
                else if (-1 == newSize) // file isn't there..
                    break;
            } while (true); // until we break when the file szie no longer changes
 
            if (false == filePath.EndsWith(".crdownload") && false == filePath.EndsWith(".part"))
            {
                if (0 == newSize)
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"File [{downloadedFilename ?? filePath}] downloaded as zero bytes", null, GPALObjectType.None);
                else if (-1 == newSize && false == fileStarted)
                    GPAL.PublishSimpleEvent(GPALEventType.INFO, $"File [{downloadedFilename ?? filePath}] NOT downloaded", null, GPALObjectType.None);
                // NOTE: if -1 and file started, we have a race condition where the file was renamed, prolly by the browser - so ignore it, it's just a temp file
            }
 
            return downloadedFilename;
        }
        internal static long WaitForFileToDownload(string filePath, int downloadTimeoutInSec, long size, out string downloadedFilename)
        {
            TimeSpan timeout = TimeSpan.FromSeconds(downloadTimeoutInSec); // Adjust timeout as needed
            DateTime startTime = DateTime.Now;
            bool fileExists = Directory.GetFiles(Path.GetDirectoryName(filePath), Path.GetFileName(filePath)).Length > 0;
            int lastPrinted = -1;  // so we print at ~30, 20, 10, 0
 
            // a scratch file is not something to wait for. the caller only ever asks about one it has already
            // seen, so it not being there is the browser having renamed it, which is the download finishing.
            // the rename never says what it renamed to, so the download is whatever arrived in that directory
            // just now, and sizing that up is what the rest of this does. waiting out the timeout for a name
            // that has already changed is how a download that worked was reported as one that never came
            if (false == fileExists && true == IsTemporaryDownload(filePath))
            {
                string arrived = NewestDownloadSince(Path.GetDirectoryName(filePath), watchArmed);
 
                if (true == string.IsNullOrEmpty(arrived))
                {
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{filePath}] has gone and nothing arrived with it, so the download did not finish.", null, GPALObjectType.None);
                    downloadedFilename = filePath;
 
                    return -1;
                }
 
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{Path.GetFileName(filePath)}] became [{Path.GetFileName(arrived)}], which is what the browser called it.", null, GPALObjectType.None);
                filePath = arrived;
                fileExists = true;
            }
 
            while (false == fileExists && (DateTime.Now - startTime) < timeout)
            {
                TimeSpan remaining = timeout - (DateTime.Now - startTime);
                int remainingSec = (int)Math.Ceiling(remaining.TotalSeconds);  // or (int)remaining.TotalSeconds
 
                // Print only when crossing a 10-second boundary (30>29, 20>19, etc.)
                if (remainingSec % 10 == 0 && remainingSec != lastPrinted)
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Waiting up tp [{remainingSec}] sec for [{filePath}].", null, GPALObjectType.None);
                // Sleep for a short interval before checking again
                System.Threading.Thread.Sleep(1000); // Check every second
                fileExists = Directory.GetFiles(Path.GetDirectoryName(filePath), Path.GetFileName(filePath)).Length > 0;
 
                // the name we were handed is one the browser may have already moved on from. it renames the file
                // as the download finishes and the name it picks is its own, so "Unconfirmed 175338" was never
                // going to turn up. a file that arrived while we sat here is that download under its new name,
                // so take it and stop waiting rather than run the timeout out on a name that has changed
                if (false == fileExists)
                {
                    string arrived = NewestDownloadSince(Path.GetDirectoryName(filePath), watchArmed);
 
                    if (false == string.IsNullOrEmpty(arrived) && false == arrived.Equals(filePath, StringComparison.OrdinalIgnoreCase))
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"[{Path.GetFileName(filePath)}] never showed up, [{Path.GetFileName(arrived)}] did, which is the name the browser gave it.", null, GPALObjectType.None);
                        filePath = arrived;
                        fileExists = true;
                    }
                }
            }
 
            if (false == fileExists)
            {
                if (false == filePath.EndsWith(".crdownload") && false == filePath.EndsWith(".part"))
                {
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"File [{filePath}] never showed up.", null, GPALObjectType.None);
                    downloadedFilename = null;
                }
                else    // if we have a temp file, we actually detected it, so return it, it existed and next step will be to check the real filename
                    downloadedFilename = filePath;
 
                return -1;
            }
 
            // we got here in case a wildcard is passed in and we have multiple matching files
 
            // use case: we expect a file dialog but the link downloads directly, so we have to figure it out post download
            // as it will not download to the correct directory, so find where it downloads to and copy to the user specified directory.
            // NOTE: CAVEAT: this depends upon the user supplied file extension which must match the expected download
            var cutoff = DateTime.Now.AddMinutes(-2);   // NOTE: hardcoded value
 
            string newest = null;
            DateTime newestTime = DateTime.MinValue;
            IEnumerable<string> filesFound = Directory.EnumerateFiles(
                                 Path.GetDirectoryName(filePath),
                                 Path.GetFileName(filePath));
 
            // there is more than one file that matches our filePath mask, so a wildcard must be in play
            if (1 < filesFound.Count())
                foreach (var f in filesFound)
                {
                    // we may have just run the auto-update, do not mistake one of these files for our download
                    if (true == f.Equals(GPAL.GPALSettings.ChromeDriverZipFilename) ||
                         true == f.Equals(GPAL.GPALSettings.EdgeDriverFilename) ||
                         true == f.Equals(GPAL.GPALSettings.FirefoxDriverFilename))
                        continue;
 
                    var t = File.GetLastWriteTime(f);
 
                    if (t >= cutoff && t > newestTime)
                    {
                        newestTime = t;
                        newest = f;
                    }
                }
            else    // only one file, get it
                newest = filesFound.FirstOrDefault();
 
            // it was there at the check above and it is gone now, which is the browser renaming it the moment it
            // finished. that is the download completing, so say what the not-found path says and let the caller go
            // looking for the name it landed under. First() on an empty match threw out of here instead, and the
            // handler that caught it turned off the watcher, so nothing was ever going to report the file
            if (null == newest)
            {
                downloadedFilename = true == IsTemporaryDownload(filePath) ? filePath : null;
 
                return -1;
            }
 
            if (true == newest.Equals(GPAL.GPALSettings.ChromeDriverZipFilename) ||
                 true == newest.Equals(GPAL.GPALSettings.EdgeDriverFilename) ||
                 true == newest.Equals(GPAL.GPALSettings.FirefoxDriverFilename))
                newest = null;
 
                // grab the newest file that showed up in the last 2 minutes. we should be waiting for just one file.
            if (null != newest)
            {
                downloadedFilename = newest;
 
                // Once the file exists, attempt to open it
                try
                {
                    using (FileStream fs = File.Open(downloadedFilename, FileMode.Open))
                    {
                        // If opening the file succeeds, the download is complete
                        if (size != fs.Length)
                            GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"File [{downloadedFilename}] downloaded with file size [{fs.Length}].", null, GPALObjectType.None);
                        return fs.Length;
                    }
                }
                catch
                {
                    try
                    {
                        // the file would not open, so it is held by whoever is writing it or by a reader. that is
                        // not an answer either way, so report the size it is at and let the caller's next sample
                        // decide: a file still being written grows, a finished one does not
                        FileInfo fileInfo = new FileInfo(downloadedFilename);
 
                        if (fileInfo.Exists)
                        {
                            // If opening the file succeeds, the download is complete
                            if (size != fileInfo.Length)
                                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"File [{downloadedFilename}] locked, using FileInfo size [{fileInfo.Length}].", null, GPALObjectType.None);
 
                            return fileInfo.Length;
                        }
                    }
                    catch (Exception ex)
                    {
                        GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"File [{downloadedFilename}] unable to determine file size.", null, GPALObjectType.None, ex);
                        return -1;
                    }
                }
            }
            else
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"File [{filePath}] not found.", null, GPALObjectType.None);
 
            downloadedFilename = null;
 
            return -1;
        }

        public static string GetDefaultDownloadDirectory(IBrowser browser)
        {
            if (BrowserType.Chrome == browser.BrowserType)
                return GetChromeDefaultDownloadDirectory(((Browser.Browser)browser).BrowserSettings.ProfileDataDirectory);
            else if (BrowserType.Edge == browser.BrowserType)
                return GetEdgeDefaultDownloadDirectory(((Browser.Browser)browser).BrowserSettings.ProfileDataDirectory);
            else if (BrowserType.FireFox == browser.BrowserType)
                return GetFirefoxDefaultDownloadDirectory(((Browser.Browser)browser).BrowserSettings.ProfileDataDirectory);
 
            // Get the default download directory used by windows
            string defaultDownloadDirectory = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + @"\Downloads\";
            return defaultDownloadDirectory;
        }

        public static string GetChromeDefaultDownloadDirectory(string profilePath = null)
        {
            // Default profile path (change "Default" to "Profile 1" etc. if using a different one)
            if (string.IsNullOrEmpty(profilePath))
            {
                string localAppData = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
                profilePath = Path.Combine(localAppData, @"Google\Chrome\User Data\Default");
            }
 
            string prefsPath = null;
            
            if (false == new DirectoryInfo(profilePath).Name.ToLower().Equals("default"))
                prefsPath = Path.Combine(profilePath, @"default\Preferences");
            else
                prefsPath = Path.Combine(profilePath, @"Preferences");
 
            if (File.Exists(prefsPath))
            {
                string json = File.ReadAllText(prefsPath);
                JObject prefs = JObject.Parse(json);
 
                // Navigate to download.default_directory
                var downloadDir = prefs["download"]?["default_directory"]?.ToString();
 
                if (!string.IsNullOrEmpty(downloadDir))
                    return downloadDir;
 
                // If not set explicitly, Chrome uses the system default Downloads folder
                return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Downloads");
            }
 
            return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Downloads"); ;
        }

        public static string GetEdgeDefaultDownloadDirectory(string profilePath = null)
        {
            // Default profile path ("Default" folder; change to "Profile 1" etc. if needed)
            if (string.IsNullOrEmpty(profilePath))
            {
                string localAppData = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
                profilePath = Path.Combine(localAppData, @"Microsoft\Edge\User Data\Default");
            }
 
            string prefsPath = Path.Combine(profilePath, @"default\Preferences");
 
            if (File.Exists(prefsPath))
            {
 
                string json = File.ReadAllText(prefsPath);
                JObject prefs = JObject.Parse(json);
 
                // download.default_directory only. savefile.default_directory is where "save page as" last put
                // something, a different setting that happens to look like this one, and it is never where a
                // download goes. reading it at all sends the download watcher somewhere the browser does not
                // download to: first when it was preferred, then again as a fallback once a profile had no
                // download directory of its own. a profile without one uses the windows Downloads folder, which
                // is what the fallback below already answers
                var downloadDir = prefs["download"]?["default_directory"]?.ToString();
 
                if (!string.IsNullOrEmpty(downloadDir))
                    return downloadDir;
 
                // Fallback: system default Downloads folder
                return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Downloads");
            }
 
            return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Downloads");
        }

        public static string GetFirefoxDefaultDownloadDirectory(string profilePath = null)
        {
            string appData = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
            string firefoxDir = Path.Combine(appData, @"Mozilla\Firefox");
            string profilesIniPath = Path.Combine(firefoxDir, "profiles.ini");
 
            if (!File.Exists(profilesIniPath))
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"Firefox profiles.ini [{profilesIniPath}] not found.");
 
            // Read and manually parse profiles.ini to find the default profile
            string[] lines = File.ReadAllLines(profilesIniPath);
            string profileDir = null;
            bool isRelative = true;
 
            for (int i = 0; i < lines.Length; i++)
            {
                string line = lines[i].Trim();
 
                if (line.StartsWith("[Profile", StringComparison.OrdinalIgnoreCase))
                {
                    // Reset for new section
                    string currentPath = null;
                    bool currentIsRelative = true;
                    bool isDefault = false;
 
                    // Look ahead in this section
                    for (int j = i + 1; j < lines.Length; j++)
                    {
                        string subLine = lines[j].Trim();
                        if (subLine.StartsWith("["))
                            break; // next section
 
                        if (subLine.StartsWith("Path=", StringComparison.OrdinalIgnoreCase))
                            currentPath = subLine.Substring(5);
                        else if (subLine.StartsWith("IsRelative=", StringComparison.OrdinalIgnoreCase))
                            currentIsRelative = subLine.Substring(11) == "1";
                        else if (subLine.StartsWith("Default=", StringComparison.OrdinalIgnoreCase))
                            isDefault = subLine.Substring(8) == "1";
                    }
 
                    // If this is the default profile, use it
                    if (isDefault && currentPath != null)
                    {
                        profileDir = currentPath;
                        isRelative = currentIsRelative;
                        break;
                    }
 
                    // Otherwise remember the first profile we find as fallback
                    if (profileDir == null && currentPath != null)
                    {
                        profileDir = currentPath;
                        isRelative = currentIsRelative;
                    }
                }
            }
 
            if (!string.IsNullOrEmpty(profileDir))
            {
 
                // Resolve full path
                string fullProfilePath = isRelative
                    ? Path.Combine(firefoxDir, profileDir)
                    : profileDir;
 
                // Allow override
                if (!string.IsNullOrEmpty(profilePath))
                    fullProfilePath = profilePath;
 
                string prefsJsPath = Path.Combine(fullProfilePath, "prefs.js");
 
                if (!File.Exists(prefsJsPath))
                    GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"prefs.js [{prefsJsPath}] not found in Firefox profile.");
                else
                {
                    string content = File.ReadAllText(prefsJsPath);
 
                    // Look for: user_pref("browser.download.dir", "C:\\Path\\To\\Folder");
                    var match = Regex.Match(content, @"user_pref\‍(""browser\.download\.dir"",\s*""([^""]*)""\‍);");
 
                    if (match.Success)
                    {
                        string dir = match.Groups[1].Value;
                        // Handle common escapes (\\ > \‍)
                        return dir.Replace("\\\\", "\\");
                    }
 
                    // Not set > fallback to user's Downloads folder
                    return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Downloads");
                }
            }
 
            return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Downloads");
        }

        internal static List<IGPALGrid<string>> TokenizeFile(UnitOfWork currentUOW, GPALFile inputFile)
        {
            currentUOW.InputFile = inputFile;
            int rowCount = FileHelper.GetGrid(((IGPALFileInternal)inputFile).FileSettings, out List<IGPALGrid<string>> tokens);
            return tokens;
        }

        private static HashSet<string> CollectDictionaryKeys(object value)
        {
            var keys = new HashSet<string>();
            if (value is IList<object> list)
            {
                foreach (var item in list)
                {
                    if (item is IDictionary dynamicDict)
                    {
                        foreach (dynamic entry in dynamicDict)
                        {
                            string key = entry.Key?.ToString();
                            if (key != null)
                            {
                                if (entry.Value is IDictionary nestedDict)
                                {
                                    foreach (string nestedKey in CollectDictionaryKeys(nestedDict))
                                    {
                                        keys.Add($"{key}.{nestedKey}");
                                    }
                                }
                                else
                                {
                                    keys.Add(key);
                                }
                            }
                        }
                    }
                }
            }
            else if (value is IDictionary dynamicDict)
            {
                foreach (dynamic entry in dynamicDict)
                {
                    string key = entry.Key?.ToString();
                    if (key != null)
                    {
                        if (entry.Value is IDictionary nestedDict)
                        {
                            foreach (string nestedKey in CollectDictionaryKeys(nestedDict))
                            {
                                keys.Add($"{key}.{nestedKey}");
                            }
                        }
                        else
                        {
                            keys.Add(key);
                        }
                    }
                }
            }
            return keys;
        }
 
        internal static object ProcessDictionaryValue(object value, ConverterSettings converterSettings, List<List<string>> gridRows = null, HashSet<string> gridColumns = null, string parentKey = null, HashSet<object> visited = null)
        {
            if (value == null)
            {
                if (gridRows != null && gridColumns == null)
                {
                    gridRows.Add(new List<string> { "" });
                }
                return null;
            }
 
            Type valueType = value.GetType();
 
            if (visited == null)
                visited = new HashSet<object>(ConverterHelper.ReferenceEqualityComparer.Instance);
 
            // Handle nested dictionaries
            if (value is IDictionary dynamicDict)
            {
                // YAML anchors/aliases can produce genuine circular object graphs (e.g. a
                // dictionary that contains itself via a nested list). Without this check
                // such cycles cause unbounded recursion and a StackOverflowException.
                if (!visited.Add(value))
                    return $"Recursion to [{parentKey ?? valueType.Name}]";
 
                try
                {
                var resultDict = new Dictionary<dynamic, dynamic>();
                var row = new List<string>();
 
                if (gridColumns != null && gridRows != null)
                {
                    row = new List<string>(new string[gridColumns.Count]);
                    int colIdx = 0;
                    foreach (string col in gridColumns)
                    {
                        string cellValue = "";
                        if (col.Contains("."))
                        {
                            var keyParts = col.Split('.');
                            object current = dynamicDict;
                            bool found = true;
                            foreach (var part in keyParts)
                            {
                                if (current is IDictionary currentDict && currentDict.Contains(part))
                                {
                                    current = currentDict[part];
                                }
                                else
                                {
                                    found = false;
                                    break;
                                }
                            }
                            if (found)
                            {
                                var processedValue = ProcessDictionaryValue(current, converterSettings, null, null, null, visited);
                                cellValue = FlattenValueForGrid(processedValue);
                            }
                        }
                        else if (dynamicDict.Contains(col))
                        {
                            var processedValue = ProcessDictionaryValue(dynamicDict[col], converterSettings, null, null, null, visited);
                            cellValue = FlattenValueForGrid(processedValue);
                        }
                        row[colIdx++] = cellValue;
                    }
                    gridRows.Add(row);
                }
 
                foreach (dynamic entry in dynamicDict)
                {
                    string key = entry.Key?.ToString();
                    if (key != null)
                    {
                        bool isTabularList = entry.Value is IList<object> list2 && list2.Count() > 0 && list2[0] is IDictionary;
                        HashSet<string> nestedColumns = isTabularList ? CollectDictionaryKeys(entry.Value) : null;
                        var processedValue = ProcessDictionaryValue(entry.Value, converterSettings, isTabularList ? gridRows : null, nestedColumns, key, visited);
                        resultDict[key] = processedValue;
                        if (!isTabularList && gridRows != null && gridColumns == null)
                        {
                            row.Add(FlattenValueForGrid(processedValue));
                        }
                    }
                }
 
                if (gridRows != null && row.Count > 0 && gridColumns == null)
                {
                    gridRows.Add(row);
                }
                return resultDict;
                }
                finally
                {
                    visited.Remove(value);
                }
            }
 
            // Handle arrays/lists
            if (value is IList<object> list && !(value is string))
            {
                if (!visited.Add(value))
                    return $"Recursion to [{parentKey ?? valueType.Name}]";
 
                try
                {
                var resultList = new List<object>();
                string itemKey = string.Empty;
 
                if (null != parentKey)
                    itemKey = parentKey;
                else
                    itemKey = "item";
 
                foreach (var item in list)
                {
                    var processedItem = ProcessDictionaryValue(item, converterSettings, gridRows, gridColumns, itemKey, visited);
                    resultList.Add(processedItem);
                }
                return resultList;
                }
                finally
                {
                    visited.Remove(value);
                }
            }
 
            // Handle simple types
            if (ConverterHelper.IsSimpleType(valueType))
            {
                string stringValue;
                if (valueType == typeof(Guid) && Guid.TryParse(value.ToString(), out var guid))
                {
                    stringValue = guid.ToString();
                    if (gridRows != null && gridColumns == null)
                    {
                        gridRows.Add(new List<string> { stringValue });
                    }
                    return guid;
                }
                if (valueType == typeof(IPAddress) && IPAddress.TryParse(value.ToString(), out var ipAddress))
                {
                    stringValue = ipAddress.ToString();
                    if (gridRows != null && gridColumns == null)
                    {
                        gridRows.Add(new List<string> { stringValue });
                    }
                    return ipAddress;
                }
                if (valueType == typeof(TimeSpan) && TimeSpan.TryParse(value.ToString(), out var timeSpan))
                {
                    stringValue = timeSpan.ToString();
                    if (gridRows != null && gridColumns == null)
                    {
                        gridRows.Add(new List<string> { stringValue });
                    }
                    return timeSpan;
                }
                if (valueType == typeof(Uri))
                {
                    try
                    {
                        var uri = new Uri(value.ToString());
                        stringValue = uri.ToString();
                        if (gridRows != null && gridColumns == null)
                        {
                            gridRows.Add(new List<string> { stringValue });
                        }
                        return uri;
                    }
                    catch
                    {
                        stringValue = value.ToString();
                        if (gridRows != null && gridColumns == null)
                        {
                            gridRows.Add(new List<string> { stringValue });
                        }
                        return stringValue;
                    }
                }
                if (valueType == typeof(System.Numerics.BigInteger))
                {
                    stringValue = value.ToString();
                    if (gridRows != null && gridColumns == null)
                    {
                        gridRows.Add(new List<string> { stringValue });
                    }
                    return stringValue;
                }
                if (valueType == typeof(System.Version))
                {
                    stringValue = value.ToString();
                    if (gridRows != null && gridColumns == null)
                    {
                        gridRows.Add(new List<string> { stringValue });
                    }
                    return stringValue;
                }
                stringValue = value.ToString();
                if (gridRows != null && gridColumns == null)
                {
                    gridRows.Add(new List<string> { stringValue });
                }
                return value;
            }
 
            // Handle complex objects
            if (ConverterHelper.IsClassType(valueType) && !valueType.IsArray)
            {
                var resultDict = ConverterHelper.ConvertClassToDictionary(value);
                var row = new List<string>();
                foreach (dynamic entry in resultDict)
                {
                    row.Add(FlattenValueForGrid(entry.Value));
                }
                if (gridRows != null && row.Count > 0 && gridColumns == null)
                {
                    gridRows.Add(row);
                }
                return resultDict;
            }
 
            // Fallback for unknown types
            GPAL.PublishSimpleEvent(GPALEventType.WARNING, $"Unknown value type [{valueType}] detected. Converting to string.", value, GPALObjectType.Other);
            string fallbackValue = value.ToString();
            if (gridRows != null && gridColumns == null)
            {
                gridRows.Add(new List<string> { fallbackValue });
            }
            return fallbackValue;
        }
 
        private static string FlattenValueForGrid(object value)
        {
            if (value == null)
            {
                return "";
            }
 
            if (value is IDictionary dynamicDict)
            {
                var values = new List<string>();
                foreach (dynamic entry in dynamicDict)
                {
                    values.Add(FlattenValueForGrid(entry.Value));
                }
                return string.Join(",", values);
            }
 
            if (value is IEnumerable<object> enumerable && !(value is string))
            {
                var values = new List<string>();
                foreach (var item in enumerable)
                {
                    values.Add(FlattenValueForGrid(item));
                }
                return string.Join(",", values);
            }
 
            return value.ToString();
        }
 
        internal static bool TokenizeFile(GPALFile inputFile, ConverterSettings myConverterSettings = null)
        {
            var xmlDoc = new XmlDocument();
            int fileIdx = 0;
            StreamReader input = null;
            IGPALGrid<string> outGrid = null;
            List<IGPALGrid<string>> tokens = new List<IGPALGrid<string>>(); ;
            bool retval = false;
 
            //if (((IGPALFileInternal)inputFile).AlreadyTokenized)
            //{
            //    return true;
            //}
 
            // Preserve a caller-supplied FirstLineIsColumnHeaders (e.g. via the converter-level
            // WithFirstLineHasColumnNames) as a fallback when the GPALFile itself doesn't specify
            // FirstLineIsColumnNames for this file index.
            bool callerFirstLineIsColumnHeaders = myConverterSettings?.InputFirstLineIsColumnHeaders ?? false;
 
            // The output format is already known by the time TokenizeFile runs, so for delimited
            // input we can avoid holding a fully-populated dictionary AND a re-derived delimited
            // string AND a grid all in memory at once - only one of those representations is
            // actually consumed downstream depending on the output format.
            DataFormat outDataFormat = myConverterSettings?.OutDataFormat ?? DataFormat.NOTSET;
            bool outputNeedsDictionary = outDataFormat == DataFormat.JSON
                || outDataFormat == DataFormat.XML
                || outDataFormat == DataFormat.YAML
                || outDataFormat == DataFormat.CLASS
                || outDataFormat == DataFormat.DICTIONARY
                || outDataFormat == DataFormat.DATABASE;
 
            ((IGPALFileInternal)inputFile).FileDictionary.Clear();
            foreach (string filename in inputFile.Filenames)
            {
                DataFormat inDataFormat = ConverterHelper.GetDataFormatFromExtension(filename);
                bool inputIsDelimited = ConverterHelper.IsDelimitedDataFormat(inDataFormat);
 
                myConverterSettings = new ConverterSettings
                {
                    InDelimiter = ((IGPALFileInternal)inputFile).Delimiter.Count() > fileIdx && ((IGPALFileInternal)inputFile).Delimiter[fileIdx].HasValue
                        ? ((IGPALFileInternal)inputFile).Delimiter[fileIdx].Value
                        : (char?)null,
                    FieldsEnclosedInQuotes = ((IGPALFileInternal)inputFile).FieldsEnclosedInQuotes.Count() > fileIdx
                        ? ((IGPALFileInternal)inputFile).FieldsEnclosedInQuotes[fileIdx]
                        : false,
                    FirstLineIsColumnHeaders = ((IGPALFileInternal)inputFile).FirstLineIsColumnNames.Count() > fileIdx
                        ? ((IGPALFileInternal)inputFile).FirstLineIsColumnNames[fileIdx]
                        : callerFirstLineIsColumnHeaders,
                    IgnoreFirstLineColumnHeaders = ((IGPALFileInternal)inputFile).IgnoreFirstLineColumnNames.Count() > fileIdx
                        ? ((IGPALFileInternal)inputFile).IgnoreFirstLineColumnNames[fileIdx]
                        : false,
                    ColumnNames = inputIsDelimited ? ((IGPALFileInternal)inputFile).FileSettings.ColumnList : new GPALGrid<string>(),
                    ExcelSheetNames = ((IGPALFileInternal)inputFile).FileSettings.ExcelSheetNames,
                    ExcelRowsPerSheet = ((IGPALFileInternal)inputFile).FileSettings.ExcelRowsPerSheet,
                    InputFile = inputFile,
                };
                if (fileIdx < ((IGPALFileInternal)inputFile).FileSettings.Delimiter.Count())
                {
                    ((IGPALFileInternal)inputFile).FileSettings.Delimiter[fileIdx] = ConverterHelper.GetDelimiterFromFormat(inDataFormat);
                }
                else
                {
                    ((IGPALFileInternal)inputFile).FileSettings.Delimiter.Add(ConverterHelper.GetDelimiterFromFormat(inDataFormat));
                }
 
                try
                {
                    string directory = Path.GetDirectoryName(filename);
                    string filenamePart = Path.GetFileName(filename);
                    foreach (string filename2 in Directory.GetFiles(string.IsNullOrEmpty(directory) ? "." : directory, filenamePart))
                    {
                        using (input = new StreamReader(filename2))
                        {
                            input.BaseStream.Position = 0;
 
                            switch (inDataFormat)
                            {
                                case DataFormat.CARET:
                                case DataFormat.CSV:
                                case DataFormat.PIPE:
                                case DataFormat.PRN:
                                case DataFormat.TAB:
                                    var dictionary = ConverterHelper.ConvertDelimitedToDictionary(myConverterSettings, input, filename2);
                                    ((IGPALFileInternal)inputFile).FileDictionary.Add(dictionary);
                                    myConverterSettings.InputDictionary = ((IGPALFileInternal)inputFile).FileDictionary;
 
                                    if (outputNeedsDictionary)
                                    {
                                        // Output consumes the dictionary directly - skip re-deriving the
                                        // delimited string/grid representations entirely.
                                        ((IGPALFileInternal)inputFile).FileData.Add(string.Empty);
                                        ((IGPALFileInternal)inputFile).FileGrid.Add(GPAL.Grid.ToGPALObject());
                                    }
                                    else
                                    {
                                        dynamic delimitedData = ConverterHelper.ConvertInputDictionaryToDelimitedAndGrid(myConverterSettings, out outGrid);
                                        ((IGPALFileInternal)inputFile).FileData.Add(delimitedData);
                                        ((IGPALFileInternal)inputFile).FileGrid.Add(outGrid);
 
                                        // The data/grid were derived from the dictionary and the output
                                        // doesn't need the dictionary itself - free its contents now.
                                        dictionary.Clear();
                                    }
                                    break;
 
                                case DataFormat.JSON:
                                    string fileString = input.ReadToEnd();
                                    input.BaseStream.Position = 0;
 
                                    dynamic jsonObject = JsonConvert.DeserializeObject<dynamic>(fileString, new JsonConverter[] { new CustomJsonConverter() });
 
                                    ((IGPALFileInternal)inputFile).FileData.Add(fileString);
 
                                    var records = new Dictionary<object, dynamic>();
                                    var gridRows = new List<List<string>>();
 
                                    // Helper to recursively convert any array (IEnumerable<dynamic>) to surrogate-keyed dict
                                    dynamic ConvertArrayToSurrogateDict(IEnumerable<dynamic> array, int startIndex = 0)
                                    {
                                        var surrogateDict = new Dictionary<object, dynamic>();
                                        int idx = startIndex;
                                        foreach (dynamic item in array)
                                        {
                                            surrogateDict[$"GPALKEY{idx:D4}"] = item;
                                            idx++;
                                        }
                                        return surrogateDict;
                                    }
 
                                    // Recursively walk the structure and replace all arrays with surrogate dicts
                                    dynamic ReplaceArrays(dynamic value)
                                    {
                                        if (value is IEnumerable<dynamic> enumerable && !(value is string))
                                        {
                                            var items = Enumerable.ToList<dynamic>(enumerable);
 
                                            // A dictionary with a complex (non-string) key serializes as an array of
                                            // {"Key": ..., "Value": ...} objects instead of a plain JSON object (see
                                            // NormalizeForCleanJson) - a complex key can't be a JSON object key at all,
                                            // and this is the standard convention (matching KeyValuePair<K,V>'s own
                                            // property names) most serializers use for this case, rather than an
                                            // invented GPAL-specific marker leaking into the file. Detect that shape
                                            // and rebuild a real dictionary here (the key may itself be a nested dict,
                                            // same as YAML's explicit-key mapping - CreateObjectFromDictionary already
                                            // reconstructs that correctly) instead of wrapping it as a plain
                                            // GPALKEY-indexed surrogate. Note: a genuine two-property {Key, Value}
                                            // data object (e.g. a real KeyValuePair-shaped list element) would also
                                            // match this shape - an acceptable, low-probability ambiguity shared by
                                            // every serializer that uses this same convention.
                                            bool looksLikePairsArray = items.Count > 0 && items.All(it =>
                                                it is IDictionary d && d.Count == 2
                                                && d.Contains("Key") && d.Contains("Value"));
                                            if (looksLikePairsArray)
                                            {
                                                var pairsDict = new Dictionary<object, dynamic>();
                                                foreach (IDictionary pairObj in items)
                                                {
                                                    dynamic pairKey = ReplaceArrays(pairObj["Key"]);
                                                    dynamic pairValue = ReplaceArrays(pairObj["Value"]);
                                                    pairsDict[pairKey] = pairValue;
                                                }
                                                return pairsDict;
                                            }
 
                                            // It's an array/list
                                            return ConvertArrayToSurrogateDict(items);
                                        }
                                        else if (value is IDictionary<string, dynamic> dict)
                                        {
                                            // It's an object - recurse into values
                                            var newDict = new Dictionary<object, dynamic>();
                                            foreach (var kvp in dict)
                                            {
                                                newDict[kvp.Key] = ReplaceArrays(kvp.Value);
                                            }
                                            return newDict;
                                        }
                                        else
                                        {
                                            // Primitive - return as-is
                                            return value;
                                        }
                                    }
 
                                    dynamic processedRoot = ReplaceArrays(jsonObject);
 
                                    // If top-level is array, wrap as surrogate dict for FileDictionary
                                    if (processedRoot is IDictionary && ConverterHelper.IsArrayLike((IDictionary)processedRoot))
                                    {
                                        records = (Dictionary<object, dynamic>)processedRoot;
                                    }
                                    else
                                    {
                                        // Single object or other - wrap under a key or just store
                                        records["root"] = processedRoot;
                                    }
 
                                    // Grid building remains the same: iterate top-level records (now surrogate-keyed)
                                    foreach (KeyValuePair<object, dynamic> pair in records.OrderBy(p => p.Key.ToString()))
                                    {
                                        if (pair.Value is IDictionary rowDict)
                                        {
                                            var row = new List<string>();
                                            foreach (dynamic kvp in rowDict)
                                            {
                                                if (kvp.Key != null)
                                                {
                                                    string key = kvp.Key.ToString();
                                                    if (key.StartsWith("GPALKEY")) continue; // skip surrogates in grid
                                                    var processedValue = ProcessDictionaryValue(kvp.Value, myConverterSettings, null, null, key);
                                                    if (!(kvp.Value is IDictionary) && !(kvp.Value is IList<object>))
                                                    {
                                                        row.Add(FlattenValueForGrid(processedValue));
                                                    }
                                                }
                                            }
                                            if (row.Count > 0)
                                            {
                                                gridRows.Add(row);
                                            }
                                        }
                                    }
 
                                    ((IGPALFileInternal)inputFile).FileDictionary.Add(records);
 
                                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"JSON FileDictionary count: [{records.Count}]", null, GPALObjectType.Other);
 
                                    var gridRow = GPAL.Grid.ToGPALObject();
                                    foreach (var row in gridRows)
                                    {
                                        gridRow.AddRow(row);
                                    }
 
                                    myConverterSettings.InputDictionary = ((IGPALFileInternal)inputFile).FileDictionary;
                                    ((IGPALFileInternal)inputFile).FileGrid.Add(gridRow);
                                    break;
 
                                case DataFormat.PDF:
                                    break;
 
                                case DataFormat.XLSX:
                                    IGPALGrid<string> grid = ExcelHelper.TokenizeExcelFile(filename2);
                                    List<string> colNames = new List<string>();
                                    List<Dictionary<object, dynamic>> returnList = new List<Dictionary<object, dynamic>>();
 
                                    ((IGPALFileInternal)inputFile).FileGrid.Add((IGPALGrid<string>)grid.Clone());
 
                                    // find out how many worksheets we have and find out the number of rows to ascertain when the grid data is another sheet with different column names
                                    using (var workbook = new XLWorkbook(filename))
                                    {
                                        List<int> rowsPerSheet = new List<int>();
                                        List<string> sheetName = new List<string>();
 
                                        foreach (IXLWorksheet worksheet in workbook.Worksheets)
                                        {
                                            int lastRowNumber = worksheet.LastRowUsed()?.RowNumber() ?? 0;
                                            // If there are blank rows after the last used row, use the sheet's defined range
                                            int totalRows = Math.Max(lastRowNumber, worksheet.RangeUsed()?.LastRow().RowNumber() ?? 0);
                                            // Or even simpler – just take the maximum row index that exists:
                                            totalRows = worksheet.Rows().LastOrDefault()?.RowNumber() ?? 0;
 
                                            rowsPerSheet.Add(totalRows);
                                            sheetName.Add(worksheet.Name);
                                        }
 
                                        ((IGPALFileInternal)inputFile).FileSettings.ExcelRowsPerSheet.AddRow(rowsPerSheet);
                                        ((IGPALFileInternal)inputFile).FileSettings.ExcelSheetNames.AddRow(sheetName);
 
                                        int columnHeaders = 0;
                                        foreach (int rowCnt in rowsPerSheet)
                                        {
                                            if (myConverterSettings.FirstLineIsColumnHeaders)
                                            {
                                                colNames = grid[columnHeaders];
                                            }
                                            else
                                            {
                                                int colIdx = 0;
                                                foreach (string columnName in grid[columnHeaders].Select(v => (string)v))
                                                {
                                                    colNames.Add($"Column{colIdx++}");
                                                }
                                            }
 
                                            myConverterSettings.ColumnNames.AddRow(colNames);
 
                                            if (0 == ((IGPALFileInternal)inputFile).FileSettings.ColumnList.Count())
                                                ((IGPALFileInternal)inputFile).FileSettings.ColumnList.AddRow(colNames);
 
                                            int rowIdx = 0;
 
                                            foreach (Dictionary<object, dynamic> row in ConverterHelper.ConvertGridToDictionary(colNames, grid, columnHeaders, rowCnt))
                                            {
                                                if (true == myConverterSettings.FirstLineIsColumnHeaders && 0 == rowIdx++)
                                                    continue;
 
                                                returnList.Add(row);
                                            }
 
                                            columnHeaders += rowCnt;
                                        }
 
                                        // now delete the column headers if denoted in settings
                                        columnHeaders = 0;
                                        int iteration = 0;
                                        foreach (int rowCnt in rowsPerSheet)
                                        {
                                            if (myConverterSettings.FirstLineIsColumnHeaders)
                                            {
                                                grid.DeleteRow(0);
                                            }
                                            columnHeaders = rowCnt - iteration++;   // we have one less row than we counted in the grid for each iteration
                                        }
                                    }
 
                                    ((IGPALFileInternal)inputFile).FileDictionary = returnList;
                                    myConverterSettings.InputDictionary = ((IGPALFileInternal)inputFile).FileDictionary;
                                    break;
 
                                case DataFormat.XML:
                                    xmlDoc.PreserveWhitespace = true;
                                    xmlDoc.Load(input);
                                    input.BaseStream.Position = 0;
                                    ((IGPALFileInternal)inputFile).FileDictionary.Add(ConverterHelper.ConvertXMLToDictionary(xmlDoc));
                                    myConverterSettings.InputDictionary = ((IGPALFileInternal)inputFile).FileDictionary;
                                    ((IGPALFileInternal)inputFile).FileData.Add(ConverterHelper.ConvertInputDictionaryToDelimitedAndGrid(myConverterSettings, out outGrid));
                                    ((IGPALFileInternal)inputFile).FileGrid.Add(outGrid);
                                    break;
 
                                case DataFormat.YAML:
                                    var deserializer = new DeserializerBuilder()
                                        .WithNamingConvention(CamelCaseNamingConvention.Instance)
                                        .Build();
 
                                    string yamlText = input.ReadToEnd();
                                    input.BaseStream.Position = 0;
                                    dynamic yamlObject = deserializer.Deserialize<object>(yamlText);
                                    ((IGPALFileInternal)inputFile).FileData.Add(yamlText);
 
                                    ((IGPALFileInternal)inputFile).FileDictionary.Clear();
                                    ((IGPALFileInternal)inputFile).FileGrid.Clear();
 
                                    var yamlGridRows = new List<List<string>>();
                                    var processedYamlDict = new Dictionary<dynamic, dynamic>();
 
                                    if (yamlObject is IDictionary yamlMapping)
                                    {
                                        foreach (var key in yamlMapping.Keys)
                                        {
                                            dynamic processedKey = (key is IDictionary || key is IList)
                                                ? ProcessDictionaryValue(key, myConverterSettings, yamlGridRows)
                                                : (key?.ToString() ?? "null_key");
                                            processedYamlDict[processedKey] = ProcessDictionaryValue(yamlMapping[key], myConverterSettings, yamlGridRows);
                                        }
                                    }
                                    else if (yamlObject is IList yamlSequence)
                                    {
                                        // Root is a YAML sequence, not a mapping - synthesize row keys
                                        // (same GPALKEYN convention used for headerless CSV rows).
                                        int rootIdx = 0;
                                        foreach (var item in yamlSequence)
                                        {
                                            processedYamlDict[$"GPALKEY{rootIdx++:D4}"] = ProcessDictionaryValue(item, myConverterSettings, yamlGridRows);
                                        }
                                    }
                                    else
                                    {
                                        // Root is a bare YAML scalar.
                                        processedYamlDict["GPALKEY0000"] = ProcessDictionaryValue(yamlObject, myConverterSettings, yamlGridRows);
                                    }
 
                                    var yamlGridRow = GPAL.Grid.ToGPALObject();
                                    foreach (var row in yamlGridRows)
                                    {
                                        yamlGridRow.AddRow(row);
                                    }
 
                                    ((IGPALFileInternal)inputFile).FileDictionary.Add(processedYamlDict);
                                    myConverterSettings.InputDictionary = ((IGPALFileInternal)inputFile).FileDictionary;
                                    ((IGPALFileInternal)inputFile).FileGrid.Add(yamlGridRow);
                                    break;
 
                                case DataFormat.HTML:
                                    ((IGPALFileInternal)inputFile).FileDictionary = ConverterHelper.ConvertHtmlToDictionary(filename2);
                                    myConverterSettings.InputDictionary = ((IGPALFileInternal)inputFile).FileDictionary;
                                    dynamic inData = ConverterHelper.ConvertInputDictionaryToDelimitedAndGrid(myConverterSettings, out outGrid);
                                    ((IGPALFileInternal)inputFile).FileData.Add(inData);
                                    ((IGPALFileInternal)inputFile).FileGrid.Add(outGrid);
                                    break;
                            }
 
                            retval = true;
                        }
                    }
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Problem tokenizing file [{filename}]. Continuing.", inputFile, GPALObjectType.GPALFile, ex);
                }
                fileIdx++;
            }
 
            if (0 == ((IGPALFileInternal)inputFile).FileSettings.FileGrid.Count)
                FileHelper.GetGrid(((IGPALFileInternal)inputFile).FileSettings, out tokens);
            else
                tokens = ((IGPALFileInternal)inputFile).FileSettings.FileGrid;
 
            ((IGPALFileInternal)inputFile).TokenList = tokens;
            ((IGPALFileInternal)inputFile).AlreadyTokenized = retval;
 
            return retval;
        }
        [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
        private static extern int GetShortPathName(string lpszLongPath, StringBuilder lpszShortPath, int cchBuffer);

        public static string GetShortPath(string longPath)
        {
            var sb = new StringBuilder(1024);
            int result = GetShortPathName(longPath, sb, sb.Capacity);
            return result > 0 ? sb.ToString() : longPath;
        }

        internal static bool IsWebAddress(string fileName)
        {
            return true == Uri.TryCreate(fileName, UriKind.Absolute, out Uri uri)
                && (Uri.UriSchemeHttp == uri.Scheme || Uri.UriSchemeHttps == uri.Scheme);
        }
 

        internal static bool FetchOrDownloadInTab(string url, string localPath, IBrowser browser)
        {
            bool gotIt = FetchToFile(url, localPath, browser);
 
            if (false == gotIt)
                gotIt = DownloadInTab(url, localPath, browser);
 
            return gotIt;
        }
        internal static bool HasStartedDownloading(string downloadDirectory, string localPath)
        {
            return true == File.Exists(localPath)
                || (true == Directory.Exists(downloadDirectory)
                    && true == Directory.EnumerateFiles(downloadDirectory).Any(file => file.EndsWith(".crdownload") || file.EndsWith(".part")));
        }

        internal static int AllowanceFor(string downloadDirectory, Browser.BrowserSettings browserSettings)
        {
            int retVal = true == IsFinalizing(downloadDirectory, watchArmed)
                       ? browserSettings.DownloadTimeoutInSec * FinalizeBudgetMultiplier
                       : browserSettings.DownloadTimeoutInSec;
 
            return retVal;
        }

        internal static void DiscardAbandonedDownload(Browser.BrowserSettings browserSettings)
        {
            foreach (string scratch in watchedScratchFiles)
                try
                {
                    if (true == File.Exists(scratch))
                    {
                        File.Delete(scratch);
                        GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{Path.GetFileName(scratch)}] was left behind by a download that never finished, removed so it does not name the next one around it", browserSettings.Browser, GPALObjectType.Browser);
                    }
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Could not remove [{scratch}]", browserSettings.Browser, GPALObjectType.Browser, ex);
                }
 
            watchedScratchFiles.Clear();
        }
 
        internal const int FinalizeBudgetMultiplier = 6;   // the browser's own finish-up is slower than a transfer and is not ours to hurry

        internal static bool IsFinalizing(string downloadDirectory, DateTime since)
        {
            bool retVal = false;
 
            try
            {
                if (true == Directory.Exists(downloadDirectory))
                    // the later of the two, for the reason NewestDownloadSince takes it: windows hands a file
                    // recreated under a name it just used the original's creation time, and a run downloading the
                    // same file over and over reuses these names constantly. by creation time alone the browser's
                    // brand new scratch file reads as belonging to the download before it, and the finalize
                    // allowance is refused to the one download that is actually finalizing
                    retVal = Directory.EnumerateFiles(downloadDirectory)
                                      .Any(file => true == IsTemporaryDownload(file)
                                                && (File.GetCreationTime(file) >= since || File.GetLastWriteTime(file) >= since));
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Could not look for a download being finished in [{downloadDirectory}]", null, GPALObjectType.None, ex);
            }
 
            return retVal;
        }
 
        internal static long DownloadProgress(string downloadDirectory, string localPath)
        {
            long retVal = -1;
 
            try
            {
                if (true == File.Exists(localPath))
                    retVal = new FileInfo(localPath).Length;
 
                if (true == Directory.Exists(downloadDirectory))
                    foreach (string scratch in Directory.EnumerateFiles(downloadDirectory).Where(f => true == IsTemporaryDownload(f)))
                        retVal = (0 > retVal ? 0 : retVal) + new FileInfo(scratch).Length;
            }
            catch (Exception ex)
            {
                GPAL.PublishSimpleEvent(GPALEventType.DEBUG, $"Could not measure the download in [{downloadDirectory}]", null, GPALObjectType.None, ex);
            }
 
            return retVal;
        }
        internal static bool DownloadInTab(string url, string localPath, IBrowser browser)
        {
            Browser.BrowserSettings browserSettings = ((Browser.Browser)browser).BrowserSettings;
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Downloading [{url}] with the browser to [{localPath}].", browser, GPALObjectType.Browser);
 
            // the browser decides which directory a download goes to, not us, so the file lands in the download
            // directory first and is moved to localPath afterwards. fetch has the bytes and writes straight to
            // localPath, which is why the two paths differ here
            string downloadDirectory = GetDefaultDownloadDirectory(browser);
            string landsAt = Path.Combine(downloadDirectory, Path.GetFileName(localPath));
 
            // ask for the filename we want. each engine takes it differently: the extension suggests a name
            // relative to the download directory, puppeteer takes the directory, selenium takes the full path
            if (true == browserSettings.UseOttoMagic)
                browserSettings.MagicHelper.WithDownloadFile(Path.GetFileName(localPath));
            else if (true == browserSettings.UsePuppeteer)
                browserSettings.PuppeteerClient.DownloadTo(downloadDirectory).Execute();
            else
                BrowserHelper.SetHeadlessDownload(browserSettings, landsAt);
 
            // SetupDownloadWatcher is what leftclickanddownload uses. it watches the directory, waits for the
            // .crdownload or .part to stop growing, moves the finished file to landsAt and sets FileDownloaded
            browserSettings.FileDownloaded = null;
            SetupDownloadWatcher(landsAt, browserSettings);
 
            browser.NewTab(url);
 
            Stopwatch waitingToStart = Stopwatch.StartNew();
 
            // FileDownloaded stays null until the watcher sets it true or false.
            // DownloadTimeoutInSec is how long to wait for the download to start, not to finish, so stop the
            // clock once a file shows up in the download directory and let the watcher take as long as it needs
            long lastSeen = long.MinValue;
 
            while (null == browserSettings.FileDownloaded || false == browserSettings.FileDownloaded)
            {
                long progress = DownloadProgress(downloadDirectory, landsAt);
 
                if (progress != lastSeen)       // it moved, so it is alive and gets the whole budget again
                {
                    lastSeen = progress;
                    waitingToStart.Restart();
                }
                else if (waitingToStart.Elapsed.TotalSeconds > AllowanceFor(downloadDirectory, browserSettings))
                    break;                      // nothing has happened for the whole allowance: it never started, or it died
 
                Thread.Sleep(100);
            }
 
            // we opened the tab so we close it. chromium usually closes it itself, firefox does not
            browser.CloseTab();
 
            bool downloaded = true == browserSettings.FileDownloaded;
 
            // move it out of the download directory into the working directory, where WithFileName said the file
            // would be. everything downstream, CopyTo and the converter included, is looking at localPath
            if (true == downloaded && false == landsAt.Equals(localPath, StringComparison.OrdinalIgnoreCase))
            {
                if (true == File.Exists(localPath))
                    File.Delete(localPath);
 
                File.Move(landsAt, localPath);
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{landsAt}] moved to [{localPath}]", browser, GPALObjectType.Browser);
            }
 
            if (false == downloaded)
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"[{url}] did not download to [{downloadDirectory}] within [{browserSettings.DownloadTimeoutInSec}] seconds.", browser, GPALObjectType.Browser);
 
            return downloaded;
        }
        internal static bool FetchToFile(string url, string localPath, IBrowser browser)
        {
            bool downloaded = false;
            string[] headers = true == File.Exists(localPath)
                ? new string[] { "If-Modified-Since", File.GetLastWriteTimeUtc(localPath).ToString("R") }
                : null;
 
            GPAL.PublishSimpleEvent(GPALEventType.INFO, $"Downloading [{url}] with the browser to [{localPath}].", browser, GPALObjectType.Browser);
 
            byte[] bytes = BrowserHelper.FetchBytes((Browser.Browser)browser, url, headers, out string lastModified);
 
            if (304 == browser.ServerResponseCode)
            {
                GPAL.PublishSimpleEvent(GPALEventType.INFO, $"[{url}] has not changed, using [{localPath}].", browser, GPALObjectType.Browser);
                downloaded = true;
            }
            else if (null == bytes)
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"[{url}] returned no content to save as [{localPath}], HTTP [{browser.ServerResponseCode}].", browser, GPALObjectType.Browser);
            else if (200 > browser.ServerResponseCode || 300 <= browser.ServerResponseCode)
                // the body is whatever the site sends with a refusal, an error page or a json message, and saving
                // that as the file would leave something that looks downloaded but is not the file
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"[{url}] returned HTTP [{browser.ServerResponseCode}], nothing saved to [{localPath}].", browser, GPALObjectType.Browser);
            else if (0 == bytes.Length)
                GPAL.PublishSimpleEvent(GPALEventType.ERROR, $"[{url}] returned an empty file, nothing saved to [{localPath}].", browser, GPALObjectType.Browser);
            else
            {
                File.WriteAllBytes(localPath, bytes);
 
                // stamp it with the server's own modified time, so next run's question is about the content
                // rather than about when we happened to fetch it
                if (true == DateTime.TryParse(lastModified, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal, out DateTime modified))
                    File.SetLastWriteTimeUtc(localPath, modified);
 
                downloaded = true;
            }
 
            return downloaded;
        }

        public static string EnsureDirectoryEndsWithBackslash(string directoryPath)
        {
            // Check if the directory path already ends with '\'
            if (!directoryPath.EndsWith("\\"))
            {
                // If not, add '\'
                directoryPath += "\\";
            }
            return directoryPath;
        }
 
        internal static bool TokenizeHierachicalCsvFile(GPALFile inputFile, ConverterSettings myConverterSettings)
        {
            bool retVal = true;
            int fileIdx = 0;
            // Initialize output dictionary
            List<Dictionary<object, dynamic>> output = new List<Dictionary<object, dynamic>>();
 
            ((IGPALFileInternal)inputFile).FileDictionary.Clear();
 
            foreach (string filename in inputFile.Filenames)
            {
                try
                {
                        myConverterSettings = new ConverterSettings()
                        {
                            InDelimiter = (0 < ((IGPALFileInternal)inputFile).Delimiter.Count() && ((IGPALFileInternal)inputFile).Delimiter[fileIdx].HasValue) ? ((IGPALFileInternal)inputFile).Delimiter[fileIdx].Value : (char?)null,
                            FieldsEnclosedInQuotes = 0 < ((IGPALFileInternal)inputFile).FieldsEnclosedInQuotes.Count() && ((IGPALFileInternal)inputFile).FieldsEnclosedInQuotes[fileIdx],
                            FirstLineIsColumnHeaders = 0 < ((IGPALFileInternal)inputFile).FirstLineIsColumnNames.Count() && ((IGPALFileInternal)inputFile).FirstLineIsColumnNames[fileIdx],
                        };
 
                    DataFormat InDataFormat = ConverterHelper.GetDataFormatFromExtension(filename);
 
                    string[] csvLines = File.ReadAllLines(filename);
 
                    // Split headers and data into lists
                    string[] headers = csvLines[0].Trim().Split(',');
                    List<dynamic[]> dataLines = new List<dynamic[]>();
 
                    // Convert data lines to arrays
                    for (int i = 1; i < csvLines.Length; i++)
                    {
                        string[] data = csvLines[i].Trim().Split(',');
                        dataLines.Add(data);
                    }
 
                    // Iterate over data lines to construct output
                    foreach (var data in dataLines)
                    {
                        Dictionary<object, dynamic> rowData = new Dictionary<object, dynamic>();
 
                        for (int i = 0; i < headers.Length; i++)
                        {
                            string[] headerParts = headers[i].Split('.');
                            Dictionary<object, dynamic> currentDict = rowData;
 
                            for (int j = 0; j < headerParts.Length; j++)
                            {
                                string part = headerParts[j];
                                bool isLastPart = (j == headerParts.Length - 1);
 
                                if (part.EndsWith("]"))
                                {
                                    int indexStart = part.LastIndexOf('[');
                                    string arrayName = part.Substring(0, indexStart);
                                    int arrayIndex = int.Parse(part.Substring(indexStart + 1, part.Length - indexStart - 2));
 
                                    // Ensure array exists
                                    if (!currentDict.ContainsKey(arrayName))
                                    {
                                        currentDict[arrayName] = new List<Dictionary<string, object>>();
                                    }
 
                                    // Ensure array index exists
                                    List<Dictionary<object, dynamic>> currentArray = (List<Dictionary<object, dynamic>>)currentDict[arrayName];
                                    while (currentArray.Count <= arrayIndex)
                                    {
                                        currentArray.Add(new Dictionary<object, dynamic>());
                                    }
 
                                    if (isLastPart)
                                    {
                                        currentArray[arrayIndex] = data[i];
                                    }
                                    else
                                    {
                                        currentDict = currentArray[arrayIndex];
                                    }
                                }
                                else
                                {
                                    if (!currentDict.ContainsKey(part))
                                    {
                                        currentDict[part] = isLastPart ? (object)data[i] : new Dictionary<object, dynamic>();
                                    }
 
                                    if (isLastPart)
                                    {
                                        currentDict[part] = data[i];
                                    }
                                    else
                                    {
                                        currentDict = (Dictionary<object, dynamic>)currentDict[part];
                                    }
                                }
                            }
                        }
 
                        // Add the constructed dictionary to the output list
                        output.Add(rowData);
                    }
                }
                catch (Exception ex)
                {
                    GPAL.PublishSimpleEvent(GPALEventType.EXCEPTION, $"Unable to open file [{filename}]. Continuing.", inputFile, GPALObjectType.GPALFile, ex);
                    retVal = false;
                }
                fileIdx++;
            }
            ((IGPALFileInternal)inputFile).FileDictionary = output;
            return retVal;
        }
        // Helper method to convert nested json Dictionary<object, object> to Dictionary<object, dynamic>
        internal static Dictionary<object, dynamic> ConvertToStringDynamicDict(dynamic inputDict)
        {
            var resultDict = new Dictionary<object, dynamic>();
            foreach (var kvp in inputDict)
            {
                if (kvp.Key != null) // Handle null keys
                {
                    string key = kvp.Key.ToString();
                    object value = kvp.Value;
                    // Check if the value is another nested dictionary
                    if (value is IDictionary nestedDict)
                    {
                        // Recursively convert the nested dictionary
                        resultDict.Add(key, ConvertToStringDynamicDict(nestedDict));
                    }
                    else
                    {
                        // Add non-dictionary value directly
                        resultDict.Add(key, value);
                    }
                }
            }
            return resultDict;
        }
    }
}