NextJsHydrationExtractor.cs

// written by grok
// inspired by the Web Scraping Club article + React Flight discussions
 
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using GenerallyPositive.Browser;
using System.Text.Json;   // for Browser instance if needed
 
namespace GenerallyPositive
{
    public class NextJsHydrationResult
    {
        public bool Success { get; set; }
        public string RawNextDataJson { get; set; } = string.Empty;     // from __NEXT_DATA__ if present
        public List<string> FlightChunks { get; set; } = new List<string>();         // raw self.__next_f.push(...) contents
        public string ParsedStructuredData { get; set; } = string.Empty; // future: cleaned/parsed version
        public string Url { get; set; } = string.Empty;
        public string Title { get; set; } = "Untitled";
        public int OriginalHtmlLength { get; set; }
        public string Message { get; set; } = string.Empty;
        public bool IsPagesRouter { get; set; }      // true if __NEXT_DATA__ was found
        public bool IsAppRouter { get; set; }        // true if flight chunks were found
 
        private JsonDocument _parsedJson;
 
        public JsonDocument GetJson()
        {
            if (_parsedJson != null)
                return _parsedJson;
 
            if (string.IsNullOrWhiteSpace(RawNextDataJson))
            {
                Message = "RawNextDataJson is empty";
                return null;
            }
 
            try
            {
                _parsedJson = JsonDocument.Parse(RawNextDataJson);
                Message = "JSON parsed successfully";
                return _parsedJson;
            }
            catch (JsonException ex)
            {
                Message = $"JSON parse failed: [{ex.Message}]";
 
                // Show where it fails
                int errorPos = (int)(ex.LineNumber > 0 ? ex.LineNumber : 0);
                string snippet = RawNextDataJson.Length > 300
                    ? RawNextDataJson.Substring(0, 300) + "..."
                    : RawNextDataJson;
 
                GPAL.PublishSimpleEvent(Enums.GPALEventType.ERROR, Message);
                GPAL.PublishSimpleEvent(Enums.GPALEventType.ERROR, $"First 300 chars: [{snippet}]");
 
                return null;
            }
            catch (Exception ex)
            {
                Message = $"Unexpected error parsing JSON: {ex.Message}";
                return null;
            }
        }
 
        public void DisposeJson()
        {
            _parsedJson?.Dispose();
            _parsedJson = null;
        }
    }
 
    internal static class NextJsHydrationExtractor
    {
        public static NextJsHydrationResult Extract(Browser.Browser browser, string htmlSource)
        {
            var result = new NextJsHydrationResult
            {
                OriginalHtmlLength = htmlSource?.Length ?? 0,
                Url = browser.GetSetCurrentUrl()
            };
 
            if (string.IsNullOrWhiteSpace(htmlSource))
            {
                result.Message = "Empty HTML";
                return result;
            }
 
            try
            {
                var doc = new HtmlDocument();
                doc.LoadHtml(htmlSource);
 
                // Always resolve a title so even a "nothing detected" file is labeled usefully.
                result.Title = ExtractTitle(doc) ?? "Untitled";
 
                // 1. Try Pages Router first (clean JSON)
                var nextDataNode = doc.DocumentNode.SelectSingleNode("//script[@id='__NEXT_DATA__']");
                if (nextDataNode != null)
                {
                    result.IsPagesRouter = true;
                    result.RawNextDataJson = nextDataNode.InnerText.Trim();
                    result.ParsedStructuredData = result.RawNextDataJson; // already JSON
                    result.Success = true;
                }
 
                // 2. App Router - collect all flight chunks
                var flightScripts = doc.DocumentNode.SelectNodes("//script")
                    ?.Where(s => s.InnerText?.Contains("self.__next_f.push") == true)
                    ?.Select(s => s.InnerText.Trim())
                    ?.ToList() ?? new List<string>();
 
                if (flightScripts.Any())
                {
                    result.IsAppRouter = true;
                    result.FlightChunks = flightScripts;
 
                    // Decode the React Flight (RSC) stream into navigable JSON. Falls back to the raw joined
                    // chunks if decoding fails, so nothing is ever lost.
                    result.ParsedStructuredData = DecodeFlightChunks(flightScripts) ?? string.Join("\n", flightScripts);
 
                    result.Success = true;
                    result.Message = $"Found {flightScripts.Count} React Flight chunks (App Router)";
                }
                else
                {
                    result.Message = "No __NEXT_DATA__ or self.__next_f.push found. Not a detectable Next.js site or SSR disabled.";
                }
            }
            catch (Exception ex)
            {
                result.Success = false;
                result.Message = ex.Message;
            }
 
            return result;
        }
 
        private static string ExtractTitle(HtmlDocument doc)
        {
            return doc.DocumentNode.SelectSingleNode("//title")?.InnerText.Trim()
                ?? doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim();
        }

        internal static string DecodeFlightChunks(List<string> flightScripts)
        {
            if (flightScripts == null || flightScripts.Count == 0)
                return null;
 
            try
            {
                // 1. Pull the string payload out of every push([...]) and concatenate in document order.
                var streamBuilder = new StringBuilder();
                foreach (var script in flightScripts)
                    foreach (var arr in ExtractPushArrays(script))
                    {
                        try
                        {
                            using (var doc = JsonDocument.Parse(arr))
                            {
                                var root = doc.RootElement;
                                if (root.ValueKind == JsonValueKind.Array && root.GetArrayLength() >= 2 &&
                                    root[1].ValueKind == JsonValueKind.String)
                                    streamBuilder.Append(root[1].GetString());
                            }
                        }
                        catch { /* skip a malformed push, keep going */ }
                    }
 
                string stream = streamBuilder.ToString();
                if (string.IsNullOrWhiteSpace(stream))
                    return null;
 
                // 2. Split into id:value rows and bucket them.
                var data = new List<string>();
                var modules = new List<string>();
                var other = new List<string>();
                var seen = new HashSet<string>(StringComparer.Ordinal);
 
                string UniqueKey(string id)
                {
                    string k = id; int n = 1;
                    while (seen.Contains(k)) k = id + "_" + (++n);
                    seen.Add(k);
                    return k;
                }
 
                string curId = null;
                var curContent = new StringBuilder();
 
                void Flush()
                {
                    if (curId == null) return;
 
                    string content = curContent.ToString();
                    string key = UniqueKey(curId);
 
                    if (IsValidJson(content))
                        data.Add($"\"{key}\":{content}");
                    else if (content.Length > 1 && char.IsLetter(content[0]) && IsValidJson(content.Substring(1)))
                    {
                        string val = content.Substring(1);
                        if ('I' == content[0]) modules.Add($"\"{key}\":{val}");
                        else other.Add($"\"{key}\":{val}");
                    }
                    else
                        other.Add($"\"{key}\":{JsonSerializer.Serialize(content)}");
 
                    curId = null;
                    curContent.Clear();
                }
 
                foreach (var line in stream.Split('\n'))
                {
                    int colon = line.IndexOf(':');
                    string prefix = colon > 0 ? line.Substring(0, colon) : null;
                    bool isRowStart = prefix != null && Regex.IsMatch(prefix, "^[0-9a-fA-F]+$");
 
                    if (isRowStart)
                    {
                        Flush();
                        curId = prefix;
                        curContent.Append(line.Substring(colon + 1));
                    }
                    else if (curId != null)
                    {
                        // continuation of a rare multi-line value
                        curContent.Append('\n').Append(line);
                    }
                }
                Flush();
 
                if (0 == data.Count && 0 == modules.Count && 0 == other.Count)
                    return null;
 
                // 3. Assemble one navigable object and pretty-print it.
                var combined = new StringBuilder();
                combined.Append("{\"data\":{").Append(string.Join(",", data)).Append("}");
                combined.Append(",\"modules\":{").Append(string.Join(",", modules)).Append("}");
                if (other.Count > 0)
                    combined.Append(",\"other\":{").Append(string.Join(",", other)).Append("}");
                combined.Append("}");
 
                return PrettyPrintOrRaw(combined.ToString());
            }
            catch
            {
                return null;
            }
        }

        private static IEnumerable<string> ExtractPushArrays(string script)
        {
            if (string.IsNullOrEmpty(script))
                yield break;
 
            int idx = 0;
            while ((idx = script.IndexOf("__next_f.push(", idx, StringComparison.Ordinal)) >= 0)
            {
                int open = script.IndexOf('[', idx);
                if (open < 0)
                    yield break;
 
                int depth = 0;
                bool inStr = false, esc = false, closed = false;
                int end = -1;
 
                for (int i = open; i < script.Length; i++)
                {
                    char c = script[i];
                    if (inStr)
                    {
                        if (esc) esc = false;
                        else if ('\\' == c) esc = true;
                        else if ('"' == c) inStr = false;
                    }
                    else
                    {
                        if ('"' == c) inStr = true;
                        else if ('[' == c) depth++;
                        else if (']' == c) { depth--; if (0 == depth) { end = i; closed = true; break; } }
                    }
                }
 
                if (false == closed)
                    yield break;
 
                yield return script.Substring(open, end - open + 1);
                idx = end + 1;
            }
        }
 
        private static bool IsValidJson(string s)
        {
            if (string.IsNullOrWhiteSpace(s))
                return false;
            char c = s[0];
            // fast reject: JSON values start with one of these
            if ('{' != c && '[' != c && '"' != c && '-' != c && false == char.IsDigit(c)
                && "tfn".IndexOf(c) < 0)
                return false;
            try { using (JsonDocument.Parse(s)) return true; }
            catch { return false; }
        }
 
        public static void SaveToFile(NextJsHydrationResult result, string filePath)
        {
            var sb = new System.Text.StringBuilder();
            sb.AppendLine($"# Next.js Hydration Data - {result.Title}");
            sb.AppendLine($"URL: {result.Url}");
            sb.AppendLine($"Type: {DescribeType(result)}");
            sb.AppendLine($"Saved: {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} UTC");
            sb.AppendLine($"Original HTML length: {result.OriginalHtmlLength / 1024} KB");
            sb.AppendLine("---");
 
            if (result.IsPagesRouter && !string.IsNullOrEmpty(result.RawNextDataJson))
            {
                sb.AppendLine("## __NEXT_DATA__ (JSON)");
                sb.AppendLine(PrettyPrintOrRaw(result.RawNextDataJson));
            }
            else if (result.IsAppRouter)
            {
                sb.AppendLine($"## React Flight (decoded from {result.FlightChunks.Count} chunks)");
                sb.AppendLine(result.ParsedStructuredData);
            }
            else
            {
                // Nothing detected: record why, so the file is not just an unexplained empty header.
                sb.AppendLine(string.IsNullOrWhiteSpace(result.Message)
                    ? "No Next.js hydration data (__NEXT_DATA__ or self.__next_f.push) found on this page."
                    : result.Message);
            }
 
            File.WriteAllText(filePath, sb.ToString());
        }

        private static string DescribeType(NextJsHydrationResult result)
        {
            if (result.IsPagesRouter) return "Pages Router (__NEXT_DATA__)";
            if (result.IsAppRouter) return "App Router (React Flight)";
            return "None detected (no Next.js SSR hydration data)";
        }

        private static string PrettyPrintOrRaw(string json)
        {
            try
            {
                using (var doc = JsonDocument.Parse(json))
                    return JsonSerializer.Serialize(doc.RootElement, new JsonSerializerOptions
                    {
                        WriteIndented = true,
                        Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping
                    });
            }
            catch
            {
                return json;
            }
        }
 
    }
}