ragflow/internal/parser/parser/markdown_parser_test.go

package parser

import (
	"context"
	"encoding/base64"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"sync/atomic"
	"testing"

	"ragflow/internal/entity"
)

func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {
	ctx := t.Context()
	p, err := NewMarkdownParser(GoMarkdown)
	if err != nil {
		t.Fatalf("NewMarkdownParser: %v", err)
	}
	md := "# Hello\n\nThis is a paragraph.\n\n* List item 1\n* List item 2\n\n```go\nfunc main() {}\n```\n"
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	if got, want := res.Produced(), true; got != want {
		t.Fatalf("produced = %t, want %t", got, want)
	}
	if len(res.JSON) == 0 {
		t.Fatal("JSON is empty; want at least one item")
	}
	// Verify heading
	if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {
		t.Fatalf("first item text = %q, want %q", got, "# Hello")
	}
	if got, _ := res.JSON[0].ToMap()["layout"].(string); got != "heading" {
		t.Fatalf("first item layout = %q, want %q", got, "heading")
	}
}

// A UTF-8 BOM in front of the first heading must not demote it to a paragraph.
func TestMarkdownParser_ParseWithResult_UTF8BOM(t *testing.T) {
	p, err := NewMarkdownParser(GoMarkdown)
	if err != nil {
		t.Fatalf("NewMarkdownParser: %v", err)
	}
	res := p.ParseWithResult(t.Context(), "bom.md", []byte("\xef\xbb\xbf# Hello\n\nBody.\n"))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {
		t.Fatalf("first item text = %q, want %q", got, "# Hello")
	}
	if got, _ := res.JSON[0].ToMap()["layout"].(string); got != "heading" {
		t.Fatalf("first item layout = %q, want %q", got, "heading")
	}
}

// TestMarkdownParser_ListItemsKeepWordBoundaries pins the shape of a list in the parsed output:
// one item per entry, marker included, and no gluing between entries.
//
// A list marker is structure, not a leaf: the marker lives on the list node (bullets on the item)
// and the entry's own text carries no marker and no trailing newline. Emitting the list as a single
// blob therefore glued the last word of one entry to the first word of the next ("GamesChina"),
// which destroys the word boundaries the tokenizer and the retrieval index depend on - a query for
// "China" can no longer match the merged token. Python's _markdown keeps one item per entry with the
// marker, so this pins the same shape.
func TestMarkdownParser_ListItemsKeepWordBoundaries(t *testing.T) {
	ctx := t.Context()
	p, err := NewMarkdownParser(GoMarkdown)
	if err != nil {
		t.Fatalf("NewMarkdownParser: %v", err)
	}
	md := "See also\n\n* China at the Asian Games\n* China at the Paralympics\n"
	res := p.ParseWithResult(ctx, "list.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	var lists []string
	for _, item := range ItemsToMaps(res.JSON) {
		if ck, _ := item["layout"].(string); ck == "list" {
			text, _ := item["text"].(string)
			lists = append(lists, text)
		}
	}
	if len(lists) != 2 {
		t.Fatalf("want one item per list entry, got %d: %#v", len(lists), lists)
	}
	for i, want := range []string{"* China at the Asian Games", "* China at the Paralympics"} {
		if lists[i] != want {
			t.Errorf("list item %d = %q, want %q", i, lists[i], want)
		}
		if strings.Contains(lists[i], "GamesChina") {
			t.Errorf("list entries are glued together: %q", lists[i])
		}
	}
}

// TestMarkdownParser_OrderedListKeepsNumbers covers the ordered branch of the marker
// reconstruction: the number and its delimiter live on the list node, so they have to be rebuilt
// from Start/Delimiter instead of being read off a leaf.
//
// Two of the three cases start above 1 on purpose. A list that starts at 1 cannot tell "kept the
// numbers the source wrote" from "renumbered from 1", and this test used to run nothing but
// "1."/"2." - it passed while the parser renumbered every ordered list in a real corpus (a
// 2,244-entry list whose source numbers start at 996 was indexed as 1..N, which removes the source
// numbers from the index and made them count as unserved words).
func TestMarkdownParser_OrderedListKeepsNumbers(t *testing.T) {
	ctx := t.Context()
	p, err := NewMarkdownParser(GoMarkdown)
	if err != nil {
		t.Fatalf("NewMarkdownParser: %v", err)
	}
	cases := []struct {
		name string
		md   string
		want []string
	}{
		{
			name: "starts at ten",
			md:   "10. a\n11. b\n",
			want: []string{"10. a", "11. b"},
		},
		{
			name: "starts in the thousands",
			md:   "996. first entry\n\n997. second entry\n\n998. third entry\n",
			want: []string{"996. first entry", "997. second entry", "998. third entry"},
		},
		{
			name: "starts at one",
			md:   "1. first item\n2. second item\n",
			want: []string{"1. first item", "2. second item"},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			res := p.ParseWithResult(ctx, "ordered.md", []byte(tc.md))
			if res.Err != nil {
				t.Fatalf("ParseWithResult: %v", res.Err)
			}
			var lists []string
			for _, item := range ItemsToMaps(res.JSON) {
				if ck, _ := item["layout"].(string); ck == "list" {
					text, _ := item["text"].(string)
					lists = append(lists, text)
				}
			}
			if len(lists) != len(tc.want) {
				t.Fatalf("want %d items, got %d: %#v", len(tc.want), len(lists), lists)
			}
			for i := range tc.want {
				if lists[i] != tc.want[i] {
					t.Errorf("ordered item %d = %q, want %q", i, lists[i], tc.want[i])
				}
			}
		})
	}
}

// TestMarkdownParser_TableKeepsCellsBeyondTheHeader covers the deliberate deviation from GFM.
//
// A row may legitimately carry more cells than its header - an unescaped '|' inside a cell,
// e.g. a Wikipedia image spec "150x150px|alt=..." - and GFM ignores every cell beyond the
// header's column count, so the tail of such a row (the Year and Description columns of a
// World Heritage list) never reaches the index. Measured on the production index that cost
// 514 documents and 18,185 words no query could reach; the header is padded to the widest row
// instead. Fails on the old behaviour: the sentinel below is dropped.
func TestMarkdownParser_TableKeepsCellsBeyondTheHeader(t *testing.T) {
	ctx := t.Context()
	p, err := NewMarkdownParser(GoMarkdown)
	if err != nil {
		t.Fatalf("NewMarkdownParser: %v", err)
	}
	const sentinel = "DESCRIPTIONSENTINEL"
	md := "| Site | Image | Year |\n" +
		"|---|---|---|\n" +
		"| Aachen Cathedral | 150x150px|alt=A Gothic building | 1978 | " + sentinel + " died in 814 |\n"
	res := p.ParseWithResult(ctx, "table.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	var joined strings.Builder
	for _, item := range ItemsToMaps(res.JSON) {
		joined.WriteString(renderedText(item))
		joined.WriteString("\n")
	}
	out := joined.String()
	// The unescaped '|' inside the image cell becomes a cell boundary of its own, so the
	// assertions are on cell contents: everything the row said has to be somewhere in the
	// output, which is exactly what the column-count padding buys.
	for _, want := range []string{"Aachen Cathedral", "150x150px", "alt=A Gothic building", "1978", sentinel} {
		if !strings.Contains(out, want) {
			t.Errorf("table cell content %q is missing from the parsed output:\n%s", want, out)
		}
	}
}

func TestMarkdownParser_ParseWithResult_EmptyInput(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	res := p.ParseWithResult(ctx, "empty.md", []byte(""))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	if len(res.JSON) != 1 {
		t.Fatalf("len(JSON) = %d, want 1", len(res.JSON))
	}
}

func TestMarkdownParser_ParseWithResult_ImageDataURI(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	// 1×1 pixel transparent PNG encoded as data URI
	pixelPNG := make([]byte, 68) // minimal 1x1 PNG header
	pixelB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-data"))
	md := "Some text with an image\n![test](data:image/png;base64," + pixelB64 + ")\n"
	_ = pixelPNG
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	// Find the image item
	found := false
	for _, item := range ItemsToMaps(res.JSON) {
		if kd, _ := item["doc_type_kwd"].(string); kd == "image" {
			found = true
			if img, ok := item["image"].(string); !ok || img != pixelB64 {
				t.Fatalf("image data = %q, want %q", img, pixelB64)
			}
			break
		}
	}
	if !found {
		t.Fatal("no item with doc_type_kwd == 'image' found")
	}
}

func TestMarkdownParser_ParseWithResult_NoImage(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	md := "# Title\n\nJust some text, no images here.\n\nMore text."
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	for _, item := range ItemsToMaps(res.JSON) {
		if kd, _ := item["doc_type_kwd"].(string); kd == "image" {
			t.Fatal("unexpected image item in text-only markdown")
		}
	}
}

func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	md := "[M03] Health check package comparison:\n\n| Check item | Basic 699 CNY | Advanced 1299 CNY |\n| --- | --- | --- |\n| Blood routine / Urine routine | Yes | Yes |\n\nNote: All packages require fasting.\n"
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	// The document has three top-level blocks: the leading paragraph, the
	// table, and the trailing note. The table is emitted as exactly ONE
	// structured doc_type_kwd:"table" item (not flattened, not duplicated as
	// a doc_type_kwd:"text" copy).
	if len(res.JSON) != 3 {
		t.Fatalf("len(JSON) = %d, want 3 (leading paragraph, table, note)", len(res.JSON))
	}
	tableCount, inlineTableCount := 0, 0
	for _, item := range ItemsToMaps(res.JSON) {
		kd, _ := item["doc_type_kwd"].(string)
		text, _ := item["text"].(string)
		switch kd {
		case "table":
			tableCount++
			td, ok := item["table"].(*entity.TableData)
			if !ok || td == nil {
				t.Fatalf("table item missing *entity.TableData: %#v", item)
			}
			if !tableRowsContain(td, "Check item") {
				t.Errorf("table item rows missing header label: %+v", td.CellTexts())
			}
		case "text":
			if strings.Contains(text, "<table") {
				inlineTableCount++
			}
		default:
			t.Fatalf("unexpected doc_type_kwd %q; want text or table", kd)
		}
		if strings.Contains(text, "| Check item |") {
			t.Fatalf("raw markdown table leaked into text: %q", text)
		}
	}
	if tableCount != 1 {
		t.Fatalf("table items = %d, want 1", tableCount)
	}
	if inlineTableCount != 0 {
		t.Fatalf("found %d doc_type_kwd:\"text\" item(s) with <table> markup; table must not be duplicated as text", inlineTableCount)
	}
}

// TestMarkdownParser_ParseWithResult_DollarIsCurrencyNotMath guards the
// regression that made a 13.8k-character billing report parse into 1.4k
// characters: CommonExtensions enables MathJax, so `$...$` spans became
// ast.Math nodes whose payload is not a Text child, and the text walker
// dropped them. With two far-apart `$` that deletes everything in between.
// A `$` in an ingested document is far more often a price than it is math, so
// the text must survive verbatim, dollar signs included.
func TestMarkdownParser_ParseWithResult_DollarIsCurrencyNotMath(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	md := "# Report\n\nFunding was provided by a $4,800 grant from the Sanctuary.\n\n" +
		"Costs: 224.50 theodolite, 48.70 binoculars, $3.75 internet.\n\n" +
		"Remaining funds for Winter 2002: $193.92\n"
	res := p.ParseWithResult(ctx, "report.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	var sb strings.Builder
	for _, item := range ItemsToMaps(res.JSON) {
		text, _ := item["text"].(string)
		sb.WriteString(text)
		sb.WriteString("\n")
	}
	got := sb.String()
	// Every sentence of the source has to survive; the price table sentence is
	// the one sitting between two '$' and used to be deleted whole.
	for _, want := range []string{
		"Funding was provided by a $4,800 grant from the Sanctuary.",
		"Costs: 224.50 theodolite, 48.70 binoculars, $3.75 internet.",
		"Remaining funds for Winter 2002: $193.92",
	} {
		if !strings.Contains(got, want) {
			t.Errorf("parsed text lost %q\n--- got ---\n%s", want, got)
		}
	}
	// `$$...$$` display math must stay literal text as well.
	mdDisplay := "Totals are shown as $$4,606.17 in the audit.\n"
	resDisplay := p.ParseWithResult(ctx, "totals.md", []byte(mdDisplay))
	if resDisplay.Err != nil {
		t.Fatalf("ParseWithResult: %v", resDisplay.Err)
	}
	var sbDisplay strings.Builder
	for _, item := range ItemsToMaps(resDisplay.JSON) {
		text, _ := item["text"].(string)
		sbDisplay.WriteString(text)
	}
	if !strings.Contains(sbDisplay.String(), "4,606.17 in the audit") {
		t.Errorf("display math span was dropped: %q", sbDisplay.String())
	}
}

// TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives guards the second
// text-loss mechanism found in the same walker: inline raw HTML is an
// ast.HTMLSpan whose payload (markup and the text inside it) lives on the node
// literal, with no child Text nodes. A 664k-character report lost 397k
// characters across 14 such spans.
func TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	// No blank line around the <table>, so gomarkdown keeps it inline inside
	// the paragraph (an HTMLSpan) instead of a block-level HTMLBlock.
	md := "Lead-in sentence before the table. " +
		"<table><tr><td>theodolite</td><td>224.50</td></tr>" +
		"<tr><td>binoculars</td><td>48.70</td></tr></table> " +
		"Trailing sentence after the table.\n"
	res := p.ParseWithResult(ctx, "report.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	var sb strings.Builder
	for _, item := range ItemsToMaps(res.JSON) {
		text, _ := item["text"].(string)
		sb.WriteString(text)
		sb.WriteString("\n")
	}
	got := sb.String()
	for _, want := range []string{
		"Lead-in sentence before the table.",
		"Trailing sentence after the table.",
	} {
		if !strings.Contains(got, want) {
			t.Errorf("parsed text lost %q\n--- got ---\n%s", want, got)
		}
	}
	// The inline HTML payload must not vanish either: the walker emits the raw
	// markup, exactly as it stands in the source.
	if !strings.Contains(got, "binoculars") {
		t.Errorf("inline raw HTML payload was dropped: %q", got)
	}
}

func TestMarkdownParser_ConfigureFromSetup(t *testing.T) {
	p, _ := NewMarkdownParser(GoMarkdown)
	p.ConfigureFromSetup(map[string]any{
		"parse_method":          "deepdoc",
		"vlm":                   map[string]any{"llm_id": "gpt-4-vision"},
		"flatten_media_to_text": false,
	})
	if p.ParseMethod != "deepdoc" {
		t.Fatalf("ParseMethod = %q, want %q", p.ParseMethod, "deepdoc")
	}
	if p.VLM == nil {
		t.Fatal("VLM is nil; want map")
	}
	if id, _ := p.VLM["llm_id"].(string); id != "gpt-4-vision" {
		t.Fatalf("VLM[llm_id] = %q, want %q", id, "gpt-4-vision")
	}
}

func TestMarkdownParser_ConfigureFromSetup_NilSafe(t *testing.T) {
	p, _ := NewMarkdownParser(GoMarkdown)
	p.ConfigureFromSetup(nil) // should not panic
	if p.ParseMethod != "" {
		t.Fatalf("ParseMethod should be empty after nil setup, got %q", p.ParseMethod)
	}
}

// TestMarkdownParser_FlattenMediaToText verifies that when
// flatten_media_to_text is true, image items are emitted with
// doc_type_kwd="text" (mirroring Python parser.py:1034). When false,
// image items keep doc_type_kwd="image".
func TestMarkdownParser_FlattenMediaToText(t *testing.T) {
	ctx := t.Context()
	pixelB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-data"))
	md := "Some text with an image\n![test](data:image/png;base64," + pixelB64 + ")\n"

	cases := []struct {
		name        string
		flatten     bool
		wantDocType string
	}{
		{"flatten=false keeps image", false, "image"},
		{"flatten=true forces text", true, "text"},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			p, _ := NewMarkdownParser(GoMarkdown)
			p.ConfigureFromSetup(map[string]any{"flatten_media_to_text": tc.flatten})
			if p.FlattenMediaToText != tc.flatten {
				t.Fatalf("FlattenMediaToText field = %v, want %v", p.FlattenMediaToText, tc.flatten)
			}
			res := p.ParseWithResult(ctx, "test.md", []byte(md))
			if res.Err != nil {
				t.Fatalf("ParseWithResult: %v", res.Err)
			}
			for _, item := range ItemsToMaps(res.JSON) {
				if kd, _ := item["doc_type_kwd"].(string); kd != tc.wantDocType {
					t.Errorf("doc_type_kwd = %q, want %q (item text=%q)",
						kd, tc.wantDocType, item["text"])
				}
			}
		})
	}
}

func TestResolveImageURL_DataURI(t *testing.T) {
	b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))
	result, raw, size, found := resolveImageURL(context.Background(), "data:image/png;base64,"+b64)
	if !found {
		t.Fatal("expected image found for data URI")
	}
	if result != b64 || raw != nil {
		t.Fatalf("got %q, want %q", result, b64)
	}
	if size != len("fakeimage") {
		t.Errorf("size = %d, want %d", size, len("fakeimage"))
	}
}

func TestResolveImageURL_LocalPathNotFetched(t *testing.T) {
	// Local / relative paths are not fetched (security); resolution fails.
	if _, _, _, found := resolveImageURL(context.Background(), "./local/image.png"); found {
		t.Fatal("expected no image resolved for a local path")
	}
}

func TestResolveImageURL_HTTPImage(t *testing.T) {
	// httptest servers bind loopback, which the SSRF guard rejects by
	// default. Allow loopback for this test so the HTTP fetch path is
	// exercised (production keeps ssrfAllowLoopback == false).
	prev := ssrfAllowLoopback
	ssrfAllowLoopback = true
	defer func() { ssrfAllowLoopback = prev }()

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.Header().Set("Content-Type", "image/png")
		w.Write([]byte("fake-png-bytes"))
	}))
	defer ts.Close()

	encoded, result, size, found := resolveImageURL(context.Background(), ts.URL+"/image.png")
	if !found {
		t.Fatal("expected image found for HTTP URL")
	}
	if encoded != "" || string(result) != "fake-png-bytes" {
		t.Fatalf("got encoded=%q raw=%q, want raw image bytes", encoded, result)
	}
	if size != len("fake-png-bytes") {
		t.Errorf("size = %d, want %d", size, len("fake-png-bytes"))
	}
}

func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {
	previous := ssrfAllowLoopback
	ssrfAllowLoopback = true
	t.Cleanup(func() { ssrfAllowLoopback = previous })

	var fetches atomic.Int32
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		fetches.Add(1)
		w.Header().Set("Content-Type", "image/png")
		_, _ = w.Write([]byte("abc"))
	}))
	defer server.Close()

	markdown := "![first](" + server.URL + "/1)\n\n" +
		"![second](" + server.URL + "/2)\n\n" +
		"![third](" + server.URL + "/3)\n"
	doc := markdownNew().Parse([]byte(markdown))
	budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}
	var items []map[string]any
	if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, budget); unresolved != 0 {
		t.Fatalf("unresolved images = %d, want 0", unresolved)
	}
	if got := fetches.Load(); got != 2 {
		t.Fatalf("remote fetches = %d, want 2 before the item limit", got)
	}
	if len(items) != 3 || items[0]["image"] != base64.StdEncoding.EncodeToString([]byte("abc")) {
		t.Fatalf("first image was not retained: %+v", items)
	}
	for _, index := range []int{1, 2} {
		if items[index]["image"] != nil || items[index]["media_omitted"] != true {
			t.Fatalf("image %d = %+v, want omitted payload", index, items[index])
		}
	}
	warnings := strings.Join(budget.warnings(), "\n")
	if !strings.Contains(warnings, "omitted 1 embedded image payload(s) after reaching the 4-byte document image budget") ||
		!strings.Contains(warnings, "stopped extracting embedded images after the 2-item document limit") {
		t.Fatalf("warnings = %q, want document byte and item limit warnings", warnings)
	}
}

// TestFindBlockImage resolves the image per-block from the AST so only the
// owning block is tagged (the fix for the whole-document scan bug).
func TestFindBlockImage(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)

	withImg := "# T\n\nText without image.\n\n![alt](data:image/png;base64,AAA)\n"
	res := p.ParseWithResult(ctx, "a.md", []byte(withImg))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	var imgItems, textItems int
	for _, item := range ItemsToMaps(res.JSON) {
		switch kd, _ := item["doc_type_kwd"].(string); kd {
		case "image":
			imgItems++
			if _, ok := item["image"].(string); !ok {
				t.Fatal("image item missing base64 payload")
			}
		case "text":
			textItems++
		}
	}
	if imgItems != 1 {
		t.Fatalf("imgItems = %d, want 1 (only the block owning the image)", imgItems)
	}
	if textItems < 2 {
		t.Fatalf("textItems = %d, want >= 2 (other blocks must stay text, not image)", textItems)
	}

	// A document with no image must not produce any image item.
	noImg := p.ParseWithResult(ctx, "b.md", []byte("# T\n\nNo images here.\n"))
	for _, item := range ItemsToMaps(noImg.JSON) {
		if kd, _ := item["doc_type_kwd"].(string); kd == "image" {
			t.Fatal("unexpected image item in text-only markdown")
		}
	}
}

func TestFetchImage_RejectsCredentials(t *testing.T) {
	_, err := fetchImage(context.Background(), "https://user:pass@example.com/img.png")
	if err == nil {
		t.Fatal("expected error for URL with credentials")
	}
}

func TestFetchImage_InvalidURL(t *testing.T) {
	withSSRFBypass(t)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusNotFound)
	}))
	defer ts.Close()

	_, err := fetchImage(context.Background(), ts.URL+"/nonexistent.png")
	if err == nil {
		t.Fatal("expected error for 404 response")
	}
}

// TestMarkdownParser_TableNotCollapsed is the core regression guard for the
// Markdown table fix: a document containing a GFM table must keep one item per
// top-level block instead of collapsing into one giant item, AND the table must
// be emitted as a SINGLE raw <table>…</table> HTML block (not scattered cell
// text) carrying doc_type_kwd:"table" in its original document position. There
// must be NO redundant inlined doc_type_kwd:"text" copy of the table. A heading,
// the table, and the trailing paragraph must each be present and in order.
func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	md := "# Title\n\nIntro paragraph.\n\n| A | B |\n| --- | --- |\n| x | y |\n\nTrailing note.\n"
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	// # Title, Intro, table item, Trailing = 4.
	if len(res.JSON) != 4 {
		t.Fatalf("len(JSON) = %d, want 4 (one per top-level block, table as one item)", len(res.JSON))
	}
	var all strings.Builder
	sawTitle, sawTrailing, sawTableItem, sawInlineTable := false, false, false, false
	tableCount := 0
	for _, item := range ItemsToMaps(res.JSON) {
		text, _ := item["text"].(string)
		all.WriteString(text)
		all.WriteString("\n")
		if text == "# Title" {
			sawTitle = true
		}
		switch kd, _ := item["doc_type_kwd"].(string); kd {
		case "text":
			// A doc_type_kwd:"text" copy must NOT carry the raw <table> HTML.
			if strings.Contains(text, "<table") && strings.Contains(text, "A") && strings.Contains(text, "B") {
				sawInlineTable = true
			}
		case "table":
			sawTableItem = true
			tableCount++
			if _, ok := item["table"].(*entity.TableData); !ok {
				t.Fatalf("table item missing *entity.TableData: %#v", item)
			}
		}
	}
	concat := all.String()
	if strings.Contains(concat, "Trailing note.") {
		sawTrailing = true
	}
	if !sawTitle {
		t.Fatal("heading item not emitted")
	}
	if sawInlineTable {
		t.Fatal("table wrongly emitted as redundant inlined doc_type_kwd:\"text\" copy")
	}
	if !sawTableItem {
		t.Fatal("separate doc_type_kwd:\"table\" item not emitted")
	}
	if tableCount != 1 {
		t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
	}
	if !sawTrailing {
		t.Fatal("trailing paragraph content not present in concatenated items")
	}
}

// TestMarkdownParser_TableWithSurroundingText is the Markdown counterpart of
// TestHTMLParser_NestedTableWithSurroundingText: a GFM table with prose both
// before and after it must emit the table as ONE structured
// doc_type_kwd:"table" item, with the surrounding paragraphs split into
// SEPARATE clean text items bracketing the table in document order — NOT
// merged into a single blob, and NOT duplicated as an inlined
// doc_type_kwd:"text" copy. This locks in the single-item-in-document-order
// contract for the common "intro / table / outro" shape.
func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	md := "Before the table.\n\n| Name | Age |\n| --- | --- |\n| Alice | 30 |\n\nAfter the table.\n"
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}

	var tableText string
	tableIdx, beforeIdx, afterIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0
	for i, item := range ItemsToMaps(res.JSON) {
		text, _ := item["text"].(string)
		switch kd, _ := item["doc_type_kwd"].(string); kd {
		case "text":
			if strings.Contains(text, "<table") {
				inlineTableCount++
			}
			if text == "Before the table." {
				beforeIdx = i
			}
			if text == "After the table." {
				afterIdx = i
			}
		case "table":
			tableText = renderedText(item)
			tableIdx = i
			tableCount++
			if _, ok := item["table"].(*entity.TableData); !ok {
				t.Fatalf("table item missing *entity.TableData: %#v", item)
			}
		}
	}

	if tableIdx < 0 {
		t.Fatalf("no structured doc_type_kwd:\"table\" item emitted; got items: %#v", res.JSON)
	}
	if tableCount != 1 {
		t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
	}
	if !strings.Contains(tableText, "Name") || !strings.Contains(tableText, "Alice") {
		t.Errorf("structured table item missing cell text: %q", tableText)
	}
	// No duplicate inline text copy of the table markup.
	if inlineTableCount != 0 {
		t.Errorf("found %d doc_type_kwd:\"text\" item(s) containing <table> markup; table must not be duplicated as inline text", inlineTableCount)
	}
	// The surrounding prose is split into clean text items bracketing the
	// table in document order — NOT collapsed into one blob.
	if beforeIdx < 0 {
		t.Fatalf("'Before the table.' text item missing")
	}
	if afterIdx < 0 {
		t.Fatalf("'After the table.' text item missing")
	}
	if !(beforeIdx < tableIdx && tableIdx < afterIdx) {
		t.Errorf("document order wrong: before=%d table=%d after=%d (prose must bracket table, not merge into one blob)", beforeIdx, tableIdx, afterIdx)
	}
}

// TestMarkdownParser_NonTableHTMLBlockNotTable guards against the blanket
// raw-table bug: a raw HTML block that is NOT a <table> (e.g. <div>,
// <style>) must be emitted as ordinary text with no layout, so downstream
// consumers (chunker) do not mistake it for a table.
func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	md := "Before.\n\n<div class=\"note\">just a div block</div>\n\n<style>.a{color:red}</style>\n\nAfter.\n"
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	for _, item := range ItemsToMaps(res.JSON) {
		text, _ := item["text"].(string)
		if strings.Contains(text, "<div") || strings.Contains(text, "<style") {
			if kd, _ := item["doc_type_kwd"].(string); kd == "table" {
				t.Fatalf("non-table HTML block wrongly tagged doc_type_kwd:\"table\": %q", text)
			}
		}
	}
}

// TestMarkdownParser_TableInCodeFenceNotRendered guards against a regression
// where pipe rows INSIDE a fenced code block would be mis-identified as a GFM
// table and rewritten into <table> HTML, corrupting the code. renderMarkdownTablesInline
// tracks fence state (inFence) so the table detector must skip lines inside a fence.
func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	md := "# Title\n\n```\n| A | B |\n| --- | --- |\n| x | y |\n```\n\nAfter fence.\n"
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	// No table item may be emitted — the pipe rows live inside a code block.
	for _, item := range ItemsToMaps(res.JSON) {
		if kd, _ := item["doc_type_kwd"].(string); kd == "table" {
			t.Fatalf("code-fence pipe rows wrongly emitted as table item: %q", item["text"])
		}
	}
	// The code block item must retain the raw pipe text and must NOT contain
	// any <table> markup.
	sawCode := false
	for _, item := range ItemsToMaps(res.JSON) {
		text, _ := item["text"].(string)
		if strings.Contains(text, "| A | B |") {
			sawCode = true
			if strings.Contains(text, "<table") {
				t.Fatalf("code block text was rewritten into table HTML: %q", text)
			}
		}
	}
	if !sawCode {
		t.Fatal("code block with pipe rows not found in output")
	}
}

// TestMarkdownParser_RawHTMLTableHandled covers a user-written raw <table> HTML
// block (not a GFM pipe table). renderMarkdownTablesInline only rewrites GFM
// pipe tables, so the raw <table> passes through and is caught by isTableHTML
// as an HTMLBlock, producing a single doc_type_kwd:"table" item in document
// order. There must be NO redundant inlined doc_type_kwd:"text" copy.
func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	md := "Before.\n\n<table><tr><td>X</td><td>Y</td></tr></table>\n\nAfter.\n"
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}
	sawInlined, sawTableItem := false, false
	tableCount := 0
	for _, item := range ItemsToMaps(res.JSON) {
		text, _ := item["text"].(string)
		switch kd, _ := item["doc_type_kwd"].(string); kd {
		case "text":
			if strings.Contains(text, "<table") && strings.Contains(text, "X") && strings.Contains(text, "Y") {
				sawInlined = true
			}
		case "table":
			sawTableItem = true
			tableCount++
			if _, ok := item["table"].(*entity.TableData); !ok {
				t.Fatalf("raw table item missing *entity.TableData: %#v", item)
			}
		}
	}
	if sawInlined {
		t.Fatal("raw <table> HTML block wrongly emitted as redundant inlined copy in text flow")
	}
	if !sawTableItem {
		t.Fatal("raw <table> HTML block not emitted as single doc_type_kwd:\"table\" item")
	}
	if tableCount != 1 {
		t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
	}
}

// TestMarkdownParser_MultipleTablesOrdering guards the ordering contract: each
// GFM table is emitted as a single doc_type_kwd:"table" item in its original
// document position (among the surrounding text blocks). Both tables' cell text
// must be present and in source order, bracketing "Middle." — NOT appended at
// the end of the stream.
func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)
	md := "# Title\n\n| A | B |\n| --- | --- |\n| x | y |\n\nMiddle.\n\n| C | D |\n| --- | --- |\n| p | q |\n\nEnd.\n"
	res := p.ParseWithResult(ctx, "test.md", []byte(md))
	if res.Err != nil {
		t.Fatalf("ParseWithResult: %v", res.Err)
	}

	var tableItemIdx []int
	var titleIdx, middleIdx, endIdx = -1, -1, -1
	for i, item := range ItemsToMaps(res.JSON) {
		text, _ := item["text"].(string)
		switch kd, _ := item["doc_type_kwd"].(string); kd {
		case "text":
			switch text {
			case "# Title":
				titleIdx = i
			case "Middle.":
				middleIdx = i
			case "End.":
				endIdx = i
			}
		case "table":
			tableItemIdx = append(tableItemIdx, i)
		}
	}

	if titleIdx < 0 || middleIdx < 0 || endIdx < 0 {
		t.Fatalf("missing anchor text item (title=%d middle=%d end=%d)", titleIdx, middleIdx, endIdx)
	}
	if len(tableItemIdx) != 2 {
		t.Fatalf("table items = %d, want 2", len(tableItemIdx))
	}
	// Table items appear in document order, bracketing "Middle.":
	// table1 before Middle, table2 between Middle and End.
	if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {
		t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)
	}
	t1 := renderedText(res.JSON[tableItemIdx[0]].ToMap())
	t2 := renderedText(res.JSON[tableItemIdx[1]].ToMap())
	if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {
		t.Fatalf("first table item missing x/y cells: %q", t1)
	}
	if !strings.Contains(t2, "p") || !strings.Contains(t2, "q") {
		t.Fatalf("second table item missing p/q cells: %q", t2)
	}
}

// TestMarkdownParser_AlignmentGolden verifies Go's ParseWithResult output is
// content-equivalent to Python's _markdown on the shared sample, using the
// shared concatenation-normalization alignment tool (align_test.go). Python
// keeps raw Markdown and splits on the delimiter set; Go emits clean per-block
// text. The comparison normalizes both (Markdown syntax, html tags, delimiters
// stripped; whitespace collapsed) and ignores the doc types the golden declares
// as accepted divergences (meta.accepted_divergences; PARSER_ALIGNMENT_HANDOFF.md §3.1).
//
// Both an English (markdown.sample.en.md) and a Chinese (markdown.sample.zh.md)
// sample are checked so markdown parsing is exercised in both Latin and CJK
// contexts — the Chinese sample also covers the full-width delimiters in the
// default delimiter set (\n!?;。；！？). Each baseline is a {meta, items}
// document whose "meta" block records how it was produced (generator
// rag/flow/parser/parser.py:_markdown, sample, delimiter, accepted
// divergences). No generator script is committed — to regenerate, call
// _markdown on the sample and dump {meta, items}. The baseline is
// reproducible from the metadata alone (an AI or human can recreate the thin
// wrapper on demand).
func TestMarkdownParser_AlignmentGolden(t *testing.T) {
	ctx := t.Context()
	p, _ := NewMarkdownParser(GoMarkdown)

	cases := []struct {
		name   string
		sample string
		golden string
	}{
		{"en", "testdata/markdown.sample.en.md", "testdata/markdown.python.en.golden.json"},
		{"zh", "testdata/markdown.sample.zh.md", "testdata/markdown.python.zh.golden.json"},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			sample, err := os.ReadFile(tc.sample)
			if err != nil {
				t.Fatalf("read sample: %v", err)
			}
			res := p.ParseWithResult(ctx, tc.sample, sample)
			if res.Err != nil {
				t.Fatalf("ParseWithResult: %v", res.Err)
			}

			gd := LoadGoldenDoc(t, tc.golden)

			// Exclude the doc types the golden declares as accepted divergences
			// (meta.accepted_divergences) on both sides — no hardcoded list in the test.
			// filterTableDivergence additionally drops the inlined <table> markup that
			// Python keeps as a "text" item, so only non-table prose is compared.
			ignore := AcceptedDivergences(gd.Meta)
			goText := filterTableDivergence(ItemsToMaps(res.JSON), ignore)
			pyText := filterTableDivergence(gd.Items, ignore)

			// filterTableDivergence drops the table from the prose comparison;
			// the table-equivalence guard below (assertTablesEquivalent) checks
			// the table cell content still matches Python, so a collapse or
			// dropped column on either side is caught independently.
			assertTablesEquivalent(t, ItemsToMaps(res.JSON), gd.Items)

			if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {
				t.Fatalf("markdown parser not aligned with Python golden:%s", diff)
			}
		})
	}
}

// tableRowsContain reports whether any cell of the structured table contains
// the substring want, so table-contract tests can assert cell content without
// depending on the legacy HTML markup shape.
func tableRowsContain(td *entity.TableData, want string) bool {
	for _, row := range td.CellTexts() {
		for _, cell := range row {
			if strings.Contains(cell, want) {
				return true
			}
		}
	}
	return false
}