From 52b757bbd35ea2e156dd2ec5ac39f44f385bde7e Mon Sep 17 00:00:00 2001 From: hectorqin Date: Thu, 29 Oct 2026 05:51:18 +0000 Subject: [PATCH] fix(web): clean web-page markup out of TXT chapters before typesetting Novels saved from a site keep the page's
,
and   in the TXT. Escaped, the tags were drawn as text and the chapter ran together as one paragraph. cleanTextMarkup turns block tags into line breaks, drops inline tags, decodes common entities and treats whitespace-only lines as blank, before paragraphs and the chapter heading are inferred. Only real tag names match; other markup is still escaped as text. Co-Authored-By: Claude Opus 5.5 --- docs/ui.md | 5 ++++ web/src/formats/segments.ts | 48 +++++++++++++++++++++++++++++++++--- web/test/reader-view.test.ts | 6 +++-- web/test/segments.test.ts | 46 ++++++++++++++++++++++++++++++++-- 4 files changed, 97 insertions(+), 8 deletions(-) diff --git a/docs/ui.md b/docs/ui.md index 40c9351..a5e4bd5 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -599,6 +599,11 @@ TXT 是唯一一种**没有作者排版**的书，所以它也是唯一一处「 传输看起来不一样。它同时负责**去掉段前空格**（`　　正文` 里那个全角空格是 「段前空格」的来源，留着就是和读者的缩进**叠加**），以及**章节标题行提升成 `

`**（`splitChapterHeading`，否则正文会变成 `第二章 落雨雨来了。`）。 + 分段之前先过 `cleanTextMarkup`：从网页另存的 TXT 常带着 `
`、`
`、 + ` `，原样转义就会把标签画成文字、每个 `
` 行都成一段空白。块级标签 + （`br`/`p`/`div`/标题/列表项）换成换行、行内标签（`span`/`font`/`b`…）去掉、 + 常见实体解码、只有空白的行算空行；只认真实的标签名，正文里的 `a` + 不动，其余标签照旧转义成文字。 `ReaderView` 在 `retypePlainText` 里对拿到的章节过这一段逻辑；如果内容带着 历史的 `.txt-body` 标记（旧服务端、旧缓存），就按块读回文本再拆，所以旧 响应不会被当成新响应渲染。 diff --git a/web/src/formats/segments.ts b/web/src/formats/segments.ts index 4226176..f1da71b 100644 --- a/web/src/formats/segments.ts +++ b/web/src/formats/segments.ts @@ -35,6 +35,10 @@ * scraped novels that need this rule are the ones that indent every paragraph * with two full-width spaces, so a rule that read the leading whitespace as * "this line continues the one above" never fired on any of them. + * 0. **Web-page markup is cleaned first.** Novels saved from a site arrive as + * TXT with the page's `
`, `
` and ` ` still in them; escaped, + * they are drawn literally and every `
` line is a gap. See + * {@link cleanTextMarkup}. * 4. **Every paragraph loses its leading whitespace.** What "段前空格" is: the * full-width spaces a scraper indented with, which the client draws itself * from the reader's own indent setting. Kept, they would be indent *added to* @@ -42,6 +46,43 @@ * needed it most. */ +/** + * HTML tags a scraped page leaves in a TXT. Only real tag names are matched, so a + * `<` in prose (`a`) is text and stays text. + */ +const BLOCK_TAG = /<\/?(?:br|p|div|h[1-6]|li|ul|ol|tr|table|tbody|section|article|blockquote|center|pre|hr|dd|dt|dl)\b[^<>]*>/gi; +const INLINE_TAG = /<\/?(?:span|font|b|i|u|em|strong|small|big|a|img|sup|sub|ins|del|s|strike|tt|code|mark|abbr|cite|q|wbr|o:p)\b[^<>]*>/gi; +const NAMED_ENTITIES: Record = { + nbsp: ' ', ensp: ' ', emsp: '\u3000', thinsp: ' ', amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", + ldquo: '“', rdquo: '”', lsquo: '‘', rsquo: '’', hellip: '…', mdash: '—', ndash: '–', middot: '·', +}; +const ENTITY = /&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]{2,8}));/gi; +/** A line holding only whitespace is a blank line: a reader cannot tell them apart. */ +const WHITESPACE_LINE = /^[ \t\u3000\u00a0]+$/gm; + +/** + * Removes the markup a web page leaves behind in a TXT, keeping its line structure. + * + * Block tags (`
`, `

`, `

`, headings, list items) become line breaks, so + * the paragraphs they marked survive as the blank lines the rules below read; + * inline tags (``, ``) are dropped; common character entities become + * the characters they stand for. Idempotent, and a file without markup is returned + * with only its whitespace-only lines emptied. The result is still *text*: every + * consumer escapes it, so nothing here can turn into markup. + */ +export function cleanTextMarkup(text: string): string { + return text + .replace(LINE_ENDINGS, '\n') + .replace(BLOCK_TAG, '\n') + .replace(INLINE_TAG, '') + .replace(ENTITY, (all, decimal: string | undefined, hex: string | undefined, name: string | undefined) => { + if (name) return NAMED_ENTITIES[name.toLowerCase()] ?? all; + const code = decimal ? Number(decimal) : Number.parseInt(hex!, 16); + return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : all; + }) + .replace(WHITESPACE_LINE, ''); +} + /** Paragraphs are separated by a blank line. */ const PARAGRAPH_BREAK = /\n{2,}/; @@ -110,8 +151,7 @@ export interface TextParagraph { * transport happened to deliver it. */ export function splitTextParagraphs(text: string): TextParagraph[] { - return text - .replace(LINE_ENDINGS, '\n') + return cleanTextMarkup(text) .split(PARAGRAPH_BREAK) .flatMap((block) => splitSentenceBreaks(block)) .map(toParagraph) @@ -271,7 +311,7 @@ function isHeadingLine(line: string): boolean { * the book rather than render it. */ export function splitChapterHeading(text: string): { heading: string; body: string } { - const normalised = text.replace(LINE_ENDINGS, '\n'); + const normalised = cleanTextMarkup(text); const lines = normalised.split('\n'); const firstIndex = lines.findIndex((line) => line.trim().length > 0); if (firstIndex === -1) return { heading: '', body: '' }; @@ -308,5 +348,5 @@ export function textToChapterHtml(text: string): string { * in the middle of a novel: invisible in any test and obvious on the page. */ export function endsAtParagraphBreak(text: string): boolean { - return /(?:\r\n?|\n)\s*$/.test(text); + return /\n\s*$/.test(cleanTextMarkup(text)); } diff --git a/web/test/reader-view.test.ts b/web/test/reader-view.test.ts index 415a4bb..dff9d5e 100644 --- a/web/test/reader-view.test.ts +++ b/web/test/reader-view.test.ts @@ -551,13 +551,15 @@ describe('plain-text chapters are typeset by the reader', () => { // assertion is on the *absence* of an element, not on the presence of the words. const container = document.createElement('div'); document.body.append(container); - const view = make(container, txtDoc('\n\n粗')); + // Known web-page tags are cleaned away (`cleanTextMarkup`); anything else stays text. + const view = make(container, txtDoc('\n\n粗\n\n')); await view.open(0, 0); const shadow = shadowOf(container); expect(shadow.querySelector('.txt-body img')).toBeNull(); expect(shadow.querySelector('.txt-body b')).toBeNull(); + expect(shadow.querySelector('.txt-body script')).toBeNull(); const paragraphs = [...shadow.querySelectorAll('.txt-body > p')].map((p) => p.textContent); - expect(paragraphs).toEqual(['', '粗']); + expect(paragraphs).toEqual(['粗', '']); }); it('re-typesets a chapter an older server sent as markup', async () => { diff --git a/web/test/segments.test.ts b/web/test/segments.test.ts index 380b72f..3dd474c 100644 --- a/web/test/segments.test.ts +++ b/web/test/segments.test.ts @@ -1,6 +1,7 @@ // @vitest-environment jsdom import { describe, expect, it } from 'vitest'; import { + cleanTextMarkup, endsAtParagraphBreak, escapeHtml, paragraphElement, @@ -126,10 +127,14 @@ describe('the markup form', () => { it('escapes a file that contains markup', () => { // A TXT is user data: a novel with a `<` in it must not be able to produce an // element, and `escapeHtml` is the one place that decides so. - const html = textToParagraphHtml('\n\n粗'); + // Known page tags (``, ``) are cleaned away as web-page residue; any + // other markup is escaped. Neither path can produce an element. + const html = textToParagraphHtml('\n\n粗\n\n'); expect(html).not.toContain(''); - expect(html).toContain('<img src=x onerror=alert(1)>'); + expect(html).not.toContain(' { expect(drawn).toEqual(splitTextParagraphs(whole).map((p) => p.text)); }); }); + +/** + * TXT files saved from a web page still carry its markup. Escaped, the tags are + * drawn as text and every `
` line becomes a gap, so they are cleaned before + * paragraphs are inferred. + */ +describe('web-page markup in a TXT', () => { + const scraped = '
\n 天斗星作为第一颗被人类发现并且移民成功的行政星，距今已经有数千年的历史了。
\n
\n
\n
\n 这颗星球大约有原本斗罗星二分之一大。
\n
'; + + it('turns the scraped block into clean paragraphs with no tags or gaps', () => { + expect(splitTextParagraphs(scraped).map((p) => p.text)).toEqual([ + '天斗星作为第一颗被人类发现并且移民成功的行政星，距今已经有数千年的历史了。', + '这颗星球大约有原本斗罗星二分之一大。', + ]); + }); + + it('renders no literal tags and no empty paragraphs', () => { + const html = textToChapterHtml(scraped); + expect(html).not.toMatch(/<|\s*<\/div>|

<\/p>/); + expect(html.match(/

/g)).toHaveLength(2); + }); + + it('keeps a heading that follows the markup', () => { + expect(splitChapterHeading('

\n第一章 天斗星
\n正文。
').heading).toBe('第一章 天斗星'); + }); + + it('drops inline tags, decodes entities and leaves prose angle brackets alone', () => { + expect(cleanTextMarkup('

他说“好” a<b！

')).toBe('\n他说“好” a 未知')).toBe('如果 a 未知'); + expect(cleanTextMarkup(cleanTextMarkup(scraped))).toBe(cleanTextMarkup(scraped)); + }); + + it('treats a trailing break tag as the end of a paragraph when streaming', () => { + expect(endsAtParagraphBreak('一段。
')).toBe(true); + expect(endsAtParagraphBreak('一段')).toBe(false); + }); +});