diff --git a/docs/ui.md b/docs/ui.md index 40c9351..a5e4bd5 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -599,6 +599,11 @@ TXT 是唯一一种**没有作者排版**的书，所以它也是唯一一处「 传输看起来不一样。它同时负责**去掉段前空格**（`　　正文` 里那个全角空格是 「段前空格」的来源，留着就是和读者的缩进**叠加**），以及**章节标题行提升成 `

`**（`splitChapterHeading`，否则正文会变成 `第二章 落雨雨来了。`）。 + 分段之前先过 `cleanTextMarkup`：从网页另存的 TXT 常带着 `
`、`
`、 + ` `，原样转义就会把标签画成文字、每个 `
` 行都成一段空白。块级标签 + （`br`/`p`/`div`/标题/列表项）换成换行、行内标签（`span`/`font`/`b`…）去掉、 + 常见实体解码、只有空白的行算空行；只认真实的标签名，正文里的 `a` + 不动，其余标签照旧转义成文字。 `ReaderView` 在 `retypePlainText` 里对拿到的章节过这一段逻辑；如果内容带着 历史的 `.txt-body` 标记（旧服务端、旧缓存），就按块读回文本再拆，所以旧 响应不会被当成新响应渲染。 diff --git a/web/src/formats/segments.ts b/web/src/formats/segments.ts index 4226176..f1da71b 100644 --- a/web/src/formats/segments.ts +++ b/web/src/formats/segments.ts @@ -35,6 +35,10 @@ * scraped novels that need this rule are the ones that indent every paragraph * with two full-width spaces, so a rule that read the leading whitespace as * "this line continues the one above" never fired on any of them. + * 0. **Web-page markup is cleaned first.** Novels saved from a site arrive as + * TXT with the page's `
`, `
` and ` ` still in them; escaped, + * they are drawn literally and every `
` line is a gap. See + * {@link cleanTextMarkup}. * 4. **Every paragraph loses its leading whitespace.** What "段前空格" is: the * full-width spaces a scraper indented with, which the client draws itself * from the reader's own indent setting. Kept, they would be indent *added to* @@ -42,6 +46,43 @@ * needed it most. */ +/** + * HTML tags a scraped page leaves in a TXT. Only real tag names are matched, so a + * `<` in prose (`a`) is text and stays text. + */ +const BLOCK_TAG = /<\/?(?:br|p|div|h[1-6]|li|ul|ol|tr|table|tbody|section|article|blockquote|center|pre|hr|dd|dt|dl)\b[^<>]*>/gi; +const INLINE_TAG = /<\/?(?:span|font|b|i|u|em|strong|small|big|a|img|sup|sub|ins|del|s|strike|tt|code|mark|abbr|cite|q|wbr|o:p)\b[^<>]*>/gi; +const NAMED_ENTITIES: Record = { + nbsp: ' ', ensp: ' ', emsp: '\u3000', thinsp: ' ', amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", + ldquo: '“', rdquo: '”', lsquo: '‘', rsquo: '’', hellip: '…', mdash: '—', ndash: '–', middot: '·', +}; +const ENTITY = /&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|([a-z]{2,8}));/gi; +/** A line holding only whitespace is a blank line: a reader cannot tell them apart. */ +const WHITESPACE_LINE = /^[ \t\u3000\u00a0]+$/gm; + +/** + * Removes the markup a web page leaves behind in a TXT, keeping its line structure. + * + * Block tags (`
`, `

`, `

`, headings, list items) become line breaks, so + * the paragraphs they marked survive as the blank lines the rules below read; + * inline tags (``, ``) are dropped; common character entities become + * the characters they stand for. Idempotent, and a file without markup is returned + * with only its whitespace-only lines emptied. The result is still *text*: every + * consumer escapes it, so nothing here can turn into markup. + */ +export function cleanTextMarkup(text: string): string { + return text + .replace(LINE_ENDINGS, '\n') + .replace(BLOCK_TAG, '\n') + .replace(INLINE_TAG, '') + .replace(ENTITY, (all, decimal: string | undefined, hex: string | undefined, name: string | undefined) => { + if (name) return NAMED_ENTITIES[name.toLowerCase()] ?? all; + const code = decimal ? Number(decimal) : Number.parseInt(hex!, 16); + return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : all; + }) + .replace(WHITESPACE_LINE, ''); +} + /** Paragraphs are separated by a blank line. */ const PARAGRAPH_BREAK = /\n{2,}/; @@ -110,8 +151,7 @@ export interface TextParagraph { * transport happened to deliver it. */ export function splitTextParagraphs(text: string): TextParagraph[] { - return text - .replace(LINE_ENDINGS, '\n') + return cleanTextMarkup(text) .split(PARAGRAPH_BREAK) .flatMap((block) => splitSentenceBreaks(block)) .map(toParagraph) @@ -271,7 +311,7 @@ function isHeadingLine(line: string): boolean { * the book rather than render it. */ export function splitChapterHeading(text: string): { heading: string; body: string } { - const normalised = text.replace(LINE_ENDINGS, '\n'); + const normalised = cleanTextMarkup(text); const lines = normalised.split('\n'); const firstIndex = lines.findIndex((line) => line.trim().length > 0); if (firstIndex === -1) return { heading: '', body: '' }; @@ -308,5 +348,5 @@ export function textToChapterHtml(text: string): string { * in the middle of a novel: invisible in any test and obvious on the page. */ export function endsAtParagraphBreak(text: string): boolean { - return /(?:\r\n?|\n)\s*$/.test(text); + return /\n\s*$/.test(cleanTextMarkup(text)); } diff --git a/web/test/reader-view.test.ts b/web/test/reader-view.test.ts index 415a4bb..dff9d5e 100644 --- a/web/test/reader-view.test.ts +++ b/web/test/reader-view.test.ts @@ -551,13 +551,15 @@ describe('plain-text chapters are typeset by the reader', () => { // assertion is on the *absence* of an element, not on the presence of the words. const container = document.createElement('div'); document.body.append(container); - const view = make(container, txtDoc('\n\n粗')); + // Known web-page tags are cleaned away (`cleanTextMarkup`); anything else stays text. + const view = make(container, txtDoc('\n\n粗\n\n')); await view.open(0, 0); const shadow = shadowOf(container); expect(shadow.querySelector('.txt-body img')).toBeNull(); expect(shadow.querySelector('.txt-body b')).toBeNull(); + expect(shadow.querySelector('.txt-body script')).toBeNull(); const paragraphs = [...shadow.querySelectorAll('.txt-body > p')].map((p) => p.textContent); - expect(paragraphs).toEqual(['', '粗']); + expect(paragraphs).toEqual(['粗', '']); }); it('re-typesets a chapter an older server sent as markup', async () => { diff --git a/web/test/segments.test.ts b/web/test/segments.test.ts index 380b72f..3dd474c 100644 --- a/web/test/segments.test.ts +++ b/web/test/segments.test.ts @@ -1,6 +1,7 @@ // @vitest-environment jsdom import { describe, expect, it } from 'vitest'; import { + cleanTextMarkup, endsAtParagraphBreak, escapeHtml, paragraphElement, @@ -126,10 +127,14 @@ describe('the markup form', () => { it('escapes a file that contains markup', () => { // A TXT is user data: a novel with a `<` in it must not be able to produce an // element, and `escapeHtml` is the one place that decides so. - const html = textToParagraphHtml('\n\n粗'); + // Known page tags (``, ``) are cleaned away as web-page residue; any + // other markup is escaped. Neither path can produce an element. + const html = textToParagraphHtml('\n\n粗\n\n'); expect(html).not.toContain(''); - expect(html).toContain('<img src=x onerror=alert(1)>'); + expect(html).not.toContain(' { expect(drawn).toEqual(splitTextParagraphs(whole).map((p) => p.text)); }); }); + +/** + * TXT files saved from a web page still carry its markup. Escaped, the tags are + * drawn as text and every `
` line becomes a gap, so they are cleaned before + * paragraphs are inferred. + */ +describe('web-page markup in a TXT', () => { + const scraped = '
\n 天斗星作为第一颗被人类发现并且移民成功的行政星，距今已经有数千年的历史了。
\n
\n
\n
\n 这颗星球大约有原本斗罗星二分之一大。
\n
'; + + it('turns the scraped block into clean paragraphs with no tags or gaps', () => { + expect(splitTextParagraphs(scraped).map((p) => p.text)).toEqual([ + '天斗星作为第一颗被人类发现并且移民成功的行政星，距今已经有数千年的历史了。', + '这颗星球大约有原本斗罗星二分之一大。', + ]); + }); + + it('renders no literal tags and no empty paragraphs', () => { + const html = textToChapterHtml(scraped); + expect(html).not.toMatch(/<|\s*<\/div>|

<\/p>/); + expect(html.match(/

/g)).toHaveLength(2); + }); + + it('keeps a heading that follows the markup', () => { + expect(splitChapterHeading('

\n第一章 天斗星
\n正文。
').heading).toBe('第一章 天斗星'); + }); + + it('drops inline tags, decodes entities and leaves prose angle brackets alone', () => { + expect(cleanTextMarkup('

他说“好” a<b！

')).toBe('\n他说“好” a 未知')).toBe('如果 a 未知'); + expect(cleanTextMarkup(cleanTextMarkup(scraped))).toBe(cleanTextMarkup(scraped)); + }); + + it('treats a trailing break tag as the end of a paragraph when streaming', () => { + expect(endsAtParagraphBreak('一段。
')).toBe(true); + expect(endsAtParagraphBreak('一段')).toBe(false); + }); +});