feat(chapter): preserve source-site order in chapter list (0.52.0)
The user-facing chapter list ordered by (number ASC, created_at ASC),
which broke the source site's order in two ways: non-numeric entries
("notice. : Officials") parsed to number=0 and clustered at the top,
even though the site placed them mid-list, and variants sharing a
number ("Ch.14 : PH" / "Ch.14 : Official") were torn apart by the
created_at tiebreak.
Capture each chapter's position in the source DOM as `source_index`
(0 = first = newest on this site) on every crawler sync, including the
UPDATE branch so a new chapter prepended on the source shifts every
existing row down by one on the next tick. The list query reverses
this with `ORDER BY source_index DESC NULLS LAST, number ASC,
created_at ASC` so the oldest chapter appears first, variants stay
adjacent in the order the site shows them, and non-numeric entries
land where the site placed them. User-uploaded chapters and pre-
migration rows keep their NULL source_index and fall through to the
prior number/created_at tiebreak via NULLS LAST.
The reader's client-side `[...chapters].sort((a,b) => a.number - b.number)`
is dropped; prev/next now walks the server-ordered array positionally
so it traverses variants and non-numeric entries in display order.
Existing data populates on the next cron tick or via admin force-resync.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
|
||||
use mangalord::crawler::source::{SourceChapterRef, SourceManga};
|
||||
use mangalord::repo::crawler::{self, ChapterDiff, UpsertStatus};
|
||||
use mangalord::repo::chapter as chapter_repo;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -961,3 +962,261 @@ async fn re_appearing_manga_clears_dropped_at(pool: PgPool) {
|
||||
assert!(dropped.0.is_none());
|
||||
assert_eq!(dropped.1, up.manga_id);
|
||||
}
|
||||
|
||||
// ---- source_index: site-order preservation ----
|
||||
//
|
||||
// The user-facing chapter list reverses the source-site order so that
|
||||
// the oldest chapter appears first. The crawler records each row's DOM
|
||||
// position in `chapters.source_index` (0 = first in source DOM = newest
|
||||
// on this site) on every sync; the list query orders by source_index
|
||||
// DESC NULLS LAST, falling through to number/created_at for rows with
|
||||
// no source row (e.g. user uploads).
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn source_index_set_on_insert_matches_dom_order(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
let m = sample_manga("foo", "Foo Manga", "hash-1");
|
||||
let up = crawler::upsert_manga_from_source(&pool, "target", "https://x.example/foo", &m)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let chapters = vec![
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "a".into(),
|
||||
number: 30,
|
||||
title: Some("Ch.30".into()),
|
||||
url: "https://x.example/foo/a".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "b".into(),
|
||||
number: 29,
|
||||
title: Some("Ch.29".into()),
|
||||
url: "https://x.example/foo/b".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "c".into(),
|
||||
number: 28,
|
||||
title: Some("Ch.28".into()),
|
||||
url: "https://x.example/foo/c".into(),
|
||||
},
|
||||
];
|
||||
crawler::sync_manga_chapters(&pool, "target", up.manga_id, &chapters)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let rows: Vec<(String, Option<i32>)> = sqlx::query_as(
|
||||
"SELECT cs.source_chapter_key, c.source_index \
|
||||
FROM chapters c \
|
||||
JOIN chapter_sources cs ON cs.chapter_id = c.id \
|
||||
WHERE c.manga_id = $1 \
|
||||
ORDER BY cs.source_chapter_key",
|
||||
)
|
||||
.bind(up.manga_id)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
("a".to_string(), Some(0)),
|
||||
("b".to_string(), Some(1)),
|
||||
("c".to_string(), Some(2)),
|
||||
],
|
||||
"source_index reflects enumerate() position in the input slice",
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn source_index_rewritten_on_resync_when_new_chapter_prepended(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
let m = sample_manga("foo", "Foo Manga", "hash-1");
|
||||
let up = crawler::upsert_manga_from_source(&pool, "target", "https://x.example/foo", &m)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let first = vec![
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "a".into(),
|
||||
number: 1,
|
||||
title: Some("Ch.1".into()),
|
||||
url: "https://x.example/foo/a".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "b".into(),
|
||||
number: 2,
|
||||
title: Some("Ch.2".into()),
|
||||
url: "https://x.example/foo/b".into(),
|
||||
},
|
||||
];
|
||||
crawler::sync_manga_chapters(&pool, "target", up.manga_id, &first)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Second sync: a brand-new chapter appears at the top of the source
|
||||
// (newest first on the site). All existing rows must shift their
|
||||
// source_index down by one so the display order stays correct.
|
||||
let second = vec![
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "new".into(),
|
||||
number: 3,
|
||||
title: Some("Ch.3".into()),
|
||||
url: "https://x.example/foo/new".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "a".into(),
|
||||
number: 1,
|
||||
title: Some("Ch.1".into()),
|
||||
url: "https://x.example/foo/a".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "b".into(),
|
||||
number: 2,
|
||||
title: Some("Ch.2".into()),
|
||||
url: "https://x.example/foo/b".into(),
|
||||
},
|
||||
];
|
||||
crawler::sync_manga_chapters(&pool, "target", up.manga_id, &second)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let rows: Vec<(String, Option<i32>)> = sqlx::query_as(
|
||||
"SELECT cs.source_chapter_key, c.source_index \
|
||||
FROM chapters c \
|
||||
JOIN chapter_sources cs ON cs.chapter_id = c.id \
|
||||
WHERE c.manga_id = $1 \
|
||||
ORDER BY cs.source_chapter_key",
|
||||
)
|
||||
.bind(up.manga_id)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
("a".to_string(), Some(1)),
|
||||
("b".to_string(), Some(2)),
|
||||
("new".to_string(), Some(0)),
|
||||
],
|
||||
"new chapter takes index 0, existing rows shift down on UPDATE",
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_for_manga_returns_source_order_reversed(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
let m = sample_manga("foo", "Foo Manga", "hash-1");
|
||||
let up = crawler::upsert_manga_from_source(&pool, "target", "https://x.example/foo", &m)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Site DOM order (top-down = newest-first):
|
||||
// ch11 (number = 11)
|
||||
// notice (number = 0, non-numeric label on the site)
|
||||
// ch10 (number = 10)
|
||||
// Numbers deliberately disagree with DOM order: a number-based sort
|
||||
// would put notice first, but the site places it between ch10 and
|
||||
// ch11. Reversed-DOM display should yield [ch10, notice, ch11].
|
||||
let chapters = vec![
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "ch11".into(),
|
||||
number: 11,
|
||||
title: Some("Ch.11 : Official".into()),
|
||||
url: "https://x.example/foo/11".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "notice".into(),
|
||||
number: 0,
|
||||
title: Some("notice. : Officials".into()),
|
||||
url: "https://x.example/foo/notice".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "ch10".into(),
|
||||
number: 10,
|
||||
title: Some("Ch.10 : Official".into()),
|
||||
url: "https://x.example/foo/10".into(),
|
||||
},
|
||||
];
|
||||
crawler::sync_manga_chapters(&pool, "target", up.manga_id, &chapters)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let listed = chapter_repo::list_for_manga(&pool, up.manga_id, 50, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
let keys: Vec<String> = listed
|
||||
.iter()
|
||||
.map(|c| c.title.clone().unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
keys,
|
||||
vec![
|
||||
"Ch.10 : Official".to_string(),
|
||||
"notice. : Officials".to_string(),
|
||||
"Ch.11 : Official".to_string(),
|
||||
],
|
||||
"list returns chapters in reversed source-DOM order, so the \
|
||||
oldest appears first and non-numeric entries land where the \
|
||||
site placed them",
|
||||
);
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "./migrations")]
|
||||
async fn list_for_manga_places_null_source_index_last(pool: PgPool) {
|
||||
crawler::ensure_source(&pool, "target", "T", "https://x.example")
|
||||
.await
|
||||
.unwrap();
|
||||
let m = sample_manga("foo", "Foo Manga", "hash-1");
|
||||
let up = crawler::upsert_manga_from_source(&pool, "target", "https://x.example/foo", &m)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Crawled chapters get source_index 0 and 1; the upload path leaves
|
||||
// it NULL. NULLS LAST plus the (number, created_at) tail means the
|
||||
// upload sits after both crawled rows even though its number is in
|
||||
// the middle.
|
||||
let crawled = vec![
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "a".into(),
|
||||
number: 1,
|
||||
title: Some("Ch.1".into()),
|
||||
url: "https://x.example/foo/a".into(),
|
||||
},
|
||||
SourceChapterRef {
|
||||
source_chapter_key: "b".into(),
|
||||
number: 3,
|
||||
title: Some("Ch.3".into()),
|
||||
url: "https://x.example/foo/b".into(),
|
||||
},
|
||||
];
|
||||
crawler::sync_manga_chapters(&pool, "target", up.manga_id, &crawled)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
chapter_repo::create(&pool, up.manga_id, 2, Some("User upload Ch.2"), None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let listed = chapter_repo::list_for_manga(&pool, up.manga_id, 50, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
let titles: Vec<String> = listed
|
||||
.iter()
|
||||
.map(|c| c.title.clone().unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
titles,
|
||||
vec![
|
||||
"Ch.3".to_string(),
|
||||
"Ch.1".to_string(),
|
||||
"User upload Ch.2".to_string(),
|
||||
],
|
||||
"crawled rows ordered by reversed source_index; user upload \
|
||||
(NULL source_index) falls through to the end",
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user