Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 72 additions & 23 deletions rust/crates/du-db/src/publication.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,19 @@ pub async fn dois(pool: &PgPool) -> Result<Vec<(PublicationId, String)>, DbError
Ok(rows.into_iter().map(|(id, doi)| (PublicationId(id), doi)).collect())
}

/// Publications the by-DOI work-list can never reach: an OpenAlex id but no DOI.
/// A discovery candidate is promoted on its OpenAlex id alone, so without this
/// pass those rows keep NULL citations / open-access status forever.
pub async fn openalex_ids_without_doi(pool: &PgPool) -> Result<Vec<(PublicationId, String)>, DbError> {
let rows: Vec<(i64, String)> = sqlx::query_as(
"SELECT id, open_alex_id FROM pubs.publication \
WHERE open_alex_id IS NOT NULL AND doi IS NULL ORDER BY id",
)
.fetch_all(pool)
.await?;
Ok(rows.into_iter().map(|(id, oa)| (PublicationId(id), oa)).collect())
}

/// Apply OpenAlex enrichment (only overwrites a column when the new value is set).
pub async fn update_openalex(pool: &PgPool, id: PublicationId, u: &OpenAlexUpdate) -> Result<bool, DbError> {
let affected = sqlx::query(
Expand Down Expand Up @@ -140,34 +153,44 @@ pub async fn enabled_search_configs(pool: &PgPool) -> Result<Vec<SearchConfig>,
.collect())
}

/// The metadata a discovery candidate is written with. `doi` is expected in bare
/// form (`du_external::openalex::normalize_doi`) — the catalog stores DOIs bare.
#[derive(Debug, Default, Clone)]
pub struct NewCandidate<'a> {
pub openalex_id: &'a str,
pub doi: Option<&'a str>,
pub title: Option<&'a str>,
pub abstract_summary: Option<&'a str>,
pub publication_date: Option<NaiveDate>,
pub journal_name: Option<&'a str>,
pub cited_by_count: Option<i32>,
pub open_access_status: Option<&'a str>,
}

/// Upsert a discovery candidate by OpenAlex id (preserves curator status/review).
/// Returns `true` when the row was newly inserted (vs an update of one already seen).
#[allow(clippy::too_many_arguments)]
pub async fn upsert_candidate(
pool: &PgPool,
openalex_id: &str,
doi: Option<&str>,
title: Option<&str>,
abstract_summary: Option<&str>,
publication_date: Option<NaiveDate>,
journal_name: Option<&str>,
) -> Result<bool, DbError> {
pub async fn upsert_candidate(pool: &PgPool, c: &NewCandidate<'_>) -> Result<bool, DbError> {
// `xmax = 0` distinguishes a fresh INSERT from a DO UPDATE on conflict.
let inserted: bool = sqlx::query_scalar(
"INSERT INTO pubs.publication_candidate \
(openalex_id, doi, title, abstract, publication_date, journal_name, status) \
VALUES ($1, $2, $3, $4, $5, $6, 'pending') \
(openalex_id, doi, title, abstract, publication_date, journal_name, \
cited_by_count, open_access_status, status) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending') \
ON CONFLICT (openalex_id) DO UPDATE SET doi = EXCLUDED.doi, title = EXCLUDED.title, \
abstract = EXCLUDED.abstract, publication_date = EXCLUDED.publication_date, \
journal_name = EXCLUDED.journal_name \
journal_name = EXCLUDED.journal_name, \
cited_by_count = COALESCE(EXCLUDED.cited_by_count, pubs.publication_candidate.cited_by_count), \
open_access_status = COALESCE(EXCLUDED.open_access_status, pubs.publication_candidate.open_access_status) \
RETURNING (xmax = 0)",
)
.bind(openalex_id)
.bind(doi)
.bind(title)
.bind(abstract_summary)
.bind(publication_date)
.bind(journal_name)
.bind(c.openalex_id)
.bind(c.doi)
.bind(c.title)
.bind(c.abstract_summary)
.bind(c.publication_date)
.bind(c.journal_name)
.bind(c.cited_by_count)
.bind(c.open_access_status)
.fetch_one(pool)
.await?;
Ok(inserted)
Expand Down Expand Up @@ -321,12 +344,15 @@ pub struct Candidate {
pub publication_date: Option<NaiveDate>,
pub journal_name: Option<String>,
pub relevance_score: Option<f64>,
pub cited_by_count: Option<i32>,
pub open_access_status: Option<String>,
pub status: String,
pub created_at: DateTime<Utc>,
}

const CAND_COLS: &str = "id, openalex_id, doi, title, abstract AS abstract_text, publication_date, \
journal_name, relevance_score::float8 AS relevance_score, status, created_at";
journal_name, relevance_score::float8 AS relevance_score, cited_by_count, open_access_status, \
status, created_at";

/// Paginated candidate queue, optionally filtered by status, newest first.
pub async fn list_candidates(
Expand Down Expand Up @@ -385,6 +411,10 @@ pub async fn review_candidate(
/// publication matching the candidate's OpenAlex id or DOI, else create one from
/// the candidate's metadata; then mark the candidate `accepted`. Returns the
/// publication id. Errors if the candidate has no title (publications require one).
///
/// Citation count and open-access status come across too, so the promoted paper
/// carries its "Open access" badge immediately rather than from the next nightly
/// enrichment run. On reuse they gap-fill an existing row without overwriting it.
pub async fn promote_candidate(pool: &PgPool, id: i64, by: Uuid) -> Result<PublicationId, DbError> {
let mut tx = pool.begin().await?;
let c: Candidate = sqlx::query_as(&format!(
Expand All @@ -410,18 +440,37 @@ pub async fn promote_candidate(pool: &PgPool, id: i64, by: Uuid) -> Result<Publi
.fetch_optional(&mut *tx)
.await?;
let pub_id: i64 = match existing {
Some(pid) => pid,
Some(pid) => {
// Gap-fill only: a publication the curators already own keeps its own
// values; this just supplies what the candidate knows and it doesn't.
sqlx::query(
"UPDATE pubs.publication SET \
cited_by_count = COALESCE(cited_by_count, $2), \
open_access_status = COALESCE(open_access_status, $3), \
updated_at = now() \
WHERE id = $1 AND (cited_by_count IS NULL OR open_access_status IS NULL)",
)
.bind(pid)
.bind(c.cited_by_count)
.bind(c.open_access_status.as_deref())
.execute(&mut *tx)
.await?;
pid
}
None => {
sqlx::query_scalar(
"INSERT INTO pubs.publication (open_alex_id, doi, title, journal, publication_date, abstract_summary) \
VALUES ($1, $2, $3, $4, $5, $6) RETURNING id",
"INSERT INTO pubs.publication (open_alex_id, doi, title, journal, publication_date, \
abstract_summary, cited_by_count, open_access_status) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id",
)
.bind(&c.openalex_id)
.bind(c.doi.as_deref())
.bind(title)
.bind(c.journal_name.as_deref())
.bind(c.publication_date)
.bind(c.abstract_text.as_deref())
.bind(c.cited_by_count)
.bind(c.open_access_status.as_deref())
.fetch_one(&mut *tx)
.await?
}
Expand Down
30 changes: 25 additions & 5 deletions rust/crates/du-db/tests/publication_candidate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,25 @@ async fn candidate_review_and_promote() {

// Discovery upserts two candidates.
du_db::publication::upsert_candidate(
&pool, "TESTPC-W1", Some("10.1234/testpc.1"), Some("A Y-DNA study"),
Some("abstract one"), None, Some("J. Phylogenetics"),
&pool,
&du_db::publication::NewCandidate {
openalex_id: "TESTPC-W1",
doi: Some("10.1234/testpc.1"),
title: Some("A Y-DNA study"),
abstract_summary: Some("abstract one"),
journal_name: Some("J. Phylogenetics"),
cited_by_count: Some(12),
open_access_status: Some("gold"),
..Default::default()
},
).await.expect("upsert 1");
du_db::publication::upsert_candidate(
&pool, "TESTPC-W2", None, Some("An off-topic paper"), None, None, None,
&pool,
&du_db::publication::NewCandidate {
openalex_id: "TESTPC-W2",
title: Some("An off-topic paper"),
..Default::default()
},
).await.expect("upsert 2");

// Both are pending.
Expand All @@ -55,6 +69,10 @@ async fn candidate_review_and_promote() {
let got = du_db::publication::get_by_id(&pool, pub_id).await.expect("get pub").expect("pub exists");
assert_eq!(got.title, "A Y-DNA study");
assert_eq!(got.doi.as_deref(), Some("10.1234/testpc.1"));
// Open-access status and citations come across with the promotion, so the
// reference list badges the paper immediately (not after the nightly job).
assert_eq!(got.open_access_status.as_deref(), Some("gold"));
assert_eq!(got.cited_by_count, Some(12));
let c1_after = du_db::publication::get_candidate(&pool, c1.id).await.unwrap().unwrap();
assert_eq!(c1_after.status, "accepted");

Expand Down Expand Up @@ -91,8 +109,10 @@ async fn bulk_system_reject_only_touches_pending() {

// Three pending candidates + one already accepted.
for oa in ["TESTPC-R1", "TESTPC-R2", "TESTPC-R3", "TESTPC-R4"] {
du_db::publication::upsert_candidate(&pool, oa, None, Some("t"), None, None, None)
.await.expect("upsert");
du_db::publication::upsert_candidate(
&pool,
&du_db::publication::NewCandidate { openalex_id: oa, title: Some("t"), ..Default::default() },
).await.expect("upsert");
}
let r4 = du_db::publication::list_candidates(&pool, None, 1, 100)
.await.unwrap().items.into_iter().find(|c| c.openalex_id == "TESTPC-R4").unwrap();
Expand Down
72 changes: 70 additions & 2 deletions rust/crates/du-external/src/openalex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ pub struct WorkMeta {
pub abstract_summary: Option<String>,
}

/// A discovered candidate publication (from a search).
/// A discovered candidate publication (from a search). Carries the same
/// open-access / citation metadata as [`WorkMeta`] so a promoted candidate is a
/// fully-formed publication without waiting for the nightly by-DOI enrichment.
#[derive(Debug, Clone, PartialEq)]
pub struct Candidate {
pub openalex_id: String,
Expand All @@ -32,6 +34,8 @@ pub struct Candidate {
pub abstract_summary: Option<String>,
pub publication_date: Option<NaiveDate>,
pub journal: Option<String>,
pub cited_by_count: Option<i32>,
pub open_access_status: Option<String>,
}

// ── wire types ────────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -102,6 +106,21 @@ fn short_id(url: &str) -> &str {
url.rsplit('/').next().unwrap_or(url)
}

/// Reduce a DOI to bare form (`10.1234/x`), stripping the `doi.org` resolver
/// prefix OpenAlex wraps around it and the `doi:` scheme people paste. Bare form
/// is what the catalog stores: the reference list builds its link as
/// `https://doi.org/{doi}`, and duplicate detection compares DOIs literally.
pub fn normalize_doi(raw: &str) -> String {
let d = raw.trim();
let lower = d.to_ascii_lowercase();
for prefix in ["https://doi.org/", "http://doi.org/", "https://dx.doi.org/", "http://dx.doi.org/", "doi:"] {
if lower.starts_with(prefix) {
return d[prefix.len()..].trim().to_string();
}
}
d.to_string()
}

fn parse_date(s: &Option<String>) -> Option<NaiveDate> {
s.as_deref().and_then(|d| NaiveDate::parse_from_str(d, "%Y-%m-%d").ok())
}
Expand Down Expand Up @@ -138,11 +157,13 @@ impl Work {
let openalex_id = self.id.clone()?;
Some(Candidate {
openalex_id,
doi: self.doi.clone(),
doi: self.doi.as_deref().map(normalize_doi).filter(|d| !d.is_empty()),
title: self.title.clone(),
abstract_summary: self.abstract_inverted_index.as_ref().and_then(reconstruct_abstract),
publication_date: parse_date(&self.publication_date),
journal: self.primary_location.and_then(|l| l.source).and_then(|s| s.display_name),
cited_by_count: self.cited_by_count.map(|c| c as i32),
open_access_status: self.open_access.and_then(|o| o.oa_status),
})
}
}
Expand Down Expand Up @@ -173,6 +194,25 @@ impl OpenAlexClient {
Ok(Some(work.into_meta()))
}

/// Fetch enrichment metadata by OpenAlex work id (bare `W…` or the full
/// `https://openalex.org/W…` URL). The by-id counterpart of
/// [`work_by_doi`](Self::work_by_doi), for catalog rows that have an OpenAlex
/// id but no DOI — a promoted discovery candidate need not have one, and the
/// by-DOI job can never reach those. Returns None on 404.
pub async fn work_by_id(&self, openalex_id: &str) -> Result<Option<WorkMeta>, ExternalError> {
let url = format!("{}/works/{}", self.base, short_id(openalex_id.trim()));
let mut req = self.http.get(url);
if let Some(m) = &self.mailto {
req = req.query(&[("mailto", m.as_str())]);
}
let resp = req.send().await?;
if resp.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(None);
}
let work: Work = resp.error_for_status()?.json().await?;
Ok(Some(work.into_meta()))
}

/// Fetch a single work's primary-topic id (short form, e.g. `T10751`). Used by
/// the topic-prune backfill to retroactively test the pending candidate queue
/// against a config's `primary_topic.id` whitelist. `openalex_id` may be a bare
Expand Down Expand Up @@ -296,5 +336,33 @@ mod tests {
assert_eq!(cands.len(), 1);
assert_eq!(cands[0].openalex_id, "https://openalex.org/W1");
assert_eq!(cands[0].title.as_deref(), Some("A"));
// The resolver prefix is stripped at ingest — the catalog stores bare DOIs.
assert_eq!(cands[0].doi.as_deref(), Some("10.1/a"));
}

/// The search payload carries open-access + citation metadata; a candidate must
/// keep it, or a promoted paper shows no OA badge until the nightly job re-fetches.
#[test]
fn candidate_keeps_open_access_and_citations() {
let json = r#"{
"id": "https://openalex.org/W9",
"doi": "https://doi.org/10.1/b",
"title": "B",
"cited_by_count": 7,
"open_access": { "oa_status": "gold" }
}"#;
let c = serde_json::from_str::<Work>(json).unwrap().into_candidate().unwrap();
assert_eq!(c.cited_by_count, Some(7));
assert_eq!(c.open_access_status.as_deref(), Some("gold"));
}

#[test]
fn normalize_doi_strips_resolver_prefixes() {
assert_eq!(normalize_doi("https://doi.org/10.1002/advs.76320"), "10.1002/advs.76320");
assert_eq!(normalize_doi("http://dx.doi.org/10.1/x"), "10.1/x");
assert_eq!(normalize_doi("DOI:10.1/x"), "10.1/x");
assert_eq!(normalize_doi(" 10.1/x "), "10.1/x");
// A DOI is not a URL — anything else is left alone.
assert_eq!(normalize_doi("https://example.org/paper"), "https://example.org/paper");
}
}
37 changes: 29 additions & 8 deletions rust/crates/du-jobs/src/publications.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,13 @@ fn to_update(m: WorkMeta) -> OpenAlexUpdate {
}
}

/// Refresh OpenAlex metadata for every publication with a DOI.
/// Refresh OpenAlex metadata for every publication with a DOI, then for those with
/// only an OpenAlex id (promoted discovery candidates need not have a DOI — the
/// by-DOI pass can never reach them).
pub async fn update_all(pool: &PgPool, client: &OpenAlexClient) -> anyhow::Result<()> {
let dois = du_db::publication::dois(pool).await?;
let total = dois.len();
let by_id = du_db::publication::openalex_ids_without_doi(pool).await?;
let total = dois.len() + by_id.len();
let (mut updated, mut missing, mut failed) = (0usize, 0usize, 0usize);
for (id, doi) in dois {
match client.work_by_doi(&doi).await {
Expand All @@ -63,6 +66,20 @@ pub async fn update_all(pool: &PgPool, client: &OpenAlexClient) -> anyhow::Resul
}
tokio::time::sleep(REQUEST_GAP).await;
}
for (id, openalex_id) in by_id {
match client.work_by_id(&openalex_id).await {
Ok(Some(meta)) => {
du_db::publication::update_openalex(pool, id, &to_update(meta)).await?;
updated += 1;
}
Ok(None) => missing += 1,
Err(e) => {
tracing::warn!(%openalex_id, error = %e, "openalex fetch by id failed");
failed += 1;
}
}
tokio::time::sleep(REQUEST_GAP).await;
}
tracing::info!(total, updated, missing, failed, "publication-update done");
Ok(())
}
Expand Down Expand Up @@ -127,12 +144,16 @@ async fn run_search(
for c in &page.candidates {
let inserted = du_db::publication::upsert_candidate(
pool,
&c.openalex_id,
c.doi.as_deref(),
c.title.as_deref(),
c.abstract_summary.as_deref(),
c.publication_date,
c.journal.as_deref(),
&du_db::publication::NewCandidate {
openalex_id: &c.openalex_id,
doi: c.doi.as_deref(),
title: c.title.as_deref(),
abstract_summary: c.abstract_summary.as_deref(),
publication_date: c.publication_date,
journal_name: c.journal.as_deref(),
cited_by_count: c.cited_by_count,
open_access_status: c.open_access_status.as_deref(),
},
)
.await?;
seen += 1;
Expand Down
Loading
Loading