Module:DPLA/sandbox
Lua
Documentation for this module may be created at Module:DPLA/sandbox/doc
Code
--[==[
Module:DPLA — helpers for the {{DPLA metadata}} template family.
This module reads Structured Data on Commons (SDC) statements that the
Digital Public Library of America partner-uploads workflow (`sdc-sync`)
writes to file pages, and renders them as the fields shown by
{{DPLA metadata}}.
All helpers filter to DPLA-authored SDC by the reference the writer
stamps on every statement it makes: a P123 (publisher) = Q2944483
(Digital Public Library of America) snak. Statements lacking that
reference — including ones that carry only the legacy P459 = Q61848113
("heuristic") determination qualifier without DPLA provenance — are
treated as non-DPLA and rendered as user-contributed.
Chunked claims: long string and monolingualtext values that exceed
Wikibase's 1500-character limit are written as multiple statements on
the same property, each carrying a P1545 (series ordinal) qualifier in
the form "<letter><ordinal>" (e.g. "A1", "A2", "B1"). The helpers
below transparently reassemble these: chunks with the same series
letter on the same property are sorted by ordinal and concatenated
into one logical value. Statements without a P1545 qualifier are
returned as their own singleton values, preserving the order each
series or singleton first appears in.
Documentation: [[Module:DPLA/doc]]
Testcases: [[Template:DPLA metadata/testcases]]
Sandbox: [[Module:DPLA/sandbox]]
Repository: https://github.com/dpla/ingest-wikimedia
]==]
local p = {}
-- ============================================================
-- Constants
-- ============================================================
-- SDC properties (mirrors what sdc-sync writes in
-- ingest_wikimedia/sdc.py — keep these in sync).
local P = {
DETERMINATION_METHOD = 'P459',
PUBLISHER = 'P123',
REFERENCE_URL = 'P854', -- reference snak: dp.la item URL (owning item)
RETRIEVED = 'P813', -- reference snak: retrieved-on date
INSTITUTION = 'P195',
CREATOR = 'P170',
DATE = 'P571',
STATED_AS = 'P1932',
SOURCING_CIRCUMSTANCES = 'P1480',
SUBJECT_STRING = 'P4272',
SUBJECT_ENTITY = 'P921',
PARTNERSHIP = 'P9126',
ROLE = 'P3831',
SOURCE_OF_FILE = 'P7482',
DESCRIBED_AT_URL = 'P973',
LOCAL_IDENTIFIER = 'P217',
NARA_IDENTIFIER = 'P1225',
AUTHOR_NAME_STRING = 'P2093',
DPLA_ID = 'P760',
PAGE = 'P304',
SERIES_ORDINAL = 'P1545',
DESCRIPTION = 'P10358',
COMMONS_CATEGORY = 'P8464',
LOGO_IMAGE = 'P154',
IMAGE = 'P18',
TITLE = 'P1476',
OTHER_VERSIONS = 'P6802',
}
local Q = {
HEURISTIC = 'Q61848113', -- determination method = "heuristic"
DPLA = 'Q2944483', -- Digital Public Library of America
-- P3831 (object has role) values, named for their true Wikidata roles.
-- A service hub's hub is an aggregator (like DPLA); its institution is
-- a repository. A content hub (NARA, Smithsonian) is itself a
-- repository; its contributing department is a custodial unit.
ROLE_AGGREGATOR = 'Q393351', -- object-has-role = "aggregator"
ROLE_REPOSITORY = 'Q108296843',-- object-has-role = "repository"
ROLE_CONTRIBUTING = 'Q108296919',-- object-has-role = "custodial unit"
NARA = 'Q518155', -- National Archives and Records Administration
CIRCA = 'Q5727902', -- sourcing circumstances = "circa"
}
-- The hub-label string the existing {{DPLA}} template branches on for
-- the NARA-identifier line. Must match the en label of Q518155.
local NARA_HUB_LABEL = 'National Archives and Records Administration'
local DEFAULT_PARTNER_LOGO = 'File:DPLA square logo.svg'
-- Maintenance categories for unresolvable partners / institutions —
-- preserve the exact names the existing {{DPLA/hub cat}} and
-- {{DPLA/inst cat}} templates use, so maintenance queries continue to
-- find files via these category names.
local UNKNOWN_PARTNER_CAT = 'Media contributed by the Digital Public Library of America with unknown partner'
local UNKNOWN_INSTITUTION_CAT = 'Media contributed by the Digital Public Library of America with unknown institution'
local MAIN_CAT = 'Media contributed by the Digital Public Library of America'
-- Tracking category for files where the rendered metadata table
-- includes at least one user-contributed (non-DPLA-attributed)
-- value — i.e. anything that surfaces in the yellow box. Emitted
-- under the same condition as the yellow box itself so the
-- category membership tracks "files with non-DPLA metadata
-- enhancements" exactly.
local ENHANCED_CAT = 'Media from the Digital Public Library of America with metadata enhancements'
-- Tracking category for a file that the SHA1-uniqueness redesign
-- centralized: it carries metadata from MORE THAN ONE DPLA item (a
-- cross-item duplicate → an extra blue box renders) OR occupies MORE
-- THAN ONE page position within a single item (a within-item duplicate →
-- a P760 with several P304 page qualifiers → the multi-line "Other pages"
-- navigation). Emitted whenever either condition holds so these merged
-- files can be found and audited.
local MULTI_CAT = 'DPLA multi-item or multi-page files'
-- Required-SDC properties tracked by the "DPLA files missing
-- required SDC statements" maintenance category. Mirrors the
-- ``{{#invoke:SDC_tracking|SDC_statement_exist}}`` calls in the
-- legacy ``{{DPLA}}`` template: any file whose MediaInfo entity
-- lacks ANY of these properties lands in the catch-all maintenance
-- category so operators can backfill from it. ``P170`` (creator)
-- is broken out into its own ``files missing creator`` category
-- by ``render_metadata_table`` because its backlog is triaged
-- separately from the other missing properties.
local REQUIRED_SDC_PROPS = {
'P760', -- DPLA ID
'P1476', -- title
'P6216', -- copyright status
'P195', -- collection
'P9126', -- partnership (DPLA + hub + institution P3831-qualifier)
'P7482', -- source of file (catalog URL etc.)
}
-- Commons-compatible DPLA-attributed rights values for P275 (copyright
-- license) and P6426 (copyright status as a creator). Derived from
-- ``rights.json`` by URI-substring classification: free Creative Commons
-- (``/licenses/by/*`` and ``/licenses/by-sa/*``) + ``/publicdomain/zero/*``
-- + ``/publicdomain/mark/*`` are eligible; RightsStatements.org NoC-* and
-- NKC-* are eligible; everything else (CC NC and/or ND restrictions,
-- ``InC*``/``UND``/``CNE`` rights statements) is treated as ineligible.
-- Anything not in this set — including Q-IDs we don't recognise — is
-- treated as potentially incompatible and the file is surfaced via the
-- ``files with potential copyright issues`` tracking category for human
-- review. A 151-element allowlist is smaller and stabler than
-- the equivalent denylist and fails safe on unknown values.
local ELIGIBLE_RIGHTS = {
['Q6938433'] = true,
['Q7257361'] = true,
['Q14946043'] = true,
['Q14947546'] = true,
['Q15914252'] = true,
['Q18195572'] = true,
['Q18199165'] = true,
['Q18199175'] = true,
['Q18810143'] = true,
['Q18810333'] = true,
['Q18810341'] = true,
['Q19068220'] = true,
['Q19113751'] = true,
['Q19125117'] = true,
['Q20007257'] = true,
['Q24331618'] = true,
['Q26116436'] = true,
['Q26259495'] = true,
['Q27940776'] = true,
['Q30942811'] = true,
['Q42716613'] = true,
['Q44282633'] = true,
['Q44282641'] = true,
['Q47001652'] = true,
['Q47530911'] = true,
['Q47530955'] = true,
['Q52555753'] = true,
['Q53859967'] = true,
['Q56292840'] = true,
['Q62619894'] = true,
['Q63241773'] = true,
['Q63340742'] = true,
['Q67918154'] = true,
['Q75209430'] = true,
['Q75434631'] = true,
['Q75443434'] = true,
['Q75445499'] = true,
['Q75446609'] = true,
['Q75446635'] = true,
['Q75450165'] = true,
['Q75452310'] = true,
['Q75457467'] = true,
['Q75457506'] = true,
['Q75460106'] = true,
['Q75460149'] = true,
['Q75466259'] = true,
['Q75470365'] = true,
['Q75470422'] = true,
['Q75474094'] = true,
['Q75475677'] = true,
['Q75476747'] = true,
['Q75477775'] = true,
['Q75486069'] = true,
['Q75487055'] = true,
['Q75488238'] = true,
['Q75491630'] = true,
['Q75494411'] = true,
['Q75500112'] = true,
['Q75501683'] = true,
['Q75504835'] = true,
['Q75506669'] = true,
['Q75663969'] = true,
['Q75665696'] = true,
['Q75705948'] = true,
['Q75706881'] = true,
['Q75759387'] = true,
['Q75759731'] = true,
['Q75760479'] = true,
['Q75761383'] = true,
['Q75761779'] = true,
['Q75762418'] = true,
['Q75762784'] = true,
['Q75763101'] = true,
['Q75764151'] = true,
['Q75764470'] = true,
['Q75764895'] = true,
['Q75765287'] = true,
['Q75766316'] = true,
['Q75767185'] = true,
['Q75767606'] = true,
['Q75768706'] = true,
['Q75770766'] = true,
['Q75771320'] = true,
['Q75771874'] = true,
['Q75775133'] = true,
['Q75775714'] = true,
['Q75776014'] = true,
['Q75776487'] = true,
['Q75777688'] = true,
['Q75778801'] = true,
['Q75779562'] = true,
['Q75779905'] = true,
['Q75789929'] = true,
['Q75850366'] = true,
['Q75850813'] = true,
['Q75850832'] = true,
['Q75851799'] = true,
['Q75852313'] = true,
['Q75852938'] = true,
['Q75853187'] = true,
['Q75853514'] = true,
['Q75853549'] = true,
['Q75854323'] = true,
['Q75856699'] = true,
['Q75857518'] = true,
['Q75858169'] = true,
['Q75859019'] = true,
['Q75859751'] = true,
['Q75866892'] = true,
['Q75882470'] = true,
['Q75889409'] = true,
['Q75894644'] = true,
['Q75894680'] = true,
['Q76631753'] = true,
['Q76767348'] = true,
['Q76769447'] = true,
['Q77014037'] = true,
['Q77021108'] = true,
['Q77131257'] = true,
['Q77132386'] = true,
['Q77133402'] = true,
['Q77135172'] = true,
['Q77136299'] = true,
['Q77143083'] = true,
['Q77352646'] = true,
['Q77355872'] = true,
['Q77361415'] = true,
['Q77362254'] = true,
['Q77363039'] = true,
['Q77363856'] = true,
['Q77364488'] = true,
['Q77364872'] = true,
['Q77365183'] = true,
['Q77365530'] = true,
['Q77366066'] = true,
['Q77366576'] = true,
['Q77367349'] = true,
['Q80837139'] = true,
['Q80837607'] = true,
['Q86239208'] = true,
['Q86239559'] = true,
['Q86239991'] = true,
['Q86240326'] = true,
['Q86240624'] = true,
['Q86241082'] = true,
['Q98755364'] = true,
['Q98960995'] = true,
['Q99438747'] = true,
}
-- ============================================================
-- Localization (label + banner text)
-- ============================================================
--
-- Hybrid strategy:
--
-- 1. Field labels that semantically match existing Commons file-info
-- messages reuse them via ``mw.message`` — those already carry
-- ~150-language coverage via translatewiki.net.
-- 2. DPLA-specific labels (Creator with archival semantics, Institution,
-- Subject, DPLA ID, Other pages, Partnership) plus the two banner
-- strings and a few inline phrases (Catalog record, File URL, IIIF
-- manifest, circa, "Page N") come from [[Module:DPLA/i18n]] — a Lua
-- data table editable by any Commons editor without admin rights.
--
-- Translators: see the "Localization" section of [[Template:DPLA metadata/doc]]
-- for step-by-step instructions on adding your language to Module:DPLA/i18n.
--
-- Per-call cache for the viewer's language. ``mw.getCurrentFrame():preprocess
-- ('{{int:lang}}')`` is moderately expensive and the same value applies to
-- every label rendered on one file page; cache it module-wide for the
-- duration of one #invoke.
local _viewerLang
local function getViewerLang()
if _viewerLang then return _viewerLang end
local ok, lang = pcall(function()
return mw.getCurrentFrame():preprocess('{{int:lang}}')
end)
_viewerLang = (ok and lang and lang ~= '') and lang or 'en'
return _viewerLang
end
-- Map our internal label names to existing MediaWiki messages that already
-- carry good translation coverage and semantically match. Creator
-- deliberately is NOT mapped to ``wm-license-information-author`` —
-- archival "creator" (per the Society of American Archivists definition)
-- can be the entity responsible for the records' aggregation, not the
-- individual author of any item in them. We register our own translation.
local REUSED_MESSAGES = {
title = { key = 'wm-license-artwork-title', en = 'Title' },
description = { key = 'wm-license-information-description',en = 'Description' },
date = { key = 'wm-license-information-date', en = 'Date' },
source = { key = 'wm-license-information-source', en = 'Source' },
permission = { key = 'wm-license-information-permission', en = 'Permission' },
}
-- Wikidata items whose labels can serve as a fallback when neither the
-- viewer's language nor its MediaWiki fallback chain is present in
-- Module:DPLA/i18n. Wikidata's community-maintained labels often cover
-- languages the i18n module hasn't reached yet, especially for common
-- single-noun concepts (circa, creator, subject).
--
-- Listed only for keys where the Wikidata item is a precise semantic
-- match in most languages. Skipped where the Wikidata label drifts
-- (Q178706 "institution" → "social institution" in fr/nl; Q728646
-- "partnership" → "Personengesellschaft" legal-business sense in de;
-- Q1069725 "page" → just "page" with no slot for the page number).
-- Those keys are translator-supplied via Module:DPLA/i18n only.
--
-- ``mw.wikibase.getLabelByLang`` accepts P-IDs and Q-IDs interchangeably,
-- so a property label (P760) works as a fallback the same way an item
-- label does.
local FALLBACK_WIKIDATA = {
creator = 'Q59275219', -- archival creator (matches SAA definition)
subject = 'Q12310021',
catalog_record = 'Q59211006',
circa = 'Q5727902',
iiif_manifest = 'Q22682088', -- International Image Interoperability Framework
dpla_id = 'P760', -- Wikidata property: "DPLA ID"
}
-- English baselines for i18n keys this module introduces that the on-wiki
-- ``Module:DPLA/i18n`` subpage does not yet carry. ``localizedLabel``
-- consults this AFTER the subpage's own ``en`` row (step 4 of its fallback
-- chain), so a freshly-deployed module renders sensible English on the
-- sandbox before translators mirror the keys onto the subpage. Values use
-- the same ``$1`` positional-placeholder convention as the subpage entries
-- and are expanded by ``substituteParams`` at the call site (like
-- ``page_n``). Keep in sync with [[Module:DPLA/i18n]].
local I18N_EN_BASELINE = {
-- Blue-box banner for one contributing DPLA item on a cross-item
-- duplicate (a Commons file carrying statements from several DPLA
-- items). Mirrors the ``banner_dpla`` voice verbatim and appends a
-- per-item scope clause so each box names the item its metadata came
-- from. ``$1`` is the ``dpla:`` interwiki link to the item.
-- Double-quoted to carry the ``institution's`` apostrophe, matching the
-- on-wiki ``banner_dpla`` string.
banner_dpla_item = "This file was uploaded by the [[w:Digital Public Library of America|Digital Public Library of America]], and the following item metadata was created by the contributing institution's staff for DPLA item $1.",
-- Per-line prefix for the multi-line "Other pages" navigation of an
-- in-item multi-page file. ``$1`` is the page number this line
-- navigates from.
page_context = 'As page $1:',
}
-- Lazy-load the Lua i18n data. mw.loadData caches across #invoke calls
-- within a parse and is the right tool for translation tables that are
-- read every render.
local function getI18nData()
local ok, data = pcall(mw.loadData, 'Module:DPLA/i18n')
if ok and type(data) == 'table' then return data end
return { en = {} }
end
-- Resolve a localized string for the given internal name. Order of
-- precedence (each step skipped silently when nothing matches):
--
-- 1. Reused MediaWiki message in the viewer's language. Covers the field
-- labels that already have community translations on translatewiki
-- via the ``wm-license-*`` message family (title, description, date,
-- source, permission).
-- 2. ``Module:DPLA/i18n`` entry for the viewer's language plus any non-
-- English entries in the MediaWiki fallback chain. Lets translators
-- override default wording without admin help.
-- 3. Wikidata label for the QID listed in ``FALLBACK_WIKIDATA``, walking
-- the same non-English fallback chain. Community-maintained labels
-- catch languages that haven't been added to Module:DPLA/i18n yet.
-- 4. English baselines (Lua i18n.en → REUSED_MESSAGES[name].en → key
-- name verbatim). Always fires as the last resort.
--
-- English is deliberately held to the end so the Wikidata fallback can
-- intervene for non-English viewers whose language is in Wikidata but not
-- yet in Module:DPLA/i18n.
local function localizedLabel(name)
local lang = getViewerLang()
-- 1. Reused MediaWiki message.
local reused = REUSED_MESSAGES[name]
if reused then
local ok, msg = pcall(function()
return mw.message.new(reused.key):inLanguage(lang):plain()
end)
if ok and msg and msg ~= '' then
-- ⧼key⧽ is MediaWiki's "message undefined" sentinel (UTF-8 bytes
-- ``\226\167\188`` and ``\226\167\189`` for the open / close
-- angle-brackets). Treat as fall-through.
local opens = msg:sub(1, 3) == '\226\167\188'
local closes = msg:sub(-3) == '\226\167\189'
if not (opens and closes) then
return msg
end
end
end
-- Build the non-English fallback chain. Holding English back lets the
-- Wikidata step (3) intervene for non-English viewers before we land
-- on the English baseline (4).
local nonEnChain = {lang}
for _, c in ipairs(mw.language.getFallbacksFor(lang) or {}) do
if c ~= 'en' then table.insert(nonEnChain, c) end
end
-- 2. Module:DPLA/i18n in the non-English chain.
local data = getI18nData()
for _, c in ipairs(nonEnChain) do
local row = data[c]
if row and row[name] then return row[name] end
end
-- 3. Wikidata label fallback (single-noun concepts only).
local qid = FALLBACK_WIKIDATA[name]
if qid then
for _, c in ipairs(nonEnChain) do
local ok, label = pcall(mw.wikibase.getLabelByLang, qid, c)
if ok and label and label ~= '' then
return label
end
end
end
-- 4. English baselines.
local en = data.en
if en and en[name] then return en[name] end
if I18N_EN_BASELINE[name] then return I18N_EN_BASELINE[name] end
if reused then return reused.en end
return name
end
-- Substitute $1, $2, ... in a translation template with positional args.
-- Replacement is non-recursive (a value containing "$1" doesn't trigger
-- further substitution) and treats the values as plain text — wikitext in
-- the arguments is left alone for the parser to handle downstream.
local function substituteParams(template, ...)
local args = {...}
return (template:gsub('%$(%d+)', function(n)
return args[tonumber(n)] or ('$' .. n)
end))
end
-- ============================================================
-- Entity loading
-- ============================================================
-- Resolve which entity to read from. By default this is the MediaInfo of
-- the current page (the file we're rendering on); pass page=File:Foo.jpg
-- to override — primarily used by /testcases.
local function getEntity(args)
if args.page and args.page ~= '' then
local title = mw.title.new(args.page)
if not title or not title.id or title.id == 0 then return nil end
return mw.wikibase.getEntity('M' .. tostring(title.id))
end
return mw.wikibase.getEntity()
end
-- ============================================================
-- Statement filtering and snak rendering
-- ============================================================
-- True iff `stmt` carries a DPLA-authored reference: a reference with a
-- P123 (publisher) = Q2944483 (DPLA) snak. This is the authoritative
-- provenance marker — sdc-sync stamps it on every statement it writes,
-- and it is the same marker the Python writer's `_is_dpla_reference`
-- keys on. We deliberately do NOT accept the P459 determination-method
-- qualifier on its own: a statement can carry that qualifier without
-- DPLA provenance (stale writes from an older sdc-sync that did not add
-- references, or foreign edits), and treating those as authoritative
-- would leak non-DPLA data into the DPLA-rendered fields. Anything left
-- out by this test surfaces as user-contributed — the intended signal
-- that the file is unsynced or carries non-DPLA SDC and wants a re-sync.
local function isDplaDetermined(stmt)
for _, ref in ipairs(stmt.references or {}) do
for _, snak in ipairs((ref.snaks or {})[P.PUBLISHER] or {}) do
if snak.snaktype == 'value'
and snak.datavalue
and snak.datavalue.value
and snak.datavalue.value.id == Q.DPLA then
return true
end
end
end
return false
end
-- All DPLA-determined statements for one property, in entity-order.
local function dplaStatements(entity, propertyId)
local out = {}
if not entity or not entity.statements then return out end
for _, stmt in ipairs(entity.statements[propertyId] or {}) do
if isDplaDetermined(stmt) then
table.insert(out, stmt)
end
end
return out
end
-- All non-DPLA-determined statements for one property — used to render
-- values that any other editor has added to the file's MediaInfo entity
-- (treated as user-contributed metadata, displayed in the yellow box).
local function nonDplaStatements(entity, propertyId)
local out = {}
if not entity or not entity.statements then return out end
for _, stmt in ipairs(entity.statements[propertyId] or {}) do
if not isDplaDetermined(stmt) then
table.insert(out, stmt)
end
end
return out
end
-- Filter predicate inverse of isDplaDetermined, for passing as the
-- ``filterFn`` argument to reassembleChunkedValues when extracting the
-- non-DPLA (user-contributed) values.
local function isNonDplaDetermined(stmt)
return not isDplaDetermined(stmt)
end
-- The DPLA item id a DPLA-determined statement belongs to. Every DPLA
-- statement carries a reference {P123=DPLA, P854=https://dp.la/item/<id>,
-- P813=retrieved}; the owning id is parsed out of that P854 URL. One Commons
-- file can carry statements from several DPLA items (a cross-item duplicate:
-- distinct source items with byte-identical media merged onto one file), and
-- P854 is the ONLY per-statement link to the owning item — the P760 mainsnak
-- gives the id for the P760 statement alone. Returns the id string, or nil for
-- a legacy pre-redesign reference that carries P123 but no P854.
local function owningDplaId(stmt)
for _, ref in ipairs(stmt.references or {}) do
local isDpla = false
for _, snak in ipairs((ref.snaks or {})[P.PUBLISHER] or {}) do
if snak.snaktype == 'value'
and snak.datavalue
and snak.datavalue.value
and snak.datavalue.value.id == Q.DPLA then
isDpla = true
break
end
end
if isDpla then
for _, snak in ipairs((ref.snaks or {})[P.REFERENCE_URL] or {}) do
if snak.snaktype == 'value'
and snak.datavalue
and type(snak.datavalue.value) == 'string' then
local id = snak.datavalue.value:match('^https://dp%.la/item/(.+)$')
if id then return id end
end
end
return nil -- DPLA ref present but no parseable P854 (legacy)
end
end
return nil
end
-- Distinct DPLA item ids on a file, in first-appearance order, read from the
-- DPLA-determined P760 (DPLA ID) statements — one per contributing item. More
-- than one => a cross-item duplicate (render a separate blue box per id).
-- Falls back to the P760 mainsnak value when the reference has no P854
-- (legacy), so old single-item files still yield exactly one id.
local function dplaItemIds(entity)
local ids, seen = {}, {}
if not entity or not entity.statements then return ids end
for _, stmt in ipairs(entity.statements[P.DPLA_ID] or {}) do
if isDplaDetermined(stmt) then
local id = owningDplaId(stmt)
if not id and stmt.mainsnak and stmt.mainsnak.snaktype == 'value'
and stmt.mainsnak.datavalue then
id = stmt.mainsnak.datavalue.value
end
if id and not seen[id] then
seen[id] = true
table.insert(ids, id)
end
end
end
return ids
end
-- Predicate factory: DPLA-determined AND owned by ``itemId``. Drops into the
-- ``filterFn`` slot of reassembleChunkedValues (chunked title/description/
-- subject) so each per-item box reassembles only its own item's chunks — this
-- also prevents P1545 series-letter collisions between items (two items both
-- using series letter 'A' would otherwise be concatenated into one value).
local function isDplaDeterminedForItem(itemId)
return function(stmt)
return isDplaDetermined(stmt) and owningDplaId(stmt) == itemId
end
end
-- Statement-selector factory scoped to one item. Drops into the
-- ``statementsFn`` slot of creatorText/dateText/institutionQidOf/subjectsText.
local function dplaStatementsForItem(itemId)
return function(entity, propertyId)
local out = {}
if not entity or not entity.statements then return out end
for _, stmt in ipairs(entity.statements[propertyId] or {}) do
if isDplaDetermined(stmt) and owningDplaId(stmt) == itemId then
table.insert(out, stmt)
end
end
return out
end
end
-- Generic statement-selector factory from a per-statement predicate.
-- ``buildDplaRows`` receives one ``filterFn`` (the chunk-path predicate)
-- and derives the ``statementsFn`` the list-path helpers want from it, so a
-- single scoping decision drives both paths. ``statementsMatching
-- (isDplaDetermined)`` is equivalent to ``dplaStatements`` and
-- ``statementsMatching(isDplaDeterminedForItem(id))`` is equivalent to
-- ``dplaStatementsForItem(id)`` — same statements, in entity order.
local function statementsMatching(filterFn)
return function(entity, propertyId)
local out = {}
if not entity or not entity.statements then return out end
for _, stmt in ipairs(entity.statements[propertyId] or {}) do
if filterFn(stmt) then
table.insert(out, stmt)
end
end
return out
end
end
-- All values for a qualifier on a statement; skips somevalue/novalue.
local function qualifierValues(stmt, qualifierId)
local out = {}
if not stmt.qualifiers or not stmt.qualifiers[qualifierId] then return out end
for _, q in ipairs(stmt.qualifiers[qualifierId]) do
if q.snaktype == 'value' and q.datavalue then
table.insert(out, q.datavalue.value)
end
end
return out
end
local function firstQualifierValue(stmt, qualifierId)
local vals = qualifierValues(stmt, qualifierId)
return vals[1]
end
-- Render a string-typed snak. Returns nil if the snak is not a string
-- value (e.g. somevalue, novalue, or a different datatype).
local function renderStringSnak(stmt)
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'string' then
return ms.datavalue.value
end
return nil
end
-- Render an {{Unknown|<role>}} credit for a creator (P170) somevalue
-- statement that carries a P3831 (object of statement has role) qualifier
-- but no P2093 (author name string) — e.g. a community "unknown photographer"
-- migrated as P170 somevalue + P3831 = Q33231 (photographer). The English
-- role label is passed to {{Unknown}}, which lower-cases and localises it
-- ("Unknown photographer"). Returns nil when there is no role qualifier (the
-- caller then renders nothing, preserving prior behaviour); a nil frame
-- (internal callers without template access) yields a plain string.
local function unknownRoleText(stmt, frame)
local role = firstQualifierValue(stmt, P.ROLE)
if type(role) ~= 'table' or not role.id then return nil end
local label = mw.wikibase.getLabelByLang(role.id, 'en')
if not label or label == '' then return nil end
if frame then
return frame:expandTemplate{ title = 'Unknown', args = {label} }
end
return 'Unknown ' .. label
end
-- Render an entity-typed snak to its plain label (no link). Returns
-- nil for non-entity values; falls back to the bare Q-ID if the label
-- is missing in the current content language.
local function renderEntitySnak(stmt)
local ms = stmt.mainsnak
if ms.snaktype ~= 'value' or not ms.datavalue then return nil end
if ms.datavalue.type ~= 'wikibase-entityid' then return nil end
local id = ms.datavalue.value.id
return mw.wikibase.getLabel(id) or id, id
end
-- ============================================================
-- Chunked-claim reassembly
-- ============================================================
-- Parse a P1545 series-ordinal value (e.g. "A1", "B12", "AA3") into
-- (letter, ordinal_number). Returns nil for malformed or non-string
-- values, in which case the caller treats the statement as a singleton.
local function parseSeriesOrdinal(value)
if type(value) ~= 'string' then return nil end
local letter, ord = value:match('^([A-Z]+)(%d+)$')
if letter and ord then
return letter, tonumber(ord)
end
return nil
end
-- Reassemble chunked claims for one property into a list of logical
-- values. ``getText`` extracts the text content of one statement's
-- mainsnak (e.g. `function(stmt) return stmt.mainsnak.datavalue.value.text end`
-- for monolingualtext, or `function(stmt) return stmt.mainsnak.datavalue.value end`
-- for string). Returns nil text → statement is skipped.
--
-- Statements with a P1545 qualifier are grouped by series letter and
-- concatenated in ordinal order — that's one logical value per series.
-- Statements without P1545 are kept as singleton values. Output order
-- follows entity statement order: each series or singleton appears at
-- the position of its first statement in the entity.
--
-- Concatenation is direct: sdc-sync chunks values at non-whitespace
-- boundaries (so neither side of a chunk join carries leading or
-- trailing whitespace), and pre-normalizes the source value to match
-- Wikibase's server-side normalization before chunking. Reassembled
-- output is bytewise identical to the canonical form sdc-sync stored.
local function reassembleChunkedValues(entity, propertyId, getText, filterFn)
filterFn = filterFn or isDplaDetermined
if not entity or not entity.statements then return {} end
local statements = entity.statements[propertyId] or {}
local seriesData = {} -- letter -> { ordinal_number -> text }
local seenSeries = {} -- letter -> true (for first-seen ordering)
local order = {} -- list of {kind='singleton'|'series', key=text|letter}
for _, stmt in ipairs(statements) do
if filterFn(stmt) then
local text = getText(stmt)
if text then
local letter, ordinal = parseSeriesOrdinal(
firstQualifierValue(stmt, P.SERIES_ORDINAL)
)
if letter then
seriesData[letter] = seriesData[letter] or {}
seriesData[letter][ordinal] = text
if not seenSeries[letter] then
seenSeries[letter] = true
table.insert(order, {kind = 'series', key = letter})
end
else
table.insert(order, {kind = 'singleton', key = text})
end
end
end
end
local out = {}
for _, entry in ipairs(order) do
if entry.kind == 'singleton' then
table.insert(out, entry.key)
else
local data = seriesData[entry.key]
local ordinals = {}
for ord in pairs(data) do table.insert(ordinals, ord) end
table.sort(ordinals)
local parts = {}
for _, ord in ipairs(ordinals) do
table.insert(parts, data[ord])
end
table.insert(out, table.concat(parts))
end
end
return out
end
-- Text extractors for the two chunkable datatypes:
local function monolingualtextOf(stmt)
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'monolingualtext' then
return ms.datavalue.value.text
end
return nil
end
local function stringOf(stmt)
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'string' then
return ms.datavalue.value
end
return nil
end
-- ============================================================
-- Partnership helpers (P9126 role lookups)
-- ============================================================
-- Find the (first) non-DPLA partner Q-ID with a particular P3831 role.
-- Returns the Q-ID string, or nil if no qualifying statement is present.
local function partnerByRole(entity, roleQid)
if not entity then return nil end
for _, stmt in ipairs(dplaStatements(entity, P.PARTNERSHIP)) do
local matchesRole = false
for _, r in ipairs(qualifierValues(stmt, P.ROLE)) do
if r.id == roleQid then
matchesRole = true
break
end
end
if matchesRole then
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid'
and ms.datavalue.value.id ~= Q.DPLA then
return ms.datavalue.value.id
end
end
end
return nil
end
-- Catalog-record link as a NARA interwiki when applicable, else nil.
-- Commons has a ``nara:`` interwiki that resolves to
-- https://catalog.archives.gov/id/<NAID>; using it produces an
-- internal-styled link instead of the external-URL chevron, but only
-- when the file's contributing institution is NARA — other partners'
-- catalog URLs route to partner-specific systems and stay as plain
-- external URLs. The "hub partnership" Q-ID is now always DPLA itself
-- (it's filtered out by partnerByRole), so we gate on P195 instead.
local function naraCatalogInterwiki(entity, statementsFn, filterFn)
statementsFn = statementsFn or dplaStatements
filterFn = filterFn or isDplaDetermined
if not entity or not entity.statements then return nil end
local instQid
for _, stmt in ipairs(statementsFn(entity, P.INSTITUTION)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid' then
instQid = ms.datavalue.value.id
break
end
end
if instQid ~= Q.NARA then return nil end
local naids = reassembleChunkedValues(entity, P.NARA_IDENTIFIER, stringOf, filterFn)
local naid = naids[1]
if not naid or naid == '' then return nil end
return '[[nara:' .. naid .. '|' .. localizedLabel('catalog_record') .. ']]'
end
-- All non-DPLA partner Q-IDs with a particular P3831 role (some roles
-- can repeat across multiple statements, e.g. several contributing
-- institutions for a joint donation).
local function allPartnersByRole(entity, roleQid)
local out = {}
if not entity then return out end
for _, stmt in ipairs(dplaStatements(entity, P.PARTNERSHIP)) do
local matchesRole = false
for _, r in ipairs(qualifierValues(stmt, P.ROLE)) do
if r.id == roleQid then
matchesRole = true
break
end
end
if matchesRole then
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid'
and ms.datavalue.value.id ~= Q.DPLA then
table.insert(out, ms.datavalue.value.id)
end
end
end
return out
end
-- Look up a partner Q-ID's Commons category property (P8464). Returns
-- the category name (without "Category:" prefix), or nil if absent.
--
-- P8464's datatype is wikibase-item: the mainsnak's value is a Q-ID
-- pointing to a Wikidata item that *represents* the Commons category.
-- That item's commonswiki sitelink holds the actual category page
-- title (e.g. "Category:Media contributed by Denver Public Library").
-- We strip the "Category:" prefix so the caller can rebuild the link
-- consistently with the rest of the categories block.
--
-- A small number of older Wikidata items still have P8464 typed as a
-- bare string. We tolerate that legacy shape too.
local function commonsCategoryFor(qid)
if not qid or qid == '' then return nil end
local partnerEntity = mw.wikibase.getEntity(qid)
if not partnerEntity then return nil end
local statements = partnerEntity:getBestStatements(P.COMMONS_CATEGORY)
if not statements then return nil end
for _, stmt in ipairs(statements) do
local ms = stmt.mainsnak
if ms.snaktype == 'value' and ms.datavalue then
if ms.datavalue.type == 'wikibase-entityid' then
local catItemQid = ms.datavalue.value.id
local sitelink = mw.wikibase.getSitelink(catItemQid, 'commonswiki')
if sitelink then
-- Parenthesize to drop gsub's count return value.
return (sitelink:gsub('^Category:', ''))
end
elseif ms.datavalue.type == 'string' then
return ms.datavalue.value
end
end
end
return nil
end
-- Look up an image-typed property (P154 / P18) on a partner Q-ID and
-- return its File: page name (with the "File:" prefix), or nil.
local function imageFileFor(qid, propertyId)
if not qid or qid == '' then return nil end
local partnerEntity = mw.wikibase.getEntity(qid)
if not partnerEntity then return nil end
local statements = partnerEntity:getBestStatements(propertyId)
if not statements then return nil end
for _, stmt in ipairs(statements) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'string' then
return 'File:' .. ms.datavalue.value
end
end
return nil
end
-- ============================================================
-- Public functions
-- ============================================================
--- p.hub — derive the regional DPLA hub's display name.
function p.hub(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
local qid = partnerByRole(entity, Q.ROLE_AGGREGATOR)
if not qid then return '' end
return mw.wikibase.getLabel(qid) or qid
end
--- p.hub_qid — the bare Q-ID of the hub (no label resolution).
function p.hub_qid(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
return partnerByRole(entity, Q.ROLE_AGGREGATOR) or ''
end
--- p.institution_qid — the Q-ID of the file's institution (P195).
function p.institution_qid(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
if not entity then return '' end
for _, stmt in ipairs(dplaStatements(entity, P.INSTITUTION)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid' then
return ms.datavalue.value.id
end
end
return ''
end
--- p.dpla_id — the file's DPLA item ID (P760 string).
-- The DPLA ID is 32 chars so it's never chunked in practice, but we
-- route through the chunking-aware reassembler for uniformity.
function p.dpla_id(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
local values = reassembleChunkedValues(entity, P.DPLA_ID, stringOf)
return values[1] or ''
end
--- p.source_catalog_url — URL to the item record on the source catalog.
function p.source_catalog_url(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
if not entity then return '' end
for _, stmt in ipairs(dplaStatements(entity, P.SOURCE_OF_FILE)) do
local url = firstQualifierValue(stmt, P.DESCRIBED_AT_URL)
if type(url) == 'string' and url ~= '' then
return url
end
end
return ''
end
-- The DPLA-authored P7482 statement carries three URL qualifiers that
-- together describe where this file's source bits live. ``sourceUrls``
-- pulls the first DPLA-authored P7482 and returns each URL (any may be
-- nil):
-- * fileUrl — P2699 (URL) qualifier, per-ordinal direct download
-- URL materialized by sdc-sync from file-list.txt
-- * iiifUrl — P6108 (IIIF manifest URL) qualifier, present only
-- when the source DPLA item ships an iiifManifest
-- * catalogUrl — P973 (described at URL) qualifier, the partner
-- catalog page for this DPLA item
local function sourceUrls(entity, statementsFn)
statementsFn = statementsFn or dplaStatements
if not entity then return nil, nil, nil end
for _, stmt in ipairs(statementsFn(entity, P.SOURCE_OF_FILE)) do
local function strQual(qid)
local v = firstQualifierValue(stmt, qid)
if type(v) == 'string' and v ~= '' then return v end
return nil
end
return strQual('P2699'), strQual('P6108'), strQual(P.DESCRIBED_AT_URL)
end
return nil, nil, nil
end
--- p.source_field — render the Information template's Source row as
-- icon-prefixed links to the file URL, IIIF manifest, and catalog
-- record. Items missing from the SDC are skipped silently; the row
-- shows however many of the three are present, joined with " | ".
-- Returns empty when none are present so the Information template's
-- Source row is hidden entirely.
--
-- Icon images (rendered at 24px wide inline):
-- * File URL : Codex icon download color-progressive.svg
-- * IIIF : International Image Interoperability Framework logo.png
-- * Catalog rec.: Breezeicons-actions-22-database-index.svg
function p.source_field(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
local fileUrl, iiifUrl, catalogUrl = sourceUrls(entity)
local cells = {}
if fileUrl then
table.insert(cells,
'[[File:Codex icon download color-progressive.svg|24px|alt=|class=skin-invert|link=]] [' ..
fileUrl .. ' ' .. localizedLabel('file_url') .. ']'
)
end
if iiifUrl then
table.insert(cells,
'[[File:International Image Interoperability Framework logo.png|24px|alt=|link=]] [' ..
iiifUrl .. ' ' .. localizedLabel('iiif_manifest') .. ']'
)
end
if catalogUrl then
local naraLink = naraCatalogInterwiki(entity)
if naraLink then
table.insert(cells,
'[[File:Breezeicons-actions-22-database-index.svg|24px|alt=|class=skin-invert|link=]] ' ..
naraLink
)
else
table.insert(cells,
'[[File:Breezeicons-actions-22-database-index.svg|24px|alt=|class=skin-invert|link=]] [' ..
catalogUrl .. ' ' .. localizedLabel('catalog_record') .. ']'
)
end
end
local parts = {}
for i, cell in ipairs(cells) do
local style = 'display:inline-block; padding: 4px 16px; vertical-align: middle;'
if i > 1 then
style = style .. ' border-left: 2px solid #a2a9b1;'
end
table.insert(parts, '<span style="' .. style .. '">' .. cell .. '</span>')
end
return table.concat(parts)
end
--- p.partner_logo — File: page name for the partner logo.
function p.partner_logo(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
if not entity then return DEFAULT_PARTNER_LOGO end
local instQid
for _, stmt in ipairs(dplaStatements(entity, P.INSTITUTION)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid' then
instQid = ms.datavalue.value.id
break
end
end
if not instQid then return DEFAULT_PARTNER_LOGO end
return imageFileFor(instQid, P.LOGO_IMAGE)
or imageFileFor(instQid, P.IMAGE)
or DEFAULT_PARTNER_LOGO
end
--- p.values — generic getter for all DPLA-determined values of a property.
-- Handles the data types DPLA SDC uses for displayable fields. For
-- string and monolingualtext, chunked values are transparently
-- reassembled via the P1545 series-ordinal convention.
--
-- Optional named args:
-- raw=1 — for entity values, return the bare Q-ID instead of the label
-- sep=… — separator between values (default ", ")
function p.values(frame)
local args = (frame and frame.args) or {}
local propId = args[1]
if not propId or propId == '' then return '' end
local entity = getEntity(args)
if not entity then return '' end
local raw = args.raw == '1' or args.raw == 'true'
local sep = args.sep or ', '
local out = {}
-- Sniff the first DPLA-determined statement's datatype to pick the
-- right rendering path.
local stmts = dplaStatements(entity, propId)
if #stmts == 0 then return '' end
local firstMS = stmts[1].mainsnak
local datatype = firstMS.datavalue and firstMS.datavalue.type or nil
if datatype == 'string' then
return table.concat(reassembleChunkedValues(entity, propId, stringOf), sep)
elseif datatype == 'monolingualtext' then
return table.concat(reassembleChunkedValues(entity, propId, monolingualtextOf), sep)
end
-- Non-chunkable datatypes — render per-statement.
for _, stmt in ipairs(stmts) do
local ms = stmt.mainsnak
if ms.snaktype == 'value' and ms.datavalue then
local t = ms.datavalue.type
if t == 'wikibase-entityid' then
local id = ms.datavalue.value.id
if raw then
table.insert(out, id)
else
table.insert(out, mw.wikibase.getLabel(id) or id)
end
elseif t == 'time' then
local v = ms.datavalue.value
local timeStr = (v.time or ''):gsub('^%+', '')
local precision = v.precision or 11
if precision >= 11 then
table.insert(out, timeStr:sub(1, 10))
elseif precision == 10 then
table.insert(out, timeStr:sub(1, 7))
else
table.insert(out, timeStr:sub(1, 4))
end
end
end
end
return table.concat(out, sep)
end
--- p.first_qid — bare Q-ID of the first DPLA-determined entity-valued
-- statement for the given property.
function p.first_qid(frame)
local args = (frame and frame.args) or {}
local propId = args[1]
if not propId or propId == '' then return '' end
local entity = getEntity(args)
if not entity then return '' end
for _, stmt in ipairs(dplaStatements(entity, propId)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid' then
return ms.datavalue.value.id
end
end
return ''
end
--- p.titles — all DPLA-determined title (P1476) values.
-- Reassembles chunked titles (P1545 series-ordinal) into one logical
-- value per series; singletons preserved as their own values.
function p.titles(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
return table.concat(
reassembleChunkedValues(entity, P.TITLE, monolingualtextOf),
args.sep or ', '
)
end
--- p.subjects — union of entity subjects (P921) and string subjects (P4272).
-- P4272 string subjects are chunking-aware; P921 entity subjects render
-- as Wikidata labels. Deduplicated by exact text match.
function p.subjects(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
if not entity then return '' end
local seen = {}
local out = {}
for _, stmt in ipairs(dplaStatements(entity, P.SUBJECT_ENTITY)) do
local label = renderEntitySnak(stmt)
-- Case-insensitive dedup key: a P921 entity label and a P4272
-- string term that differ only in case (e.g. "assassination" vs
-- "Assassination") are the same subject to a reader.
local key = label and mw.ustring.lower(label)
if label and not seen[key] then
seen[key] = true
table.insert(out, label)
end
end
for _, s in ipairs(reassembleChunkedValues(entity, P.SUBJECT_STRING, stringOf)) do
local key = mw.ustring.lower(s)
if not seen[key] then
seen[key] = true
table.insert(out, s)
end
end
return table.concat(out, args.sep or ', ')
end
--- p.creator — render the creator(s).
function p.creator(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
if not entity then return '' end
local out = {}
for _, stmt in ipairs(dplaStatements(entity, P.CREATOR)) do
local ms = stmt.mainsnak
if ms.snaktype == 'somevalue' then
local name = firstQualifierValue(stmt, P.AUTHOR_NAME_STRING)
if type(name) == 'string' then
table.insert(out, name)
else
local u = unknownRoleText(stmt, frame)
if u then table.insert(out, u) end
end
elseif ms.snaktype == 'value' and ms.datavalue then
if ms.datavalue.type == 'string' then
table.insert(out, ms.datavalue.value)
elseif ms.datavalue.type == 'wikibase-entityid' then
local id = ms.datavalue.value.id
-- QID → ``{{creator|Wikidata=Q…}}`` Commons Creator
-- infobox, matching hand-authored ``{{Creator:Foo}}``
-- wikitext. See ``creatorText`` for the row-render
-- variant used by buildDplaRows / buildUserRows.
table.insert(out, frame:expandTemplate{
title = 'creator',
args = {Wikidata = id},
})
end
end
end
return table.concat(out, args.sep or ', ')
end
--- p.date — render the date(s).
-- For each DPLA-determined P571 statement:
-- * value-typed time: format with the snak's recorded precision
-- (day → YYYY-MM-DD, month → YYYY-MM, year → YYYY).
-- * somevalue + P1932 (stated as): use the qualifier text — the
-- legacy shape for unparseable date strings.
-- "Approximate" parsed dates carry a P1480 (sourcing circumstances)
-- qualifier with value Q5727902 (circa); we prefix those with "circa ".
function p.date(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
if not entity then return '' end
local out = {}
for _, stmt in ipairs(dplaStatements(entity, P.DATE)) do
local ms = stmt.mainsnak
local isCirca = false
for _, sc in ipairs(qualifierValues(stmt, P.SOURCING_CIRCUMSTANCES)) do
if type(sc) == 'table' and sc.id == Q.CIRCA then
isCirca = true
break
end
end
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'time' then
local v = ms.datavalue.value
local timeStr = (v.time or ''):gsub('^%+', '')
local precision = v.precision or 11
local rendered
if precision >= 11 then
rendered = timeStr:sub(1, 10)
elseif precision == 10 then
rendered = timeStr:sub(1, 7)
elseif precision == 9 then
rendered = timeStr:sub(1, 4)
elseif precision == 8 then
-- decade
rendered = timeStr:sub(1, 3) .. '0s'
else
rendered = timeStr:sub(1, 4)
end
if isCirca then
rendered = localizedLabel('circa') .. ' ' .. rendered
end
table.insert(out, rendered)
elseif ms.snaktype == 'somevalue' then
local stated = firstQualifierValue(stmt, P.STATED_AS)
if type(stated) == 'string' then
table.insert(out, stated)
elseif type(stated) == 'table' and stated.text then
table.insert(out, stated.text)
end
end
end
return table.concat(out, args.sep or ', ')
end
--- p.local_id — render the local identifier with the appropriate label.
-- Reassembles chunked P217 / P1225 values (long source identifiers)
-- before rendering.
function p.local_id(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
if not entity then return '' end
if p.hub(frame) == NARA_HUB_LABEL then
local naids = reassembleChunkedValues(entity, P.NARA_IDENTIFIER, stringOf)
if #naids > 0 then
return '* National Archives Identifier: ' .. table.concat(naids, ', ')
end
return ''
end
local locals = reassembleChunkedValues(entity, P.LOCAL_IDENTIFIER, stringOf)
if #locals > 0 then
local instLabel
for _, inst in ipairs(dplaStatements(entity, P.INSTITUTION)) do
instLabel = renderEntitySnak(inst)
if instLabel then break end
end
local joined = table.concat(locals, ', ')
if instLabel and instLabel ~= '' then
return '* ' .. instLabel .. ' identifier: ' .. joined
end
return '* Identifier: ' .. joined
end
return ''
end
-- ============================================================
-- Multipage navigation (P304 on P760)
-- ============================================================
-- All P304 page numbers (integers, deduped, sorted ascending) carried as
-- qualifiers on ONE P760 statement. A single DPLA-item statement can now
-- carry several P304 qualifiers — an in-item duplicate, where one Commons
-- file occupies multiple page positions in the same DPLA item. Returns {}
-- when the statement has no integer P304 qualifier. Per-statement so a
-- per-item box could navigate only its own item's pages.
local function pageNumbersOfStatement(stmt)
local seen, out = {}, {}
for _, v in ipairs(qualifierValues(stmt, P.PAGE)) do
if type(v) == 'string' then
local n = tonumber(v)
if n and not seen[n] then
seen[n] = true
table.insert(out, n)
end
end
end
table.sort(out)
return out
end
-- Every P304 page number this file occupies within ONE DPLA item — the
-- first DPLA-determined P760 statement (matching ``filterFn``, default
-- ``isDplaDetermined``) that carries at least one page — deduped and
-- sorted. Returns {} when no P304 qualifier is present, i.e. the file is a
-- singleton in its extension series and the "Other pages" field should be
-- omitted; this reproduces the old ``currentPageNumber`` selection ("first
-- P304 of the first DPLA P760 with a page").
--
-- Deliberately reads a SINGLE statement rather than aggregating across a
-- cross-item file's several P760 statements: the current file's title
-- encodes one item's page position, and title-based prev/next navigation
-- only makes sense within that one item's series — mixing another item's
-- page numbers in would emit sibling links in the wrong series. A per-item
-- box can pass ``isDplaDeterminedForItem(id)`` to navigate exactly that
-- item's pages; an in-item duplicate carries all its pages on one P760
-- statement, so a single statement is the complete set for that item.
local function allPageNumbers(entity, filterFn)
filterFn = filterFn or isDplaDetermined
if not entity or not entity.statements then return {} end
for _, stmt in ipairs(entity.statements[P.DPLA_ID] or {}) do
if filterFn(stmt) then
local nums = pageNumbersOfStatement(stmt)
if #nums > 0 then return nums end
end
end
return {}
end
-- Build a sibling file's title by swapping the "(page N)" suffix in the
-- current file's title. Returns the candidate title string (without
-- the "File:" prefix), or nil if the current title doesn't carry a
-- "(page N)" suffix at all (in which case we can't find siblings by
-- title-pattern enumeration).
local function siblingTitleFor(currentTitle, newPageNumber)
if not currentTitle then return nil end
-- Strip File: prefix if present.
local base = currentTitle:gsub('^File:', '')
-- Match "...(page N).ext" — preserve the bits before/after.
local prefix, _, ext = base:match('^(.-)%(page (%d+)%)(%..+)$')
if not prefix then return nil end
return prefix .. '(page ' .. tostring(newPageNumber) .. ')' .. ext
end
-- Resolve a sibling File: title to an mw.title object when the page exists
-- (redirects count as existing), or nil otherwise.
local function siblingExists(siblingBaseTitle)
if not siblingBaseTitle then return nil end
local title = mw.title.makeTitle('File', siblingBaseTitle)
if title and title.exists then
return title
end
return nil
end
-- True when a sibling ordinal is really THIS page wearing another page
-- number -- either the identical title, or a redirect that resolves back to
-- the current file. Such an ordinal carries no new content (it is an in-item
-- duplicate merged into this page under the SHA1-uniqueness rules); rendering
-- it would show the current file's own thumbnail and loop the reader back to
-- the canonical, so it must be skipped.
local function resolvesToSelf(siblingTitleObj, currentTitle)
if not siblingTitleObj then return false end
if siblingTitleObj.fullText == currentTitle then return true end
if siblingTitleObj.isRedirect then
local target = siblingTitleObj.redirectTarget
if target and target.fullText == currentTitle then return true end
end
return false
end
-- Walk outward from the current ordinal in `step` direction (+1 for next,
-- -1 for prev), skipping any ordinal that resolves back to this same file,
-- until a genuinely different page is found. Returns (siblingTitleObj,
-- ordinal) for the first such page, or nil when the run of pages ends (a
-- missing ordinal) before one is found. A sibling that redirects to some
-- OTHER page of the item is a valid target: its thumbnail and link resolve
-- through the redirect and we label it by its own ordinal -- merged pages
-- behave like ordinary pages in every respect but the shared description.
local function findNavSibling(currentTitle, currentOrdinal, step)
local i = currentOrdinal + step
while i >= 1 do
local sib = siblingExists(siblingTitleFor(currentTitle, i))
if not sib then return nil end -- ran off the end of the item
if not resolvesToSelf(sib, currentTitle) then
return sib, i
end
i = i + step -- self-duplicate ordinal: skip
end
return nil
end
-- Render one prev/next thumbnail cell for a resolved sibling. `arrow` is
-- 'prev' or 'next' (controls the arrow side); `ordinal` labels the cell.
local function navCell(sib, ordinal, arrow)
local label = substituteParams(localizedLabel('page_n'), tostring(ordinal))
local thumb = '[[File:' .. sib.text .. '|100px|alt=' .. label ..
'|link=' .. sib.fullText .. ']]'
local link = '[[:' .. sib.fullText .. '|' .. label .. ']]'
local caption
if arrow == 'prev' then
caption = '<span style="font-size:1.2em;" aria-hidden="true">←</span> ' .. link
else
caption = link .. ' <span style="font-size:1.2em;" aria-hidden="true">→</span>'
end
return '<div style="display:inline-block;text-align:center;margin:0 8px;">' ..
thumb .. '<br/>' .. caption .. '</div>'
end
--- p.other_pages -- render the "Other pages" navigation cell.
-- For a page of a multi-page DPLA item, show thumbnails of the previous and
-- next pages, each linking to that page. Sibling discovery uses the DPLA
-- uploader's title convention ``... - DPLA - <dpla_id> (page N).<ext>``: the
-- current file's own ordinal is read from its "(page N)" title and the
-- neighbouring ordinals are probed by title.
--
-- In-item duplicates are merged under the SHA1-uniqueness rules: the
-- duplicate page becomes a redirect to the page it duplicates. Navigation
-- treats those redirect ordinals as ordinary pages EXCEPT that an ordinal
-- resolving back to the current file (a duplicate of THIS page) is skipped --
-- otherwise the cell would display this page's own thumbnail and loop the
-- reader back to the canonical. ``findNavSibling`` skips such ordinals and
-- continues outward to the next genuinely different page, so the reader can
-- still click up and down the item. When every other ordinal resolves to
-- this page (e.g. an item whose remaining pages are all duplicates of page
-- 1), neither direction yields a sibling and the field is suppressed.
--
-- Returns '' when the file is not part of a paged item, when its title has
-- no "(page N)" ordinal, or when no navigable sibling exists.
function p.other_pages(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
if #allPageNumbers(entity) == 0 then return '' end
local titleObj
if args.page and args.page ~= '' then
titleObj = mw.title.new(args.page)
else
titleObj = mw.title.getCurrentTitle()
end
if not titleObj then return '' end
local currentTitle = titleObj.fullText
-- Current file's own ordinal, from its "(page N)" title suffix.
local ordStr = currentTitle:gsub('^File:', ''):match('%(page (%d+)%)%..+$')
if not ordStr then return '' end
local currentOrdinal = tonumber(ordStr)
local cells = {}
local prevSib, prevOrd = findNavSibling(currentTitle, currentOrdinal, -1)
if prevSib then
table.insert(cells, navCell(prevSib, prevOrd, 'prev'))
end
local nextSib, nextOrd = findNavSibling(currentTitle, currentOrdinal, 1)
if nextSib then
table.insert(cells, navCell(nextSib, nextOrd, 'next'))
end
if #cells == 0 then return '' end
return table.concat(cells, ' ')
end
--- p.categories — emit the full set of DPLA partner / institution
-- categories for this file. Three categories, matching the legacy
-- ``{{DPLA}}`` template's tracking output:
--
-- * MAIN_CAT — always
-- * Hub category — the regional aggregator (e.g. "Plains to Peaks
-- Collective"). Identified by a non-DPLA P9126 statement carrying
-- ``P3831 = Q393351`` (ROLE_AGGREGATOR); ``partnerByRole`` filters
-- out DPLA itself so the remaining publisher is the regional
-- hub. For partners where the contributing institution is also
-- its own hub (NARA, Smithsonian), the bot omits the redundant
-- hub-ROLE_AGGREGATOR entry, so we fall back to the institution
-- Q-ID — same category on Commons, no spurious "unknown partner"
-- classification.
-- * Institution category — the contributing institution, read from
-- ``P195`` (collection). Falls back to UNKNOWN_INSTITUTION_CAT
-- when the P195 entity has no Commons sitelink.
-- Render the three "Media contributed by..." category links from a
-- pre-resolved (hubQid, instQid) pair. Either argument may be nil,
-- in which case its slot falls back to the UNKNOWN_* maintenance
-- category — same shape the legacy ``{{DPLA/hub cat}}`` /
-- ``{{DPLA/inst cat}}`` templates emitted, preserved here so existing
-- maintenance queries keep finding their files.
local function buildCategoryList(hubQid, instQid)
local parts = {'[[Category:' .. MAIN_CAT .. ']]'}
local hubCat = commonsCategoryFor(hubQid)
if hubCat then
table.insert(parts, '[[Category:' .. hubCat .. ']]')
else
table.insert(parts, '[[Category:' .. UNKNOWN_PARTNER_CAT .. ']]')
end
local instCat = commonsCategoryFor(instQid)
if instCat then
table.insert(parts, '[[Category:' .. instCat .. ']]')
else
table.insert(parts, '[[Category:' .. UNKNOWN_INSTITUTION_CAT .. ']]')
end
return table.concat(parts, '\n')
end
-- Resolve (hubQid, instQid) for category emission from an SDC entity.
-- The DPLA SDC writer encodes the partnership graph across two
-- properties:
-- * ``P195`` (collection) — usually the hub-level Q-ID, kept for
-- legacy ``{{DPLA/hub cat}}`` compatibility.
-- * ``P9126`` (object of statement has role) — three entries per
-- file: DPLA itself (role = ROLE_AGGREGATOR), the hub (role =
-- ROLE_REPOSITORY), and the granular contributing institution
-- (role = ROLE_CONTRIBUTING). The contributing institution can
-- be more specific than P195 — e.g. ``National Archives at
-- College Park - Cartographic`` versus the parent ``National
-- Archives and Records Administration`` in P195.
-- Prefer the role-tagged P9126 values so the category emission lines
-- up with the granular institution where one exists. Fall back to
-- P195 when the role-tagged entries are absent (older bot writes,
-- partner-equals-institution cases like NARA-direct uploads).
local function resolveCategoryQidsFromEntity(entity)
if not entity then return nil, nil end
-- P195 (collection) always carries the contributing institution
-- Q-ID — every code path that writes SDC writes one, and where
-- no separate hub exists (direct uploads) the institution is also
-- the hub.
local p195Qid
for _, stmt in ipairs(dplaStatements(entity, P.INSTITUTION)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid' then
p195Qid = ms.datavalue.value.id
break
end
end
-- The hub is the non-DPLA, non-institution Q-ID in the P9126
-- partnership chain. Role qualifiers (P3831) are deliberately
-- ignored here: the two hub models tag roles differently — a
-- service hub's hub is an aggregator, a content hub's hub is a
-- repository — so no single role test finds the hub across both.
-- Set-based identification gets the correct hub Q-ID regardless
-- of model.
local hubQid
for _, stmt in ipairs(dplaStatements(entity, P.PARTNERSHIP)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid' then
local qid = ms.datavalue.value.id
if qid ~= Q.DPLA and qid ~= p195Qid then
hubQid = qid
break
end
end
end
-- A content hub (NARA, Smithsonian) records its granular contributing
-- institution on a P9126 partner carrying the custodial-unit role.
-- When present, P195 holds the *hub* (the content hub itself, e.g.
-- NARA) and this partner is the institution — e.g. P195 = NARA,
-- contributing = "John F. Kennedy Presidential Library & Museum". A
-- service hub never sets this role, so its institution stays on P195
-- (handled below).
local contributingQid = partnerByRole(entity, Q.ROLE_CONTRIBUTING)
if contributingQid then
return p195Qid, contributingQid
end
-- Fall back to institution-is-its-own-hub when no separate hub
-- Q-ID is present (BPL-direct, NARA-direct files where DPLA
-- flattened the partnership). buildCategoryList renders a single
-- category in that case via MediaWiki's natural categorylinks
-- dedup.
return hubQid or p195Qid, p195Qid
end
function p.categories(frame)
local args = (frame and frame.args) or {}
local entity = getEntity(args)
if not entity then return '' end
local hubQid, instQid = resolveCategoryQidsFromEntity(entity)
return buildCategoryList(hubQid, instQid)
end
-- ============================================================
-- Custom metadata-table rendering (replaces {{Information}})
-- ============================================================
-- Edit-pencil affordances are intentionally NOT emitted. DPLA SDC is
-- authoritative and synced from the source catalog by the bot; we don't
-- want to invite ad-hoc edits to the rendered fields. The propertyId
-- argument to ``tableRow`` is preserved for forward compatibility but
-- currently unused.
-- HTML id="..." attributes that Commons' machine-readable scanners
-- (including AntiCompositeBot's NoLicense tagger and external
-- file-metadata consumers) look for to extract structured per-file
-- fields. Each canonical row in the rendered table carries the
-- matching id so a file rendered through Module:DPLA stays out of
-- the ``Files with no machine-readable <field>`` maintenance
-- categories. Keys here mirror the label keys passed to ``add()``
-- in ``buildDplaRows`` / ``buildUserRows``. See
-- https://commons.wikimedia.org/wiki/Commons:Machine-readable_data
-- for the full list of recognised ids and the bot scanners that
-- key on them.
local MACHINE_READABLE_ID = {
description = 'fileinfotpl_desc',
creator = 'fileinfotpl_aut',
date = 'fileinfotpl_date',
source = 'fileinfotpl_src',
permission = 'fileinfotpl_perm',
}
local function tableRow(label, value, _propertyId, fieldIdAttr)
if value == nil or value == '' then return nil end
-- The fileinfotpl_* machine-readable id must sit on the LABEL cell, not
-- the value cell. Commons' CommonsMetadata extension (which populates the
-- ``Files with no machine-readable <field>`` categories and the
-- ``extmetadata`` API) locates the element by id and then reads its NEXT
-- SIBLING cell as the field value — mirroring {{Information}}, whose id
-- sits on the "Author"/"Source"/… label cell. With the id on the value
-- cell its sibling is empty, so every field extracted as nothing and the
-- file landed in the maintenance categories regardless of content.
local idAttr = fieldIdAttr and (' id="' .. fieldIdAttr .. '"') or ''
return '<tr style="vertical-align: top">'
.. '<td class="fileinfo-paramfield"' .. idAttr .. '>' .. label .. '</td>'
.. '<td>' .. value .. '</td>'
.. '</tr>'
end
-- Top banner row mirroring {{Artwork}}'s title header: the file's title
-- spans the full table width, centered, in larger and bolder type.
-- Returns nil when no title can be derived so the dispatcher can omit
-- the banner row.
local function titleBanner(title)
if not title or title == '' then return nil end
return '<tr><th colspan="2" style="text-align:center; font-size:1.25em; padding:0.6em; '
.. 'background-color:var(--background-color-interactive, #e0e0ee); color:inherit;">'
.. title .. '</th></tr>'
end
-- Each field-text helper accepts an optional ``statementsFn`` that
-- selects which statements to render (``dplaStatements`` by default,
-- ``nonDplaStatements`` for the yellow user-contributed box). The
-- ``filterFn`` parallel argument is used inside chunked-value
-- reassembly, which iterates the raw statement list itself.
local function titlesText(entity, statementsFn, filterFn)
filterFn = filterFn or isDplaDetermined
return table.concat(reassembleChunkedValues(entity, P.TITLE, monolingualtextOf, filterFn), ', ')
end
local function descriptionText(entity, statementsFn, filterFn)
filterFn = filterFn or isDplaDetermined
-- Multi-value descriptions get a blank-line break between values,
-- not commas — descriptions are often paragraph-length, and a
-- comma between two paragraphs reads as one continuous wall of
-- text. ``reassembleChunkedValues`` already concatenates chunked
-- single-values (statements sharing a P1545 ordinal letter
-- prefix, used to bypass the per-statement string-length limit)
-- into one entry, so the separator only fires between truly
-- distinct descriptions.
return table.concat(
reassembleChunkedValues(entity, P.DESCRIPTION, monolingualtextOf, filterFn),
'<br /><br />'
)
end
-- ``frame`` is threaded so a P170 QID-valued statement can expand
-- ``{{creator|Wikidata=Q…}}`` — the Commons Creator infobox —
-- matching what a hand-authored ``{{Creator:Foo}}`` wikitext link
-- would render on the file page. Mirrors the ``Institution`` row's
-- expandTemplate pattern already used elsewhere in this module.
-- Without ``frame`` (test harnesses that call the helper directly),
-- falls back to ``mw.wikibase.getLabel`` — bare label instead of
-- the full infobox.
local function creatorText(entity, statementsFn, frame)
statementsFn = statementsFn or dplaStatements
if not entity then return '' end
local out = {}
for _, stmt in ipairs(statementsFn(entity, P.CREATOR)) do
local ms = stmt.mainsnak
if ms.snaktype == 'somevalue' then
local name = firstQualifierValue(stmt, P.AUTHOR_NAME_STRING)
if type(name) == 'string' then
table.insert(out, name)
else
local u = unknownRoleText(stmt, frame)
if u then table.insert(out, u) end
end
elseif ms.snaktype == 'value' and ms.datavalue then
if ms.datavalue.type == 'string' then
table.insert(out, ms.datavalue.value)
elseif ms.datavalue.type == 'wikibase-entityid' then
local id = ms.datavalue.value.id
if frame then
table.insert(out, frame:expandTemplate{
title = 'creator',
args = {Wikidata = id},
})
else
table.insert(out, mw.wikibase.getLabel(id) or id)
end
end
end
end
return table.concat(out, ', ')
end
-- Format an ISO-prefix date string ("YYYY-MM-DD" or "YYYY-MM") to a
-- human-readable form via the content language's date formatter (e.g.
-- "1967-03-28" → "28 March 1967", matching the {{Artwork}} renderer's
-- output). Falls back to the raw ISO string on any formatter error so a
-- BC date or other unsupported input doesn't blow up the whole row.
local function formatWikibaseTime(timeStr, precision)
if not timeStr or timeStr == '' then return '' end
local lang = mw.language.getContentLanguage()
local fmt, input
if precision >= 11 then
fmt = 'j F Y'
input = timeStr:sub(1, 10)
elseif precision == 10 then
-- formatDate parses "YYYY-MM-01" reliably; fabricating the day
-- doesn't affect the "F Y" output (month + year only).
fmt = 'F Y'
input = timeStr:sub(1, 7) .. '-01'
elseif precision == 9 then
return timeStr:sub(1, 4)
elseif precision == 8 then
return timeStr:sub(1, 3) .. '0s'
else
return timeStr:sub(1, 4)
end
-- Reject BC / year-zero values; ``formatDate`` accepts them but the
-- output for negative years is platform-dependent. Show raw instead.
if input:sub(1, 1) == '-' or input:sub(1, 4) == '0000' then
return timeStr:sub(1, #input)
end
local ok, result = pcall(function() return lang:formatDate(fmt, input) end)
if ok and result and result ~= '' then return result end
return timeStr:sub(1, #input)
end
-- Parse a year-range string into (start_year, end_year) with
-- start <= end, or return nil for non-range / unparseable inputs.
-- Mirror of ingest_wikimedia.sdc.parse_date_range; the Python side
-- uses this same canonicalisation when deciding whether a legacy
-- ``{{Artwork}}`` date claim is a duplicate of DPLA's own range
-- claim, and the SDC reconciler uses it to remove inferred-from-
-- Wikitext claims that are now equivalent to a DPLA-sourced one.
-- Module:DPLA mirrors the canonicalisation at render time so the
-- yellow box stays clean between sync runs.
--
-- Recognised shapes (post wikitext-expansion or raw):
-- * YYYY [- / – —] YYYY
-- * between YYYY and YYYY (case-insensitive)
-- * {{other date|between|YYYY|YYYY}} (raw wikitext fallback)
--
-- Lua patterns have no alternation or {n,m} quantifier — captures
-- pull "one or more digits" and a helper validates the year is
-- plausible (100..9999) so e.g. "5-100" can't pretend to be a year
-- range. Year 0 is rejected for the same reason parse_dpla_date
-- rejects it: proleptic Gregorian has no year 0.
local function _yearRange(a, b)
local ya, yb = tonumber(a), tonumber(b)
if not ya or not yb then return nil end
if ya < 100 or ya > 9999 then return nil end
if yb < 100 or yb > 9999 then return nil end
if ya > yb then ya, yb = yb, ya end
return ya, yb
end
local function parseDateRange(s)
if type(s) ~= 'string' or s == '' then return nil end
local trimmed = s:gsub('^%s+', ''):gsub('%s+$', '')
if trimmed == '' then return nil end
local lower = trimmed:lower()
-- "between X and Y" (lowercased input → ASCII pattern)
local a, b = lower:match('^between%s+(%d+)%s+and%s+(%d+)$')
if a then
local ya, yb = _yearRange(a, b)
if ya then return ya, yb end
end
-- "{{other date|between|X|Y}}" — tolerates ``other_date`` alias
-- and optional whitespace around the pipes.
a, b = lower:match('^{{%s*other[ _]date%s*|%s*between%s*|%s*(%d+)%s*|%s*(%d+)%s*}}$')
if a then
local ya, yb = _yearRange(a, b)
if ya then return ya, yb end
end
-- "YYYY [sep] YYYY" — Lua patterns can't alternate over
-- separators, so try each one. ``-`` and ``/`` are ASCII; ``–``
-- (en-dash) and ``—`` (em-dash) are multi-byte UTF-8 sequences
-- that Lua's byte-oriented patterns still match literally.
for _, sep in ipairs({'%-', '/', '–', '—'}) do
a, b = trimmed:match('^(%d+)%s*' .. sep .. '%s*(%d+)$')
if a then
local ya, yb = _yearRange(a, b)
if ya then return ya, yb end
end
end
return nil
end
-- Produce a comparable key for a P571 statement that compares equal
-- across equivalent encodings (value-typed time vs. somevalue+P1932
-- single date, range-shaped P1932 across format variants, raw P1932
-- prose strings). Mirror of ``_statement_comparable_value`` in
-- tools/sdc_sync.py; used to suppress non-DPLA P571 statements whose
-- value duplicates a DPLA-attributed one in the yellow box.
--
-- Returns nil for shapes we can't compare — caller treats nil as
-- "render normally", not as a match.
local function dateStatementComparable(stmt)
local ms = stmt and stmt.mainsnak
if not ms then return nil end
if ms.snaktype == 'value' and ms.datavalue
and ms.datavalue.type == 'time' then
local v = ms.datavalue.value or {}
local circa = ''
for _, q in ipairs((stmt.qualifiers or {}).P1480 or {}) do
if q.snaktype == 'value' and q.datavalue
and q.datavalue.type == 'wikibase-entityid'
and q.datavalue.value
and q.datavalue.value.id == Q.CIRCA then
circa = '|circa'
break
end
end
return 'time|' .. (v.time or '') .. '|P' .. tostring(v.precision or '') .. circa
end
if ms.snaktype == 'somevalue' then
for _, q in ipairs((stmt.qualifiers or {}).P1932 or {}) do
if q.snaktype == 'value' and q.datavalue
and q.datavalue.type == 'string' then
local s = (q.datavalue.value or ''):gsub('^%s+', ''):gsub('%s+$', '')
if s == '' then return nil end
local ya, yb = parseDateRange(s)
if ya then
return 'range|' .. ya .. '|' .. yb
end
return 'p1932|' .. s
end
end
end
return nil
end
local function dateText(entity, statementsFn, frame)
statementsFn = statementsFn or dplaStatements
if not entity then return '' end
-- When rendering the yellow (non-DPLA) box, build a set of
-- DPLA-attributed comparables so non-DPLA statements that encode
-- the same fact (e.g. an inferred-from-Wikitext range claim left
-- behind by an older legacy-migration run) are suppressed at
-- render time. The Python reconciler removes these on the next
-- sync — see ``_reconcile_inferred_from_wikitext_dupes`` in
-- tools/sdc_sync.py — but the module needs to dedup between
-- syncs so the page reads cleanly today.
--
-- Gate positively on the yellow box's selector (``nonDplaStatements``)
-- rather than "anything that isn't ``dplaStatements``": the blue box
-- may now pass a per-item-scoped selector (``statementsMatching`` of an
-- item filter), and deduping those against ALL DPLA dates would wrongly
-- blank the blue-box date field.
local dplaComparables
if statementsFn == nonDplaStatements then
dplaComparables = {}
for _, dplaStmt in ipairs(dplaStatements(entity, P.DATE)) do
local key = dateStatementComparable(dplaStmt)
if key then dplaComparables[key] = true end
end
end
local out = {}
for _, stmt in ipairs(statementsFn(entity, P.DATE)) do
local skip = false
if dplaComparables then
local key = dateStatementComparable(stmt)
if key and dplaComparables[key] then
skip = true
end
end
if not skip then
local ms = stmt.mainsnak
local isCirca = false
for _, sc in ipairs(qualifierValues(stmt, P.SOURCING_CIRCUMSTANCES)) do
if type(sc) == 'table' and sc.id == Q.CIRCA then
isCirca = true
break
end
end
if ms.snaktype == 'value' and ms.datavalue and ms.datavalue.type == 'time' then
local v = ms.datavalue.value
local timeStr = (v.time or ''):gsub('^%+', '')
local rendered = formatWikibaseTime(timeStr, v.precision or 11)
if isCirca then
rendered = localizedLabel('circa') .. ' ' .. rendered
end
table.insert(out, rendered)
elseif ms.snaktype == 'somevalue' then
-- ``P1932`` (stated as) preserves the editor's original source
-- string verbatim. Legacy {{Artwork}} migration imports it
-- raw, so the value can legitimately contain wikitext markup
-- (e.g. ``{{other date|~|1911}}``). MediaWiki does NOT re-
-- expand templates inside Scribunto output, so emit through
-- ``frame:preprocess`` to render any embedded template /
-- parser-function. Plain-text passes through unchanged. Same
-- pattern Module:Information / Module:Artwork use for their
-- editor-contributed text fields.
local stated = firstQualifierValue(stmt, P.STATED_AS)
local text
if type(stated) == 'string' then
text = stated
elseif type(stated) == 'table' and stated.text then
text = stated.text
end
if text then
if frame then text = frame:preprocess(text) end
table.insert(out, text)
end
end
end -- close ``if not skip then``
end
return table.concat(out, ', ')
end
local function institutionQidOf(entity, statementsFn)
statementsFn = statementsFn or dplaStatements
if not entity then return nil end
for _, stmt in ipairs(statementsFn(entity, P.INSTITUTION)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid' then
return ms.datavalue.value.id
end
end
return nil
end
local function subjectsText(entity, statementsFn, filterFn)
statementsFn = statementsFn or dplaStatements
filterFn = filterFn or isDplaDetermined
if not entity then return '' end
local seen = {}
local out = {}
for _, stmt in ipairs(statementsFn(entity, P.SUBJECT_ENTITY)) do
local label = renderEntitySnak(stmt)
-- Case-insensitive dedup key: a P921 entity label and a P4272
-- string term that differ only in case (e.g. "assassination" vs
-- "Assassination") are the same subject to a reader.
local key = label and mw.ustring.lower(label)
if label and not seen[key] then
seen[key] = true
table.insert(out, label)
end
end
for _, s in ipairs(reassembleChunkedValues(entity, P.SUBJECT_STRING, stringOf, filterFn)) do
local key = mw.ustring.lower(s)
if not seen[key] then
seen[key] = true
table.insert(out, s)
end
end
return table.concat(out, ', ')
end
-- DPLA ID rendered as an interwiki link to the DPLA item page.
-- The ``dpla:`` interwiki prefix on Commons resolves to
-- https://dp.la/item/<id>, so ``[[dpla:<id>|<id>]]`` produces an
-- internal-looking link with the bare ID as visible text.
-- Render a DPLA item id as a ``dpla:`` interwiki link, or '' when blank.
-- Shared by the blue box (``dplaIdText``, reading SDC) and the yellow box
-- (reading the wikitext ``dpla_id`` param) so both render identically.
local function dplaIdLink(id)
if not id or id == '' then return '' end
return '[[dpla:' .. id .. '|' .. id .. ']]'
end
local function dplaIdText(entity, filterFn)
local values = reassembleChunkedValues(entity, P.DPLA_ID, stringOf, filterFn)
return dplaIdLink(values[1])
end
-- Build the Source-row icon cells from explicit URL arguments. Extracted
-- from ``sourceFieldText`` so the yellow box can call the same renderer
-- after parsing the user-side ``{{DPLA|...}}`` (legacy) or flat-param
-- (new uploads) shape — sharing one cell builder keeps the two
-- representations indistinguishable in the rendered HTML.
--
-- ``naraInterwikiLink`` is the optional pre-built ``[[w:..]]`` wikilink
-- the NARA-specific catalog cell uses; pass nil for any other partner
-- and the cell becomes a plain external link to ``catalogUrl``.
local function sourceFieldFromUrls(fileUrl, iiifUrl, catalogUrl, naraInterwikiLink)
local cells = {}
if fileUrl and fileUrl ~= '' then
table.insert(cells,
'[[File:Codex icon download color-progressive.svg|24px|alt=|class=skin-invert|link=]] [' ..
fileUrl .. ' ' .. localizedLabel('file_url') .. ']'
)
end
if iiifUrl and iiifUrl ~= '' then
table.insert(cells,
'[[File:International Image Interoperability Framework logo.png|24px|alt=|link=]] [' ..
iiifUrl .. ' ' .. localizedLabel('iiif_manifest') .. ']'
)
end
if catalogUrl and catalogUrl ~= '' then
if naraInterwikiLink then
table.insert(cells,
'[[File:Breezeicons-actions-22-database-index.svg|24px|alt=|class=skin-invert|link=]] ' ..
naraInterwikiLink
)
else
table.insert(cells,
'[[File:Breezeicons-actions-22-database-index.svg|24px|alt=|class=skin-invert|link=]] [' ..
catalogUrl .. ' ' .. localizedLabel('catalog_record') .. ']'
)
end
end
local parts = {}
for i, cell in ipairs(cells) do
local style = 'display:inline-block; padding: 4px 16px; vertical-align: middle;'
if i > 1 then
style = style .. ' border-left: 2px solid #a2a9b1;'
end
table.insert(parts, '<span style="' .. style .. '">' .. cell .. '</span>')
end
return table.concat(parts)
end
local function sourceFieldText(entity, statementsFn, filterFn)
local fileUrl, iiifUrl, catalogUrl = sourceUrls(entity, statementsFn)
return sourceFieldFromUrls(
fileUrl, iiifUrl, catalogUrl,
naraCatalogInterwiki(entity, statementsFn, filterFn)
)
end
-- Partnership card rendered as pure HTML (no wikitable). The legacy
-- ``{{DPLA/layout}}`` template uses wikitable syntax (``{| ... |}``)
-- which MediaWiki only parses at certain top-level positions; embedding
-- it inside a surrounding <td> via frame:expandTemplate left the
-- ``{|`` literal in the rendered output. Building the 3-column layout
-- as a single <div> with flex avoids the parse issue.
--
-- ``partnershipFromQids`` is the parametric core, callable from either
-- the SDC entity path (blue box) or the wikitext-param path (yellow
-- box for new uploads or legacy-shape pages whose source param the
-- module parses below). Same HTML, same accessibility behaviour:
-- the alt text on both logos is empty because the institution name is
-- already inside the adjacent text, and the DPLA name is in the banner
-- above the row.
local function partnershipFromQids(instQid, hubQid)
if (not instQid or instQid == '') and (not hubQid or hubQid == '') then
return ''
end
local instLabel = (instQid and instQid ~= '')
and (mw.wikibase.getLabel(instQid) or instQid)
or 'an institution'
local hubLabel = (hubQid and hubQid ~= '')
and (mw.wikibase.getLabel(hubQid) or hubQid)
or ''
local instMarkup = (instQid and instQid ~= '')
and ('[[d:' .. instQid .. '|' .. instLabel .. ']]')
or instLabel
local text
if hubLabel ~= '' then
local hubMarkup = '[[d:' .. hubQid .. '|' .. hubLabel .. ']]'
text = substituteParams(
localizedLabel('partnership_text_with_hub'), instMarkup, hubMarkup
)
else
text = substituteParams(
localizedLabel('partnership_text_without_hub'), instMarkup
)
end
local logo = ((instQid and instQid ~= '') and imageFileFor(instQid, P.LOGO_IMAGE))
or ((instQid and instQid ~= '') and imageFileFor(instQid, P.IMAGE))
or DEFAULT_PARTNER_LOGO
return '<div style="display:flex; align-items:center; gap:15px; padding:0.4em;">'
.. '<div>[[' .. logo .. '|70px|alt=]]</div>'
.. '<div style="flex:1;">' .. text .. '</div>'
.. '<div>[[File:DPLA square logo.svg|70px|alt=]]</div>'
.. '</div>'
end
local function partnershipMarkup(entity, _frame)
if not entity then return '' end
return partnershipFromQids(
institutionQidOf(entity),
partnerByRole(entity, Q.ROLE_AGGREGATOR)
)
end
-- P6426 Q-IDs that we have to emit a Commons template for directly,
-- because Module:License (the engine behind ``{{License from structured
-- data}}``) misreads our DPLA-attributed statements — it picks up the
-- determination-method qualifier value (Q61848113) instead of the
-- mainsnak Q-ID and produces a 'missing main template (P1424)' error
-- in the permission row. Each template takes one positional arg = the
-- contributing institution Q-ID, matching the legacy ``permission =
-- {{NoC-US|Q...}}`` / ``{{NKC|Q...}}`` shape uploaded files use. Other
-- eligible NoC-* values (NoC-CR, NoC-NC, NoC-OKLR) have no Commons
-- template yet and are also not used by any DPLA file today; add them
-- here if either condition changes.
local RIGHTS_TEMPLATE_BY_P6426 = {
['Q47530911'] = 'NoC-US', -- No Copyright - United States
['Q47530955'] = 'NKC', -- No Known Copyright
}
-- Resolve a copyright-license item (e.g. Q20007257 for CC BY 4.0) to the
-- corresponding Commons template basename (``Cc-by-4.0``) via its Wikidata
-- ``P1424`` (topic's main template) sitelink to commonswiki. Returns nil
-- when the license item has no ``P1424`` or its sitelink is missing / not a
-- template. Used as a fallback when Module:License bails out on
-- multi-P6216 files (see permissionMarkup).
local function licenseTemplateFromLicenseQid(qid)
if not qid then return nil end
local ok, licenseEntity = pcall(mw.wikibase.getEntity, qid)
if not ok or not licenseEntity then return nil end
local stmts = licenseEntity:getBestStatements('P1424')
for _, stmt in ipairs(stmts) do
local ms = stmt.mainsnak
if ms.snaktype == 'value' and ms.datavalue and ms.datavalue.type == 'wikibase-entityid' then
local sitelink = mw.wikibase.getSitelink(ms.datavalue.value.id, 'commonswiki')
if sitelink and sitelink:sub(1, 9) == 'Template:' then
return sitelink:sub(10)
end
end
end
return nil
end
local function permissionMarkup(entity, frame)
if not entity then return '' end
-- DPLA writes the rights cluster across two properties depending on
-- the source URI: ``P6426`` (copyright status as a creator) for
-- public-domain / no-known-copyright items, ``P275`` (copyright
-- license) for Creative Commons items. Detect either — the row
-- should render whenever DPLA has contributed any rights claim.
local p6426
for _, stmt in ipairs(dplaStatements(entity, 'P6426')) do
local ms = stmt.mainsnak
if ms.snaktype == 'value' and ms.datavalue and ms.datavalue.type == 'wikibase-entityid' then
p6426 = ms.datavalue.value.id
break
end
end
local p275Qid
for _, stmt in ipairs(dplaStatements(entity, 'P275')) do
local ms = stmt.mainsnak
if ms.snaktype == 'value' and ms.datavalue and ms.datavalue.type == 'wikibase-entityid' then
p275Qid = ms.datavalue.value.id
break
end
end
-- DPLA-attributed P6216 (copyright status). The write shape for source
-- rights = CC Public Domain Mark (PDM) is ``P6216 = Q19652`` (public
-- domain) with no P275 or P6426 — PDM is a rights-statement declaration,
-- not a copyright license, so Module:License can't dispatch it. Detect
-- the shape here and route directly to ``{{PD-US}}``.
local p6216Qid
for _, stmt in ipairs(dplaStatements(entity, 'P6216')) do
local ms = stmt.mainsnak
if ms.snaktype == 'value' and ms.datavalue and ms.datavalue.type == 'wikibase-entityid' then
p6216Qid = ms.datavalue.value.id
break
end
end
-- Suppress the row entirely when DPLA hasn't contributed any rights
-- claim. Without this guard the row would fall through to
-- {{License from structured data}} and render whatever license is on
-- the file regardless of DPLA provenance — defeating the
-- "DPLA-attributed fields only" contract of this template.
if not p6426 and not p275Qid and not p6216Qid then return '' end
local templateName = RIGHTS_TEMPLATE_BY_P6426[p6426]
if templateName then
-- Each template requires the institution Q-ID as its first
-- positional arg; without one it renders the raw ``{{{1}}}``
-- placeholder. Fall back to DPLA's own Q-ID when the file has
-- no DPLA-authored P195 — semantically accurate (DPLA bot is
-- the entity making the rights determination; the contributing
-- institution is just the original source).
local instQid = institutionQidOf(entity)
if not instQid or instQid == '' then
instQid = Q.DPLA
end
return frame:expandTemplate{title = templateName, args = {[1] = instQid}}
end
-- PDM shape: DPLA-attributed P6216=Q19652 (public domain) with no
-- P275/P6426 rights template on the entity. Module:License can't route
-- this — no Wikidata item sitelinks to ``Template:PD-US``, so its
-- P1424-based dispatch has nothing to follow. Emit ``{{PD-US}}``
-- directly. The neutral wording ("This work is in the public domain in
-- the United States") matches what the source institution has
-- declared via PDM without asserting a specific reason (e.g. >95
-- years old) that our upstream data doesn't warrant. Also emits
-- ``{{SDC-PD-US}}`` for the machine-readable license marker so files
-- clear ``Category:Files with no machine-readable license``.
if p6216Qid == 'Q19652' and not p275Qid then
return frame:expandTemplate{title = 'PD-US'}
end
-- ``{{License from structured data}}`` dispatches to the correct
-- license template (Cc-by-4.0, PD-USGov, etc.) from whichever of
-- P275/P6426/P6216 is set. Pass the entity's own id so it reads the
-- SAME MediaInfo the rest of the box rendered from: under the
-- {{Extracted from}} fallback that's the source file, so a cropped
-- CC-licensed derivative still gets its licence rather than reading
-- its own empty SDC. In the normal case ``entity.id`` is just the
-- current file and the result is unchanged.
local rendered = frame:expandTemplate{
title = 'License from structured data',
args = {id = entity.id},
}
-- Fallback: {{License from structured data}} (Module:License) bundles
-- rights via a branch table that assumes one of two mutually exclusive
-- shapes — N × P6216 with license/etc. as qualifiers (Wikidata style),
-- or 1 × P6216 with N × P275 top-level (Commons style). A community
-- edit that adds a second, contradictory P6216 statement (e.g.
-- ``Q19652`` public domain alongside our DPLA-attributed
-- ``Q50423863`` copyrighted) breaks both assumptions and Module:License
-- returns empty rather than pick a bundle. When that happens, expand
-- the DPLA-attributed P275 license template directly, taking the same
-- Wikidata P1424 (main template) → commonswiki sitelink path
-- Module:License uses internally. Loses author attribution and
-- ``{{Self}}`` wrapping, but DPLA files don't set either.
if p275Qid and (rendered == '' or rendered:match('^%s*$')) then
local fallback = licenseTemplateFromLicenseQid(p275Qid)
if fallback then
return frame:expandTemplate{title = fallback}
end
end
return rendered
end
-- Template-parameter alias map. The yellow user-contributed box accepts
-- either {{Information}} param names (description, author, source,
-- permission) or {{Artwork}} param names (description, artist,
-- institution, source, permission); ``firstParam`` resolves the first
-- non-empty match from a list of candidate names.
local function firstParam(args, ...)
if not args then return nil end
for _, name in ipairs({...}) do
local v = args[name]
if v and v ~= '' then return v end
end
return nil
end
-- Return the trimmed value iff it is a bare Wikibase entity ID (``Q`` followed
-- by digits); otherwise nil. Used to gate flat ``hub`` / ``institution`` params
-- before they reach a Wikibase lookup: Scribunto pre-expands args, so a value
-- written as ``institution = {{Institution|wikidata=Q...}}`` arrives as the
-- rendered vcard HTML and a hand-typed ``institution = <name>`` as plain text —
-- neither is an entity ID, and passing one to ``mw.wikibase`` /
-- ``{{Institution|wikidata=...}}`` throws ("The ID ... is not valid").
local function asBareQid(v)
if type(v) ~= 'string' then return nil end
return v:match('^%s*(Q%d+)%s*$')
end
-- Return true when ``wikitext_value`` is the same value the DPLA-attributed
-- SDC for ``prop`` already carries — i.e. when the wikitext is a verbatim
-- duplicate of authoritative DPLA data, not a user contribution. The
-- comparison depends on ``kind``:
--
-- * 'monolingualtext' — compare to ``mainsnak.datavalue.value.text`` on
-- each DPLA-determined statement (P1476 title, P10358 description).
-- * 'string' — compare to ``mainsnak.datavalue.value`` (P2093 stated-as
-- creator, P217 local identifier).
-- * 'time' — compare to the P1932 (stated as) qualifier on each
-- DPLA-determined statement. The mainsnak's structured time value
-- would have to be re-formatted to the original string for a direct
-- comparison; the P1932 qualifier preserves the original DPLA-supplied
-- display string verbatim (see ingest_wikimedia.sdc._build_date_claim),
-- so comparing to it is exact.
-- * 'wikibase-item' — wikitext value is expected to be a Q-ID string;
-- compare to ``mainsnak.datavalue.value.id`` on each DPLA-determined
-- statement (P195 institution, P9126 hub).
--
-- All comparisons trim surrounding whitespace to match the renderer's
-- behaviour. Returns false on missing entity / unknown kind so an
-- unrecognised property kind is "play it safe and render".
local function valueMatchesDplaSdc(entity, prop, wikitext_value, kind)
if not entity or not prop or not wikitext_value then return false end
local stripped = wikitext_value:gsub('^%s+', ''):gsub('%s+$', '')
if stripped == '' then return false end
for _, stmt in ipairs(dplaStatements(entity, prop)) do
local ms = stmt.mainsnak
if kind == 'monolingualtext' then
if ms.snaktype == 'value' and ms.datavalue
and ms.datavalue.type == 'monolingualtext'
and ms.datavalue.value
and ms.datavalue.value.text == stripped then
return true
end
elseif kind == 'string' then
-- Top-level string mainsnak (e.g. P217 local identifier).
-- For the creator field, the canonical SDC shape is a
-- P170 ``somevalue`` mainsnak with a P2093 qualifier, not
-- a top-level P2093 statement — see ``creator-name``
-- below; the ``string`` kind here only matches a direct
-- string mainsnak on the queried property.
if ms.snaktype == 'value' and ms.datavalue
and ms.datavalue.type == 'string'
and ms.datavalue.value == stripped then
return true
end
elseif kind == 'creator-name' then
-- DPLA's creator SDC: P170 mainsnak is typically
-- ``somevalue`` (we don't have a Wikidata item for the
-- creator), with the user-visible name carried on the
-- P2093 (stated as) qualifier (see sdc.py's
-- _build_creator_claim). Compare the wikitext creator
-- string to each DPLA-attributed P170 statement's P2093
-- qualifier; a direct P2093 statement is rare but is
-- caught by the ``string`` kind path when callers pass
-- prop='P2093' instead.
for _, q in ipairs((stmt.qualifiers or {}).P2093 or {}) do
if q.snaktype == 'value' and q.datavalue
and q.datavalue.value == stripped then
return true
end
end
elseif kind == 'time' then
-- The original display string lives on the P1932 qualifier;
-- the time mainsnak may be at year precision while the
-- editor wrote a full ISO date, so the mainsnak isn't a
-- reliable comparand.
--
-- Two-stage compare: direct byte equality first (covers
-- "1934 - 1948" vs. "1934 - 1948"), then range canonical
-- equivalence so ``{{other date|between|1934|1948}}`` /
-- "between 1934 and 1948" / "1934-1948" / "1934/1948" all
-- dedup against each other. The range fallback only runs
-- when both sides parse as canonical ranges, so single
-- dates and free prose are unaffected.
local strippedRangeA, strippedRangeB = parseDateRange(stripped)
for _, q in ipairs((stmt.qualifiers or {}).P1932 or {}) do
if q.snaktype == 'value' and q.datavalue then
if q.datavalue.value == stripped then
return true
end
if strippedRangeA then
local a, b = parseDateRange(q.datavalue.value or '')
if a == strippedRangeA and b == strippedRangeB then
return true
end
end
end
end
elseif kind == 'wikibase-item' then
if ms.snaktype == 'value' and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid'
and ms.datavalue.value
and ms.datavalue.value.id == stripped then
return true
end
end
end
return false
end
-- Special-case institution lookup that walks P195 *and* P9126
-- (partnership) statements for a matching Q-ID. The DPLA SDC writer
-- emits the hub-level institution on P195 (e.g. Q518155 = NARA for
-- every NARA-contributed file) and the more specific data-provider /
-- contributing institution on P9126 with a role qualifier (e.g.
-- Q59661040 = "National Archives at College Park - Still Pictures").
-- The wikitext ``institution =`` param holds the *data provider* Q-ID
-- — which matches the P9126 statement, not P195 — so checking only
-- P195 would miss the equality and leak a redundant Institution row
-- into the yellow box. Walking both properties catches either form.
-- Resolve a Q-ID through a Wikidata redirect to its canonical target. A
-- wikitext institution pointing at a redirect (e.g. Q69487402 -> Q5132215
-- "Cleveland Public Library") is the SAME institution the DPLA SDC carries
-- under the canonical ID, but a literal string compare misses it and leaks a
-- redundant Institution row into the yellow box. mw.wikibase.getEntity follows
-- entity redirects and :getId() returns the target's ID; non-Q input or an
-- unknown/invalid ID is returned unchanged (cheap no-op).
local function resolveRedirect(qid)
if type(qid) ~= 'string' or not qid:match('^%s*Q%d+%s*$') then
return qid
end
local ok, ent = pcall(mw.wikibase.getEntity, qid)
if ok and ent then
local id = ent:getId()
if id then
return id
end
end
return qid
end
local function institutionQidMatchesDplaSdc(entity, wikitext_value)
-- Compare the wikitext Q-ID AND its redirect-resolved canonical form against
-- both P195 (institution) and P9126 (partnership), so a redirected-but-
-- equivalent institution still de-duplicates against DPLA SDC.
local seen = {}
for _, q in ipairs({ wikitext_value, resolveRedirect(wikitext_value) }) do
if q and not seen[q] then
seen[q] = true
if valueMatchesDplaSdc(entity, P.INSTITUTION, q, 'wikibase-item')
or valueMatchesDplaSdc(entity, P.PARTNERSHIP, q, 'wikibase-item') then
return true
end
end
end
return false
end
-- Pick the user-contributed value for a field: explicit template
-- parameter wins, otherwise fall back to the non-DPLA SDC renderer.
--
-- When ``dplaProp`` (and ``dplaKind``) is supplied, the wikitext value
-- is first checked against the DPLA-attributed SDC for that property
-- — if it matches, the value is treated as not user-contributed and
-- the SDC fallback runs instead. This catches the post-SDC pre-strip
-- case where the wikitext still carries a value DPLA wrote at upload
-- time and SDC sync has already mirrored. Module:DPLA's yellow box
-- is for user contributions only; a wikitext value that's identical
-- to authoritative DPLA SDC is by definition not a contribution.
--
-- Without ``dplaProp``, behavior is unchanged from the original
-- ``userValue`` (no redundancy check), which is the right default
-- for fields whose redundancy can't be reliably detected (e.g.
-- ``subject``, where the SDC representation is a mix of P921 items
-- and P4272 strings the renderer would have to reassemble).
--
local function userValue(args, paramNames, sdcFallback, entity, dplaProp, dplaKind)
for _, name in ipairs(paramNames) do
local v = args and args[name]
if v and v ~= '' then
if dplaProp and valueMatchesDplaSdc(entity, dplaProp, v, dplaKind) then
break -- drop into SDC fallback
end
-- Prepend a newline so block-level wikitext (----, bulleted
-- lists, blank-line paragraph breaks, headings) at the
-- start of the value renders as HTML. MediaWiki's parser
-- only fires block-level transformations on lines starting
-- at column 0; ``tableRow`` concatenates value directly
-- after ``<td>``, so a leading ``----`` in the value would
-- otherwise sit at column-6 relative to the cell start and
-- render as literal characters. A leading newline places
-- the first content line at column 0. Do NOT add a trailing
-- newline — that promotes the whole value to a block and
-- MediaWiki wraps it in a spurious ``<p>``, breaking the
-- ordinary scalar-row rendering. Extension tags like
-- <gallery> already fire in inline context and are
-- unaffected. SDC-fallback content is already-rendered
-- HTML and never touches this path.
return '\n' .. v
end
end
return sdcFallback() or ''
end
-- Legacy creator-shape extractor for files where the creator value
-- lives inside ``Other fields N = {{InFi | Creator | <value> |
-- id=fileinfotpl_aut}}`` (the pre-#291 legacy template carried it
-- this way; some files still have it post-{{Artwork}}→{{DPLA metadata}}
-- migration because the strip pass doesn't know how to normalise
-- ``Other fields N`` keys yet). Without this fallback the renderer
-- never sees a creator value for these files, the creator row is
-- omitted, and the ``fileinfotpl_aut`` machine-readable id never
-- makes it into the rendered HTML — putting the file in
-- ``Files with no machine-readable author``.
--
-- Reads the *raw* page wikitext via ``mw.title:getContent()`` because
-- by the time Lua sees ``args``, Scribunto has already pre-expanded
-- the ``{{InFi|...}}`` sub-template into its rendered HTML — there's
-- no longer a literal ``{{InFi|Creator|...}}`` string to match. The
-- raw-content path is the same one ``parseLegacyFromPageContent``
-- uses for the legacy ``source = {{DPLA|...}}`` block; ``getContent``
-- is cached within a single parse so the additional call is
-- effectively free when both legacy shapes coexist on the page.
-- Wrap a list of <tr> rows in the same outer table/div structure the
-- legacy renderer used so Commons CSS still styles the labels.
local function wrapInTable(rows)
if #rows == 0 then return nil end
return '<div class="hproduct commons-file-information-table">\n'
.. '<table class="fileinfotpl-type-information vevent" dir="ltr">\n'
.. table.concat(rows, '\n')
.. '\n</table></div>'
end
-- Banner strings now live in [[Module:DPLA/i18n]] keyed as 'banner_dpla'
-- and 'banner_user'. Resolved at render time via ``localizedLabel`` so the
-- viewer sees them in their preferred language (falling back to English).
-- Build the DPLA (blue) table rows from DPLA-determined SDC only.
--
-- ``filterFn`` (default ``isDplaDetermined``) scopes which DPLA statements
-- feed this box. On a cross-item duplicate — one Commons file carrying
-- statements from several DPLA items — ``render_metadata_table`` draws one
-- box per contributing item, passing ``isDplaDeterminedForItem(id)`` so
-- each box stays cohesive (only that item's statements) and merged metadata
-- from different items doesn't bleed together. Scoping the chunk path per
-- item also prevents P1545 series-letter collisions between items (two
-- items both using letter 'A' would otherwise be concatenated). The
-- list-path helpers (creator/date/institution/subject entities) take the
-- equivalent ``statementsFn`` derived from ``filterFn``.
--
-- ``emitMachineIds`` (default true) controls whether the ``fileinfotpl_*``
-- machine-readable ids are placed on the label cells. Only the FIRST box on
-- a cross-item file emits them; the others must not, because CommonsMetadata
-- requires those ids to be unique per page.
local function buildDplaRows(entity, frame, suppressKeys, filterFn, emitMachineIds)
filterFn = filterFn or isDplaDetermined
if emitMachineIds == nil then emitMachineIds = true end
-- Statement selector for the list-path helpers, equivalent to
-- ``dplaStatements`` (default) or ``dplaStatementsForItem(id)`` (per-item).
local statementsFn = statementsMatching(filterFn)
local rows = {}
-- ``add`` takes a label *key* (looked up in i18n via
-- ``localizedLabel``) rather than the already-resolved label
-- string, so the same key can also drive the optional
-- ``fileinfotpl_*`` machine-readable id on the value cell. Keys
-- with no entry in ``MACHINE_READABLE_ID`` render as plain rows.
--
-- ``suppressKeys`` (optional set of label keys) is populated only by
-- the {{Extracted from}} fallback: when the blue box is drawn from a
-- *source* file's SDC, any field the user supplied on the derivative
-- itself must take precedence, so the matching fallback field is
-- dropped here and surfaces in the yellow box instead. Nil/empty in
-- the normal single-entity render, where the file's own DPLA SDC is
-- authoritative over user-contributed values.
local function add(labelKey, value)
if suppressKeys and suppressKeys[labelKey] then return end
local fieldId = emitMachineIds and MACHINE_READABLE_ID[labelKey] or nil
local row = tableRow(
localizedLabel(labelKey),
value,
nil,
fieldId
)
if row then table.insert(rows, row) end
end
local title = titlesText(entity, nil, filterFn)
-- Drop the big title banner too when the derivative carries its own
-- (user-contributed) title — it renders as a yellow row instead.
if not (suppressKeys and suppressKeys['title']) then
local banner = titleBanner(title)
if banner then table.insert(rows, banner) end
end
add('title', title)
add('creator', creatorText(entity, statementsFn, frame))
add('description', descriptionText(entity, nil, filterFn))
add('date', dateText(entity, statementsFn, frame))
local instQid = institutionQidOf(entity, statementsFn)
if instQid then
add('institution', frame:expandTemplate{
title = 'Institution',
args = {wikidata = instQid},
})
end
add('subject', subjectsText(entity, statementsFn, filterFn))
add('source', sourceFieldText(entity, statementsFn, filterFn))
add('dpla_id', dplaIdText(entity, filterFn))
add('other_pages', p.other_pages(frame))
add('partnership', partnershipMarkup(entity, frame))
add('permission', permissionMarkup(entity, frame))
return rows
end
-- Parse a legacy ``source = {{DPLA|<inst>|hub=...|url=...|dpla_id=...|local_id=...}}``
-- invocation, given the raw (unexpanded) wikitext value of the source
-- param. Returns a flat dict keyed by hub / inst_qid / url / dpla_id /
-- local_id, or nil when the value doesn't match the legacy shape.
-- Tolerant of inner whitespace and newlines (the upload template_string
-- emits a multi-line form).
local function parseLegacySourceParam(s)
if not s or s == '' then return nil end
local body = s:match('{{%s*[Dd][Pp][Ll][Aa]%s*|(.-)}}%s*$')
if not body then return nil end
local out = {}
for piece in (body .. '|'):gmatch('(.-)|') do
piece = piece:match('^%s*(.-)%s*$') or ''
if piece ~= '' then
local eq = piece:find('=', 1, true)
if eq then
local k = piece:sub(1, eq - 1):match('^%s*(.-)%s*$'):lower()
local v = piece:sub(eq + 1):match('^%s*(.-)%s*$')
if k == 'hub' then out.hub = v end
if k == 'url' then out.url = v end
if k == 'dpla_id' then out.dpla_id = v end
if k == 'local_id' then out.local_id = v end
elseif not out.inst_qid then
out.inst_qid = piece
end
end
end
return out
end
-- Locate the body of a top-level ``{{<name>|...}}`` template invocation
-- in ``content``, returning the wikitext between the opening ``{{`` and
-- the matching closing ``}}`` (not including the braces themselves).
-- Handles nesting — an inner ``{{...}}`` within the template's args
-- counts up the depth so the matching close belongs to the outer
-- envelope, not to the inner sub-template's close.
--
-- Lua patterns don't support balanced matching, so this is a manual
-- scan. Linear in content length; fine for the ~kilobyte file pages
-- this runs on at render time.
local function extractTemplateBody(content, name)
local i = content:find('{{%s*' .. name, 1)
if not i then return nil end
-- Skip past the opening ``{{`` and the template name itself.
local _, openEnd = content:find('{{%s*' .. name, i)
local j = openEnd + 1
local depth = 1
while j <= #content do
local two = content:sub(j, j + 1)
if two == '{{' then
depth = depth + 1
j = j + 2
elseif two == '}}' then
depth = depth - 1
if depth == 0 then
return content:sub(openEnd + 1, j - 1)
end
j = j + 2
else
j = j + 1
end
end
return nil
end
-- Walk the body of a ``{{DPLA metadata|...}}`` invocation and return
-- the value of each top-level (depth-0) named argument as a flat
-- dict. Nested ``{{...}}`` and ``[[...]]`` get carried inside the
-- containing arg's value; only ``|`` characters at depth 0 separate
-- top-level args.
local function splitTopLevelArgs(body)
local args = {}
local depth = 0
local linkDepth = 0
local current = {}
local positional = 0
local function flush()
local piece = table.concat(current)
current = {}
local eq = piece:find('=', 1, true)
if eq then
local k = piece:sub(1, eq - 1):match('^%s*(.-)%s*$'):lower()
local v = piece:sub(eq + 1):match('^%s*(.-)%s*$')
args[k] = v
else
-- Positional argument: store under its 1-based index. Never
-- clobber an explicit ``N=`` form (explicit wins, matching
-- MediaWiki template-argument semantics); the empty piece
-- before the first ``|`` is skipped.
local v = piece:match('^%s*(.-)%s*$')
if v ~= '' then
positional = positional + 1
local key = tostring(positional)
if args[key] == nil then args[key] = v end
end
end
end
local i = 1
while i <= #body do
local two = body:sub(i, i + 1)
if two == '{{' then
depth = depth + 1
table.insert(current, two)
i = i + 2
elseif two == '}}' then
depth = depth - 1
table.insert(current, two)
i = i + 2
elseif two == '[[' then
linkDepth = linkDepth + 1
table.insert(current, two)
i = i + 2
elseif two == ']]' then
linkDepth = linkDepth - 1
table.insert(current, two)
i = i + 2
elseif body:sub(i, i) == '|' and depth == 0 and linkDepth == 0 then
flush()
i = i + 1
else
table.insert(current, body:sub(i, i))
i = i + 1
end
end
flush()
return args
end
-- ============================================================
-- {{Extracted from}} SDC fallback (CropTool derivatives)
-- ============================================================
--
-- CropTool (and similar derivative-creation gadgets) copy a file's
-- *wikitext* — now just ``{{DPLA metadata}}`` — onto the new cropped file,
-- but do NOT copy its structured data. The derivative is left with the
-- template and an empty MediaInfo, so ``{{DPLA metadata}}`` renders blank
-- with no licence and the file gets tagged "no licence". The derivative
-- does, though, carry a ``{{Extracted from|<source>}}`` pointer back to the
-- file it was cropped from. When the current file has no DPLA SDC of its
-- own, we follow that pointer to the source file's MediaInfo and render
-- from it, so the derivative inherits the original's description,
-- categories and licence.
-- Maximum number of {{Extracted from}} pointers to follow (crop-of-a-crop
-- chains). Bounds the expensive getContent/getEntity calls and guards
-- against cyclic pointers.
local MAX_EXTRACTED_FROM_HOPS = 4
-- True iff the entity carries DPLA-authored structured data of its own.
-- Keys on the two markers every DPLA upload writes — P760 (DPLA ID) and
-- P7482 (source of file), each stamped with the DPLA P123 reference. A
-- file with neither is either a non-DPLA file or an un-synced derivative
-- (e.g. a fresh CropTool crop), and so a candidate for the fallback.
local function hasOwnDplaSdc(entity)
return #dplaStatements(entity, P.DPLA_ID) > 0
or #dplaStatements(entity, P.SOURCE_OF_FILE) > 0
end
-- Return the source filename named by an {{Extracted from}} template in
-- ``content`` (its first parameter, written positionally or as ``1=``),
-- or nil. Returned verbatim — may or may not carry a File:/Image: prefix.
local function extractedFromSource(content)
if not content then return nil end
local body = extractTemplateBody(content, '[Ee]xtracted from')
if not body then return nil end
local src = splitTopLevelArgs(body)['1']
if not src or src == '' then return nil end
return src
end
-- Follow {{Extracted from}} pointers up the derivative chain and return
-- the first ancestor file that carries its own DPLA SDC, or nil. ``content``
-- is the starting file's raw wikitext (the derivative's). Bounded to
-- MAX_EXTRACTED_FROM_HOPS so a crop-of-a-crop still resolves but cyclic or
-- runaway pointers can't. Each hop reads one page's wikitext and one
-- MediaInfo entity — both expensive parser functions, hence the bound.
local function resolveExtractedFromEntity(content)
for _ = 1, MAX_EXTRACTED_FROM_HOPS do
local src = extractedFromSource(content)
if not src then return nil end
local title = mw.title.new(src, 'File')
if not title or not title.id or title.id == 0 then return nil end
local entity = mw.wikibase.getEntity('M' .. tostring(title.id))
if hasOwnDplaSdc(entity) then return entity end
-- Ancestor is itself an un-synced derivative — climb to its source.
content = title:getContent()
if not content then return nil end
end
return nil
end
-- Read the raw (unexpanded) wikitext of the current file page and
-- extract the legacy ``source = {{DPLA|...}}`` and ``Institution =
-- {{Institution|wikidata=Q...}}`` values, parsing each into the flat
-- dict the parametric renderers consume.
--
-- This indirection exists because Scribunto's ``frame:getParent().args``
-- returns pre-expanded values: by the time the Lua module sees
-- ``args.source``, the inner ``{{DPLA|...}}`` has been expanded to its
-- rendered wikitext (which contains ``{|...|}`` table syntax that
-- leaks raw inside the surrounding HTML <td>). Fetching the raw
-- wikitext via mw.title gives us back the literal sub-template
-- invocation, which we can parse confidently.
--
-- Returns an empty dict when getContent fails (the current title isn't
-- a file page, or the page doesn't exist yet during preview); the
-- caller's behaviour then degrades gracefully to "no parsed legacy
-- params" — same effect as a flat-shape file with no source fields.
local function parseLegacyFromPageContent()
local title = mw.title.getCurrentTitle()
if not title then return {} end
local content = title:getContent()
if not content then return {} end
local envelope = extractTemplateBody(content, 'DPLA metadata')
if not envelope then return {} end
local topArgs = splitTopLevelArgs(envelope)
local out = {}
local sourceVal = topArgs['source']
if sourceVal then
local parsed = parseLegacySourceParam(sourceVal)
if parsed then
out.hub = parsed.hub
out.inst_qid = parsed.inst_qid
out.url = parsed.url
out.dpla_id = parsed.dpla_id
out.local_id = parsed.local_id
end
end
-- The Institution param is conventionally written with a capital
-- ``I`` in the legacy form (``Institution = {{Institution|...}}``);
-- splitTopLevelArgs lowercases the key.
local instVal = topArgs['institution']
if instVal and not out.inst_qid then
out.inst_qid = instVal:match(
'{{%s*[Ii]nstitution%s*|%s*[Ww]ikidata%s*=%s*([^|}%s]+)'
)
end
-- Content a community editor appended INSIDE the institution param beyond
-- the {{Institution|wikidata=Q}} sub-template (e.g. a {{NARA|...}} record
-- template on the next line) has no institution-ID meaning but is a real
-- contribution. Strip the sub-template and hand back the remainder so the
-- renderer can preserve it verbatim in the yellow box instead of dropping
-- it once the institution Q is extracted/deduped.
if instVal and instVal:match('{{%s*[Ii]nstitution%s*|') then
local extra = instVal:gsub(
'{{%s*[Ii]nstitution%s*|%s*[Ww]ikidata%s*=%s*[^|}%s]+%s*}}', ''
)
if extra:match('%S') then
out.inst_extra = extra
end
end
return out
end
-- Legacy creator-shape extractor for files where the creator value
-- lives inside ``Other fields N = {{InFi | Creator | <value> |
-- id=fileinfotpl_aut}}`` (the pre-#291 legacy template carried it
-- this way; some files still have it post-{{Artwork}}→{{DPLA metadata}}
-- migration because the strip pass doesn't know how to normalise
-- ``Other fields N`` keys yet). Without this fallback the renderer
-- never sees a creator value for these files, the creator row is
-- omitted, and the ``fileinfotpl_aut`` machine-readable id never
-- makes it into the rendered HTML — putting the file in
-- ``Files with no machine-readable author``.
--
-- Reads the *raw* page wikitext via ``mw.title:getContent()`` because
-- by the time Lua sees ``args``, Scribunto has already pre-expanded
-- the ``{{InFi|...}}`` sub-template into its rendered HTML — there's
-- no longer a literal ``{{InFi|Creator|...}}`` string to match. The
-- raw-content path is the same one ``parseLegacyFromPageContent``
-- uses for the legacy ``source = {{DPLA|...}}`` block; ``getContent``
-- is cached within a single parse so the additional call is
-- effectively free when both legacy shapes coexist on the page.
local function legacyOtherFieldsCreator()
local title = mw.title.getCurrentTitle()
if not title then return nil end
local content = title:getContent()
if not content then return nil end
local envelope = extractTemplateBody(content, 'DPLA metadata')
if not envelope then return nil end
local topArgs = splitTopLevelArgs(envelope)
for i = 1, 9 do
local v = topArgs['other fields ' .. i]
if v and v ~= '' then
-- ``{{ InFi | <Field> | <Value> | id=<id> }}`` shape;
-- the field label preceding the value is ``Creator``,
-- ``Author``, or ``Artist`` (case-insensitive, with
-- optional surrounding whitespace) on the InFi
-- invocations used by the legacy DPLA uploader.
local extracted = v:match(
'{{%s*[Ii]n[Ff]i%s*|%s*[Cc]reator%s*|%s*(.-)%s*|'
) or v:match(
'{{%s*[Ii]n[Ff]i%s*|%s*[Aa]uthor%s*|%s*(.-)%s*|'
) or v:match(
'{{%s*[Ii]n[Ff]i%s*|%s*[Aa]rtist%s*|%s*(.-)%s*|'
)
if extracted and extracted ~= '' then
return extracted
end
end
end
return nil
end
-- Resolve the user-side source/institution data into the flat shape the
-- parametric renderers consume, dual-pathed across:
-- 1. Flat params on ``{{DPLA metadata}}`` (hub, institution, url,
-- dpla_id, local_id) — what new uploads emit.
-- 2. Legacy nested params: ``source = {{DPLA|...}}`` and
-- ``Institution = {{Institution|wikidata=...}}`` — what historic
-- and pre-flat uploads carry. The literal sub-template invocations
-- are pulled from ``mw.title.getCurrentTitle():getContent()``
-- because Scribunto's args are pre-expanded.
--
-- Returns a table with hub/inst_qid/url/dpla_id/local_id; any field
-- may be nil. Flat-param values take precedence over legacy-parsed
-- ones so a file that's been migrated to flat shape but still has
-- stale legacy params (mid-transition) renders using the new shape.
local function resolveUserSourceFields(args)
-- ``hub`` and ``institution`` are entity-ID params. Accept them only when
-- they are a bare ``Q...`` (see asBareQid); a pre-expanded
-- ``{{Institution|wikidata=Q...}}`` value or plain text is not an ID and
-- must not reach a Wikibase lookup. Non-ID institution values are recovered
-- either by the legacy raw-wikitext parse below (which extracts the Q) or,
-- for a genuine literal, rendered verbatim by the caller.
local hub = asBareQid(firstParam(args, 'hub'))
local instQid = asBareQid(firstParam(args, 'institution'))
local url = firstParam(args, 'url')
local dplaId = firstParam(args, 'dpla_id')
local localId = firstParam(args, 'local_id')
if hub and url and dplaId and localId then
return {
hub = hub, inst_qid = instQid, url = url,
dpla_id = dplaId, local_id = localId,
}
end
-- Only pay the page-content fetch cost when there's actually a
-- legacy ``source`` or ``Institution`` param to resolve. Files in
-- the flat-shape pipeline never hit this path; files lacking any
-- source/institution context (a rare hand-written edit) also skip
-- it. mw.title:getContent() registers a transclusion dependency
-- whose invalidation pulse triggers on every page edit, so the
-- gate keeps the cache-invalidation graph minimal.
local hasLegacySourceArg = firstParam(args, 'source') ~= nil
local hasLegacyInstArg = firstParam(args, 'Institution') ~= nil
or firstParam(args, 'institution') ~= nil
if not hasLegacySourceArg and not hasLegacyInstArg then
return {
hub = hub, inst_qid = instQid, url = url,
dpla_id = dplaId, local_id = localId,
}
end
local legacy = parseLegacyFromPageContent()
return {
hub = hub or legacy.hub,
inst_qid = instQid or legacy.inst_qid,
inst_extra = legacy.inst_extra,
url = url or legacy.url,
dpla_id = dplaId or legacy.dpla_id,
local_id = localId or legacy.local_id,
}
end
-- Build the user-contributed (yellow) table rows. Each field prefers
-- an explicit template parameter; if absent, falls back to non-DPLA
-- SDC statements on the same property. For DPLA-bot uploads (legacy
-- ``{{DPLA|...}}`` shape or new flat-param shape), the source and
-- partnership rows render from the parsed wikitext params via the
-- same parametric helpers the blue box uses — keeping the yellow-box
-- presentation aligned with the blue box and avoiding the ``{|``
-- raw-markup leak the legacy sub-template caused inside an HTML cell.
local function buildUserRows(entity, args, frame, hasDplaBox)
local rows = {}
-- ``add`` mirrors ``buildDplaRows``'s helper — takes a label
-- *key* so the same key drives both the localized label and
-- the optional ``fileinfotpl_*`` machine-readable id on the
-- value cell. ``emittedKeys`` records every field that actually
-- produced a row (i.e. the user contributed it); it is returned so
-- the {{Extracted from}} fallback can suppress those same fields in
-- the source-derived blue box.
local emittedKeys = {}
local function add(labelKey, value)
local row = tableRow(
localizedLabel(labelKey),
value,
nil,
MACHINE_READABLE_ID[labelKey]
)
if row then
table.insert(rows, row)
emittedKeys[labelKey] = true
end
end
local title = userValue(args, {'title'}, function()
return entity and titlesText(entity, nil, isNonDplaDetermined) or ''
end, entity, P.TITLE, 'monolingualtext')
-- Only render the yellow box's own title banner when the blue box
-- isn't present. When the blue box is rendering above, it already
-- emits the file's title in the big-bold banner; duplicating it (or
-- worse, promoting a user-contributed translation to a secondary
-- big header) reads as two coequal titles. The user-contributed
-- title still renders below as a normal row via add('title', ...).
if not hasDplaBox then
local banner = titleBanner(title)
if banner then table.insert(rows, banner) end
end
add('title', title)
add('creator', userValue(
args, {'author', 'artist', 'creator'},
function()
-- SDC fallback: prefer a non-DPLA P170 statement, then
-- fall back to the legacy ``Other fields N = {{InFi |
-- Creator | ...}}`` shape some mid-migration files still
-- carry. The InFi extraction is critical for the
-- machine-readable ``fileinfotpl_aut`` id: without it,
-- the creator row never renders on legacy-shape files
-- and the page lands in
-- ``Files with no machine-readable author``.
local sdcVal = entity and creatorText(entity, nonDplaStatements, frame) or ''
if sdcVal and sdcVal ~= '' then return sdcVal end
return legacyOtherFieldsCreator() or ''
end,
-- Check P170 (creator) statements for a P2093 (stated as)
-- qualifier matching the wikitext value — that's the canonical
-- DPLA SDC shape. See the ``creator-name`` branch of
-- ``valueMatchesDplaSdc`` for the comparison logic.
entity, P.CREATOR, 'creator-name'
))
add('description', userValue(
args, {'description'},
function()
return entity and descriptionText(entity, nil, isNonDplaDetermined) or ''
end,
entity, P.DESCRIPTION, 'monolingualtext'
))
add('date', userValue(
args, {'date'},
function()
return entity and dateText(entity, nonDplaStatements, frame) or ''
end,
entity, P.DATE, 'time'
))
-- Source-row fields (hub/institution/url/dpla_id/local_id) are
-- consolidated once because the institution Q-ID is used by both
-- the Institution row and the Partnership row, and the legacy
-- ``source = {{DPLA|...}}`` carried it alongside the URL fields.
-- Resolving once also means the legacy-shape parse only runs once
-- per file.
local userSource = resolveUserSourceFields(args)
-- Institution row. Prefers the resolved Q-ID (works for both flat
-- ``institution = Q...`` uploads and legacy ``Institution =
-- {{Institution|wikidata=Q...}}`` uploads), falling back to a
-- non-DPLA P195 SDC statement when neither template form is
-- present. The wikitext Q-ID is suppressed when it matches the
-- DPLA-attributed P195 — same redundancy rule the scalar rows
-- above apply: a wikitext value identical to DPLA SDC is by
-- definition not a user contribution.
local userInstQid = userSource.inst_qid
-- Did the institution param resolve to a structured entity ID (a bare
-- ``Q...`` flat param, or a Q extracted from a legacy
-- ``{{Institution|wikidata=Q...}}`` value)? This gates the two no-Q
-- branches below: a structured Q that duplicates DPLA SDC is SUPPRESSED
-- (the blue box already shows it), while a genuinely non-ID value the user
-- typed is rendered VERBATIM instead of dropped.
local structuredInstQid = userInstQid ~= nil
if userInstQid and institutionQidMatchesDplaSdc(entity, userInstQid) then
userInstQid = nil -- fall through to non-DPLA SDC below
end
local userInstLiteral = firstParam(args, 'institution', 'Institution')
if userInstQid then
add('institution', frame:expandTemplate{
title = 'Institution',
args = {wikidata = userInstQid},
})
elseif not structuredInstQid
and userInstLiteral and userInstLiteral:match('%S')
and not userInstLiteral:match('^%s*Q%d+%s*$') then
-- The user supplied a non-entity-ID institution value: an arbitrary
-- template's (already-expanded) output or plain text. Render it
-- verbatim in the yellow box rather than dropping it or forcing it
-- through {{Institution|wikidata=...}} — whose Wikibase lookup throws
-- on a non-ID and lands the file in Category:Pages with script errors.
-- A legacy {{Institution|wikidata=Q...}} value never reaches here: its
-- Q is extracted upstream (structuredInstQid is true), so it renders
-- via the branch above with SDC de-duplication.
add('institution', userInstLiteral)
elseif entity then
-- Skip a non-DPLA P195 whose Q-ID is already a DPLA-sourced
-- P9126 custodial unit (P3831 = ROLE_CONTRIBUTING). Same fact,
-- different property — the P9126 custodial-unit statement is
-- the canonical shape for a departmental unit, and rendering
-- a matching P195 in the yellow box surfaces a redundant
-- Institution row that looks like a community override.
local custodialQids = allPartnersByRole(entity, Q.ROLE_CONTRIBUTING)
local isCustodialUnit = {}
for _, q in ipairs(custodialQids) do
isCustodialUnit[q] = true
end
local sdcInstQid
for _, stmt in ipairs(nonDplaStatements(entity, P.INSTITUTION)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid'
and not isCustodialUnit[ms.datavalue.value.id] then
sdcInstQid = ms.datavalue.value.id
break
end
end
if sdcInstQid then
add('institution', frame:expandTemplate{
title = 'Institution',
args = {wikidata = sdcInstQid},
})
end
end
-- Preserve any non-institution content the user appended inside the
-- institution param (e.g. a {{NARA|...}} record template) — see
-- parseLegacyFromPageContent's inst_extra. The institution Q itself is
-- rendered/deduped above; this keeps the appended extra from being silently
-- dropped once the Q is extracted. Rendered verbatim (preprocess) in the
-- yellow box.
if userSource.inst_extra and userSource.inst_extra:match('%S') then
add('institution', frame:preprocess(userSource.inst_extra))
end
add('subject', userValue(args, {'subject'}, function()
return entity and subjectsText(entity, nonDplaStatements, isNonDplaDetermined) or ''
end))
-- Source row. Builds the icon-cell HTML directly from the resolved
-- catalog URL — same parametric helper the SDC-driven blue box
-- uses. The legacy ``source = {{DPLA|...}}`` sub-template path
-- used to fall through to a verbatim pass-through here, which
-- leaked the sub-template's ``{|`` wikitable syntax into the
-- surrounding HTML <td> (MediaWiki doesn't restart table scanning
-- inside an HTML cell). ``resolveUserSourceFields`` parses both
-- the legacy and the flat shape into the same flat dict, so the
-- icon-cell renderer handles either identically.
--
-- Only ``catalogUrl`` is populated from wikitext params — the
-- per-ordinal direct-file URL and IIIF manifest URL live only in
-- SDC (P2699 / P6108) and aren't carried in the upload-time
-- wikitext. The yellow box therefore shows just the catalog-link
-- cell; the blue box shows up to three cells once SDC is written.
-- Source-row redundancy guard: if there's any DPLA-attributed
-- P7482 (described at) statement, the blue box already shows the
-- canonical catalog/file/IIIF URLs. A wikitext-derived source row
-- showing the same catalog URL would be redundant — suppress.
local hasDplaSource = entity and #dplaStatements(entity, P.SOURCE_OF_FILE) > 0
if userSource.url and not hasDplaSource then
local naraInterwiki
if userSource.inst_qid == Q.NARA and userSource.local_id then
naraInterwiki = '[[nara:' .. userSource.local_id
.. '|' .. localizedLabel('catalog_record') .. ']]'
end
add('source', sourceFieldFromUrls(
nil, nil, userSource.url, naraInterwiki
))
elseif not userSource.url then
-- Pass-through fallback: a Commons editor may have written a
-- free-text source description that's neither the DPLA-bot
-- ``{{DPLA|...}}`` invocation nor a flat-param shape. Preserve
-- it verbatim — the editor put it there deliberately, and
-- losing the row would silently discard their contribution.
-- The ``{|`` parsing problem only affects the specific
-- ``{{DPLA|...}}`` legacy shape (its inner {{DPLA/layout}}
-- emits wikitable syntax); free-text or other-template
-- values render fine in an HTML cell.
local rawSource = firstParam(args, 'source')
if rawSource then
add('source', rawSource)
end
end
-- DPLA ID row. Mirrors the blue box's ``dpla:`` interwiki link, built
-- from the wikitext ``dpla_id`` param (legacy ``{{DPLA|...}}`` or flat
-- shape) so a file rendering from upload-time wikitext — before SDC
-- sync runs — still shows its DPLA ID instead of silently dropping it.
-- Suppress when a DPLA-attributed P760 exists: the blue box renders
-- the canonical ID then and a wikitext duplicate would be redundant
-- (same redundancy rule the Source row above applies).
local hasDplaId = entity and #dplaStatements(entity, P.DPLA_ID) > 0
if userSource.dpla_id and not hasDplaId then
add('dpla_id', dplaIdLink(userSource.dpla_id))
end
-- Partnership row. Renders from the resolved institution + hub
-- Q-IDs the Institution row used. Surfaced in the yellow box for
-- the upload → SDC-sync interim window (no SDC yet, wikitext
-- params are the only source). Once SDC is written and carries
-- the canonical P9126 partnership, the wikitext-derived row would
-- be a redundant duplicate — suppress when any DPLA-attributed
-- P9126 statement exists. The blue box's partnership card renders
-- from that same SDC.
--
-- Partnership is added BEFORE Permission so the yellow box's row
-- order matches the blue box's (see ``buildDplaRows`` above). The
-- mismatch was a real, visible reordering on files with no SDC
-- yet — the rendered yellow box put the big Permission block
-- above the Partnership card, which read as Partnership being
-- demoted to a footer.
local hasDplaPartnership = entity
and #dplaStatements(entity, P.PARTNERSHIP) > 0
if (userSource.inst_qid or userSource.hub) and not hasDplaPartnership then
add('partnership',
partnershipFromQids(userSource.inst_qid, userSource.hub))
end
-- Permission row. A wikitext ``permission = {{NoC-US|...}}`` /
-- ``{{Cc-zero}}`` / similar template renders to a clean permission
-- box inside a cell when present. Suppress when any DPLA-attributed
-- P6426 (copyright license) statement is on the entity — the blue
-- box already renders the permission from that SDC. Comparing the
-- specific wikitext template name (``NoC-US``, ``Cc-zero``, etc.)
-- to the P6426 Q-ID (Q47530911, Q6938433, etc.) requires a mapping
-- table that doesn't exist here; the existence-based suppression
-- is a safe over-suppression because the wikitext permission was,
-- in practice, always written by the bot to mirror SDC.
local hasDplaPermission = entity
and #dplaStatements(entity, 'P6426') > 0
if not hasDplaPermission then
add('permission', firstParam(args, 'permission') or '')
end
-- Other versions row. Preserves the ``{{Artwork}}`` "Other versions"
-- field a community editor may have populated with ``{{other
-- version|Filename.jpg}}`` links. The wikitext value passes through
-- verbatim; the SDC fallback iterates P6802 (related image) statements
-- on the entity — including inferred-from-Wikitext claims written by
-- our migration — and expands each as ``{{other version|<filename>}}``.
-- ``nonDplaStatements`` is correct here because inferred-from-Wikitext
-- references (P887=Q131783016 + P4656=<permalink>) don't carry the
-- DPLA-publisher marker that ``isDplaDetermined`` keys on, so they
-- correctly surface in the yellow user-contributed box.
add('other_versions', userValue(args, {'other_versions', 'other versions', 'Other versions'},
function()
if not entity then return '' end
local parts = {}
for _, stmt in ipairs(nonDplaStatements(entity, P.OTHER_VERSIONS)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'string' then
table.insert(parts, frame:expandTemplate{
title = 'other version',
args = {[1] = ms.datavalue.value},
})
end
end
return table.concat(parts, ' ')
end
))
-- other_fields: arbitrary user-added display rows, mirroring
-- {{Information}}. Each {{Information field}}/{{InFi}} transcludes to a
-- <tr>, so the arg arrives as already-rendered row HTML — inject it
-- verbatim (NOT via add(), which would wrap it in a second <tr>). Supports
-- the canonical single ``other_fields`` and the legacy numbered ``Other
-- fields N`` shape. Values are opaque to us, so multiple fields (several
-- InFi calls concatenated) and multi-value cells (e.g. {{ubl|...}}) work
-- with no special handling.
--
-- Strip any ``id="fileinfotpl_*"`` a passthrough row carries (an
-- {{InFi|Creator|...}} sets id="fileinfotpl_aut"): the DPLA box owns those
-- machine-readable ids, and a duplicate on the page trips CommonsMetadata's
-- error categories. other_fields is for fields DPLA doesn't already model,
-- so the scrub is a no-op for intended use. Not recorded in emittedKeys —
-- these are free display fields, not DPLA-SDC-suppressible keys.
local function addPassthroughRows(raw)
if type(raw) ~= 'string' or not raw:match('%S') then return end
table.insert(rows, (raw:gsub('id%s*=%s*"fileinfotpl_[%w_]*"', '')))
end
addPassthroughRows(args.other_fields)
for i = 1, 20 do
addPassthroughRows(args['other fields ' .. i] or args['Other fields ' .. i])
end
return rows, emittedKeys
end
local function divbox(color, banner, body, frame)
return frame:expandTemplate{
title = 'divbox',
args = {[1] = color, [2] = banner, [3] = body},
}
end
--- p.render_metadata_table — entry point invoked from the
-- ``Template:DPLA metadata`` template. Builds up to two stacked boxes:
--
-- * Blue box: DPLA-determined SDC, with the standard provenance
-- banner. Rendered iff at least one DPLA field is present.
-- * Yellow box: explicit template parameters (matching {{Information}}
-- or {{Artwork}} param names) plus any non-DPLA SDC on the same
-- properties. Rendered iff at least one user-contributed value
-- exists.
--
-- Either box is silently omitted when its rows are empty; on a file
-- with no DPLA SDC and no template params, render_metadata_table
-- returns the empty string.
function p.render_metadata_table(frame)
local invokeArgs = (frame and frame.args) or {}
local parent = frame and frame:getParent() or nil
local userArgs = (parent and parent.args) or {}
local selfEntity = getEntity(invokeArgs)
-- {{Extracted from}} fallback: a derivative (e.g. a CropTool crop)
-- inherits its source's ``{{DPLA metadata}}`` wikitext but not its
-- SDC, so on its own it renders blank and unlicensed. When the file
-- has no DPLA SDC of its own, follow any ``{{Extracted from}}`` pointer
-- in its wikitext to the source file and use its MediaInfo as a
-- *fallback*. Only engaged for a wholly un-synced file.
--
-- The "subject" file is the ``page=`` target when given (so
-- /testcases can exercise this path against a real derivative), else
-- the page being rendered on. ``getEntity`` already honours ``page=``
-- for ``selfEntity``; we mirror it here for the wikitext lookup so
-- ``page=`` is a complete simulation rather than reading whichever
-- page the test happens to render on.
local subject = (invokeArgs.page and invokeArgs.page ~= '')
and mw.title.new(invokeArgs.page)
or mw.title.getCurrentTitle()
local sourceEntity = nil
if subject and not hasOwnDplaSdc(selfEntity) then
local content = subject:getContent()
if content then
sourceEntity = resolveExtractedFromEntity(content)
end
end
-- Effective entity for the categories and missing-SDC tracking below:
-- the fallback source when engaged (so the derivative is categorised
-- with its source and isn't flagged as missing required SDC), else the
-- file's own MediaInfo.
local entity = sourceEntity or selfEntity
-- Register the ``{{Infobox template tag}}`` transclusion so
-- ``{{DPLA metadata}}`` is recognised by YiFeiBot's
-- "Media missing infobox template" scanner. The tag is an
-- empty marker template; the ``expandTemplate`` call has no
-- observable output but registers the transclusion edge the
-- bot looks for. Same pattern ``Module:Information`` and
-- ``Module:Artwork`` use — see the comment in Module:Information:
-- "all official infoboxes transclude {{Infobox template tag}}
-- so files without that tag do not have an infobox".
if frame then
frame:expandTemplate{ title = 'Infobox template tag' }
end
-- The blue box(es) render from the file's own DPLA SDC (``selfEntity``),
-- or from a resolved {{Extracted from}} source when the file has none.
-- ``suppressKeys`` is populated only in the fallback case.
local blueEntity = sourceEntity or selfEntity
local suppressKeys = nil
local userRows
if sourceEntity then
-- Fallback precedence: the file's own DPLA SDC is what normally
-- outranks user-contributed values, and here the file has none —
-- so the derivative's OWN user content (its wikitext params and
-- any non-DPLA SDC on the file itself) wins over the source's
-- fallback SDC. Build the yellow box from the derivative first, then
-- suppress those same fields in the source-derived blue box. A
-- user-contributed value therefore shows in the yellow box and
-- replaces the blue-box field; the source fills in only the
-- fields the user left blank. ``hasDplaBox`` is forced true: the
-- source supplies the primary (blue) box, so a user-contributed
-- title renders as a normal yellow row rather than a banner.
local userKeys
userRows, userKeys = buildUserRows(selfEntity, userArgs, frame, true)
suppressKeys = userKeys
end
-- One blue box per contributing DPLA item. A single-item file (every
-- file today) yields exactly one box built with the default filter —
-- behaviour-identical to before. A cross-item duplicate (statements from
-- several DPLA items merged onto one file) yields one cohesive box per
-- item, so merged metadata from different items stays separated. The
-- machine-readable ``fileinfotpl_*`` ids are emitted only on the FIRST
-- box (CommonsMetadata requires them unique per page).
local ids = dplaItemIds(blueEntity)
local blueBoxes = {} -- list of { id = <item id>, table = <html>, single = <bool> }
if #ids > 1 then
for i, id in ipairs(ids) do
local rows = buildDplaRows(
blueEntity, frame, suppressKeys,
isDplaDeterminedForItem(id),
i == 1
)
local tbl = wrapInTable(rows)
if tbl then
table.insert(blueBoxes, { id = id, table = tbl })
end
end
else
-- Single-item (or, defensively, zero-item) file: the original
-- single-box path — default filter, machine ids on.
local rows = blueEntity and buildDplaRows(blueEntity, frame, suppressKeys) or {}
local tbl = wrapInTable(rows)
if tbl then
table.insert(blueBoxes, { table = tbl, single = true })
end
end
local hasDplaBox = #blueBoxes > 0
-- In the normal (non-fallback) render the yellow box's banner depends
-- on whether any blue box rendered, so build it now.
if not sourceEntity then
userRows = buildUserRows(selfEntity, userArgs, frame, hasDplaBox)
end
local out = {}
local userTable = wrapInTable(userRows)
-- ID for the anchor link the blue banner points at when both
-- boxes render. Kept as a stable string so translations of the
-- jump-link text land on the same anchor and any external
-- deep-links from documentation continue to resolve. Suggested
-- by [[User:Jmabel]] on the DPLA-bot talk thread 2026-07-07.
local USER_BOX_ANCHOR = 'dpla-user-contributions'
for i, box in ipairs(blueBoxes) do
-- Blue banner: provenance sentence + maintenance note, on a single
-- line because Template:Divbox embeds arg 2 into an HTML ``title=""``
-- attribute — a ``<br />`` inside it breaks the outer div. On a
-- cross-item file each box names its own contributing item via a
-- ``dpla:`` interwiki link.
local banner
if box.single then
banner = localizedLabel('banner_dpla')
else
banner = substituteParams(
localizedLabel('banner_dpla_item'), dplaIdLink(box.id)
)
end
banner = banner .. ' ' .. localizedLabel('banner_dpla_maintenance_note')
-- Jump-link at the END of the LAST blue box's banner — the box
-- sitting directly above the yellow box it points at — after the
-- maintenance note.
if userTable and i == #blueBoxes then
banner = banner
.. ' [[#' .. USER_BOX_ANCHOR .. '|'
.. localizedLabel('banner_dpla_jump_to_user')
.. ']]'
end
table.insert(out, divbox('blue', banner, box.table, frame))
end
if userTable then
-- Anchor placed ABOVE the yellow divbox (not inside its body)
-- so the jump-link scrolls to the top of the yellow box —
-- including the "The following metadata was added by Wikimedia
-- users" heading — rather than skipping it. Suggested by
-- [[User:Jmabel]] after eyeballing the earlier sandbox render
-- where the anchor landed BELOW that heading and readers
-- missed the disclaimer.
table.insert(out, '<span id="' .. USER_BOX_ANCHOR .. '"></span>')
-- The yellow box's "added by Wikimedia users" disclaimer only
-- makes sense as a contrast against the blue DPLA-attributed
-- box. When the blue box is suppressed (no DPLA SDC), drop the
-- banner so the yellow box stands on its own.
local yellowBanner = hasDplaBox and localizedLabel('banner_user') or ''
table.insert(out, divbox('yellow', yellowBanner, userTable, frame))
end
-- "Media contributed by..." categories — DPLA, hub, institution.
-- Dual-source: prefer SDC-resolved Qids when present; otherwise
-- fall back to the parsed wikitext source (legacy
-- ``source = {{DPLA|...}}`` or flat ``hub`` / ``institution``
-- params). The wikitext fallback covers legacy ``{{DPLA}}`` /
-- dual-render files that haven't had their SDC synced yet — they
-- still need to appear in the hub / institution maintenance
-- categories the legacy ``{{DPLA/hub cat}}`` / ``{{DPLA/inst cat}}``
-- templates emitted. A file with neither source identifying it
-- as a DPLA upload stays uncategorised.
local catHubQid, catInstQid = resolveCategoryQidsFromEntity(entity)
if not catHubQid and not catInstQid then
local src = resolveUserSourceFields(userArgs)
if src.hub or src.inst_qid then
catHubQid = src.hub or src.inst_qid
catInstQid = src.inst_qid
end
end
if catHubQid or catInstQid then
table.insert(out, buildCategoryList(catHubQid, catInstQid))
end
-- Tracking category for "files with non-DPLA metadata enhancements":
-- emit iff the yellow box renders WITH its banner — i.e. there's
-- a DPLA-attributed blue box AND user-contributed yellow rows on
-- top of it. The banner is suppressed (``hasDplaBox`` false) on a
-- just-uploaded file where SDC sync hasn't run yet; the yellow
-- box still renders there because the wikitext params haven't
-- been stripped, but those values aren't real "enhancements" —
-- they're the upload's own metadata pending SDC migration. Gating
-- on ``hasDplaBox and userTable`` keeps the category tied to the
-- post-SDC steady state where genuine community contributions
-- show up.
if hasDplaBox and userTable then
table.insert(out, '[[Category:' .. ENHANCED_CAT .. ']]')
end
-- Tracking category for centralized (multi-item / multi-page) files.
-- Multi-item: more than one contributing DPLA item, i.e. an extra blue
-- box renders (``#ids > 1`` — the same list that drove the box loop
-- above). Multi-page: a single item's P760 carries more than one P304
-- page position (``allPageNumbers`` returns >1), i.e. the file is a
-- within-item duplicate with multi-line "Other pages" navigation. Read
-- from ``blueEntity`` so the category matches exactly what rendered
-- (including a resolved {{Extracted from}} source). Either condition
-- alone is sufficient; a file that is both still gets the one category.
if #ids > 1 or #allPageNumbers(blueEntity) > 1 then
table.insert(out, '[[Category:' .. MULTI_CAT .. ']]')
end
-- Missing-SDC tracking categories — mirror the legacy
-- {{DPLA}} template's ``{{#invoke:SDC_tracking|SDC_statement_exist}}``
-- block. Files lacking any of the canonical required properties
-- land in the "missing required SDC statements" maintenance
-- category (operators backfill from there); ``P170`` (creator)
-- gets its own bucket so its backlog can be triaged separately.
-- The check matches ``Module:SDC_tracking._statement_exist`` —
-- simple ``entity.statements[property]`` existence, no
-- DPLA-attribution filter — so wikitext-contributed claims also
-- count as "has statement".
--
-- Always fires on DPLA-rendered files (the template is
-- bot-only); a file rendering through Module:DPLA is by
-- definition a DPLA file and gets the tracking. Categorylinks
-- dedupes, so multiple missing required props still emit the
-- "missing required" category just once.
local missingRequired = false
if not entity or not entity.statements then
missingRequired = true
else
for _, prop in ipairs(REQUIRED_SDC_PROPS) do
if not entity.statements[prop] then
missingRequired = true
break
end
end
end
if missingRequired then
table.insert(
out,
'[[Category:Digital Public Library of America files missing required SDC statements]]'
)
end
local hasCreator = entity
and entity.statements
and entity.statements[P.CREATOR]
if not hasCreator then
table.insert(
out,
'[[Category:Digital Public Library of America files missing creator]]'
)
end
-- Surface files whose DPLA-authored P275/P6426 rights statement carries
-- a Q-ID that is not in our Commons-eligibility allowlist. Typically the
-- contributing institution updated their rights metadata to a non-free
-- license (CC-BY-NC*, CC-BY-*-ND, InC*) after the file was already
-- uploaded under an eligible license; DPLA's SDC sync correctly mirrors
-- the institution's authoritative state, so the file ends up with
-- accurate-but-Commons-incompatible structured data. Silent — no
-- rendered output, just the tracking category for human review.
local hasIneligibleRights = false
if entity and entity.statements then
for _, prop in ipairs({'P275', 'P6426'}) do
for _, stmt in ipairs(dplaStatements(entity, prop)) do
local ms = stmt.mainsnak
if ms.snaktype == 'value'
and ms.datavalue
and ms.datavalue.type == 'wikibase-entityid'
and not ELIGIBLE_RIGHTS[ms.datavalue.value.id] then
hasIneligibleRights = true
break
end
end
if hasIneligibleRights then break end
end
end
if hasIneligibleRights then
table.insert(
out,
'[[Category:Media from the Digital Public Library of America with potential copyright issues]]'
)
end
return table.concat(out, '\n')
end
return p