Balisage Paper: Unlocking an archive of scholarly journal articles with interactive XQuery

app.xqm

The module app.xqm contains functions with RESTXQ annotations that serve the web application. The excerpt shown here supports the most of the functionality of the web-based user interface as well as providing API endpoints that are used by the Jupyter Notebook interface.

(:~ Display find-query page. :)
declare
  %rest:path("/banyan/find-query")
  %rest:produces("text/html")
  %output:method("text")
  %output:media-type("text/html") 
function app:find-query-html () {
  common:no-cache-header(),
  unparsed-text('find-query.html')
  => common:renderTemplate(common:templateData({
    'inline-script': common:no-cache-script(),
    'help_content': unparsed-text('find-query-help.html')
  }))
};

(:~ Run find-query and return results. :)
declare
  %rest:path("/banyan/find-query.xq")
  %rest:POST
  %rest:form-param("identifiers", "{$identifiers}")
  %rest:form-param("identifierType", "{$identifierType}")
  %rest:form-param("contentFormat", "{$contentFormat}")
  %rest:form-param("xquery", "{$xquery}")
  %rest:form-param("longRunning", "{$longRunning}")
  %rest:form-param("saveOutputAsName", "{$saveOutputAsName}")
  %output:indent('yes')
function app:find-query-xq (
  $identifiers as xs:string,
  $identifierType as xs:string,
  $xquery as xs:string,
  $contentFormat as xs:string,
  $longRunning as xs:string?,
  $saveOutputAsName as xs:string?
) {
  let $user := common:auth-key()
  return if (not(common:known-user($user))) then
    'User is not recognized.'
  else (
    common:restore-session-user-key($user),
    let $bindings := {
      "identifiers": $identifiers,
      "identifierType": $identifierType,
      "xquery": $xquery,
      "contentFormat": $contentFormat,
      "saveOutputAsName": $saveOutputAsName,
      "longRunning": $longRunning,
      "user": $user
    }
    let $query := xs:anyURI('find-query.xq')
    return try {
      if ($longRunning) then job:eval($query, $bindings, {'cache': true()})
      else (
        let $jobId := job:eval($query, $bindings, {'cache': true()})
        return (
          job:wait($jobId),
          job:result($jobId, {'keep': true()})
        )
      )
    } catch * {
      $err:description
    }
  )
};

declare
  %rest:path("/banyan/find-query-api/status")
  %output:method("json")
  %output:indent("yes")
function app:find-query-status () {
  app:find-query-status((), ())
};

declare
  %rest:path("/banyan/find-query-api/status/{$jobId=job\d+}")
  %rest:query-param("saved", "{$saved}")
  %output:method("json")
  %output:indent("yes")
function app:find-query-status (
  $jobId as xs:string?,
  $saved as xs:string*
) {
  common:restore-session-user-key(common:auth-key()),
  {
    "jobs": array:build(
      for $job in 
        if (exists($jobId)) then job:list-details()[@id = $jobId]
        else job:list-details()[ends-with(., 'find-query.xq')]
        => reverse()
      let $bindings := job:bindings($job/@id)
      return map:merge((
        {
          "id": string($job/@id),
          "state": string($job/@state),
          "time": string($job/@time),
          "ago": string(df:system-dateTime() - xs:dateTime($job/@time))
        },
        { "user": replace($bindings?user, '@.+', '')},
        $bindings
      ))
    ),
    "saved": array:build(
      for $file in 
        if (exists($saved)) then df:get-list()[@name = $saved]
        else df:get-list()
      order by $file/@name
      return {
        "name": string($file/@name),
        "modified": string($file/@modified-date),
        "size": string($file/@size)
      }
    ),
    "logs": array:build(
      current-date()
      => format-date('[Y0000]-[M00]-[D00]')
      => admin:logs()
      => filter(fn($e){
        (exists($jobId) and starts-with($e, $jobId))
        or matches($e, '^job\d+')
      })
      => slice(start := -15)
      => reverse()
      => for-each(fn($e){{'time': string($e/@time), 'message': string($e)}})
    )
  }
};

declare
  %rest:path("/banyan/find-query-api/cancel")
  %rest:POST
  %rest:form-param("jobId", "{$jobId}")
function app:find-query-cancel (
  $jobId as xs:string
) as xs:string? {
  let $user := common:auth-key()
  let $jobUser as xs:string? := job:bindings($jobId)?user
  return if ($user = $jobUser) then (
    $jobId || ' cancelled',
    admin:write-log($jobId || ' cancelled, state: ' || job:list-details($jobId)/@state),
    job:remove($jobId)
  )
};

find-query.xq

This is the main query that processes queries submitted by users.

Lines 58 – 88 apply guardrails to user-provided XQuery.

import module namespace df = "https://jats.taylorandfrancis.com/ns/darts/functions" at "../util/functions.xqm";
import module namespace yield = "https://jats.taylorandfrancis.com/ns/darts/loader/yield" at "../loader/yield.xqm";

declare namespace da = "https://jats.taylorandfrancis.com/ns/darts/archive";

(:~ User-provided identifiers :)
declare variable $identifiers as xs:string? external := '';

(:~ User-provided identifier type :)
declare variable $identifierType as enum(
  'doi', 'articleid', 'filename', 'package', 'journal', 'date-loaded-on', 'date-loaded-since'
) external := 'articleid';

(:~ User-provided Content Format :)
declare variable $contentFormat as enum('any', 'jats', 'tfjats', 'tfja') external := 'any';

(:~ User-provided XQuery Expression :)
declare variable $xquery as xs:string? external := '';

(:~ User-provided name to save the output using df:put :)
declare variable $saveOutputAsName as xs:string? external := '';

(:~ Parse user-provided identifiers string into a sequence :)
declare variable $ids as xs:string* := df:tokenize($identifiers);

(:~ Add preamble with function library and common namespaces to user-provided XQuery Expression :)
declare variable $query := df:xquery-preamble-declarations() || $xquery;

(:~ Flag for presence of a user-provided XQuery Expression :)
declare variable $runQuery as xs:boolean := string-length(normalize-space($xquery)) gt 0;

(:~ Constant for the name of the index database :)
declare variable $db-index := df:config('db-index');

(:~ Capture the timestamp when this query started :)
declare variable $startedAt := df:system-dateTime();

(:~ 
 : Echo the data, and if a name is provided then save the data using df:put
 :)
declare function echo-save (
  $data as item()*
) as item()* {
  $data,
  if (string-length($saveOutputAsName) gt 0) then (
    df:put($data, $saveOutputAsName)
  )
};

(:~ 
 : Begin by logging the user's input with the jobId.
 : Every run will typically create 4 log entries that begin with the jobId and " find-query ".
 :)
admin:write-log(job:current() || ' find-query user: ' || df:user() || '; save: ' || $saveOutputAsName),
admin:write-log(job:current() || ' find-query format: ' || $contentFormat || '; ' || $identifierType || ': ' || string-join($ids, '; ')),
admin:write-log(job:current() || ' find-query query: ' || $xquery),

(:~ 
 : Apply guardrails to user-provided XQuery Expression.
 : The query is first parsed, which may identify problems such as syntax errors.
 : A query is blocked if it attempts any unsafe actions that would: 
 : update or delete anything on the server, access files on the server, or run arbitrary code.
 :)
try {
  if ($runQuery) then
  let $parsed := xquery:parse($query)
  let $unsafe := (
    'xquery:eval', 'xquery:eval-update', 'job:eval', 'job:execute', 'job:bindings',
    'proc:system', 'proc:execute', 'proc:fork',
    'user:create', 'user:grant', 'user:drop', 'user:alter', 'user:password', 'user:update-info',
    'admin:logs', 'admin:delete-logs',
    'repo:install', 'repo:delete',
    'xslt:transform', 'xslt:transform-text', 'xslt:transform-report',
    'file:list', 'file:children', 'file:descendants', 'file:list-roots', 'file:read-binary', 'file:read-text', 'file:read-text-lines', 'file:create-dir', 'file:delete', 'file:write', 'file:write-binary', 'file:write-text', 'file:write-text-lines', 'file:append', 'file:append-binary', 'file:append-text', 'file:append-text-lines', 'file:copy', 'file:move', 'file:exists', 'file:is-dir', 'file:is-absolute', 'file:is-file', 'file:last-modified', 'file:size', 'file:temp-dir', 'file:current-dir', 'file:base-dir'
  )
  let $blocked as xs:string* := $parsed//@name[. = $unsafe]
  let $requested as xs:string* := distinct-values($blocked[contains($xquery, .)])
  (: a blocked function may be in the optimized query rather than what the user requested, e.g., df:put safely uses file:write :)
  return if (exists($requested))
  then error(xs:QName('error'), 'Function not allowed: ' || string-join($requested, ', '))
  else if (exists($parsed//@updating[. = 'true']))
  then error(xs:QName('error'), 'Updating expressions are not allowed.')
  else if (contains(normalize-space($xquery), 'import module namespace '))
  then error(xs:QName('error'), 'Importing modules is not allowed.')
} catch * {
  admin:write-log(job:current() || ' find-query blocked: ' || $err:description),
  error($err:code, 'XQuery error: ' || $err:description)
},

(:~ 
 : Yield when updates are running to avoid resource contention.
 :)
if (yield:updates-running())
then (
  admin:write-log(job:current() || ' find-query blocked: Loading activity is currently in progress.'),
  error(xs:QName('yield'), 'Please try again later. Loading activity is currently in progress.')
),

(:~ 
 : If there is an XQuery Expression and no identifiers, then evaluate the XQuery Expression.
 :)
if (empty($ids) and $runQuery)
then xquery:eval($query) => echo-save()

(:~ 
 : Based on the Identifier Type and Content Format, find metadata in the index database, then:
 : If no documents are found, then output a message.
 : If one document is found, and there is an XQuery Expression, then evaluate the expression on the document and return the results.
 : If one document is found, and there is no XQuery Expression, then return the document.
 : If two or more documents are found, then output is wrapped in df:wrap and:
 : If there is an XQuery Expression, evaluate the expression for each document and wrap the results with df:found metadata to identify the document.
 : If there is no XQuery Expression, wrap each document with df:found metadata to identify the document.
 :)
else if ($identifierType = ('doi', 'articleid', 'filename', 'package', 'date-loaded-on', 'date-loaded-since')) then (
  let $metas := (
    switch ($identifierType) {
      case 'doi' return
        db:get($db-index)/da:item[@doi = $ids]
      case 'articleid' return
        db:get($db-index)/da:item[@articleid = $ids]
      case 'filename' return
        db:get($db-index)/da:item[@filename = $ids]
      case 'package' return
        let $packages := $ids =!> replace('\.zip', '', 'i')
        return db:get($db-index)/da:item[@package_filename = $packages]
      case 'date-loaded-on' return
        for $date in $ids
        let $end := xs:date($date) + xs:dayTimeDuration('P1D')
        return
        db:attribute-range($db-index, string($date), string($end), 'date_loaded')/..
      case 'date-loaded-since' return
        let $validate := fn($d) {
          if (empty($d) or matches($d, '^\d\d\d\d(-\d\d(-\d\d)?)?$')) then $d
          else error(xs:QName('error'), `Wrong date format: '{$d}' (try YYYY-MM-DD or YYYY-MM or YYYY).`)
        }
        let $date := $ids => sort() => head() => $validate()
        let $end := ($ids => sort() => tail() => foot() => $validate()) otherwise df:current-dateTime()
        return
        db:attribute-range($db-index, string($date), string($end), 'date_loaded')/..
      default return ()
    }) ! (
    switch ($contentFormat) {
      case 'tfja' return .[@content_format = 'tfja']
      case 'tfjats' return .[@content_format = 'tfjats']
      case 'jats' return .[@content_format = ('jats1_2', 'jats1_4')]
      default return .
    })

  return if (empty($metas))
  then "No documents found"
  else if (empty($metas[2]))
  then echo-save(
    let $doc := db:get($metas/@database, $metas/@document)
    return if ($runQuery)
    then xquery:eval($query, {
      '': $doc,
      xs:QName('df:database'): $metas/@database,
      xs:QName('df:document'): $metas/@document,
      xs:QName('df:metadata'): $metas
    })
    else $doc
  )
  else echo-save(df:wrap(
    for $meta in $metas
    let $doc := db:get($meta/@database, $meta/@document)
    return if ($runQuery)
    then (
      let $result := xquery:eval($query, {
        '': $doc, 
        xs:QName('df:database'): $meta/@database, 
        xs:QName('df:document'): $meta/@document,
        xs:QName('df:metadata'): $meta
      })
      return if (exists($result)) then df:found($result, $meta)
    ) else df:found($doc, $meta)
  ))
) 
(:~ 
 : Based on the Identifier Type "journal" and Content Format, find metadata in relevant databases, then:
 : If no documents are found, then output a message.
 : Output is wrapped with df:wrap.
 : If there is an XQuery Expression, evaluate the expression for each document and wrap the results with df:found metadata to identify the document.
 : If there is no XQuery Expression, wrap each document with df:found metadata to identify the document.
 :)
else if ($identifierType = 'journal') then (
  try {
    echo-save(df:wrap(one-or-more(
      let $databases as xs:string* :=
        let $list := db:list()
        for $journal in $ids
        let $pattern := 
          if ($contentFormat eq 'any')
          then '^darts_[a-z]+_' || lower-case($journal) || '_'
          else '^darts_' || $contentFormat || '_' || lower-case($journal) || '_'
        return $list[matches(., $pattern)]
      for $db in $databases
      for $meta in db:get($db)/da:item
      let $doc := db:get($db, $meta/@document)
      return if ($runQuery)
      then (
        let $result := xquery:eval($query, {
          '': $doc, 
          xs:QName('df:database'): $db, 
          xs:QName('df:document'): $meta/@document,
          xs:QName('df:metadata'): $meta
        })
        return if (exists($result)) then df:found($result, $meta)
      ) else df:found($doc, $meta)
    )))
  } catch err:FORG0004 {
    "No documents found"
  }
),

(:~ End by logging the time duration with the jobId :)
admin:write-log(job:current() || ' find-query finished; time: ' || (df:system-dateTime() - $startedAt))

functions.xqm

The module functions.xqm contains a library of functions that are available for users to use in their queries. This module is also used in find-query.xq. The code listing shown here is an excerpt the module functions.xqm.

module namespace df = "https://jats.taylorandfrancis.com/ns/darts/functions";

declare namespace da = "https://jats.taylorandfrancis.com/ns/darts/archive";

declare type df:content_format as enum('jats', 'tfjats', 'tfja');

declare variable $df:database as xs:string? external := ();
declare variable $df:document as xs:string? external := ();
declare variable $df:metadata as element(da:item)? external := ();

declare variable $df:SESSION-USER-KEY := 'banyan-user-key';


(:~ config.xqm module is loaded dynamically because the location of this functions.xqm module might vary :)
declare variable $df:config-uri := 'https://jats.taylorandfrancis.com/ns/darts/loader/config';

declare variable $df:config := load-xquery-module(
  $df:config-uri,
  {'location-hints': file:resolve-path('src/main/loader/config.xqm')}
);

(:~ Call function config:config() in config.xqm :)
declare function df:config () as element(config)+ {
  $df:config?functions(QName($df:config-uri, 'config'))?0()
};

(:~ Return variable defined in config.xqm :)
declare function df:config ($variable as xs:string) as item()* {
  $df:config?variables(QName($df:config-uri, $variable))
};

(:~
 : Returns the Unicode Case Insensitive Collation URI for use in functions such as contains()
 : see https://qt4cg.org/specifications/xpath-functions-40/Overview.html#unicode-case-insensitive-collation
 :)
declare function df:case-insensitive() as xs:string {
  'http://www.w3.org/2005/xpath-functions/collation/unicode-case-insensitive'
};

(:~
 : Returns the current dateTime converted to UTC time zone
 : based on fn:current-dateTime so the value returned is the same throught the execution of a query.
 :)
declare function df:current-dateTime () as xs:dateTime {
  adjust-dateTime-to-timezone(current-dateTime(), xs:dayTimeDuration('PT0H'))
};


(:~
 : Returns the system dateTime converted to UTC time zone
 : based on prof:current-ms so the value returned is the actual current time.
 :)
declare function df:system-dateTime () as xs:dateTime {
  prof:current-ms() => convert:integer-to-dateTime() => adjust-dateTime-to-timezone(xs:dayTimeDuration('PT0H'))
};

(:~ 
 : Convert JATS date or pub-date element containing elements day, month, year or any element with @iso-8601-date to YYYY-MM-DD format.
 : The value returned might not be valid as an actual xs:date, e.g., it might be a year without month and day, but may be usable, e.g., for date string comparisons.
 :)
declare function df:date-from-jats ($date as element()) as xs:string? {
  let $a := analyze-string($date/@iso-8601-date, '(\d{4}(?:-\d{2}){0,2})')/fn:match/fn:group/string()
  return if (exists($a)) then $a
  else 
  let $d := $date/day =!> number() =!> format-number('00')
  let $m := $date/month =!> number() =!> format-number('00')
  let $y := $date/year =!> number() =!> format-number('0000')
  return string-join(($y, $m, $d), '-')[.]
};

(:~
 : Wrap item(s) in an element, which is useful when chaining instructions together and to consolidate namespace declarations.
 :)
declare function df:wrap($item as item()* := .) as element(df:wrap) {
    element df:wrap { df:namespace-declarations(), $item }
};


(:~ create namespace nodes to declare all frequently used namespaces :)
declare function df:namespace-declarations() as namespace-node()* {
  namespace df {'https://jats.taylorandfrancis.com/ns/darts/functions'},
  namespace da {'https://jats.taylorandfrancis.com/ns/darts/archive'},
  namespace xlink {'http://www.w3.org/1999/xlink'},
  namespace mml {'http://www.w3.org/1998/Math/MathML'},
  namespace tf {'http://cats.informa.com/tfjats'},
  namespace jats {'http://jats.nlm.nih.gov'},
  namespace oasis {'http://www.niso.org/standards/z39-96/ns/oasis-exchange/table'},
  namespace ali {'http://www.niso.org/schemas/ali/1.0/'},
  namespace xsi {'http://www.w3.org/2001/XMLSchema-instance'}
};

(:~ 
 : XQuery preamble to import this functions module and declare all frequently used namespaces.
 :)
declare function df:xquery-preamble-declarations() as xs:string {
``[
import module namespace df = 'https://jats.taylorandfrancis.com/ns/darts/functions';
declare namespace da = 'https://jats.taylorandfrancis.com/ns/darts/archive';
declare namespace xlink = 'http://www.w3.org/1999/xlink';
declare namespace mml = 'http://www.w3.org/1998/Math/MathML';
declare namespace tf = 'http://cats.informa.com/tfjats';
declare namespace jats = 'http://jats.nlm.nih.gov';
declare namespace oasis = 'http://www.niso.org/standards/z39-96/ns/oasis-exchange/table';
declare namespace ali = 'http://www.niso.org/schemas/ali/1.0/';
declare namespace xsi = 'http://www.w3.org/2001/XMLSchema-instance';
]``
};

(:~
 : Returns a reference to the index database that contains metadata for every document.
 :)
declare function df:db-index() as element(da:item)* {
  db:get(df:config('db-index'))/da:item
};

(:~ 
 : Retrieve document metadata for a given node.
 :)
declare function df:metadata($node as node()) as element(da:item)? {
    let $db := db:name($node)
    let $path := db:path($node) => replace('_J\.xml$', '_M.xml')
    return db:get($db, $path)/da:item
};

(:~
 : Retrieve the file name from a given path or URI.
 : Similar to file:name but works on any URI
 :)
declare function df:file-name($path as xs:string?) as xs:string {
  string(tokenize($path, '[/\\]')[.][last()])
};


(:~
 : Retrieve the parent directory of a given path or URI.
 : Similar to file:parent but works on any URI
 :)
declare function df:file-parent($path as xs:string?) as xs:string {
  $path => replace('[/\\]$', '') => replace('[^/\\]+$', '')
};

(:~
 : The file name extension in lower case, without the dot, or the empty sequence if the file name has no extension.
 : A file name that only has a leading dot (such as ".gitignore") is considered to have no extension.
 :)
declare function df:file-name-ext ($filePath as xs:string) as xs:string? {
  ($filePath => replace('^.*[\\/]', '', 's') => replace('^(?:.+\.([^.]+)|.*)$', '$1', 's') => normalize-space() => lower-case())[.]
};

(:~ fn:path configured with common namespaces :)
declare function df:path (
  $node     as node()?  := .,
  $options  as map(*)   := {}
) as xs:string? {
  let $opt := 
    map:merge((
      {'namespaces': map:merge((
        $options?namespaces,
        df:namespace-declarations() ! map:entry(name(), string()),
        {'fn': 'http://www.w3.org/2005/xpath-functions'}
      ))}, 
      $options
    ))
  return path($node, $opt)
};

(:~ Wrap a given node with an element whose attributes identify the origin of the node using xpath and, if available, the database name and document path :)
declare function df:location (
  $node     as node()?  := .,
  $options  as map(*)   := {},
  $dbdoc    as xs:boolean := false()
) as element(df:location) {
  element df:location {
    attribute xpath {df:path($node, $options)},
    if ($dbdoc) then try {
      attribute database {db:name($node)},
      attribute document {db:path($node)}
    } catch db:node {},
    $node
  }
};

(:~
 : This function can be used in queries to wrap individual result items with useful information.
 : Metadata about the result item is derived from the first item in $content
 : 
 : @return element(df:found) that contains $content and provides metadata in attributes 
 :)
declare function df:found($content as item()+ := .) as element(df:found) {
    df:found($content, head(($content/df:location/node(), $content)))
};

(:~
 : This function can be used in queries to wrap individual result items with useful information.
 : Metadata about the result item is derived from the database node given in the second parameter.
 : 
 : @return element(df:found) that contains $content and provides metadata in attributes 
 :)
declare function df:found($content as item()*, $node as node()) as element(df:found) {
  let $db := db:name($node)
  let $path := db:path($node)
  let $meta := 
    if ($node instance of element(da:item)) then $node
    else df:metadata($node)
  return element df:found {
    $meta/@database,
    $meta/@document,
    $meta/@loaded_item_id,
    $meta/@content_format,
    $meta/@package_filename,
    $meta/@filename,
    $meta/@doi,
    $meta/@articleid,
    $meta/@journal_code,
    $meta/@volume_num,
    $meta/@issue_num,
    $meta/@year_of_publication,
    $meta/@vor_date,
    $meta/@seq,
    $content
  }
};


(:~ 
 : Filter out older versions and return only newest version based on doi and loaded_item_id
 :)
declare function df:dedup-articles ($found as element(df:found)*) as element(df:found)* {
  for $group in $found
  group by $doi := $group/@doi
  return if (string($doi) = '') then $group
  else head(
    for $item in $group order by $item/number(@loaded_item_id) descending return $item
  )
};

(:~ 
 : List databases for a given format.
 : Use with db:get to query selected databases.
 :)
declare function df:databases (
  $content_format as df:content_format
) as xs:string* {
  db:list()[starts-with(., 'darts_' || $content_format || '_')]
};

(:~ 
 : List databases for a given format and journal.
 : Use with db:get to query selected databases.
 :)
declare function df:databases (
  $content_format as df:content_format,
  $journal_code as xs:string*
) as xs:string* {
  let $list := db:list()
  for $journal in $journal_code
  let $prefix := 'darts_' || $content_format || '_' || lower-case($journal) || '_'
  return $list[starts-with(., $prefix)]
};


(:~ 
 : Tokenize a string that contains a list of identifiers
 : separated by spaces, semicolons, or commas
 : and remove quotes.
 :)
declare function df:tokenize (
  $identifiers as xs:string
) as xs:string* {
  ($identifiers
  => replace('["'“”]', ' ')
  => normalize-space()
  => tokenize('[\s;,]')
  )[.]
};

(:~ Return the flattened text content of an element. :)
declare function df:normalize-text($element as element()?) as xs:string? {
    if ($element//text()) 
    then normalize-space(string-join($element//text(), ' '))[.]
    else ()
};

(:~ 
 : Save data for later retrieval.
 :)
declare function df:put (
  $data as item()*,
  $name as xs:string,
  $echo as xs:boolean := false()
) as item()* {
  let $safe := $name => replace('[^a-zA-Z0-9_\.\-]+', '-')
  return (
    admin:write-log(job:current() || ' df:put name: ' || $safe || '; user: ' || df:user()),
    file:write(file:resolve-path(df:dir() || $safe || '.xml'), $data, {"method": "basex"})
  ),
  if ($echo) then $data
};

(:~ 
 : Retrieve data saved with df:put().
 :)
declare function df:get (
  $name as xs:string
) as (document-node() | xs:string)? {
  let $path := file:resolve-path(df:dir() || $name || '.xml')
  return if (file:exists($path)) then 
  try {
    fetch:doc($path)
  } catch fetch:open {
    fetch:text($path)
  }
};

(:~ 
 : Retrieve data saved with df:put().
 :)
declare function df:get-binary (
  $name as xs:string
) as (xs:base64Binary)? {
  let $path := file:resolve-path(df:dir() || $name || '.xml')
  return if (file:exists($path)) then 
  file:read-binary($path)
};

(:~ 
 : Remove (delete) results saved with df:put().
 :)
declare function df:delete (
  $name as xs:string
) as empty-sequence() {
  let $path := file:resolve-path(df:dir()|| $name || '.xml')
  return if (file:exists($path)) then file:delete($path)
};

(:~ 
 : List files saved with df:put() that can be retrieved using df:get().
 :)
declare function df:get-list () as element(resource)* {
  for $file in file:list(df:dir())
  let $name := replace($file, '\.xml$', '')
  let $size := file:size(df:dir() || $file)
  let $time := file:last-modified(df:dir() || $file)
  return <resource name="{$name}" modified-date="{$time}" size="{$size}"/>
};

(:~ Directory for df:put, df:get, and df:get-list :)
declare function df:dir () as xs:string {
  let $dir := file:resolve-path(db:option('dbpath') || '/.banyan/' || df:user() || '/')
  return (
    if (not(file:exists($dir))) { file:create-dir($dir) },
    $dir
  )
};

(:~ Returns the current user :)
declare function df:user() as xs:string? {
  let $user := 
    (try {
      session:get($df:SESSION-USER-KEY)
    } catch basex:http { () }) 
    otherwise (job:bindings(job:current())?user)
  return replace($user, '@.+', '')[.]
};

common.xqm

The module common.xqm provides functions that support app.xqm.

module namespace common = "https://jats.taylorandfrancis.com/ns/banyan-grove/common";

declare function common:href-home () as xs:string {
  '/banyan/home'
};

(:~
 : Extracts the API key from the Authorization header. Requires the
 : "Bearer " scheme prefix; a header without it is treated as no key at all.
 : @return the API key, or the empty sequence if no valid Authorization header was sent
 :)
declare function common:auth-key () as xs:string? {
  request:header('Authorization')[.][matches(., '^[Bb]earer\s')] ! replace(., '^[Bb]earer\s+', '')
};

(:~
 : Standard denial response for a missing or unrecognized API key.
 : @return a 401 response with a WWW-Authenticate header
 :)
declare function common:unauthorized () as element(rest:response) {
  web:response-header(
    (),
    { 'WWW-Authenticate': 'Bearer' },
    { 'status': 401, 'message': 'A valid API key is required in the Authorization header.' }
  )
};


(:~ Partial implementation of Mustache templates.
 : For now, only supports replacing simple {{variables}} with values from a provided map.
 :)
declare function common:renderTemplate (
  $template as xs:string,
  $data as map(xs:string, xs:anyAtomicType?)
) as xs:string {
  fold-left(map:keys($data), $template, fn($t, $k) { replace($t, '\{\{' || $k || '\}\}', replace($data($k), '\$', '\\\$')) })
};

declare function common:templateData (
  $data as map(xs:string, xs:anyAtomicType?)? := {}
) as map(xs:string, xs:anyAtomicType?) {
  map:merge(($data, {
    "user": common:user-key(),
    "inline-script": (),
    "href_base": 
      try {
        request:scheme() || '://' || request:hostname() || ':' || request:port()
      } catch basex:http {()},
    "app_environment": common:app-environment()
  }))
};

declare function common:app-environment() as xs:string? {
  environment-variable('APP_ENVIRONMENT') otherwise (try { request:hostname() } catch basex:http {()})
};

(:~
 : Returns a Cache-Control response header to prevent the browser from caching a secured page.
 : Include in the response sequence of any route that requires authentication.
 :)
declare function common:no-cache-header() as element(rest:response) {
  web:response-header((), map { 'Cache-Control': 'no-store, no-cache, must-revalidate', 'Pragma': 'no-cache' })
};

(:~
 : Returns Javascript to prevent a secured page from being displayed from the browser's
 : back/forward cache (bfcache) after the user has logged out. Include on any secured page.
 :
 : Paired with common:no-cache-script-session-end() on the login page.
 :
 : The flow:
 : 1. On page load, a sessionStorage flag is set to record that the user is logged in.
 : 2. On pagehide (fired when the user navigates away), the page body is hidden so that
 :    the bfcache snapshot is stored in an invisible state.
 : 3. On pageshow with e.persisted=true (fired when the browser restores the page from
 :    bfcache), the flag is checked:
 :    - Flag absent (cleared by common:no-cache-script-session-end() on the login page):
 :      redirect immediately to the login page without a server round-trip.
 :    - Flag present (user is still logged in): reload the page to re-validate the session
 :      with the server.
 :)
declare function common:no-cache-script() as xs:string {
  (: this minimal version works but allows a brief display of a cached page before redirecting:
  ``[window.addEventListener('pageshow', function(e) { if (e.persisted) location.reload(); });]``
  :)
  ``[
sessionStorage.setItem('loggedIn', '1');
window.addEventListener('pagehide', function() {
  document.body.style.visibility = 'hidden';
});
window.addEventListener('pageshow', function(e) {
  if (e.persisted) {
    if (sessionStorage.getItem('loggedIn')) location.reload();
    else location.replace('/banyan/login');
  }
});
]``
};

(:~
 : Returns Javascript to clear the loggedIn sessionStorage flag.
 : Include on the login page so that bfcache restores of secured pages redirect without a server round-trip.
 :)
declare function common:no-cache-script-session-end() as xs:string {
  ``[sessionStorage.removeItem('loggedIn');]``
};