Generating the eCTD XML Backbone with Python

The eCTD backbone is the index.xml (and its regional companion) that tells a health authority’s review software which files exist, where they sit in the Common Technical Document tree, and how each relates to earlier submissions. It is generated once per sequence and validated before anything leaves the building. This walkthrough builds a correct backbone with lxml, and it is the concrete implementation behind eCTD Sequence Assembly & Lifecycle Management, part of the Electronic Submission Gateway & E-Signature Automation domain. Every generated backbone is recorded in the append-only audit log so the sequence can be reconstructed from a known toolchain.

Why naive backbone generation fails

String-templating the XML — the first instinct — fails in ways that only surface at the gateway, after the submission clock has started:

  • Unescaped paths and titles. A leaf title containing an ampersand or an accented investigator name produces malformed XML that a naive template emits verbatim. A real XML builder escapes content correctly.
  • Relative xlink:href that does not resolve. The backbone references each file by a path relative to the backbone itself. Hand-built paths drift from the on-disk layout and produce “file not found” rejections.
  • Missing or wrong checksums. The eCTD specification mandates an MD5 checksum per leaf. Omitting it, or computing it over the wrong bytes, is a high-severity technical rejection.
  • No DTD validation before send. Without validating the generated tree against the published DTD, structural errors ship unseen. The whole point of generating in code is to catch them at the transformation boundary.

Architecture overview

The generator is a short pipeline: model each leaf, serialize the tree, checksum every referenced file, and validate the result against the DTD before recording it.

eCTD backbone generation pipeline Five stages left to right: leaf models, lxml tree builder, serialize index.xml, checksum and DTD validation, and audit record. Each stage feeds the next with a single arrow. Leaf models Tree builder Serialize DTD check Audit record

Setup and configuration

Install the maintained XML library and pin it. lxml provides both serialization and DTD validation; the standard library covers hashing and configuration.

pip install "lxml>=5.2"

Configuration is read from the environment, never hardcoded, so the same generator runs against a sandbox and a production layout without a code change:

"""Environment-driven paths for backbone generation."""
from __future__ import annotations

import logging
import os
from dataclasses import dataclass


@dataclass(frozen=True)
class BackboneConfig:
    sequence_root: str   # directory that will contain index.xml
    dtd_path: str        # published eCTD DTD to validate against

    @classmethod
    def from_env(cls) -> "BackboneConfig":
        def required(name: str) -> str:
            value = os.environ.get(name)
            if not value:
                raise RuntimeError(f"Missing required environment variable: {name}")
            return value

        return cls(
            sequence_root=required("ECTD_SEQUENCE_ROOT"),
            dtd_path=required("ECTD_DTD_PATH"),
        )


logging.basicConfig(level=logging.INFO, format="%(message)s")
logger = logging.getLogger("ectd.backbone")

Full working implementation

The generator builds the tree with correct escaping and relative hrefs, checksums each referenced file, validates against the DTD, and returns serialized bytes plus a manifest for the audit trail.

"""Generate and validate an eCTD index.xml backbone."""
from __future__ import annotations

import hashlib
import posixpath
from dataclasses import dataclass
from pathlib import Path

from lxml import etree

XLINK = "http://www.w3.org/1999/xlink"


def xlink_attr(local_name: str) -> str:
    """Return the namespaced attribute name in Clark notation."""
    return etree.QName(XLINK, local_name).text


@dataclass(frozen=True)
class BackboneLeaf:
    leaf_id: str
    module: str
    title: str
    href: str            # path relative to the backbone
    operation: str       # new | replace | append | delete
    modified_leaf_id: str | None = None


def _md5(path: Path) -> str:
    return hashlib.md5(path.read_bytes()).hexdigest()


def build_index_xml(config: BackboneConfig, leaves: list[BackboneLeaf]) -> tuple[bytes, list[dict]]:
    """Return (serialized index.xml, per-leaf manifest) after DTD validation."""
    root = etree.Element("ectd", nsmap={"xlink": XLINK})
    manifest: list[dict] = []
    for leaf in leaves:
        target = Path(config.sequence_root) / leaf.href
        if not target.is_file():
            raise FileNotFoundError(f"leaf {leaf.leaf_id!r} href does not resolve: {leaf.href}")
        checksum = _md5(target)
        node = etree.SubElement(root, "leaf")
        node.set("ID", leaf.leaf_id)
        node.set("operation", leaf.operation)
        node.set("checksum", checksum)
        node.set("checksum-type", "md5")
        node.set(xlink_attr("href"), posixpath.normpath(leaf.href))
        if leaf.modified_leaf_id:
            node.set("modified-file", leaf.modified_leaf_id)
        title = etree.SubElement(node, "title")
        title.text = leaf.title  # lxml escapes &, <, > and unicode correctly
        manifest.append({"leaf_id": leaf.leaf_id, "md5": checksum, "operation": leaf.operation})

    xml_bytes = etree.tostring(root, xml_declaration=True, encoding="UTF-8", pretty_print=True)
    _validate_against_dtd(xml_bytes, config.dtd_path)
    return xml_bytes, manifest


def _validate_against_dtd(xml_bytes: bytes, dtd_path: str) -> None:
    dtd = etree.DTD(dtd_path)
    doc = etree.fromstring(xml_bytes)
    if not dtd.validate(doc):
        errors = "; ".join(str(e) for e in dtd.error_log)
        raise ValueError(f"backbone failed DTD validation: {errors}")

The title text is assigned through lxml, which escapes reserved characters and preserves Unicode, eliminating the malformed-XML failure class entirely. The xlink:href is normalized with posixpath so it stays forward-slashed and relative regardless of the host operating system.

Validation and edge-case handling

Three gotchas account for most late rejections; detect each before transport:

  • A leaf that moved after checksumming. Compute the MD5 immediately before serialization, not from a cached value — the code above reads the bytes at generation time, so a file swapped underneath the assembler is caught by the subsequent DTD and downstream technical validation.
  • A replace with a dangling modified-file. The DTD does not verify that the referenced prior leaf exists in the application’s lifecycle; that cross-sequence check belongs in the assembler, as covered in Handling eCTD Lifecycle Operations.
  • Non-ASCII in a title. Investigator names and site cities routinely carry diacritics; because the serializer emits UTF-8 and escapes correctly, these are safe — but the encoding must be declared, which xml_declaration=True guarantees.

Testing and verification

A focused pytest fixture confirms the generator escapes content, computes the right checksum, and rejects an unresolved href.

"""Tests for the eCTD backbone generator."""
from __future__ import annotations

import hashlib
from pathlib import Path

import pytest
from lxml import etree


def test_title_is_escaped(tmp_path: Path, config_factory) -> None:
    leaf_file = tmp_path / "m1" / "cover.pdf"
    leaf_file.parent.mkdir(parents=True)
    leaf_file.write_bytes(b"%PDF-1.7 test")
    config = config_factory(sequence_root=str(tmp_path))
    leaf = BackboneLeaf(
        leaf_id="l1", module="m1", title="Smith & Associates",
        href="m1/cover.pdf", operation="new",
    )
    xml_bytes, manifest = build_index_xml(config, [leaf])
    assert b"Smith &amp; Associates" in xml_bytes
    assert manifest[0]["md5"] == hashlib.md5(leaf_file.read_bytes()).hexdigest()


def test_unresolved_href_raises(tmp_path: Path, config_factory) -> None:
    config = config_factory(sequence_root=str(tmp_path))
    leaf = BackboneLeaf(
        leaf_id="l2", module="m1", title="Missing",
        href="m1/nope.pdf", operation="new",
    )
    with pytest.raises(FileNotFoundError):
        build_index_xml(config, [leaf])

The first test proves the ampersand is escaped and the checksum matches the on-disk bytes; the second proves a broken reference fails loudly at generation time rather than silently at the gateway. Together they defend the two most common backbone defects against regression.