dicom-index-send/dicom_indexer.py
2026-07-27 15:00:26 +07:00

1129 lines
44 KiB
Python

#!/usr/bin/env python3
"""
DICOM Folder Indexer (CLI) — MongoDB edition
=============================================
Recursively scans a folder for DICOM files, extracts key metadata
(patient, study, series, instance level), and indexes it into MongoDB.
Files whose path is already present in the index are skipped on
subsequent scans, so you can re-run `scan` on a growing folder and only
new files will be read/indexed.
Dependencies:
pip install pydicom pymongo python-dotenv
MongoDB connection config:
Connection settings are read from a .env file (in the current directory,
or pointed to with --env-file) instead of being passed as CLI flags.
Create a .env file like this (see .env.example):
MONGO_URI=mongodb://localhost:27017
MONGO_DB=dicom_index
MONGO_COLLECTION=files
MONGO_USERNAME=myuser
MONGO_PASSWORD=mypassword
MONGO_USERNAME/MONGO_PASSWORD are optional — skip them if your MongoDB
has no auth, or if you'd rather embed credentials directly in
MONGO_URI (e.g. mongodb://user:pass@host:27017). Using the separate
fields avoids having to URL-encode special characters in the password.
CLI flags --mongo-uri/--db/--collection/--mongo-username/--mongo-password
are still available and take priority over the .env file if you ever
need a one-off override.
Usage:
# Build/update an index from a folder (uses .env for Mongo connection)
python dicom_indexer.py scan /path/to/dicom_folder
# Force re-reading files that are already in the index (e.g. metadata
# extraction logic changed, or files were modified in place)
python dicom_indexer.py scan /path/to/dicom_folder --rescan
# Search the index
python dicom_indexer.py search --patient "Doe" --modality CT
# Search across all fields for a keyword
python dicom_indexer.py search -q "chest"
# Show summary statistics for the index
python dicom_indexer.py summary
# Export the current index to CSV
python dicom_indexer.py export -o index.csv
"""
import os
import sys
import csv
import argparse
import traceback
from datetime import datetime
# ---------------------------------------------------------------------------
# Dependency check with a friendly message.
# ---------------------------------------------------------------------------
MISSING = []
try:
import pydicom
except ImportError:
MISSING.append("pydicom")
try:
from pymongo import MongoClient, UpdateOne
from pymongo.errors import PyMongoError
except ImportError:
MISSING.append("pymongo")
if MISSING:
print("Missing required packages: " + ", ".join(MISSING))
print("Install them with:")
print(f" pip install {' '.join(MISSING)}")
sys.exit(1)
# ---------------------------------------------------------------------------
# .env file loading.
#
# Uses python-dotenv if it's installed (handles quoting, comments, export
# prefixes, etc. correctly). If it's not installed, falls back to a small
# manual KEY=VALUE parser so the script still works without the extra
# dependency — just with fewer edge cases covered.
# ---------------------------------------------------------------------------
def load_env_file(path):
"""Load KEY=VALUE pairs from an env file into os.environ (without
overriding variables already set in the real environment)."""
if not os.path.isfile(path):
return False
try:
from dotenv import load_dotenv
load_dotenv(path, override=False)
return True
except ImportError:
pass
# Minimal fallback parser: KEY=VALUE per line, '#' comments, optional
# surrounding quotes, blank lines ignored.
try:
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, value = line.partition("=")
key = key.strip()
value = value.strip().strip('"').strip("'")
if key and key not in os.environ:
os.environ[key] = value
return True
except OSError:
return False
# ---------------------------------------------------------------------------
# Metadata fields to extract from each DICOM file.
# Format: (Mongo field name, DICOM keyword)
# Add/remove entries here to customize what gets indexed.
# ---------------------------------------------------------------------------
DICOM_FIELDS = [
("PatientName", "PatientName"),
("PatientID", "PatientID"),
("PatientBirthDate", "PatientBirthDate"),
("PatientSex", "PatientSex"),
("PatientAge", "PatientAge"),
("StudyDate", "StudyDate"),
("StudyTime", "StudyTime"),
("StudyDescription", "StudyDescription"),
("StudyInstanceUID", "StudyInstanceUID"),
("AccessionNumber", "AccessionNumber"),
("Modality", "Modality"),
("SeriesDescription", "SeriesDescription"),
("SeriesNumber", "SeriesNumber"),
("SeriesInstanceUID", "SeriesInstanceUID"),
("InstanceNumber", "InstanceNumber"),
("SOPInstanceUID", "SOPInstanceUID"),
("Manufacturer", "Manufacturer"),
("ManufacturerModelName", "ManufacturerModelName"),
("InstitutionName", "InstitutionName"),
("BodyPartExamined", "BodyPartExamined"),
("Rows", "Rows"),
("Columns", "Columns"),
("SliceThickness", "SliceThickness"),
]
# Maps CLI search flags -> Mongo field name
SEARCH_FLAG_TO_COLUMN = {
"patient": "PatientName",
"patient_id": "PatientID",
"study_date": "StudyDate",
"study": "StudyDescription",
"modality": "Modality",
"series": "SeriesDescription",
"body_part": "BodyPartExamined",
"accession": "AccessionNumber",
}
# Columns actually stored in every document (used for CSV export / display)
ALL_FIELDS = ["FilePath", "FileName"] + [c for c, _ in DICOM_FIELDS] + ["FileSizeKB", "IndexedAt"]
# ---------------------------------------------------------------------------
# Extension-based fast skip.
#
# DICOM files often have NO extension at all (common with PACS/modality
# exports), or use .dcm/.dicom/.ima/.img. Because a missing extension is
# normal and valid, we do NOT filter to an allow-list of DICOM extensions
# only — that would silently skip real DICOM files.
#
# Instead we skip files whose extension is UNAMBIGUOUSLY something else
# (images, documents, archives, executables, etc.) without even opening
# them. Everything else (no extension, .dcm, or anything unrecognized)
# still goes through extract_metadata() as before, so correctness is
# preserved while common junk/sidecar files are skipped cheaply.
# ---------------------------------------------------------------------------
SKIP_EXTENSIONS = {
# common non-DICOM image formats
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tif", ".tiff", ".webp", ".ico", ".svg",
# documents / text / data
".txt", ".md", ".csv", ".json", ".xml", ".html", ".htm", ".pdf", ".doc", ".docx",
".xls", ".xlsx", ".ppt", ".pptx", ".log", ".ini", ".yaml", ".yml",
# archives
".zip", ".rar", ".7z", ".tar", ".gz", ".bz2", ".xz",
# executables / scripts / libraries
".exe", ".dll", ".so", ".bat", ".sh", ".py", ".js", ".jar",
# media
".mp3", ".mp4", ".avi", ".mov", ".wav",
# misc OS/editor cruft
".ds_store", ".db", ".tmp", ".bak", ".lnk", ".url",
}
# Used only when --strict-extension is passed: files must have one of these
# extensions (or no extension) to be opened at all.
DICOM_EXTENSIONS = {".dcm", ".dicom", ".dic", ".ima", ".img", ".dcm30"}
DEFAULT_MONGO_URI = "mongodb://localhost:27017"
DEFAULT_DB_NAME = "dicom_index"
DEFAULT_COLLECTION_NAME = "files"
def format_dicom_date(value):
"""Convert DICOM DA format (YYYYMMDD) to YYYY-MM-DD for readability."""
if not value:
return ""
value = str(value)
if len(value) == 8 and value.isdigit():
try:
return datetime.strptime(value, "%Y%m%d").strftime("%Y-%m-%d")
except ValueError:
return value
return value
def extract_metadata(filepath):
"""
Read a single file with pydicom and return a dict of extracted fields,
or None if the file is not a valid DICOM file.
"""
try:
ds = pydicom.dcmread(filepath, stop_before_pixels=True, force=False)
except Exception:
# Try once more with force=True in case the file is missing the
# standard preamble but is still valid DICOM (common with exported
# files that strip the 128-byte header).
try:
ds = pydicom.dcmread(filepath, stop_before_pixels=True, force=True)
if "SOPClassUID" not in ds and "Modality" not in ds:
return None
except Exception:
return None
filepath_abs = os.path.abspath(filepath)
row = {"FilePath": filepath_abs, "FileName": os.path.basename(filepath)}
for col_name, keyword in DICOM_FIELDS:
try:
value = getattr(ds, keyword, "")
except Exception:
value = ""
if value is None:
value = ""
value = str(value).strip()
if keyword in ("StudyDate", "PatientBirthDate"):
value = format_dicom_date(value)
row[col_name] = value
try:
size_bytes = os.path.getsize(filepath)
except OSError:
size_bytes = 0
row["FileSizeKB"] = round(size_bytes / 1024, 1)
row["IndexedAt"] = datetime.utcnow().isoformat()
return row
# ---------------------------------------------------------------------------
# MongoDB helpers
# ---------------------------------------------------------------------------
def get_collection(mongo_uri, db_name, collection_name, username=None, password=None):
"""Connect to MongoDB and return the target collection, with indexes set up."""
client_kwargs = {"serverSelectionTimeoutMS": 5000}
if username:
client_kwargs["username"] = username
if password:
client_kwargs["password"] = password
try:
client = MongoClient(mongo_uri, **client_kwargs)
client.admin.command("ping")
except PyMongoError as exc:
print(f"Error: could not connect to MongoDB at '{mongo_uri}': {exc}", file=sys.stderr)
sys.exit(1)
coll = client[db_name][collection_name]
# FilePath is unique so re-scanning the same file upserts instead of
# duplicating, and so we can pre-load "already indexed" paths cheaply.
coll.create_index("FilePath", unique=True)
coll.create_index("PatientID")
coll.create_index("StudyInstanceUID")
coll.create_index("SeriesInstanceUID")
coll.create_index("SOPInstanceUID")
coll.create_index("Modality")
return coll
def scan_folder(root_folder, already_indexed, show_progress=True, strict_extension=False):
"""
Walk root_folder recursively and yield a metadata dict for every valid,
not-yet-indexed DICOM file found, one at a time.
`already_indexed` is a set of absolute FilePaths already present in the
Mongo collection; files whose path is in this set are skipped without
even being opened. Pass an empty set to force re-reading everything.
This is a generator so callers can write each row to Mongo as it's
produced, instead of holding every row in memory at once.
"""
count_seen = 0
count_found = 0
count_skipped_ext = 0
count_skipped_indexed = 0
for dirpath, _dirnames, filenames in os.walk(root_folder):
for fn in filenames:
filepath = os.path.join(dirpath, fn)
filepath_abs = os.path.abspath(filepath)
count_seen += 1
if filepath_abs in already_indexed:
count_skipped_indexed += 1
else:
ext = os.path.splitext(fn)[1].lower()
if strict_extension:
should_skip_ext = ext != "" and ext not in DICOM_EXTENSIONS
else:
should_skip_ext = ext in SKIP_EXTENSIONS
if should_skip_ext:
count_skipped_ext += 1
else:
row = extract_metadata(filepath)
if row is not None:
count_found += 1
yield row
if show_progress and count_seen % 25 == 0:
sys.stdout.write(
f"\rScanning... {count_seen} files checked — {count_found} new DICOM found "
f"({count_skipped_indexed} already indexed, {count_skipped_ext} skipped by extension)"
)
sys.stdout.flush()
if show_progress and count_seen:
sys.stdout.write(
f"\rScanning... {count_seen} files checked — {count_found} new DICOM found "
f"({count_skipped_indexed} already indexed, {count_skipped_ext} skipped by extension)\n"
)
sys.stdout.flush()
# ---------------------------------------------------------------------------
# Subcommand: scan
# ---------------------------------------------------------------------------
BULK_WRITE_BATCH_SIZE = 500
def cmd_scan(args):
folder = args.folder
if not os.path.isdir(folder):
print(f"Error: '{folder}' is not a valid folder.", file=sys.stderr)
sys.exit(1)
coll = get_collection(args.mongo_uri, args.db, args.collection, args.mongo_username, args.mongo_password)
if args.rescan:
already_indexed = set()
print("--rescan given: re-reading all files, even ones already in the index.")
else:
print("Loading list of already-indexed files...")
already_indexed = {doc["FilePath"] for doc in coll.find({}, {"FilePath": 1, "_id": 0})}
print(f"Found {len(already_indexed)} file(s) already in the index.")
print(f"Scanning '{folder}' for DICOM files...")
row_count = 0
modality_counts = {}
patient_ids = set()
study_uids = set()
series_uids = set()
batch = []
def flush_batch():
if not batch:
return
try:
coll.bulk_write(batch, ordered=False)
except PyMongoError as exc:
print(f"\nWarning: a batch write to MongoDB failed: {exc}", file=sys.stderr)
batch.clear()
try:
for row in scan_folder(
folder,
already_indexed,
show_progress=not args.quiet,
strict_extension=args.strict_extension,
):
batch.append(UpdateOne({"FilePath": row["FilePath"]}, {"$set": row}, upsert=True))
row_count += 1
modality_counts[row.get("Modality", "")] = modality_counts.get(row.get("Modality", ""), 0) + 1
if row.get("PatientID"):
patient_ids.add(row["PatientID"])
if row.get("StudyInstanceUID"):
study_uids.add(row["StudyInstanceUID"])
if row.get("SeriesInstanceUID"):
series_uids.add(row["SeriesInstanceUID"])
if len(batch) >= BULK_WRITE_BATCH_SIZE:
flush_batch()
flush_batch()
except Exception as exc:
print(f"Error during scan: {exc}", file=sys.stderr)
if args.verbose:
traceback.print_exc()
sys.exit(1)
if row_count == 0:
print("No new DICOM files found (nothing to add to the index).")
sys.exit(0)
print(f"Indexed {row_count} new DICOM file(s) into MongoDB "
f"({args.db}.{args.collection} at {args.mongo_uri}).")
if not args.quiet:
print("\n--- New Files This Scan ---")
print(f"Files indexed: {row_count}")
print(f"Unique patients: {len(patient_ids)}")
print(f"Unique studies: {len(study_uids)}")
print(f"Unique series: {len(series_uids)}")
print("Modalities:")
for modality, count in sorted(modality_counts.items(), key=lambda kv: -kv[1]):
label = modality if modality else "(unknown)"
print(f" {label:<10} {count}")
print()
# ---------------------------------------------------------------------------
# Subcommand: search
# ---------------------------------------------------------------------------
DISPLAY_LIMIT = 200 # cap rows kept in memory for screen display
def cmd_search(args):
coll = get_collection(args.mongo_uri, args.db, args.collection, args.mongo_username, args.mongo_password)
mongo_filter = {}
for flag, column in SEARCH_FLAG_TO_COLUMN.items():
value = getattr(args, flag, None)
if value:
mongo_filter[column] = {"$regex": value, "$options": "i"}
if args.accession_missing:
if args.accession:
print("Note: --accession-missing overrides --accession; ignoring the --accession value.")
mongo_filter.pop("AccessionNumber", None)
# Matches AccessionNumber that is missing, null, or an empty/whitespace-only string.
mongo_filter["$or"] = [
{"AccessionNumber": {"$exists": False}},
{"AccessionNumber": None},
{"AccessionNumber": {"$regex": r"^\s*$"}},
]
if args.query:
searchable_columns = [c for c, _ in DICOM_FIELDS] + ["FileName", "FilePath"]
mongo_filter["$or"] = [
{col: {"$regex": args.query, "$options": "i"}} for col in searchable_columns
]
match_count = coll.count_documents(mongo_filter)
print(f"{match_count} matching record(s) found.\n")
if match_count == 0:
return
cursor = coll.find(mongo_filter, {"_id": 0})
display_rows = []
out_f = None
writer = None
try:
if args.output:
out_f = open(args.output, "w", newline="", encoding="utf-8")
writer = csv.DictWriter(out_f, fieldnames=ALL_FIELDS)
writer.writeheader()
for row in cursor:
if writer is not None:
writer.writerow({k: row.get(k, "") for k in ALL_FIELDS})
if len(display_rows) < DISPLAY_LIMIT:
display_rows.append(row)
finally:
if out_f is not None:
out_f.close()
display_columns = args.columns.split(",") if args.columns else [
"PatientName", "PatientID", "AccessionNumber", "StudyDate", "StudyDescription",
"Modality", "SeriesDescription", "SeriesNumber", "InstanceNumber",
"FileName",
]
display_columns = [c for c in display_columns if c in display_rows[0]]
_print_table(display_rows, display_columns)
if match_count > len(display_rows):
print(f"\n... showing first {len(display_rows)} of {match_count} matches. Use -o to save all results.")
if args.output:
print(f"\nFiltered results ({match_count} rows) saved to: {args.output}")
# ---------------------------------------------------------------------------
# Subcommand: summary
# ---------------------------------------------------------------------------
def cmd_summary(args):
coll = get_collection(args.mongo_uri, args.db, args.collection, args.mongo_username, args.mongo_password)
total = coll.count_documents({})
if total == 0:
print("Index is empty.")
return
patient_count = len(coll.distinct("PatientID", {"PatientID": {"$ne": ""}}))
study_count = len(coll.distinct("StudyInstanceUID", {"StudyInstanceUID": {"$ne": ""}}))
series_count = len(coll.distinct("SeriesInstanceUID", {"SeriesInstanceUID": {"$ne": ""}}))
modality_pipeline = [
{"$group": {"_id": "$Modality", "count": {"$sum": 1}}},
{"$sort": {"count": -1}},
]
modality_counts = list(coll.aggregate(modality_pipeline, allowDiskUse=True))
group_pipeline = [
{"$group": {
"_id": {
"PatientName": "$PatientName", "PatientID": "$PatientID",
"StudyDescription": "$StudyDescription", "SeriesDescription": "$SeriesDescription",
},
"count": {"$sum": 1},
}},
]
group_counts = list(coll.aggregate(group_pipeline, allowDiskUse=True))
print("\n--- Index Summary ---")
print(f"Total files indexed: {total}")
print(f"Unique patients: {patient_count}")
print(f"Unique studies: {study_count}")
print(f"Unique series: {series_count}")
print("Modalities:")
for entry in modality_counts:
label = entry["_id"] if entry["_id"] else "(unknown)"
print(f" {label:<10} {entry['count']}")
print("\nPatients / Studies / Series:")
for entry in group_counts:
g = entry["_id"]
print(f" {g.get('PatientName','')} | {g.get('PatientID','')} | "
f"{g.get('StudyDescription','')} | {g.get('SeriesDescription','')} -> {entry['count']} file(s)")
print()
# ---------------------------------------------------------------------------
# Subcommand: export
# ---------------------------------------------------------------------------
def cmd_export(args):
coll = get_collection(args.mongo_uri, args.db, args.collection, args.mongo_username, args.mongo_password)
total = coll.count_documents({})
if total == 0:
print("Index is empty, nothing to export.")
return
with open(args.output, "w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=ALL_FIELDS)
writer.writeheader()
for row in coll.find({}, {"_id": 0}):
writer.writerow({k: row.get(k, "") for k in ALL_FIELDS})
print(f"Exported {total} row(s) to: {args.output}")
# ---------------------------------------------------------------------------
# Subcommand: dedupe
#
# Since FilePath is the unique key for indexing, the index can never have
# two documents with the identical path. But the same DICOM instance can
# legitimately end up indexed at more than one path — e.g. a file copied to
# two folders, or re-exported by the PACS to a new location. Those share
# the same SOPInstanceUID (globally unique per DICOM standard), which is
# what this command uses to detect true duplicates.
# ---------------------------------------------------------------------------
def cmd_dedupe(args):
coll = get_collection(args.mongo_uri, args.db, args.collection, args.mongo_username, args.mongo_password)
pipeline = [
{"$match": {"SOPInstanceUID": {"$nin": [None, ""]}}},
{"$group": {
"_id": "$SOPInstanceUID",
"count": {"$sum": 1},
"docs": {"$push": {"FilePath": "$FilePath", "IndexedAt": "$IndexedAt"}},
}},
{"$match": {"count": {"$gt": 1}}},
]
groups = list(coll.aggregate(pipeline, allowDiskUse=True))
if not groups:
print("No duplicate DICOM instances found (checked by SOPInstanceUID).")
return
total_dupe_files = sum(g["count"] for g in groups)
to_remove = 0
print(f"Found {len(groups)} DICOM instance(s) indexed under more than one file path "
f"({total_dupe_files} file entries total).")
if args.dry_run:
print("(--dry-run given: reporting only, NOT deleting anything from MongoDB)")
print(f"Keep strategy: {args.keep} entry per duplicate group "
f"(files that no longer exist on disk are never kept if a valid copy is available).\n")
paths_to_delete = []
shown = 0
for g in groups:
docs = g["docs"]
existing = [d for d in docs if os.path.isfile(d["FilePath"])]
candidates = existing if existing else docs
candidates_sorted = sorted(
candidates, key=lambda d: d.get("IndexedAt") or "", reverse=(args.keep == "newest")
)
keep = candidates_sorted[0]
remove = [d for d in docs if d["FilePath"] != keep["FilePath"]]
paths_to_delete.extend(d["FilePath"] for d in remove)
to_remove += len(remove)
if not args.quiet and shown < DISPLAY_LIMIT:
print(f"SOPInstanceUID {g['_id']}{len(docs)} copies indexed:")
print(f" KEEP {keep['FilePath']}")
for d in remove:
stale = "" if os.path.isfile(d["FilePath"]) else " (file no longer on disk)"
print(f" REMOVE {d['FilePath']}{stale}")
print()
shown += 1
if not args.quiet and len(groups) > DISPLAY_LIMIT:
print(f"... and {len(groups) - DISPLAY_LIMIT} more duplicate group(s) not shown.\n")
print(f"Total index entries that would be removed: {to_remove}")
if args.dry_run:
print("\nRe-run without --dry-run to actually remove these from MongoDB.")
return
if to_remove:
result = coll.delete_many({"FilePath": {"$in": paths_to_delete}})
deleted = getattr(result, "deleted_count", to_remove)
print(f"Removed {deleted} duplicate index entrie(s) from MongoDB.")
print()
# ---------------------------------------------------------------------------
# Subcommand: recheck-accession
#
# For files already in the index whose AccessionNumber came back missing,
# null, or blank, go back to the actual file on disk and re-read its DICOM
# metadata fresh (not from the index). This catches cases like: the
# original scan happened before the file was fully written, the file was
# re-exported by the modality/PACS with the field filled in afterward, or
# the field is genuinely absent from the file itself.
# ---------------------------------------------------------------------------
def _missing_accession_filter():
return {
"$or": [
{"AccessionNumber": {"$exists": False}},
{"AccessionNumber": None},
{"AccessionNumber": {"$regex": r"^\s*$"}},
]
}
def resolve_target_files(coll, files, filenames, note_prefix="recheck"):
"""
Given raw --file paths and --filename patterns, return a de-duplicated
list of absolute file paths to operate on. --filename is resolved by
looking up the indexed FileName field (substring, case-insensitive) to
find the corresponding FilePath(s); --file paths are used as-is.
"""
target_files = list(files or [])
for fn in (filenames or []):
matches = list(coll.find({"FileName": {"$regex": fn, "$options": "i"}}, {"_id": 0, "FilePath": 1}))
if not matches:
print(f"Note: no indexed file found matching filename '{fn}' — skipping.")
else:
for m in matches:
target_files.append(m["FilePath"])
# De-duplicate while preserving order, in case --file/--filename overlap
# or a --filename pattern matched the same file more than once.
return [os.path.abspath(f) for f in dict.fromkeys(target_files)]
# ---------------------------------------------------------------------------
# Subcommand: inspect
#
# Read-only: re-reads a file's DICOM metadata straight from disk and prints
# it, without touching MongoDB at all (no index writes, no upserts). Useful
# for eyeballing what a file actually contains before deciding whether to
# rescan/recheck it, or just to sanity-check a specific file/filename.
# ---------------------------------------------------------------------------
def read_dicom_dataset(filepath):
"""
Read a file with pydicom and return the raw Dataset (not a flattened
dict like extract_metadata), or None if it's not a valid DICOM file.
Used by --full inspect mode to show every tag actually present.
"""
try:
return pydicom.dcmread(filepath, stop_before_pixels=True, force=False)
except Exception:
try:
ds = pydicom.dcmread(filepath, stop_before_pixels=True, force=True)
if "SOPClassUID" not in ds and "Modality" not in ds:
return None
return ds
except Exception:
return None
def _format_all_tags(ds, indent=0):
"""Return a list of printable lines covering every element in a Dataset,
recursing into sequences (SQ) and truncating/summarizing binary values
(pixel data, overlays, etc.) instead of dumping raw bytes."""
lines = []
pad = " " * indent
for elem in ds:
keyword = elem.keyword or "(unknown/private tag)"
tag_str = str(elem.tag)
if elem.VR == "SQ":
items = elem.value or []
lines.append(f"{pad}{tag_str} {keyword:<32} VR=SQ ({len(items)} item(s))")
for i, item in enumerate(items, start=1):
lines.append(f"{pad} [Item {i}]")
lines.extend(_format_all_tags(item, indent + 2))
continue
if elem.VR in ("OB", "OW", "OF", "OL", "OD", "UN") or elem.tag == 0x7FE00010:
try:
length = len(elem.value) if elem.value is not None else 0
except Exception:
length = 0
lines.append(f"{pad}{tag_str} {keyword:<32} VR={elem.VR:<3} <binary data, {length} bytes>")
continue
try:
value_str = str(elem.value)
except Exception:
value_str = "<unreadable value>"
if len(value_str) > 150:
value_str = value_str[:150] + "...(truncated)"
lines.append(f"{pad}{tag_str} {keyword:<32} VR={elem.VR:<3} {value_str}")
return lines
def cmd_inspect(args):
# Only need MongoDB if resolving --filename patterns to a path; if the
# person only passed --file, we never touch the database.
coll = None
if args.filename:
coll = get_collection(args.mongo_uri, args.db, args.collection, args.mongo_username, args.mongo_password)
target_files = resolve_target_files(coll, args.file, args.filename)
if not target_files:
print("Nothing to inspect. Pass --file <path> or --filename <name/pattern>.")
return
for filepath in target_files:
print(f"\n=== {filepath} ===")
if not os.path.isfile(filepath):
print(" File not found on disk.")
continue
if args.full:
ds = read_dicom_dataset(filepath)
if ds is None:
print(" Could not be read as a valid DICOM file.")
continue
for line in _format_all_tags(ds):
print(f" {line}")
continue
row = extract_metadata(filepath)
if row is None:
print(" Could not be read as a valid DICOM file.")
continue
for col_name, _ in DICOM_FIELDS:
value = row.get(col_name, "")
print(f" {col_name:<22} {value if value != '' else '(empty)'}")
print(f" {'FileSizeKB':<22} {row.get('FileSizeKB', '')}")
print()
def cmd_recheck_accession(args):
coll = get_collection(args.mongo_uri, args.db, args.collection, args.mongo_username, args.mongo_password)
target_files = resolve_target_files(coll, getattr(args, "file", None), getattr(args, "filename", None))
if (getattr(args, "file", None) or getattr(args, "filename", None)) and not target_files:
print("Nothing to recheck.")
return
if target_files:
# Targeted mode: recheck exactly these file(s), regardless of their
# current AccessionNumber status (present, blank, or not indexed at
# all yet). Always writes back what's found on disk (an upsert),
# since the point is "make sure this specific file is correct in
# the index" rather than "only fix missing accession numbers".
to_check = [{"FilePath": os.path.abspath(f)} for f in target_files]
scope_desc = "specified file(s) targeted for recheck"
else:
mongo_filter = _missing_accession_filter()
if args.patient_id:
mongo_filter = {"$and": [mongo_filter, {"PatientID": {"$regex": args.patient_id, "$options": "i"}}]}
to_check = list(coll.find(mongo_filter, {"_id": 0, "FilePath": 1}))
if args.limit:
to_check = to_check[: args.limit]
scope_desc = "indexed file(s) currently have a missing/blank AccessionNumber"
if args.patient_id:
scope_desc += f" for PatientID matching '{args.patient_id}'"
total = len(to_check)
print(f"{total} {scope_desc}.")
if total == 0:
return
if args.dry_run:
print("(--dry-run given: re-reading files but NOT writing any updates back to MongoDB)\n")
print("Re-reading each file's metadata from disk...\n")
recovered = []
still_missing = 0
not_found = []
read_errors = []
batch = []
for i, doc in enumerate(to_check, start=1):
filepath = doc["FilePath"]
if not os.path.isfile(filepath):
not_found.append(filepath)
else:
row = extract_metadata(filepath)
if row is None:
read_errors.append(filepath)
else:
has_accession = bool(row.get("AccessionNumber"))
if has_accession:
recovered.append((filepath, row["AccessionNumber"]))
else:
still_missing += 1
if not args.dry_run:
# Targeted-file mode upserts unconditionally (may be a
# brand new file not in the index yet). Missing-filter
# mode only writes back when something was actually
# recovered, since "still missing" files are unchanged.
if target_files or has_accession:
batch.append(UpdateOne({"FilePath": filepath}, {"$set": row}, upsert=bool(target_files)))
if not args.quiet and i % 25 == 0:
sys.stdout.write(f"\rChecked {i}/{total}...")
sys.stdout.flush()
if len(batch) >= BULK_WRITE_BATCH_SIZE:
coll.bulk_write(batch, ordered=False)
batch.clear()
if batch:
coll.bulk_write(batch, ordered=False)
if not args.quiet and total:
sys.stdout.write(f"\rChecked {total}/{total}.\n")
print("\n--- Recheck Results ---")
print(f"Files re-checked: {total}")
print(f"Accession number recovered: {len(recovered)}" + ("" if args.dry_run else " (updated in MongoDB)"))
print(f"Still missing/blank on disk: {still_missing}")
print(f"File no longer found on disk: {len(not_found)}")
print(f"File unreadable / not DICOM: {len(read_errors)}")
if recovered:
print("\nRecovered accession numbers:")
for filepath, acc in recovered[:DISPLAY_LIMIT]:
print(f" {acc:<20} {filepath}")
if len(recovered) > DISPLAY_LIMIT:
print(f" ... and {len(recovered) - DISPLAY_LIMIT} more.")
if not_found:
print("\nFiles missing from disk (index may be stale for these):")
for filepath in not_found[:DISPLAY_LIMIT]:
print(f" {filepath}")
if len(not_found) > DISPLAY_LIMIT:
print(f" ... and {len(not_found) - DISPLAY_LIMIT} more.")
if read_errors:
print("\nFiles that could not be read as DICOM on recheck:")
for filepath in read_errors[:DISPLAY_LIMIT]:
print(f" {filepath}")
if len(read_errors) > DISPLAY_LIMIT:
print(f" ... and {len(read_errors) - DISPLAY_LIMIT} more.")
print()
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _print_table(rows, columns):
"""Print a list of dicts as a simple aligned text table for the given columns."""
if not rows:
return
col_widths = {
col: min(max(len(col), max(len(str(r.get(col, ""))) for r in rows)), 40)
for col in columns
}
header = " ".join(col.ljust(col_widths[col]) for col in columns)
print(header)
print("-" * len(header))
for row in rows:
line = " ".join(
str(row.get(col, ""))[: col_widths[col]].ljust(col_widths[col])
for col in columns
)
print(line)
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
def _add_mongo_args(p):
p.add_argument("--env-file", default=".env",
help="Path to a .env file with MONGO_URI/MONGO_DB/MONGO_COLLECTION/MONGO_USERNAME/MONGO_PASSWORD (default: .env).")
p.add_argument("--mongo-uri", default=None,
help=f"MongoDB connection URI. Overrides .env / $MONGO_URI. (default: {DEFAULT_MONGO_URI})")
p.add_argument("--db", default=None,
help=f"MongoDB database name. Overrides .env / $MONGO_DB. (default: {DEFAULT_DB_NAME})")
p.add_argument("--collection", default=None,
help=f"MongoDB collection name. Overrides .env / $MONGO_COLLECTION. (default: {DEFAULT_COLLECTION_NAME})")
p.add_argument("--mongo-username", default=None,
help="MongoDB username. Overrides .env / $MONGO_USERNAME. Not needed if credentials are already in --mongo-uri.")
p.add_argument("--mongo-password", default=None,
help="MongoDB password. Overrides .env / $MONGO_PASSWORD. Not needed if credentials are already in --mongo-uri.")
def resolve_mongo_args(args):
"""
Fill in args.mongo_uri/db/collection/username/password from (in priority
order):
1. CLI flags, if explicitly given
2. The .env file (args.env_file)
3. Real environment variables
4. Hard-coded defaults (uri/db/collection only — username/password
default to None, meaning "no separate auth, rely on the URI as-is")
"""
load_env_file(args.env_file)
if args.mongo_uri is None:
args.mongo_uri = os.environ.get("MONGO_URI", DEFAULT_MONGO_URI)
if args.db is None:
args.db = os.environ.get("MONGO_DB", DEFAULT_DB_NAME)
if args.collection is None:
args.collection = os.environ.get("MONGO_COLLECTION", DEFAULT_COLLECTION_NAME)
if args.mongo_username is None:
args.mongo_username = os.environ.get("MONGO_USERNAME") or None
if args.mongo_password is None:
args.mongo_password = os.environ.get("MONGO_PASSWORD") or None
def build_parser():
parser = argparse.ArgumentParser(
prog="dicom_indexer.py",
description="Index DICOM folders by metadata into MongoDB, and search the resulting index.",
)
subparsers = parser.add_subparsers(dest="command", required=True)
# scan
p_scan = subparsers.add_parser("scan", help="Scan a folder and index new DICOM files into MongoDB.")
p_scan.add_argument("folder", help="Path to the folder containing DICOM files (scanned recursively).")
_add_mongo_args(p_scan)
p_scan.add_argument("-q", "--quiet", action="store_true", help="Suppress progress output and summary.")
p_scan.add_argument("--verbose", action="store_true", help="Show full error tracebacks on failure.")
p_scan.add_argument(
"--rescan", action="store_true",
help="Re-read and re-index files that are already in the index (normally they're skipped).",
)
p_scan.add_argument(
"--strict-extension", action="store_true",
help=(
"Only attempt to read files with a .dcm/.dicom/.ima/.img extension "
"(or no extension). Fastest option, but will silently skip DICOM "
"files saved with an unusual extension. Off by default."
),
)
p_scan.set_defaults(func=cmd_scan)
# search
p_search = subparsers.add_parser("search", help="Search/filter the DICOM index in MongoDB.")
_add_mongo_args(p_search)
p_search.add_argument("-q", "--query", help="Free-text search across all fields.")
p_search.add_argument("--patient", help="Filter by patient name (substring match).")
p_search.add_argument("--patient-id", dest="patient_id", help="Filter by patient ID (substring match).")
p_search.add_argument("--study", help="Filter by study description (substring match).")
p_search.add_argument("--study-date", dest="study_date", help="Filter by study date, e.g. 2024-01-15 (substring match).")
p_search.add_argument("--modality", help="Filter by modality, e.g. CT, MR, US.")
p_search.add_argument("--series", help="Filter by series description (substring match).")
p_search.add_argument("--body-part", dest="body_part", help="Filter by body part examined.")
p_search.add_argument("--accession", help="Filter by accession number.")
p_search.add_argument(
"--accession-missing", action="store_true",
help="Only show records where AccessionNumber is missing, null, or blank (overrides --accession).",
)
p_search.add_argument("--columns", help="Comma-separated list of columns to display (default: a sensible subset).")
p_search.add_argument("-o", "--output", help="Save filtered results to this CSV path.")
p_search.set_defaults(func=cmd_search)
# summary
p_summary = subparsers.add_parser("summary", help="Show summary statistics for the MongoDB index.")
_add_mongo_args(p_summary)
p_summary.set_defaults(func=cmd_summary)
# export
p_export = subparsers.add_parser("export", help="Export the MongoDB index to a CSV file.")
_add_mongo_args(p_export)
p_export.add_argument("-o", "--output", default="dicom_index.csv", help="Output CSV path (default: dicom_index.csv).")
p_export.set_defaults(func=cmd_export)
# dedupe
p_dedupe = subparsers.add_parser(
"dedupe",
help="Find and remove duplicate index entries — the same DICOM instance indexed under more than one file path.",
)
_add_mongo_args(p_dedupe)
p_dedupe.add_argument(
"--dry-run", action="store_true",
help="Report duplicates found without deleting anything from MongoDB.",
)
p_dedupe.add_argument(
"--keep", choices=["newest", "oldest"], default="newest",
help="Which copy to keep per duplicate group, by IndexedAt (default: newest). "
"A copy whose file still exists on disk is always preferred over one that's gone missing.",
)
p_dedupe.add_argument("-q", "--quiet", action="store_true", help="Suppress per-group detail, show totals only.")
p_dedupe.set_defaults(func=cmd_dedupe)
# recheck-accession
p_recheck = subparsers.add_parser(
"recheck-accession",
help="Re-read on-disk DICOM metadata for indexed files whose AccessionNumber is missing/blank.",
)
_add_mongo_args(p_recheck)
p_recheck.add_argument(
"--dry-run", action="store_true",
help="Re-read files and report findings only; don't write any updates back to MongoDB.",
)
p_recheck.add_argument(
"--limit", type=int, default=None,
help="Only check the first N matching files (useful to sample a large index before a full run).",
)
p_recheck.add_argument(
"--patient-id", dest="patient_id", default=None,
help="Only recheck missing-accession files for this PatientID (substring match).",
)
p_recheck.add_argument(
"--file", action="append", default=None,
help=(
"Recheck one specific file path instead of scanning the whole index for "
"missing accession numbers. Can be given multiple times to check several "
"files. Works even if the file isn't in the index yet."
),
)
p_recheck.add_argument(
"--filename", action="append", default=None,
help=(
"Recheck file(s) by filename (substring match against the indexed FileName) "
"instead of a full path — handy when you know the filename but not its full "
"location. Can be given multiple times. Only matches files already in the index."
),
)
p_recheck.add_argument("-q", "--quiet", action="store_true", help="Suppress progress output.")
p_recheck.set_defaults(func=cmd_recheck_accession)
# inspect
p_inspect = subparsers.add_parser(
"inspect",
help="Read-only: show all DICOM metadata for specific file(s), straight from disk. No MongoDB writes.",
)
_add_mongo_args(p_inspect)
p_inspect.add_argument(
"--file", action="append", default=None,
help="Full path to a file to inspect. Can be given multiple times. Doesn't need to be indexed.",
)
p_inspect.add_argument(
"--filename", action="append", default=None,
help="Inspect file(s) by filename (substring match against the indexed FileName). Can be given multiple times.",
)
p_inspect.add_argument(
"--full", action="store_true",
help=(
"Dump EVERY DICOM tag found in the file, not just the fields normally "
"indexed. Useful for locating unusual/site-specific fields (e.g. a "
"'registration number' stored under OtherPatientIDs or a private tag)."
),
)
p_inspect.set_defaults(func=cmd_inspect)
return parser
def main():
parser = build_parser()
args = parser.parse_args()
resolve_mongo_args(args)
args.func(args)
if __name__ == "__main__":
main()