im_anca/dicom-index-send
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add fix

Browse Source
This commit is contained in:
Anca 2026-07-27 15:43:40 +07:00
parent 56c07edbbd
commit a8a1b62f31
2 changed files with 167 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.env
View File

@ -5,3 +5,4 @@
MONGO_URI=mongodb://root:AtHEntRyPrOchite@172.16.44.35:27017/
MONGO_DB=rsabhk_simrs
MONGO_COLLECTION=dicom-index
PATH_MAPPINGS_FILE=/app/config.json

173
dicom_indexer.py
View File

@ -53,6 +53,7 @@ Usage:
"""
import os
import json
import sys
import csv
import argparse
@ -263,6 +264,21 @@ def extract_metadata(filepath):
# ---------------------------------------------------------------------------
# MongoDB helpers
# ---------------------------------------------------------------------------
def _redact_uri(uri):
"""Hide any inline username:password in a Mongo URI for safe printing."""
if "@" not in uri:
return uri
scheme_sep = uri.find("://")
if scheme_sep == -1:
return uri
scheme = uri[: scheme_sep + 3]
rest = uri[scheme_sep + 3:]
creds, _, host_part = rest.partition("@")
if ":" in creds:
return f"{scheme}***:***@{host_part}"
return f"{scheme}***@{host_part}"
def get_collection(mongo_uri, db_name, collection_name, username=None, password=None):
"""Connect to MongoDB and return the target collection, with indexes set up."""
client_kwargs = {"serverSelectionTimeoutMS": 5000}
@ -271,6 +287,9 @@ def get_collection(mongo_uri, db_name, collection_name, username=None, password=
if password:
client_kwargs["password"] = password
auth_note = " (using --mongo-username/--mongo-password)" if username else ""
print(f"Connecting to MongoDB: {_redact_uri(mongo_uri)}{auth_note} -> db='{db_name}' collection='{collection_name}'")
try:
client = MongoClient(mongo_uri, **client_kwargs)
client.admin.command("ping")
@ -292,6 +311,74 @@ def get_collection(mongo_uri, db_name, collection_name, username=None, password=
return coll
def _slashify(p):
"""Forward-slash version of a path string, used only for in-memory
comparisons — never written back to the index."""
return p.replace("\\", "/")
def load_path_mappings(path):
"""
Load a list of (from, to) path prefix mappings from a JSON file. Accepts
either the same shape as a DICOM router config (a dict with a
"path_mappings" key), or a bare list of {"from": ..., "to": ...}
objects — so you can point this at the exact same config file your
router uses.
"""
if not path:
return []
if not os.path.isfile(path):
print(f"Warning: path-mappings file '{path}' not found; proceeding without path mappings.", file=sys.stderr)
return []
try:
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
except Exception as exc:
print(f"Warning: could not parse path-mappings file '{path}': {exc}", file=sys.stderr)
return []
if isinstance(data, dict) and "path_mappings" in data:
data = data["path_mappings"]
if not isinstance(data, list):
print(f"Warning: path-mappings file '{path}' doesn't contain a list of "
f"{{'from', 'to'}} objects; ignoring it.", file=sys.stderr)
return []
mappings = []
for entry in data:
frm, to = entry.get("from"), entry.get("to")
if frm and to:
mappings.append((frm, to))
return mappings
def build_mapped_lookup_set(paths, mappings):
"""
Given the raw FilePath values already in the index and a list of
(from, to) path mappings, return an expanded set — normalized to
forward slashes for comparison only, the originals in MongoDB are
never touched — that also includes each path translated through every
mapping, in both directions.
This lets `scan` recognize a file as already indexed even when it's
being scanned from a different environment than it was originally
indexed from (e.g. it was indexed from a Windows dev path but is now
being scanned from its Linux production mount, or vice versa), instead
of re-indexing it as if it were new.
"""
expanded = set()
norm_mappings = [(_slashify(f).rstrip("/"), _slashify(t).rstrip("/")) for f, t in mappings]
for p in paths:
norm_p = _slashify(p)
expanded.add(norm_p)
for frm, to in norm_mappings:
if norm_p.startswith(frm):
expanded.add(to + norm_p[len(frm):])
if norm_p.startswith(to):
expanded.add(frm + norm_p[len(to):])
return expanded
def scan_folder(root_folder, already_indexed, show_progress=True, strict_extension=False):
"""
Walk root_folder recursively and yield a metadata dict for every valid,
@ -312,9 +399,10 @@ def scan_folder(root_folder, already_indexed, show_progress=True, strict_extensi
for fn in filenames:
filepath = os.path.join(dirpath, fn)
filepath_abs = os.path.abspath(filepath)
lookup_key = _slashify(filepath_abs)
count_seen += 1
if filepath_abs in already_indexed:
if lookup_key in already_indexed:
count_skipped_indexed += 1
else:
ext = os.path.splitext(fn)[1].lower()
@ -365,8 +453,30 @@ def cmd_scan(args):
print("--rescan given: re-reading all files, even ones already in the index.")
else:
print("Loading list of already-indexed files...")
already_indexed = {doc["FilePath"] for doc in coll.find({}, {"FilePath": 1, "_id": 0})}
print(f"Found {len(already_indexed)} file(s) already in the index.")
raw_paths = {doc["FilePath"] for doc in coll.find({}, {"FilePath": 1, "_id": 0})}
mappings = []
for pair in (getattr(args, "path_mapping", None) or []):
if "=" not in pair:
print(f"Warning: ignoring malformed --path-mapping '{pair}' (expected FROM=TO).", file=sys.stderr)
continue
frm, _, to = pair.partition("=")
mappings.append((frm, to))
mappings_file = getattr(args, "path_mappings_file", None) or os.environ.get("PATH_MAPPINGS_FILE")
if mappings_file:
mappings.extend(load_path_mappings(mappings_file))
if mappings:
print(f"Using {len(mappings)} path mapping(s) to recognize already-indexed files "
f"across environments (e.g. a Windows dev path vs. its Linux production mount).")
already_indexed = build_mapped_lookup_set(raw_paths, mappings)
else:
# Normalize for comparison only — what's actually stored in
# MongoDB (raw_paths) is never altered.
already_indexed = {_slashify(p) for p in raw_paths}
print(f"Found {len(raw_paths)} file(s) already in the index.")
print(f"Scanning '{folder}' for DICOM files...")
@ -378,13 +488,18 @@ def cmd_scan(args):
batch = []
write_errors = []
def flush_batch():
if not batch:
return
try:
coll.bulk_write(batch, ordered=False)
except PyMongoError as exc:
print(f"\nWarning: a batch write to MongoDB failed: {exc}", file=sys.stderr)
msg = f"Warning: a batch write to MongoDB failed ({len(batch)} file(s) in this batch): {exc}"
print(f"\n{msg}")
print(msg, file=sys.stderr)
write_errors.append(msg)
batch.clear()
try:
@ -418,10 +533,27 @@ def cmd_scan(args):
if row_count == 0:
print("No new DICOM files found (nothing to add to the index).")
server_total = coll.count_documents({})
print(f"(For reference: {server_total} document(s) currently exist in {args.db}.{args.collection} on the server.)")
sys.exit(0)
print(f"Indexed {row_count} new DICOM file(s) into MongoDB "
f"({args.db}.{args.collection} at {args.mongo_uri}).")
f"({args.db}.{args.collection} at {_redact_uri(args.mongo_uri)}).")
if write_errors:
print(f"\n{len(write_errors)} batch write(s) failed during this scan — the index is "
f"INCOMPLETE. See warnings above for details.")
server_total = coll.count_documents({})
print(f"Total documents now in {args.db}.{args.collection} on the server: {server_total}")
if server_total < row_count:
print(
"WARNING: the server-side count is lower than the number of files just indexed. "
"This usually means writes are landing in a different database/collection than "
"you're checking, or a batch write failed silently earlier in the log above. "
"Double-check --env-file / MONGO_URI / MONGO_DB / MONGO_COLLECTION match what "
"you're using to verify the index."
)
if not args.quiet:
print("\n--- New Files This Scan ---")
@ -435,6 +567,9 @@ def cmd_scan(args):
print(f" {label:<10} {count}")
print()
if write_errors:
sys.exit(1)
# ---------------------------------------------------------------------------
# Subcommand: search
@ -817,7 +952,7 @@ def cmd_recheck_accession(args):
# all yet). Always writes back what's found on disk (an upsert),
# since the point is "make sure this specific file is correct in
# the index" rather than "only fix missing accession numbers".
to_check = [{"FilePath": os.path.abspath(f)} for f in target_files]
to_check = [{"FilePath": f} for f in target_files]
scope_desc = "specified file(s) targeted for recheck"
else:
mongo_filter = _missing_accession_filter()
@ -964,7 +1099,13 @@ def resolve_mongo_args(args):
4. Hard-coded defaults (uri/db/collection only — username/password
default to None, meaning "no separate auth, rely on the URI as-is")
"""
load_env_file(args.env_file)
env_loaded = load_env_file(args.env_file)
env_abs_path = os.path.abspath(args.env_file)
if env_loaded:
print(f"Loaded config from: {env_abs_path}")
else:
print(f"Note: no .env file found at '{env_abs_path}' — using CLI flags / real "
f"environment variables / built-in defaults instead.")
if args.mongo_uri is None:
args.mongo_uri = os.environ.get("MONGO_URI", DEFAULT_MONGO_URI)
@ -1003,6 +1144,24 @@ def build_parser():
"files saved with an unusual extension. Off by default."
),
)
p_scan.add_argument(
"--path-mappings-file", default=None,
help=(
"Path to a JSON file of {\"from\": ..., \"to\": ...} path prefix mappings "
"(the same shape/file your DICOM router config uses is fine, e.g. it can "
"contain a top-level \"path_mappings\" key). Used only to recognize files "
"as already indexed when scanning from a different environment/root than "
"they were originally indexed from — it never changes what's stored. "
"Can also be set via $PATH_MAPPINGS_FILE."
),
)
p_scan.add_argument(
"--path-mapping", action="append", default=None, metavar="FROM=TO",
help=(
"Add a single path mapping inline, e.g. --path-mapping \"Z:/2026=/mnt/data/2026\". "
"Can be given multiple times; combines with --path-mappings-file if both are given."
),
)
p_scan.set_defaults(func=cmd_scan)
# search