From a8a1b62f31d7553bd6232604dbc87150bcd1037c Mon Sep 17 00:00:00 2001 From: Anca Date: Mon, 27 Jul 2026 15:43:40 +0700 Subject: [PATCH] add fix --- .env | 1 + dicom_indexer.py | 173 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 167 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 9523f79..37996fc 100644 --- a/.env +++ b/.env @@ -5,3 +5,4 @@ MONGO_URI=mongodb://root:AtHEntRyPrOchite@172.16.44.35:27017/ MONGO_DB=rsabhk_simrs MONGO_COLLECTION=dicom-index +PATH_MAPPINGS_FILE=/app/config.json \ No newline at end of file diff --git a/dicom_indexer.py b/dicom_indexer.py index f1df09c..40dbc08 100644 --- a/dicom_indexer.py +++ b/dicom_indexer.py @@ -53,6 +53,7 @@ Usage: """ import os +import json import sys import csv import argparse @@ -263,6 +264,21 @@ def extract_metadata(filepath): # --------------------------------------------------------------------------- # MongoDB helpers # --------------------------------------------------------------------------- +def _redact_uri(uri): + """Hide any inline username:password in a Mongo URI for safe printing.""" + if "@" not in uri: + return uri + scheme_sep = uri.find("://") + if scheme_sep == -1: + return uri + scheme = uri[: scheme_sep + 3] + rest = uri[scheme_sep + 3:] + creds, _, host_part = rest.partition("@") + if ":" in creds: + return f"{scheme}***:***@{host_part}" + return f"{scheme}***@{host_part}" + + def get_collection(mongo_uri, db_name, collection_name, username=None, password=None): """Connect to MongoDB and return the target collection, with indexes set up.""" client_kwargs = {"serverSelectionTimeoutMS": 5000} @@ -271,6 +287,9 @@ def get_collection(mongo_uri, db_name, collection_name, username=None, password= if password: client_kwargs["password"] = password + auth_note = " (using --mongo-username/--mongo-password)" if username else "" + print(f"Connecting to MongoDB: {_redact_uri(mongo_uri)}{auth_note} -> db='{db_name}' collection='{collection_name}'") + try: client = MongoClient(mongo_uri, **client_kwargs) client.admin.command("ping") @@ -292,6 +311,74 @@ def get_collection(mongo_uri, db_name, collection_name, username=None, password= return coll +def _slashify(p): + """Forward-slash version of a path string, used only for in-memory + comparisons — never written back to the index.""" + return p.replace("\\", "/") + + +def load_path_mappings(path): + """ + Load a list of (from, to) path prefix mappings from a JSON file. Accepts + either the same shape as a DICOM router config (a dict with a + "path_mappings" key), or a bare list of {"from": ..., "to": ...} + objects — so you can point this at the exact same config file your + router uses. + """ + if not path: + return [] + if not os.path.isfile(path): + print(f"Warning: path-mappings file '{path}' not found; proceeding without path mappings.", file=sys.stderr) + return [] + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + except Exception as exc: + print(f"Warning: could not parse path-mappings file '{path}': {exc}", file=sys.stderr) + return [] + + if isinstance(data, dict) and "path_mappings" in data: + data = data["path_mappings"] + if not isinstance(data, list): + print(f"Warning: path-mappings file '{path}' doesn't contain a list of " + f"{{'from', 'to'}} objects; ignoring it.", file=sys.stderr) + return [] + + mappings = [] + for entry in data: + frm, to = entry.get("from"), entry.get("to") + if frm and to: + mappings.append((frm, to)) + return mappings + + +def build_mapped_lookup_set(paths, mappings): + """ + Given the raw FilePath values already in the index and a list of + (from, to) path mappings, return an expanded set — normalized to + forward slashes for comparison only, the originals in MongoDB are + never touched — that also includes each path translated through every + mapping, in both directions. + + This lets `scan` recognize a file as already indexed even when it's + being scanned from a different environment than it was originally + indexed from (e.g. it was indexed from a Windows dev path but is now + being scanned from its Linux production mount, or vice versa), instead + of re-indexing it as if it were new. + """ + expanded = set() + norm_mappings = [(_slashify(f).rstrip("/"), _slashify(t).rstrip("/")) for f, t in mappings] + for p in paths: + norm_p = _slashify(p) + expanded.add(norm_p) + for frm, to in norm_mappings: + if norm_p.startswith(frm): + expanded.add(to + norm_p[len(frm):]) + if norm_p.startswith(to): + expanded.add(frm + norm_p[len(to):]) + return expanded + + def scan_folder(root_folder, already_indexed, show_progress=True, strict_extension=False): """ Walk root_folder recursively and yield a metadata dict for every valid, @@ -312,9 +399,10 @@ def scan_folder(root_folder, already_indexed, show_progress=True, strict_extensi for fn in filenames: filepath = os.path.join(dirpath, fn) filepath_abs = os.path.abspath(filepath) + lookup_key = _slashify(filepath_abs) count_seen += 1 - if filepath_abs in already_indexed: + if lookup_key in already_indexed: count_skipped_indexed += 1 else: ext = os.path.splitext(fn)[1].lower() @@ -365,8 +453,30 @@ def cmd_scan(args): print("--rescan given: re-reading all files, even ones already in the index.") else: print("Loading list of already-indexed files...") - already_indexed = {doc["FilePath"] for doc in coll.find({}, {"FilePath": 1, "_id": 0})} - print(f"Found {len(already_indexed)} file(s) already in the index.") + raw_paths = {doc["FilePath"] for doc in coll.find({}, {"FilePath": 1, "_id": 0})} + + mappings = [] + for pair in (getattr(args, "path_mapping", None) or []): + if "=" not in pair: + print(f"Warning: ignoring malformed --path-mapping '{pair}' (expected FROM=TO).", file=sys.stderr) + continue + frm, _, to = pair.partition("=") + mappings.append((frm, to)) + + mappings_file = getattr(args, "path_mappings_file", None) or os.environ.get("PATH_MAPPINGS_FILE") + if mappings_file: + mappings.extend(load_path_mappings(mappings_file)) + + if mappings: + print(f"Using {len(mappings)} path mapping(s) to recognize already-indexed files " + f"across environments (e.g. a Windows dev path vs. its Linux production mount).") + already_indexed = build_mapped_lookup_set(raw_paths, mappings) + else: + # Normalize for comparison only — what's actually stored in + # MongoDB (raw_paths) is never altered. + already_indexed = {_slashify(p) for p in raw_paths} + + print(f"Found {len(raw_paths)} file(s) already in the index.") print(f"Scanning '{folder}' for DICOM files...") @@ -378,13 +488,18 @@ def cmd_scan(args): batch = [] + write_errors = [] + def flush_batch(): if not batch: return try: coll.bulk_write(batch, ordered=False) except PyMongoError as exc: - print(f"\nWarning: a batch write to MongoDB failed: {exc}", file=sys.stderr) + msg = f"Warning: a batch write to MongoDB failed ({len(batch)} file(s) in this batch): {exc}" + print(f"\n{msg}") + print(msg, file=sys.stderr) + write_errors.append(msg) batch.clear() try: @@ -418,10 +533,27 @@ def cmd_scan(args): if row_count == 0: print("No new DICOM files found (nothing to add to the index).") + server_total = coll.count_documents({}) + print(f"(For reference: {server_total} document(s) currently exist in {args.db}.{args.collection} on the server.)") sys.exit(0) print(f"Indexed {row_count} new DICOM file(s) into MongoDB " - f"({args.db}.{args.collection} at {args.mongo_uri}).") + f"({args.db}.{args.collection} at {_redact_uri(args.mongo_uri)}).") + + if write_errors: + print(f"\n{len(write_errors)} batch write(s) failed during this scan — the index is " + f"INCOMPLETE. See warnings above for details.") + + server_total = coll.count_documents({}) + print(f"Total documents now in {args.db}.{args.collection} on the server: {server_total}") + if server_total < row_count: + print( + "WARNING: the server-side count is lower than the number of files just indexed. " + "This usually means writes are landing in a different database/collection than " + "you're checking, or a batch write failed silently earlier in the log above. " + "Double-check --env-file / MONGO_URI / MONGO_DB / MONGO_COLLECTION match what " + "you're using to verify the index." + ) if not args.quiet: print("\n--- New Files This Scan ---") @@ -435,6 +567,9 @@ def cmd_scan(args): print(f" {label:<10} {count}") print() + if write_errors: + sys.exit(1) + # --------------------------------------------------------------------------- # Subcommand: search @@ -817,7 +952,7 @@ def cmd_recheck_accession(args): # all yet). Always writes back what's found on disk (an upsert), # since the point is "make sure this specific file is correct in # the index" rather than "only fix missing accession numbers". - to_check = [{"FilePath": os.path.abspath(f)} for f in target_files] + to_check = [{"FilePath": f} for f in target_files] scope_desc = "specified file(s) targeted for recheck" else: mongo_filter = _missing_accession_filter() @@ -964,7 +1099,13 @@ def resolve_mongo_args(args): 4. Hard-coded defaults (uri/db/collection only — username/password default to None, meaning "no separate auth, rely on the URI as-is") """ - load_env_file(args.env_file) + env_loaded = load_env_file(args.env_file) + env_abs_path = os.path.abspath(args.env_file) + if env_loaded: + print(f"Loaded config from: {env_abs_path}") + else: + print(f"Note: no .env file found at '{env_abs_path}' — using CLI flags / real " + f"environment variables / built-in defaults instead.") if args.mongo_uri is None: args.mongo_uri = os.environ.get("MONGO_URI", DEFAULT_MONGO_URI) @@ -1003,6 +1144,24 @@ def build_parser(): "files saved with an unusual extension. Off by default." ), ) + p_scan.add_argument( + "--path-mappings-file", default=None, + help=( + "Path to a JSON file of {\"from\": ..., \"to\": ...} path prefix mappings " + "(the same shape/file your DICOM router config uses is fine, e.g. it can " + "contain a top-level \"path_mappings\" key). Used only to recognize files " + "as already indexed when scanning from a different environment/root than " + "they were originally indexed from — it never changes what's stored. " + "Can also be set via $PATH_MAPPINGS_FILE." + ), + ) + p_scan.add_argument( + "--path-mapping", action="append", default=None, metavar="FROM=TO", + help=( + "Add a single path mapping inline, e.g. --path-mapping \"Z:/2026=/mnt/data/2026\". " + "Can be given multiple times; combines with --path-mappings-file if both are given." + ), + ) p_scan.set_defaults(func=cmd_scan) # search