From 7e80e38918de9f79bd41302678b6f19daafa4af7 Mon Sep 17 00:00:00 2001
From: Mike Gerber <mike@mike-gerber.de>
Date: Thu, 3 Sep 2026 13:38:31 +0200
Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Find=20duplicates=20across=20direct?=
 =?UTF-8?q?ories.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The new CLI flag --across now lets users list duplicates only if they
are found in multiple of the given directories.

Closes gh-5.

This also removes support for arbitrary arguments to `git annex find`,
as we need to know which of the CLI arguments are directories to search
in. This might be re-introduced later.

Closes gh-4.
---
 README.md               | 20 +++++++-------
 git_annex_duplicates.py | 58 +++++++++++++++++++++++++++++++++++------
 2 files changed, 61 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 0d88c16..25df4f8 100644
--- a/README.md
+++ b/README.md
@@ -57,19 +57,21 @@ Exclude one or more path patterns with `--exclude`:
 git-annex-duplicates --exclude '*.iso' --exclude 'archive/**'
 ```
 
-Additional arguments are passed to `git annex find`. Place `--` before options
-that belong to `git annex find`, so they are not interpreted as options to
-`git-annex-duplicates`:
+Find duplicates in the given directories:
 
 ```sh
-# Only report duplicates whose content is present in this repository
-git-annex-duplicates -- --in=here
-
-# Combine a local exclusion with a git-annex find option
-git-annex-duplicates --exclude 'archive/**' -- --in=here
+git-annex-duplicates dir_a/ dir_b/
 ```
 
-See `git annex find --help` for the available filters.
+### Finding duplicates across directories
+
+To find duplicates that are duplicated across directories, use the `--across` flag:
+
+```sh
+git-annex-duplicates --across data/ new_data/
+```
+
+This will only list duplicate files that are found in more than one of the given directories.
 
 
 ## How it works
diff --git a/git_annex_duplicates.py b/git_annex_duplicates.py
index 42d0aee..662e60b 100644
--- a/git_annex_duplicates.py
+++ b/git_annex_duplicates.py
@@ -1,21 +1,23 @@
 import subprocess
 import collections
+import os.path
 import re
+
+from pathlib import Path
+
 import click
 
 
-@click.command()
-@click.option("--exclude", multiple=True, default=[])
-@click.argument("arguments", nargs=-1)
-def main(exclude, arguments):
+def collect_files(exclude, directories):
     # build command
-    find_cmd = ["git", "annex", "find", "--include", "*", "--format=${key}:${file}\n", *arguments]
+    find_cmd = ["git", "annex", "find", "--include", "*", "--format=${key}:${file}\n", *directories]
     for e in exclude:
         find_cmd += ["--exclude", e]
 
     # run
     result = subprocess.run(find_cmd, capture_output=True)
 
+    # collect output
     files = collections.defaultdict(list)
     for l in result.stdout.decode("utf-8").split("\n"):
         if l == "":
@@ -24,11 +26,51 @@ def main(exclude, arguments):
 
         files[key].append(file)
 
+    return files
+
+
+def print_files(fs):
+    for f in fs:
+        print(f)
+
+
+def resolve_parent(p):
+    return p.parent.resolve() / p.stem
+
+
+@click.command()
+@click.option("--exclude", multiple=True, default=[])
+@click.option("--across", is_flag=True, default=False)
+@click.argument("directories", nargs=-1, default=["."], type=click.Path(exists=True, file_okay=False, path_type=Path))
+def main(exclude, across, directories):
+
+    if across and len(directories) < 2:
+        raise click.UsageError("--across requires at least two directories")
+
+    directories = [d.resolve() for d in directories]  # for is_relative_to()
+
+    files = collect_files(exclude, directories)
+
     for k in files:
         if len(files[k]) > 1:
-            for f in files[k]:
-                print(f)
-            print()
+            if not across:
+                print_files(files[k])
+                print()
+            else:
+                # A file is a duplicate across the directories iff it's in more than one
+                # directory.
+                count = 0
+                found_across = False
+                for directory in directories:
+                    if any(resolve_parent(Path(f)).is_relative_to(directory) for f in files[k]):
+                       count += 1
+
+                    if count > 1:
+                        found_across = True
+                        break
+                if found_across:
+                    print_files(files[k])
+                    print()
 
 
 if __name__ == "__main__":