✨ Find duplicates across directories.

The new CLI flag --across now lets users list duplicates only if they
are found in multiple of the given directories.

Closes gh-5.

This also removes support for arbitrary arguments to `git annex find`,
as we need to know which of the CLI arguments are directories to search
in. This might be re-introduced later.

Closes gh-4.

README.md

@@ -57,19 +57,21 @@ Exclude one or more path patterns with `--exclude`:
 git-annex-duplicates --exclude '*.iso' --exclude 'archive/**'
 ```
 
-Additional arguments are passed to `git annex find`. Place `--` before options
+Find duplicates in the given directories:
-that belong to `git annex find`, so they are not interpreted as options to
-`git-annex-duplicates`:
 
 ```sh
-# Only report duplicates whose content is present in this repository
+git-annex-duplicates dir_a/ dir_b/
-git-annex-duplicates -- --in=here
-
-# Combine a local exclusion with a git-annex find option
-git-annex-duplicates --exclude 'archive/**' -- --in=here
 ```
 
-See `git annex find --help` for the available filters.
+### Finding duplicates across directories
+
+To find duplicates that are duplicated across directories, use the `--across` flag:
+
+```sh
+git-annex-duplicates --across data/ new_data/
+```
+
+This will only list duplicate files that are found in more than one of the given directories.
 
 
 ## How it works

git_annex_duplicates.py

@@ -1,21 +1,23 @@
 import subprocess
 import collections
+import os.path
 import re
+
+from pathlib import Path
+
 import click
 
 
-@click.command()
+def collect_files(exclude, directories):
-@click.option("--exclude", multiple=True, default=[])
-@click.argument("arguments", nargs=-1)
-def main(exclude, arguments):
     # build command
-    find_cmd = ["git", "annex", "find", "--include", "*", "--format=${key}:${file}\n", *arguments]
+    find_cmd = ["git", "annex", "find", "--include", "*", "--format=${key}:${file}\n", *directories]
     for e in exclude:
         find_cmd += ["--exclude", e]
 
     # run
     result = subprocess.run(find_cmd, capture_output=True)
 
+    # collect output
     files = collections.defaultdict(list)
     for l in result.stdout.decode("utf-8").split("\n"):
         if l == "":
@@ -24,10 +26,50 @@ def main(exclude, arguments):
 
         files[key].append(file)
 
+    return files
+
+
+def print_files(fs):
+    for f in fs:
+        print(f)
+
+
+def resolve_parent(p):
+    return p.parent.resolve() / p.stem
+
+
+@click.command()
+@click.option("--exclude", multiple=True, default=[])
+@click.option("--across", is_flag=True, default=False)
+@click.argument("directories", nargs=-1, default=["."], type=click.Path(exists=True, file_okay=False, path_type=Path))
+def main(exclude, across, directories):
+
+    if across and len(directories) < 2:
+        raise click.UsageError("--across requires at least two directories")
+
+    directories = [d.resolve() for d in directories]  # for is_relative_to()
+
+    files = collect_files(exclude, directories)
+
     for k in files:
         if len(files[k]) > 1:
-            for f in files[k]:
+            if not across:
-                print(f)
+                print_files(files[k])
+                print()
+            else:
+                # A file is a duplicate across the directories iff it's in more than one
+                # directory.
+                count = 0
+                found_across = False
+                for directory in directories:
+                    if any(resolve_parent(Path(f)).is_relative_to(directory) for f in files[k]):
+                       count += 1
+
+                    if count > 1:
+                        found_across = True
+                        break
+                if found_across:
+                    print_files(files[k])
                     print()