diff --git a/README.md b/README.md index 0d88c16..25df4f8 100644 --- a/README.md +++ b/README.md @@ -57,19 +57,21 @@ Exclude one or more path patterns with `--exclude`: git-annex-duplicates --exclude '*.iso' --exclude 'archive/**' ``` -Additional arguments are passed to `git annex find`. Place `--` before options -that belong to `git annex find`, so they are not interpreted as options to -`git-annex-duplicates`: +Find duplicates in the given directories: ```sh -# Only report duplicates whose content is present in this repository -git-annex-duplicates -- --in=here - -# Combine a local exclusion with a git-annex find option -git-annex-duplicates --exclude 'archive/**' -- --in=here +git-annex-duplicates dir_a/ dir_b/ ``` -See `git annex find --help` for the available filters. +### Finding duplicates across directories + +To find duplicates that are duplicated across directories, use the `--across` flag: + +```sh +git-annex-duplicates --across data/ new_data/ +``` + +This will only list duplicate files that are found in more than one of the given directories. ## How it works diff --git a/git_annex_duplicates.py b/git_annex_duplicates.py index 42d0aee..662e60b 100644 --- a/git_annex_duplicates.py +++ b/git_annex_duplicates.py @@ -1,21 +1,23 @@ import subprocess import collections +import os.path import re + +from pathlib import Path + import click -@click.command() -@click.option("--exclude", multiple=True, default=[]) -@click.argument("arguments", nargs=-1) -def main(exclude, arguments): +def collect_files(exclude, directories): # build command - find_cmd = ["git", "annex", "find", "--include", "*", "--format=${key}:${file}\n", *arguments] + find_cmd = ["git", "annex", "find", "--include", "*", "--format=${key}:${file}\n", *directories] for e in exclude: find_cmd += ["--exclude", e] # run result = subprocess.run(find_cmd, capture_output=True) + # collect output files = collections.defaultdict(list) for l in result.stdout.decode("utf-8").split("\n"): if l == "": @@ -24,11 +26,51 @@ def main(exclude, arguments): files[key].append(file) + return files + + +def print_files(fs): + for f in fs: + print(f) + + +def resolve_parent(p): + return p.parent.resolve() / p.stem + + +@click.command() +@click.option("--exclude", multiple=True, default=[]) +@click.option("--across", is_flag=True, default=False) +@click.argument("directories", nargs=-1, default=["."], type=click.Path(exists=True, file_okay=False, path_type=Path)) +def main(exclude, across, directories): + + if across and len(directories) < 2: + raise click.UsageError("--across requires at least two directories") + + directories = [d.resolve() for d in directories] # for is_relative_to() + + files = collect_files(exclude, directories) + for k in files: if len(files[k]) > 1: - for f in files[k]: - print(f) - print() + if not across: + print_files(files[k]) + print() + else: + # A file is a duplicate across the directories iff it's in more than one + # directory. + count = 0 + found_across = False + for directory in directories: + if any(resolve_parent(Path(f)).is_relative_to(directory) for f in files[k]): + count += 1 + + if count > 1: + found_across = True + break + if found_across: + print_files(files[k]) + print() if __name__ == "__main__":