Merge pull request #1014 from vhoulbreque-withings/960-fix-requirements-txt-fixer-duplicated-elements
Remove duplicated packages
diff --git a/pre_commit_hooks/requirements_txt_fixer.py b/pre_commit_hooks/requirements_txt_fixer.py
index 5884394..261acc9 100644
--- a/pre_commit_hooks/requirements_txt_fixer.py
+++ b/pre_commit_hooks/requirements_txt_fixer.py
@@ -45,6 +45,11 @@
elif requirement.value == b'\n':
return False
else:
+ # if 2 requirements have the same name, the one with comments
+ # needs to go first (so that when removing duplicates, the one
+ # with comments is kept)
+ if self.name == requirement.name:
+ return bool(self.comments) > bool(requirement.comments)
return self.name < requirement.name
def is_complete(self) -> bool:
@@ -113,10 +118,14 @@
if req.value != b'pkg-resources==0.0.0\n'
]
+ # sort the requirements and remove duplicates
+ prev = None
for requirement in sorted(requirements):
after.extend(requirement.comments)
assert requirement.value, requirement.value
- after.append(requirement.value)
+ if prev is None or requirement.value != prev.value:
+ after.append(requirement.value)
+ prev = requirement
after.extend(rest)
after_string = b''.join(after)
diff --git a/tests/requirements_txt_fixer_test.py b/tests/requirements_txt_fixer_test.py
index b725afa..c400be1 100644
--- a/tests/requirements_txt_fixer_test.py
+++ b/tests/requirements_txt_fixer_test.py
@@ -68,6 +68,12 @@
b'f<=2\n'
b'g<2\n',
),
+ (b'a==1\nb==1\na==1\n', FAIL, b'a==1\nb==1\n'),
+ (
+ b'a==1\nb==1\n#comment about a\na==1\n',
+ FAIL,
+ b'#comment about a\na==1\nb==1\n',
+ ),
(b'ocflib\nDjango\nPyMySQL\n', FAIL, b'Django\nocflib\nPyMySQL\n'),
(
b'-e git+ssh://git_url@tag#egg=ocflib\nDjango\nPyMySQL\n',
