Check for duplicates across dictionaries (#1501) * Check for duplicates across dictionaries And no longer allow self-corrections within these checks (given we exclude them elsewhere already) * Improve the "is an error itself in another dictionary file" logging * Try @larsoner 's proposed improvement to global dictionary checking * Fix the syntax error and I think fix the cross-dictionary detection * Ensure we catch all the error cases * ENH: Refactor * Move chack back to rare * Fix the substitution variables for an error * Move wen back from rare to names * Finish moving chack to rare * Remove a few en-GB to en-US conversions from rare Co-authored-by: Eric Larson <larson.eric.d@gmail.com>
diff --git a/.travis.yml b/.travis.yml index 535d8c0..2f301f7 100644 --- a/.travis.yml +++ b/.travis.yml
@@ -30,7 +30,7 @@ - source venv/bin/activate - python --version # just to check - pip install -U pip wheel # upgrade to latest pip find 3.5 wheels; wheel to avoid errors - - retry pip install pytest pytest-cov flake8 coverage codecov chardet setuptools docutils + - retry pip install pytest pytest-cov pytest-dependency flake8 coverage codecov chardet setuptools docutils - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then retry pip install aspell-python-py2; fi - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then retry pip install aspell-python-py3; fi - cd $SRC_DIR
diff --git a/appveyor.yml b/appveyor.yml index 201e011..d7a9999 100644 --- a/appveyor.yml +++ b/appveyor.yml
@@ -16,7 +16,7 @@ install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "pip install pytest pytest-cov setuptools flake8 coverage chardet codecov" + - "pip install pytest pytest-cov pytest-dependency setuptools flake8 coverage chardet codecov" - "python setup.py develop" build: false # Not a C# project, build stuff at the test step instead.
diff --git a/codespell_lib/data/dictionary.txt b/codespell_lib/data/dictionary.txt index 1e2ffb9..5b65e67 100644 --- a/codespell_lib/data/dictionary.txt +++ b/codespell_lib/data/dictionary.txt
@@ -4223,7 +4223,6 @@ chache->cache chached->cached chacheline->cacheline -chack->check, chalk, chaeck->check chaecked->checked chaecker->checker
diff --git a/codespell_lib/data/dictionary_names.txt b/codespell_lib/data/dictionary_names.txt index e58d4c7..a9113db 100644 --- a/codespell_lib/data/dictionary_names.txt +++ b/codespell_lib/data/dictionary_names.txt
@@ -6,5 +6,4 @@ que->queue sargent->sergeant, argent, tim->time -wen->when -wight->weight, white, right, write, +wen->we, when,
diff --git a/codespell_lib/data/dictionary_rare.txt b/codespell_lib/data/dictionary_rare.txt index f79ee60..cd425c0 100644 --- a/codespell_lib/data/dictionary_rare.txt +++ b/codespell_lib/data/dictionary_rare.txt
@@ -13,8 +13,7 @@ busses->buses calculatable->calculable cant->can't -catalogue->catalog -chack->check, cheque, +chack->check, chalk, cheque, chancel->cancel chancels->cancels circularly->circular @@ -79,7 +78,6 @@ marge->merge mater->matter, master, mother, medias->media, mediums, -memorise->memorize midwifes->midwives moil->soil, mohel, mot->not @@ -133,12 +131,11 @@ wan->want want's->wants wee->we -wen->we, when, whats->what's whet->when, what, wet, whiling->while -wight->weight, white, right, -wights->weights, whites, rights, +wight->weight, white, right, write, +wights->weights, whites, rights, writes, wit->with withe->with wither->either, whether, weather,
diff --git a/codespell_lib/tests/test_dictionary.py b/codespell_lib/tests/test_dictionary.py index a71e4c7..2349874 100644 --- a/codespell_lib/tests/test_dictionary.py +++ b/codespell_lib/tests/test_dictionary.py
@@ -28,6 +28,8 @@ ws = re.compile(r'.*\s.*') # whitespace comma = re.compile(r'.*,.*') # comma +global_err_dicts = dict() +global_pairs = set() # Filename, should be seen as errors in aspell or not _data_dir = op.join(op.dirname(__file__), '..', 'data') @@ -169,21 +171,78 @@ _check_err_rep(err, rep, (err_aspell, rep_aspell), 'dummy') +# allow some duplicates, like "m-i-n-i-m-i-s-e", or "c-a-l-c-u-l-a-t-a-b-l-e" +allowed_dups = { + ('dictionary.txt', 'dictionary_en-GB_to_en-US.txt'), + ('dictionary.txt', 'dictionary_rare.txt'), +} + + @fname_params +@pytest.mark.dependency(name='dictionary loop') def test_dictionary_looping(fname, in_aspell): """Test that all dictionary entries are valid.""" - err_dict = dict() + this_err_dict = dict() + short_fname = op.basename(fname) with open(fname, 'rb') as fid: for line in fid: err, rep = line.decode('utf-8').split('->') err = err.lower() - assert err not in err_dict, 'error %r already exists' % err + assert err not in this_err_dict, \ + 'error %r already exists in %s' % (err, short_fname) rep = rep.rstrip('\n') reps = [r.strip() for r in rep.lower().split(',')] reps = [r for r in reps if len(r)] - err_dict[err] = reps - # check for corrections that are errors (but not self replacements) - for err in err_dict: - for r in err_dict[err]: - assert (r not in err_dict) or (r in err_dict[r]), \ - ('error %s: correction %s is an error itself' % (err, r)) + this_err_dict[err] = reps + # 1. check the dict against itself (diagonal) + for err in this_err_dict: + for r in this_err_dict[err]: + assert r not in this_err_dict, \ + ('error %s: correction %s is an error itself in the same ' + 'dictionary file %s' % (err, r, short_fname)) + pair = (short_fname, short_fname) + assert pair not in global_pairs + global_pairs.add(pair) + for other_fname, other_err_dict in global_err_dicts.items(): + # error duplication (eventually maybe we should just merge?) + for err in this_err_dict: + assert err not in other_err_dict, \ + ('error %r in dictionary %s already exists in dictionary ' + '%s' % (err, short_fname, other_fname)) + # 2. check corrections in this dict against other dicts (upper) + pair = (short_fname, other_fname) + if pair not in allowed_dups: + for err in this_err_dict: + assert err not in other_err_dict, \ + ('error %r in dictionary %s already exists in dictionary ' + '%s' % (err, short_fname, other_fname)) + for r in this_err_dict[err]: + assert r not in other_err_dict, \ + ('error %s: correction %s from dictionary %s is an ' + 'error itself in dictionary %s' + % (err, r, short_fname, other_fname)) + assert pair not in global_pairs + global_pairs.add(pair) + # 3. check corrections in other dicts against this dict (lower) + pair = (other_fname, short_fname) + if pair not in allowed_dups: + for err in other_err_dict: + for r in other_err_dict[err]: + assert r not in this_err_dict, \ + ('error %s: correction %s from dictionary %s is an ' + 'error itself in dictionary %s' + % (err, r, other_fname, short_fname)) + assert pair not in global_pairs + global_pairs.add(pair) + global_err_dicts[short_fname] = this_err_dict + + +@pytest.mark.dependency(depends=['dictionary loop']) +def test_ran_all(): + """Test that all pairwise tests ran.""" + for f1, _ in _fnames_in_aspell: + f1 = op.basename(f1) + for f2, _ in _fnames_in_aspell: + f2 = op.basename(f2) + assert (f1, f2) in global_pairs + assert len(global_pairs) == len(_fnames_in_aspell) ** 2