Update data pipeline for creating dictionary

fastdatascience · Aug 7, 2024 · 3391ea6 · 3391ea6
1 parent eed3358
commit 3391ea6
Show file tree

Hide file tree

Showing 8 changed files with 1 addition and 1 deletion.
diff --git a/...source/drugbank_01_download_vocabulary.py → ...source/01_drugbank_download_vocabulary.py b/...source/drugbank_01_download_vocabulary.py → ...source/01_drugbank_download_vocabulary.py
diff --git a/...mp_and_extract_drug_names_and_synonyms.py → ...mp_and_extract_drug_names_and_synonyms.py b/...mp_and_extract_drug_names_and_synonyms.py → ...mp_and_extract_drug_names_and_synonyms.py
diff --git a/...m_source/nhs_01_get_drug_data_from_api.py → ...m_source/03_nhs_get_drug_data_from_api.py b/...m_source/nhs_01_get_drug_data_from_api.py → ...m_source/03_nhs_get_drug_data_from_api.py
diff --git a/...names_and_synonyms_from_wikipedia_dump.py → ...names_and_synonyms_from_wikipedia_dump.py b/...names_and_synonyms_from_wikipedia_dump.py → ...names_and_synonyms_from_wikipedia_dump.py
diff --git a/...ta_from_source/02_combine_data_sources.py → ...ta_from_source/05_combine_data_sources.py b/...ta_from_source/02_combine_data_sources.py → ...ta_from_source/05_combine_data_sources.py
@@ -195,7 +195,7 @@ def get_brand_names_nhs(description: str):
 
 print("Finding all drugs that are also in the NLTK list of English words.")
 
-all_english_vocab = set(words.words())
+all_english_vocab = set([w.lower() for w in words.words()])
 
 words_to_check_with_ai = set()
 for word in list(drug_variant_to_canonical):

diff --git a/...source/optional_03_check_words_with_ai.py → ...source/06_optional_check_words_with_ai.py b/...source/optional_03_check_words_with_ai.py → ...source/06_optional_check_words_with_ai.py
diff --git a/...ce/optional_04_augment_inclusions_list.py → ...ce/07_optional_augment_inclusions_list.py b/...ce/optional_04_augment_inclusions_list.py → ...ce/07_optional_augment_inclusions_list.py
diff --git a/src/drug_named_entity_recognition/drug_ner_dictionary.pkl.bz2 b/src/drug_named_entity_recognition/drug_ner_dictionary.pkl.bz2