-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
28 changed files
with
1,467 additions
and
388 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| ############################################################################### | ||
| # Copyright 2015-2021 University of Florida. All rights reserved. | ||
| # This file is part of UF CTS-IT's NACCulator project. | ||
| # Use of this source code is governed by the license found in the LICENSE file. | ||
| ############################################################################### | ||
|
|
||
| import csv | ||
| import os | ||
| import re | ||
| import sys | ||
|
|
||
|
|
||
| def convert_rule_to_python(name: str, rule: str) -> bool: | ||
| """ | ||
| Converts the text `rule` into a python function. | ||
| The returned function accepts one argument of type `Packet`. | ||
| Example: | ||
| packet["FOO"] = "I should be blank!" | ||
| packet["BAR"] = 0 | ||
| r = convert_rule_to_python("FOO", "Blank if Question 1 BAR = 0 (No)") | ||
| if packet["FOOBAR"] != "" and r(packet): | ||
| raise RuleError("FOO should be blank, but is not!") | ||
| :param name: Canonical name of the field | ||
| :param rule: Blanking rule text | ||
| """ | ||
|
|
||
| special_cases = { | ||
|
|
||
| } | ||
|
|
||
| single_value = re.compile( | ||
| r"Blank if( Question(s?))? *\w+ (?P<key>\w+)" | ||
| r" *(?P<eq>=|ne|is|not =|!=) (?P<value>\d+)([^-]|$)") | ||
| range_values = re.compile( | ||
| r"Blank if( Question(s?))? *\w+ (?P<key>\w+)" | ||
| r" *(?P<eq>=|ne|is|not =|!=) (?P<start>\d+)-(?P<stop>\d+)( |$)") | ||
| blank_value = re.compile( | ||
| r"Blank if( Question(s?))? *\w+ (?P<key>\w+) *(?P<eq>=|ne|is|not =) blank") | ||
| not_answered = re.compile( | ||
| r"Blank if question not answered") | ||
|
|
||
| # First, check to see if the rule is a "Special Case" | ||
| if name in special_cases: | ||
| return special_cases[name](rule) | ||
|
|
||
| # Then, check to see if the rule is of the within-range type | ||
| m = range_values.match(rule) | ||
| if m: | ||
| return _blanking_rule_check_within_range( | ||
| m.group('key'), m.group('eq'), m.group('start'), m.group('stop')) | ||
|
|
||
| # Next, check to see if the rule is of the single-value type | ||
| m = single_value.match(rule) | ||
| if m: | ||
| return _blanking_rule_check_single_value( | ||
| m.group('key'), m.group('eq'), m.group('value')) | ||
|
|
||
| # Next, check to see if the rule is of the "blank if _ = blank" type | ||
| m = blank_value.match(rule) | ||
| if m: | ||
| return _blanking_rule_check_blank_value( | ||
| m.group('key'), m.group('eq')) | ||
|
|
||
| # For the FTLD forms, we need to also check to see if | ||
| # "Blank if question not answered" is included in the blanking rules | ||
| m = not_answered.match(rule) | ||
| if m: | ||
| return lambda packet: False | ||
|
|
||
| # Finally, raise an error since we do not know how to handle the rule | ||
| raise Exception("Could not parse Blanking rule: "+name) | ||
|
|
||
|
|
||
| def extract_blanks(csvfile): | ||
| with open(csvfile) as fp: | ||
| reader = csv.DictReader(fp) | ||
| blanks_fieldnames = [f for f in reader.fieldnames if 'BLANKS' in f] | ||
| for row in reader: | ||
| rules = '\t'.join([row[f] for f in blanks_fieldnames]).strip() | ||
| if rules: | ||
| yield "%s:\t%s" % (row['Data Element'], rules) | ||
|
|
||
|
|
||
| def _blanking_rule_check_single_value(key, eq, value): | ||
| def should_be_blank(packet): | ||
| """ Returns True if the value should be blank according to the rule """ | ||
| if '=' == eq or 'is' == eq: | ||
| return packet[key] == value | ||
| elif 'ne' == eq or 'not =' == eq or '!=' == eq: | ||
| return packet[key] != value | ||
| else: | ||
| raise ValueError("'eq' must be '=' or 'ne', not '%s'." % eq) | ||
|
|
||
| return should_be_blank | ||
|
|
||
|
|
||
| def _blanking_rule_check_within_range(key, eq, start, stop): | ||
| def should_be_blank(packet): | ||
| """ Returns True if the value should be blank according to the rule """ | ||
| first = int(start) | ||
| last = int(stop)+1 | ||
| if '=' == eq: | ||
| return packet[key] in range(first, last) | ||
| elif 'ne' == eq: | ||
| return packet[key] not in list(range(first, last)) | ||
| else: | ||
| raise ValueError("'eq' must be '=' or 'ne', not '%s'." % eq) | ||
|
|
||
| return should_be_blank | ||
|
|
||
|
|
||
| def _blanking_rule_check_blank_value(key, eq, value=None): | ||
| def should_be_blank(packet): | ||
| """ Returns True if the value should be blank according to the rule """ | ||
| if '=' == eq: | ||
| return packet[key] == value | ||
| elif 'ne' == eq: | ||
| return packet[key] != value | ||
| else: | ||
| raise ValueError("'eq' must be '=' or 'ne', not '%s'." % eq) | ||
|
|
||
| return should_be_blank | ||
|
|
||
|
|
||
| def _blanking_rule_dummy(rule): | ||
| return lambda packet: False | ||
|
|
||
|
|
||
| def main(): | ||
| """ | ||
| Extracts all blanking rules from all DED files in a specified directory. | ||
| Usage: | ||
| python blanks.py ./ded_ivp | ||
| Note: this module is more useful as an imported module; see | ||
| `convert_rule_to_python`. | ||
| """ | ||
| data_dict_path = './ded_ivp' | ||
| if len(sys.argv) > 1: | ||
| data_dict_path = sys.argv[1] | ||
|
|
||
| deds = [f for f in os.listdir(data_dict_path) if f.endswith('.csv')] | ||
| for ded in deds: | ||
| for rule in extract_blanks(os.path.join(data_dict_path, ded)): | ||
| print(rule) | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| main() |
Oops, something went wrong.