diff --git a/src/ncdiff/composer.py b/src/ncdiff/composer.py index 3053a97..230c2aa 100755 --- a/src/ncdiff/composer.py +++ b/src/ncdiff/composer.py @@ -108,12 +108,9 @@ def model_name(self): ret = re.search(Tag.BRACE[0], self.path[0]) if ret: url_to_name = {i[2]: i[0] for i in self.device.namespaces - if i[1] is not None} + if i[1] is not None and i[2] == ret.group(1)} if ret.group(1) in url_to_name: - raise ModelMissing("please load model '{}' by calling " - "method load_model() of device {}" - .format(url_to_name[ret.group(1)], - self.device)) + return url_to_name[ret.group(1)] else: raise ModelMissing("unknown model url '{}'" .format(ret.group(1))) @@ -123,7 +120,10 @@ def model_name(self): @property def model_ns(self): - return self.device.models[self.model_name].url + name_to_url = {i[0]: i[2] for i in self.device.namespaces + if i[1] is not None and i[0] == self.model_name} + return name_to_url[self.model_name] if self.model_name in name_to_url \ + else None @property def is_config(self): diff --git a/src/ncdiff/manager.py b/src/ncdiff/manager.py index f4cea8b..06421a1 100755 --- a/src/ncdiff/manager.py +++ b/src/ncdiff/manager.py @@ -271,7 +271,9 @@ def scan_models(self, folder='./yang', download='check'): if download in ['check', 'force']: d = ModelDownloader(self, folder) d.download_all(check_before_download=(download == 'check')) - self.compiler = ModelCompiler(folder) + self.compiler = ModelCompiler(folder, context=d.context) + else: + self.compiler = ModelCompiler(folder) def load_model(self, model): '''load_model diff --git a/src/ncdiff/model.py b/src/ncdiff/model.py index 8ebc5e4..d5290e7 100755 --- a/src/ncdiff/model.py +++ b/src/ncdiff/model.py @@ -8,7 +8,8 @@ from copy import deepcopy from ncclient import operations from threading import Thread, current_thread -from pyang import statements +from pyang import statements, xpath_parser, syntax, util +from pyang import xpath as xp try: from pyang.repository import FileRepository except ImportError: @@ -19,6 +20,11 @@ from pyang import Context from .errors import ModelError +from .composer import Tag +from .tailf import is_deprecated_without_replacement +from .tailf import is_tailf_ordering, get_tailf_ordering +from .tailf import add_tailf_annotation, set_ordering_xpath +from .xpath import chk_xpath_path # create a logger for this module @@ -27,7 +33,11 @@ logging.getLogger('ncclient.operations').setLevel(logging.WARNING) PARSER = etree.XMLParser(encoding='utf-8', remove_blank_text=True) - +PREFIX = syntax.prefix +IDENTIFIER = PREFIX + r'|\*' +KEYWORD = '((' + PREFIX + '):)?(' + IDENTIFIER + ')' +RE_SCHEMA_NODE_ID_PART = re.compile('/' + KEYWORD) +RE_ANNOTATE_STATEMENT = re.compile(r'^(.+)\[name=[\'|"](.+)[\'|"]\]') def write_xml(filename, element): element_tree = etree.ElementTree(element) @@ -547,7 +557,7 @@ def run(self): class CompilerContext(Context): - def __init__(self, repository): + def __init__(self, repository, modeldevice=None): Context.__init__(self, repository) self.dependencies = None self.modulefile_queue = None @@ -555,6 +565,7 @@ def __init__(self, repository): self.num_threads = 2 else: self.num_threads = 1 + self.modeldevice = modeldevice def _get_latest_revision(self, modulename): latest = None @@ -670,8 +681,8 @@ def update_dependencies(self, module_statement): for stmt in module_statement.search(node_name): for substmt in stmt.substmts: if ( + isinstance(substmt.keyword, tuple) and 'tailf' in substmt.keyword[0] and - len(substmt.keyword) == 2 and substmt.keyword[1] == 'hidden' ): break @@ -697,6 +708,138 @@ def read_dependencies(self): ) self.dependencies = read_xml(dependencies_file) + def check_data_tree_xpath(self, xpath_stmt, node_stmt): + if not hasattr(xpath_stmt, 'i_orig_module'): + logger.warning(f"Statement at {xpath_stmt.pos} does not have " + "attribute 'i_orig_module'") + return None + + p = xpath_parser.parse(xpath_stmt.arg) + if isinstance(p, list): + node = chk_xpath_path( + self, + xpath_stmt, + node_stmt, + node_stmt, + p, + ) + elif isinstance(p, tuple): + if p[0] == 'absolute': + node = chk_xpath_path( + self, + xpath_stmt, + node_stmt, + 'root', + p[1], + ) + elif p[0] == 'relative': + node = chk_xpath_path( + self, + xpath_stmt, + node_stmt, + node_stmt, + p[1], + ) + else: + logger.warning(f"Failed to understand Xpath '{xpath_stmt.arg}' " + f"in data tree at {xpath_stmt.pos}") + return None + else: + logger.warning(f"Failed to parse Xpath '{xpath_stmt.arg}' in data " + f"tree at {xpath_stmt.pos}") + return None + if node is None: + logger.warning(f"Failed to find annotated statement by the Xpath " + f"'{xpath_stmt.arg}' in data tree at " + f"{xpath_stmt.pos}") + else: + xpath_stmt.i_annotate_node = node + return node + + def check_schema_tree_xpath(self, xpath_stmt): + if xpath_stmt.arg.startswith('/'): + is_absolute = True + arg = xpath_stmt.arg + else: + is_absolute = False + arg = "/" + xpath_stmt.arg + + # Parse the path into a list of two-tuples of (prefix, identifier) + path = [(m[1], m[2]) for m in RE_SCHEMA_NODE_ID_PART.findall(arg)] + + # Find the module of the first node in the path + if not isinstance(path, list) or len(path) == 0: + logger.warning(f"Failed to parse Xpath {xpath_stmt.arg} in schema " + f"tree at {xpath_stmt.pos}") + return None + (prefix, identifier) = path[0] + module = util.prefix_to_module( + xpath_stmt.i_module, prefix, xpath_stmt.pos, self.errors) + if module is None: + logger.warning(f"Failed to find a module by the prefix {prefix} " + f"at {xpath_stmt.pos}") + return None + if is_absolute: + node = statements.search_data_keyword_child(module.i_children, + module.i_modulename, + identifier) + if node is None: + # Check all our submodules + for inc in module.search('include'): + submod = self.get_module(inc.arg) + if submod is not None: + node = statements.search_data_keyword_child( + submod.i_children, + submod.i_modulename, + identifier) + if node is not None: + break + if node is None: + logger.warning("Failed to find annotated statement by the " + f"identifier {prefix}:{identifier} at " + f"{xpath_stmt.pos}") + return None + path = path[1:] + else: + if hasattr(xpath_stmt.parent, 'i_annotate_node'): + node = xpath_stmt.parent.i_annotate_node + else: + logger.warning("Parent statement does not have a resolved " + f"target: {xpath_stmt.pos}") + return None + + # Recurse down the path + for prefix, identifier in path: + if hasattr(node, 'i_children'): + children = node.i_children + else: + children = [] + if prefix == '' and identifier == '*': + return children + module = util.prefix_to_module( + xpath_stmt.i_module, prefix, xpath_stmt.pos, self.errors) + if module is None: + logger.warning("Failed to find a module by the prefix " + f"{prefix}: {xpath_stmt.pos}") + return None + child = statements.search_data_keyword_child(children, + module.i_modulename, + identifier) + if child is None: + logger.warning("Failed to find annotated statement by the " + f"identifier {prefix}:{identifier} at " + f"{xpath_stmt.pos}") + return None + node = child + xpath_stmt.i_annotate_node = node + return node + + def get_xpath_from_schema_node(self, schema_node, type=Tag.XPATH): + if self.modeldevice is None: + return None + else: + return self.modeldevice.get_xpath(schema_node, type=type, instance=False) + def load_context(self): self.modulefile_queue = queue.Queue() for filename in os.listdir(self.repository.dirs[0]): @@ -711,6 +854,157 @@ def load_context(self): self.modulefile_queue.join() self.write_dependencies() + def process_annotation_module(self, preprocessing=True): + + def tailf_annotate(context, annotating_stmt): + target = context.check_schema_tree_xpath(annotating_stmt) + if target is not None: + for annitating_substmt in annotating_stmt.substmts: + if annitating_substmt.keyword == ( + 'tailf-common', + 'annotate', + ): + tailf_annotate(context, annitating_substmt) + else: + if isinstance(target, list): + for t in target: + append_annotation(t, annitating_substmt) + else: + append_annotation(target, annitating_substmt) + + def tailf_annotate_module(context, module_stmt): + for substmt in module_stmt.substmts: + if ( + isinstance(substmt.keyword, tuple) and + 'tailf' in substmt.keyword[0] and + substmt.keyword[1] == 'annotate-module' + ): + annotated_module = context.get_module(substmt.arg) + if annotated_module is None: + logger.warning("Failed to find annotated module " + f"{substmt.arg} at {substmt.pos}") + continue + substmt.i_annotate_node = annotated_module + for annotating_substmt in substmt.substmts: + if isinstance(substmt.raw_keyword, tuple): + prefix, identifier = annotating_substmt.raw_keyword + m, rev = util.prefix_to_modulename_and_revision( + annotating_substmt.i_module, + prefix, + annotating_substmt.pos, + context.errors, + ) + if ( + m == 'tailf-common' and + identifier == 'annotate-statement' + ): + tailf_annotate_statement( + context, annotating_substmt) + else: + append_annotation( + annotated_module, annotating_substmt) + else: + append_annotation( + annotated_module, annotating_substmt) + + def tailf_annotate_statement(context, annotating_stmt): + annotated_stmt = annotating_stmt.parent.i_annotate_node + match = re.match(RE_ANNOTATE_STATEMENT, annotating_stmt.arg) + if match: + matched_stmts = [ + s for s in annotated_stmt.substmts + if s.keyword == match.group(1) and s.arg == match.group(2) + ] + if len(matched_stmts) == 0: + logger.warning("Annotating statement at " + f"{annotating_stmt.pos}: Failed to find a " + f"matching sub-statement '{match.group(1)} " + f"{match.group(2)}' under the annotated " + f"statement at {annotated_stmt.pos}") + return + elif len(matched_stmts) > 1: + logger.warning("Annotating statement at " + f"{annotating_stmt.pos}: Found more than " + "one matching sub-statement " + f"'{match.group(1)} {match.group(2)}' " + "under the annotated statement at " + f"{annotated_stmt.pos}") + return + elif annotating_stmt.arg == 'type': + matched_stmts = [ + s for s in annotated_stmt.substmts + if s.keyword == 'type' + ] + if len(matched_stmts) == 0: + logger.warning("Annotating statement at " + f"{annotating_stmt.pos}: 'type' not found " + "under the annotated statement at " + f"{annotated_stmt.pos}") + return + elif len(matched_stmts) > 1: + logger.warning("Annotating statement at " + f"{annotating_stmt.pos}: found more than " + "one 'type' under the annotated statement " + f"at {annotated_stmt.pos}") + return + else: + logger.warning("Annotating statement at " + f"{annotating_stmt.pos}: Invalid arg " + f"{annotating_stmt.arg}") + return + + annotating_stmt.i_annotate_node = matched_stmts[0] + for substmt in annotating_stmt.substmts: + if isinstance(substmt.raw_keyword, tuple): + annotate_statement = False + prefix, identifier = substmt.raw_keyword + m, rev = util.prefix_to_modulename_and_revision( + substmt.i_module, + prefix, + substmt.pos, + context.errors, + ) + if ( + m == 'tailf-common' and + identifier == 'annotate-statement' + ): + tailf_annotate_statement(context, substmt) + else: + append_annotation(matched_stmts[0], substmt) + else: + append_annotation(matched_stmts[0], substmt) + + def append_annotation(target_stmt, annotation_stmt): + new_stmt = statements.new_statement( + annotation_stmt.top, + target_stmt, + annotation_stmt.pos, + annotation_stmt.keyword, + annotation_stmt.arg, + ) + new_stmt.raw_keyword = annotation_stmt.raw_keyword + new_stmt.i_orig_module = annotation_stmt.top + if hasattr(target_stmt, 'i_module'): + new_stmt.i_module = target_stmt.i_module + target_stmt.substmts.append(new_stmt) + for substmt in annotation_stmt.substmts: + append_annotation(new_stmt, substmt) + + mudule_names = [k[0] for k in self.modules] + for mudule_name in mudule_names: + if mudule_name.endswith('-ann'): + module_statement = self.get_module(mudule_name) + if module_statement is None: + logger.warning(f"Failed to find annotation module {mudule_name}") + elif preprocessing: + tailf_annotate_module(self, module_statement) + logger.debug(f"Pre-processed tailf:annotate-module in {mudule_name}") + else: + for substmt in module_statement.substmts: + if substmt.keyword == ('tailf-common', 'annotate'): + tailf_annotate(self, substmt) + logger.debug(f"Post-processed tailf:annotate in {mudule_name}") + def validate_context(self): revisions = {} for mudule_name, module_revision in self.modules: @@ -720,17 +1014,36 @@ def validate_context(self): ): revisions[mudule_name] = module_revision self.sort_modules() + + # Initialize annotation modules + annotation_modules = [m for k, m in self.modules.items() + if k[0].endswith("-ann")] + for m in annotation_modules: + statements.v_init_module(self, m) + + # Process annotation modules as a pre-processing step + self.process_annotation_module(preprocessing=True) + self.validate() if 'prune' in dir(statements.Statement): for mudule_name, module_revision in revisions.items(): self.modules[(mudule_name, module_revision)].prune() + # Process annotation modules as a post-processing step + self.process_annotation_module(preprocessing=False) + def sort_modules(self): - submodules = {k: m for k, m in self.modules.items() - if m.keyword == "submodule"} - for k in submodules: - del self.modules[k] - self.modules.update(submodules) + modulename_revision = {k[0]: k for k in self.modules.keys()} + submodules = sorted([ + k[0] for k, m in self.modules.items() if m.keyword == "submodule" + ]) + modules = sorted([ + k for k in modulename_revision if k not in submodules + ]) + self.modules = { + modulename_revision[k]: self.modules[modulename_revision[k]] + for k in modules + submodules + } def internal_reset(self): self.modules = {} @@ -779,7 +1092,7 @@ def __init__(self, nc_device, folder): 'capabilities.txt', ) repo = FileRepository(path=self.dir_yang) - self.context = CompilerContext(repository=repo) + self.context = CompilerContext(repository=repo, modeldevice=nc_device) self.download_queue = queue.Queue() self.num_threads = 2 @@ -933,17 +1246,30 @@ class ModelCompiler(object): call pyang.error.err_to_str() to print out detailed error messages. ''' - def __init__(self, folder): + def __init__(self, folder, context=None): ''' __init__ instantiates a ModelCompiler instance. ''' self.dir_yang = os.path.abspath(folder) - self.context = None + self.context = context self.module_prefixes = {} self.module_namespaces = {} self.identity_deps = {} self.build_dependencies() + self.ordering_stmt_leafref = {} + self.ordering_stmt_tailf = {} + self.ordering_xpath_leafref = {} + self.ordering_xpath_tailf = {} + self.ordering_match = {} + self.ordering = {} + self._dependencies = {} + + self.exclude_obsolete = False + self.exclude_deprecated = False + self.include_deprecated_without_replacement = False + self.include_xpaths = set() + self.exclude_xpaths = set() @property def pyang_errors(self): @@ -996,27 +1322,41 @@ def get_dependencies(self, module): ------- tuple - A tuple with two elements: a set of imports and a set of depends. + A tuple with three elements: a set of imports, a set of includes + and a set of other depends. ''' + def find_all_depends(depends, dependencies): + depends_copy = set(depends) + for m in dependencies: + if ( + list(filter(lambda i: i.get('module') in depends, + m.findall('./imports/import'))) or + list(filter(lambda i: i.get('module') in depends, + m.findall('./includes/include'))) + ): + depends.add(m.get('id')) + return depends_copy != depends + if self.context is None or self.context.dependencies is None: self.build_dependencies() dependencies = self.context.dependencies imports = set() + includes = set() for m in list(filter(lambda i: i.get('id') == module, dependencies.findall('./module'))): imports.update(set(i.get('module') for i in m.findall('./imports/import'))) - depends = set() - for m in dependencies: - if list(filter(lambda i: i.get('module') == module, - m.findall('./imports/import'))): - depends.add(m.get('id')) - if list(filter(lambda i: i.get('module') == module, - m.findall('./includes/include'))): - depends.add(m.get('id')) - return (imports, depends) + includes.update(set(i.get('module') + for i in m.findall('./includes/include'))) + + depends = imports | includes + while find_all_depends(depends, dependencies): + pass + self._dependencies[module] = ( + imports, includes, depends - imports - includes) + return self._dependencies[module] def compile(self, module): '''compile @@ -1041,8 +1381,8 @@ def compile(self, module): return Model(cached_tree) varnames = Context.add_module.__code__.co_varnames - imports, depends = self.get_dependencies(module) - required_module_set = imports | depends + imports, includes, depends = self.get_dependencies(module) + required_module_set = imports | includes | depends required_module_set.add(module) self.context.internal_reset() for m in required_module_set: @@ -1114,6 +1454,11 @@ def compile(self, module): else: self.identity_deps[b_idn].append(curr_idn) + self.ordering_stmt_leafref[module] = [] + self.ordering_stmt_tailf[module] = [] + self.ordering_match[module] = [] + self.ordering[module] = {} + for child in vm.i_children: if child.keyword in statements.data_definition_keywords: self.depict_a_schema_node(vm, st, child) @@ -1125,6 +1470,7 @@ def compile(self, module): self.depict_a_schema_node(vm, st, child, mode='notification') self._write_to_cache(module, st) + set_ordering_xpath(self, module) return Model(st) @@ -1138,6 +1484,15 @@ def depict_a_schema_node(self, module, parent, child, mode=None): sm = child.search_one('status') if sm is not None and sm.arg in ['deprecated', 'obsolete']: n.set('status', sm.arg) + if is_deprecated_without_replacement(child): + n.set('deprecated-without-replacement', 'true') + + if self.skip(child, n): + parent.remove(n) + return + if not hasattr(child, 'schema_node'): + child.schema_node = n + sm = child.search('default') if sm is not None and len(sm) > 0: n.set('default', ",".join(map(lambda x: x.arg, sm))) @@ -1160,7 +1515,7 @@ def depict_a_schema_node(self, module, parent, child, mode=None): if cases: n.set('values', '|'.join(cases)) elif child.keyword in ['leaf', 'leaf-list']: - self.set_leaf_datatype_value(child, n) + self.set_leaf_datatype_value(module.arg, child, n) sm = child.search_one('mandatory') if ( sm is not None and sm.arg == 'true' or @@ -1180,19 +1535,37 @@ def depict_a_schema_node(self, module, parent, child, mode=None): for ch in child.substmts: if ( isinstance(ch.keyword, tuple) and - 'tailf' in ch.keyword[0] + ch.keyword[0] == 'tailf-common' ): if ( ch.keyword[0] in self.module_namespaces and len(ch.keyword) == 2 ): - n.set( - etree.QName(self.module_namespaces[ch.keyword[0]], - ch.keyword[1]), - ch.arg if ch.arg else '', - ) + if ch.keyword[1] == 'non-strict-leafref': + # Do not treat non-strict-leafref as a leafref for now + # as it is not clear how this may impact the CLI + # ordering. + # p = ch.search_one('path') + # if p is not None: + # self.set_ordering_stmt_leafref( + # module.arg, child, p, n) + pass + elif is_tailf_ordering(ch): + target = self.context.check_data_tree_xpath( + ch, child) + if target is not None: + ordering = get_tailf_ordering( + self.context, ch, target) + self.ordering_stmt_tailf[module.arg].append(( + child, + target, + ordering, + ch, + )) + else: + add_tailf_annotation(self.module_namespaces, ch, n) else: - logger.warning("Special Tailf annotation at {}, " + logger.warning("Unknown Tailf annotation at {}, " "keyword = {}" .format(ch.pos, ch.keyword)) @@ -1212,6 +1585,14 @@ def depict_a_schema_node(self, module, parent, child, mode=None): else: self.depict_a_schema_node(module, n, c, mode=mode) + def get_xpath_from_schema_node(self, schema_node, type=Tag.XPATH): + from .manager import ModelDevice + + if self.context.modeldevice is None: + self.context.modeldevice = ModelDevice(None, None) + self.context.modeldevice.compiler = self + return self.context.get_xpath_from_schema_node(schema_node, type=type) + @staticmethod def set_access(statement, node, mode): if ( @@ -1230,7 +1611,28 @@ def set_access(statement, node, mode): else: node.set('access', 'read-only') - def set_leaf_datatype_value(self, leaf_statement, leaf_node): + def set_ordering_stmt_leafref(self, module, leaf_statement, path_statement, + leaf_node): + # Consider leafref as a dpendency for ordering purpose + if not self.skip(leaf_statement, leaf_node): + target_stmt = self.context.check_data_tree_xpath( + path_statement, leaf_statement) + if target_stmt is not None: + self.ordering_stmt_leafref[module].append(( + leaf_statement, + target_stmt, + [ + ('create', 'after', 'create'), + ('modify', 'after', 'create'), + ('create', 'after', 'modify'), + ('delete', 'before', 'modify'), + ('modify', 'before', 'delete'), + ('delete', 'before', 'delete'), + ], + path_statement, + )) + + def set_leaf_datatype_value(self, module, leaf_statement, leaf_node): sm = leaf_statement.search_one('type') if sm is None: datatype = '' @@ -1238,6 +1640,9 @@ def set_leaf_datatype_value(self, leaf_statement, leaf_node): if sm.arg == 'leafref': p = sm.search_one('path') if p is not None: + self.set_ordering_stmt_leafref( + module, leaf_statement, p, leaf_node) + # Try to make the path as compact as possible. # Remove local prefixes, and only use prefix when # there is a module change in the path. @@ -1253,12 +1658,12 @@ def set_leaf_datatype_value(self, leaf_statement, leaf_node): else: target.append(prefix + ':' + name) curprefix = prefix - datatype = "-> %s" % "/".join(target) + datatype = f'leafref {"/".join(target)}' else: datatype = sm.arg elif sm.arg == 'identityref': idn_base = sm.search_one('base') - datatype = sm.arg + ":" + idn_base.arg + datatype = f'identityref {idn_base.arg}' else: datatype = sm.arg leaf_node.set('datatype', datatype) @@ -1324,6 +1729,37 @@ def type_identityref_values(self, type_statement): return '|'.join(value_stmts) return '' + def skip(self, statement, schema_node): + xpath = self.get_xpath_from_schema_node( + schema_node, type=Tag.LXML_XPATH) + for in_xpath in self.include_xpaths: + if in_xpath == xpath or in_xpath.startswith(xpath + '/'): + return False + for ex_xpath in self.exclude_xpaths: + if ex_xpath == xpath or xpath.startswith(ex_xpath + '/'): + return True + + # i_not_implemented should be set to True when features in the context + # are not met + if getattr(statement, "i_not_implemented", None) is True: + return True + + # Statement status is checked + status = schema_node.get('status', default=None) + deprecated_without_replacement = schema_node.get( + 'deprecated-without-replacement', default=None) + if ( + status == 'obsolete' and + self.exclude_obsolete or + status == 'deprecated' and + self.exclude_deprecated and not ( + self.include_deprecated_without_replacement and + deprecated_without_replacement == 'true' + ) + ): + return True + return False + class ModelDiff(object): '''ModelDiff diff --git a/src/ncdiff/ref.py b/src/ncdiff/ref.py index f992b0e..fad2cc2 100755 --- a/src/ncdiff/ref.py +++ b/src/ncdiff/ref.py @@ -362,7 +362,7 @@ def parse_square_bracket(self, to_node=None): self.cut(start_idx+1, end_idx-2, tag, 2) start_idx = None else: - if re.search('^\[[1-9][0-9]*\]$', substring): + if re.search(r'^\[[1-9][0-9]*\]$', substring): numbers = substring[1:-1] self.cut(start_idx+1, end_idx-1, numbers, 2) else: diff --git a/src/ncdiff/tailf.py b/src/ncdiff/tailf.py new file mode 100755 index 0000000..0de35ec --- /dev/null +++ b/src/ncdiff/tailf.py @@ -0,0 +1,609 @@ +import logging +from os import path +from lxml import etree +from pyang import util, statements + +from .composer import Tag + +logger = logging.getLogger(__name__) + + +DEPENDENCY_TYPE = { + ("create", "create"): 1, + ("create", "delete"): 2, + ("create", "modify"): 4, + ("delete", "create"): 8, + ("delete", "delete"): 16, + ("delete", "modify"): 32, + ("modify", "create"): 64, + ("modify", "delete"): 128, + ("modify", "modify"): 256, +} + + +def is_tailf_ordering(stmt): + if isinstance(stmt.keyword, tuple): + m, identifier = stmt.keyword + return m == 'tailf-common' and identifier in { + 'cli-diff-after', 'cli-diff-before', + 'cli-diff-create-after', 'cli-diff-create-before', + 'cli-diff-delete-after', 'cli-diff-delete-before', + 'cli-diff-modify-after', 'cli-diff-modify-before', + 'cli-diff-set-after', 'cli-diff-set-before', + 'cli-diff-dependency', + } + else: + return False + + +def is_deprecated_without_replacement(stmt): + for substmt in stmt.search(('Cisco-IOS-XE-types', 'yang-meta-data')): + if substmt.arg == 'deprecated-without-replacement': + return True + return False + + +def get_tailf_ordering(context, stmt, target_stmt): + symmetric = is_symmetric_tailf_ordering(context, stmt, target_stmt) + if stmt.keyword[1] in ['cli-diff-after', 'cli-diff-before']: + conj = 'after' if stmt.keyword[1] == 'cli-diff-after' else 'before' + valid_substmts = { + 'cli-when-target-set', + 'cli-when-target-create', + 'cli-when-target-modify', + 'cli-when-target-delete', + } + substmts = [s for s in stmt.substmts if s.keyword[1] in valid_substmts] + if len(substmts) == 0: + return [ + ('create', conj, 'create'), + ('modify', conj, 'create'), + ('delete', conj, 'create'), + ('create', conj, 'modify'), + ('modify', conj, 'modify'), + ('delete', conj, 'modify'), + ('create', conj, 'delete'), + ('modify', conj, 'delete'), + ('delete', conj, 'delete'), + ] + ordering = [] + for substmt in substmts: + if substmt.keyword[1] == 'cli-when-target-set': + ordering.extend([ + ('create', conj, 'create'), + ('modify', conj, 'create'), + ('delete', conj, 'create'), + ('create', conj, 'modify'), + ('modify', conj, 'modify'), + ('delete', conj, 'modify'), + ]) + elif substmt.keyword[1] == 'cli-when-target-create': + ordering.extend([ + ('create', conj, 'create'), + ('modify', conj, 'create'), + ('delete', conj, 'create'), + ]) + elif substmt.keyword[1] == 'cli-when-target-modify': + ordering.extend([ + ('create', conj, 'modify'), + ('modify', conj, 'modify'), + ('delete', conj, 'modify'), + ]) + elif substmt.keyword[1] == 'cli-when-target-delete': + ordering.extend([ + ('create', conj, 'delete'), + ('modify', conj, 'delete'), + ('delete', conj, 'delete'), + ]) + return ordering + elif stmt.keyword[1] in ['cli-diff-create-after', 'cli-diff-create-before']: + conj = 'after' if stmt.keyword[1] == 'cli-diff-create-after' else 'before' + valid_substmts = [ + 'cli-when-target-set', + 'cli-when-target-create', + 'cli-when-target-modify', + 'cli-when-target-delete', + ] + substmts = [s for s in stmt.substmts if s.keyword[1] in valid_substmts] + if len(substmts) == 0: + if symmetric: + return [ + ('create', conj, 'modify'), + ('create', conj, 'delete'), + ] + else: + return [ + ('create', conj, 'create'), + ('create', conj, 'modify'), + ('create', conj, 'delete'), + ] + ordering = [] + for substmt in substmts: + if substmt.keyword[1] == 'cli-when-target-set': + ordering.extend([ + ('create', conj, 'create'), + ('create', conj, 'modify'), + ]) + elif substmt.keyword[1] == 'cli-when-target-create': + ordering.append( + ('create', conj, 'create'), + ) + elif substmt.keyword[1] == 'cli-when-target-modify': + ordering.append( + ('create', conj, 'modify'), + ) + elif substmt.keyword[1] == 'cli-when-target-delete': + ordering.append( + ('create', conj, 'delete'), + ) + return ordering + elif stmt.keyword[1] in ['cli-diff-delete-after', 'cli-diff-delete-before']: + conj = 'after' if stmt.keyword[1] == 'cli-diff-delete-after' else 'before' + valid_substmts = { + 'cli-when-target-set', + 'cli-when-target-create', + 'cli-when-target-modify', + 'cli-when-target-delete', + } + substmts = [s for s in stmt.substmts if s.keyword[1] in valid_substmts] + if len(substmts) == 0: + if symmetric: + return [ + ('delete', conj, 'create'), + ('delete', conj, 'modify'), + ] + else: + return [ + ('delete', conj, 'create'), + ('delete', conj, 'modify'), + ('delete', conj, 'delete'), + ] + ordering = [] + for substmt in substmts: + if substmt.keyword[1] == 'cli-when-target-set': + ordering.extend([ + ('delete', conj, 'create'), + ('delete', conj, 'modify'), + ]) + elif substmt.keyword[1] == 'cli-when-target-create': + ordering.append( + ('delete', conj, 'create'), + ) + elif substmt.keyword[1] == 'cli-when-target-modify': + ordering.append( + ('delete', conj, 'modify'), + ) + elif substmt.keyword[1] == 'cli-when-target-delete': + ordering.append( + ('delete', conj, 'delete'), + ) + return ordering + elif stmt.keyword[1] in ['cli-diff-modify-after', 'cli-diff-modify-before']: + conj = 'after' if stmt.keyword[1] == 'cli-diff-modify-after' else 'before' + valid_substmts = { + 'cli-when-target-set', + 'cli-when-target-create', + 'cli-when-target-modify', + 'cli-when-target-delete', + } + substmts = [s for s in stmt.substmts if s.keyword[1] in valid_substmts] + if len(substmts) == 0: + return [ + ('modify', conj, 'create'), + ('modify', conj, 'modify'), + ('modify', conj, 'delete'), + ] + ordering = [] + for substmt in substmts: + if substmt.keyword[1] == 'cli-when-target-set': + ordering.extend([ + ('modify', conj, 'create'), + ('modify', conj, 'modify'), + ]) + elif substmt.keyword[1] == 'cli-when-target-create': + ordering.append( + ('modify', conj, 'create'), + ) + elif substmt.keyword[1] == 'cli-when-target-modify': + ordering.append( + ('modify', conj, 'modify'), + ) + elif substmt.keyword[1] == 'cli-when-target-delete': + ordering.append( + ('modify', conj, 'delete'), + ) + return ordering + elif stmt.keyword[1] in ['cli-diff-set-after', 'cli-diff-set-before']: + conj = 'after' if stmt.keyword[1] == 'cli-diff-set-after' else 'before' + valid_substmts = { + 'cli-when-target-set', + 'cli-when-target-create', + 'cli-when-target-modify', + 'cli-when-target-delete', + } + substmts = [s for s in stmt.substmts if s.keyword[1] in valid_substmts] + if len(substmts) == 0: + return [ + ('create', conj, 'create'), + ('modify', conj, 'create'), + ('create', conj, 'modify'), + ('modify', conj, 'modify'), + ('create', conj, 'delete'), + ('modify', conj, 'delete'), + ] + ordering = [] + for substmt in substmts: + if substmt.keyword[1] == 'cli-when-target-set': + ordering.extend([ + ('create', conj, 'create'), + ('modify', conj, 'create'), + ('create', conj, 'modify'), + ('modify', conj, 'modify'), + ]) + elif substmt.keyword[1] == 'cli-when-target-create': + ordering.extend([ + ('create', conj, 'create'), + ('modify', conj, 'create'), + ]) + elif substmt.keyword[1] == 'cli-when-target-modify': + ordering.extend([ + ('create', conj, 'modify'), + ('modify', conj, 'modify'), + ]) + elif substmt.keyword[1] == 'cli-when-target-delete': + ordering.extend([ + ('create', conj, 'delete'), + ('modify', conj, 'delete'), + ]) + return ordering + elif stmt.keyword[1] == 'cli-diff-dependency': + valid_substmts = { + 'cli-trigger-on-set', + 'cli-trigger-on-delete', + 'cli-trigger-on-all', + } + substmts = [s for s in stmt.substmts if s.keyword[1] in valid_substmts] + # This is the ordering retrived from TailF confd test result: + ordering = [ + ('create', 'after', 'create'), + ('modify', 'after', 'create'), + ('delete', 'before', 'modify'), + ('create', 'before', 'delete'), + ('modify', 'before', 'delete'), + ('delete', 'before', 'delete'), + ] + # Test result from TailF confd 8.4.7.1: + # 1 depends on 2 + # ('create', 'after', 'create'), + # ('modify', 'after', 'create'), + # ('delete', 'before', 'create'), + # ('create', 'before', 'modify'), + # ('modify', 'before', 'modify'), + # ('delete', 'before', 'modify'), + # ('create', 'before', 'delete'), + # ('modify', 'before', 'delete'), + # ('delete', 'before', 'delete'), + # 2 depends on 1 + # ('create', 'after', 'create'), + # ('modify', 'after', 'create'), + # ('delete', 'after', 'create'), + # ('create', 'after', 'modify'), + # ('modify', 'after', 'modify'), + # ('delete', 'before', 'modify'), + # ('create', 'before', 'delete'), + # ('modify', 'before', 'delete'), + # ('delete', 'before', 'delete'), + if len(substmts) == 0: + return ordering + ordering = [] + for substmt in substmts: + if substmt.keyword[1] == 'cli-trigger-on-set': + ordering.extend([ + ('create', 'after', 'create'), + ('modify', 'after', 'create'), + ('create', 'after', 'modify'), + ('modify', 'after', 'modify'), + ]) + # Test result from TailF confd 8.4.7.1: + # 1 depends on 2 + # ('create', 'after', 'create'), + # ('modify', 'after', 'create'), + # ('delete', 'before', 'create'), + # ('create', 'after', 'modify'), + # ('modify', 'after', 'modify'), + # ('delete', 'before', 'modify'), + # ('create', 'after', 'delete'), + # ('modify', 'after', 'delete'), + # ('delete', 'before', 'delete'), + # 2 depends on 1 + # ('create', 'after', 'create'), + # ('modify', 'after', 'create'), + # ('delete', 'after', 'create'), + # ('create', 'after', 'modify'), + # ('modify', 'after', 'modify'), + # ('delete', 'after', 'modify'), + # ('create', 'after', 'delete'), + # ('modify', 'after', 'delete'), + # ('delete', 'after', 'delete'), + elif substmt.keyword[1] == 'cli-trigger-on-delete': + ordering.extend([ + ('create', 'after', 'create'), + ('modify', 'after', 'create'), + ('delete', 'before', 'modify'), + ('delete', 'before', 'delete'), + ]) + # Test result from TailF confd 8.4.7.1: + # 1 depends on 2 + # ('create', 'after', 'create'), + # ('modify', 'after', 'create'), + # ('delete', 'before', 'create'), + # ('create', 'before', 'modify'), + # ('modify', 'before', 'modify'), + # ('delete', 'before', 'modify'), + # ('create', 'before', 'delete'), + # ('modify', 'before', 'delete'), + # ('delete', 'before', 'delete'), + # 2 depends on 1 + # ('create', 'after', 'create'), + # ('modify', 'after', 'create'), + # ('delete', 'after', 'create'), + # ('create', 'after', 'modify'), + # ('modify', 'after', 'modify'), + # ('delete', 'before', 'modify'), + # ('create', 'before', 'delete'), + # ('modify', 'before', 'delete'), + # ('delete', 'before', 'delete'), + elif substmt.keyword[1] == 'cli-trigger-on-all': + return [ + ('create', 'after', 'create'), + ('modify', 'after', 'create'), + ('delete', 'after', 'create'), + ('create', 'after', 'modify'), + ('modify', 'after', 'modify'), + ('delete', 'after', 'modify'), + ('create', 'after', 'delete'), + ('modify', 'after', 'delete'), + ('delete', 'after', 'delete'), + ] + # Test result from TailF confd 8.4.7.1: + # 1 depends on 2 + # ('create', 'after', 'create'), + # ('modify', 'after', 'create'), + # ('delete', 'after', 'create'), + # ('create', 'after', 'modify'), + # ('modify', 'after', 'modify'), + # ('delete', 'after', 'modify'), + # ('create', 'after', 'delete'), + # ('modify', 'after', 'delete'), + # ('delete', 'after', 'delete'), + # 2 depends on 1 + # ('create', 'after', 'create'), + # ('modify', 'after', 'create'), + # ('delete', 'after', 'create'), + # ('create', 'after', 'modify'), + # ('modify', 'after', 'modify'), + # ('delete', 'after', 'modify'), + # ('create', 'after', 'delete'), + # ('modify', 'after', 'delete'), + # ('delete', 'after', 'delete'), + return ordering + return [] + +def add_tailf_annotation(module_namespaces, stmt, node): + if len(stmt.substmts) > 0: + sub_sm_dict = { + sub.keyword[1]: sub.arg if sub.arg is not None else '' + for sub in stmt.substmts + if ( + isinstance(sub.keyword, tuple) and + 'tailf' in sub.keyword[0] + ) + } + node.set( + etree.QName(module_namespaces[stmt.keyword[0]], + stmt.keyword[1]), + repr(sub_sm_dict) if sub_sm_dict else '', + ) + else: + node.set( + etree.QName(module_namespaces[stmt.keyword[0]], + stmt.keyword[1]), + stmt.arg if stmt.arg else '', + ) + + +def set_ordering_xpath(compiler, module): + # There are cases where a leafref node has TailF ordering annotations + # defined. For example: + # leaf nve { + # description + # "Network virtualization endpoint interface"; + # tailf:cli-allow-join-with-value { + # tailf:cli-display-joined; + # } + # tailf:cli-diff-create-after "/ios:native/ios:interface/ios:nve/ios:name" { + # tailf:cli-when-target-set; + # } + # tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:nve/ios:name" { + # tailf:cli-when-target-delete; + # } + # type leafref { + # path "/ios:native/ios:interface/ios:nve/ios:name"; + # } + # } + # In this case, we treat TailF ordering annotations as higher priority and + # ignore the default leafref ordering constraints. To support this, we + # define a dictionary to track existing nodes that have TailF ordering + # annotations applied. + tailf_ordering = {} + + for constraint_type in ["ordering_stmt_tailf", "ordering_stmt_leafref"]: + if ( + hasattr(compiler, constraint_type) and + module in getattr(compiler, constraint_type) + ): + update_ordering_xpath( + compiler, module, constraint_type, tailf_ordering) + + +def get_xpath(compiler, stmt): + schema_node = getattr(stmt, 'schema_node', None) + if schema_node is None: + return '' + if not hasattr(stmt, 'schema_xpath'): + stmt.schema_xpath = compiler.get_xpath_from_schema_node( + schema_node, type=Tag.LXML_XPATH) + return stmt.schema_xpath + + +def update_ordering_xpath(compiler, module, constraint_type, tailf_ordering): + constraints = [] + stmt = {} + xpath = {} + constraint_info = getattr(compiler, constraint_type)[module] + + for stmt[0], stmt[1], cinstraint_list, xpath_stmt in constraint_info: + + for i in range(2): + xpath[i] = get_xpath(compiler, stmt[i]) + + # Skip entries with missing Xpath. Missing Xpaths might be in a + # different module not compiled or due to other deviations. + if xpath[i] == '': + break + else: + + # Track nodes that have TailF ordering annotations applied. + if constraint_type == "ordering_stmt_tailf": + if stmt[0] not in tailf_ordering: + tailf_ordering[stmt[0]] = {} + if stmt[1] not in tailf_ordering[stmt[0]]: + tailf_ordering[stmt[0]][stmt[1]] = True + + # Skip leafref entries where it already has TailF ordering + # annotations applied. + if constraint_type == "ordering_stmt_leafref": + if ( + stmt[0] in tailf_ordering and + stmt[1] in tailf_ordering[stmt[0]] + ): + continue + + ordering_match = set_ordering_match(compiler, module, xpath_stmt) + x0_before_x1 = 0 + x1_before_x0 = 0 + + for oper_0, sequence, oper_1 in cinstraint_list: + + # Skip entries with same Xpath and same operation. + if xpath[0] == xpath[1] and oper_0 == oper_1: + continue + + if sequence == 'before': + constraints.append(( + xpath[0], oper_0, xpath[1], oper_1, xpath_stmt)) + update_schema_tree(stmt[0], oper_0, stmt[1], oper_1) + x0_before_x1 += DEPENDENCY_TYPE[(oper_0, oper_1)] + else: + constraints.append(( + xpath[1], oper_1, xpath[0], oper_0, xpath_stmt)) + update_schema_tree(stmt[1], oper_1, stmt[0], oper_0) + x1_before_x0 += DEPENDENCY_TYPE[(oper_1, oper_0)] + + if hasattr(compiler, "ordering") and module in compiler.ordering: + if x0_before_x1 > 0: + if xpath[0] not in compiler.ordering[module]: + compiler.ordering[module][xpath[0]] = [] + compiler.ordering[module][xpath[0]].append( + (xpath[1], f"{x0_before_x1:03x}", ordering_match, "1") + ) + if x1_before_x0 > 0: + if xpath[1] not in compiler.ordering[module]: + compiler.ordering[module][xpath[1]] = [] + compiler.ordering[module][xpath[1]].append( + (xpath[0], f"{x1_before_x0:03x}", ordering_match, "0") + ) + + attribute_name = "ordering_xpath_leafref" \ + if constraint_type == "ordering_stmt_leafref" \ + else "ordering_xpath_tailf" + getattr(compiler, attribute_name)[module] = constraints + + + + +def set_ordering_match(compiler, module, xpath_stmt): + if ( + not hasattr(xpath_stmt, 'ordering_match') or + module not in compiler.ordering_match + ): + return None + match_table_indexes = [] + for node_0, operator, node_1, function in xpath_stmt.ordering_match: + if isinstance(node_1, str): + item = ( + get_xpath(compiler, node_0), + operator, + node_1, + function, + ) + else: + item = ( + get_xpath(compiler, node_0), + operator, + get_xpath(compiler, node_1), + function, + ) + + if get_xpath(compiler, node_1) == '': + compiler.xpath_stmt = xpath_stmt + + if item not in compiler.ordering_match[module]: + compiler.ordering_match[module].append(item) + match_table_indexes.append(compiler.ordering_match[module].index(item)) + return " ".join(map(str, match_table_indexes)) + + +def update_schema_tree(stmt_0, oper_0, stmt_1, oper_1): + schema_node_1 = getattr(stmt_0, 'schema_node', None) + if schema_node_1 is None: + logger.warning( + f"Schema node not found for statement {stmt_0.keyword} " + f"at {stmt_0.pos}") + return + ordering_str = schema_node_1.get("before") + if ordering_str is None: + schema_node_1.set("before", repr({ + oper_0: {stmt_1.schema_xpath: [oper_1]} + })) + else: + ordering = eval(ordering_str) + if oper_0 in ordering: + if stmt_1.schema_xpath in ordering[oper_0]: + if oper_1 in ordering[oper_0][stmt_1.schema_xpath]: + return + else: + ordering[oper_0][stmt_1.schema_xpath].append(oper_1) + else: + ordering[oper_0][stmt_1.schema_xpath] = [oper_1] + else: + ordering[oper_0] = {stmt_1.schema_xpath: [oper_1]} + schema_node_1.set("before", repr(ordering)) + + +def is_symmetric_tailf_ordering(context, stmt, target_stmt): + if len(stmt.substmts) != 0: + return False + substmts = { + s for s in target_stmt.substmts + if isinstance(s.keyword, tuple) and + 'tailf' in s.keyword[0] and + len(s.substmts) == 0 and + s.keyword[1] == stmt.keyword[1] + } + for substmt in substmts: + target = context.check_data_tree_xpath( + substmt, target_stmt) + if target == stmt.parent: + return True + return False diff --git a/src/ncdiff/tests/test_tailf_annotation.py b/src/ncdiff/tests/test_tailf_annotation.py new file mode 100755 index 0000000..196a362 --- /dev/null +++ b/src/ncdiff/tests/test_tailf_annotation.py @@ -0,0 +1,637 @@ +#!/bin/env python +""" Unit tests for the ncdiff cisco-shared package. """ + +import os +import unittest +from ncdiff.composer import Tag +from ncdiff.model import ModelCompiler +from ncdiff.tailf import is_tailf_ordering, get_tailf_ordering +from ncdiff.tailf import is_symmetric_tailf_ordering +from ncdiff.tailf import is_deprecated_without_replacement + + +curr_dir = os.path.dirname(os.path.abspath(__file__)) + + +def delete_xml_files(folder): + for filename in os.listdir(folder): + if filename.endswith(".xml"): + file_path = os.path.join(folder, filename) + os.remove(file_path) + + +class TestNative(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.compiler = ModelCompiler(os.path.join(curr_dir, 'yang')) + cls.compiler.exclude_obsolete = True + cls.compiler.exclude_deprecated = True + cls.compiler.include_deprecated_without_replacement = True + delete_xml_files(cls.compiler.dir_yang) + cls.native = cls.compiler.compile('Cisco-IOS-XE-native') + # cls.oc_interfaces = cls.compiler.compile('openconfig-interfaces') + + def test_dependencies(self): + self.assertIsNotNone(self.compiler.context) + self.assertEqual(self.native.tree.tag, 'Cisco-IOS-XE-native') + imports, includes, depends = \ + self.compiler._dependencies['Cisco-IOS-XE-native'] + self.assertIn('Cisco-IOS-XE-features', imports) + self.assertIn('Cisco-IOS-XE-interfaces', includes) + self.assertIn('Cisco-IOS-XE-sla', depends) + self.assertIn('Cisco-IOS-XE-sla-ann', depends) + + def test_check_data_tree_xpath(self): + # Line 52 in Cisco-IOS-XE-sla-ann.yang: + # tailf:annotate-module Cisco-IOS-XE-sla { + # tailf:annotate-statement "grouping[name='config-ip-sla-grouping']" { + # tailf:annotate-statement "container[name='sla']" { + # tailf:annotate-statement "list[name='entry']" { + # tailf:annotate-statement "choice[name='sla-param'] " { + # tailf:annotate-statement "case[name='path-echo-case'] " { + # tailf:annotate-statement "container[name='path-echo']" { + # tailf:annotate-statement "leaf[name='source-ip']" { + # tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-set; + # } + # tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-delete; + # } + # } + # } + # } + # } + # } + # } + # } + # } + stmt = self.compiler.context.get_module('Cisco-IOS-XE-sla') + for arg in [ + "config-ip-sla-grouping", + "sla", + "entry", + "sla-param", + "path-echo-case", + "path-echo", + "source-ip", + ]: + stmts = [i for i in stmt.substmts if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + stmts = [i for i in stmt.substmts + if i.keyword == ("tailf-common", "cli-diff-create-after")] + self.assertEqual(len(stmts), 1) + xpath_stmt = stmts[0] + + target = self.compiler.context.check_data_tree_xpath(xpath_stmt, stmt) + + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "interface", + "GigabitEthernet", + "carrier-delay", + "delay-choice", + "seconds", + "seconds", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + + self.assertIs(target, stmt) + + def test_check_schema_tree_xpath(self): + # Line 75 in Cisco-IOS-XE-sla-ann.yang: + # tailf:annotate "/ios:native/ios:ip/ios-sla:sla/ios-sla:entry/ios-sla:sla-param/ios-sla:path-echo-case/ios-sla:path-echo/ios-sla:dst-ip" { + # tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-delete; + # } + # tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-create; + # } + # } + stmt = self.compiler.context.get_module('Cisco-IOS-XE-sla-ann') + stmts = [i for i in stmt.substmts + if i.keyword == ("tailf-common", "annotate")] + self.assertEqual(len(stmts), 1) + annotating_stmt = stmts[0] + + target = self.compiler.context.check_schema_tree_xpath(annotating_stmt) + + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "ip", + "sla", + "entry", + "sla-param", + "path-echo-case", + "path-echo", + "dst-ip", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + + self.assertIs(target, stmt) + + def test_get_xpath_from_schema_node(self): + xpath = "/ios:native/ios:ip/ios-sla:sla/ios-sla:entry" \ + "/ios-sla:sla-param/ios-sla:path-echo-case/ios-sla:path-echo" \ + "/ios-sla:dst-ip" + self.assertIsNotNone(self.native.tree) + matches = self.native.tree.xpath( + "/Cisco-IOS-XE-native" + xpath, + namespaces=self.native.prefixes, + ) + self.assertEqual(len(matches), 1) + schema_node = matches[0] + result_xpath = self.compiler.context.get_xpath_from_schema_node( + schema_node, type=Tag.LXML_XPATH) + self.assertEqual(result_xpath, xpath) + + def test_process_annotation_module_1(self): + # Line 52 in Cisco-IOS-XE-sla-ann.yang: + # tailf:annotate-module Cisco-IOS-XE-sla { + # tailf:annotate-statement "grouping[name='config-ip-sla-grouping']" { + # tailf:annotate-statement "container[name='sla']" { + # tailf:annotate-statement "list[name='entry']" { + # tailf:annotate-statement "choice[name='sla-param'] " { + # tailf:annotate-statement "case[name='path-echo-case'] " { + # tailf:annotate-statement "container[name='path-echo']" { + # tailf:annotate-statement "leaf[name='source-ip']" { + # tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-set; + # } + # tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-delete; + # } + # } + # } + # } + # } + # } + # } + # } + # } + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "ip", + "sla", + "entry", + "sla-param", + "path-echo-case", + "path-echo", + "source-ip", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + + stmts = [i for i in stmt.substmts + if i.keyword == ("tailf-common", "cli-diff-create-after") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + self.assertEqual(len(annotation.substmts), 1) + annotation_substmt = annotation.substmts[0] + self.assertEqual( + annotation_substmt.keyword, + ("tailf-common", "cli-when-target-set"), + ) + + stmts = [i for i in stmt.substmts + if i.keyword == ("tailf-common", "cli-diff-delete-before") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + self.assertEqual(len(annotation.substmts), 1) + annotation_substmt = annotation.substmts[0] + self.assertEqual( + annotation_substmt.keyword, + ("tailf-common", "cli-when-target-delete"), + ) + + def test_process_annotation_module_2(self): + # Line 75 in Cisco-IOS-XE-sla-ann.yang: + # tailf:annotate "/ios:native/ios:ip/ios-sla:sla/ios-sla:entry/ios-sla:sla-param/ios-sla:path-echo-case/ios-sla:path-echo/ios-sla:dst-ip" { + # tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-delete; + # } + # tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-create; + # } + # } + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "ip", + "sla", + "entry", + "sla-param", + "path-echo-case", + "path-echo", + "dst-ip", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + + stmts = [i for i in stmt.substmts + if i.keyword == ("tailf-common", "cli-diff-create-after") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + self.assertEqual(len(annotation.substmts), 1) + annotation_substmt = annotation.substmts[0] + self.assertEqual( + annotation_substmt.keyword, + ("tailf-common", "cli-when-target-delete"), + ) + + stmts = [i for i in stmt.substmts + if i.keyword == ("tailf-common", "cli-diff-delete-before") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + self.assertEqual(len(annotation.substmts), 1) + annotation_substmt = annotation.substmts[0] + self.assertEqual( + annotation_substmt.keyword, + ("tailf-common", "cli-when-target-create"), + ) + + def test_ordering_stmt_leafref(self): + self.assertIn( + 'Cisco-IOS-XE-native', + self.compiler.ordering_stmt_leafref, + ) + + # Line 159 in Cisco-IOS-XE-parser.yang: + # leaf view-name { + # type leafref { + # path "../../../view-name-list/name"; + # } + # } + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "parser", + "view", + "view-name-superview-list", + "view", + "view-name", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + leafref = stmt + + tuples = [ + i + for i in self.compiler.ordering_stmt_leafref['Cisco-IOS-XE-native'] + if i[0] is leafref + ] + self.assertEqual(len(tuples), 1) + leafref_stmt, target_stmt, ordering, xpath_stmt = tuples[0] + + type_stmt = leafref.search_one('type') + self.assertIsNotNone(type_stmt) + path_stmt = type_stmt.search_one('path') + self.assertIsNotNone(path_stmt) + target = self.compiler.context.check_data_tree_xpath( + path_stmt, leafref) + + self.assertIs(leafref, leafref_stmt) + self.assertIs(target, target_stmt) + self.assertIsInstance(ordering, list) + self.assertIn('Cisco-IOS-XE-parser.yang:161', str(xpath_stmt.pos)) + + def test_ordering_stmt_tailf(self): + self.assertIn('Cisco-IOS-XE-native', self.compiler.ordering_stmt_tailf) + + # Line 75 in Cisco-IOS-XE-sla-ann.yang: + # tailf:annotate "/ios:native/ios:ip/ios-sla:sla/ios-sla:entry/ios-sla:sla-param/ios-sla:path-echo-case/ios-sla:path-echo/ios-sla:dst-ip" { + # tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-delete; + # } + # tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-create; + # } + # } + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "ip", + "sla", + "entry", + "sla-param", + "path-echo-case", + "path-echo", + "dst-ip", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + node = stmt + + # tailf:cli-diff-create-after + stmts = [i for i in node.substmts + if i.keyword == ("tailf-common", "cli-diff-create-after") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + self.assertEqual(len(annotation.substmts), 1) + annotation_substmt = annotation.substmts[0] + self.assertEqual( + annotation_substmt.keyword, + ("tailf-common", "cli-when-target-delete"), + ) + + # tailf:cli-diff-delete-before + stmts = [i for i in node.substmts + if i.keyword == ("tailf-common", "cli-diff-delete-before") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + self.assertEqual(len(annotation.substmts), 1) + annotation_substmt = annotation.substmts[0] + self.assertEqual( + annotation_substmt.keyword, + ("tailf-common", "cli-when-target-create"), + ) + + tuples = [ + i + for i in self.compiler.ordering_stmt_tailf['Cisco-IOS-XE-native'] + if i[0] is node + ] + self.assertEqual(len(tuples), 2) + + target = self.compiler.context.check_data_tree_xpath( + annotation, node) + positions = [ + 'Cisco-IOS-XE-sla-ann.yang:76', + 'Cisco-IOS-XE-sla-ann.yang:79', + ] + for node_stmt, target_stmt, ordering, xpath_stmt in tuples: + self.assertIs(target, target_stmt) + self.assertIsInstance(ordering, list) + for p in positions: + if p in str(xpath_stmt.pos): + positions.remove(p) + break + self.assertEqual(len(positions), 0) + + def test_datatype_leafref(self): + # Line 159 in Cisco-IOS-XE-parser.yang: + # leaf view-name { + # type leafref { + # path "../../../view-name-list/name"; + # } + # } + xpath = "/ios:native/ios:parser/ios:view" \ + "/ios:view-name-superview-list/ios:view/ios:view-name" + matches = self.native.tree.xpath( + "/Cisco-IOS-XE-native" + xpath, + namespaces=self.native.prefixes, + ) + self.assertEqual(len(matches), 1) + schema_node = matches[0] + datatype = schema_node.get("datatype", None) + self.assertEqual(datatype, "leafref ../../../view-name-list/name") + + def test_has_tailf_ordering(self): + # Line 75 in Cisco-IOS-XE-sla-ann.yang: + # tailf:annotate "/ios:native/ios:ip/ios-sla:sla/ios-sla:entry/ios-sla:sla-param/ios-sla:path-echo-case/ios-sla:path-echo/ios-sla:dst-ip" { + # tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-delete; + # } + # tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-create; + # } + # } + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "ip", + "sla", + "entry", + "sla-param", + "path-echo-case", + "path-echo", + "dst-ip", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + node = stmt + func_result = is_tailf_ordering(node) + self.assertFalse(func_result) + + stmts = [i for i in node.substmts + if i.keyword == ("tailf-common", "cli-diff-create-after") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + func_result = is_tailf_ordering(annotation) + self.assertTrue(func_result) + + def test_get_tailf_ordering(self): + # Line 75 in Cisco-IOS-XE-sla-ann.yang: + # tailf:annotate "/ios:native/ios:ip/ios-sla:sla/ios-sla:entry/ios-sla:sla-param/ios-sla:path-echo-case/ios-sla:path-echo/ios-sla:dst-ip" { + # tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-delete; + # } + # tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-create; + # } + # } + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "ip", + "sla", + "entry", + "sla-param", + "path-echo-case", + "path-echo", + "dst-ip", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + node = stmt + + stmts = [i for i in node.substmts + if i.keyword == ("tailf-common", "cli-diff-create-after") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + target = self.compiler.context.check_data_tree_xpath( + annotation, node) + ordering = get_tailf_ordering(self.compiler.context, annotation, target) + self.assertEqual(ordering, [('create', 'after', 'delete')]) + + stmts = [i for i in node.substmts + if i.keyword == ("tailf-common", "cli-diff-delete-before") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + target = self.compiler.context.check_data_tree_xpath( + annotation, node) + ordering = get_tailf_ordering(self.compiler.context, annotation, target) + self.assertEqual(ordering, [('delete', 'before', 'create')]) + + def test_is_symmetric_tailf_ordering(self): + # Line 75 in Cisco-IOS-XE-sla-ann.yang: + # tailf:annotate "/ios:native/ios:ip/ios-sla:sla/ios-sla:entry/ios-sla:sla-param/ios-sla:path-echo-case/ios-sla:path-echo/ios-sla:dst-ip" { + # tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-delete; + # } + # tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + # tailf:cli-when-target-create; + # } + # } + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "ip", + "sla", + "entry", + "sla-param", + "path-echo-case", + "path-echo", + "dst-ip", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + node = stmt + + stmts = [i for i in node.substmts + if i.keyword == ("tailf-common", "cli-diff-create-after") and + i.arg == "/ios:native/ios:interface/ios:GigabitEthernet" + "/ios-eth:carrier-delay/ios-eth:seconds"] + self.assertEqual(len(stmts), 1) + annotation = stmts[0] + target = self.compiler.context.check_data_tree_xpath( + annotation, node) + func_result = is_symmetric_tailf_ordering(self.compiler.context, annotation, target) + self.assertFalse(func_result) + + def test_is_deprecated_without_replacement(self): + # Line 1523 in Cisco-IOS-XE-lisp.yang: + # grouping router-lisp-ip-grouping { + # leaf alt-vrf { + # description + # "Activate LISP-ALT functionality in VRF"; + # status deprecated; + # ios-types:yang-meta-data "deprecated-without-replacement"; + # type string; + # } + # ... + # } + module_stmt = self.compiler.context.get_module('Cisco-IOS-XE-native') + stmts = [i for i in module_stmt.substmts if i.arg == "native"] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + for arg in [ + "router", + "lisp", + "ipv4", + "alt-vrf", + ]: + stmts = [i for i in stmt.i_children if i.arg == arg] + self.assertEqual(len(stmts), 1) + stmt = stmts[0] + + # Node ipv4 does not have deprecated-without-replacement + func_result = is_deprecated_without_replacement(stmt.parent) + self.assertFalse(func_result) + + # Node alt-vrf has deprecated-without-replacement + func_result = is_deprecated_without_replacement(stmt) + self.assertTrue(func_result) + + # Check that the alt-vrf node is present in the compiled tree, even + # though it is deprecated + nodes = self.native.tree.xpath( + "//ios:native/ios:router/ios-lisp:lisp/ios-lisp:ipv4/ios-lisp:alt-vrf", + namespaces=self.native.prefixes) + self.assertEqual(len(nodes), 1) + + +class TestOpenConfigInterfaces(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.compiler = ModelCompiler(os.path.join(curr_dir, 'yang')) + delete_xml_files(cls.compiler.dir_yang) + cls.oc_interfaces = cls.compiler.compile('openconfig-interfaces') + + def test_datatype_identityref(self): + # Line 254 in Cisco-IOS-XE-interfaces.yang: + # leaf type { + # type identityref { + # base ietf-if:interface-type; + # } + # mandatory true; + # description + # "[adapted from IETF interfaces model (RFC 7223)] + + # The type of the interface. + + # When an interface entry is created, a server MAY + # initialize the type leaf with a valid value, e.g., if it + # is possible to derive the type from the name of the + # interface. + + # If a client tries to set the type of an interface to a + # value that can never be used by the system, e.g., if the + # type is not supported or if the type does not match the + # name of the interface, the server MUST reject the request. + # A NETCONF server MUST reply with an rpc-error with the + # error-tag 'invalid-value' in this case."; + # reference + # "RFC 2863: The Interfaces Group MIB - ifType"; + # } + xpath = "/oc-if:interfaces/oc-if:interface/oc-if:config/oc-if:type" + matches = self.oc_interfaces.tree.xpath( + "/openconfig-interfaces" + xpath, + namespaces=self.oc_interfaces.prefixes, + ) + self.assertEqual(len(matches), 1) + schema_node = matches[0] + datatype = schema_node.get("datatype", None) + self.assertEqual(datatype, "identityref ietf-if:interface-type") diff --git a/src/ncdiff/tests/yang/Cisco-IOS-XE-lisp.yang b/src/ncdiff/tests/yang/Cisco-IOS-XE-lisp.yang index f485164..7f9a844 100644 --- a/src/ncdiff/tests/yang/Cisco-IOS-XE-lisp.yang +++ b/src/ncdiff/tests/yang/Cisco-IOS-XE-lisp.yang @@ -71,7 +71,7 @@ module Cisco-IOS-XE-lisp { } - //router lisp new grouping starts + //router lisp new grouping starts //router lisp service route import database protocol grouping router-lisp-inst-service-ip-route-import-database-protocol-grouping { container application { @@ -339,13 +339,13 @@ module Cisco-IOS-XE-lisp { } } - //router lisp inst service ipv6 router-import + //router lisp inst service ipv6 router-import grouping router-lisp-inst-service-ipv6-route-import-protocol-grouping { uses router-lisp-inst-service-ipv6-route-import-database-protocol-grouping; uses router-lisp-inst-service-ipv6-route-import-map-cache-protocol-grouping; } - //router lisp inst service ipv4 router-import + //router lisp inst service ipv4 router-import grouping router-lisp-inst-service-ipv4-route-import-protocol-grouping { uses router-lisp-inst-service-ipv4-route-import-database-protocol-grouping; uses router-lisp-inst-service-ipv4-route-import-map-cache-protocol-grouping; @@ -373,7 +373,7 @@ module Cisco-IOS-XE-lisp { } } } - + //router lisp etr grouping router-lisp-etr-grouping { container etr-enable { @@ -401,7 +401,7 @@ module Cisco-IOS-XE-lisp { } } } - } + } //router lisp database mapping limit grouping router-lisp-database-mapping-limit-grouping { @@ -422,7 +422,7 @@ module Cisco-IOS-XE-lisp { range "1..100"; } } - } + } } //router lisp map cache @@ -466,7 +466,7 @@ module Cisco-IOS-XE-lisp { type inet:ipv6-address; } } - + //router lisp map request source any grouping router-lisp-map-request-source-any-grouping { leaf map-request-source { @@ -550,7 +550,7 @@ module Cisco-IOS-XE-lisp { } } - //router lisp service common + //router lisp service common grouping router-lisp-service-common-grouping { container database-mapping { description @@ -686,8 +686,8 @@ module Cisco-IOS-XE-lisp { uses router-lisp-map-cache-persistent-grouping; uses router-lisp-proxy-grouping; uses router-lisp-route-export-grouping; - uses router-lisp-sgt-grouping; - uses router-lisp-use-petr-grouping; + uses router-lisp-sgt-grouping; + uses router-lisp-use-petr-grouping; } //router lisp service ipv4 @@ -702,7 +702,7 @@ module Cisco-IOS-XE-lisp { uses router-lisp-map-request-source-ipv6-grouping; } - //router lisp four key + //router lisp four key grouping router-lisp-four-key-grouping { leaf unc-pwd { description @@ -724,10 +724,10 @@ module Cisco-IOS-XE-lisp { "The ENCRYPTED password"; type string; } - } + } //router lisp key hash function - grouping router-lisp-key-hash-function-grouping { + grouping router-lisp-key-hash-function-grouping { leaf hash-function { description "authentication type"; @@ -757,8 +757,8 @@ module Cisco-IOS-XE-lisp { uses router-lisp-key-hash-function-grouping; } } - - //router lisp passowd key-7 + + //router lisp passowd key-7 grouping router-lisp-password-key-7-grouping { container key-7 { leaf ak-7 { @@ -867,7 +867,7 @@ module Cisco-IOS-XE-lisp { } } } - + //router lisp use-petr grouping router-lisp-use-petr-grouping { list use-petr { @@ -906,28 +906,28 @@ module Cisco-IOS-XE-lisp { "LISP routes installed in the ALT table"; type uint8 { range 1..255; - } + } } leaf away { description "Administrative distance for RIB route installation"; type uint8 { range 1..255; - } + } } leaf dyn-eid { description "LISP installed routes of type dynamic-EID"; type uint8 { range 1..255; - } + } } leaf site-registrations { description "LISP installed routes of type site-registrations"; type uint8 { range 1..255; - } + } } } } @@ -961,7 +961,7 @@ module Cisco-IOS-XE-lisp { description "Configures which Locators from a set are preferred"; type uint8 { range 0..255; - } + } } leaf weight { description "Traffic load-spreading among Locators"; @@ -969,14 +969,14 @@ module Cisco-IOS-XE-lisp { range 0..100; } } - leaf down { + leaf down { description "Configure this database mapping down"; type empty; } } } - //router lisp inst database mapping common + //router lisp inst database mapping common grouping router-lisp-inst-database-mapping-common-grouping { leaf locator-set { description @@ -997,7 +997,7 @@ module Cisco-IOS-XE-lisp { key "address"; leaf address { type inet:ipv6-address; - } + } uses router-lisp-inst-database-mapping-option-grouping; } @@ -1034,7 +1034,7 @@ module Cisco-IOS-XE-lisp { //router lisp inst service ethernet grouping router-lisp-inst-service-ethernet-grouping { container eid-table { - description "Bind an eid-table"; + description "Bind an eid-table"; leaf vlan { description "VLAN configuration"; type uint16 { @@ -1042,7 +1042,7 @@ module Cisco-IOS-XE-lisp { } } } - container broadcast-underlay { + container broadcast-underlay { description "Multicast group to use for underlay"; leaf ipv4-multicast { description "IPv4 multicast group address"; @@ -1187,9 +1187,9 @@ module Cisco-IOS-XE-lisp { uses router-lisp-map-cache-persistent-grouping; uses router-lisp-proxy-grouping; uses router-lisp-route-export-grouping; - uses router-lisp-sgt-grouping; - uses router-lisp-use-petr-grouping; - } + uses router-lisp-sgt-grouping; + uses router-lisp-use-petr-grouping; + } //router lisp inst service ipv4 grouping grouping router-lisp-inst-service-ipv4-grouping { @@ -1243,7 +1243,7 @@ module Cisco-IOS-XE-lisp { } } - //router lisp inst + //router lisp inst grouping router-lisp-inst-grouping { container decapsulation { description @@ -1421,7 +1421,7 @@ module Cisco-IOS-XE-lisp { } } container service { - description + description "Configure lisp service type"; presence true; container ipv4 { @@ -1444,11 +1444,11 @@ module Cisco-IOS-XE-lisp { uses router-lisp-inst-service-ethernet-grouping; } uses router-lisp-inst-service-ethernet-grouping; - } + } } } - //router lisp new grouping ends + //router lisp new grouping ends grouping router-lisp-ip-route-import-map-cache-grouping { container map-cache-container { @@ -1462,7 +1462,7 @@ module Cisco-IOS-XE-lisp { } grouping router-lisp-ip-route-import-database-grouping { - container lisp-ip-route-import { + container lisp-ip-route-import { leaf route-map { description "Route map for route selection filtering"; @@ -1525,6 +1525,7 @@ module Cisco-IOS-XE-lisp { description "Activate LISP-ALT functionality in VRF"; status deprecated; + ios-types:yang-meta-data "deprecated-without-replacement"; type string; } container database-mapping { @@ -2173,10 +2174,10 @@ module Cisco-IOS-XE-lisp { description "Configures a LISP Egress Tunnel Router (ETR)"; container map-server { - description + description "Configures map server for ETR registration"; leaf source-address { - description + description "Configures map server source address"; type string; } @@ -2393,7 +2394,7 @@ module Cisco-IOS-XE-lisp { } } - //router lisp site common grouping + //router lisp site common grouping grouping router-lisp-site-common-grouping { container authentication-key { description @@ -2451,13 +2452,13 @@ module Cisco-IOS-XE-lisp { description "Accept registrations for any L2 EID records"; type empty; - } + } } leaf any-mac { description "Accept registrations for any L2 EID records"; type empty; - } + } } container eid-record { description @@ -2477,13 +2478,13 @@ module Cisco-IOS-XE-lisp { description "Accept registrations for any L2 EID records"; type empty; - } + } } leaf any-mac { description "Accept registrations for any L2 EID records"; type empty; - } + } } leaf site-id { description @@ -2492,9 +2493,9 @@ module Cisco-IOS-XE-lisp { range "0..4294967295"; } } - } - - //router lisp site grouping + } + + //router lisp site grouping grouping rouer-lisp-site-grouping { list site { description @@ -2507,7 +2508,7 @@ module Cisco-IOS-XE-lisp { } container default { uses router-lisp-site-common-grouping; - } + } uses router-lisp-site-common-grouping; } } @@ -2827,7 +2828,7 @@ module Cisco-IOS-XE-lisp { type empty; } } - + uses rouer-lisp-site-grouping; leaf site-id { diff --git a/src/ncdiff/tests/yang/Cisco-IOS-XE-sla-ann.yang b/src/ncdiff/tests/yang/Cisco-IOS-XE-sla-ann.yang new file mode 100644 index 0000000..c3e8921 --- /dev/null +++ b/src/ncdiff/tests/yang/Cisco-IOS-XE-sla-ann.yang @@ -0,0 +1,83 @@ +module Cisco-IOS-XE-sla-ann { + namespace "http://cisco.com/ns/yang/Cisco-IOS-XE-sla-ann"; + prefix ios-sla-ann; + + import Cisco-IOS-XE-native { + prefix ios; + } + + import Cisco-IOS-XE-sla { + prefix ios-sla; + } + + import Cisco-IOS-XE-ethernet { + prefix ios-eth; + } + import tailf-common { + prefix tailf; + } + + organization + "Cisco Systems, Inc."; + + contact + "Cisco Systems, Inc. + Customer Service + Postal: 170 W Tasman Drive + San Jose, CA 95134 + Tel: +1 1800 553-NETS + E-mail: cs-yang@cisco.com"; + + description + "Cisco XE Native Service Level Agreements (SLA) Annotation Yang Model. + Copyright (c) 2019-2021 by Cisco Systems, Inc. + All rights reserved."; + + revision 2020-07-01 { + description + "Removed annotations for ethernet-monitor container nodes as they are hardened in + 17.1.1 release"; + } + + revision 2019-12-04 { + description + "Added annotations for all non-hardened nodes to hide them from + controller"; + } + + revision 2019-03-28 { + description "Initial revision"; + } + + tailf:annotate-module Cisco-IOS-XE-sla { + tailf:annotate-statement "grouping[name='config-ip-sla-grouping']" { + tailf:annotate-statement "container[name='sla']" { + tailf:annotate-statement "list[name='entry']" { + tailf:annotate-statement "choice[name='sla-param'] " { + tailf:annotate-statement "case[name='path-echo-case'] " { + tailf:annotate-statement "container[name='path-echo']" { + tailf:annotate-statement "leaf[name='source-ip']" { + tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + tailf:cli-when-target-set; + } + tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + tailf:cli-when-target-delete; + } + } + } + } + } + } + } + } + } + + tailf:annotate "/ios:native/ios:ip/ios-sla:sla/ios-sla:entry/ios-sla:sla-param/ios-sla:path-echo-case/ios-sla:path-echo/ios-sla:dst-ip" { + tailf:cli-diff-create-after "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + tailf:cli-when-target-delete; + } + tailf:cli-diff-delete-before "/ios:native/ios:interface/ios:GigabitEthernet/ios-eth:carrier-delay/ios-eth:seconds" { + tailf:cli-when-target-create; + } + } +} diff --git a/src/ncdiff/tests/yang/Cisco-IOS-XE-types.yang b/src/ncdiff/tests/yang/Cisco-IOS-XE-types.yang index 3533b39..67593de 100644 --- a/src/ncdiff/tests/yang/Cisco-IOS-XE-types.yang +++ b/src/ncdiff/tests/yang/Cisco-IOS-XE-types.yang @@ -26,6 +26,16 @@ module Cisco-IOS-XE-types { Copyright (c) 2016-2017 by Cisco Systems, Inc. All rights reserved."; + // ========================================================================= + // EXTENSION + // ========================================================================= + + extension yang-meta-data { + argument value; + description "Extra information associated with the yang node to tell + model compiler to compile it in a specific way"; + } + // ========================================================================= // REVISION // ========================================================================= @@ -685,8 +695,8 @@ module Cisco-IOS-XE-types { } } } - - // Comma-separated numbers with ranges + + // Comma-separated numbers with ranges typedef range-string { type string { pattern diff --git a/src/ncdiff/xpath.py b/src/ncdiff/xpath.py new file mode 100755 index 0000000..c47ccc0 --- /dev/null +++ b/src/ncdiff/xpath.py @@ -0,0 +1,340 @@ +import pyang +import logging + + +logger = logging.getLogger(__name__) + + +def set_ordering_match(ctx, xpath_stmt, initial, node1, node2, operator): + if not hasattr(xpath_stmt, 'raw_ordering_match'): + xpath_stmt.raw_ordering_match = [] + match_item = (node1, operator, node2) + if match_item not in xpath_stmt.raw_ordering_match: + xpath_stmt.raw_ordering_match.append(match_item) + + n2 = get_function(node2, xpath_stmt) + if n2 is not None: + if not hasattr(xpath_stmt, 'ordering_match'): + xpath_stmt.ordering_match = [] + if isinstance(n2, tuple): + match_item = (node1, operator) + n2 + else: + match_item = (node1, operator, n2, None) + if match_item not in xpath_stmt.ordering_match: + xpath_stmt.ordering_match.append(match_item) + +def get_function(tuple_info, xpath_stmt): + """tuple_info is a tuple of the form (type, inputs). For example, + ('number', [('object', ('substring-before', [('string', ), ('string', '.')]))])""" + if not isinstance(tuple_info, tuple): + return tuple_info + func, args = tuple_info + if func in ['substring-before', 'substring-after']: + if len(args) != 2: + logger.error(f"{func}() should have 2 arguments but actually has " + f"{len(args)}:\n{xpath_stmt.pos}") + return None + if args[1] != ('string', '.'): + logger.warning(f"{func}() should have the 2nd argument as '.' but " + f"actually has {args[1]}:\n{xpath_stmt.pos}") + return None + return (args[0][1], func) + elif func == 'string': + if len(args) != 1: + logger.error(f"string() should have 1 argument but actually has " + f"{len(args)}:\n{xpath_stmt.pos}") + return None + return args[0][1] + elif func == 'number': + if len(args) != 1: + logger.error(f"number() should have 1 argument but actually has " + f"{len(args)}:\n{xpath_stmt.pos}") + return None + return get_function(args[0][1], xpath_stmt) + else: + logger.warning(f"{func}() is not supported by the value matching " + f"feature\n{xpath_stmt.pos}") + return None + +def chk_xpath_expr(ctx, xpath_stmt, initial, node, q, t): + mod = xpath_stmt.i_orig_module + pos = xpath_stmt.pos + + if isinstance(q, list): + return chk_xpath_path(ctx, xpath_stmt, initial, node, q) + elif isinstance(q, tuple): + if q[0] == 'absolute': + return chk_xpath_path(ctx, xpath_stmt, initial, 'root', q[1]) + elif q[0] == 'relative': + return chk_xpath_path(ctx, xpath_stmt, initial, node, q[1]) + elif q[0] == 'union': + return [ + chk_xpath_path(ctx, xpath_stmt, initial, node, qa) + for qa in q[1] + ] + elif q[0] == 'comp': + node1 = chk_xpath_expr(ctx, xpath_stmt, initial, node, q[2], None) + node2 = chk_xpath_expr(ctx, xpath_stmt, initial, node, q[3], None) + set_ordering_match(ctx, xpath_stmt, initial, node1, node2, q[1]) + elif q[0] == 'arith': + chk_xpath_expr(ctx, xpath_stmt, initial, node, q[2], None) + chk_xpath_expr(ctx, xpath_stmt, initial, node, q[3], None) + elif q[0] == 'bool': + chk_xpath_expr(ctx, xpath_stmt, initial, node, q[2], None) + chk_xpath_expr(ctx, xpath_stmt, initial, node, q[3], None) + elif q[0] == 'negative': + chk_xpath_expr(ctx, xpath_stmt, initial, node, q[1], None) + elif q[0] == 'function_call': + rettype, ret, inputs = chk_xpath_function(ctx, xpath_stmt, initial, node, q[1], q[2]) + if ret is None: + return q[1], inputs + else: + return ret + elif q[0] == 'path_expr': + return chk_xpath_expr(ctx, xpath_stmt, initial, node, q[1], t) + # return sth + elif q[0] == 'path': # q[1] == 'filter' + chk_xpath_expr(ctx, xpath_stmt, initial, node, q[2], None) + chk_xpath_expr(ctx, xpath_stmt, initial, node, q[3], None) + elif q[0] == 'var': + # NOTE: check if the variable is known; currently we don't + # have any variables in YANG xpath expressions + pyang.error.err_add(ctx.errors, pos, 'XPATH_VARIABLE', q[1]) + elif q[0] == 'literal': + # kind of hack to detect qnames, and mark the prefixes + # as being used in order to avoid warnings. + s = q[1] + if s[0] == s[-1] and s[0] in ("'", '"'): + s = s[1:-1] + i = s.find(':') + # make sure there is just one : present + # FIXME: more colons should possibly be reported, instead + if i != -1 and s.find(':', i + 1) == -1: + prefix = s[:i] + tag = s[i + 1:] + if (pyang.syntax.re_identifier.search(prefix) is not None and + pyang.syntax.re_identifier.search(tag) is not None): + # we don't want to report an error; just mark the + # prefix as being used. + my_errors = [] + pyang.util.prefix_to_module(mod, prefix, pos, my_errors) + for pos0, code, arg in my_errors: + if code == 'PREFIX_NOT_DEFINED' and t == 'qstring': + # we know for sure that this is an error + pyang.error.err_add(ctx.errors, pos0, + 'PREFIX_NOT_DEFINED', arg) + else: + # this may or may not be an error; + # report a warning + pyang.error.err_add(ctx.errors, pos0, + 'WPREFIX_NOT_DEFINED', arg) + return s + + +def chk_xpath_function(ctx, xpath_stmt, initial, node, func, args): + mod = xpath_stmt.i_orig_module + pos = xpath_stmt.pos + + signature = None + if func in pyang.xpath.core_functions: + signature = pyang.xpath.core_functions[func] + elif func in pyang.xpath.yang_xpath_functions: + signature = pyang.xpath.yang_xpath_functions[func] + elif mod.i_version != '1' and func in pyang.xpath.yang_1_1_xpath_functions: + signature = pyang.xpath.yang_1_1_xpath_functions[func] + elif ctx.strict and func in pyang.xpath.extra_xpath_functions: + pyang.error.err_add(ctx.errors, pos, 'STRICT_XPATH_FUNCTION', func) + return None + elif not ctx.strict and func in pyang.xpath.extra_xpath_functions: + signature = pyang.xpath.extra_xpath_functions[func] + if signature is None: + pyang.error.err_add(ctx.errors, pos, 'XPATH_FUNCTION', func) + return None + # check that the number of arguments are correct + nexp = len(signature[0]) + nargs = len(args) + if nexp == 0: + if nargs != 0: + pyang.error.err_add(ctx.errors, pos, 'XPATH_FUNC_ARGS', + (func, nexp, nargs)) + elif signature[0][-1] == '?': + if nargs != (nexp - 1) and nargs != (nexp - 2): + pyang.error.err_add(ctx.errors, pos, 'XPATH_FUNC_ARGS', + (func, "%s-%s" % (nexp - 2, nexp - 1), nargs)) + elif signature[0][-1] == '*': + if nargs < (nexp - 1): + pyang.error.err_add(ctx.errors, pos, 'XPATH_FUNC_ARGS', + (func, "at least %s" % (nexp - 1), nargs)) + elif nexp != nargs: + pyang.error.err_add(ctx.errors, pos, 'XPATH_FUNC_ARGS', + (func, nexp, nargs)) + # check the arguments - FIXME check type + i = 0 + args_signature = signature[0][:] + if func == 'deref': + arg = args[0] + tgt = chk_xpath_path(ctx, xpath_stmt, initial, node, arg) + if tgt is not None: + if not hasattr(tgt, 'i_leafref_ptr') or tgt.i_leafref_ptr is None: + # not a leafref; + type_ = tgt.search_one('type') + if (type_ is None or + not isinstance(type_.i_type_spec, + pyang.types.InstanceIdentifierTypeSpec)): + pyang.error.err_add(ctx.errors, pos, 'XPATH_DEREF_TARGET', tgt) + # tgt = None + return (signature[1], None, [(args_signature[0], arg)]) + return (signature[1], tgt, [(args_signature[0], arg)]) + elif func == 'current': + return (signature[1], initial, []) + else: + inputs = [] + for arg in args: + obj = chk_xpath_expr(ctx, xpath_stmt, initial, node, arg, args_signature[i]) + inputs.append((args_signature[i], obj)) + if args_signature[i] == '*': + args_signature.append('*') + i = i + 1 + return (signature[1], None, inputs) + + +def chk_xpath_path(ctx, xpath_stmt, initial, node, path): + mod = xpath_stmt.i_orig_module + pos = xpath_stmt.pos + + if len(path) == 0: + return node + head = path[0] + if head == 'relative': + return chk_xpath_path(ctx, xpath_stmt, initial, node, path[1]) + if head[0] == 'var': + # check if the variable is known as a node-set + # currently we don't have any variables, so this fails + pyang.error.err_add(ctx.errors, pos, 'XPATH_VARIABLE', head[1]) + elif head[0] == 'function_call': + func = head[1] + args = head[2] + (rettype, tgt, inputs) = chk_xpath_function( + ctx, xpath_stmt, initial, node, func, args) + if rettype is not None: + # known function, check that it returns a node set + if rettype != 'node-set': + pyang.error.err_add(ctx.errors, pos, 'XPATH_FUNCTION_RET_VAL', + (func, 'node-set')) + if func == 'current': + return chk_xpath_path(ctx, xpath_stmt, initial, initial, path[1:]) + elif func == 'deref': + t = None + if tgt is not None: + (t, _pos) = tgt.i_leafref_ptr + return chk_xpath_path(ctx, xpath_stmt, initial, t, path[1:]) + elif head[0] == 'step': + axis = head[1] + nodetest = head[2] + preds = head[3] + node1 = None + if axis == 'self': + node1 = node + pass + elif nodetest[0] == 'name': + prefix = nodetest[1] + name = nodetest[2] + if prefix is None: + if initial is None: + pmodule = None + elif initial.keyword == 'module': + pmodule = initial + else: + pmodule = initial.i_module + else: + pmodule = pyang.util.prefix_to_module(mod, prefix, pos, ctx.errors) + # pmodule = prefix_to_module(mod, prefix, pos, ctx.errors) + # if node and initial are None, it means we're checking an XPath + # expression when it is defined in a grouping or augment, i.e., + # when the full tree is not expanded. in this case we can't check + # the paths + if pmodule is not None and node is not None and initial is not None: + if axis == 'child': + if node == 'root': + children = pmodule.i_children + else: + children = getattr(node, 'i_children', None) or [] + child = pyang.util.search_data_node( + children, pmodule.i_modulename, name) + if child is None and node == 'root': + pyang.error.err_add(ctx.errors, pos, 'XPATH_NODE_NOT_FOUND2', + (pmodule.i_modulename, name, pmodule.arg)) + elif child is None and node.i_module is not None: + pyang.error.err_add(ctx.errors, pos, 'XPATH_NODE_NOT_FOUND1', + (pmodule.i_modulename, name, + node.i_module.i_modulename, node.arg)) + elif child is None: + pyang.error.err_add(ctx.errors, pos, 'XPATH_NODE_NOT_FOUND2', + (pmodule.i_modulename, name, node.arg)) + elif (getattr(initial, 'i_config', None) is True + and getattr(child, 'i_config', None) is False): + pyang.error.err_add(ctx.errors, pos, 'XPATH_REF_CONFIG_FALSE', + (pmodule.i_modulename, name)) + else: + node1 = child + elif axis == 'ancestor' or axis == 'ancestor-or-self': + p = node + if axis == 'ancestor': + if node == 'root': + pyang.error.err_add(ctx.errors, pos, 'XPATH_ANCESTOR_NOT_FOUND', + (pmodule.i_modulename, name, + node.i_module.i_modulename, node.arg)) + else: + p = pyang.util.data_node_up(node) + while (p is not None and + not(p.arg == name and + p.i_module and + p.i_module.i_modulename == pmodule.i_modulename)): + p = pyang.util.data_node_up(p) + if p is None: + pyang.error.err_add( + ctx.errors, pos, 'XPATH_ANCESTOR_NOT_FOUND', ( + pmodule.i_modulename, name, + node.i_module.i_modulename, node.arg)) + else: + node1 = p + # we have now found one matching ancestor. + # NOTE: we don't handle multiple matching ancestors, + # so we check for this + p = pyang.util.data_node_up(p) + while (p is not None and + not(p.arg == name and + p.i_module and + p.i_module.i_modulename == + pmodule.i_modulename)): + p = pyang.util.data_node_up(p) + if p is not None: + # multiple ancestors; give a warning and continue + pyang.error.err_add( + ctx.errors, pos, 'XPATH_MULTIPLE_ANCESTORS', ( + node.i_module.i_modulename, node.arg, + pmodule.i_modulename, name)) + node1 = None + else: + # we can't validate the steps on other axis, but we can + # validate functions etc. + pass + elif axis == 'parent' and nodetest == ('node_type', 'node'): + if node is None: + pass + elif node == 'root': + pyang.error.err_add(ctx.errors, pos, 'XPATH_PATH_TOO_MANY_UP', ()) + else: + p = pyang.util.data_node_up(node) + if p is None: + pyang.error.err_add(ctx.errors, pos, 'XPATH_PATH_TOO_MANY_UP', ()) + else: + node1 = p + else: + # we can't validate the steps on other axis, but we can + # validate functions etc. + pass + for p in preds: + # pyang.xpath.chk_xpath_expr(ctx, mod, pos, initial, node1, p, None) + chk_xpath_expr(ctx, xpath_stmt, initial, node1, p, None) + return chk_xpath_path(ctx, xpath_stmt, initial, node1, path[1:])