# # Copyright 2021 Espressif Systems (Shanghai) CO LTD # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import collections import fnmatch import itertools from collections import namedtuple from entity import Entity from fragments import Mapping, Scheme, Sections from ldgen_common import LdGenFailure from output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress class Placement(): def __init__(self, node, sections, target, flags, explicit, force=False, dryrun=False): self.node = node self.sections = sections self.target = target self.flags = flags self.exclusions = set() self.subplacements = set() # Force this placement to be output self.force = force # This placement was created from a mapping # fragment entry. self.explicit = explicit # Find basis placement. A basis placement is a placement # on the parent (or parent's parent and so on and so forth) # that operates on the same section as this one. parent = node.parent candidate = None while parent: try: candidate = parent.placements[sections] except KeyError: pass if candidate and candidate.is_significant(): break else: parent = parent.parent self.basis = candidate if self.is_significant() and not dryrun and self.basis: self.basis.add_exclusion(self) def is_significant(self): # Check if the placement is significant. Significant placements # are the end of a basis chain (not self.basis) or a change # in target (self.target != self.basis.target) # # Placement can also be a basis if it has flags # (self.flags) or its basis has flags (self.basis.flags) return (not self.basis or self.target != self.basis.target or (self.flags and not self.basis.flags) or (not self.flags and self.basis.flags) or self.force) def force_significant(self): if not self.is_significant(): self.force = True if self.basis: self.basis.add_exclusion(self) def add_exclusion(self, exclusion): self.exclusions.add(exclusion) def add_subplacement(self, subplacement): self.subplacements.add(subplacement) class EntityNode(): def __init__(self, parent, name): self.children = [] self.parent = parent self.name = name self.child_t = EntityNode self.entity = None self.placements = dict() def add_child(self, entity): child_specificity = self.entity.specificity.value + 1 assert(child_specificity <= Entity.Specificity.SYMBOL.value) name = entity[Entity.Specificity(child_specificity)] assert(name and name != Entity.ALL) child = [c for c in self.children if c.name == name] assert(len(child) <= 1) if not child: child = self.child_t(self, name) self.children.append(child) else: child = child[0] return child def get_output_commands(self): commands = collections.defaultdict(list) def process_commands(cmds): for (target, commands_list) in cmds.items(): commands[target].extend(commands_list) # Process the commands generated from this node node_commands = self.get_node_output_commands() process_commands(node_commands) # Process the commands generated from this node's children # recursively for child in sorted(self.children, key=lambda c: c.name): children_commands = child.get_output_commands() process_commands(children_commands) return commands def get_node_output_commands(self): commands = collections.defaultdict(list) for sections in self.get_output_sections(): placement = self.placements[sections] if placement.is_significant(): assert(placement.node == self) keep = False sort = None surround_type = [] placement_flags = placement.flags if placement.flags is not None else [] for flag in placement_flags: if isinstance(flag, Mapping.Keep): keep = True elif isinstance(flag, Mapping.Sort): sort = (flag.first, flag.second) else: # SURROUND or ALIGN surround_type.append(flag) for flag in surround_type: if flag.pre: if isinstance(flag, Mapping.Surround): commands[placement.target].append(SymbolAtAddress('_%s_start' % flag.symbol)) else: # ALIGN commands[placement.target].append(AlignAtAddress(flag.alignment)) # This is for expanded object node and symbol node placements without checking for # the type. placement_sections = frozenset(placement.sections) command_sections = sections if sections == placement_sections else placement_sections command = InputSectionDesc(placement.node.entity, command_sections, [e.node.entity for e in placement.exclusions], keep, sort) commands[placement.target].append(command) # Generate commands for intermediate, non-explicit exclusion placements here, so that they can be enclosed by # flags that affect the parent placement. for subplacement in placement.subplacements: if not subplacement.flags and not subplacement.explicit: command = InputSectionDesc(subplacement.node.entity, subplacement.sections, [e.node.entity for e in subplacement.exclusions], keep, sort) commands[placement.target].append(command) for flag in surround_type: if flag.post: if isinstance(flag, Mapping.Surround): commands[placement.target].append(SymbolAtAddress('_%s_end' % flag.symbol)) else: # ALIGN commands[placement.target].append(AlignAtAddress(flag.alignment)) return commands def self_placement(self, sections, target, flags, explicit=True, force=False): placement = Placement(self, sections, target, flags, explicit, force) self.placements[sections] = placement return placement def child_placement(self, entity, sections, target, flags, sections_db): child = self.add_child(entity) child.insert(entity, sections, target, flags, sections_db) def insert(self, entity, sections, target, flags, sections_db): if self.entity.specificity == entity.specificity: # Since specificities match, create the placement in this node. self.self_placement(sections, target, flags) else: # If not, create a child node and try to create the placement there. self.child_placement(entity, sections, target, flags, sections_db) def get_output_sections(self): return sorted(self.placements.keys(), key=' '.join) class SymbolNode(EntityNode): def __init__(self, parent, name): EntityNode.__init__(self, parent, name) self.entity = Entity(self.parent.parent.name, self.parent.name) class ObjectNode(EntityNode): def __init__(self, parent, name): EntityNode.__init__(self, parent, name) self.child_t = SymbolNode self.entity = Entity(self.parent.name, self.name) self.subplacements = list() def child_placement(self, entity, sections, target, flags, sections_db): child = self.add_child(entity) sym_placement = Placement(child, sections, target, flags, True, dryrun=True) # The basis placement for sym_placement can either be # an existing placement on this node, or nonexistent. if sym_placement.is_significant(): try: obj_sections = self.placements[sections].sections except KeyError: obj_sections = None if not obj_sections or obj_sections == sections: # Expand this section for the first time found_sections = sections_db.get_sections(self.parent.name, self.name) obj_sections = [] for s in sections: obj_sections.extend(fnmatch.filter(found_sections, s)) if obj_sections: symbol = entity.symbol remove_sections = [s.replace('.*', '.%s' % symbol) for s in sections if '.*' in s] filtered_sections = [s for s in obj_sections if s not in remove_sections] if set(filtered_sections) != set(obj_sections): if sym_placement.basis: subplace = False try: # If existing placement exists, make sure that # it is emitted. obj_placement = self.placements[sections] except KeyError: # Create intermediate placement. obj_placement = self.self_placement(sections, sym_placement.basis.target, None, False) if obj_placement.basis.flags: subplace = True if subplace: obj_placement.basis.add_subplacement(obj_placement) self.subplacements.append(sections) else: obj_placement.force_significant() obj_placement.sections = filtered_sections sym_placement.basis = obj_placement sym_placement.sections = remove_sections child.placements[sections] = sym_placement def get_output_sections(self): output_sections = [key for key in self.placements if key not in self.subplacements] return sorted(output_sections, key=' '.join) class ArchiveNode(EntityNode): def __init__(self, parent, name): EntityNode.__init__(self, parent, name) self.child_t = ObjectNode self.entity = Entity(self.name) class RootNode(EntityNode): def __init__(self): EntityNode.__init__(self, None, Entity.ALL) self.child_t = ArchiveNode self.entity = Entity('*') class Generation: """ Implements generation of placement based on collected sections, scheme and mapping fragment. """ # Processed mapping, scheme and section entries EntityMapping = namedtuple('EntityMapping', 'entity sections_group target flags') def __init__(self, check_mappings=False, check_mapping_exceptions=None): self.schemes = {} self.placements = {} self.mappings = {} self.check_mappings = check_mappings if check_mapping_exceptions: self.check_mapping_exceptions = check_mapping_exceptions else: self.check_mapping_exceptions = [] def _prepare_scheme_dictionary(self): scheme_dictionary = collections.defaultdict(dict) # Collect sections into buckets based on target name for scheme in self.schemes.values(): sections_bucket = collections.defaultdict(list) for (sections_name, target_name) in scheme.entries: # Get the sections under the bucket 'target_name'. If this bucket does not exist # is is created automatically sections_in_bucket = sections_bucket[target_name] try: sections = self.placements[sections_name] except KeyError: message = GenerationException.UNDEFINED_REFERENCE + " to sections '" + sections_name + "'." raise GenerationException(message, scheme) sections_in_bucket.append(sections) scheme_dictionary[scheme.name] = sections_bucket # Search for and raise exception on first instance of sections mapped to multiple targets for (scheme_name, sections_bucket) in scheme_dictionary.items(): for sections_a, sections_b in itertools.combinations(sections_bucket.values(), 2): set_a = set() set_b = set() for sections in sections_a: set_a.update(sections.entries) for sections in sections_b: set_b.update(sections.entries) intersection = set_a.intersection(set_b) # If the intersection is a non-empty set, it means sections are mapped to multiple # targets. Raise exception. if intersection: scheme = self.schemes[scheme_name] message = 'Sections ' + str(intersection) + ' mapped to multiple targets.' raise GenerationException(message, scheme) return scheme_dictionary def _prepare_entity_mappings(self, scheme_dictionary, entities): # Prepare entity mappings processed from mapping fragment entries. def get_section_strs(section): s_list = [Sections.get_section_data_from_entry(s) for s in section.entries] return frozenset([item for sublist in s_list for item in sublist]) entity_mappings = dict() for mapping in self.mappings.values(): archive = mapping.archive for (obj, symbol, scheme_name) in mapping.entries: entity = Entity(archive, obj, symbol) # Check the entity exists if (self.check_mappings and entity.specificity.value > Entity.Specificity.ARCHIVE.value and mapping.name not in self.check_mapping_exceptions): if not entities.check_exists(entity): message = "'%s' not found" % str(entity) raise GenerationException(message, mapping) if (obj, symbol, scheme_name) in mapping.flags.keys(): flags = mapping.flags[(obj, symbol, scheme_name)] # Check if all section->target defined in the current # scheme. for (s, t, f) in flags: if (t not in scheme_dictionary[scheme_name].keys() or s not in [_s.name for _s in scheme_dictionary[scheme_name][t]]): message = "%s->%s not defined in scheme '%s'" % (s, t, scheme_name) raise GenerationException(message, mapping) else: flags = None # Create placement for each 'section -> target' in the scheme. for (target, sections) in scheme_dictionary[scheme_name].items(): for section in sections: # Find the applicable flags _flags = [] if flags: for (s, t, f) in flags: if (s, t) == (section.name, target): _flags.extend(f) sections_str = get_section_strs(section) key = (entity, section.name) try: existing = entity_mappings[key] except KeyError: existing = None if not existing: entity_mappings[key] = Generation.EntityMapping(entity, sections_str, target, _flags) else: # Check for conflicts. if (target != existing.target): raise GenerationException('Sections mapped to multiple targets.', mapping) # Combine flags here if applicable, to simplify # insertion logic. if (_flags or existing.flags): if ((_flags and not existing.flags) or (not _flags and existing.flags)): _flags.extend(existing.flags) entity_mappings[key] = Generation.EntityMapping(entity, sections_str, target, _flags) elif (_flags == existing.flags): pass else: raise GenerationException('Conflicting flags specified.', mapping) # Sort the mappings by specificity, so as to simplify # insertion logic. res = list(entity_mappings.values()) res.sort(key=lambda m: m.entity) return res def generate(self, entities): scheme_dictionary = self._prepare_scheme_dictionary() entity_mappings = self._prepare_entity_mappings(scheme_dictionary, entities) root_node = RootNode() for mapping in entity_mappings: (entity, sections, target, flags) = mapping try: root_node.insert(entity, sections, target, flags, entities) except ValueError as e: raise GenerationException(str(e)) # Traverse the tree, creating the placements commands = root_node.get_output_commands() return commands def add_fragments_from_file(self, fragment_file): for fragment in fragment_file.fragments: dict_to_append_to = None if isinstance(fragment, Mapping) and fragment.deprecated and fragment.name in self.mappings.keys(): self.mappings[fragment.name].entries |= fragment.entries else: if isinstance(fragment, Scheme): dict_to_append_to = self.schemes elif isinstance(fragment, Sections): dict_to_append_to = self.placements else: dict_to_append_to = self.mappings # Raise exception when the fragment of the same type is already in the stored fragments if fragment.name in dict_to_append_to.keys(): stored = dict_to_append_to[fragment.name].path new = fragment.path message = "Duplicate definition of fragment '%s' found in %s and %s." % (fragment.name, stored, new) raise GenerationException(message) dict_to_append_to[fragment.name] = fragment class GenerationException(LdGenFailure): """ Exception for linker script generation failures such as undefined references/ failure to evaluate conditions, duplicate mappings, etc. """ UNDEFINED_REFERENCE = 'Undefined reference' def __init__(self, message, fragment=None): self.fragment = fragment self.message = message def __str__(self): if self.fragment: return "%s\nIn fragment '%s' defined in '%s'." % (self.message, self.fragment.name, self.fragment.path) else: return self.message