import email.feedparser import email.header import email.message import email.parser import email.policy import sys import typing from typing import Dict, List, Optional, Tuple, Union, cast if sys.version_info >= (3, 8): # pragma: no cover from typing import TypedDict else: # pragma: no cover if typing.TYPE_CHECKING: from typing_extensions import TypedDict else: try: from typing_extensions import TypedDict except ImportError: class TypedDict: def __init_subclass__(*_args, **_kwargs): pass # The RawMetadata class attempts to make as few assumptions about the underlying # serialization formats as possible. The idea is that as long as a serialization # formats offer some very basic primitives in *some* way then we can support # serializing to and from that format. class RawMetadata(TypedDict, total=False): """A dictionary of raw core metadata. Each field in core metadata maps to a key of this dictionary (when data is provided). The key is lower-case and underscores are used instead of dashes compared to the equivalent core metadata field. Any core metadata field that can be specified multiple times or can hold multiple values in a single field have a key with a plural name. Core metadata fields that can be specified multiple times are stored as a list or dict depending on which is appropriate for the field. Any fields which hold multiple values in a single field are stored as a list. """ # Metadata 1.0 - PEP 241 metadata_version: str name: str version: str platforms: List[str] summary: str description: str keywords: List[str] home_page: str author: str author_email: str license: str # Metadata 1.1 - PEP 314 supported_platforms: List[str] download_url: str classifiers: List[str] requires: List[str] provides: List[str] obsoletes: List[str] # Metadata 1.2 - PEP 345 maintainer: str maintainer_email: str requires_dist: List[str] provides_dist: List[str] obsoletes_dist: List[str] requires_python: str requires_external: List[str] project_urls: Dict[str, str] # Metadata 2.0 # PEP 426 attempted to completely revamp the metadata format # but got stuck without ever being able to build consensus on # it and ultimately ended up withdrawn. # # However, a number of tools had started emiting METADATA with # `2.0` Metadata-Version, so for historical reasons, this version # was skipped. # Metadata 2.1 - PEP 566 description_content_type: str provides_extra: List[str] # Metadata 2.2 - PEP 643 dynamic: List[str] # Metadata 2.3 - PEP 685 # No new fields were added in PEP 685, just some edge case were # tightened up to provide better interoptability. _STRING_FIELDS = { "author", "author_email", "description", "description_content_type", "download_url", "home_page", "license", "maintainer", "maintainer_email", "metadata_version", "name", "requires_python", "summary", "version", } _LIST_STRING_FIELDS = { "classifiers", "dynamic", "obsoletes", "obsoletes_dist", "platforms", "provides", "provides_dist", "provides_extra", "requires", "requires_dist", "requires_external", "supported_platforms", } def _parse_keywords(data: str) -> List[str]: """Split a string of comma-separate keyboards into a list of keywords.""" return [k.strip() for k in data.split(",")] def _parse_project_urls(data: List[str]) -> Dict[str, str]: """Parse a list of label/URL string pairings separated by a comma.""" urls = {} for pair in data: # Our logic is slightly tricky here as we want to try and do # *something* reasonable with malformed data. # # The main thing that we have to worry about, is data that does # not have a ',' at all to split the label from the Value. There # isn't a singular right answer here, and we will fail validation # later on (if the caller is validating) so it doesn't *really* # matter, but since the missing value has to be an empty str # and our return value is dict[str, str], if we let the key # be the missing value, then they'd have multiple '' values that # overwrite each other in a accumulating dict. # # The other potentional issue is that it's possible to have the # same label multiple times in the metadata, with no solid "right" # answer with what to do in that case. As such, we'll do the only # thing we can, which is treat the field as unparseable and add it # to our list of unparsed fields. parts = [p.strip() for p in pair.split(",", 1)] parts.extend([""] * (max(0, 2 - len(parts)))) # Ensure 2 items # TODO: The spec doesn't say anything about if the keys should be # considered case sensitive or not... logically they should # be case-preserving and case-insensitive, but doing that # would open up more cases where we might have duplicate # entries. label, url = parts if label in urls: # The label already exists in our set of urls, so this field # is unparseable, and we can just add the whole thing to our # unparseable data and stop processing it. raise KeyError("duplicate labels in project urls") urls[label] = url return urls def _get_payload(msg: email.message.Message, source: Union[bytes, str]) -> str: """Get the body of the message.""" # If our source is a str, then our caller has managed encodings for us, # and we don't need to deal with it. if isinstance(source, str): payload: str = msg.get_payload() return payload # If our source is a bytes, then we're managing the encoding and we need # to deal with it. else: bpayload: bytes = msg.get_payload(decode=True) try: return bpayload.decode("utf8", "strict") except UnicodeDecodeError: raise ValueError("payload in an invalid encoding") # The various parse_FORMAT functions here are intended to be as lenient as # possible in their parsing, while still returning a correctly typed # RawMetadata. # # To aid in this, we also generally want to do as little touching of the # data as possible, except where there are possibly some historic holdovers # that make valid data awkward to work with. # # While this is a lower level, intermediate format than our ``Metadata`` # class, some light touch ups can make a massive difference in usability. # Map METADATA fields to RawMetadata. _EMAIL_TO_RAW_MAPPING = { "author": "author", "author-email": "author_email", "classifier": "classifiers", "description": "description", "description-content-type": "description_content_type", "download-url": "download_url", "dynamic": "dynamic", "home-page": "home_page", "keywords": "keywords", "license": "license", "maintainer": "maintainer", "maintainer-email": "maintainer_email", "metadata-version": "metadata_version", "name": "name", "obsoletes": "obsoletes", "obsoletes-dist": "obsoletes_dist", "platform": "platforms", "project-url": "project_urls", "provides": "provides", "provides-dist": "provides_dist", "provides-extra": "provides_extra", "requires": "requires", "requires-dist": "requires_dist", "requires-external": "requires_external", "requires-python": "requires_python", "summary": "summary", "supported-platform": "supported_platforms", "version": "version", } def parse_email(data: Union[bytes, str]) -> Tuple[RawMetadata, Dict[str, List[str]]]: """Parse a distribution's metadata. This function returns a two-item tuple of dicts. The first dict is of recognized fields from the core metadata specification. Fields that can be parsed and translated into Python's built-in types are converted appropriately. All other fields are left as-is. Fields that are allowed to appear multiple times are stored as lists. The second dict contains all other fields from the metadata. This includes any unrecognized fields. It also includes any fields which are expected to be parsed into a built-in type but were not formatted appropriately. Finally, any fields that are expected to appear only once but are repeated are included in this dict. """ raw: Dict[str, Union[str, List[str], Dict[str, str]]] = {} unparsed: Dict[str, List[str]] = {} if isinstance(data, str): parsed = email.parser.Parser(policy=email.policy.compat32).parsestr(data) else: parsed = email.parser.BytesParser(policy=email.policy.compat32).parsebytes(data) # We have to wrap parsed.keys() in a set, because in the case of multiple # values for a key (a list), the key will appear multiple times in the # list of keys, but we're avoiding that by using get_all(). for name in frozenset(parsed.keys()): # Header names in RFC are case insensitive, so we'll normalize to all # lower case to make comparisons easier. name = name.lower() # We use get_all() here, even for fields that aren't multiple use, # because otherwise someone could have e.g. two Name fields, and we # would just silently ignore it rather than doing something about it. headers = parsed.get_all(name) # The way the email module works when parsing bytes is that it # unconditionally decodes the bytes as ascii using the surrogateescape # handler. When you pull that data back out (such as with get_all() ), # it looks to see if the str has any surrogate escapes, and if it does # it wraps it in a Header object instead of returning the string. # # As such, we'll look for those Header objects, and fix up the encoding. value = [] # Flag if we have run into any issues processing the headers, thus # signalling that the data belongs in 'unparsed'. valid_encoding = True for h in headers: # It's unclear if this can return more types than just a Header or # a str, so we'll just assert here to make sure. assert isinstance(h, (email.header.Header, str)) # If it's a header object, we need to do our little dance to get # the real data out of it. In cases where there is invalid data # we're going to end up with mojibake, but there's no obvious, good # way around that without reimplementing parts of the Header object # ourselves. # # That should be fine since, if mojibacked happens, this key is # going into the unparsed dict anyways. if isinstance(h, email.header.Header): # The Header object stores it's data as chunks, and each chunk # can be independently encoded, so we'll need to check each # of them. chunks: List[Tuple[bytes, Optional[str]]] = [] for bin, encoding in email.header.decode_header(h): try: bin.decode("utf8", "strict") except UnicodeDecodeError: # Enable mojibake. encoding = "latin1" valid_encoding = False else: encoding = "utf8" chunks.append((bin, encoding)) # Turn our chunks back into a Header object, then let that # Header object do the right thing to turn them into a # string for us. value.append(str(email.header.make_header(chunks))) # This is already a string, so just add it. else: value.append(h) # We've processed all of our values to get them into a list of str, # but we may have mojibake data, in which case this is an unparsed # field. if not valid_encoding: unparsed[name] = value continue raw_name = _EMAIL_TO_RAW_MAPPING.get(name) if raw_name is None: # This is a bit of a weird situation, we've encountered a key that # we don't know what it means, so we don't know whether it's meant # to be a list or not. # # Since we can't really tell one way or another, we'll just leave it # as a list, even though it may be a single item list, because that's # what makes the most sense for email headers. unparsed[name] = value continue # If this is one of our string fields, then we'll check to see if our # value is a list of a single item. If it is then we'll assume that # it was emitted as a single string, and unwrap the str from inside # the list. # # If it's any other kind of data, then we haven't the faintest clue # what we should parse it as, and we have to just add it to our list # of unparsed stuff. if raw_name in _STRING_FIELDS and len(value) == 1: raw[raw_name] = value[0] # If this is one of our list of string fields, then we can just assign # the value, since email *only* has strings, and our get_all() call # above ensures that this is a list. elif raw_name in _LIST_STRING_FIELDS: raw[raw_name] = value # Special Case: Keywords # The keywords field is implemented in the metadata spec as a str, # but it conceptually is a list of strings, and is serialized using # ", ".join(keywords), so we'll do some light data massaging to turn # this into what it logically is. elif raw_name == "keywords" and len(value) == 1: raw[raw_name] = _parse_keywords(value[0]) # Special Case: Project-URL # The project urls is implemented in the metadata spec as a list of # specially-formatted strings that represent a key and a value, which # is fundamentally a mapping, however the email format doesn't support # mappings in a sane way, so it was crammed into a list of strings # instead. # # We will do a little light data massaging to turn this into a map as # it logically should be. elif raw_name == "project_urls": try: raw[raw_name] = _parse_project_urls(value) except KeyError: unparsed[name] = value # Nothing that we've done has managed to parse this, so it'll just # throw it in our unparseable data and move on. else: unparsed[name] = value # We need to support getting the Description from the message payload in # addition to getting it from the the headers. This does mean, though, there # is the possibility of it being set both ways, in which case we put both # in 'unparsed' since we don't know which is right. try: payload = _get_payload(parsed, data) except ValueError: unparsed.setdefault("description", []).append( parsed.get_payload(decode=isinstance(data, bytes)) ) else: if payload: # Check to see if we've already got a description, if so then both # it, and this body move to unparseable. if "description" in raw: description_header = cast(str, raw.pop("description")) unparsed.setdefault("description", []).extend( [description_header, payload] ) elif "description" in unparsed: unparsed["description"].append(payload) else: raw["description"] = payload # We need to cast our `raw` to a metadata, because a TypedDict only support # literal key names, but we're computing our key names on purpose, but the # way this function is implemented, our `TypedDict` can only have valid key # names. return cast(RawMetadata, raw), unparsed