# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
"""Defines layers for containing data.

One layer may combine other layers, map data based on the data itself,
or map a procedure (such as decryption) across another layer of data.
"""
import collections.abc
import functools
import logging
import math
import multiprocessing
import multiprocessing.managers
import threading
import traceback
import types
from abc import ABCMeta, abstractmethod
from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple, Union

from volatility3.framework import constants, exceptions, interfaces

vollog = logging.getLogger(__name__)

ProgressValue = Union["DummyProgress", multiprocessing.managers.ValueProxy]
IteratorValue = Tuple[List[Tuple[str, int, int]], int]


class ScannerInterface(
    interfaces.configuration.VersionableInterface, metaclass=ABCMeta
):
    """Class for layer scanners that return locations of particular values from
    within the data.

    These are designed to be given a chunk of data and return a generator which yields
    any found items.  They should NOT perform complex/time-consuming tasks, these should
    be carried out by the consumer of the generator on the items returned.

    They will be provided all *available* data (therefore not necessarily contiguous)
    in ascending offset order, in chunks no larger than chunk_size + overlap where
    overlap is the amount of data read twice once at the end of an earlier chunk and
    once at the start of the next chunk.

    It should be noted that the scanner can maintain state if necessary.
    Scanners should balance the size of chunk based on the amount of time
    scanning the chunk will take (ie, do not set an excessively large chunksize
    and try not to take a significant amount of time in the __call__ method).

    Scanners must NOT return results found *after* self.chunk_size (ie, entirely contained
    within the overlap).  It is the responsibility of the scanner not to return such
    duplicate results.

    Scanners can mark themselves as thread_safe, if they do not require state
    in either their own class or the context.  This will allow the scanner to be run
    in parallel against multiple blocks.
    """

    thread_safe = False

    _required_framework_version = (2, 0, 0)

    def __init__(self) -> None:
        super().__init__()
        self.chunk_size = 0x1000000  # Default to 16Mb chunks
        self.overlap = 0x1000  # A page of overlap by default
        self._context: Optional[interfaces.context.ContextInterface] = None
        self._layer_name: Optional[str] = None

    @property
    def context(self) -> Optional["interfaces.context.ContextInterface"]:
        return self._context

    @context.setter
    def context(self, ctx: "interfaces.context.ContextInterface") -> None:
        """Stores the context locally in case the scanner needs to access the
        layer."""
        self._context = ctx

    @property
    def layer_name(self) -> Optional[str]:
        return self._layer_name

    @layer_name.setter
    def layer_name(self, layer_name: str) -> None:
        """Stores the layer_name being scanned locally in case the scanner
        needs to access the layer."""
        self._layer_name = layer_name

    @abstractmethod
    def __call__(self, data: bytes, data_offset: int) -> Iterable[Any]:
        """Searches through a chunk of data for a particular value/pattern/etc
        Always returns an iterator of the same type of object (need not be a
        volatility object)

        data is the chunk of data to search through data_offset is the
        offset within the layer that the data being searched starts at
        """


class DataLayerInterface(
    interfaces.configuration.ConfigurableInterface, metaclass=ABCMeta
):
    """A Layer that directly holds data (and does not translate it).

    This is effectively a leaf node in a layer tree. It directly
    accesses a data source and exposes it within volatility.
    """

    _direct_metadata: Mapping = {"architecture": "Unknown", "os": "Unknown"}

    def __init__(
        self,
        context: "interfaces.context.ContextInterface",
        config_path: str,
        name: str,
        metadata: Optional[Dict[str, Any]] = None,
    ) -> None:
        super().__init__(context, config_path)
        self._name = name
        self._metadata = metadata or {}

    # Standard attributes

    @property
    def name(self) -> str:
        """Returns the layer name."""
        return self._name

    @property
    @abstractmethod
    def maximum_address(self) -> int:
        """Returns the maximum valid address of the space."""

    @property
    @abstractmethod
    def minimum_address(self) -> int:
        """Returns the minimum valid address of the space."""

    @property
    def address_mask(self) -> int:
        """Returns a mask which encapsulates all the active bits of an address
        for this layer."""
        return (1 << int(math.ceil(math.log2(self.maximum_address)))) - 1

    @abstractmethod
    def is_valid(self, offset: int, length: int = 1) -> bool:
        """Returns a boolean based on whether the entire chunk of data (from
        offset to length) is valid or not.

        Args:
            offset: The address to start determining whether bytes are readable/valid
            length: The number of bytes from offset of which to test the validity

        Returns:
             Whether the bytes are valid and accessible
        """

    @abstractmethod
    def read(self, offset: int, length: int, pad: bool = False) -> bytes:
        """Reads an offset for length bytes and returns 'bytes' (not 'str') of
        length size.

        If there is a fault of any kind (such as a page fault), an exception will be thrown
        unless pad is set, in which case the read errors will be replaced by null characters.

        Args:
            offset: The offset at which to being reading within the layer
            length: The number of bytes to read within the layer
            pad: A boolean indicating whether exceptions should be raised or bad bytes replaced with null characters

        Returns:
            The bytes read from the layer, starting at offset for length bytes
        """

    @abstractmethod
    def write(self, offset: int, data: bytes) -> None:
        """Writes a chunk of data at offset.

        Any unavailable sections in the underlying bases will cause an exception to be thrown.
        Note: Writes are not guaranteed atomic, therefore some data may have been written, even if an exception is thrown.
        """

    def destroy(self) -> None:
        """Causes a DataLayer to close any open handles, etc.

        Systems that make use of Data Layers should call destroy when
        they are done with them. This will close all handles, and make
        the object unreadable (exceptions will be thrown using a
        DataLayer after destruction)
        """
        pass

    @classmethod
    def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
        """Returns a list of Requirement objects for this type of layer."""
        return super().get_requirements()

    @property
    def dependencies(self) -> List[str]:
        """A list of other layer names required by this layer.

        Note:
            DataLayers must never define other layers
        """
        return []

    # ## General scanning methods

    def scan(
        self,
        context: interfaces.context.ContextInterface,
        scanner: ScannerInterface,
        progress_callback: constants.ProgressCallback = None,
        sections: Iterable[Tuple[int, int]] = None,
    ) -> Iterable[Any]:
        """Scans a Translation layer by chunk.

        Note: this will skip missing/unmappable chunks of memory

        Args:
             context: The context containing the data layer
             scanner: The constructed Scanner object to be applied
             progress_callback: Method that is called periodically during scanning to update progress
             sections: A list of (start, size) tuples defining the portions of the layer to scan

        Returns:
             The output iterable from the scanner object having been run against the layer
        """
        if progress_callback is not None and not callable(progress_callback):
            raise TypeError("Progress_callback is not callable")

        scanner.context = context
        scanner.layer_name = self.name

        if sections is None:
            sections = [
                (self.minimum_address, self.maximum_address - self.minimum_address)
            ]

        sections = list(self._coalesce_sections(sections))

        try:
            progress: ProgressValue = DummyProgress()
            scan_iterator = functools.partial(self._scan_iterator, scanner, sections)
            scan_metric = self._scan_metric(scanner, sections)
            if (
                not scanner.thread_safe
                or constants.PARALLELISM == constants.Parallelism.Off
            ):
                progress = DummyProgress()
                scan_chunk = functools.partial(self._scan_chunk, scanner, progress)
                for value in scan_iterator():
                    if progress_callback:
                        progress_callback(
                            scan_metric(progress.value),
                            f"Scanning {self.name} using {scanner.__class__.__name__}",
                        )
                    yield from scan_chunk(value)
            else:
                progress = multiprocessing.Manager().Value("Q", 0)
                parallel_module: types.ModuleType = multiprocessing
                if constants.PARALLELISM == constants.Parallelism.Threading:
                    progress = DummyProgress()
                    parallel_module = threading
                scan_chunk = functools.partial(self._scan_chunk, scanner, progress)
                with parallel_module.Pool() as pool:
                    result = pool.map_async(scan_chunk, scan_iterator())
                    while not result.ready():
                        if progress_callback:
                            # Run the progress_callback
                            progress_callback(
                                scan_metric(progress.value),
                                f"Scanning {self.name} using {scanner.__class__.__name__}",
                            )
                        # Ensures we don't burn CPU cycles going round in a ready waiting loop
                        # without delaying the user too long between progress updates/results
                        result.wait(0.1)
                    for result_value in result.get():
                        yield from result_value
        except Exception as e:
            # We don't care the kind of exception, so catch and report on everything, yielding nothing further
            vollog.debug(f"Scan Failure: {str(e)}")
            vollog.log(
                constants.LOGLEVEL_VVV,
                "\n".join(
                    traceback.TracebackException.from_exception(e).format(chain=True)
                ),
            )

    def _coalesce_sections(
        self, sections: Iterable[Tuple[int, int]]
    ) -> Iterable[Tuple[int, int]]:
        """Take a list of (start, length) sections and coalesce any adjacent
        sections."""
        result: List[Tuple[int, int]] = []
        position = 0
        for start, length in sorted(sections):
            if result and start <= position:
                initial_start, _ = result.pop()
                result.append((initial_start, (start + length) - initial_start))
            else:
                result.append((start, length))
            position = start + length

        while result and result[0] < (self.minimum_address, 0):
            first_start, first_length = result[0]
            if first_start + first_length < self.minimum_address:
                result = result[1:]
            elif first_start < self.minimum_address:
                result[0] = (
                    self.minimum_address,
                    (first_start + first_length) - self.minimum_address,
                )
        while result and result[-1] > (self.maximum_address, 0):
            last_start, last_length = result[-1]
            if last_start > self.maximum_address:
                result.pop()
            elif last_start + last_length > self.maximum_address:
                result[1] = (last_start, self.maximum_address - last_start)
        return result

    def _scan_iterator(
        self, scanner: "ScannerInterface", sections: Iterable[Tuple[int, int]]
    ) -> Iterable[IteratorValue]:
        """Iterator that indicates which blocks in the layer are to be read by
        for the scanning.

        Returns a list of blocks (potentially in lower layers) that make
        up this chunk contiguously. Chunks can be no bigger than
        scanner.chunk_size + scanner.overlap DataLayers by default are
        assumed to have no holes
        """
        for section_start, section_length in sections:
            offset, mapped_offset, length, layer_name = (
                section_start,
                section_start,
                section_length,
                self.name,
            )
            while length > 0:
                chunk_size = min(length, scanner.chunk_size + scanner.overlap)
                yield [(layer_name, mapped_offset, chunk_size)], offset + chunk_size
                # If we've got more than the scanner's chunk_size, only move up by the chunk_size
                if chunk_size > scanner.chunk_size:
                    chunk_size -= scanner.overlap
                length -= chunk_size
                mapped_offset += chunk_size
                offset += chunk_size

    # We ignore the type due to the iterator_value, actually it only needs to match the output from _scan_iterator
    def _scan_chunk(
        self,
        scanner: "ScannerInterface",
        progress: "ProgressValue",
        iterator_value: IteratorValue,
    ) -> List[Any]:
        data_to_scan, chunk_end = iterator_value
        data = b""
        for layer_name, address, chunk_size in data_to_scan:
            try:
                data += self.context.layers[layer_name].read(address, chunk_size)
            except exceptions.InvalidAddressException:
                vollog.debug(
                    "Invalid address in layer {} found scanning {} at address {:x}".format(
                        layer_name, self.name, address
                    )
                )

        if len(data) > scanner.chunk_size + scanner.overlap:
            vollog.debug(f"Scan chunk too large: {hex(len(data))}")

        progress.value = chunk_end
        return list(scanner(data, chunk_end - len(data)))

    def _scan_metric(
        self, _scanner: "ScannerInterface", sections: List[Tuple[int, int]]
    ) -> Callable[[int], float]:
        if not sections:
            raise ValueError("Sections have no size, nothing to scan")
        last_section, last_length = sections[-1]
        min_address, _ = sections[0]
        max_address = last_section + last_length

        def _actual_scan_metric(value: int) -> float:
            return max(0, ((value - min_address) * 100) / (max_address - min_address))

        return _actual_scan_metric

    def build_configuration(self) -> interfaces.configuration.HierarchicalDict:
        config = super().build_configuration()

        # Translation Layers are constructable, and therefore require a class configuration variable
        config["class"] = self.__class__.__module__ + "." + self.__class__.__name__
        return config

    # ## Metadata methods

    @property
    def metadata(self) -> Mapping:
        """Returns a ReadOnly copy of the metadata published by this layer."""
        maps = [
            self.context.layers[layer_name].metadata for layer_name in self.dependencies
        ]
        return interfaces.objects.ReadOnlyMapping(
            collections.ChainMap(self._metadata, self._direct_metadata, *maps)
        )


class TranslationLayerInterface(DataLayerInterface, metaclass=ABCMeta):
    """Provides a layer that translates or transforms another layer or layers.

    Translation layers always depend on another layer (typically
    translating offsets in a virtual offset space into a smaller
    physical offset space).
    """

    @abstractmethod
    def mapping(
        self, offset: int, length: int, ignore_errors: bool = False
    ) -> Iterable[Tuple[int, int, int, int, str]]:
        """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer)
        mappings.

        ignore_errors will provide all available maps with gaps, but
        their total length may not add up to the requested length This
        allows translation layers to provide maps of contiguous regions
        in one layer
        """
        return []

    @property
    @abstractmethod
    def dependencies(self) -> List[str]:
        """Returns a list of layer names that this layer translates onto."""
        return []

    def _decode_data(
        self, data: bytes, mapped_offset: int, offset: int, output_length: int
    ) -> bytes:
        """Decodes any necessary data.  Note, additional data may need to be read from the lower layer, such as lookup
        tables or similar.  The data provided to this layer is purely that data which encompasses the requested data
        range.

        Args:
            data: The bytes of data necessary for decoding
            mapped_offset: The offset in the underlying layer where the data would begin
            offset: The offset in the higher-layer where the data would begin
            output_length: The expected length of the returned data

        Returns:
             The data to be read from the underlying layer."""
        return data

    def _encode_data(
        self, layer_name: str, mapped_offset: int, offset: int, value: bytes
    ) -> bytes:
        """Encodes any necessary data.

        Args:
            layer_name: The layer to write data back to
            mapped_offset: The offset in the underlying layer where the data would begin
            offset: The offset in the higher-layer where the data would begin
            value: The new value to encode

        Returns:
             The data to be rewritten at mapped_offset."""
        return value

    # ## Read/Write functions for mapped pages

    @functools.lru_cache(maxsize=512)
    def read(self, offset: int, length: int, pad: bool = False) -> bytes:
        """Reads an offset for length bytes and returns 'bytes' (not 'str') of
        length size."""
        current_offset = offset
        output: bytes = b""
        for (
            layer_offset,
            sublength,
            mapped_offset,
            mapped_length,
            layer,
        ) in self.mapping(offset, length, ignore_errors=pad):
            if not pad and layer_offset > current_offset:
                raise exceptions.InvalidAddressException(
                    self.name,
                    current_offset,
                    f"Layer {self.name} cannot map offset: {current_offset}",
                )
            elif layer_offset > current_offset:
                output += b"\x00" * (layer_offset - current_offset)
                current_offset = layer_offset
            # The layer_offset can be less than the current_offset in non-linearly mapped layers
            # it does not suggest an overlap, but that the data is in an encoded block
            if mapped_length > 0:
                unprocessed_data = self._context.layers.read(
                    layer, mapped_offset, mapped_length, pad
                )
                processed_data = self._decode_data(
                    unprocessed_data, mapped_offset, layer_offset, sublength
                )
                if len(processed_data) != sublength:
                    raise ValueError(
                        "ProcessedData length does not match expected length of chunk"
                    )
                output += processed_data
                current_offset += sublength
        return output + (b"\x00" * (length - len(output)))

    def write(self, offset: int, value: bytes) -> None:
        """Writes a value at offset, distributing the writing across any
        underlying mapping."""
        current_offset = offset
        length = len(value)
        for (
            layer_offset,
            sublength,
            mapped_offset,
            mapped_length,
            layer,
        ) in self.mapping(offset, length):
            if layer_offset > current_offset:
                raise exceptions.InvalidAddressException(
                    self.name,
                    current_offset,
                    f"Layer {self.name} cannot map offset: {current_offset}",
                )

            value_chunk = value[
                layer_offset - offset : layer_offset - offset + sublength
            ]
            new_data = self._encode_data(
                layer, mapped_offset, layer_offset, value_chunk
            )
            self._context.layers.write(layer, mapped_offset, new_data)

            current_offset += len(new_data)

    def _scan_iterator(
        self,
        scanner: "ScannerInterface",
        sections: Iterable[Tuple[int, int]],
        linear: bool = False,
    ) -> Iterable[IteratorValue]:
        """Iterator that indicates which blocks in the layer are to be read by
        for the scanning.

        Returns a list of blocks (potentially in lower layers) that make
        up this chunk contiguously. Chunks can be no bigger than
        scanner.chunk_size + scanner.overlap DataLayers by default are
        assumed to have no holes
        """
        for section_start, section_length in sections:
            output: List[Tuple[str, int, int]] = []

            # Hold the offsets of each chunk (including how much has been filled)
            chunk_start = chunk_position = 0

            # For each section, find out which bits of its exists and where they map to
            # This is faster than cutting the entire space into scan_chunk sized blocks and then
            # finding out what exists (particularly if most of the space isn't mapped)
            for mapped in self.mapping(
                section_start, section_length, ignore_errors=True
            ):
                offset, sublength, mapped_offset, mapped_length, layer_name = mapped

                # Setup the variables for this block
                block_start = offset
                block_end = offset + sublength

                # Setup the necessary bits for non-linear mappings
                # For linear we give one layer down and mapped offsets (therefore the conversion)
                # This saves an tiny amount of time not have to redo lookups we've already done
                # For non-linear layers, we give the layer name and the offset in the layer name
                # so that the read/conversion occurs properly
                conversion = mapped_offset - offset if linear else 0
                return_name = layer_name if linear else self.name

                # If this isn't contiguous, start a new chunk
                if chunk_position < block_start:
                    yield output, chunk_position
                    output = []
                    chunk_start = chunk_position = block_start

                # Halfway through a chunk, finish the chunk, then take more
                if chunk_position != chunk_start:
                    chunk_size = min(
                        chunk_position - chunk_start,
                        scanner.chunk_size + scanner.overlap,
                    )
                    output += [(return_name, chunk_position + conversion, chunk_size)]
                    chunk_start = chunk_position + chunk_size
                    chunk_position = chunk_start

                # Pack chunks, if we're enter the loop (starting a new chunk) and there's already chunk there, ship it
                for chunk_start in range(chunk_position, block_end, scanner.chunk_size):
                    if output:
                        yield output, chunk_position
                        output = []
                        chunk_position = chunk_start
                    # Take from chunk_position as far as the block can go,
                    # or as much left of a scanner chunk as we can
                    chunk_size = min(
                        block_end - chunk_position,
                        scanner.chunk_size
                        + scanner.overlap
                        - (chunk_position - chunk_start),
                    )
                    output += [(return_name, chunk_position + conversion, chunk_size)]
                    chunk_start = chunk_position + chunk_size
                    chunk_position = chunk_start

            # Ship anything that might be left
            if output:
                yield output, chunk_position


class LayerContainer(collections.abc.Mapping):
    """Container for multiple layers of data."""

    def __init__(self) -> None:
        self._layers: Dict[str, DataLayerInterface] = {}

    def read(self, layer: str, offset: int, length: int, pad: bool = False) -> bytes:
        """Reads from a particular layer at offset for length bytes.

        Returns 'bytes' not 'str'

        Args:
            layer: The name of the layer to read from
            offset: Where to begin reading within the layer
            length: How many bytes to read from the layer
            pad: Whether to raise exceptions or return null bytes when errors occur

        Returns:
            The result of reading from the requested layer
        """
        return self[layer].read(offset, length, pad)

    def __eq__(self, other):
        return dict(self) == dict(other)

    def write(self, layer: str, offset: int, data: bytes) -> None:
        """Writes to a particular layer at offset for length bytes."""
        self[layer].write(offset, data)

    def add_layer(self, layer: DataLayerInterface) -> None:
        """Adds a layer to memory model.

        This will throw an exception if the required dependencies are not met

        Args:
            layer: the layer to add to the list of layers (based on layer.name)
        """
        if layer.name in self._layers:
            raise exceptions.LayerException(
                layer.name, f"Layer already exists: {layer.name}"
            )
        if isinstance(layer, TranslationLayerInterface):
            missing_list = [
                sublayer
                for sublayer in layer.dependencies
                if sublayer not in self._layers
            ]
            if missing_list:
                raise exceptions.LayerException(
                    layer.name,
                    f"Layer {layer.name} has unmet dependencies: {', '.join(missing_list)}",
                )
        self._layers[layer.name] = layer

    def del_layer(self, name: str) -> None:
        """Removes the layer called name.

        This will throw an exception if other layers depend upon this layer

        Args:
            name: The name of the layer to delete
        """
        for layer in self._layers:
            if name in self._layers[layer].dependencies:
                raise exceptions.LayerException(
                    self._layers[layer].name,
                    f"Layer {self._layers[layer].name} is depended upon by {layer}",
                )
        # Otherwise, wipe out the layer
        self._layers[name].destroy()
        del self._layers[name]

    def free_layer_name(self, prefix: str = "layer") -> str:
        """Returns an unused layer name to ensure no collision occurs when
        inserting a layer.

        Args:
            prefix: A descriptive string with which to prefix the layer name

        Returns:
            A string containing a name, prefixed with prefix, not currently in use within the LayerContainer
        """
        if prefix not in self:
            return prefix
        count = 1
        while f"{prefix}_{count}" in self:
            count += 1
        return f"{prefix}_{count}"

    def __getitem__(self, name: str) -> DataLayerInterface:
        """Returns the layer of specified name."""
        return self._layers[name]

    def __len__(self) -> int:
        return len(self._layers)

    def __iter__(self):
        return iter(self._layers)

    def check_cycles(self) -> None:
        """Runs through the available layers and identifies if there are cycles
        in the DAG."""
        # TODO: Is having a cycle check necessary?
        raise NotImplementedError("Cycle checking has not yet been implemented")


class DummyProgress(object):
    """A class to emulate Multiprocessing/threading Value objects."""

    def __init__(self) -> None:
        self.value = 0
