Source code for hansken_extraction_plugin.api.data_context
+"""This module contains the definition of the DataContext."""
+fromdataclassesimportdataclass
+
+
+
+[docs]
+@dataclass(frozen=True)
+classDataContext:
+"""This class contains the data context of a plugin that is processing a trace."""
+
+ data_type:str#: the named data type that is being processed
+ data_size:int#: the size / total length of the data stream that is being processed
+
+ def__eq__(self,other):
+ # override the default equality check, a subclass is considered equal as long as it matches all the fields this
+ # type describes
+ returnisinstance(other,DataContext)and(self.data_type,self.data_size)==(other.data_type,other.data_size)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/_modules/hansken_extraction_plugin/api/extraction_plugin.html b/0.9.1/_modules/hansken_extraction_plugin/api/extraction_plugin.html
new file mode 100644
index 0000000..28a18f7
--- /dev/null
+++ b/0.9.1/_modules/hansken_extraction_plugin/api/extraction_plugin.html
@@ -0,0 +1,227 @@
+
+
+
+
+
+
+
+ hansken_extraction_plugin.api.extraction_plugin — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Source code for hansken_extraction_plugin.api.extraction_plugin
+"""
+This module contains the different types of Extraction Plugins.
+
+The types of Extraction Plugins differ in their process functions.
+"""
+fromabcimportABC,abstractmethod
+importinspect
+fromtypingimportList
+
+fromhansken_extraction_plugin.api.data_contextimportDataContext
+fromhansken_extraction_plugin.api.extraction_traceimportExtractionTrace,MetaExtractionTrace
+fromhansken_extraction_plugin.api.plugin_infoimportPluginInfo
+fromhansken_extraction_plugin.api.trace_searcherimportTraceSearcher
+fromhansken_extraction_plugin.api.transformerimportTransformer
+fromhansken_extraction_plugin.decorators.transformerimporttransformer_registry
+
+
+
+[docs]
+classBaseExtractionPlugin(ABC):
+"""All Extraction Plugins are derived from this class."""
+
+
+[docs]
+ @abstractmethod
+ defplugin_info(self)->PluginInfo:
+"""Return information about this extraction plugin."""
+
+
+ @property
+ deftransformers(self)->List[Transformer]:
+"""
+ Dynamically retrieves the transformer methods that were decorated with @transform.
+
+ Note: This method will retrieve transformers for superclasses as well.
+ """
+ # Retrieve all super classes of a plugin so that we can also retrieve transformers of super classes.
+ # Note: This also contains the more specific instance this method might be called on as well.
+ base_classes=inspect.getmro(self.__class__)
+
+ # Check for each (super) class of the plugin if transformers have been registered.
+ transformers=[
+ transformerforclinbase_classes
+ ifcl.__name__intransformer_registry
+ fortransformerintransformer_registry[cl.__name__]
+ ]
+
+ returntransformers
+
+
+
+
+[docs]
+classExtractionPlugin(BaseExtractionPlugin):
+"""Default extraction plugin, that processes a trace and one of its datastreams."""
+
+
+[docs]
+ @abstractmethod
+ defprocess(self,trace:ExtractionTrace,data_context:DataContext):
+"""
+ Process a given trace.
+
+ This method is called for every trace that is processed by this tool.
+
+ :param trace: Trace that is being processed
+ :param data_context: Data data_context describing the data stream that is being processed
+ """
+
+
+
+
+
+[docs]
+classMetaExtractionPlugin(BaseExtractionPlugin):
+"""Extraction Plugin that processes a trace only with its metadata, without processing its data."""
+
+
+[docs]
+ @abstractmethod
+ defprocess(self,trace:MetaExtractionTrace):
+"""
+ Process a given trace.
+
+ This method is called for every trace that is processed by this tool.
+
+ :param trace: Trace that is being processed
+ """
+
+
+
+
+
+[docs]
+classDeferredExtractionPlugin(BaseExtractionPlugin):
+"""
+ Extraction Plugin that can be run at a different extraction stage.
+
+ This type of plugin also allows accessing other traces using the searcher.
+ """
+
+
+[docs]
+ @abstractmethod
+ defprocess(self,trace:ExtractionTrace,data_context:DataContext,searcher:TraceSearcher):
+"""
+ Process a given trace.
+
+ This method is called for every trace that is processed by this tool.
+
+ :param trace: Trace that is being processed
+ :param data_context: Data data_context describing the data stream that is being processed
+ :param searcher: TraceSearcher that can be used to obtain more traces
+ """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/_modules/hansken_extraction_plugin/api/extraction_trace.html b/0.9.1/_modules/hansken_extraction_plugin/api/extraction_trace.html
new file mode 100644
index 0000000..9bf3d56
--- /dev/null
+++ b/0.9.1/_modules/hansken_extraction_plugin/api/extraction_trace.html
@@ -0,0 +1,394 @@
+
+
+
+
+
+
+
+ hansken_extraction_plugin.api.extraction_trace — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Source code for hansken_extraction_plugin.api.extraction_trace
+"""
+This module contains the different Trace apis.
+
+Note that there are a couple of different traces:
+ * The ExtractionTrace and MetaExtractionTrace, which are offered to the process function.
+ * ExtractionTraceBuilder, which is a trace that can be built; it does not exist in hansken yet, but it is added after
+ building.
+ * SearchTrace, which represents an immutable trace which is returned after searching for traces.
+"""
+fromabcimportABC,abstractmethod
+fromioimportBufferedReader,BufferedWriter,TextIOBase
+fromtypingimportAny,Literal,Mapping,Optional,Union
+
+fromhansken_extraction_plugin.api.traceletimportTracelet
+fromhansken_extraction_plugin.api.transformationimportTransformation
+
+
+
+[docs]
+classExtractionTraceBuilder(ABC):
+"""
+ ExtractionTrace that can be build.
+
+ Represents child traces.
+ """
+
+
+[docs]
+ @abstractmethod
+ defupdate(self,key_or_updates:Optional[Union[Mapping,str]]=None,value:Optional[Any]=None,
+ data:Optional[Mapping[str,bytes]]=None)->'ExtractionTraceBuilder':
+"""
+ Update or add metadata properties for this `.ExtractionTraceBuilder`.
+
+ Can be used to update the name of the Trace represented by this builder,
+ if not already set.
+
+ :param key_or_updates: either a `str` (the metadata property to be
+ updated) or a mapping supplying both keys and values to be updated
+ :param value: the value to update metadata property *key* to (used
+ only when *key_or_updates* is a `str`, an exception will be thrown
+ if *key_or_updates* is a mapping)
+ :param data: a `dict` mapping data type / stream name to bytes to be
+ added to the trace
+ :return: this `.ExtractionTraceBuilder`
+ """
+
+
+
+[docs]
+ @abstractmethod
+ defadd_tracelet(self,
+ tracelet:Union[Tracelet,str],
+ value:Optional[Mapping[str,Any]]=None)->'ExtractionTraceBuilder':
+"""
+ Add a `.Tracelet` to this `.ExtractionTraceBuilder`.
+
+ :param tracelet: the Tracelet or tracelet type (supplied as a `str`) to add
+ :param value: the tracelet properties to add (only applicable when *tracelet* is a `str`)
+ :return: this `.ExtractionTraceBuilder`
+ """
+
+
+
+[docs]
+ @abstractmethod
+ defadd_transformation(self,data_type:str,transformation:Transformation)->'ExtractionTraceBuilder':
+"""
+ Update or add transformations for this `.ExtractionTraceBuilder`.
+
+ :param data_type: data type of the Transformation
+ :param transformation: the Transformation to add
+ :return: this `.ExtractionTraceBuilder`
+ """
+
+
+
+[docs]
+ @abstractmethod
+ defchild_builder(self,name:Optional[str]=None)->'ExtractionTraceBuilder':
+"""
+ Create a new `.TraceBuilder` to build a child trace to the trace to be represented by this builder.
+
+ .. note::
+ Traces should be created and built in depth first order,
+ parent before child (pre-order).
+
+ :return: a `.TraceBuilder` set up to save a new trace as the child
+ trace of this builder
+ """
+
+
+
+[docs]
+ defadd_data(self,stream:str,data:bytes)->'ExtractionTraceBuilder':
+"""
+ Add data to this trace as a named stream.
+
+ :param stream: name of the data stream to be added
+ :param data: data to be attached
+ :return: this `.ExtractionTraceBuilder`
+ """
+ returnself.update(data={stream:data})
+
+
+
+[docs]
+ @abstractmethod
+ defopen(self,data_type:Optional[str]=None,offset:int=0,size:Optional[int]=None,
+ mode:Literal['rb','wb','w','wt']='rb',encoding='utf-8',buffer_size:Optional[int]=None) \
+ ->Union[BufferedReader,BufferedWriter,TextIOBase]:
+"""
+ Open a data stream to read or write data from or to the `.ExtractionTrace`.
+
+ :param data_type: the data type of the datastream, 'raw' by default
+ :param offset: byte offset to start the stream on when reading
+ :param size: the number of bytes to make available when reading
+ :param mode: 'rb' for reading, 'wb' for writing
+ :param encoding: encoding for writing text, used to convert `str` values to bytes, \
+ only valid for modes 'w' and 'wt'
+ :param buffer_size: buffer size for reading (cache read back/ahead) or writing (cache for flush) data
+ :return: a file-like object to read or write bytes from the named stream
+ """
+
+
+
+[docs]
+ @abstractmethod
+ defbuild(self)->str:
+"""
+ Save the trace being built by this builder to remote.
+
+ .. note::
+ Building more than once will result in an error being raised.
+
+ :return: the new trace' id
+ """
+
+
+
+
+
+[docs]
+classTrace(ABC):
+"""All trace classes should be able to return values."""
+
+
+[docs]
+ @abstractmethod
+ defget(self,key:str,default:Optional[Any]=None)->Any:
+"""
+ Return metadata properties for this `.ExtractionTrace`.
+
+ :param key: the metadata property to be retrieved
+ :param default: value returned if property is not set
+ :return: the value of the requested metadata property
+ """
+
+
+
+
+
+[docs]
+classSearchTrace(Trace):
+"""SearchTraces represent traces returned when searching for traces."""
+
+
+[docs]
+ @abstractmethod
+ defopen(self,stream:str='raw',offset:int=0,size:Optional[int]=None,
+ buffer_size:Optional[int]=None)->BufferedReader:
+"""
+ Open a data stream of the data that is being processed.
+
+ :param stream: data stream of trace to open. defaults to raw. other examples are html, text, etc.
+ :param offset: byte offset to start the stream on
+ :param size: the number of bytes to make available
+ :param buffer_size: buffer size for reading data
+ :return: a file-like object to read bytes from the named stream
+ """
+
+
+
+
+
+[docs]
+classMetaExtractionTrace(Trace):
+"""
+ MetaExtractionTraces contain only metadata.
+
+ This class represenst traces during the extraction of an extraction plugin without a data stream.
+ """
+
+
+[docs]
+ @abstractmethod
+ defupdate(self,key_or_updates:Optional[Union[Mapping,str]]=None,value:Optional[Any]=None,
+ data:Optional[Mapping[str,bytes]]=None)->None:
+"""
+ Update or add metadata properties for this `.ExtractionTrace`.
+
+ :param key_or_updates: either a `str` (the metadata property to be
+ updated) or a mapping supplying both keys and values to be updated
+ :param value: the value to update metadata property *key* to (used
+ only when *key_or_updates* is a `str`, an exception will be thrown
+ if *key_or_updates* is a mapping)
+ :param data: a `dict` mapping data type / stream name to bytes to be
+ added to the trace
+ """
+
+
+
+[docs]
+ @abstractmethod
+ defadd_tracelet(self,
+ tracelet:Union[Tracelet,str],
+ value:Optional[Mapping[str,Any]]=None)->None:
+"""
+ Add a `.Tracelet` to this `.MetaExtractionTrace`.
+
+ :param tracelet: the Tracelet or tracelet type to add
+ :param value: the tracelet properties to add (only applicable when *tracelet* is a tracelet type)
+ """
+
+
+
+[docs]
+ @abstractmethod
+ defadd_transformation(self,data_type:str,transformation:Transformation)->None:
+"""
+ Update or add transformations for this `.ExtractionTraceBuilder`.
+
+ :param data_type: data type of the Transformation
+ :param transformation: the Transformation to add
+ """
+
+
+
+[docs]
+ @abstractmethod
+ defchild_builder(self,name:Optional[str]=None)->ExtractionTraceBuilder:
+"""
+ Create a `.TraceBuilder` to build a trace to be saved as a child of this `.Trace`.
+
+ A new trace will only be added to the index once explicitly saved (e.g.
+ through `.TraceBuilder.build`).
+
+ .. note::
+ Traces should be created and built in depth first order,
+ parent before child (pre-order).
+
+ :param name: the name for the trace being built
+ :return: a `.TraceBuilder` set up to create a child trace of this `.MetaExtractionTrace`
+ """
+
+
+
+
+
+[docs]
+classExtractionTrace(MetaExtractionTrace):
+"""Trace offered to be processed."""
+
+
+[docs]
+ @abstractmethod
+ defopen(self,data_type:Optional[str]=None,offset:int=0,size:Optional[int]=None,
+ mode:Literal['rb','wb','w','wt']='rb',encoding='utf-8',buffer_size:Optional[int]=None) \
+ ->Union[BufferedReader,BufferedWriter,TextIOBase]:
+"""
+ Open a data stream to read or write data from or to the `.ExtractionTrace`.
+
+ :param data_type: the data type of the datastream, 'raw' by default
+ :param offset: byte offset to start the stream on when reading
+ :param size: the number of bytes to make available when reading
+ :param mode: 'rb' for reading, 'wb' for writing
+ :param encoding: encoding for writing text, used to convert `str` values to bytes, \
+ only valid for modes 'w' and 'wt'
+ :param buffer_size: buffer size for reading (cache read back/ahead) or writing (cache for flush) data
+ :return: a file-like object to read or write bytes from the named stream
+ """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/_modules/hansken_extraction_plugin/api/plugin_info.html b/0.9.1/_modules/hansken_extraction_plugin/api/plugin_info.html
new file mode 100644
index 0000000..3807620
--- /dev/null
+++ b/0.9.1/_modules/hansken_extraction_plugin/api/plugin_info.html
@@ -0,0 +1,237 @@
+
+
+
+
+
+
+
+ hansken_extraction_plugin.api.plugin_info — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Source code for hansken_extraction_plugin.api.plugin_info
+"""This module contains all definitions to describe meta data of a plugin, a.k.a. PluginInfo."""
+fromdataclassesimportdataclass,field
+fromenumimportEnum
+fromtypingimportDict,List,Optional
+
+
+
+[docs]
+@dataclass(frozen=True)
+classAuthor:
+"""
+ The author of an Extraction Plugin.
+
+ This information can be retrieved by an end-user from Hansken.
+ """
+
+ name:str
+ email:str
+ organisation:str
+
+
+
+
+[docs]
+classMaturityLevel(Enum):
+"""This class represents the maturity level of an extraction plugin."""
+
+ PROOF_OF_CONCEPT=0
+ READY_FOR_TEST=1
+ PRODUCTION_READY=2
+
+
+
+
+[docs]
+@dataclass(frozen=True)
+classPluginId:
+"""Identifier of a plugin, consisting of domain, category and name. Needs to be unique among all tools/plugins."""
+
+ domain:str
+ category:str
+ name:str
+
+ def__str__(self):
+ returnf'{self.domain}/{self.category}/{self.name}'.lower()
+
+
+
+
+[docs]
+@dataclass(frozen=True)
+classPluginResources:
+"""PluginResources contains information about how many resources will be used for a plugin."""
+
+ maximum_cpu:Optional[float]=None
+"""
+ CPU resources are measured in cpu units. One cpu is equivalent to 1 vCPU/Core for cloud providers and 1 hyperthread
+ on bare-metal Intel processors. Also, fractional requests are allowed. A plugin that asks 0.5 CPU uses half as
+ much CPU as one that asks for 1 CPU.
+ """
+ maximum_memory:Optional[int]=None
+"""Max usable memory for a plugin, measured in megabytes."""
+ maximum_workers:Optional[int]=None
+"""The number of concurrent workers(i.e. traces that can be processed)."""
+
+ def__post_init__(self):
+ ifself.maximum_cpuisnotNoneandself.maximum_cpu<0:
+ raiseValueError(f'maximum_cpu cannot be < 0: {self.maximum_cpu}')
+ ifself.maximum_memoryisnotNoneandself.maximum_memory<0:
+ raiseValueError(f'maximum_memory cannot be < 0: {self.maximum_memory}')
+ ifself.maximum_workersisnotNoneandself.maximum_workers<0:
+ raiseValueError(f'maximum_workers cannot be < 0: {self.maximum_workers}')
+
+
+
+
+[docs]
+@dataclass(frozen=True)
+classTransformerLabel:
+"""
+ TransformerLabel contains information about a transformer method that a plugin provides.
+
+ It is mainly used for storing the properties (name, arguments, return type) of a transformer in PluginInfo objects.
+ Unlike the Transformer class it does not contain the actual function reference to the transformer itself.
+ """
+
+"""The method name of the transformer. For example: my_method"""
+ method_name:str
+
+"""The parameters of the function where the key is the parameter name and the value is the type of the parameter."""
+ parameters:Dict[str,str]
+
+"""The return type of the parameter. See api.Transformer class for supported types."""
+ return_type:str
+
+
+
+
+[docs]
+@dataclass
+classPluginInfo:
+"""
+ This information is used by Hansken to identify and run the plugin.
+
+ Note that the build_plugin.py build script is used to build a plugin docker image with PluginInfo docker labels.
+ """
+
+ id:PluginId#: a plugin's unique identifier, see PluginId
+ version:str#: version of the plugin
+ description:str#: short description of the functionality of the plugin
+ author:Author#: the plugin's author, see Author
+ maturity:MaturityLevel#: maturity level, see MaturityLevel
+ matcher:str#: this matcher selects the traces offered to the plugin
+ webpage_url:str#: plugin url
+ license:Optional[str]=None#: license of this plugin
+ deferred_iterations:int=1#: number of deferred iterations (1 to 20), nly for deferred plugins (optional)
+ resources:Optional[PluginResources]=None#: resources to be reserved for a plugin (optional)
+
+"""Populated dynamically in pack.plugin_info by collecting all @transformer methods. Do not assign manually."""
+ transformers:List[TransformerLabel]=field(default_factory=list)
+
+ def__post_init__(self):
+ ifnot1<=self.deferred_iterations<=20:
+ raiseValueError(f'Invalid value for deferred_iterations: {self.deferred_iterations}. '
+ f'Valid values are 1 =< 20.')
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/_modules/hansken_extraction_plugin/api/search_result.html b/0.9.1/_modules/hansken_extraction_plugin/api/search_result.html
new file mode 100644
index 0000000..9cf21d1
--- /dev/null
+++ b/0.9.1/_modules/hansken_extraction_plugin/api/search_result.html
@@ -0,0 +1,198 @@
+
+
+
+
+
+
+
+ hansken_extraction_plugin.api.search_result — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Source code for hansken_extraction_plugin.api.search_result
+"""This module contains a representation of a search result."""
+fromabcimportABC,abstractmethod
+fromitertoolsimportislice
+fromtypingimportIterable,List,Optional
+
+fromhansken_extraction_plugin.api.extraction_traceimportSearchTrace
+
+
+
+[docs]
+classSearchResult(ABC,Iterable):
+"""
+ Class representing a stream of traces, returned when performing a search request.
+
+ This result can only be iterated once. Results can be retrieved in three ways:
+
+ Treating the result as an iterable:
+
+ .. code-block:: python
+
+ for trace in result:
+ print(trace.name)
+
+ Calling `.take` to process one or more batches of traces:
+
+ .. code-block:: python
+
+ first_100 = result.take(100)
+ process_batch(first_100)
+
+ Calling `.takeone` to get a single trace:
+
+ .. code-block:: python
+
+ first = result.takeone()
+ second = result.takeone()
+
+ print(first.name, second.name)
+
+ """
+
+
+[docs]
+ @abstractmethod
+ deftotal_results(self)->int:
+"""
+ Return the total number of hits.
+
+ :return: Total number of hits
+ """
+ pass
+
+
+
+[docs]
+ deftakeone(self)->Optional[SearchTrace]:
+"""
+ Return a single trace, if this stream is not exhausted.
+
+ :return: A searchtrace, or None if no trace is available
+ """
+ returnnext(self.__iter__(),None)
+
+
+
+[docs]
+ deftake(self,num:int)->List[SearchTrace]:
+"""
+ Return a list containing at most num number of traces, or less if they are not available.
+
+ :param num: Number of traces to take
+ :return: List containing zero or more traces
+ """
+ returnlist(islice(self.__iter__(),num))
+
+
+
+[docs]
+ defclose(self):
+"""
+ Close this SearchResult if no more traces are to be retrieved.
+
+ Required to keep compatibility with hansken.py.
+ """
+ pass
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/_modules/hansken_extraction_plugin/api/trace_searcher.html b/0.9.1/_modules/hansken_extraction_plugin/api/trace_searcher.html
new file mode 100644
index 0000000..1242e31
--- /dev/null
+++ b/0.9.1/_modules/hansken_extraction_plugin/api/trace_searcher.html
@@ -0,0 +1,150 @@
+
+
+
+
+
+
+
+ hansken_extraction_plugin.api.trace_searcher — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Source code for hansken_extraction_plugin.api.trace_searcher
+"""This module contains the definition of a trace searcher."""
+fromabcimportabstractmethod
+fromenumimportEnum
+fromtypingimportUnion
+
+fromhansken_extraction_plugin.api.search_resultimportSearchResult
+
+
+
+[docs]
+classSearchScope(str,Enum):
+"""Scope to describe the search context for TraceSearcher.search calls."""
+
+ image='image'
+ project='project'
+
+
+
+
+[docs]
+classTraceSearcher:
+"""This class can be used to search for traces, using the search method."""
+
+
+[docs]
+ @abstractmethod
+ defsearch(self,query:str,count:int,scope:Union[str,SearchScope]=SearchScope.image)->SearchResult:
+"""
+ Search for indexed traces in Hansken using provided query returning at most count results.
+
+ :param query: HQL-query used for searching
+ :param count: Maximum number of traces to return
+ :param scope: Select search scope: 'image' to search only search for other traces within the image of the trace
+ that is being processed, or 'project' to search in the scope of the full project (either Scope-
+ enum value can be used, or the str-values directly).
+ :return: SearchResult containing found traces
+ """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/_modules/hansken_extraction_plugin/api/tracelet.html b/0.9.1/_modules/hansken_extraction_plugin/api/tracelet.html
new file mode 100644
index 0000000..44ddd60
--- /dev/null
+++ b/0.9.1/_modules/hansken_extraction_plugin/api/tracelet.html
@@ -0,0 +1,142 @@
+
+
+
+
+
+
+
+ hansken_extraction_plugin.api.tracelet — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Source code for hansken_extraction_plugin.api.tracelet
+"""This module contains the definition of a Tracelet."""
+fromtypingimportAny,Mapping
+
+
+
+[docs]
+classTracelet:
+"""
+ A tracelet contains the values of a single fvt (Few Valued Type).
+
+ A few valued type is a trace property type that is a collection of tracelets. A trace can contain multiple few
+ valued types containing one or more tracelets. For example, the `trace.identity`` type may look like this::
+
+ {emailAddress: 'interesting@notreally.com'},
+ {firstName: 'piet'},
+ {emailAddress: 'anotheremail@notreally.com'},
+
+ The trace.identity few valued types contains three different tracelets.
+ """
+
+ def__init__(self,name:str,value:Mapping[str,Any]):
+"""
+ Initialize a tracelet.
+
+ :param name: name or type of the tracelet. In the example this would be ``identity``.
+ :param value: Mapping of keys to properties of this tracelet. In the example this could be
+ ``{emailAddress: 'anotheremail@notreally.com'}``.
+ """
+ self.name=name
+ self.value=value
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/_modules/hansken_extraction_plugin/api/transformation.html b/0.9.1/_modules/hansken_extraction_plugin/api/transformation.html
new file mode 100644
index 0000000..54c1c25
--- /dev/null
+++ b/0.9.1/_modules/hansken_extraction_plugin/api/transformation.html
@@ -0,0 +1,193 @@
+
+
+
+
+
+
+
+ hansken_extraction_plugin.api.transformation — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Source code for hansken_extraction_plugin.api.transformation
+"""This module contains the definition of a Transformation."""
+fromabcimportABC
+fromdataclassesimportdataclass
+fromtypingimportList
+
+
+
+[docs]
+classTransformation(ABC):
+"""A super class for data transformations. Currently only :class:RangedTransformation is supported."""
+
+ pass
+
+
+
+
+[docs]
+@dataclass(frozen=True)
+classRange:
+"""A Range describes a range of bytes with a offset and length."""
+
+ offset:int#: the starting point of the data
+ length:int#: the size of the data
+
+
+
+
+[docs]
+classRangedTransformation(Transformation):
+"""A :class:RangedTransformation describes a data transformation consisting of a list of :class:Range."""
+
+ def__init__(self,ranges:List[Range]):
+""":type ranges: list of :class:Range."""
+ self.ranges=ranges
+
+
+[docs]
+ @staticmethod
+ defbuilder():
+""":return a Builder."""
+ returnRangedTransformation.Builder()
+
+
+
+[docs]
+ classBuilder:
+"""Helper class that implements a transformation builder."""
+
+ def__init__(self)->None:
+"""Initialize a Builder."""
+ self._ranges:List[Range]=[]
+
+
+[docs]
+ defadd_range(self,offset:int,length:int)->'RangedTransformation.Builder':
+"""
+ Add a range to a ranged transformation by providing the range's offset and length.
+
+ :param offset the offset of the data transformation
+ :param length the length of the data transformation
+ :return: this `.RangedTransformation.Builder`
+ """
+ ifoffsetisNone:
+ raiseValueError('offset is required')
+ iflengthisNone:
+ raiseValueError('length is required')
+ self._ranges.append(Range(offset=offset,length=length))
+ returnself
+
+
+
+[docs]
+ defbuild(self)->'RangedTransformation':
+"""
+ Return a RangedTransformation.
+
+ :return: a :class:RangedTransformation
+ """
+ returnRangedTransformation(ranges=self._ranges)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/_modules/hansken_extraction_plugin/api/transformer.html b/0.9.1/_modules/hansken_extraction_plugin/api/transformer.html
new file mode 100644
index 0000000..1171756
--- /dev/null
+++ b/0.9.1/_modules/hansken_extraction_plugin/api/transformer.html
@@ -0,0 +1,243 @@
+
+
+
+
+
+
+
+ hansken_extraction_plugin.api.transformer — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Source code for hansken_extraction_plugin.api.transformer
+"""
+This module contains the Transformer class that holds the function reference of the transformer.
+
+Instances of this class are constructed by BaseExtractionPlugin when retrieving transformers dynamically.
+It also validates whether the method to which @transformer is applied adheres to the requirements of a transformer.
+"""
+
+fromdatetimeimportdatetime
+importinspect
+fromtypingimportMapping,Sequence
+
+fromhansken.utilimportGeographicLocation,Vector
+
+fromhansken_extraction_plugin.api.plugin_infoimportTransformerLabel
+fromhansken_extraction_plugin.utility.type_conversionimportget_type_name
+
+
+
+[docs]
+classTransformer:
+"""
+ A transformer is an exposed method of a plugin that can be executed remotely outside extraction-time.
+
+ This allows for on-demand analysis during an investigation.
+ """
+
+"""
+ This dictionary holds the supported types that transformer methods can as arguments and return types.
+
+ The keys are the supported Python types and the values the generic type names as in the Hansken trace model.
+ Note that these types should also be defined in _primitive_matchers in runtime.pack for successful serialization.
+ In order to not create circular dependencies and separate the runtime module and the api module this is defined
+ separately here.
+ """
+ supported_primitives={bytes:'binary',
+ bool:'boolean',
+ int:'long',
+ float:'double',
+ str:'string',
+ datetime:'date',
+ GeographicLocation:'latLong',
+ Vector:'vector',
+ Sequence:'list',
+ Mapping:'map'}
+
+ def__init__(self,function):
+"""Create a transformer and validate whether the passed function meets the requirements."""
+ self.function=function
+
+ # Retrieve the signature so that we can validate whether it complies to the transformer requirements.
+ signature=inspect.signature(function)
+
+ # Validate that @transformer was applied to a function/method and not any other type of object (i.e. a class)
+ ifnotfunction.__class__.__name__=='function':
+ raiseException('@transformer was applied to something other than a function/method. '
+ '@transformer may only be applied to methods of classes derived from BaseExtractionPlugin.')
+
+ # Validate that @transformer was applied to a method instead of a function.
+ if'.'notinfunction.__qualname__:
+ raiseException(
+ '@transformer was applied to a function instead of a method of a class derived from '
+ 'BaseExtractionPlugin. '
+ '@transformer may only be applied to methods of classes derived from BaseExtractionPlugin.')
+
+ # Extract the method name for ease of use.
+ self.method_name=function.__qualname__.split('.')[1]
+
+ # Validate if this function is not a static method.
+ # Note: This is not entirely foolproof since the self parameter may officially also be named differently or as
+ # a non-first argument.
+ if'self'notinsignature.parameters:
+ raiseException('@transformer may not be applied to static methods.')
+
+ # Validate all the parameters and store them.
+ self.parameters={}
+ forparameterinsignature.parameters.values():
+
+ # Other than validating the self property we don't include it in the parameters field because we do not
+ # want to expose it externally.
+ # Note: This check is not fool-proof since parameters called self can be defined as non-first parameters.
+ ifparameter.name=='self':
+ ifparameter.annotationisnotinspect.Parameter.empty:
+ raiseException('@transformer methods should have a parameter self without a type annotation.')
+ else:
+ continue
+
+ # Validate if annotations are present on each parameter.
+ ifparameter.annotationisinspect.Parameter.empty:
+ raiseException('The parameters of @transformer methods must have type hints.')
+
+ # Validate that parameters do not have default values.
+ ifparameter.defaultisnotinspect.Parameter.empty:
+ raiseException('@transformer methods are currently not allowed to have parameters with a '
+ 'default value.')
+
+ # Do not allow variable arguments or positional only arguments.
+ ifparameter.kindin[parameter.POSITIONAL_ONLY,parameter.VAR_POSITIONAL,parameter.VAR_KEYWORD]:
+ raiseException('@transformer methods are currently not allowed to have positional only parameters or '
+ 'variable parameters (like *args and **kwargs).')
+
+ # Validate if a parameter is one of the supported serializable types.
+ ifparameter.annotationnotinTransformer.supported_primitives.keys():
+ raiseException(f'The parameters of @transformer methods should be one of '
+ f'{[get_type_name(x)forxinTransformer.supported_primitives]}')
+
+ self.parameters[parameter.name]=parameter.annotation
+
+ # Validate if the return annotation is present and one of the supported serializable types.
+ ifsignature.return_annotationnotinTransformer.supported_primitives.keys():
+ raiseException(f'The return type of @transformer methods should be one of '
+ f'{[get_type_name(x)forxinTransformer.supported_primitives.keys()]}')
+
+ self.return_type=signature.return_annotation
+
+
+[docs]
+ defgenerate_label(self)->TransformerLabel:
+"""
+ Generate a TransformerLabel given the transformer method. TransformerLabels are used in PluginInfo objects.
+
+ Unlike Transformers TransformerLabels can be serialized and sent to a client that wishes to call a transformer.
+ The specific Python types are converted to the generic types that are used in the Hansken trace model.
+ """
+ # Convert the parameters to a generic Hansken parameter names.
+ # No checks are needed here because they are already performed upon initialization of a Transformer.
+ parameters={name:self.supported_primitives[param_type]forname,param_typeinself.parameters.items()}
+ return_type=self.supported_primitives[self.return_type]
+ returnTransformerLabel(method_name=self.method_name,
+ parameters=parameters,
+ return_type=return_type)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/_modules/index.html b/0.9.1/_modules/index.html
new file mode 100644
index 0000000..36a2709
--- /dev/null
+++ b/0.9.1/_modules/index.html
@@ -0,0 +1,119 @@
+
+
+
+
+
+
+
+ Overview: module code — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Starting from the Overview page, you can browse the documentation using the links in each page, and in the navigation bar at the top of each page. The Index and Search box allow you to navigate to specific declarations and summary pages, including: All Packages, All Classes and Interfaces
+
+
Search
+
You can search for definitions of modules, packages, types, fields, methods, system properties and other terms defined in the API, using some or all of the name, optionally using "camelCase" abbreviations. For example:
+
+
j.l.obj will match "java.lang.Object"
+
InpStr will match "java.io.InputStream"
+
HM.cK will match "java.util.HashMap.containsKey(Object)"
+The following sections describe the different kinds of pages in this collection.
+
+
Overview
+
The Overview page is the front page of this API document and provides a list of all packages with a summary for each. This page can also contain an overall description of the set of packages.
+
+
+
Package
+
Each package has a page that contains a list of its classes and interfaces, with a summary for each. These pages may contain the following categories:
+
+
Interfaces
+
Classes
+
Enums
+
Exceptions
+
Errors
+
Annotation Types
+
+
+
+
Class or Interface
+
Each class, interface, nested class and nested interface has its own separate page. Each of these pages has three sections consisting of a declaration and description, member summary tables, and detailed member descriptions. Entries in each of these sections are omitted if they are empty or not applicable.
+
+
Class Inheritance Diagram
+
Direct Subclasses
+
All Known Subinterfaces
+
All Known Implementing Classes
+
Class or Interface Declaration
+
Class or Interface Description
+
+
+
+
Nested Class Summary
+
Enum Constant Summary
+
Field Summary
+
Property Summary
+
Constructor Summary
+
Method Summary
+
Required Element Summary
+
Optional Element Summary
+
+
+
+
Enum Constant Details
+
Field Details
+
Property Details
+
Constructor Details
+
Method Details
+
Element Details
+
+
Note: Annotation interfaces have required and optional elements, but not methods. Only enum classes have enum constants. The components of a record class are displayed as part of the declaration of the record class. Properties are a feature of JavaFX.
+
The summary entries are alphabetical, while the detailed descriptions are in the order they appear in the source code. This preserves the logical groupings established by the programmer.
+
+
+
Other Files
+
Packages and modules may contain pages with additional information related to the declarations nearby.
+
+
+
Use
+
Each documented package, class and interface has its own Use page. This page describes what packages, classes, methods, constructors and fields use any part of the given class or package. Given a class or interface A, its Use page includes subclasses of A, fields declared as A, methods that return A, and methods and constructors with parameters of type A. You can access this page by first going to the package, class or interface, then clicking on the USE link in the navigation bar.
+
+
+
Tree (Class Hierarchy)
+
There is a Class Hierarchy page for all packages, plus a hierarchy for each package. Each hierarchy page contains a list of classes and a list of interfaces. Classes are organized by inheritance structure starting with java.lang.Object. Interfaces do not inherit from java.lang.Object.
+
+
When viewing the Overview page, clicking on TREE displays the hierarchy for all packages.
+
When viewing a particular package, class or interface page, clicking on TREE displays the hierarchy for only that package.
+
+
+
+
Deprecated API
+
The Deprecated API page lists all of the API that have been deprecated. A deprecated API is not recommended for use, generally due to shortcomings, and a replacement API is usually given. Deprecated APIs may be removed in future implementations.
+
+
+
All Packages
+
The All Packages page contains an alphabetic index of all packages contained in the documentation.
+
+
+
All Classes and Interfaces
+
The All Classes and Interfaces page contains an alphabetic index of all classes and interfaces contained in the documentation, including annotation interfaces, enum classes, and record classes.
+
+
+
Index
+
The Index contains an alphabetic index of all classes, interfaces, constructors, methods, and fields in the documentation, as well as summary pages such as All Packages, All Classes and Interfaces.
+
+
+
+This help file applies to API documentation generated by the standard doclet.
+
+
Read from the data sequence, returning the read bytes as an array.The data will be read from the current
+ position and the amount of bytes read will equal count, unless the sequence contains
+ fewer remaining bytes.
This the base class for types of Extraction Plugins, and cannot be used solely as a superclass for a plugin.
+ Implement one of its subclasses that have a 'process' method.
+ Extraction plugins can be used by Hansken to process data during the extraction process.
The type of data (see data()) that is being processed. The type is never empty or null.
+ This is the type of data the given data sequence represents, for example "raw" for raw
+ bytes or "plain" for plaintext.
+
+ The data type will be one of those defined on the trace model associated with the trace being processed.
+ Note: the received output stream should not be closed by the user. It should
+ also only be used within the scope of the function, other usage may be guarded against or otherwise
+ result in undefined behaviour.
Deferred extraction plugins can be used by Hansken to process traces during the extraction process.
+ A processed trace can be enriched with new information and new child traces can also be created.
+
+ The difference between this and a normal ExtractionPlugin is that this plugin is able to run a secondary query
+ for traces and combine the results with previously retrieved traces.
+
+ When a plugin matches on the trace which is currently processed by Hansken
+ (for example, because it has certain properties), the plugin will receive the
+ matched trace in order to process it (see process(Trace, DataContext, TraceSearcher)).
+ Note: the given trace should only be modified within the scope of this method.
+ Any modifications afterwards may be guarded against or result in undefined behaviour.
+
+
Parameters:
+
trace - the trace to process
+
dataContext - data context of this trace extraction
+
searcher - the searcher for the trace
+
Throws:
+
ExecutionException - when an exception occurs while searching for traces
+
InterruptedException - when a thread gets interrupted while searching for traces
Extraction plugins can be used by Hansken to process traces during the extraction process.
+ A processed trace can be enriched with new information and new child traces can also be created.
+
+ When a plugin matches on the trace which is currently processed by Hansken
+ (for example, because it has certain properties or a certain type of data sequence), the plugin will receive the
+ matched trace and data in order to process it (see process(Trace, DataContext)).
+ A trace can have multiple data sequences of different types. Because of this, a certain trace might
+ be processed multiple times (depending on if this plugin triggers on the different data types).
+
+ Note: the given trace should only be modified within the scope of this method.
+ Any modifications afterwards may be guarded against or result in undefined behavior.
+
+
Parameters:
+
trace - the trace to process
+
dataContext - data context of the traces data stream that is being processed
A trace contains information about processed data. A trace should conform to the trace model defined by Hansken.
+ Unlike a Trace, an ImmutableTrace instance lacks methods to modify or update its properties and data
+ streams after the instance is created.
Get the value of the property with given name on this trace.
+
+ Note: the method is declared with a type parameter to allow for
+ automatic casting. It is an unchecked cast, so implementers have to make sure that
+ the type of the value returned is of type T.
public staticLatLongof(double latitude,
+ double longitude)
+
Create a new geographical point with given latitude
+ and longitude.
+
+ Note: this does not do any input validation. Valid
+ values should be between -90 and +90 inclusive for latitude and
+ between -180 and +180 inclusive for longitude.
Returns the enum constant of this type with the specified name.
+The string must match exactly an identifier used to declare an
+enum constant in this type. (Extraneous whitespace characters are
+not permitted.)
+
+
Parameters:
+
name - the name of the enum constant to be returned.
Meta extraction plugins can be used by Hansken to process traces during the extraction process.
+ A processed trace can be enriched with new information and new child traces can also be created.
+
+ The difference between this and a normal ExtractionPlugin is that this plugin does not receive
+ or processes any data, only a trace itself.
+
+ When a plugin matches on the trace which is currently processed by Hansken
+ (for example, because it has certain properties), the plugin will receive the
+ matched trace in order to process it (see process(Trace)).
+
+ Note for Hansken core developers: specifying 'meta' in the matcher is
+ not necessary, the framework takes care of this.
+ A trace can have multiple data sequences of different types. Because of this, a certain trace might
+ be processed multiple times (depending on if this plugin triggers on the different data types).
+
+ Note: the given trace should only be modified within the scope of this method.
+ Any modifications afterwards may be guarded against or result in undefined behavior.
Start processing a trace without any of its associated data streams.
+ When processing a given trace, new properties may be set on it. New children can be added using
+ Trace.newChild(String, ThrowingConsumer).
+
+ Note: the given trace should only be modified within the scope of this method.
+ Any modifications afterwards may be guarded against or result in undefined behaviour.
Create a unique identifier for a plugin, consisting of domain, category and name.
+
+
Parameters:
+
domain - the domain of the organisation, for example "nfi.nl"
+
category - the action group of the plugin, for example `extract`, `carve`, `classify` (read the SDK documentation for more details).
+
name - the name of the plugin, or in the classic sense, a description detailing the action(s) of the plugin. Note that the name can contain (forward) slashes.
+ example: "nfi.nl/extract/ocr/detection/plugin".
+ in this example nfi.nl is the domain, extract is the category, and ocr/detection/plugin is the name.
Set the number of extraction iterations needed for this deferred plugin.
+ Only relevant for deferred plugins.
+ If this method is not called upon, default number of iterations will be set to 1.
+
+
Parameters:
+
deferredIterations - number of iterations needed for this plugin. Should be between 1 and 20
Set the unique id of this plugin, consisting of domain, category and name.
+
+ example: "nfi.nl/extract/ocr/detection/plugin".
+ in this example nfi.nl is the domain, extract is the category, and ocr/detection/plugin is the name.
+
+
Parameters:
+
domain - the domain of the organisation, for example "nfi.nl"
+
category - the action group of the plugin, for example `extract`, `carve`, `classify` (read the SDK documentation for more details).
+
name - the name of the plugin, or in the classic sense, a description detailing the action(s) of the plugin. Note that the name can contain (forward) slashes.
public final class PluginResources
+extends Object
+
PluginResources contains information about how many resources will be used for a plugin. The most common resources to specify are CPU and memory (RAM).
+
+ CPU resources are measured in cpu units. One cpu is equivalent to 1 vCPU/Core for cloud providers and 1 hyperthread on bare-metal Intel processors.
+ Also, fractional requests are allowed. A plugin that asks 0.5 CPU uses half as much CPU as one that asks for 1 CPU.
+
+ Memory resources are measured in Megabytes.
+
+ Here is an example to set resources for a plugin:
+
Returns the enum constant of this type with the specified name.
+The string must match exactly an identifier used to declare an
+enum constant in this type. (Extraneous whitespace characters are
+not permitted.)
+
+
Parameters:
+
name - the name of the enum constant to be returned.
Read from the data sequence, returning the read bytes as an array.The data will be read from the current
+ position and the amount of bytes read will equal count, unless the sequence contains
+ fewer remaining bytes.
Read bytes into the given buffer, starting at position 0 in the buffer. The data will be read from the
+ current position and the amount of bytes copied will equal the length of the buffer, unless
+ the data sequence contains fewer remaining bytes. In that case, data is read until the end of the sequence.
defaultintread(byte[] buffer,
+ int count)
+ throws IOException
+
Read bytes into the given buffer, starting at position 0 in the buffer. The data will be read from the
+ current position and the amount of bytes copied will equal count, unless the data
+ sequence contains fewer remaining bytes. In that case, data is read until the end of the sequence.
intread(byte[] buffer,
+ int offset,
+ int count)
+ throws IOException
+
Read data into the given buffer, starting at position offset in the buffer. The data will be read from
+ the current position and the amount of bytes read will equal count, unless the
+ sequence contains fewer remaining bytes. In that case, data is read until the end of the sequence.
+
+
Parameters:
+
buffer - the buffer to read into
+
offset - the offset in the buffer from which to start writing
+
count - the amount of bytes to read
+
Returns:
+
the number of bytes actually read
+
Throws:
+
IllegalArgumentException - if count or offset is negative,
+ or if offset + count is larger than the size of the buffer
Read from the data sequence, returning the read bytes as an array.The data will be read from the current
+ position and the amount of bytes read will equal count, unless the sequence contains
+ fewer remaining bytes. In that case, data is read until the end of the sequence and a smaller array is returned,
+ with a length equal to the number of read bytes.
+
+ Note: this method will allocate a new buffer each time it is called. It is intended for simple
+ cases where it is convenient to read a specified number of bytes into a byte array.
+
+
Parameters:
+
count - the amount of bytes to read
+
Returns:
+
a buffer containing the read bytes, or an empty array if we were at the end of the stream
A trace contains information about processed data. A trace should conform to the trace model defined by Hansken.
+ Unlike a Trace, the SearchTrace contains information about the searched trace. It is able to
+ directly retrieve all the data contexts belonging to a searched trace. When data type is known a
+ RandomAccessData can be retrieved directly.
public static class Trace.Tracelet
+extends Object
+
a Tracelet represents tracedata that can be present multiple times within a trace.
+ The API doesn't specify the cardinality , but the implementation is limited to
+ cardinality Few.
A trace contains information about processed data. A trace should conform to the trace model defined by Hansken.
+
+ A trace can have multiple types (e.g. file or chat). For these types, the trace can contain a set of properties and
+ associated values. A trace can be associated with a set of data sequences.
+ During extraction, an extraction plugin may receive a currently processed trace
+ and accompanying data sequence.
+
+ Implementers of a plugin should ensure that setting a property on a trace is valid as per the trace model
+ which is defined for the traces generated by the plugin.
+
+ Note: implementations are not required to implement any kind of thread safety.
+ It is up to the client to ensure this if necessary.
Add a data stream of a given type to this Trace (for example, 'raw' or 'html').
+ A trace can only have a single data stream for each data type. A callback function can be passed
+ which receives an OutputStream to write the data to.
+
+ Note: the received output stream should not be closed by the user. It should
+ also only be used within the scope of the callback, other usage may be guarded against or otherwise
+ result in undefined behaviour.
+
+ Example usage:
+
+ final PacketSequencer sequencer = ...;
+ final RandomAccessData input = dataContext.data();
+
+ trace.setData("packets", data -> {
+ sequencer.process(input).forEach(packet -> {
+ data.write(packet);
+ });
+ });
+
+
+
Parameters:
+
dataType - the type of the data stream to add
+
writer - callback that receives the stream to write to as input
Set a series of data transformations for a specific dataType.
+
+ Transformations are pointers to actual raw data. They describe how data can be obtained by reading on certain
+ positions, using decryption, or combinations of these. The benefit of using Transformations is that they take up
+ less space than the actual data.
Create and store new child trace of this trace. A callback function
+ can be passed in order to enrich the created child trace (which will have no types
+ or properties set yet), or even recursively add new children under it. After the
+ trace goes out of scope of the callback, it can no longer be updated.
+
+ As an example, say we have a Node type with the following API
+
Returns the enum constant of this type with the specified name.
+The string must match exactly an identifier used to declare an
+enum constant in this type. (Extraneous whitespace characters are
+not permitted.)
+
+
Parameters:
+
name - the name of the enum constant to be returned.
Description of a transform method of a plugin.
+ The transform method is a method, specified by method name, of a Transformer capable plugin
+ that transforms input to output of a specified type (returnType)
Creates a Vector from a collection of numbers.
+ Note that all numbers are converted to floats internally, so loss of precision may occur when doubles or longs are offered.
Creates a vector from a binary representation.
+ Vector.asBinary() can be used to obtain a binary representation.
+ Note that this directly sets the internal state of the Vector, use {code asVector(bytes.clone()} to store a safe, immutable copy.
+
+
Parameters:
+
bytes - the bytes to convert to a vector.
+
Returns:
+
a vector
+
+
+
+
+
+
asBinary
+
publicbyte[]asBinary()
+
Returns the binary representation of the Vector.
+ Vector.asVector(byte[]) can be used to convert the binary representation back to the original vector.
+ The format of the returned bytes is a sequence of the floating point values of the vector, stored as big-endian IEEE 754 encoded 32-bit floating point values.
+ Note that this exposes the internal state of the Vector, use {code asBinary().clone()} to obtain a safely mutable copy.
+
+
Returns:
+
a binary representation of the vector.
+
+
+
+
+
+
size
+
publicintsize()
+
Returns the number of dimensions of the vector.
+
+
Returns:
+
the number of dimensions of the vector.
+
+
+
+
+
+
values
+
publicfloat[]values()
+
Returns the values of the Vector as an array of floats.
public static<T>TargNotNull(String name,
+ T value)
+
Check that the value with given name is not null, otherwise throw an exception.
+
+ Example usage:
+
+
+ private final Author author;
+
+ public Book(final Author author) {
+ // this throws a NullPointerException when the author is null
+ this.author = argNotNull("author", author);
+ }
+
+
+
+
Type Parameters:
+
T - the type of the value
+
Parameters:
+
name - a descriptive name for the value (most likely the argument name)
+
value - the value itself
+
Returns:
+
the value, so it can be used to inline in an assignment
public static<T>List<T>argsIsType(String name,
+ List<T> value,
+ Class<?> type)
+
Check that the value with given name is of type type, otherwise throw an exception.
+
+ Example usage:
+
+
+ public Book(final List<Object> authors) {
+ // this throws a ClassCastException when authors contains a type other than Author.class
+ argsIsType("author", authors, Author.class);
+ }
+
+
+
+
Type Parameters:
+
T - the type of the value
+
Parameters:
+
name - a descriptive name for the value (most likely the argument name)
+
value - the value itself
+
type - the class type we want to check against value
+
Returns:
+
the value, so it can be used to inline in an assignment
Check that not all values with given name are null, otherwise throw an exception.
+
+ Example usage:
+
+
+ private final Author author;
+
+ public Book(final Author... authors) {
+ // this throws a NullPointerException when the author is null
+ this.author = argNotAllNull("authors", authors);
+ }
+
+
+
+
Parameters:
+
name - a descriptive name for the values (most likely the argument name)
+
values - the values, which can be different types
+
Returns:
+
the values, so it can be used to inline in an assignment
public static<T>T[]argNotEmpty(String name,
+ T[] value)
+
Check that the array value with given name is not empty, otherwise throw an exception.
+
+ Example usage:
+
+
+ private final Author[] author;
+
+ public Book(final Author[] author) {
+ // this throws a NullPointerException when the author array is empty
+ this.author = argNotEmpty("author", author);
+ }
+
+
+
+
Type Parameters:
+
T - the type of the value
+
Parameters:
+
name - a descriptive name for the value (most likely the argument name)
+
value - the value itself
+
Returns:
+
the value, so it can be used to inline in an assignment
Check that the collection value with given name is not empty, otherwise throw an exception.
+
+ Example usage:
+
+
+ private final Collection<Author> author;
+
+ public Book(final Collection<Author> author) {
+ // this throws a NullPointerException when the author collection is empty
+ this.author = argNotEmpty("author", author);
+ }
+
+
+
+
Type Parameters:
+
T - the type of the value
+
Parameters:
+
name - a descriptive name for the value (most likely the argument name)
+
value - the value itself
+
Returns:
+
the value, so it can be used to inline in an assignment
Check that the string with given name is not null or empty, otherwise throw an exception.
+
+ Example usage:
+
+
+ private final String title;
+
+ public Book(final String title) {
+ // this throws a NullPointerException when the author is null
+ // or an IllegalArgumentException when author is an empty String
+ this.title = argNotEmpty("title", title);
+ }
+
+
+
+
Parameters:
+
name - a descriptive name for the string (most likely the argument name)
+
value - the string itself
+
Returns:
+
the string, so it can be used to inline in an assignment
public staticintargNotNegative(String name,
+ int value)
+
Check that the int with given name is not negative, otherwise throw an exception.
+ Example usage:
+
+
+ private final int pageCount;
+
+ public Book(final int pageCount) {
+ // this throws an IllegalArgumentException when pageCount is negative
+ this.pageCount = argNotNegative("pageCount", pageCount);
+ }
+
+
+
+
Parameters:
+
name - a descriptive name for the int (most likely the argument name)
+
value - the int itself
+
Returns:
+
the int, so it can be used in an inline assignment
public staticlongargNotNegative(String name,
+ long value)
+
Check that the long with given name is not negative, otherwise throw an exception.
+ Example usage:
+
+
+ private final long pageCount;
+
+ public Book(final long pageCount) {
+ // this throws an IllegalArgumentException when pageCount is negative
+ this.pageCount = argNotNegative("pageCount", pageCount);
+ }
+
+
+
+
Parameters:
+
name - a descriptive name for the long (most likely the argument name)
+
value - the long itself
+
Returns:
+
the long, so it can be used in an inline assignment
Represents an operation that accepts a single input argument and returns no result.
+ Replaces the normal Java Consumer and enables throwing an Exception from the callback. Used to enable
+ lambda callbacks in for example Trace.newChild(String, ThrowingConsumer).
The help page provides an introduction to the scope and syntax of JavaDoc search.
+
You can use the <ctrl> or <cmd> keys in combination with the left and right arrow keys to switch between result tabs in this page.
+
The URL template below may be used to configure this page as a search engine in browsers that support this feature. It has been tested to work in Google Chrome and Mozilla Firefox. Note that other browsers may not support this feature or require a different URL format.
+link
+
+
+
+
+
Loading search index...
+
+
+
+
+
+
+
+
+
+
diff --git a/0.9.1/_static/javadoc/search.js b/0.9.1/_static/javadoc/search.js
new file mode 100644
index 0000000..2246cdd
--- /dev/null
+++ b/0.9.1/_static/javadoc/search.js
@@ -0,0 +1,354 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+var noResult = {l: "No results found"};
+var loading = {l: "Loading search index..."};
+var catModules = "Modules";
+var catPackages = "Packages";
+var catTypes = "Types";
+var catMembers = "Members";
+var catSearchTags = "Search Tags";
+var highlight = "$&";
+var searchPattern = "";
+var fallbackPattern = "";
+var RANKING_THRESHOLD = 2;
+var NO_MATCH = 0xffff;
+var MIN_RESULTS = 3;
+var MAX_RESULTS = 500;
+var UNNAMED = "";
+function escapeHtml(str) {
+ return str.replace(//g, ">");
+}
+function getHighlightedText(item, matcher, fallbackMatcher) {
+ var escapedItem = escapeHtml(item);
+ var highlighted = escapedItem.replace(matcher, highlight);
+ if (highlighted === escapedItem) {
+ highlighted = escapedItem.replace(fallbackMatcher, highlight)
+ }
+ return highlighted;
+}
+function getURLPrefix(ui) {
+ var urlPrefix="";
+ var slash = "/";
+ if (ui.item.category === catModules) {
+ return ui.item.l + slash;
+ } else if (ui.item.category === catPackages && ui.item.m) {
+ return ui.item.m + slash;
+ } else if (ui.item.category === catTypes || ui.item.category === catMembers) {
+ if (ui.item.m) {
+ urlPrefix = ui.item.m + slash;
+ } else {
+ $.each(packageSearchIndex, function(index, item) {
+ if (item.m && ui.item.p === item.l) {
+ urlPrefix = item.m + slash;
+ }
+ });
+ }
+ }
+ return urlPrefix;
+}
+function createSearchPattern(term) {
+ var pattern = "";
+ var isWordToken = false;
+ term.replace(/,\s*/g, ", ").trim().split(/\s+/).forEach(function(w, index) {
+ if (index > 0) {
+ // whitespace between identifiers is significant
+ pattern += (isWordToken && /^\w/.test(w)) ? "\\s+" : "\\s*";
+ }
+ var tokens = w.split(/(?=[A-Z,.()<>[\/])/);
+ for (var i = 0; i < tokens.length; i++) {
+ var s = tokens[i];
+ if (s === "") {
+ continue;
+ }
+ pattern += $.ui.autocomplete.escapeRegex(s);
+ isWordToken = /\w$/.test(s);
+ if (isWordToken) {
+ pattern += "([a-z0-9_$<>\\[\\]]*?)";
+ }
+ }
+ });
+ return pattern;
+}
+function createMatcher(pattern, flags) {
+ var isCamelCase = /[A-Z]/.test(pattern);
+ return new RegExp(pattern, flags + (isCamelCase ? "" : "i"));
+}
+var watermark = 'Search';
+$(function() {
+ var search = $("#search-input");
+ var reset = $("#reset-button");
+ search.val('');
+ search.prop("disabled", false);
+ reset.prop("disabled", false);
+ search.val(watermark).addClass('watermark');
+ search.blur(function() {
+ if ($(this).val().length === 0) {
+ $(this).val(watermark).addClass('watermark');
+ }
+ });
+ search.on('click keydown paste', function() {
+ if ($(this).val() === watermark) {
+ $(this).val('').removeClass('watermark');
+ }
+ });
+ reset.click(function() {
+ search.val('').focus();
+ });
+ search.focus()[0].setSelectionRange(0, 0);
+});
+$.widget("custom.catcomplete", $.ui.autocomplete, {
+ _create: function() {
+ this._super();
+ this.widget().menu("option", "items", "> :not(.ui-autocomplete-category)");
+ },
+ _renderMenu: function(ul, items) {
+ var rMenu = this;
+ var currentCategory = "";
+ rMenu.menu.bindings = $();
+ $.each(items, function(index, item) {
+ var li;
+ if (item.category && item.category !== currentCategory) {
+ ul.append("
The following page lists all (technical) changes in the extraction plugin SDK.
+
Programming language specific API changes are described in more detail on API changelog pages.
+These pages list new API functionalities, and describe how to update your plugins when API changes are in order.
+For the API changelog pages see:
HANSKEN-20128: Scope trace searches by default to the image under extraction, but allow project-wide searches by passing an optional argument (Java).
+
HANSKEN-20127: Scope trace searches by default to the image under extraction, but allow project-wide searches by passing an optional argument (Python).
+
HANSKEN-20552: Support trace property types of list[float] (Python), List (Java), List (Java), and List (Java).
HANSKEN-16632: Fix execution of meta-extraction plugins with hansken.py runner which failed with an ‘expected $data in matcher’ error
+
HANSKEN-16634: Allow forward compatibility with Hansken by introducing an GRPC API version
+
HANSKEN-16489: Let build pipeline publish Java artifacts to community
+
HANSKEN-16558: Serve SDK test framework errors in a more developer-friendly way
+
HANSKEN-16489: Removed incompatibility warning for All In One with Hansken.py
+
HANSKEN-16403: Fixed running markdownlint with tox-emarkdownlint
+
HANSKEN-16268: Added * value support to the HQL-Lite term matcher, improved documentation by using HQL default property:value instead of property=value
+
HANSKEN-16258: Fixed Jenkins build
+
HANSKEN-16257: Fixed docker stop command in test framework
HANSKEN-14879: Allow SDK releases to be published on PyPI
+
HANSKEN-14738: Shade NFI internal projects into the SDK testframework jar
+
HANSKEN-14841: Bugfix where RpcStringMap wasn’t being unpacked properly in Python, which was discovered during a flits test.
+
HANSKEN-14703: Let Python plugins exit gracefully on SIGTERM
+
HANSKEN-14844: SDK: move serve from test_framework to runtime
+
HANSKEN-14793: Add ExtractionPluginBuilder.add_data method in python API
+
HANSKEN-14777: Add an extra check to ignore and log unsupported types during RpcStart gRPC serialization.
+
HANSKEN-14739: License: Distribute ExtractionPluginSDK under the Apache License 2.0
+
HANSKEN-14763: Bugfix where some python plugins were not loaded correctly when using serve or test-plugin commands
+
HANSKEN-14737: Move serve.py from plugin examples to SDK
+
HANSKEN-14720: Add option to use with when using the trace.open method in python
+
HANSKEN-14582: Add option to write data using the python api
+
HANSKEN-14660: Move _test.py files from plugin examples repo to SDK repo
+
HANSKEN-14618: Add validation for unexpected extra data streams to test framework
+
HANSKEN-14704: Fix shading of the runtime super pom
+
HANSKEN-14619: Allow propagation of IOException in plugin new child callback
+
HANSKEN-14632: Add Java gRPC support for writing raw data streams on a trace
+
HANSKEN-14591: Split into three modules in the SDK
+
HANSKEN-14580: Add proto message definitions for raw data stream writing
+
HANSKEN-14635: Trace format which containes name/id can now be deserialized by testframework
+
HANSKEN-14131: Added ‘verbose’ logging for test-framework HQL matching
+
HANSKEN-14130: Updated StandaloneTestRunner to expose more errors & exceptions
+
HANSKEN-13784: Add meta support to test-framework
+
HANSKEN-14581: Extend Trace API with raw data writing capabilities
+
HANSKEN-14547: Deploy Java sources JAR for improved client debugging
+
HANSKEN-14531: Fix Python release. Python needs only one build step, which is either a snapshot or a release build. The separate Python release step was removed and merged with the first Python build step. Repository paths were corrected for the release version.
+
HANSKEN-14531: Fix python release - The python release is no longer a separate step in the build pipeline, since there is no actual difference between a snapshot and a release, apart from the version numbering scheme and the test-framework is downloaded from a repository location depending on the release parameter (see comments in Jenkinsfile)
+
HANSKEN-14161: Add Python test-framework wrapper around Java test-framework and add test-framework.tgz to whl
+
HANSKEN-14234: Restructure build pipeline to build and release Java first
+
HANSKEN-13799: Extraction Plugin: support meta extraction
+
HANSKEN-14286: Create adapter from RandomAccessData to InputStream
+
HANSKEN-14314: Don’t send child name when sending enrichment message
+
HANSKEN-14318: Flush cached children before flushing root in case of error with python gRPC server
+
HANSKEN-13414: Generate shaded jar for runtime that shades Guava, Protobuf, gRPC, and Netty (fix)
+
HANSKEN-14283: Allow passing a configuration of retry policy for the extraction plugin client
+
HANSKEN-14234: Restructure build pipeline to build and release Java first
+
HANSKEN-13414: Generate shaded jar for runtime that shades Guava, Protobuf, gRPC, and Netty
+
HANSKEN-14234: Restructure build pipeline to build and release Java first
+
HANSKEN-14135: Improve testing Python plugins in integration step and test reading large chunks
+
HANSKEN-14134: Fix releasing python plugins
+
HANSKEN-14128: Validate gRPC message limit for Python server instances
+
HANSKEN-14122: Fix missing comma in dependencies which broke the release
+
HANSKEN-14092: The type and total size of the data currently being processed can now be retrieved from an extractioncontext object passed to the process function
+
HANSKEN-13668: Added support for lists of longs, Hansken maps and LatLong to Java and Python API
+
HANSKEN-14104: Set Python gRPC limit to 64 MB(including message overhead)
+
HANSKEN-14079: Add logging to the SDK
+
HANSKEN-14010: Add support for serializing datetime in python API
+
HANSKEN-14083: Fix releasing python sdk
+
HANSKEN-14035: Add static type checks to python project
+
HANSKEN-14030: Allow test framework to be executed standalone for non-java extraction plugins
+
HANSKEN-14073: Support gRPC extraction plugins in test framework
+
HANSKEN-14074: Add support for serialization of Maps.
+
Hansken-14090: Use new Hansken python-api children call for creating nested children
+
HANSKEN-13774: Add support for creating children in python API
+
HANSKEN-13776: Add error handling to the Python based server and send error messages to the client
+
HANSKEN-14060: Remove mapping-interface from python extraction for API consistency
+
HANSKEN-13773: Update trace properties in Python API
+
HANSKEN-14044: Make sure python testing code is linted as well, enforce single quotes
+
HANSKEN-13772: Expose trace properties in Python API
+
HANSKEN-13775: Python - added trace.open() functionality to read from data streams
+
HANSKEN-14031: Split Trace interfaces (hansken.py trace vs external plugin trace)
+
HANSKEN-14037: Make sure pytest is always used for python tests
+
HANSKEN-14011: Implement unpack for trace
+
HANSKEN-14009: Implement pack for trace and trace enrichment
+
HANSKEN-14008: Move generated hql-lite parsers to different package (conflicts with hql package)
+
HANSKEN-13777: Added utility to run Python Extraction Plugin implementations with Hansken.py
+
HANSKEN-13771: Implement Extraction Plugin Info for Python plugins
+
HANSKEN-13966: Give socketproxy disconnect some time to disconnect (fixes flaky unit test)
+
HANSKEN-13810: Add extraction plugin python server code
+
HANSKEN-13676: Added support for ZonedDateTime over gRPC
+
HANSKEN-13922: Add webpage url to PluginInfo
+
HANSKEN-13676: Changed the way of creating child Traces to using a consumer.
+
HANSKEN-13655: Added server/client disconnect tests and implemented initial handling server-side
+
HANSKEN-13809: Added missing gRPC exception handles of process()
+
HANSKEN-13801: Made HQL-Lite matchers immutable
+
HANSKEN-13761: Seperated HQL-Lite type matcher implementation
+
HANSKEN-13756: Added HQL-Lite datastream matchers
+
HANSKEN-13800: (Temporarily) remove meta from Extraction Plugin API
+
HANSKEN-13798: Propagate exception on failure of START serialization
+
HANSKEN-13706: Create basic test framework implementation
+
HANSKEN-13769: Make jenkins run tests and a linter for python
+
HANSKEN-13705: Add support for creating child traces over gRPC
+
HANSKEN-13713: Make (partial) extraction plugin errors visible for clients
+
HANSKEN-13709: Send partial result when an external plugin errors out
+
HANSKEN-13733: Add script to generate gRPC Python files
+
HANSKEN-13660: Copied the Hql definition from Hansken to enable the HQL-Lite implementation
+
HANSKEN-13656: Test and handle invalid protocol messages
+
HANSKEN-13663: Add matcher interface to PluginInfo
+
HANSKEN-13714: Add IOException to ExtractionPlugin.process() interface
+
HANSKEN-13658: test(s) for non-grpc connection with a grpc server
+
HANSKEN-13650: Add basic support for writing trace information over gRPC
+
HANSKEN-13648: Add basic support for reading trace information over gRPC
+
HANSKEN-13651: Default implementations for RandomAccessData interface
+
HANSKEN-13643: Add basic support for reading trace information over gRPC
+
HANSKEN-13643: Add basic support for reading from trace data over gRPC
Chat with us on Discord. You can find members of the extraction plugin SDK development team
+in the Hansken Community server, in the extraction-plugins channel. This is a Hansken community-private server. If you
+don’t have access to the Hansken Community server, please contact your Hansken business owner. He or she can invite you
+to the Hansken Community server. If you don’t know who to contact, feel free to fill in
+the contact form for further questions/contact.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/concepts.html b/0.9.1/dev/concepts.html
new file mode 100644
index 0000000..4fcd398
--- /dev/null
+++ b/0.9.1/dev/concepts.html
@@ -0,0 +1,148 @@
+
+
+
+
+
+
+
+
+ General concepts — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
The pluginInfo() method returns a PluginInfo object. Hansken needs this object to be able to know the capabilities
+of the plugin, and to show the plugin in the list of tools. The most important fields that must be set on PluginInfo
+are the following:
+
+
id | The identifier of the plugin. This will be used as a unique name for the plugin Hansken. |
+
description | A description of the plugin that is shown in Hansken. |
+
author | The author of the plugin that is shown in Hansken. |
+
license | The type of license of the plugin that is shown in Hansken. |
+
matcher | This matcher is used by Hansken to determine which Traces are sent to the Plugin during extraction. |
During extraction, Hansken calls the process() method for every matching trace. The matcher attribute of
+the PluginInfo is very important as it determines which traces will be sent to the process method.
+
Although the plugin developer is free to program whatever seems useful, the following tasks are typically performed
+within the process() method:
+
+
Creating child-traces
+
Reading trace properties
+
Adding trace properties
+
Reading the data that the Trace represents
+
Writing data on a Trace
+
+
Depending on the type of plugin that is implemented, different functionality is available in the process() method.
+See Plugin Types for more details.
This describes the process of running a plugin from the perspective of Hansken. The perspective of the user is described
+in Hansken Extraction Plugins.
Hansken manages a list of tools that can be used in extractions. The available plugins must be added to this list so
+that the user can select them. To accomplish this, Hansken scans the Docker registry for docker images that are plugins.
+Each image is started up, and a call is done to its pluginInfo() method. If the call resulted in a valid PluginInfo
+object, the Extraction Plugin is added to the list of tools visible to users. After the PluginInfo is retrieved, the
+docker image is shutdown again.
Hansken checks if any plugins are selected by the user that started the extraction. For each selected plugin, at least
+one docker image will be started. See Kubernetes autoscaling for more details on expanding
+the number of instances for each plugin.
During an extraction, Hansken will iteratively loop through all selected tools, including Extraction Plugins. For each
+trace that matches on a tool, Hansken will call its process method. For Extraction Plugins, this means that
+the process method is called via the gRPC protocol. The trace to be processed is sent over gRPC to the plugin, and any
+other communication between Hansken and the Extraction Plugin (like created properties and child traces, search requests
+and written data) are done using gRPC.
Extraction Plugins can create new data-streams on a Trace through data transformations. Data
+transformations describe how data can be obtained from a source. Data transformations are preferred over storing blobs
+because they take less space. This is because they only describe the data instead of specifying the actual data.
+
The following figure shows how Hansken visualizes data transformations:
+
+
Note that transformations can be applied on transformations. The SDK only supports range transformations at the moment,
+while this image also shows some transformations that are currently available in Hansken but not in the SDK.
+
An example case is an extraction plugin that processes an archive file. The plugin creates a child trace per entry in
+the archive file. Each child trace will have a data stream that is a transformation that marks the start and length of
+the entry in the original archive data. By just describing the data instead of specifying the actual data, a lot of
+space is saved.
+
Although Hansken supports various transformations, the Extraction Plugins SDK for now only supports ranged data
+transformations. Ranged data transformations define data as a list of ranges, each range with an offset and length in a
+bytearray.
Hansken Extraction Plugins can be built in Java or Python by implementing an interface. Which interface you choose
+depends on the type of plugin you choose to make, see Plugin Types. For more information on coding
+your own plugin, see the Extraction Plugin Examples.
+
The plugin can then be tested using the Test Framework. This way you make sure everything works as
+expected before taking further steps.
To upload a plugin into Hansken, a docker image for this plugin must be uploaded to the docker registry. First, the
+plugin container image must be packaged.
+A plugin is packaged into an OCI image (also known as Docker image).
Hansken finds the plugins by scanning a docker registry. It will try to load all docker images with a certain prefix as
+Extraction Plugins. The settings for this are defined in Hansken properties:
+
+
registry.extraction.plugins.registry.uri defines the registry
+
registry.extraction.plugins.registry.prefix defines the prefix plugins must have
+
+
When the image is packaged locally, it needs to be pushed (uploaded) to the docker registry. These commands provide an
+outline of how to do this:
+
+
dockerlogin<docker-registry> (make sure you are logged in to the registry)
+
dockerpush<docker-registry><prefix><pluginname> (push the plugin docker image to the registry)
+
+
+
Note
+
For more information about uploading plugins and running them in Hansken, see the ‘Using extraction plugins in
+Hansken’ chapter of the Hansken User Guide.
Hansken checks which plugins are available at startup. The list of available plugins can also be refreshed by calling
+the following endpoint: <hansken-domain>/gatekeeper/tools?refresh=true
If everything went well, the list of available tools in Hansken should now feature your plugin. To run the plugin in an
+extraction, be sure to select its checkbox in the extraction tools dialog.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/concepts/hql_lite.html b/0.9.1/dev/concepts/hql_lite.html
new file mode 100644
index 0000000..fd441c1
--- /dev/null
+++ b/0.9.1/dev/concepts/hql_lite.html
@@ -0,0 +1,575 @@
+
+
+
+
+
+
+
+
+ HQL-Lite — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
HQL-Lite is a query language derived from Hanskens full HQL human. HQL stands for Hansken Query Language and can be
+used to search or match traces. Since not all elements of full HQL can be used in the context of an extraction,
+extraction plugins use HQL-Lite, a lightweight version of HQL. This document describes the usage of HQL-Lite in the
+context of extraction plugins.
To reduce the unnecessary overhead of processing all traces (even the ones the tool cannot actually process), Hansken
+implements the concept of a matcher for each tool. This matcher basically checks the trace for “matching
+conditionsâ€, that would allow the tool to process it.
+
Sometimes these matching conditions can be as simple as a specific filename or extension, but are often more
+elaborate in the sense that they check multiple factors that require some intimate knowledge of Hansken.
HQL-Lite is a language based on HQL (Hansken Query Language) that allows plugin developers to write matchers for
+Hansken Extraction Plugins. It could be said that HQL-Lite contains a subset of HQL features, plus some HQL-Lite unique
+features that are only interesting for matchers.
+
+
Note
+
Please note that even though the HQL-Lite query is part of the plugin, it is compiled and stored in Hansken during
+startup to achieve performance.
HQL was designed to search for traces stored in the Elasticsearch database. As such, some of its features are tightly
+coupled to the Elasticsearch implementation, making it difficult to re-implement them for plugins.
+
Also, even though HQL is more complex than the requirements for matching in plugins, a couple of minor features that
+are absolutely necessary for matching are not implemented in HQL, as they don’t make much sense from a search point of
+view. This is because HQL was designed to be used with finished extractions with all the traces stored in the
+database, while HQL-Lite was designed for active extractions.
an empty string translates to match for all traces
+
+
And
+
foo:1ANDbar:2
+
the case-sensitive AND operator behaves like a logical AND of 2 conditions
+
+
Not
+
NOTfoo or -foo
+
the case-sensitive NOT or - negates the expression that follows
+
+
Range
+
foo>1 or 1<=foo<10
+
a numbered-range check with a min or/and max range(s)
+
+
Or
+
foo:1ORbar:2
+
the case-sensitive OR operator behaves like a logical OR of 2 conditions
+
+
Data
+
$data.foo:1
+
see $data section below
+
+
DataType
+
$data.type:raw
+
this query matches against the type of the current datastream
+
+
Types
+
type:email
+
this query checks if the trace contains a certain trace type as defined in the Hansken trace model
+
+
+
+
There are also a couple of general guidelines that apply to all matchers:
+
+
Equals/not equals:
+
+
: or = : The most basic of left equals right statements. note that = is also valid.
+
!= : The opposite of equals, not equals. Note that !: is NOT supported.
+
+
+
Wildcards:
+
+
? : Match against any single character. E.g. foo:r?w will match against raw,row but not against rowing.
+
* : Match against any chars. E.g. foo:r* will match against r,ra,raw,raaaaaw but not against aw.
+
+
+
Exact match: By surrounding a value with quotes, we tell the parser that it is a single value. This is especially
+helpful for values that might contain separators. E.g. foo:'hellohql-lite'.
+
CSV: Currently only the type query supports multiple values to check against. E.g. type:email,chatMessage will only
+return true if both types exist for this trace.
+
() grouping: You can group statements by putting brackets around them. E.g. foo:1AND(bar:2ORbla:3) which
+translates to foo:1 plus one of the statements in the brackets.
+
Escaping \"\.\t\r\n:=><!()~/,[]{}: Some characters are used internally by HQL-Lite, and need to be escaped if they
+are used in the value side of the key-value pair. These values can be escaped by adding prepending \\ to the
+character(s). Example: foo:foobar should be foo:foo\\bar, foo.bar:foo:bar should be foo.bar:foo\\:bar
+…etc.
+
+
The only exceptions to this rule are unix paths:
+
+
Acceptable paths:
+
+
foo:/
+
foo:bar/baz
+
foo:/bar/baz
+
foo:'/bar/baz/hello'
+
foo:*bar/baz*
+
+
+
Unacceptable paths:
+
+
foo:/bar/ -> this is the regex matcher, which is unsupported in HQL-lite
+
foo:c:\ -> should be foo:c\:\\, both the colon and the slash need to be escaped
+
foo:'c:\' -> should be foo:'c:\\', the slash still needs to be escaped
+
+
+
Note
+
the backslash is the universal escape character, so it always needs to be escaped.
In Hansken, a trace can have multiple datastreams. The exact content of said datastreams is
+discussed elsewhere, but the basic idea is that a trace can have multiple representations. For example, a trace might
+have a raw datastream, but after we identify that the raw bytes contain a text file, we might add a separate
+datastream text.
+
+
Note
+
The process() method of each plugin is called for each datastream of each trace. This is explained
+in How does Hansken work? . Subsequently, you might have the same property for a
+different datastream. For example: you might have a data.raw.size and a data.text.size property. The reason you
+might have the same property multiple times, is because it could have a different meaning.
+
+
For example:
+
+
data.raw.size: is the size in bytes
+
data.text.size: is the number of bytes in the text representation of the raw stream
+
+
If we want to check if either of these properties is not empty by using a $data matcher, we do:
The easiest way would be to only allow traces with the .pdf extension. Looking at the Hansken trace model (or a
+Hansken extraction), we can see that there’s a property file
+which contains a property extension.
+
So what would that look like in HQL-lite? Something like
+
file.extension=pdf
+
+
+
+
Warning
+
This of course only works if the file has the correct extension (note that matchers are case-sensitive).
+
+
So what do we do, if we also want to match pdf files that are (un)intentionally misnamed?
Looking at Wikipedia, we see that pdf has a couple of mime-types. In return looking at our extraction and the
+trace-model, we see both at data.raw.mimeType, with a further explanation in the Hansken trace model that
+the raw portion of the property is the data type of the datastream.
+
If we don’t know which datastream has the mimeType property beforehand, we could use the broad-scoped $data. matcher
+to look at every datastream.
+
So our matcher becomes:
+
file.extension=pdf OR
+(
+ $data.mimeType=application\\/pdf OR
+ $data.mimeType=application\\/x-pdf
+)
+
Some pdf files can be huge, meaning that parsing them will need a lot of resources. Could we add a data size check to
+the matcher? According to the Hansken trace modeldata has a property size (similar to mimeType) that we
+could use for this.
+
+
Note
+
This is also a good way to check if a file is empty or not.
+
+
Let’s say our cutoff limit is 1 MB, meaning our matcher becomes:
+
0 < $data.size < 1000000 AND
+(
+ file.extension=pdf OR
+ (
+ $data.mimeType=application\\/pdf OR
+ $data.mimeType=application\\/x-pdf
+ )
+)
+
It is not uncommon to have some overlap between tools/plugins. For example:
+
+
PdfPlugin: a plugin that only supports pdf documents
+
DocumentPlugin: this plugin supports a lot of document types, including pdf.
+
+
So how would we prevent our plugin from processing a trace that has already been processed by the DocumentPlugin?
+
The easiest solution would be to check if a certain property has already been set. Meaning, that if both plugins set
+the foo.bar property, we check if said property has already been set.
+
So we only process the trace if foo.bar is empty, meaning our matcher becomes:
+
foo.bar!=* AND
+0 < $data.size < 1000000 AND
+(
+ file.extension=pdf OR
+ (
+ $data.mimeType=application\\/pdf OR
+ $data.mimeType=application\\/x-pdf
+ )
+)
+
It is also not uncommon to exclude certain paths from your plugin. These paths might contain invalid or encrypted files,
+for example.
+
So let’s say we want to exclude all files under in the /tmp/virus path. How do we go about it?
+
Again, we check our extraction/Hansken trace model, and we see that file.path looks promising.
+
So excluding /tmp/virus would look something like:
+
-file.path=/tmp/virus* AND
+foo.bar!=* AND
+0 < $data.size < 1000000 AND
+(
+ file.extension=pdf OR
+ (
+ $data.mimeType=application\\/pdf OR
+ $data.mimeType=application\\/x-pdf
+ )
+)
+
+
+
+
+
Match on specific datastream type, an anti-patternïƒ
+
+
Warning
+
Matching on specific datastream types is an anti-pattern! It is not recommended to match on a datastream
+type. Instead one should match on other datastream properties, such as fileType, mimeType or mimeClass.
+The reason for this is explained in the paragraph below.
+
+
Using a matcher that is too loose or too tight can yield unexpected results. Usually one will match on properties
+of a datastream like fileType, mimeType or mimeClass as these are reliable properties that have been added by
+Hansken tools. Matching on a specific datastream says nothing about the type of file. For example a PDF file may be
+available in a raw as well as in a decrypted datastream. By matching on the datastream type one may exclude traces
+that were not intended to be excluded.
+Contrarily, note that matching on a datastream type may include more traces than you expected as well. For example,
+someone may think “Plugin A puts data on the plain datastream, so I’ll match on the plain datastream with Plugin Bâ€,
+forgetting that plain may be used by other tools as well. In other words, there may be traces with that datastream
+type that you did not know of, potentially crashing your plugin. See Data streams for more information.
+
Now that you know why it is an anti-pattern, lets explain how it would be done (for those edge cases where it’s needed):
+Lets say we want our PdfPlugin to ONLY process raw datastreams.
+The best way to do this would be to match
+on $data.type:raw. Note that $data.type matches against the type of the current datastream, so in this case it
+matches only when the current datastream is of type raw.
+
An incorrect way to do it would be to replace $data. matcher(s) with data.raw.. This means the matcher
+will match whenever a trace has this datastream type, even if the current datastream type is different.
+Remember that the process method of an extraction plugin is always called once for each datastream on each trace.
+For example, lets say a trace has two datastreams, raw and text. The matcher would match for both the datastreams
+because the trace has a raw datastream (even though the current datastream type may be text). This results in the
+process method being called twice (for raw and for text), which may lead to other bugs if the developer doesn’t
+know this. For example, the second time the plugin may be trying to overwrite data on a trace which is prohibited.
+
So, using $data.type, our matcher would look like:
+
$data.type:raw AND
+-file.path=/tmp/virus* AND
+foo.bar!=* AND
+0 < $data.size < 1000000 AND
+(
+ file.extension=pdf OR
+ (
+ $data.mimeType=application\\/pdf OR
+ $data.mimeType=application\\/x-pdf
+ )
+)
+
In practice, only you as the plugin dev can answer this question.
+
Know that from the point of view of Hansken, we only care that the plugin:
+
+
Should not crash: If a matcher does not compile, then your plugin will not be available in Hansken. Tip: be sure
+to test your plugin with the test framework.
+
Should not be slow: Matching is designed to be extremely fast, but of course, if you make it too complex it can
+take longer than we want. In the example above, we calculated that 1 second extra for 1 million traces is 11 days of
+extra CPU time. Unlike processing, matching is done for every trace, in every extraction iteration, so be careful!
+
Should match on the bare minimum: Don’t go too far by matching 50 different criteria before allowing a trace to be
+processed. Note that a lot of (if not all) of these criteria depend on properties set by other tools, and you don’t
+really have any control on how these tools work.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/concepts/isolation.html b/0.9.1/dev/concepts/isolation.html
new file mode 100644
index 0000000..958a83c
--- /dev/null
+++ b/0.9.1/dev/concepts/isolation.html
@@ -0,0 +1,164 @@
+
+
+
+
+
+
+
+
+ Plugin isolation — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Extraction plugins allows arbitrary code to be executed during a Hansken extraction. This code is executed inside the
+Hansken cluster. Extraction plugins are subjected
+to Hanskens design principles
+such as security, privacy and transparency. To ensure that plugins are compliant to these principles, each plugin will
+be executed in isolation. This page describes the isolation measures that are in place.
Plugins are only allowed to call a limited set of (Linux) system calls. This ensures that a plugin can be executed in a
+secure manner within the Hansken platform.
+
Hansken uses Kubernetes to run extraction plugins. The Kubernetes RuntimeDefault secure computing mode (seccomp) is
+enabled to provide a sane default of available system calls.
Extraction Plugins can be run in a Kubernetes cluster. This can be the same cluster Hansken run on, or another external
+cluster. For each plugin, a pod is created by Hansken. Each plugin will have 12 threads by default to process traces
+separately within one pod.
Hansken will create a Horizontal Pod Autoscaler (HPA) for each pod. HPA’s manage the number of replica’s of a pod
+based on metrics. The Extraction Plugins SDK provides two metrics to be set in the PluginInfo:
+
+
Observed CPU utilization
+
Observed memory usage
+
+
For more info on how to set these metrics follow these links:
If a pod reaches the CPU or memory usage provided with the PluginInfo, the HPA will increase the number of replicas for
+that plugin. Scaling down is done automatically. The maximum number of replicas per pod is specified within Hansken
+properties and can be adapted by an operator (needs restart).
This depends on the kubernetes cluster settings, the nodes it runs on, and of course the plugin itself. Monitor the
+number of replica’s and resource metrics while an extraction is running and adapt accordingly.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/concepts/plugin_naming_convention.html b/0.9.1/dev/concepts/plugin_naming_convention.html
new file mode 100644
index 0000000..93b3c38
--- /dev/null
+++ b/0.9.1/dev/concepts/plugin_naming_convention.html
@@ -0,0 +1,233 @@
+
+
+
+
+
+
+
+
+ Plugin naming convention — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Each extraction plugin has a unique identifier. The identifier consists of three fields. These three fields combined
+form the plugin name.
+
The three fields of a plugin identifier are: domain, category, and name. The fields are described in more detail
+below.
+
domain
+The domain name of the organisation where the plugin is created. If an organisation has multiple domain names, the
+shortest name is preferred over the longer domain names. Examples: nfi.nl, politie.nl, fiod.nl, hansken.org.
+
category
+A type of action that the plugin performs. The category is a free text field, but the following table gives some
+recommendations.
+
+
+
Category
+
Description
+
+
+
+
extract
+
The plugin parses a clear data structure
+
+
carve
+
The plugin parses data fragments to reassemble traces in the absence of filesystem metadata
+
+
classify
+
The plugin categorizes a plugin based on its content, e.g. detecting money on traces of type picture
+
+
digest
+
The plugin digests data to compute a hash
+
+
ocr
+
The plugin applies ocr (optical character recognition) to read text on pictures or scanned documents
+
+
match
+
The plugin matches a trace against a database, and reports whether there was hit or miss, e.g. matching a trace to a well known files database
+
+
+
+
name
+The name of the plugin, or in the classic sense, a description detailing what the plugin processes. Note that the name
+can contain (forward) slashes.
The following table shows a list of plugin identifiers. The last column of the table shows the derived full plugin name.
+The derived full plugin name will be shown in Hansken.
+
+
+
Domain
+
Category
+
Name
+
Derived plugin name
+
Explanation
+
+
+
+
hansken.org
+
extract
+
archive
+
hansken.org/extract/archive
+
A plugin created by the Hansken development team that extracts traces from an arbitrary archive format
+
+
nfi.nl
+
extract
+
archive/zip
+
nfi.nl/extract/archive/zip
+
A plugin created by an NFI team that extracts traces from a specific archive format: zip
+
+
politie.nl
+
extract
+
archive/zip
+
politie.nl/extract/archive/zip
+
The same as the previous example, but now the plugin is created by a different organisation: politie.nl
+
+
hansken.org
+
carve
+
archive/zip
+
hansken.org/carve/archive/zip
+
A plugin that carves data to detect a specific archive format: zip
+
+
hansken.org
+
digest
+
sha256
+
hansken.org/digest/sha256
+
A plugin that digests data to compute a sha256 hash
+
+
hansken.org
+
ocr
+
tesseract
+
hansken.org/ocr/tesseract
+
A plugin that performs ocr using tesseract
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/concepts/plugin_types.html b/0.9.1/dev/concepts/plugin_types.html
new file mode 100644
index 0000000..c44e331
--- /dev/null
+++ b/0.9.1/dev/concepts/plugin_types.html
@@ -0,0 +1,196 @@
+
+
+
+
+
+
+
+
+ Extraction plugin types — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Meta Extraction Plugins can only process and produce trace properties without the need (or possibility) for processing
+actual trace data. This includes:
These plugins have two additional features that are not included with Standard Extraction Plugins:
+
+
a developer can choose to defer their execution
+
information about other traces can be obtained while processing the current extraction trace. Code examples can be
+found here: Java, Python.
+
+
Deferring execution
+A single Hansken image extraction consists of multiple iterations. Within every iteration, every Hansken tool and
+plugin is executed on matching traces, which produces new traces or modifies existing traces. If a tool can be executed
+another time because of these additions or modifications, another iteration is started.
+
A regular plugin is executed in the same iteration a trace is matched. Deferred plugins are executed in a different
+iteration; they are always deferred for at least one iteration. This is very useful when searching for traces, because
+you are certain the deferred plugin is executed after all other tools performed their modifications.
+
Sometimes, executing your plugin in the next iteration is not enough; it needs to be executed in a different iteration.
+This is why the SDK allows setting a deferredIterations parameter in the plugin info. After the plugin matches with a
+trace, it will be executed after deferredIterations. The execution can currently be deferred by a maximum of 20
+iterations and the default is 1.
+
Searching for traces
+This type of plugin can perform a search to look for extracted traces in the current image (default) or
+project (optional). This search is performed using a provided HQL query.
+A maximum of 50 traces is returned for a given search request.
+
+
Warning
+
Please note that HQL-Lite specific syntax such as the Data or DataType matchers is NOT supported.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/concepts/test_framework.html b/0.9.1/dev/concepts/test_framework.html
new file mode 100644
index 0000000..5074bb6
--- /dev/null
+++ b/0.9.1/dev/concepts/test_framework.html
@@ -0,0 +1,428 @@
+
+
+
+
+
+
+
+
+ Test framework — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
The SDK provides the FLITS Test Framework for integration testing. This allows us to test/validate the plugin input
+and output without having a running Hansken instance.
+
To use the test framework, three components are required:
The test data is independent of which programming language is used for the plugin (Java or Python). This section
+describes the setup of the test data, while the sections thereafter will link to the language specific documentation.
The inputs folder contains all traces that will be processed during the test. These ‘input traces’ are defined in
+files with the ‘.trace’ extension, using JSON. This JSON structure is explained in section
+Trace format. Each trace may have various data-streams. The data for each trace
+is put into separate files for each data-stream. The data-stream files need to have the same name as their corresponding
+trace file but differ in extension. They can have any extension, for example ‘raw’, ‘text’ or ‘jpeg’. Note that one
+input trace will always have one ‘.trace’ file, and can have none, one or many data files. Also note that if the
+plugin doesn’t match on any of the input files and there are no result files yet, the test will succeed.
+
+
Note
+
The test-framework uses the extension of the input test file(s) _(other than __.trace__)_ as type of the
+current data-stream.
+
+
The expected results (which are also traces) are stored in a separate results folder next to the inputs folder. The
+file names in the results folder correspond to the file names in the inputs folder. Note that the name of the plugin
+is added between the file basename and the file extension. This can be useful if one maintains a single test input and
+output test datasets for multiple extraction plugins.
+
+
Note
+
It is possible to let the test framework regenerate the results files automatically. See the
+Java and Python sections on testing on how to do this. If no files are
+being generated, check if the plugin matcher is actually matching the input files.
+
+
The test runner will invoke the extraction plugin for each input trace. The test runner collects the plugin output and
+compares it against the trace defined in the results folder. If there is a mismatch, the test runner will fail with an
+exit code 1. If all tests pass the test runner will finish with exit code 0.
+
Given the files in the example above, the test runner will invoke the extraction plugin three times:
+
+
+
Input
+
Result
+
+
+
+
example1.trace with data streamexample1.raw
+
example1.raw.PluginName.trace
+
+
example1.trace with data streamexample1.text
+
example1.text.PluginName.trace
+
+
example2.trace with data streamexample2.text
+
example2.raw.PluginName.trace
+
+
+
+
+
+
Test data structure for deferred extraction pluginsïƒ
+
Deferred extration plugins have the unique ability to search traces with a query.
+The input test data should be extended to contain the results of searches done by deferred extraction plugins. These
+search traces are stored in separate folders that follow the naming format ‘{deferred trace name}/searchtraces/’. Below
+is an example test data directory structure for a deferred extraction plugin that searches for
+a deferredExampleSearch.trace:
The plugin will try to match on all traces in the input folder, including traces used for search results (
+of deferred extraction plugins). This means that it is impossible to search on traces that match the same deferred
+extraction plugin, as it would create an infinite loop.
+
+
Given the files in the example above, the test runner will invoke the extraction plugin one time:
+
+
+
Input
+
Result
+
+
+
+
deferredExample.trace with data stream deferredExample.raw
+
deferredExample.raw.DeferredPluginName.trace
+
+
+
+
+
Warning
+
The search query should be written in HQL, as that is how Hansken will interpret it. However, the test
+framework interprets the query using its HQL-lite interpreter. Therefore, not all queries will be supported.
Input and result traces both stored in a JSON structure. There is however a slight difference between the two: The
+result trace may store additional values that are purely there for testing purposes. The input format will first be
+discussed, followed by the result format.
Input traces start with a trace key, which contains a mapping of properties. The property names are split in a
+dictionary structure. The example below shows a serialized trace with six properties: data.raw.mimeClass and the five
+data types that are currently supported by the test-framework.
+
The data key defines the data-streams of the trace. When adding a data-stream make sure you also
+add the corresponding input data file, as described above.
The extraction plugin SDK and the test framework have no knowledge of the
+trace model. This means that when properties are used that don’t
+comply with the trace model, this will not cause the test to fail, but it will fail when running your plugin in Hansken.
The result traces have the same format as the input traces, namely a trace key which contains the full input trace
+with all its properties. However, the result traces may have two additional keys children and data (which are
+explained in-depth below). These are added for testing purposes. If the plugin adds child traces
+or writes data transformations to Hansken, this would normally not reflect on the JSON of the
+trace. However, the test framework adds these to the result JSON structure to be able to test them.
+
Consequently, result traces are stored in a JSON structure that may consist of up to three parts, namely the always
+present trace and the occasional children and data:
+
+
trace: The key trace contains a mapping of its properties, in exactly the same way as is done for input traces.
+
children: :ref:Childtraces<childtraces> that have been created by the plugin during the test are stored under a
+reserved field children, which is a list of traces. The example trace below contains a child trace with a property
+name.
+
data: Data transformations that have been created by the plugin during the test are
+stored under a reserved field data. For each data-stream type there is a descriptor field describing the data
+transformation in a JSON format. The example trace has a ranged data transformation for the raw data-stream. Note that
+this data is entirely different from the data key that may be present inside the trace!
Some scenarios may throw exceptions and this can be part of your tests too. For example, an input file that has the
+wrong format can be part of your integration tests. When an exception occurs during the test, it will be written to the
+result file. This can be deliberately used to test exceptions. However, it is often impractical to match against a full
+exception. For example, the row numbers in the exception are very much prone to change due to circumstances irrelevant
+to the case being tested. Therefore, the testframework provides some options to match only on those parts of result
+files that are relevant to the test.
+
The following sections will explain these partial result matchers using the following example exception:
+
{
+"class":"org.hansken.plugin.extraction.runtime.grpc.client.ExtractionPluginException",
+"message":"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris faucibus varius sodales."
+}
+
The containsInOrder partial result matcher requires a list of strings as a parameter. The result will be valid if
+every string in the list can be found in that same order in the actual result.
+
{
+"class":"org.hansken.plugin.extraction.runtime.grpc.client.ExtractionPluginException",
+"message.containsInOrder":[
+"Lorem ipsum dolor sit amet,",
+"consectetur adipiscing elit.",
+"Mauris faucibus varius sodales."
+]
+}
+
The Test Framework itself is built in Java. When building extraction plugins with Java, it can be incorporated in your
+unit tests, as shown in Using the Test Framework in Java.
The Python SDK also uses the Java based Test Framework. This is done by providing a wrapper to make calls to an included
+Test Framework ‘jar’ file. See Advanced use of the Test Framework in Python for documentation
+and examples on how to use FLITS for testing your Python plugin.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/concepts/traces.html b/0.9.1/dev/concepts/traces.html
new file mode 100644
index 0000000..459eca5
--- /dev/null
+++ b/0.9.1/dev/concepts/traces.html
@@ -0,0 +1,340 @@
+
+
+
+
+
+
+
+
+ Traces & Trace model — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Traces are structured data objects produced by tools/plugins during an extraction. A trace represents a piece of
+information found in an evidence file.
+
The following figure shows the main elements of a trace. Each element is described in more detail in the following
+paragraphs.
A trace has properties that describe the information of it by means of a property value. Trace properties are grouped by
+a trace type. A trace can have multiple types.
+
All types and properties that can be set are defined in the Hansken trace model.
+
An example of a type is document, which could have the properties application and createdOn. The trace will have a
+type document, and can the following properties with values:
A trace has several intrinsic properties. These are properties that are not related to a trace type. The intrinsic
+properties available to extraction plugins are:
+
+
id: a unique identifier of the trace, generated when a trace is created
+
name: a name given to a trace when it is created
+
path: a logical path of the trace, of which the elements are the names of traces from the root trace until this
+trace
Typically, a trace represents a piece of data found in an evidence file. This data is part of the trace and available as
+a data stream. A trace can have multiple data streams. Each data stream has a type. Data streams can also have
+properties that apply to the data stream itself. The data stream properties are modeled as properties of the trace, in
+the following pattern:
+data.<datastreamtype>.propertyname (where <datastreamtype> is substituted by the actual type of the data stream).
+
The set of data stream types and data stream properties is fixed. All allowed types and properties are defined in the
+Hansken trace model (see data).
+
An important data stream property is the fileType property. This property contains a textual description of the
+detected file type for the data stream. An example of a fileType is ‘Adobe Pdf’. The fileType is a good candidate
+to use in a extraction plugin ‘matcher’. This fileType is detected by Hansken using file type heuristics, which are
+primarily based on the data stream bytes itself, and secondarily on other metadata such as a file extension.
+(N.B. The fileType is detected in Hansken by the extraction tool Firefli.) For more information on how datastream
+properties can be used for matching, see here.
+
Note that not all traces have data streams. In these cases it is a trace of meta-data derived from another trace.
+
Usually, each trace with data has a data stream of type raw. This data stream contains the bytes of the traces as they
+were found when the trace was created. In some occasions, the raw data can be represented in a different form before it
+can be processed further, for example if the data can be decoded or decrypted. Hansken tools and plugins can decode
+the raw data stream to a standard UTF-8 data stream, or can decrypt the data if a decryption key is present. Hansken
+tools and extraction plugins can store the new data at the new trace in a new data stream. This new data stream has a
+different type than the raw type.
A trace can have child traces. For example, a trace of type archive can have children, where each child is a trace
+that represents an entry in the archive.
+
With an extraction plugin it is possible to create child traces for a trace that is being processed. New properties,
+data streams, and other child traces can be set on the new child traces. When a child trace is created, the plugin
+should provide a name for the child trace. The id of the child trace is generated, in the following
+form: parenttraceid-childnumber. For example, if the parent has an id 0-0-0-0-0:0-9, the first child gets the
+id 0-0-0-0-0:0-9-1, the second child gets the id 0-0-0-0-0:0-9-2, and so on.
+
Note that a trace does not have (direct) access to its parent trace.
A vector is a data type that can be used to store points in n-dimensional space as an array of floating point values.
+Once indexed, the vectors can then be used in a gui or other client to search for traces that have a nearby vectors.
+For example, it is possible to use a neural network that provides embeddings of human faces as vectors. Once indexed,
+the vectors can then be used to find pictures with similar faces. To do this, the search rest api can be used to
+sort by the euclidean- or manhattan distance, or cosine similarity to a given vector.
A Tracelet is a bundle of property values that belong to a single type. It is a property on a trace that can have
+multiple properties itself, making it a list of key/value pairs. The API doesn’t specify the cardinality, but the
+implementation is limited to cardinality ‘Few’. In Hansken these are called FVT’s (Few Valued Types).
+
+
Note
+
MVT’s (Many Valued Types) are currently not supported in the SDK and will be added in a future release.
+
+
An example of a tracelet is the prediction property, which describes a category or class a trace belongs to. It is
+possible for a trace to have multiple predictions. Therefore prediction is a tracelet. Other examples of
+tracelets are identity and collection.
All traces in Hansken are based on a specific version of the trace model, and must comply to that version of the trace
+model. This is a nested data structure composed of origins, categories, types and properties.
+
All non-inrinsic trace properties are optional and are grouped by type. These types are defined under the trace
+model section ‘categories’. Every category has a list of allowed types. When a trace is identified as being a
+document, it will get this set of predefined document properties. Trace types can have different origins. The
+possible origins are defined in the trace model section ‘origins’. An example of this is the processed types that are
+always generated by the system during an extraction.
+
The details of the current trace model can be retrieved using the /tracemodel
+REST call on the Gatekeeper endpoint of Hansken, or check the Hansken Documentation on the trace model.
The extraction plugin SDK has no knowledge of the trace model
+
+
The Extraction Plugins SDK has no knowledge of the trace model at this time. It is however possible to create new traces
+with plugins. If any newly created Traces don’t comply to the model, Hansken will not accept them and mark the plugin
+execution as failed. The Extraction Plugins SDK and the provided Test Framework don’t check this.
+Please make sure to use the right naming when creating new Traces, as provided by the trace model.
+
If an erroneous trace property is set, Hansken will show an error. The error can be found in the Hansken Expert UI
+interface by double-clicking on the trace. Then the trace details screen will be opened and the error will be displayed
+as follows:
+
+
This error describes that a property does not exist in the trace model. To get more information about the error, the
+extraction log can be viewed. In the extraction log you have to search
+for java.lang.IllegalArgumentException:nosuchtype to find out which property is not supported by the trace model.
+
In the example extraction log below, the property this_property_does_not_exist could not be found 681 times.
You will first need git access to the Hansken developer community. Here you can find started guides and examples. If you
+have no access yet, you can get access by following the next steps:
After you have created your account, you should request access to the â€
+Hansken Community†group . Do this by contacting your organisations Hansken business owner. If you don’t know who to
+contact as your business owner, please read the Contact page.
The SDK contains an API and tools to write a Hansken extraction plugin in Java or Python. The Java API can be used to
+develop extraction plugins in JVM-compatible languages, such as Scala and Kotlin.
Probably not. It takes time and effort to create a proper SDK. If you think there is a good use case to support
+language foobar, and there is gRPC support, feel free to contact us. We can discuss the options to add support for
+Hansken extraction plugins with foobar.
+
Under the hood, extraction plugins use gRPC to communicate with Hansken. In theory, all programming languages that have
+a gRPC implementation can be used to write Hansken extraction plugins.
+
+
+
Can I reuse or modify the Extraction Plugins SDK?ïƒ
+
The SDK is distributed under the Apache 2.0 License, see the LICENSE file in the SDK for more details.
We are doing everything to make sure Extraction Plugins are as safe as possible, however note that the Extraction Plugin
+SDK is still in beta. Use it at your own risk. For more information on security see Isolation.
+
+
+
Can my Extraction Plugin be embedded into Hansken for performance reasons?ïƒ
+
Embedding an Extraction Plugin into Hansken requires access to the Hansken source code. If you have access to the source
+code then please ../contact us for assistance. Please note that embedded Extraction Plugins are not officially
+supported.
+
If you do not have access to the Hansken source code, then please contact your own Business Owner, and ask them to
+contact the Hansken Team.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/introduction.html b/0.9.1/dev/introduction.html
new file mode 100644
index 0000000..34a61a5
--- /dev/null
+++ b/0.9.1/dev/introduction.html
@@ -0,0 +1,171 @@
+
+
+
+
+
+
+
+
+ Introduction — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
The Extraction Plugin Software Development Kit can be used to develop a Hansken
+extraction plugin.
+
Hansken is designed to give access to and insight in digital data and traces originating from seized and demanded
+digital material. One aspect of the Hansken platform is the extraction engine (extraction framework). The extraction
+engine contains digital forensics knowledge, which is used to find traces in digital material. With extraction plugins
+case investigators can add new digital forensics knowledge to the extraction framework. In this way, Hansken is enabled
+to understand new digital formats, and thus is able to find new types of traces in the seized and demanded material.
+
Examples of digital forensics knowledge that can be added to Hansken with extraction plugins:
+
+
new file formats (e.g. a new crypto currency wallet)
+
combine traces to find new information (e.g. use a windows registry entry required to read a file from disk)
+
apply algorithms on traces (e.g. speech to text from audio files)
+
+
The primary goals of this SDK are:
+
+
to make it as easy as possible to add new digital forensics knowledge to Hansken.
+
be able to share digital forensics knowledge with other Hansken community members
This document summarizes all important API changes in the Extraction Plugin API. This document only shows changes that
+are important to plugin developers. For a full list of changes per version, please refer to the general
+changelog.
The trace property imageId is renamed to image. This is to be in line with the Hansken REST API and Python API.
+When updating your plugin, please update your calls trace.get("imageId") to trace.get("image").
+
#774
+By default, deferred extraction plugin searches are now scoped to the image
+of the trace that is currently being processed. Optionally, a project-wide
+search can be done by passing an optional scope argument.
+
@Override
+publicvoidprocess(finalTracetrace,finalExtractionContextcontext,finalTraceSearchersearcher){
+// only search for traces inside the same image as the trace that is being processed
+finalSearchResultresult=searcher.search("file.extension=asc",10);
+finalSearchResultresult=searcher.search("file.extension=asc",10,TraceSearcher.SearchScope.IMAGE);
+
+// search for all traces inside the same project as the trace that is being processed
+finalSearchResultresult=searcher.search("file.extension=asc",10,TraceSearcher.SearchScope.PROJECT);
+}
+
+
+
+
Support trace properties of type List<Integer>, List<Double>, and List<Float>.
+This enables you to write multiple offsets and confidence scores in tracelets of type prediction.
Escaping the / character in matchers is optional.
+This simplifies and aims for better HQL and HQL-Lite compatability.
+See for more information and examples the HQL-Lite syntax documentation.
Hansken returns file.path properties as a String property, instead of a List<String>.
+Example: trace.get("file.path") now returns "/dev/null", this was ["dev","null"].
A plugin can now write multiple data streams to a single trace concurrently,
+e.g. write both decrypted and ocr at the same time. See the “Adding data to a trace†code snippets for
+general examples on adding data to a trace.
It is highly recommended to upgrade your plugin to this new version.
+See the migration steps below.
+
+
+
Extraction plugin container images are now labeled with PluginInfo. This
+allows Hansken to efficiently load extraction plugins.
+
By default, extraction plugin version is managed in the plugin’s pom.xml.
+The .pluginVersion(..) can be removed from the PluginInfo builder.
+
Migration steps from earlier versions – for plugins that use the Java
+extraction plugin SuperPOM:
+
+
Update the SDK version in your pom.xml
+
If you come from a version prior to 0.4.0, or if you use a plugin name
+instead of a plugin id in your pluginInfo(), switch to the plugin id style
+(read instructions for version 0.4.0)
+
Set your plugin version in your project’s pom.xml, and remove the
+following from your PluginInfo.Builder:
+
.pluginVersion(...)
+
+
+
+
Update your build scripts to build your plugin (Docker) container image.
+You should build your plugin container image with the following command:
+
mvnpackagedocker:build`
+
+
+
This will generate a plugin image:
+
+
The extraction plugin is added to your local image registry
+(dockerimages),
+
The image name is extraction-plugin/PLUGINID, e.g.
+extraction-plugin/nfi.nl/extract/chat/whatsapp,
+
The image is tagged with two tags: latest, and your plugin version.
+
+
Nb. If Docker is not available in your environment, podman can be used
+as an alternative. See packaging for more
+details.
When writing input search traces for tests, it is no longer required to explicitly set an id property.
+These are automatically generated when executing tests.
A new convenience method id(String,String,String) is added to the PluginInfo builder. This removes some
+boilerplate code when setting the pluginId. More details on the plugin naming conventions can be found at the
+Plugin naming convention section.
+
PluginInfo.builderFor(this)
+.id("nfi.nl","extract","TestPlugin")// new style
+.id(newPluginId("nfi.nl","extract","TestPlugin"))// old style, but also works
+...
+
It is now possible to specify maximum system resources in the PluginInfo. To run a plugin with 0.5 cpu (= 0.5
+vCPU/Core/hyperthread) and 1 gb memory, for example, the following configuration can be added to PluginInfo:
Extraction Plugins are now identified with a PluginInfo.PluginId containing a domain, category and name. The
+method PluginInfo.name(pluginName) has been replaced by PluginInfo.id(newPluginId(domain,category,name). More
+details on the plugin naming conventions can be found at the Plugin naming convention section.
+
PluginInfo.name() is now deprecated (but will still work for backwards compatibility).
+
A new license field PluginInfo.license has also been added in this release.
+
The following example creates a PluginInfo for a plugin with the name TestPlugin, licensed under
+the ApacheLicense2.0 license:
Extraction Plugins can now create new datastreams on a Trace through data transformations. Data transformations
+describe how data can be obtained from a source.
+
An example case is an extraction plugin that processes an archive file. The plugin creates a child trace per entry in
+the archive file. Each child trace will have a datastream that is a transformation that marks the start and length of
+the entry in the original archive data. By just describing the data instead of specifying the actual data, a lot of
+space is saved.
+
Although Hansken supports various transformations, the Extraction Plugins SDK for now only supports ranged data
+transformations. Ranged data transformations define data as a list of ranges, each range with an offset and length in
+a bytearray.
+
The following example sets a new datastream with dataType html on a trace, by setting a ranged data transformation:
The following example creates a child trace and sets a new datastream with dataType raw on it, by setting a ranged
+data transformation with two ranges:
This is an API breaking change. Plugins created with an earlier version of the extraction plugin SDK are
+not compatible with Hansken that uses 0.2.0 or later.
+
+
+
Introduced a new extraction plugin type DeferredExtractioPlugin. Deferred Extraction plugins can be run at a
+different extraction stage. This type of plugin also allows accessing other traces using the searcher.
+
The class ExtractionContext has been renamed to DataContext. The new name DataContext represents the class
+contents better. Plugins have to update matching import statements and the type in ExtractionPlugin.process()
+implementation in the same way. This change has no functional side effects.
To debug a plugin locally, it is recommended to start the plugin via the IDE by running the integration test. This has
+the advantage that breakpoints can easily be put in the code instead of printing log statements, for example.
If the Docker image is not built, run the following command to build the Docker image:
+
mvnpackagedocker:build
+
+
+
+
+
Run the Docker image with specific Java tool optionsïƒ
+
In Java, the remote debug functionality is not enabled by default. To enable the remote debug functionality, the
+following environments variable must be set in the Docker container:
This environment variable allows the debugger to connect to the debuggee (application being debugged). To start the
+Docker image with the JAVA_TOOL_OPTIONS environment variable, the following command can be used:
The next step is to attach the debugger to the debuggee.
+For Intellij, the instructions are clearly described on the following
+page: Tutorial: Remote debug
The logging of the extraction plugin is displayed in the console after running the dockerrun command. In addition,
+the logging is also displayed in the IntelliJ console while debugging.
An HQL query can be debugged by overriding the isVerboseLoggingEnabled() method of the ExtractionPluginFlits class.
+The example below shows an example of an embedded FLITS test with verbose logging enabled.
The following output will then be displayed in the console:
+
HQL match found for:
+$data.type=jpg
+With trace:
+dataType=jpg
+types={file, data}
+properties={data.raw.mimeType=image/jpg, path=/test-input-trace, file.name=image.jpg, name=test-input-trace, id=0}
+
+
+
If the HQL query contains an error, it will be shown in the generated test results. An example of an invalid query
+is $data.mimeType=image/jpg (slash not escaped). This query will produce an error like the one shown below.
The error is only shown in the generated trace, so to find out the ParseException override
+the regenerate() method from Flits and then let this method return true.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/java/javadoc.html b/0.9.1/dev/java/javadoc.html
new file mode 100644
index 0000000..8a0a92b
--- /dev/null
+++ b/0.9.1/dev/java/javadoc.html
@@ -0,0 +1,130 @@
+
+
+
+
+
+
+
+
+ Javadoc — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Extraction plugins are packaged as OCI images (also known as Docker images).
+The OCI images are labeled with the PluginInfo.
+To automate packaging of a Java plugin and labeling the OCI image, the Extraction Plugin SuperPom has been configured to automate this for you.
+
If your project uses the Extraction Plugin SuperPom (see Prerequisites), Packaging an extraction plugin is handled by Maven.
+To package your plugin into a container image, the following command can be used:
+
mvnpackagedocker:build
+
+
+
This will generate a plugin image:
+
+
The extraction plugin is added to your local image registry
+(dockerimages),
+
The image name is extraction-plugin/PLUGINID, e.g.
+extraction-plugin/nfi.nl/extract/chat/whatsapp,
+
The image is labeled with two tags: latest, and your plugin version.
+
+
It is possible to apply extra arguments to the docker command as described here.
+For example, to specify a proxy, use the following command:
Once your plugin is packaged, it can be published or ‘uploaded’ to Hansken.
+See “Upload the plugin to Hansken†for instructions.
+
Note: if your build environment does not have Docker available, you can use
+podman as an alternative. Install podman on your machine
+or build agent, and run the following commands before invoking the
+mvnpackagedocker:build command:
Docker for packaging and publishing plugins
+(or use a Docker alternative such as podman)
+
Maven (recommended, build automation tool)
+
+
Required dependencies:
+
+
All required project dependencies to build extraction plugins are published on the public Maven Central, under org.hansken.plugin.extraction:plugin-super-pom.
+For maven based extraction plugins, the following pom.xml snippet can be used as basis of a plugin:
In Java, InputStream is a common type to pass data to another class or method. The SDK provides a simple utility to
+use a RandomAccessData as InputStream.
Next we can create an InputStream from the RandomAccessData as shown in the following snippet. Note that the
+InputStream is created using a try-with-resources-statement. This ensures that the InputStream is correctly closed
+when the InputStream is no longer required.
+
RandomAccessDatatraceData=...;
+try(InputStreamasInputStream=RandomAccessDatas.asInputStream(traceData)){
+// use the InputStream here
+}
+
+
+
Notes:
+
+
the created InputStream is not thread-safe,
+
the created InputStream changes state in the provided RandomAccessData
+(e.g. when data is read, the position updated in both the InputStreamand
+the RandomAccessData instances),
+
for more details on the implementation of the InputStream, refer to the RandomAccessDataInputStream JavaDoc.
In the following Java example, a “classification†tracelet is added to a trace. The tracelet consists
+of a list of four properties, namely “classâ€, “confidenceâ€, “modelName†and “modelVersionâ€.
The following example creates a child trace and sets a new datastream with dataType raw on it, by setting a ranged
+data transformation with two ranges:
It is not always possible to create a transormation for the data that has to be
+added to a trace. For example the data is a result of a computation, and not
+a direct subset of another data stream..
+
The following examples show how to creates a new data stream of dataType raw on a trace.
+
In case all data is stored in a byte[], we can add the byte array to the data stream with:
In the PluginInfo you can specify maximum system resource metrics for a plugin. These are used for scaling the
+number of pods as described here. To run a plugin with 0.5 cpu (=
+0.5 vCPU/Core/hyperthread), 1 gb memory and 10 (concurrent) cpu workers (threads), for example, the following configuration can be added to PluginInfo:
Using a deferred plugin requires inheriting the DeferredExtractionPlugin base class. This allows access to
+a TraceSearcher object in the process function to search for traces.
a HQL query (note: this is the traditional HQL query, and not the matchers HQL-lite variant),
+
the maximum number of traces the return (currently hard-limited to a maximum of 50 traces),
+
(optional) a scope, which can be either TraceSearcher.SearchScope.IMAGE (default), or TraceSearcher.SearchScope.PROJECT.
+When set to IMAGE, the searcher will only search for traces within the same image as the trace that is being processed.
+
+
The traces contained in the SearchResult are returned as a stream.
The logging is provided by Log4j 2 with a SLF4J binding. The Log4j 2 SLF4J binding allows applications coded to the
+SLF4J API to use Log4j 2 as the implementation.
Here is an example illustrating how to log something with SLF4J. It begins by getting a logger with the name “LOGâ€. This
+logger is in turn used to log the message I'mloggingavariable1234!.
+
importorg.slf4j.Logger;
+importorg.slf4j.LoggerFactory;
+
+publicclassExample{
+privatestaticfinalLoggerLOG=LoggerFactory.getLogger(Example.class);
+
+publicvoidexample(){
+finalintaNumber=1234;
+// logs to console: I'm logging a variable 1234!
+LOG.info("I'm logging a variable {}!",aNumber);
+}
+}
+
It’s easy to change the logging format with a file called log4j2.xml. If desired, this file must be in the resources
+folder, for example src/main/resources/log4j2.xml
More information about customizing the logging can be found here.
+
+
+
Note
+
The default logger is pre-configured to log INFO to STDOUT (see the configuration above)
+
+
+
Note
+
Log4j 2 supports various logging formats, including xml, yaml, json, properties, etc.
+Currently, only the xml format is supported.
+
+
+
Note
+
Contact your Hansken administrator for more information on where to find logs for your Hansken environment.
+
+
+
+
+
[EXPERIMENTAL FEATURE] Adding previews to a traceïƒ
+
+
Warning
+
This is an experimental feature, which might change or get removed in future releases.
+
+
Example:
+
publicclassExamplePluginextendsExtractionPlugin{
+@Override
+publicPluginInfopluginInfo();
+
+@Override
+publicvoidprocess(finalTracetrace,finalDataContextcontext){
+finalbyte[]previewData;
+// set the preview data for the image/png MIME-type
+trace.set("preview.image/png",previewData);
+trace.set("preview.image/png",previewData);
+}
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/java/testing.html b/0.9.1/dev/java/testing.html
new file mode 100644
index 0000000..a97e289
--- /dev/null
+++ b/0.9.1/dev/java/testing.html
@@ -0,0 +1,287 @@
+
+
+
+
+
+
+
+
+ Using the Test Framework in Java — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
There are ways of integration testing a plugin with the Test Framework:
+
+
Embedded testing: Here the plugin is run directly from a JUnit test without using the gRPC layer.
+
Remote testing: Here the test will start an ExtractionPluginServer that will serve the plugin. All communication
+is between server and plugin is done using gRPC.
+
+
See below for an example of each way of testing.
+The Extraction Plugin Examples contains many
+more examples.
import staticnl.minvenj.nfi.flits.util.FlitsUtil.srcPath;
+
+importjava.nio.file.Path;
+
+importorg.hansken.plugin.extraction.api.ExtractionPlugin;
+importorg.hansken.plugin.extraction.test.EmbeddedExtractionPluginFlits;
+
+/**
+ * An integration test for MyPlugin.
+ */
+classMyPluginITextendsEmbeddedExtractionPluginFlits{
+
+@Override
+protectedExtractionPluginpluginToTest(){
+// MyPlugin is a class implementing the ExtractionPlugin interface,
+// with pluginInfo() and process() methods.
+returnnewMyPlugin();
+}
+
+@Override
+publicPathtestPath(){
+// Provide the folder containing input files. For examples, see
+// https://git.eminjenv.nl/hanskaton/hansken-extraction-plugin-sdk/examples.
+returnsrcPath("integration/inputs");
+}
+
+@Override
+publicPathresultPath(){
+// Provide the folder containing result files. For examples, see
+// https://git.eminjenv.nl/hanskaton/hansken-extraction-plugin-sdk/examples.
+returnsrcPath("integration/results");
+}
+
+@Override
+publicbooleanregenerate(){
+// Returning false means the test will fail if the result files don't
+// match the outcome of the=++ test. Returning true means the test create
+// new result files .
+returnfalse;
+}
+}
+
Note that the following example serves the plugin by using ExtractionServer.
+
import staticnl.minvenj.nfi.flits.util.FlitsUtil.srcPath;
+
+importjava.nio.file.Path;
+
+importorg.hansken.plugin.extraction.runtime.grpc.client.ExtractionPluginClient;
+importorg.hansken.plugin.extraction.runtime.grpc.server.ExtractionPluginServer;
+importorg.hansken.plugin.extraction.test.plugins.DataTransformationsPlugin;
+importorg.junit.jupiter.api.AfterAll;
+importorg.junit.jupiter.api.BeforeAll;
+
+publicclassRemoteTransformationPluginFlitsITextendsRemoteExtractionPluginFlits{
+
+privatestaticExtractionPluginServer_server;
+privatestaticExtractionPluginClient_client;
+
+@BeforeAll
+publicstaticvoidinit()throwsException{
+finalintport=8999;
+
+// Serve MyPlugin.
+// MyPlugin is a class implementing the ExtractionPlugin interface, with PluginInfo and Process methods.
+_server=ExtractionPluginServer.serve(port,MyPlugin::new);
+
+// Create an ExtractionPluginClient
+_client=newExtractionPluginClient("localhost",_server.getListeningPort());
+}
+
+@AfterAll
+publicstaticvoiddestruct(){
+// At the end of the test, make sure the server and client are closed.
+if(_server!=null){
+_server.close();
+}
+if(_client!=null){
+_client.close();
+}
+}
+
+@Override
+publicPathtestPath(){
+// Provide the folder containing input files. For examples, see https://git.eminjenv.nl/hanskaton/hansken-extraction-plugin-sdk/examples.
+returnsrcPath("integration/inputs");
+}
+
+@Override
+publicPathresultPath(){
+// Provide the folder containing result files. For examples, see https://git.eminjenv.nl/hanskaton/hansken-extraction-plugin-sdk/examples.
+returnsrcPath("integration/results");
+}
+
+@Override
+protectedExtractionPluginClientpluginToTest(){
+// For Remote testing, the test won't talk directly to the plugin, but to the client.
+// The client will use gRPC to communicate with the served plugin.
+return_client;
+}
+
+@Override
+publicbooleanregenerate(){
+// Returning false means the test will fail if the result files don't match the outcome of the test.
+// Returning true means the test create new result files.
+returnfalse;
+}
+}
+
+
+
+
Note
+
Note that with a RemoteTransformationPluginFlitsIT it is possible to start a docker image of a plugin and
+run remote tests against it using your own testdata. To do this, simply remove all _server code and manually start
+your plugin in a docker container. Then run the test against the docker container by setting the correct url and
+port, presumably newExtractionPluginClient("localhost",8999).
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/python.html b/0.9.1/dev/python.html
new file mode 100644
index 0000000..48bf990
--- /dev/null
+++ b/0.9.1/dev/python.html
@@ -0,0 +1,158 @@
+
+
+
+
+
+
+
+
+ Python — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Update or add metadata properties for this .ExtractionTraceBuilder.
+
Can be used to update the name of the Trace represented by this builder,
+if not already set.
+
+
Parameters:
+
+
key_or_updates – either a str (the metadata property to be
+updated) or a mapping supplying both keys and values to be updated
+
value – the value to update metadata property key to (used
+only when key_or_updates is a str, an exception will be thrown
+if key_or_updates is a mapping)
+
data – a dict mapping data type / stream name to bytes to be
+added to the trace
Update or add metadata properties for this .ExtractionTrace.
+
+
Parameters:
+
+
key_or_updates – either a str (the metadata property to be
+updated) or a mapping supplying both keys and values to be updated
+
value – the value to update metadata property key to (used
+only when key_or_updates is a str, an exception will be thrown
+if key_or_updates is a mapping)
+
data – a dict mapping data type / stream name to bytes to be
+added to the trace
CPU resources are measured in cpu units. One cpu is equivalent to 1 vCPU/Core for cloud providers and 1 hyperthread
+on bare-metal Intel processors. Also, fractional requests are allowed. A plugin that asks 0.5 CPU uses half as
+much CPU as one that asks for 1 CPU.
TransformerLabel contains information about a transformer method that a plugin provides.
+
It is mainly used for storing the properties (name, arguments, return type) of a transformer in PluginInfo objects.
+Unlike the Transformer class it does not contain the actual function reference to the transformer itself.
Search for indexed traces in Hansken using provided query returning at most count results.
+
+
Parameters:
+
+
query – HQL-query used for searching
+
count – Maximum number of traces to return
+
scope – Select search scope: ‘image’ to search only search for other traces within the image of the trace
+that is being processed, or ‘project’ to search in the scope of the full project (either Scope-
+enum value can be used, or the str-values directly).
+
+
+
Returns:
+
SearchResult containing found traces
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/python/api/hansken_extraction_plugin.api.tracelet.html b/0.9.1/dev/python/api/hansken_extraction_plugin.api.tracelet.html
new file mode 100644
index 0000000..7254b79
--- /dev/null
+++ b/0.9.1/dev/python/api/hansken_extraction_plugin.api.tracelet.html
@@ -0,0 +1,171 @@
+
+
+
+
+
+
+
+
+ hansken_extraction_plugin.api.tracelet — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
A tracelet contains the values of a single fvt (Few Valued Type).
+
A few valued type is a trace property type that is a collection of tracelets. A trace can contain multiple few
+valued types containing one or more tracelets. For example, the trace.identity` type may look like this:
This module contains the Transformer class that holds the function reference of the transformer.
+
Instances of this class are constructed by BaseExtractionPlugin when retrieving transformers dynamically.
+It also validates whether the method to which @transformer is applied adheres to the requirements of a transformer.
Generate a TransformerLabel given the transformer method. TransformerLabels are used in PluginInfo objects.
+
Unlike Transformers TransformerLabels can be serialized and sent to a client that wishes to call a transformer.
+The specific Python types are converted to the generic types that are used in the Hansken trace model.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/python/api_changelog.html b/0.9.1/dev/python/api_changelog.html
new file mode 100644
index 0000000..3ab367c
--- /dev/null
+++ b/0.9.1/dev/python/api_changelog.html
@@ -0,0 +1,638 @@
+
+
+
+
+
+
+
+
+ Python API Changelog — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
This document summarizes all important API changes in the Extraction Plugin API. This document only shows changes that
+are important to plugin developers. For a full list of changes per version, please refer to the general
+changelog.
This release introduces a simple flow-control mechanism that fixes connectionRST errors that can occur when a
+plugin produces traces too fast. Plugins built with this version are not backwards compatible.
+
Python plugins now support transformers, remote methods which can be executed using the Hansken REST API. More information can be found in the docs.
+
The build_plugin and label_plugin utilities prematurely shut down containers if the building and labeling process takes too long causing the process for slow containers to fail. If your plugin takes a long time to start, you may want to increase the timeout before the script stops trying to connect and aborts the process of building the plugin. This can be done using the new optional --timeout argument. The default is set to 30 seconds.
+
The optional image name argument of build_plugin is changed to a flag. Build scripts can be updated using --target-nameDOCKER_IMAGE_NAME.
âš ï¸ This release is deprecated, please upgrade to 0.8.3
+
The build_plugin utility has been updated and the deprecation status has been removed.
+As with label_plugin, build_plugin now no longer requires a full (virtual) environment
+with all plugin dependencies and resources. This will greatly reduce build times for plugins with
+big dependencies and/or large models.
+
The first argument of the command (a pointer to your plugin.py file) has been removed.
+Please do not forget to remove the first argument of build_plugin in your tox.ini or other build tooling.
The default read-buffer of trace.open('rb') as been changed from 1 Megabyte to 6 Megabyte to reduce overhead while data reading.
+
The data stream writer of trace.open('wb') is now buffered as well. This means that multiple small writes will be flushed after every 6 Megabytes of data has been written (or when the writer is closed).
+
The read-buffer or write-buffer size can be overridden by the user, by passing the buffer_size= argument to trace.open():
+
withtrace.open('rb',buffer_size=1024*1024):# set a 1 Megabyte buffer size
+ pass
+
+withtrace.open('wb',buffer_size=1024*1024*12):# set a 12 Megabyte buffer size
+ pass
+
+withtrace.open('wb',buffer_size=1):# a buffer_size of 1 effectively disables the buffer:
+ pass# each write will be flushed to Hansken directly
+
+
+
+
It is now possible to write str values to trace.open(..). To do so, pass mode='w' as additional argument.
+By default, it is assumed that the written text is ‘utf-8’ encoded. The default can be overwritten by using the 'encoding=' argument.
+
In a future Hansken update, Hansken will set the correct data-stream properties for your text stream (mimeType, mimeClass, and fileType).
+
Example use cases are:
+
+
write picture-to-text (OCR) data to a trace
+
write translations to a trace
+
write audio-to-text (audio transcriptions) to a trace
+
write the results of a JSON dump, e.g.: json.dump(['your','data'],text_writer)
+
+
Examples in code:
+
withtrace.open(data_type='raw',mode='w',encoding='utf-8')astext_writer:
+ text_writer.write('hello.world')# write strings directly to it
+ json.dump({'hello':'world'},text_writer)# or pass the writer to json.dump
+
The trace property imageId is renamed to image. This is to be in line with the Hansken REST API and Python API.
+When updating your plugin, please update your calls trace.get('imageId') to trace.get('image').
+
#774
+By default, deferred extraction plugin searches are now scoped to the image
+of the trace that is currently being processed. Optionally, a project-wide
+search can be done by passing an optional scope argument.
+
defprocess(trace,data_context,searcher):
+ # only search for traces inside the same image as the trace that is being processed
+ searcher.search('*')
+ searcher.search('*',scope='image')# explicit alternative, using a str
+ searcher.search('*',scope=SearchScope.image)# explicit alternative, using the SearchScope enum
+
+ # only search for traces inside the same image as the trace that is being processed
+ searcher.search('*',scope='project')
+ searcher.search('*',scope=SearchScope.project)
+
+
+
+
Support trace properties of type list[float]. This enables you to write
+multiple offsets and confidence scores in tracelets of type prediction.
+
For example:
+
trace.add_tracelet('prediction',{
+ 'modelName':'my_cat_detector',
+ 'modelVersion':'0.0.BETA',
+ 'type':'classification',
+ 'label':'cat',
+
+ # the best score
+ 'offset':3.0,
+ 'confidence':0.4,
+
+ # all scores
+ 'offsets':[0.0,3.0,6.0,9.0],
+ 'confidences':[0.1,0.4,0.03,0.09],
+})
+
This version introduces a new docker image build utility label_plugin.
+This utility will eventually replace build_plugin. build_plugin is now deprecated.
+
label_plugin is a utility to add labels to an extraction plugin image. Labeling a plugin is required for
+Hansken to detect extraction plugins in a plugin image registry.
+
To label a plugin, first build the plugin image with docker build;
+for example by using one of the following commands:
Next, run the label_plugin utility to label the build plugin container:
+
label_pluginmy_plugin
+
+
+
The result of label_plugin is a plugin image that can be uploaded to Hansken.
+
label_plugin is preferred over build_plugin, as it does not require a full (virtual) environment
+with all plugin dependencies and resources. This is especially preferred when the plugin uses (big)
+data models or (external) dependencies.
Escaping the / character in matchers is optional.
+This simplifies and aims for better HQL and HQL-Lite compatability.
+See for more information and examples the HQL-Lite syntax documentation.
Hansken returns file.path properties (outside the scope of matchers) as a String property,
+instead of a list of strings.
+Example: trace.get('file.path') now returns '/dev/null', this was ['dev','null'].
+
Improved plugin loading when using serve_plugin and build_plugin:
+import statements now work for modules (python files) that are located the same directory structure of a plugin.
+
A plugin can now stream data to a trace using trace.open(mode='wb').
+This removes the limit on the size of data that could be written.
+See also the python code snippet.
The docker image build script build_plugin has been updated to allow for extension of the docker command.
+This can be especially handy for specifying a proxy. You should build your plugin container image with the following
+command:
This is an API breaking change.
+Upgrading your plugin to this version will require code changes.
+Plugins built with previous versions of the SDK from 0.3.0 will still work with Hansken.
+
+
+
Warning
+
It is strongly recommended to upgrade your plugins to this new version because it significantly improves
+the start-up time of Hansken. See the migration steps below.
+
+
This release contains both build pipeline changes and API changes.
+Please read all changes carefully.
Extraction plugin container images are now labeled with PluginInfo. This
+allows Hansken to efficiently load extraction plugins.
+Migration steps from earlier versions:
+
+
Update the SDK version in your setup.py / requirements.txt
+
If you come from a version prior to 0.4.0, or if you use a plugin name
+instead of a plugin id in your pluginInfo(), switch to the plugin id style
+(read instructions for version 0.4.0)
+
Update your build scripts to build your plugin (Docker) container image.
+Be sure to have the Extraction Plugins SDK installed.
+Then, you should build your plugin container image with the following command:
# no need for a builder, declare resources by direct instantiation
+returnPluginInfo(
+ ...,
+ resources=PluginResources(maximum_cpu=2.0,maximum_memory=2048)
+)
+# or, as before, specify just on resource
+returnPluginInfo(
+ ...,
+ resources=PluginResources(maximum_memory=4096)
+)
+
Simplify tracelet properties by making the tracelet type prefix optional.
+
# using a Tracelet object
+trace.add_tracelet(Tracelet("prediction",{
+ "type":"example",
+ "confidence":0.8
+}))
+# or without a Tracelet object
+trace.add_tracelet("identity",{"name":"John Doe","status":"online"})
+
When writing input search traces for tests, it is no longer required to explicitly set an id property.
+These are automatically generated when executing tests.
More $data matchers are supported in Hansken.py plugin runner. Before this improvement it was only possible to match
+on $data.type. Now it is also possible to match for example on $data.mimeType and $data.mimeClass. The $data
+matcher should still be at the end of the query as before.
It is now possible to specify maximum system resources in the PluginInfo. To run a plugin with 0.5 cpu (= 0.5
+vCPU/Core/hyperthread) and 1 gb memory, for example, the following configuration can be added to PluginInfo:
Extraction Plugins are now identified with a PluginInfo.PluginId containing a domain, category and name. The
+method PluginInfo.name(pluginName) has been replaced by PluginInfo.id(newPluginId(domain,category,name). More
+details on the plugin naming conventions can be found at the Plugin naming convention section.
+
PluginInfo.name() is now deprecated (but will still work for backwards compatibility).
+
A new license field PluginInfo.license has also been added in this release.
+
The following example creates a PluginInfo for a plugin with the name TestPlugin, licensed under
+the ApacheLicense2.0 license:
Extraction Plugins can now create new datastreams on a Trace through data transformations. Data transformations
+describe how data can be obtained from a source.
+
An example case is an extraction plugin that processes an archive file. The plugin creates a child trace per entry in
+the archive file. Each child trace will have a datastream that is a transformation that marks the start and length of
+the entry in the original archive data. By just describing the data instead of specifying the actual data, a lot of
+space is saved.
+
Although Hansken supports various transformations, the Extraction Plugins SDK for now only supports ranged data
+transformations. Ranged data transformations define data as a list of ranges, each range with an offset and length in
+a bytearray.
+
The following example sets a new datastream with dataType html on a trace, by setting a ranged data transformation:
The following example creates a child trace and sets a new datastream with dataType raw on it, by setting a ranged
+data transformation with two ranges:
This is an API breaking change.
+Plugins created with an earlier version of the extraction plugin
+SDK are not compatible with Hansken that uses 0.2.0 or later.
+
+
+
Introduced a new extraction plugin type api.extraction_plugin.DeferredExtractioPlugin.
+Deferred Extraction plugins can be run at a different extraction stage.
+This type of plugin also allows accessing other traces using the searcher.
+
The class api.extraction_context.ExtractionContext has been renamed to api.data_context.DataContext.
+The new name DataContext represents the class contents better.
+Plugins have to update matching import statements accordingly.
+Plugins should also update the named argument context to data_context of the plugin process() method.
+This change has no functional changes.
Moved api.author.Author to api.plugin_info.Author, and moved api.maturity_level.MaturityLevel
+to api.plugin_info.MaturityLevel
+This is a more pythonic way of grouping of classes into modules. This change has no functional side effects.
+
Plugins have to update matching import statements accordingly.
Removed DataContext.get_first_bytes() from the public API.
+
Removed api.extraction_trace.validate_update_arguments(..) from the public API. This method is still invoked
+implicitly when setting trace properties.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/python/debugging.html b/0.9.1/dev/python/debugging.html
new file mode 100644
index 0000000..309f436
--- /dev/null
+++ b/0.9.1/dev/python/debugging.html
@@ -0,0 +1,303 @@
+
+
+
+
+
+
+
+
+ How to debug an Extraction Plugin — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
To debug a plugin locally, it is recommended to start the plugin via the IDE. This has the advantage that breakpoints
+can easily be put in the code instead of printing log statements, for example. To start a plugin locally, a piece of
+code must be added, see Testing for more information.
Debugging an extraction plugin via docker is a bit trickier. In order to debug in Python, a debugger must be added to
+the extraction plugin. There are several debug modules for Python available, but one debug module that works well with
+Visual Studio Code is debugpy. This package is developed by Microsoft
+specifically for use in Visual Studio Code with Python.
+
+
Note
+
debugpy implements the Debug Adapter Protocol (DAP), which is a standardised way for development tools to
+communicate with debuggers.
+
+
Using debugpy with Docker containers requires 4 distinct steps:
+
+
Install debugpy
+
Configuring debugpy in Python
+
Build a docker image
+
Configuring the connection to the Docker container
At the beginning of your script, import debugpy, and call debugpy.listen() to start the debug adapter, passing
+a (host,port) tuple as the first argument. Use the debugpy.wait_for_client() function to block program execution
+until the client is attached.
+
importdebugpy
+
+debugpy.listen(("0.0.0.0",5678))
+debugpy.wait_for_client()# blocks execution until client is attached
+
+# your extraction plugin code
+
If the Docker image is not built, first build the image as described
+here.
+
+
+
Configuring the connection to the Docker containerïƒ
+
debugpy is now set up to accept connections inside a Docker container. To connect to debugpy in the docker
+container, port 5678 must be published. To make a port available to services outside of Docker, use the –publish or -p
+flag. This creates a firewall rule which maps a container port to a port on the Docker host to the outside world.
+
To run the extraction plugin with the published port the following command can be used:
+
dockerrun-p5678:5678your_extraction_plugin_name
+
+
+
The next step is to configure Visual Studio Code. A launch.json file must be created in order for Visual Studio Code
+to connect to the extraction plugin in Docker. This minimal launch.json example below tells the debugger to attach
+to localhost on port 5678.
The logging of the extraction plugin is displayed in the console after running the dockerrun command. In addition,
+the logging is also displayed in the Visual Studio Code console while debugging.
An HQL query can be debugged by running the test framework with the --verbose option enabled. The found HQL matches
+will then be displayed in the console. To test a plugin in python with the --verbose option enabled use the following
+command:
The following output will then be displayed in the console:
+
HQL match found for:
+$data.type=jpg
+With trace:
+dataType=jpg
+types={file, data}
+properties={data.raw.mimeType=image/jpg, path=/test-input-trace, file.name=image.jpg, name=test-input-trace, id=0}
+
+
+
If the HQL query contains an error, it will be shown in the generated test results. An example of an invalid query
+is $data.mimeType=image/jpg (slash not escaped). This query will produce an error like the one shown below.
The following section describes how to set up a fully working development environment for extraction plugins with Python.
+This is written for those who are not comfortable setting up a working build environment.
+This is optional; advanced users may choose a different development environment setup, and can skip this section completely.
+
If you fail to set up a development environment, feel free to ask for help at our Discord channel.
In order to be able to develop Hansken Extraction Plugins in Python on Ubuntu, the following build tools need be installed on your system: python, pip, tox, Java, Docker.
+
+
Python, pip, tox, Java
+To install, run the following commands in your terminal:
You can check whether all versions are installed correctly by running the following commands (and validate the output):
+
python--version
+# should return version 3.10.15 or higher
+
+pip3--version
+# should return version 20.0.2 or higher
+
+tox--version
+# should return version 3.11.2 or higher
+
+java-version
+# should return version 11.0.4 or higher
+
+
+
+
+
If the above software is installed, you can continue with “Install Dockerâ€,
+or if you don’t want to install Docker, continue with “Install your IDE: Pycharmâ€.
In order to be able to develop Hansken Extraction Plugins in Python on Windows, follow the next steps:
+
+
For verification of the installation of each program we will use commands on the command prompt.
+This can be opened by clicking the Windows Start button and typing:
+
cmd
+
+
+
Then hit Enter . This will open the command prompt where you can enter the commands given in the following steps.
+
+
Python 3.10 or higher & pip
+
Download the installer from python.org/Downloads (click the yellow “Download Python†button)
+and run it to install Python and pip.
+Pip is the standard package manager for Python.
+It allows you to install and manage additional packages that are not part of the Python standard library, like the Extraction Plugins SDK.
+
Be sure to select the option “Add python to PATHâ€.
+
When the installation is complete, verify the installation by checking the Python and pip versions:
+
python--version
+# should return your downloaded Python version (>3.10.4)
+
+pip3--version
+# should return 20.2.3 or higher
+
+
+
+
tox
+Tox is used to automate and standardize testing in Python. Use Pip to install Tox with this command:
+
pipinstalltox
+
+
+
Verify that Tox is installed by running:
+
tox--version
+# should return 3.23.0 or higher
+
+
+
+
Java JDK 11.0.4 (or higher)
+Java JDK 11.0.4 or higher is needed to run the test framework of the Extraction Plugins SDK.
+This enables you to test without actually deploying the plugin in Hansken.
+Installing Java on Windows can be done in many different way, but for now we can not recommend one method.
+Please have a look at Install the Microsoft Build of OpenJDK for more details.
+
java-version
+# should return 11.0.4 or higher
+
+
+
N.b. Make sure to set the environment variable in case the installed JDK is not detected by the system.
+You can follow the below-mentioned steps to set the environment variable:
+
+
Click the Windows start button
+
Type “advanced system settingsâ€
+
Hit enter
+
Now click on Environment Variables, select Path under System Variables section and click on Edit. We need to add the path of installed JDK to system Path.
+
Click on New Button and add the path to installed JDK bin which is C:javajava-11jdk-11.0.4bin in our case.
+
Press OK Button 3 times to close all the windows. This sets the JDK 11 on system environment variables to access the same from the console.
+
+
+
+
If the above software is installed, you can continue with “Install Dockerâ€,
+or if you don’t want to install Docker, continue with “Install your IDE: Pycharmâ€.
Installing Docker is a bit more complicated. The Docker website describes the installation instructions in detail.
+Please follow the instructions on docs.docker.gom/get-docker/ to install docker.
+
+
To check you have Docker installed correctly, run
+
docker--version
+# should return version 20.10.17 or higher
+
+
+
+
+
Note: if you run Docker inside a managed network, you might also need to configure proxies and/or certificates.
+Please contact your system administrator if you need help with this.
To verify that your system has been setup correctly, you can run the test suite in the Extraction Plugin Skeleton:
+
+
First, press alt-F12 at the same time to open a terminal in PyCharm in the project root folder.
+
To run the tests of the Skeleton, run this command in the terminal from the root folder:
+
tox
+
+
+
The first time running tox may take a few minutes! Please be patient 😊
+
Tox will install all required plugin dependencies, and start your tests.
+N.b. The plugin template demonstrates how to build a plugin with tox.
+Some plugin developers choose a different tool than tox.
+
+
If your system has been set up correctly, the output should end with a summary like this:
+
py38:commandssucceeded
+congratulations:)
+
+
+
This means the setup is finished. You now have everything installed to start coding your own plugin!
Hansken.py is a Python client to Hansken’s REST API, developed and maintained by the Netherlands Forensic Institute.
+
With Hansken.py, you can run your Python plugin on a project on your Hansken installation that has already been
+extracted. This way of running your plugin is useful during your plugin development. It is not required to upload your
+plugin to Hansken and start an extraction, making the development cycle faster. However, please note that this is only
+useful during the development stage of your plugin, as the execution of your plugin will be much slower compared to
+running the plugin during a Hansken extraction.
+
+
Note
+
The execution of your plugin will be much slower in Hansken.py compared to running the plugin during a Hansken
+extraction.
+
+
+
How to run python extraction plugins standalone with Hansken.pyïƒ
+
Running python extraction plugins standalone with Hansken.py is easy. It is just one command. This section explains how
+to setup this command for your specific environment.
Before you can enter the command that runs your extraction plugin with Hansken.py, you need to find three values:
+
+
HANSKEN_PROJECT_ID project id on which you want to run your plugin
+
YOUR_GATEKEEPER_URL the URL to the Hansken gatekeeper
+
YOUR_KEYSTORE_URL the URL to your keystore
+
+
The correct values of these variables can be found in the Expert UI. Go to the search-page of your project in the
+ExpertUI. Next to the search-bar hit the button “Save queryâ€.
+
+
A new dialog shows up. At the bottom of this dialog, you will find your gatekeeper and keystore urls as well as the
+project id.
Next, open a terminal in the project root folder of your plugin and enter the following command. It will run your
+extraction plugin with Hansken.py in Hansken. Replace the three variables with their respective values.
If your command runs well, you might be prompted for your username and password. There will be some output (note that
+the output may vary depending on your system setup and project content):
+
[2021-03-16 12:59:45.344248+0000] INFO: hansken.auth: selected IDP ID (...) with SOAP endpoint (...)
+[2021-03-16 12:59:45.344450+0000] WARNING: hansken.auth: IDP url known, user+pass auth required but no username supplied
+username []: testaccount
+[2021-03-16 12:59:48.423245+0000] INFO: hansken.auth: user acknowledged environment username or supplied custom username: testaccount
+password for user testaccount:
+[2021-03-16 12:59:53.799668+0000] INFO: hansken.auth: identity provider url and user+pass provided or known, using Keycloak SAML with Basic auth
+[2021-03-16 12:59:53.805538+0000] INFO: hansken_extraction_plugin.runtime.extraction_plugin_runner: PluginRunner is running plugin class Plugin
+[2021-03-16 12:59:53.859299+0000] INFO: hansken.auth: posting SAML request with authorization for user testaccount to IDP endpoint (...)
+[2021-03-16 12:59:54.240290+0000] INFO: plugin.extraction_plugin: processing trace 54197e67-8135-40c3-93f1-3d73a5552693
+[2021-03-16 12:59:54.240753+0000] INFO: plugin.extraction_plugin: processing trace OCRimage
+[2021-03-16 12:59:54.240753+0000] INFO: plugin.extraction_plugin: processing trace (...)
+
+
+
Note that the arguments -v and -l- are passed to enable logging. To find out what other options can be passed to
+this command, please have a look at the hansken.py documentation, or simply run the following command:
Extraction plugins are packaged as OCI images (also known as Docker images).
+The OCI images are labeled with the PluginInfo.
+To automate packaging of a Python plugin and labeling the OCI image,
+the Extraction Plugin SDK comes with two utility applications: label_plugin,
+and build_plugin.
+
To package a plugin, make sure that the Extraction Plugins SDK is installed, as well as Docker.
+Next build and label your plugin as described in the following sections.
+
To verify that the image has been built, use the following command to view all local images:
+
dockerimages
+
+
+
Once your plugin is packaged and labelled, it can be published or ‘uploaded’ to Hansken.
+See “Upload the plugin to Hansken†for instructions.
label_plugin is a utility to add labels to an extraction plugin image.
+To label a plugin, first build the plugin image with docker build;
+for example by using one of the following commands:
Next, run the label_plugin utility to label the build plugin container:
+
label_pluginmy_plugin
+
+
+
This utility will briefly start your plugin using Docker, and requests the PluginInfo from the plugin.
+The information from the PluginInfo will be added as labels to the plugin image.
+The result of label_plugin is a plugin image that can be published to a docker/OCI image registry.
The build_plugin extends label_plugin by also taking care of the docker build command.
+Use this as an one-liner to both build and label your plugin image.
+
To build your plugin container image you can use the following command:
The extraction plugin is added to your local image registry (dockerimages),
+
Note that the variables $http_proxy and $https_proxy are put in quotes, this is needed in case they contain
+spaces,
+
The image is tagged with two tags: latest, and your plugin version.
+
+
Arguments:
+
+
DOCKER_FILE_DIRECTORY: Path to the directory containing the Dockerfile of the plugin.
+
(Optional) DOCKER\_IMAGE\_NAME: Name of the docker image without tag. Note that docker image names cannot start with
+a period or dash. If it starts with a dash, it will be interpreted as an additional docker argument (see
+DOCKER_ARGS). If no name is given the name defaults to extraction-plugin/PLUGINID, e.g.
+extraction-plugin/nfi.nl/extract/chat/whatsapp.
+
(Optional) DOCKER\_ARGS: Additional arguments for the docker command, which can be as many arguments as you like.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/0.9.1/dev/python/prerequisites.html b/0.9.1/dev/python/prerequisites.html
new file mode 100644
index 0000000..e9ac4f6
--- /dev/null
+++ b/0.9.1/dev/python/prerequisites.html
@@ -0,0 +1,139 @@
+
+
+
+
+
+
+
+
+ Prerequisites — Hansken Extraction Plugins for plugin developers SNAPSHOT
+ documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Use update
+to add trace types and their properties to an
+ExtractionTrace.
+Example:
+
defprocess(self,trace,data_context):
+ # get the name of the file
+ file_name=trace.get('file.name')
+ # set the chat application property on the trace
+ trace.update('chatConversation.application',f'DemoApp {file_name}')
+
+
+
All types and properties that can be set are defined in the Hansken trace model.
If the information, which must be added as a property, does not match any of the existing properties of Hansken trace
+model, use the category “misc†(miscellaneous). When part of the category “miscâ€, any name can be given to a property.
+The values of miscellaneous properties are expected to be of data-type string. Example:
+
defprocess(self,trace,data_context):
+ trace.update({
+ 'file.misc.notes':'Some additional notes about the file trace.',
+ 'file.misc.anyName':'Even more notes.'
+ })
+
In the following Python example, a “prediction†tracelet is added to a trace. The tracelet consists
+of a list of four properties, namely “classâ€, “confidenceâ€, “modelName†and “modelVersionâ€.
The following example creates a child trace and sets a new datastream with dataType raw on it, by setting a ranged
+data transformation with two ranges:
It is not always possible to create a transformation for the data that has to be
+added to a trace. For example, if the data is a result of a computation, and not
+a direct subset of another data stream..
+
The following snippet shows how to create a new data stream of dataType raw on a trace from a blob stored in bytes:
Streaming data does not work with the Hansken.py runner because Hansken.py does not support it. It does
+work when running your plugin in Hansken and in the test framework.
+
+
When dealing with large quantities of data, it is possible to keep the memory usage
+of the plugin within manageable limits by streaming the data from the plugin to Hansken in smaller chunks.
+To do this, use the withtrace.open(data_type=...,mode='wb') syntax. Here are some examples:
To write str values directly, use mode w (or wt).
+By default, it is assumed that the written text is ‘utf-8’ encoded. The default encoding can be overwritten by using the 'encoding=' argument.
+
(In a future Hansken update) Hansken will set the correct data-stream properties for your text stream (mimeType, mimeClass, and fileType).
+
withtrace.open(data_type='raw',mode='w',encoding='utf-8')astext_writer:
+ text_writer.write('hello.world')# write strings directly to the writer
+ json.dump({'hello':'world'},text_writer)# or pass the writer to json.dump
+
+
+
It is recommended to pass utf-8 explictly as encoding.
It is possible to specify system resources hints in the PluginInfo. To run a plugin with at least 0.5 cpu (= 0.5
+vCPU/Core/hyperthread), 1 gb memory and 10 (concurrent) cpu workers (threads), for example, the following configuration can be added to PluginInfo:
a HQL query (note: this is the traditional HQL query, and not the matchers HQL-lite variant),
+
(optional) the maximum number of traces to return (currently hard-limited to a maximum of 50 traces),
+
(optional) a scope, which can be either image, or project. When set to image, the searcher will only search for traces
+within the same image as the trace that is being processed.
+
+
The returned SearchResult
+should be closed, for example by using with. The resulting search result is an iterable, which will be exhausted when
+no more traces are available. The search result allows taking one or more traces by calling :py:
+meth:take<hansken_extraction_plugin.api.search_result.SearchResult.take> or
+takeone.
+
+
Note
+
The command trace.open(datastream_type) will fail on search result traces that do not originate from the
+same image (evidence item) as the trace that is being processed.
We use Logbook to log messages in Python. Logbook is a logging system for Python that replaces the standard library’s
+logging module.
+
To enable logging in your plugin, add the following to the top of your plugin code:
+
fromlogbookimportLogger
+
+log=Logger(__name__)
+
+
+
From there on the logging is pretty straight forward:
+
log.info(f'Logging a variable: {my_variable}')
+
+
+
The default log level is WARNING. There are two ways to set the logging level. You can use the -v (or -vv or -vvv) option of serve_plugin.py to increase the log level. This is typically done in the plugin Dockerfile. Another option is to use an environment variable, LOG_LEVEL. Available levels are WARNING, NOTICE, INFO and DEBUG. The environment variable overrides the option.
+
+
Warning
+
Be careful with logging sensitive information.
+
+
+
Note
+
Contact your Hansken administrator for more information on where to find logs for your Hansken environment.
+
+
+
+
[EXPERIMENTAL FEATURE] Adding previews to a traceïƒ
+
+
Warning
+
This is an experimental feature, which might change or get removed in future releases.
By default, the build scripts as described in the Getting Started section will automatically run
+tests. The appropriate commands have been added to the tox.ini directly. This section gives a little more detail on the
+test commands and options.
+
One can simply create unit tests for a plugin directly. However, we also provide a test-framework for testing them over
+gRPC. The test-framework serves a running instance of a Python plugin, and feeds it input files and compares the results
+against an expected result set.
+
Note that the test-framework is implemented in Java, hence the Java 11 requirement. A jar file is included in the Python
+SDK which is called from a Python wrapper.
The test runner is a script called test_plugin which is available in the SDK.
+
To get started, cd into the directory of the plugin you want to test and run:
+
test_plugin--standaloneplugin/chat_plugin.py
+
+
+
Note that the argument provided to the option --standalone must be the relative path to the plugin py file which is
+to be tested. This test accepts input files from the directory testdata/input and compares the results to the result
+files in found in testdata/results. Use the optional argument --regenerate to regenerate the expected results for
+the test when needed.
+
This standalone test is also used by the tox.ini file to validate the plugin. Simply calling tox should be enough to
+install all dependencies and run the test.
Replace the ‘extraction-plugin-examples-chat’ with the docker image you want to test. Run the following command to see
+which docker images are available:
The third option for testing is a manually started plugin. Start the plugin service in a terminal by executing:
+
serve_plugin-vvvplugin/my_plugin.py
+
+
+
This will spin up the chat plugin at port 8999. Here also the argument must be a path to the plugin’s .py file. In
+another terminal window, run the test with:
To start the extraction plugin from code, create a __main__ method which calls the _test_validate_standalone
+method of the test framework (see the example below). This method causes the extraction plugin to be started and
+supplied with data by the FLITS test framework. In this way the test can be started from the IDE, which has the
+advantage that it is easier to debug.
+
fromhansken_extraction_plugin.test_framework.test_pluginimport_test_validate_standalone
+fromhansken_extraction_plugin.api.extraction_pluginimportExtractionPlugin
+
+
+classPluginToTest(ExtractionPlugin):
+
+ defplugin_info(self):
+ # return plugin info
+ pass
+
+ defprocess(self,trace,data_context):
+ # process the data/trace here
+ pass
+
+
+if__name__=='__main__':
+ _test_validate_standalone(PluginToTest,'testdata/input','testdata/result',False)
+
The sdk includes a script to test transformer functions. This script will start an extraction plugin, executes a single transformer function with the provided arguments and stops the plugin. Try it with:
Transformers are methods inside a plugin that can be called remotely at any moment.
+This allows for live plugin execution independent of extraction.
+Examples on how transformers could be used:
+
+
For searching images using text (i.e. a purple car).
+
For translating text in traces so that an investigator can read the text in their preferred language in a UI.
Transformers can be implemented in extraction plugins. Using the Hansken REST API, calls can be made to a
+specific plugin’s transformer by specifying the transformer one wishes to call as well as its arguments.
+Hansken can automatically discover transformers before plugins are actually started.
+Once it has received a request for invoking a transformer it can choose to start the plugin’s Docker container if it
+is not already started and send the request once it is up and running.
A transformer can be easily defined by using the transformer decorator. By creating a method inside your plugin and
+decorating it with the transformer decorator it will automatically be made available to be remotely called.
+
fromhansken_extraction_plugin.api.extraction_pluginimportExtractionPlugin
+fromhansken_extraction_plugin.decorators.transformerimporttransformer
+classPlugin(ExtractionPlugin):
+
+ defplugin_info(self):
+ ...
+
+ defprocess(self,trace,data_context):
+ ...
+
+ @transformer
+ deftranslate_text(self,text:str,language:str)->str:
+ # To implement: Translate the text here.
+ return"Translated text"
+
However, there are some limitations on which methods can be turned into a decorator method:
+
+
Transformers may only be defined on methods of a class that derives (indirectly) from BaseExtractionPlugin.
+
+
Note: ExtractionPlugin derives from BaseExtractionPlugin and is therefore allowed.
+
+
+
Transformer may not be static methods.
+
All parameters and the return type must be annotated with type hints (except the self parameter).
+
Parameters may not be positional-only or contain variable parameters like *args or **kwarg.
+
Parameters and return types may only be of the following:
+
+
bool
+
int
+
float
+
str
+
bytes
+
bytearray
+
datetime.datetime
+
hansken.util.GeographicLocation
+
hansken.util.Vector
+
typing.Sequence
+
typing.Mapping
+
+
+
+
Upon starting a plugin every method decorated with the transformer decorator will be automatically validated to see if they adhere to these requirements. An exception will be thrown when a method does not adhere to all of these requirements.
If at some point a transformer wishes to signal to the caller of the transformer that something has gone wrong
+it can simply throw a suitable exception. Any exceptions being thrown will automatically be propagated to the client
+calling the transformer by wrapping the exception in a gRPC exception. The stack trace will be provided as well for
+debugging purposes.
If you use the Java or Python extraction plugin SDK, you don’t have
+to worry about these specifications. The Java and Python SDKs makes
+sure your plugin is compiled and packaged conform to the extraction
+plugin specifications.
+
+
This page describes the specifications that define an extraction plugin.
+The spec contains two major parts: a plugin protocol, and the plugin packaging
+method.
+
This specification applies to plugins that are not embedded within Hansken,
+but to plugins that developed and distributed outside the scope of the Hansken
+platform development.
An extraction plugin is a process that implements a GRPC
+service ExtractionPluginService. The service defines a protocol that is
+used to allow communication between Hansken and an extraction plugin. The
+GRPC and protocol definitions can be found in the extraction plugin source
+code, under the folder grpc.
+
+
Note
+
The source code of the Extraction Plugin is currently not available
+outside the scope of the Hansken core development teams. If you are
+interested in the GRPC definitions, please Contact the
+Hansken development team.
An extraction plugin is packaged as a container image – conform the open
+container initiative image spec.
+An extraction plugin can be
+
The ENTRYPOINT of the container image should be a process that starts a GRPC
+server that implements the plugin protocol. The GRPC protocol should run on
+port 8999.
+
The container image should be labeled with the plugin info.
+The plugin info returned by the plugin plugin-info call and container labels
+are required to match. If not, Hansken will not accept your plugin during
+extractions - as it is unsure if the intended plugin is processing traces.
org.hansken.plugin-resource-max_cpu (in milicpu, optional)
+
org.hansken.plugin-resource-max_mem (in mbs, optional)
+
org.hansken.plugin-transformers (the signatures of the transformer methods as JSON). The transformers field contains a JSON array and is structured as follows:
Welcome to the Hansken extraction plugin SDK documentation for plugin developers.
+This documentation describes the Hansken extraction plugin Software Development Kit (SDK).
+If you are new here, you can start by reading the introduction.
+
+
Attention
+
Hansken extraction plugins is a technology preview.
+Please don’t consider the SDK and integration in Hansken to be fully stable.
+
PluginInfo.Builder.id(PluginId)instead.