From: allfro Date: Mon, 24 Dec 2012 06:04:47 +0000 (-0500) Subject: Tons of bug fixes and support for link colors, thickness, style and entity notes... X-Git-Url: https://git.nothing2do.fr/?a=commitdiff_plain;h=285071749a11b200201398ac497b11904a324862;p=get-hack-src.git Tons of bug fixes and support for link colors, thickness, style and entity notes, and bookmarks --- diff --git a/src/canari/commands/create_package.py b/src/canari/commands/create_package.py index 5b69432..d6b11e6 100644 --- a/src/canari/commands/create_package.py +++ b/src/canari/commands/create_package.py @@ -13,7 +13,7 @@ __copyright__ = 'Copyright 2012, Canari Project' __credits__ = [] __license__ = 'GPL' -__version__ = '0.2' +__version__ = '0.5' __maintainer__ = 'Nadeem Douba' __email__ = 'ndouba@gmail.com' __status__ = 'Development' @@ -159,7 +159,8 @@ def run(args): 'email' : '', 'maintainer' : getuser(), 'example' : True, - 'description' : '' + 'description' : '', + 'canari_version' : __version__ } ask_user(values) diff --git a/src/canari/commands/install_package.py b/src/canari/commands/install_package.py index 1238eb6..d0f1f0d 100644 --- a/src/canari/commands/install_package.py +++ b/src/canari/commands/install_package.py @@ -208,7 +208,7 @@ def installmtz(package, prefix): category = xml.get('category') catdir = path.join(prefix, category) if not path.exists(catdir): - mkdir(catdir, mode=0755) + mkdir(catdir) p = path.join(catdir, path.basename(e)) print 'Installing entity %s to %s...' % (e, p) with open(p, 'wb') as f: @@ -250,7 +250,7 @@ def installmachines(package, prefix): if path.exists(n): e = XML(file(n).read()) if not path.exists(prefix): - mkdir(prefix, 0755) + mkdir(prefix) package = '%s.resources.maltego' % package for m in filter(lambda x: x.endswith('.machine'), resource_listdir(package, '')): src = resource_filename(package, m) diff --git a/src/canari/maltego/message.py b/src/canari/maltego/message.py index 20c8af7..904a893 100644 --- a/src/canari/maltego/message.py +++ b/src/canari/maltego/message.py @@ -129,7 +129,7 @@ class Field(MaltegoElement): super(Field, self).__init__(self.__class__.__name__) self.name = name self.matchingrule = kwargs.get('matchingrule', self.matchingrule) - self.displayname = kwargs.get('displayname', name.title()) + self.displayname = kwargs.get('displayname', None) self.text = str(value) if not isinstance(value, basestring) else value @@ -137,7 +137,7 @@ class StringEntityField(object): def __init__(self, name, displayname=None, decorator=None, matchingrule=MatchingRule.Strict): self.name = name - self.displayname = name.title() if displayname is None else displayname + self.displayname = displayname self.decorator = decorator self.matchingrule = matchingrule @@ -168,9 +168,9 @@ class StringEntityField(object): class EnumEntityField(StringEntityField): - def __init__(self, name, displayname=None, choices=[], decorator=None): + def __init__(self, name, displayname=None, choices=[], decorator=None, matchingrule=MatchingRule.Strict): self.choices = [ str(c) if not isinstance(c, basestring) else c for c in choices ] - super(EnumEntityField, self).__init__(name, displayname, decorator) + super(EnumEntityField, self).__init__(name, displayname, decorator, matchingrule) def __set__(self, obj, val): val = str(val) if not isinstance(val, basestring) else val @@ -234,12 +234,15 @@ class EntityFieldType(object): class EntityField(object): - def __init__(self, **kwargs): + def __init__(self, link=False, **kwargs): self.name = kwargs.get('name') if self.name is None: raise ValueError("Keyword argument 'name' is required.") self.property = kwargs.get('propname', sub('[^\w]+', '_', self.name)) - self.displayname = kwargs.get('displayname', self.name.title()) + if not link: + self.displayname = kwargs.get('displayname', self.name.title()) + else: + self.displayname = kwargs.get('displayname', None) self.type = kwargs.get('type', EntityFieldType.String) self.required = kwargs.get('required', False) self.choices = kwargs.get('choices') @@ -248,12 +251,27 @@ class EntityField(object): def __call__(self, cls): if self.type is EntityFieldType.Enum: - setattr(cls, self.property, self.type(self.name, self.displayname, self.choices, self.decorator)) + setattr( + cls, + self.property, + self.type(self.name, self.displayname, self.choices, self.decorator, self.matchingrule) + ) else: - setattr(cls, self.property, self.type(self.name, self.displayname, self.decorator)) + setattr( + cls, + self.property, + self.type(self.name, self.displayname, self.decorator, self.matchingrule) + ) return cls +class EntityLinkField(EntityField): + + def __init__(self, **kwargs): + kwargs['name'] = 'link#%s' % kwargs.get('name') + super(EntityLinkField, self).__init__(link=True, **kwargs) + + class UIMessageType(object): Fatal = "FatalError" Partial = "PartialError" @@ -277,6 +295,17 @@ class UIMessage(MaltegoElement): @XMLSubElement(name='DisplayInformation', propname='labels', type=XSSubElementType.List) @XMLSubElement(name='Value', propname='value') @XMLAttribute(name='Type', propname='type') +@EntityField(name='notes#', propname='notes', link=True, matchingrule=MatchingRule.Loose) +@EntityField(name='bookmark#', propname='bookmark', type=EntityFieldType.Integer, matchingrule=MatchingRule.Loose, + link=True) +@EntityLinkField(name='maltego.link.label', propname='linklabel', matchingrule=MatchingRule.Loose) +@EntityLinkField(name='maltego.link.style', propname='linkstyle', matchingrule=MatchingRule.Loose, + type=EntityFieldType.Integer) +@EntityLinkField(name='maltego.link.show-label', propname='linkshowlabel', matchingrule=MatchingRule.Loose, + type=EntityFieldType.Enum, choices=[0, 1]) +@EntityLinkField(name='maltego.link.color', propname='linkcolor', matchingrule=MatchingRule.Loose) +@EntityLinkField(name='maltego.link.thickness', propname='linkthickness', matchingrule=MatchingRule.Loose, + type=EntityFieldType.Integer) class Entity(MaltegoElement): namespace = 'maltego' @@ -286,7 +315,7 @@ class Entity(MaltegoElement): super(Entity, self).__init__("Entity") type = kwargs.pop('type', None) if type is None: - self.type = '%s.%s' % (self.namespace, self.__class__.__name__ if self.name is None else self.name) + self.type = '%s.%s' % (self.namespace, (self.__class__.__name__ if self.name is None else self.name)) else: self.type = type self.value = value @@ -300,13 +329,11 @@ class Entity(MaltegoElement): def appendelement(self, other): if isinstance(other, Field): - display_name = other.get('DisplayName') - if display_name is None: - name = other.get('Name') - if name in self.fields.keys(): - other.set('DisplayName', self.fields[name]) - else: - other.set('DisplayName', name.title()) +# display_name = other.get('DisplayName') +# if display_name is None: +# name = other.get('Name') +# if name in self.fields.keys(): +# other.set('DisplayName', self.fields[name]) self.fields += other elif isinstance(other, Label): self.labels += other diff --git a/src/canari/maltego/utils.py b/src/canari/maltego/utils.py index d811ea2..57bc7cd 100644 --- a/src/canari/maltego/utils.py +++ b/src/canari/maltego/utils.py @@ -5,7 +5,7 @@ from message import MaltegoMessage, Message, MaltegoTransformExceptionMessage, M from signal import signal, SIGTERM, SIGINT from sys import exit, argv, stderr, stdout from cStringIO import StringIO -from re import split, sub +from re import split, sub, search __author__ = 'Nadeem Douba' @@ -59,8 +59,8 @@ def parseargs(args=argv): exit(-1) arg_script = args[1] - arg_field = args[-1] if '=' in args[-1] else None - arg_value = args[-1] if arg_field is None else args[-2] + arg_field = args[-1] if search(r'(?<=[^\\])=', args[-1]) is not None else None + arg_value = (args[-1] if arg_field is None else args[-2]).replace('\=', '=') arg_param = [] if arg_field is None and len(args) > 3: @@ -72,7 +72,15 @@ def parseargs(args=argv): if arg_field is not None: fs = split(r'(?<=[^\\])#', arg_field) if fs is not None: - fields = dict(map(lambda x: x.split('=', 1), fs)) + fields = dict( + map( + lambda x: [ + c.replace('\#', '#').replace('\=', '=').replace('\\\\', '\\') + for c in split(r'(?<=[^\\])=', x, 1) + ], + fs + ) + ) return arg_script, arg_param, arg_value, fields diff --git a/src/canari/resources/template/setup.plate b/src/canari/resources/template/setup.plate index e095f20..ea41491 100755 --- a/src/canari/resources/template/setup.plate +++ b/src/canari/resources/template/setup.plate @@ -14,7 +14,7 @@ setup( '' : [ '*.gif', '*.png', '*.conf', '*.mtz', '*.machine' ] # list of resources }, install_requires=[ - # Name of packages required for easy_install + 'canari=${canari_version}' ], dependency_links=[ # custom links for the install_requires diff --git a/src/canari/xmltools/fixetree.py b/src/canari/xmltools/fixetree.py new file mode 100644 index 0000000..7831894 --- /dev/null +++ b/src/canari/xmltools/fixetree.py @@ -0,0 +1,322 @@ +# +# ElementTree +# $Id: ElementPath.py 3375 2008-02-13 08:05:08Z fredrik $ +# +# limited xpath support for element trees +# +# history: +# 2003-05-23 fl created +# 2003-05-28 fl added support for // etc +# 2003-08-27 fl fixed parsing of periods in element names +# 2007-09-10 fl new selection engine +# 2007-09-12 fl fixed parent selector +# 2007-09-13 fl added iterfind; changed findall to return a list +# 2007-11-30 fl added namespaces support +# 2009-10-30 fl added child element value filter +# +# Copyright (c) 2003-2009 by Fredrik Lundh. All rights reserved. +# +# fredrik@pythonware.com +# http://www.pythonware.com +# +# -------------------------------------------------------------------- +# The ElementTree toolkit is +# +# Copyright (c) 1999-2009 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + +# Licensed to PSF under a Contributor Agreement. +# See http://www.python.org/psf/license for licensing details. + +## +# Implementation module for XPath support. There's usually no reason +# to import this module directly; the ElementTree does this for +# you, if needed. +## +import xml.etree.ElementPath as EP +import re + +EP.xpath_tokenizer_re = xpath_tokenizer_re = re.compile( + "(" + "'[^']*'|\"[^\"]*\"|" + "::|" + "//?|" + "\.\.|" + "\(\)|" + "[/.*:\[\]\(\)@=])|" + "((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|" + "\s+" + ) + +def xpath_tokenizer(pattern, namespaces=None): + for token in xpath_tokenizer_re.findall(pattern): + tag = token[1] + if tag and tag[0] != "{" and ":" in tag: + try: + prefix, uri = tag.split(":", 1) + if not namespaces: + raise KeyError + yield token[0], "{%s}%s" % (namespaces[prefix], uri) + except KeyError: + raise SyntaxError("prefix %r not found in prefix map" % prefix) + else: + yield token + +EP.xpath_tokenizer = xpath_tokenizer + +def get_parent_map(context): + parent_map = context.parent_map + if parent_map is None: + context.parent_map = parent_map = {} + for p in context.root.iter(): + for e in p: + parent_map[e] = p + return parent_map + +EP.get_parent_map = get_parent_map + +def prepare_child(next, token): + tag = token[1] + def select(context, result): + for elem in result: + for e in elem: + if e.tag == tag: + yield e + return select + +EP.prepare_child = prepare_child + +def prepare_star(next, token): + def select(context, result): + for elem in result: + for e in elem: + yield e + return select + +EP.prepare_star = prepare_star + +def prepare_self(next, token): + def select(context, result): + for elem in result: + yield elem + return select + +EP.prepare_self = prepare_self + +def prepare_descendant(next, token): + token = next() + if token[0] == "*": + tag = "*" + elif not token[0]: + tag = token[1] + else: + raise SyntaxError("invalid descendant") + def select(context, result): + for elem in result: + for e in elem.iter(tag): + if e is not elem: + yield e + return select + +EP.prepare_descendant = prepare_descendant + +def prepare_parent(next, token): + def select(context, result): + # FIXME: raise error if .. is applied at toplevel? + parent_map = get_parent_map(context) + result_map = {} + for elem in result: + if elem in parent_map: + parent = parent_map[elem] + if parent not in result_map: + result_map[parent] = None + yield parent + return select + +EP.prepare_parent = prepare_parent + +def prepare_predicate(next, token): + # FIXME: replace with real parser!!! refs: + # http://effbot.org/zone/simple-iterator-parser.htm + # http://javascript.crockford.com/tdop/tdop.html + signature = [] + predicate = [] + while 1: + token = next() + if token[0] == "]": + break + if token[0] and token[0][:1] in "'\"": + token = "'", token[0][1:-1] + signature.append(token[0] or "-") + predicate.append(token[1]) + signature = "".join(signature) + # use signature to determine predicate type + if signature == "@-": + # [@attribute] predicate + key = predicate[1] + def select(context, result): + for elem in result: + if elem.get(key) is not None: + yield elem + return select + if signature == "@-='": + # [@attribute='value'] + key = predicate[1] + value = predicate[-1] + def select(context, result): + for elem in result: + if elem.get(key) == value: + yield elem + return select + if signature == "-" and not re.match("\d+$", predicate[0]): + # [tag] + tag = predicate[0] + def select(context, result): + for elem in result: + if elem.find(tag) is not None: + yield elem + return select + if signature == "-='" and not re.match("\d+$", predicate[0]): + # [tag='value'] + tag = predicate[0] + value = predicate[-1] + def select(context, result): + for elem in result: + for e in elem.findall(tag): + if "".join(e.itertext()) == value: + yield elem + break + return select + if signature == "-" or signature == "-()" or signature == "-()-": + # [index] or [last()] or [last()-index] + if signature == "-": + index = int(predicate[0]) - 1 + else: + if predicate[0] != "last": + raise SyntaxError("unsupported function") + if signature == "-()-": + try: + index = int(predicate[2]) - 1 + except ValueError: + raise SyntaxError("unsupported expression") + else: + index = -1 + def select(context, result): + parent_map = get_parent_map(context) + for elem in result: + try: + parent = parent_map[elem] + # FIXME: what if the selector is "*" ? + elems = list(parent.findall(elem.tag)) + if elems[index] is elem: + yield elem + except (IndexError, KeyError): + pass + return select + raise SyntaxError("invalid predicate") + +EP.prepare_predicate = prepare_predicate + +EP.ops = ops = { + "": prepare_child, + "*": prepare_star, + ".": prepare_self, + "..": prepare_parent, + "//": prepare_descendant, + "[": prepare_predicate, + } + +EP._cache = _cache = {} + +class _SelectorContext: + parent_map = None + def __init__(self, root): + self.root = root + +EP._SelectorContext = _SelectorContext +# -------------------------------------------------------------------- + +## +# Generate all matching objects. + +def iterfind(elem, path, namespaces=None): + # compile selector pattern + if path[-1:] == "/": + path = path + "*" # implicit all (FIXME: keep this?) + try: + selector = _cache[path] + except KeyError: + if len(_cache) > 100: + _cache.clear() + if path[:1] == "/": + raise SyntaxError("cannot use absolute path on element") + next = iter(xpath_tokenizer(path, namespaces)).next + token = next() + selector = [] + while 1: + try: + selector.append(ops[token[0]](next, token)) + except StopIteration: + raise SyntaxError("invalid path") + try: + token = next() + if token[0] == "/": + token = next() + except StopIteration: + break + _cache[path] = selector + # execute selector pattern + result = [elem] + context = _SelectorContext(elem) + for select in selector: + result = select(context, result) + return result + +EP.iterfind = iterfind + +## +# Find first matching object. +def find(elem, path, namespaces=None): + try: + return iterfind(elem, path, namespaces).next() + except StopIteration: + return None + +EP.find = find +## +def findall(elem, path, namespaces=None): + return list(iterfind(elem, path, namespaces)) + +EP.findall = findall +## +# Find text for first matching object. + +def findtext(elem, path, default=None, namespaces=None): + try: + elem = iterfind(elem, path, namespaces).next() + return elem.text or "" + except StopIteration: + return default +EP.findtext = findtext diff --git a/src/canari/xmltools/oxml.py b/src/canari/xmltools/oxml.py index 02addd6..ea0d24c 100644 --- a/src/canari/xmltools/oxml.py +++ b/src/canari/xmltools/oxml.py @@ -56,7 +56,9 @@ class XSStringAttribute(object): self.default = default def __get__(self, obj, objtype): - return obj.attrib.setdefault(self.name, self.default) + if self.default is not None: + return obj.attrib.setdefault(self.name, self.default) + return obj.attrib.get(self.name, None) def __set__(self, obj, val): if val is None: @@ -88,7 +90,7 @@ class XSEnumAttribute(XSStringAttribute): class XSIntegerAttribute(XSStringAttribute): def __get__(self, obj, objtype): - return int(super(XSIntegerAttribute, self).__get__(obj, objtype)) + return int(super(XSIntegerAttribute, self).__get__(obj, objtype) or 0) def __set__(self, obj, val): if not isinstance(val, Number): @@ -113,7 +115,7 @@ class XSBooleanAttribute(XSStringAttribute): class XSFloatAttribute(XSStringAttribute): def __get__(self, obj, objtype): - return float(super(XSFloatAttribute, self).__get__(obj, objtype)) + return float(super(XSFloatAttribute, self).__get__(obj, objtype) or 0) def __set__(self, obj, val): if not isinstance(val, Number): @@ -124,7 +126,7 @@ class XSFloatAttribute(XSStringAttribute): class XSLongAttribute(XSStringAttribute): def __get__(self, obj, objtype): - return long(super(XSLongAttribute, self).__get__(obj, objtype)) + return long(super(XSLongAttribute, self).__get__(obj, objtype) or 0) def __set__(self, obj, val): if not isinstance(val, Number): @@ -218,7 +220,7 @@ class XSEnumSubElement(XSStringSubElement): class XSIntegerSubElement(XSStringSubElement): def __get__(self, obj, objtype): - return int(super(XSIntegerSubElement, self).__get__(obj, objtype)) + return int(super(XSIntegerSubElement, self).__get__(obj, objtype) or 0) def __set__(self, obj, val): if not isinstance(val, Number): @@ -243,7 +245,7 @@ class XSBooleanSubElement(XSStringSubElement): class XSFloatSubElement(XSStringSubElement): def __get__(self, obj, objtype): - return float(super(XSFloatSubElement, self).__get__(obj, objtype)) + return float(super(XSFloatSubElement, self).__get__(obj, objtype) or 0) def __set__(self, obj, val): if not isinstance(val, Number): @@ -254,7 +256,7 @@ class XSFloatSubElement(XSStringSubElement): class XSLongSubElement(XSStringSubElement): def __get__(self, obj, objtype): - return long(super(XSLongSubElement, self).__get__(obj, objtype)) + return long(super(XSLongSubElement, self).__get__(obj, objtype) or 0) def __set__(self, obj, val): if not isinstance(val, Number):