next »

[Maltego Forum]

Maltego Forum » Development » User Transforms
Person->Phone & Person->Location using directories.ch (swiss yellow page)

[TristanLeiter]

Here is my litlle contribution to the Maltego project.

I wrote two local transforms to browse directories.ch which is the "swiss yellow page". The first one finds the address (location) given a person and the second finds the phone number.

I used beautiful soup, and andrew''s lib MaltegoTransform.

----------------person2location.py--------------------$

from BeautifulSoup import BeautifulSoup
import re
import urllib2
import sys
from MaltegoTransform import *
# store entity value
entity_value = str(sys.argv[1])

# divide entity value into first name and last name
index_blank = entity_value.find(" ")
first_name = entity_value[0:index_blank]
last_name = entity_value[index_blank+1:]

# open url to local.ch with search parameters
url = "http://tel.local.ch/en/q/?what=" + last_name + "+" + first_name + "&where=&suchen=Search"
page = urllib2.urlopen(url)
all = BeautifulSoup(page)

#init Maltego Transform
me = MaltegoTransform();

# parser if more than 1 entry
entry_box_tags = all.findAll(name="div", attrs={"class" : "entrybox"})
for entry_box_tag in entry_box_tags:
   # for each entry...
   entry = BeautifulSoup(str(entry_box_tag))
   # get firstname and lastname tags
   full_name_tag = entry.find(name="a", attrs={"class" : "fn"})
   full_name_entry = BeautifulSoup(str(full_name_tag))
   first_name_string = str(full_name_entry.find(name="span", attrs={"class" : "highlight"}, text= first_name))
   last_name_string = str(full_name_entry.find(name="span", attrs={"class" : "highlight"}, text= last_name))
   # if firstname and lastname match entry value
   if first_name_string != "None" and last_name_string != "None":
      #find street and locality using regex
      patern_street = re.compile(r"""<span class="street-address">(.*?)</span>""", re.DOTALL)
      patern_locality = re.compile(r"""<span class="locality">(.*?)</span>""", re.DOTALL)
      street = patern_street.findall(unicode(entry))
      locality = patern_locality.findall(unicode(entry))
      if len(locality)>0 and len(street)>0 :
         location = street[0] + ", " + locality[0]
         new_entity = me.addEntity("Location", location);
         new_entity.setWeight(70); #Set the Weight of the entity
# parse if 1 entry
single_entry = all.find(name="div", attrs={"id" : "singleview"})
if single_entry:
   patern_street = re.compile(r"""<span class="street-address">(.*?)</span>""", re.DOTALL)
   patern_locality = re.compile(r"""<span class="locality">(.*?)</span>""", re.DOTALL)
   street = patern_street.findall(unicode(single_entry))
   locality = patern_locality.findall(unicode(single_entry))
   if len(locality)>0 and len(street)>0 :
      location = street[0] + ", " + locality[0]
      new_entity = me.addEntity("Location", location);
      new_entity.setWeight(100); #Set the Weight of the entity
me.returnOutput();


------------------------------------------------------


----------------person2phone.py--------------------

from BeautifulSoup import BeautifulSoup
import re
import urllib2
import sys
from MaltegoTransform import *

# store entity value
entity_value = str(sys.argv[1])

# divide entity value into first name and last name
index_blank = entity_value.find(" ")
first_name = entity_value[0:index_blank]
last_name = entity_value[index_blank+1:]

# open url to local.ch with search parameters
url = "http://tel.local.ch/en/q/?what=" + last_name + "+" + first_name + "&where=&suchen=Search"
page = urllib2.urlopen(url)
all = BeautifulSoup(page)

#init Maltego Transform
me = MaltegoTransform();
# parser if more than 1 entry
entryboxTags = all.findAll(name="div", attrs={"class" : "entrybox"})
for entryboxTag in entryboxTags:
   # for each entry...
   entry = BeautifulSoup(str(entryboxTag))
   # get firstname and lastname tags
   full_name_tag = entry.find(name="a", attrs={"class" : "fn"})
   full_name_entry = BeautifulSoup(str(full_name_tag))
   first_name_string = str(full_name_entry.find(name="span", attrs={"class" : "highlight"}, text= first_name))
   last_name_string = str(full_name_entry.find(name="span", attrs={"class" : "highlight"}, text= last_name))
   # if firstname and lastname match entry value
   if first_name_string != "None" and last_name_string != "None":
      #print (firstnameString +" "+ lastnameString +".")
      rawPhoneNumber = str(entry.find(name="span", attrs={"class" : "tel"}).renderContents())
      phone_number = rawPhoneNumber.replace("*&#xA0;","")
      phone_number = phone_number.replace(" ","")
      phone_number_format = "+41 "+ phone_number[1:4] + " " + phone_number[4:7] + " " + phone_number[7:10]
      new_entity = me.addEntity("PhoneNumber",phone_number_format);
      new_entity.setWeight(70); #Set the Weight of the entity
# parse if 1 entry
singleEntry = all.find(name="div", attrs={"id" : "singleview"})
if singleEntry:
   rawPhoneNumber = str(singleEntry.find(name="td", attrs={"class" : "tel"}).renderContents())
   phone_number = rawPhoneNumber.replace("*&#xA0;","")
   phone_number = phone_number.replace(" ","")
   phone_number_format = "+41 "+ phone_number[1:4] + " " + phone_number[4:7] + " " + phone_number[7:10]
   new_entity = me.addEntity("PhoneNumber",phone_number_format);
   new_entity.setWeight(100); #Set the Weight of the entity
me.returnOutput();


------------------------------------------------------

[RT]

Tristan,

Nice work! These seems to work well. I would perhaps also use the DisplayInfo to add some extra goodies in there (like adding the location to phone number entity, and adding the phone number to the location). Also - is reverse lookup possible? Then Phone2Person/Phone2Location would be awesome! Something else to consider is to make the Location a Phrase Entity and put quotes around it - then you can directly start searching from there. The whole idea would be to make entities that can be used for a second round of transforms...if you catch my drift.

But hey! - it works and TADA - you are our first non-Paterva local transform writer - congrats - we owe you a beer!
RT

[TristanLeiter]

Hello,

Hehe, I''m proud to be the first one 

I did some modifications and added 2 more transforms. You can now reverse lookup from phone to person and from location to person so it means we now have something like that :

         --------->          <----------
Phone               Person                   Location
         <---------          ----------->

I added some "display info" too like you suggested.

P.S. I have to copy/past the code of the script bc the forum doesn''t accept text file as an attachment.

location2person.py

from BeautifulSoup import BeautifulSoup
import re
import urllib2
import sys
import datetime
from MaltegoTransform import *
#init Maltego Transform
me = MaltegoTransform();
# store entity value
entity_value = str(sys.argv[1])
location = entity_value.replace(" ", "+")
# open url to local.ch with search parameters and format it
url = "http://tel.local.ch/fr/q/" + location + ".html"
page = urllib2.urlopen(url)
all = BeautifulSoup(page)
# parser if more than 1 entry (find all entrybox)
entryboxTags = all.findAll(name="div", attrs={"class" : "entrybox"})
for entryboxTag in entryboxTags:
   # for each entry...$
   entry = BeautifulSoup(str(entryboxTag))
   
   # get Person
   full_name_tag = entry.find(name="a", attrs={"class" : "fn"})
   full_name = full_name_tag.renderContents()
   index_blank = full_name.find(" ")
   last_name = full_name[0:index_blank]
   first_name = full_name[index_blank+1:]
   # if Person found
   if full_name != "None" :
      rawPhoneNumber = str(entry.find(name="span", attrs={"class" : "tel"}).renderContents())
      phone_number = rawPhoneNumber.replace("*&#xA0;","")
      phone_number = phone_number.replace(" ","")
      phone_number_format = "+41 "+ phone_number[1:4] + " " + phone_number[4:7] + " " + phone_number[7:10]
      
      new_entity = me.addEntity("Person", first_name + " " + last_name);
      new_entity.setWeight(70); #Set the Weight of the entity
      new_entity.addAdditionalFields(fieldName="Phone",displayName="Phone",matchingRule=False, value=phone_number_format)
      table = """<table width=100% border=1 rules=cols frame=box cellpadding=2>
<tr><td class=one colspan=3 align=center>GENERATION INFORMATION</td></tr>
<tr> <td class=three width=20%>Result</td>
<td class=value align=center>""" + full_name + """</td>
<td class=three width=30% align=center>(Person)</td> </tr>
<tr> <td class=three width=20%>Source</td>
<td class=two align=center>""" + entity_value + """</td>
<td class=three width=30% align=center>(Loctation)</td> </tr>
<tr> <td class=three width=20%>Gen. by</td>
<td class=two align=center>To Person [directories.ch]</td>
<td class=three width=30% align=center>"""+str(datetime.date.today())+"""</td> </tr>
</table>"""
      new_entity.setDisplayInformation(table)
   
# parse if 1 entry
single_entry = all.find(name="div", attrs={"id" : "singleview"})
if single_entry:
   # get Name
   full_name_tag = single_entry.find(name="h2", attrs={"class" : "fn"})
   full_name = full_name_tag.renderContents()
   index_blank = full_name.find(" ")
   last_name = full_name[0:index_blank]
   first_name = full_name[index_blank+1:]
   
   rawPhoneNumber = str(single_entry.find(name="td", attrs={"class" : "tel"}).renderContents())
   phone_number = rawPhoneNumber.replace("*&#xA0;","")
   phone_number = phone_number.replace(" ","")
   phone_number_format = "+41 "+ phone_number[1:4] + " " + phone_number[4:7] + " " + phone_number[7:10]


   new_entity = me.addEntity("Person", first_name + " " + last_name);
   new_entity.setWeight(70); #Set the Weight of the entity
   new_entity.addAdditionalFields(fieldName="Phone",displayName="Phone",matchingRule=False, value=phone_number_format)
   table = """<table width=100% border=1 rules=cols frame=box cellpadding=2>
<tr><td class=one colspan=3 align=center>GENERATION INFORMATION</td></tr>
<tr> <td class=three width=20%>Result</td>
<td class=value align=center>""" + full_name + """</td>
<td class=three width=30% align=center>(Person)</td> </tr>
<tr> <td class=three width=20%>Source</td>
<td class=two align=center>""" + entity_value + """</td>
<td class=three width=30% align=center>(Loctation)</td> </tr>
<tr> <td class=three width=20%>Gen. by</td>
<td class=two align=center>To Person [directories.ch]</td>
<td class=three width=30% align=center>"""+str(datetime.date.today())+"""</td> </tr>
</table>"""
   new_entity.setDisplayInformation(table)
me.returnOutput();

person2location.py

from BeautifulSoup import BeautifulSoup
import re
import urllib2
import sys
import datetime
from MaltegoTransform import *
# store entity value
entity_value = str(sys.argv[1])

# divide entity value into first name and last name
index_blank = entity_value.find(" ")
first_name = entity_value[0:index_blank]
last_name = entity_value[index_blank+1:]

# open url to local.ch with search parameters
url = "http://tel.local.ch/en/q/?what=" + last_name + "+" + first_name + "&where=&suchen=Search"
page = urllib2.urlopen(url)
all = BeautifulSoup(page)

#init Maltego Transform
me = MaltegoTransform();

# parser if more than 1 entry
entry_box_tags = all.findAll(name="div", attrs={"class" : "entrybox"})
for entry_box_tag in entry_box_tags:
   # for each entry...
   entry = BeautifulSoup(str(entry_box_tag))
   # get firstname and lastname tags
   full_name_tag = entry.find(name="a", attrs={"class" : "fn"})
   full_name_entry = BeautifulSoup(str(full_name_tag))
   first_name_string = str(full_name_entry.find(name="span", attrs={"class" : "highlight"}, text= first_name))
   last_name_string = str(full_name_entry.find(name="span", attrs={"class" : "highlight"}, text= last_name))
   # if firstname and lastname match entry value
   if first_name_string != "None" and last_name_string != "None":
      #find street and locality using regex
      patern_street = re.compile(r"""<span class="street-address">(.*?)</span>""", re.DOTALL)
      patern_locality = re.compile(r"""<span class="locality">(.*?)</span>""", re.DOTALL)
      street = patern_street.findall(unicode(entry))
      locality = patern_locality.findall(unicode(entry))
      rawPhoneNumber = str(entry.find(name="span", attrs={"class" : "tel"}).renderContents())
      phone_number = rawPhoneNumber.replace("*&#xA0;","")
      phone_number = phone_number.replace(" ","")
      phone_number_format = "+41 "+ phone_number[1:4] + " " + phone_number[4:7] + " " + phone_number[7:10]
      if len(locality)>0 and len(street)>0 :
         location = street[0] + ", " + locality[0]
         new_entity = me.addEntity("Location", location);
         new_entity.setWeight(70); #Set the Weight of the entity
         new_entity.addAdditionalFields(fieldName="Phone",displayName="Phone",matchingRule=False,value=phone_number_format)
         table = """<table width=100% border=1 rules=cols frame=box cellpadding=2>
<tr><td class=one colspan=3 align=center>GENERATION INFORMATION</td></tr>
<tr> <td class=three width=20%>Result</td>
<td class=value align=center>""" + location + """</td>
<td class=three width=30% align=center>(Location)</td> </tr>
<tr> <td class=three width=20%>Source</td>
<td class=two align=center>""" + entity_value + """</td>
<td class=three width=30% align=center>(Person)</td> </tr>
<tr> <td class=three width=20%>Gen. by</td>
<td class=two align=center>To Location [directories.ch]</td>
<td class=three width=30% align=center>"""+str(datetime.date.today())+"""</td> </tr>
</table>"""
         new_entity.setDisplayInformation(table)
         
# parse if 1 entry
single_entry = all.find(name="div", attrs={"id" : "singleview"})
if single_entry:
   patern_street = re.compile(r"""<span class="street-address">(.*?)</span>""", re.DOTALL)
   patern_locality = re.compile(r"""<span class="locality">(.*?)</span>""", re.DOTALL)
   street = patern_street.findall(unicode(single_entry))
   locality = patern_locality.findall(unicode(single_entry))
   
   rawPhoneNumber = str(single_entry.find(name="td", attrs={"class" : "tel"}).renderContents())
   phone_number = rawPhoneNumber.replace("*&#xA0;","")
   phone_number = phone_number.replace(" ","")
   phone_number_format = "+41 "+ phone_number[1:4] + " " + phone_number[4:7] + " " + phone_number[7:10]
   
   if len(locality)>0 and len(street)>0 :
      location = street[0] + ", " + locality[0]
      new_entity = me.addEntity("Location", location);
      new_entity.setWeight(100); #Set the Weight of the entity
      new_entity.addAdditionalFields(fieldName="Phone",displayName="Phone",matchingRule=False,value=phone_number_format)
      table = """<table width=100% border=1 rules=cols frame=box cellpadding=2>
<tr><td class=one colspan=3 align=center>GENERATION INFORMATION</td></tr>
<tr> <td class=three width=20%>Result</td>
<td class=value align=center>""" + location + """</td>
<td class=three width=30% align=center>(Location)</td> </tr>
<tr> <td class=three width=20%>Source</td>
<td class=two align=center>""" + entity_value + """</td>
<td class=three width=30% align=center>(Person)</td> </tr>
<tr> <td class=three width=20%>Gen. by</td>
<td class=two align=center>To Location [directories.ch]</td>
<td class=three width=30% align=center>"""+str(datetime.date.today())+"""</td> </tr>
</table>"""
      new_entity.setDisplayInformation(table)
me.returnOutput();

person2phone.py

from BeautifulSoup import BeautifulSoup
import re
import urllib2
import sys
import datetime
from MaltegoTransform import *

# store entity value
entity_value = str(sys.argv[1])
full_name = entity_value.replace(" ", "+" )
# divide entity value into first name and last name
index_blank = entity_value.find(" ")
first_name = entity_value[0:index_blank]
last_name = entity_value[index_blank+1:]

# open url to local.ch with search parameters
url = "http://tel.local.ch/en/q/?what=" + full_name + "&where=&suchen=Search"

page = urllib2.urlopen(url)
all = BeautifulSoup(page)

#init Maltego Transform
me = MaltegoTransform();
# parser if more than 1 entry
entryboxTags = all.findAll(name="div", attrs={"class" : "entrybox"})
for entryboxTag in entryboxTags:
   # for each entry...
   entry = BeautifulSoup(str(entryboxTag))
   # get firstname and lastname tags
   full_name_tag = entry.find(name="a", attrs={"class" : "fn"})
   full_name_entry = BeautifulSoup(str(full_name_tag))
   first_name_string = str(full_name_entry.find(name="span", attrs={"class" : "highlight"}, text= first_name))
   last_name_string = str(full_name_entry.find(name="span", attrs={"class" : "highlight"}, text= last_name))
   # if firstname and lastname match entry value
   if first_name_string != "None" and last_name_string != "None":
      rawPhoneNumber = str(entry.find(name="span", attrs={"class" : "tel"}).renderContents())
      phone_number = rawPhoneNumber.replace("*&#xA0;","")
      phone_number = phone_number.replace(" ","")
      phone_number_format = "+41 "+ phone_number[1:4] + " " + phone_number[4:7] + " " + phone_number[7:10]
      
      #find street and locality using regex
      patern_street = re.compile(r"""<span class="street-address">(.*?)</span>""", re.DOTALL)
      patern_locality = re.compile(r"""<span class="locality">(.*?)</span>""", re.DOTALL)
      street = patern_street.findall(unicode(entry))
      locality = patern_locality.findall(unicode(entry))
      location = street[0] + ", " + locality[0]
      
      new_entity = me.addEntity("PhoneNumber",phone_number_format);
      new_entity.setWeight(70); #Set the Weight of the entity
      new_entity.addAdditionalFields(fieldName="Location",displayName="Location",matchingRule=False,value=location)
      table = """<table width=100% border=1 rules=cols frame=box cellpadding=2>
<tr><td class=one colspan=3 align=center>GENERATION INFORMATION</td></tr>
<tr> <td class=three width=20%>Result</td>
<td class=value align=center>""" + phone_number_format + """</td>
<td class=three width=30% align=center>(Phone)</td> </tr>
<tr> <td class=three width=20%>Source</td>
<td class=two align=center>""" + entity_value + """</td>
<td class=three width=30% align=center>(Person)</td> </tr>
<tr> <td class=three width=20%>Gen. by</td>
<td class=two align=center>To Location [directories.ch]</td>
<td class=three width=30% align=center>"""+str(datetime.date.today())+"""</td> </tr>
</table>"""
      new_entity.setDisplayInformation(table)
# parse if 1 entry
single_entry = all.find(name="div", attrs={"id" : "singleview"})
if single_entry:
   rawPhoneNumber = str(single_entry.find(name="td", attrs={"class" : "tel"}).renderContents())
   phone_number = rawPhoneNumber.replace("*&#xA0;","")
   phone_number = phone_number.replace(" ","")
   phone_number_format = "+41 "+ phone_number[1:4] + " " + phone_number[4:7] + " " + phone_number[7:10]

   patern_street = re.compile(r"""<span class="street-address">(.*?)</span>""", re.DOTALL)
   patern_locality = re.compile(r"""<span class="locality">(.*?)</span>""", re.DOTALL)
   street = patern_street.findall(unicode(single_entry))
   locality = patern_locality.findall(unicode(single_entry))

   new_entity = me.addEntity("PhoneNumber",phone_number_format);
   new_entity.setWeight(100); #Set the Weight of the entity
   if len(locality)>0 and len(street)>0 :
      location = street[0] + ", " + locality[0]
      new_entity.addAdditionalFields(fieldName="Location",displayName="Location",matchingRule=False,value=location)
   table = """<table width=100% border=1 rules=cols frame=box cellpadding=2>
<tr><td class=one colspan=3 align=center>GENERATION INFORMATION</td></tr>
<tr> <td class=three width=20%>Result</td>
<td class=value align=center>""" + phone_number_format + """</td>
<td class=three width=30% align=center>(Phone)</td> </tr>
<tr> <td class=three width=20%>Source</td>
<td class=two align=center>""" + entity_value + """</td>
<td class=three width=30% align=center>(Person)</td> </tr>
<tr> <td class=three width=20%>Gen. by</td>
<td class=two align=center>To Location [directories.ch]</td>
<td class=three width=30% align=center>"""+str(datetime.date.today())+"""</td> </tr>
</table>"""
   new_entity.setDisplayInformation(table)
me.returnOutput();

phone2person.py

from BeautifulSoup import BeautifulSoup
import re
import urllib2
import sys
import datetime
from MaltegoTransform import *
#init Maltego Transform
me = MaltegoTransform();
# store entity value
entity_value = str(sys.argv[1])
phone = entity_value.replace("+", "")
phone = phone.replace(" ", "+")
# open url to local.ch with search parameters and format it
url = "http://tel.local.ch/fr/q/" + phone + ".html"
page = urllib2.urlopen(url)
all = BeautifulSoup(page)
# parser if more than 1 entry (find all entrybox)
entryboxTags = all.findAll(name="div", attrs={"class" : "entrybox"})
for entryboxTag in entryboxTags:
   # for each entry...$
   entry = BeautifulSoup(str(entryboxTag))
   
   # get Person
   full_name_tag = entry.find(name="a", attrs={"class" : "fn"})
   full_name = full_name_tag.renderContents()
   index_blank = full_name.find(" ")
   last_name = full_name[0:index_blank]
   first_name = full_name[index_blank+1:]
   # if Person found
   if full_name != "None" :
      rawPhoneNumber = str(entry.find(name="span", attrs={"class" : "tel"}).renderContents())
      phone_number = rawPhoneNumber.replace("*&#xA0;","")
      phone_number = phone_number.replace(" ","")
      phone_number_format = "+41 "+ phone_number[1:4] + " " + phone_number[4:7] + " " + phone_number[7:10]
      
      new_entity = me.addEntity("Person", first_name + " " + last_name);
      new_entity.setWeight(70); #Set the Weight of the entity
      table = """<table width=100% border=1 rules=cols frame=box cellpadding=2>
<tr><td class=one colspan=3 align=center>GENERATION INFORMATION</td></tr>
<tr> <td class=three width=20%>Result</td>
<td class=value align=center>""" + full_name + """</td>
<td class=three width=30% align=center>(Person)</td> </tr>
<tr> <td class=three width=20%>Source</td>
<td class=two align=center>""" + entity_value + """</td>
<td class=three width=30% align=center>(Loctation)</td> </tr>
<tr> <td class=three width=20%>Gen. by</td>
<td class=two align=center>To Person [directories.ch]</td>
<td class=three width=30% align=center>"""+str(datetime.date.today())+"""</td> </tr>
</table>"""
      new_entity.setDisplayInformation(table)
   
# parse if 1 entry
single_entry = all.find(name="div", attrs={"id" : "singleview"})
if single_entry:
   # get Name
   full_name_tag = single_entry.find(name="h2", attrs={"class" : "fn"})
   full_name = full_name_tag.renderContents()
   index_blank = full_name.find(" ")
   last_name = full_name[0:index_blank]
   first_name = full_name[index_blank+1:]
   
   

   new_entity = me.addEntity("Person", first_name + " " + last_name);
   new_entity.setWeight(70); #Set the Weight of the entity
   table = """<table width=100% border=1 rules=cols frame=box cellpadding=2>
<tr><td class=one colspan=3 align=center>GENERATION INFORMATION</td></tr>
<tr> <td class=three width=20%>Result</td>
<td class=value align=center>""" + full_name + """</td>
<td class=three width=30% align=center>(Person)</td> </tr>
<tr> <td class=three width=20%>Source</td>
<td class=two align=center>""" + entity_value + """</td>
<td class=three width=30% align=center>(Phone)</td> </tr>
<tr> <td class=three width=20%>Gen. by</td>
<td class=two align=center>To Person [directories.ch]</td>
<td class=three width=30% align=center>"""+str(datetime.date.today())+"""</td> </tr>
</table>"""
   new_entity.setDisplayInformation(table)
me.returnOutput();

[RT]

Tristan,

I''ve tried the new transforms - there seem to be some problems with the *->Phone transforms. Also if I go from Person->Location->Person I get some strange results. Is the transforms looking for only the first name (or the last name)?

RT