domain_tools_rod (#5)

- build: update dependencies (1c6e88d9c66188ba8c0defe1ef75a86fa55a21b8)
- feat: add tools and interfaces to inspect domains (dc8fed07898d310f9ae83f65cdb1a3a47f559832)
- feat: add tools (1fbf466dca5c9cc8729ccc9633828e679b1d9848)
- feat: add tools to app (35aff7c1b7c710519a5d82c8cf86adba26f2fb3a)
- Merge branch 'main' into pr/5 (e7e6c15215589f9c781732dc0dbffe1fb84e7868)


Co-authored-by: Rodrigo Domínguez Sanz <RodDoSanz@users.noreply.huggingface.co>

app.py

@@ -2,6 +2,12 @@ from typing import NamedTuple
 
 import gradio as gr
 
+from tdagent.tools.get_domain_information import (
+    dns_enumeration_tool,
+    extractor_of_ioc_from_threatfox_tool,
+    geo_location_tool,
+    scrap_subdomains_tool,
+)
 from tdagent.tools.get_url_content import gr_get_url_http_content
 from tdagent.tools.internal_company_user_search import gr_internal_company
 from tdagent.tools.lookup_company_cloud_account_information import (
@@ -33,6 +39,10 @@ TOOLS = (
     # ToolInfo("Query WHOIS", gr_query_whois),
     ToolInfo("Query RDAP", gr_query_rdap),
     ToolInfo("Virus Total URL info", gr_virus_total_url_info),
+    ToolInfo("Get IP's Location", geo_location_tool),
+    ToolInfo("DNS Enumerator", dns_enumeration_tool),
+    ToolInfo("Subdomain Retriever", scrap_subdomains_tool),
+    ToolInfo("Extractor of IoCs", extractor_of_ioc_from_threatfox_tool),
     ## Fake tools
     ToolInfo("Fake company directory", gr_internal_company),
     ToolInfo(

pyproject.toml

@@ -14,6 +14,7 @@ readme = "README.md"
 license = ""
 dependencies = [
     "cachetools>=6.0.0",
+    "dnspython>=2.7.0",
     "gradio[mcp]>=5.32.1",
     "python-whois>=0.9.5",
     "requests>=2.32.3",

requirements-dev.txt

@@ -1,109 +1,309 @@
 # This file was autogenerated by uv via the following command:
 #    uv export --format requirements-txt --no-hashes --group dev --group test -o requirements-dev.txt
 aiofiles==24.1.0
+    # via
+    #   gradio
+    #   vt-py
 aiohappyeyeballs==2.6.1
+    # via aiohttp
 aiohttp==3.12.8
+    # via vt-py
 aiosignal==1.3.2
+    # via aiohttp
 annotated-types==0.7.0
+    # via pydantic
 anyio==4.9.0
+    # via
+    #   gradio
+    #   httpx
+    #   mcp
+    #   sse-starlette
+    #   starlette
 async-timeout==5.0.1 ; python_full_version < '3.11'
+    # via aiohttp
 attrs==25.3.0
+    # via aiohttp
 audioop-lts==0.2.1 ; python_full_version >= '3.13'
+    # via gradio
 boolean-py==5.0
+    # via license-expression
 cachecontrol==0.14.3
+    # via pip-audit
 cachetools==6.0.0
+    # via tdagent
 certifi==2025.4.26
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
 cfgv==3.4.0
+    # via pre-commit
 charset-normalizer==3.4.2
+    # via requests
 click==8.2.1 ; sys_platform != 'emscripten'
+    # via
+    #   typer
+    #   uvicorn
 colorama==0.4.6 ; sys_platform == 'win32'
+    # via
+    #   click
+    #   pytest
+    #   tqdm
 coverage==7.8.2
+    # via pytest-cov
 cyclonedx-python-lib==9.1.0
+    # via pip-audit
 defusedxml==0.7.1
+    # via py-serializable
 distlib==0.3.9
+    # via virtualenv
+dnspython==2.7.0
+    # via tdagent
 exceptiongroup==1.3.0 ; python_full_version < '3.11'
+    # via
+    #   anyio
+    #   pytest
 fastapi==0.115.12
+    # via gradio
 ffmpy==0.6.0
+    # via gradio
 filelock==3.18.0
+    # via
+    #   cachecontrol
+    #   huggingface-hub
+    #   virtualenv
 frozenlist==1.6.2
+    # via
+    #   aiohttp
+    #   aiosignal
 fsspec==2025.5.1
+    # via
+    #   gradio-client
+    #   huggingface-hub
 gradio==5.32.1
+    # via tdagent
 gradio-client==1.10.2
+    # via gradio
 groovy==0.1.2
+    # via gradio
 h11==0.16.0
+    # via
+    #   httpcore
+    #   uvicorn
 hf-xet==1.1.2 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'
+    # via huggingface-hub
 httpcore==1.0.9
+    # via httpx
 httpx==0.28.1
+    # via
+    #   gradio
+    #   gradio-client
+    #   mcp
+    #   safehttpx
 httpx-sse==0.4.0
+    # via mcp
 huggingface-hub==0.32.4
+    # via
+    #   gradio
+    #   gradio-client
 identify==2.6.12
+    # via pre-commit
 idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
 iniconfig==2.1.0
+    # via pytest
 jinja2==3.1.6
+    # via gradio
 license-expression==30.4.1
+    # via cyclonedx-python-lib
 markdown-it-py==3.0.0
+    # via rich
 markupsafe==3.0.2
+    # via
+    #   gradio
+    #   jinja2
 mcp==1.9.0
+    # via gradio
 mdurl==0.1.2
+    # via markdown-it-py
 msgpack==1.1.0
+    # via cachecontrol
 multidict==6.4.4
+    # via
+    #   aiohttp
+    #   yarl
 mypy==1.16.0
 mypy-extensions==1.1.0
+    # via mypy
 nodeenv==1.9.1
+    # via pre-commit
 numpy==2.2.6
+    # via
+    #   gradio
+    #   pandas
 orjson==3.10.18
+    # via gradio
 packageurl-python==0.16.0
+    # via cyclonedx-python-lib
 packaging==25.0
+    # via
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   pip-audit
+    #   pip-requirements-parser
+    #   pytest
 pandas==2.2.3
+    # via gradio
 pathspec==0.12.1
+    # via mypy
 pillow==11.2.1
+    # via gradio
 pip==25.1.1
+    # via pip-api
 pip-api==0.0.34
+    # via pip-audit
 pip-audit==2.9.0
 pip-requirements-parser==32.0.1
+    # via pip-audit
 platformdirs==4.3.8
+    # via
+    #   pip-audit
+    #   virtualenv
 pluggy==1.6.0
+    # via pytest
 pre-commit==3.8.0
 propcache==0.3.1
+    # via
+    #   aiohttp
+    #   yarl
 py-serializable==2.0.0
+    # via cyclonedx-python-lib
 pydantic==2.11.5
+    # via
+    #   fastapi
+    #   gradio
+    #   mcp
+    #   pydantic-settings
 pydantic-core==2.33.2
+    # via pydantic
 pydantic-settings==2.9.1
+    # via mcp
 pydub==0.25.1
+    # via gradio
 pygments==2.19.1
+    # via rich
 pyparsing==3.2.3
+    # via pip-requirements-parser
 pytest==7.4.4
+    # via
+    #   pytest-cov
+    #   pytest-randomly
 pytest-cov==4.1.0
 pytest-randomly==3.16.0
 python-dateutil==2.9.0.post0
+    # via
+    #   pandas
+    #   python-whois
 python-dotenv==1.1.0
+    # via pydantic-settings
 python-multipart==0.0.20
+    # via
+    #   gradio
+    #   mcp
 python-whois==0.9.5
+    # via tdagent
 pytz==2025.2
+    # via pandas
 pyyaml==6.0.2
+    # via
+    #   gradio
+    #   huggingface-hub
+    #   pre-commit
 requests==2.32.3
+    # via
+    #   cachecontrol
+    #   huggingface-hub
+    #   pip-audit
+    #   tdagent
 rich==14.0.0
+    # via
+    #   pip-audit
+    #   typer
 ruff==0.11.12
+    # via gradio
 safehttpx==0.1.6
+    # via gradio
 semantic-version==2.10.0
+    # via gradio
 shellingham==1.5.4 ; sys_platform != 'emscripten'
+    # via typer
 six==1.17.0
+    # via python-dateutil
 sniffio==1.3.1
+    # via anyio
 sortedcontainers==2.4.0
+    # via cyclonedx-python-lib
 sse-starlette==2.3.6
+    # via mcp
 starlette==0.46.2
+    # via
+    #   fastapi
+    #   gradio
+    #   mcp
 toml==0.10.2
+    # via pip-audit
 tomli==2.2.1 ; python_full_version <= '3.11'
+    # via
+    #   coverage
+    #   mypy
+    #   pytest
 tomlkit==0.13.2
+    # via gradio
 tqdm==4.67.1
+    # via huggingface-hub
 typer==0.16.0 ; sys_platform != 'emscripten'
+    # via gradio
 typing-extensions==4.14.0
+    # via
+    #   anyio
+    #   exceptiongroup
+    #   fastapi
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   multidict
+    #   mypy
+    #   pydantic
+    #   pydantic-core
+    #   rich
+    #   typer
+    #   typing-inspection
+    #   uvicorn
 typing-inspection==0.4.1
+    # via
+    #   pydantic
+    #   pydantic-settings
 tzdata==2025.2
+    # via pandas
 urllib3==2.4.0
+    # via
+    #   gradio
+    #   requests
 uvicorn==0.34.3 ; sys_platform != 'emscripten'
+    # via
+    #   gradio
+    #   mcp
 virtualenv==20.31.2
+    # via pre-commit
 vt-py==0.21.0
+    # via tdagent
 websockets==15.0.1
+    # via gradio-client
 xdoctest==1.2.0
 yarl==1.20.0
+    # via aiohttp

requirements.txt

@@ -1,84 +1,212 @@
 # This file was autogenerated by uv via the following command:
-#    uv export --format requirements-txt --no-hashes --no-dev -o requirements.txt
+#    uv pip compile pyproject.toml -o requirements.txt
 aiofiles==24.1.0
+    # via
+    #   gradio
+    #   vt-py
 aiohappyeyeballs==2.6.1
+    # via aiohttp
 aiohttp==3.12.8
+    # via vt-py
 aiosignal==1.3.2
+    # via aiohttp
 annotated-types==0.7.0
+    # via pydantic
 anyio==4.9.0
-async-timeout==5.0.1 ; python_full_version < '3.11'
+    # via
+    #   gradio
+    #   httpx
+    #   mcp
+    #   sse-starlette
+    #   starlette
 attrs==25.3.0
-audioop-lts==0.2.1 ; python_full_version >= '3.13'
+    # via aiohttp
+audioop-lts==0.2.1
+    # via gradio
 cachetools==6.0.0
+    # via tdagent (pyproject.toml)
 certifi==2025.4.26
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
 charset-normalizer==3.4.2
-click==8.2.1 ; sys_platform != 'emscripten'
-colorama==0.4.6 ; sys_platform == 'win32'
-coverage==7.8.2
-exceptiongroup==1.3.0 ; python_full_version < '3.11'
+    # via requests
+click==8.2.1
+    # via
+    #   typer
+    #   uvicorn
+dnspython==2.7.0
+    # via tdagent (pyproject.toml)
 fastapi==0.115.12
+    # via gradio
 ffmpy==0.6.0
+    # via gradio
 filelock==3.18.0
+    # via huggingface-hub
 frozenlist==1.6.2
+    # via
+    #   aiohttp
+    #   aiosignal
 fsspec==2025.5.1
+    # via
+    #   gradio-client
+    #   huggingface-hub
 gradio==5.32.1
+    # via tdagent (pyproject.toml)
 gradio-client==1.10.2
+    # via gradio
 groovy==0.1.2
+    # via gradio
 h11==0.16.0
-hf-xet==1.1.2 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'
+    # via
+    #   httpcore
+    #   uvicorn
+hf-xet==1.1.2
+    # via huggingface-hub
 httpcore==1.0.9
+    # via httpx
 httpx==0.28.1
+    # via
+    #   gradio
+    #   gradio-client
+    #   mcp
+    #   safehttpx
 httpx-sse==0.4.0
+    # via mcp
 huggingface-hub==0.32.4
+    # via
+    #   gradio
+    #   gradio-client
 idna==3.10
-iniconfig==2.1.0
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
 jinja2==3.1.6
-markdown-it-py==3.0.0 ; sys_platform != 'emscripten'
+    # via gradio
+markdown-it-py==3.0.0
+    # via rich
 markupsafe==3.0.2
+    # via
+    #   gradio
+    #   jinja2
 mcp==1.9.0
-mdurl==0.1.2 ; sys_platform != 'emscripten'
+    # via gradio
+mdurl==0.1.2
+    # via markdown-it-py
 multidict==6.4.4
+    # via
+    #   aiohttp
+    #   yarl
 numpy==2.2.6
+    # via
+    #   gradio
+    #   pandas
 orjson==3.10.18
+    # via gradio
 packaging==25.0
+    # via
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
 pandas==2.2.3
+    # via gradio
 pillow==11.2.1
-pluggy==1.6.0
+    # via gradio
 propcache==0.3.1
+    # via
+    #   aiohttp
+    #   yarl
 pydantic==2.11.5
+    # via
+    #   fastapi
+    #   gradio
+    #   mcp
+    #   pydantic-settings
 pydantic-core==2.33.2
+    # via pydantic
 pydantic-settings==2.9.1
+    # via mcp
 pydub==0.25.1
-pygments==2.19.1 ; sys_platform != 'emscripten'
-pytest==7.4.4
-pytest-cov==4.1.0
-pytest-randomly==3.16.0
+    # via gradio
+pygments==2.19.1
+    # via rich
 python-dateutil==2.9.0.post0
+    # via
+    #   pandas
+    #   python-whois
 python-dotenv==1.1.0
+    # via pydantic-settings
 python-multipart==0.0.20
+    # via
+    #   gradio
+    #   mcp
 python-whois==0.9.5
+    # via tdagent (pyproject.toml)
 pytz==2025.2
+    # via pandas
 pyyaml==6.0.2
+    # via
+    #   gradio
+    #   huggingface-hub
 requests==2.32.3
-rich==14.0.0 ; sys_platform != 'emscripten'
-ruff==0.11.12 ; sys_platform != 'emscripten'
+    # via
+    #   tdagent (pyproject.toml)
+    #   huggingface-hub
+rich==14.0.0
+    # via typer
+ruff==0.11.12
+    # via gradio
 safehttpx==0.1.6
+    # via gradio
 semantic-version==2.10.0
-shellingham==1.5.4 ; sys_platform != 'emscripten'
+    # via gradio
+shellingham==1.5.4
+    # via typer
 six==1.17.0
+    # via python-dateutil
 sniffio==1.3.1
+    # via anyio
 sse-starlette==2.3.6
+    # via mcp
 starlette==0.46.2
-tomli==2.2.1 ; python_full_version <= '3.11'
+    # via
+    #   fastapi
+    #   gradio
+    #   mcp
 tomlkit==0.13.2
+    # via gradio
 tqdm==4.67.1
-typer==0.16.0 ; sys_platform != 'emscripten'
+    # via huggingface-hub
+typer==0.16.0
+    # via gradio
 typing-extensions==4.14.0
+    # via
+    #   fastapi
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   pydantic
+    #   pydantic-core
+    #   typer
+    #   typing-inspection
 typing-inspection==0.4.1
+    # via
+    #   pydantic
+    #   pydantic-settings
 tzdata==2025.2
+    # via pandas
 urllib3==2.4.0
-uvicorn==0.34.3 ; sys_platform != 'emscripten'
+    # via requests
+uvicorn==0.34.3
+    # via
+    #   gradio
+    #   mcp
 vt-py==0.21.0
+    # via tdagent (pyproject.toml)
 websockets==15.0.1
-xdoctest==1.2.0
+    # via gradio-client
 yarl==1.20.0
+    # via aiohttp

subdomains/subdomains.txt

@@ -0,0 +1,999 @@
+www
+mail
+ftp
+localhost
+webmail
+smtp
+pop
+ns1
+webdisk
+ns2
+cpanel
+whm
+autodiscover
+autoconfig
+m
+imap
+test
+ns
+blog
+pop3
+dev
+www2
+admin
+forum
+news
+vpn
+ns3
+mail2
+new
+mysql
+old
+lists
+support
+mobile
+mx
+static
+docs
+beta
+shop
+sql
+secure
+demo
+cp
+calendar
+wiki
+web
+media
+email
+images
+img
+www1
+intranet
+portal
+video
+sip
+dns2
+api
+cdn
+stats
+dns1
+ns4
+www3
+dns
+search
+staging
+server
+mx1
+chat
+wap
+my
+svn
+mail1
+sites
+proxy
+ads
+host
+crm
+cms
+backup
+mx2
+lyncdiscover
+info
+apps
+download
+remote
+db
+forums
+store
+relay
+files
+newsletter
+app
+live
+owa
+en
+start
+sms
+office
+exchange
+ipv4
+mail3
+help
+blogs
+helpdesk
+web1
+home
+library
+ftp2
+ntp
+monitor
+login
+service
+correo
+www4
+moodle
+it
+gateway
+gw
+i
+stat
+stage
+ldap
+tv
+ssl
+web2
+ns5
+upload
+nagios
+smtp2
+online
+ad
+survey
+data
+radio
+extranet
+test2
+mssql
+dns3
+jobs
+services
+panel
+irc
+hosting
+cloud
+de
+gmail
+s
+bbs
+cs
+ww
+mrtg
+git
+image
+members
+poczta
+s1
+meet
+preview
+fr
+cloudflare-resolve-to
+dev2
+photo
+jabber
+legacy
+go
+es
+ssh
+redmine
+partner
+vps
+server1
+sv
+ns6
+webmail2
+av
+community
+cacti
+time
+sftp
+lib
+facebook
+www5
+smtp1
+feeds
+w
+games
+ts
+alumni
+dl
+s2
+phpmyadmin
+archive
+cn
+tools
+stream
+projects
+elearning
+im
+iphone
+control
+voip
+test1
+ws
+rss
+sp
+wwww
+vpn2
+jira
+list
+connect
+gallery
+billing
+mailer
+update
+pda
+game
+ns0
+testing
+sandbox
+job
+events
+dialin
+ml
+fb
+videos
+music
+a
+partners
+mailhost
+downloads
+reports
+ca
+router
+speedtest
+local
+training
+edu
+bugs
+manage
+s3
+status
+host2
+ww2
+marketing
+conference
+content
+network-ip
+broadcast-ip
+english
+catalog
+msoid
+mailadmin
+pay
+access
+streaming
+project
+t
+sso
+alpha
+photos
+staff
+e
+auth
+v2
+web5
+web3
+mail4
+devel
+post
+us
+images2
+master
+rt
+ftp1
+qa
+wp
+dns4
+www6
+ru
+student
+w3
+citrix
+trac
+doc
+img2
+css
+mx3
+adm
+web4
+hr
+mailserver
+travel
+sharepoint
+sport
+member
+bb
+agenda
+link
+server2
+vod
+uk
+fw
+promo
+vip
+noc
+design
+temp
+gate
+ns7
+file
+ms
+map
+cache
+painel
+js
+event
+mailing
+db1
+c
+auto
+img1
+vpn1
+business
+mirror
+share
+cdn2
+site
+maps
+tickets
+tracker
+domains
+club
+images1
+zimbra
+cvs
+b2b
+oa
+intra
+zabbix
+ns8
+assets
+main
+spam
+lms
+social
+faq
+feedback
+loopback
+groups
+m2
+cas
+loghost
+xml
+nl
+research
+art
+munin
+dev1
+gis
+sales
+images3
+report
+google
+idp
+cisco
+careers
+seo
+dc
+lab
+d
+firewall
+fs
+eng
+ann
+mail01
+mantis
+v
+affiliates
+webconf
+track
+ticket
+pm
+db2
+b
+clients
+tech
+erp
+monitoring
+cdn1
+images4
+payment
+origin
+client
+foto
+domain
+pt
+pma
+directory
+cc
+public
+finance
+ns11
+test3
+wordpress
+corp
+sslvpn
+cal
+mailman
+book
+ip
+zeus
+ns10
+hermes
+storage
+free
+static1
+pbx
+banner
+mobil
+kb
+mail5
+direct
+ipfixe
+wifi
+development
+board
+ns01
+st
+reviews
+radius
+pro
+atlas
+links
+in
+oldmail
+register
+s4
+images6
+static2
+id
+shopping
+drupal
+analytics
+m1
+images5
+images7
+img3
+mx01
+www7
+redirect
+sitebuilder
+smtp3
+adserver
+net
+user
+forms
+outlook
+press
+vc
+health
+work
+mb
+mm
+f
+pgsql
+jp
+sports
+preprod
+g
+p
+mdm
+ar
+lync
+market
+dbadmin
+barracuda
+affiliate
+mars
+users
+images8
+biblioteca
+mc
+ns12
+math
+ntp1
+web01
+software
+pr
+jupiter
+labs
+linux
+sc
+love
+fax
+php
+lp
+tracking
+thumbs
+up
+tw
+campus
+reg
+digital
+demo2
+da
+tr
+otrs
+web6
+ns02
+mailgw
+education
+order
+piwik
+banners
+rs
+se
+venus
+internal
+webservices
+cm
+whois
+sync
+lb
+is
+code
+click
+w2
+bugzilla
+virtual
+origin-www
+top
+customer
+pub
+hotel
+openx
+log
+uat
+cdn3
+images0
+cgi
+posta
+reseller
+soft
+movie
+mba
+n
+r
+developer
+nms
+ns9
+webcam
+construtor
+ebook
+ftp3
+join
+dashboard
+bi
+wpad
+admin2
+agent
+wm
+books
+joomla
+hotels
+ezproxy
+ds
+sa
+katalog
+team
+emkt
+antispam
+adv
+mercury
+flash
+myadmin
+sklep
+newsite
+law
+pl
+ntp2
+x
+srv1
+mp3
+archives
+proxy2
+ps
+pic
+ir
+orion
+srv
+mt
+ocs
+server3
+meeting
+v1
+delta
+titan
+manager
+subscribe
+develop
+wsus
+oascentral
+mobi
+people
+galleries
+wwwtest
+backoffice
+sg
+repo
+soporte
+www8
+eu
+ead
+students
+hq
+awstats
+ec
+security
+school
+corporate
+podcast
+vote
+conf
+magento
+mx4
+webservice
+tour
+s5
+power
+correio
+mon
+mobilemail
+weather
+international
+prod
+account
+xx
+pages
+pgadmin
+bfn2
+webserver
+www-test
+maintenance
+me
+magazine
+syslog
+int
+view
+enews
+ci
+au
+mis
+dev3
+pdf
+mailgate
+v3
+ss
+internet
+host1
+smtp01
+journal
+wireless
+opac
+w1
+signup
+database
+demo1
+br
+android
+career
+listserv
+bt
+spb
+cam
+contacts
+webtest
+resources
+1
+life
+mail6
+transfer
+app1
+confluence
+controlpanel
+secure2
+puppet
+classifieds
+tunet
+edge
+biz
+host3
+red
+newmail
+mx02
+sb
+physics
+ap
+epaper
+sts
+proxy1
+ww1
+stg
+sd
+science
+star
+www9
+phoenix
+pluto
+webdav
+booking
+eshop
+edit
+panelstats
+xmpp
+food
+cert
+adfs
+mail02
+cat
+edm
+vcenter
+mysql2
+sun
+phone
+surveys
+smart
+system
+twitter
+updates
+webmail1
+logs
+sitedefender
+as
+cbf1
+sugar
+contact
+vm
+ipad
+traffic
+dm
+saturn
+bo
+network
+ac
+ns13
+webdev
+libguides
+asp
+tm
+core
+mms
+abc
+scripts
+fm
+sm
+test4
+nas
+newsletters
+rsc
+cluster
+learn
+panelstatsmail
+lb1
+usa
+apollo
+pre
+terminal
+l
+tc
+movies
+sh
+fms
+dms
+z
+base
+jwc
+gs
+kvm
+bfn1
+card
+web02
+lg
+editor
+metrics
+feed
+repository
+asterisk
+sns
+global
+counter
+ch
+sistemas
+pc
+china
+u
+payments
+ma
+pics
+www10
+e-learning
+auction
+hub
+sf
+cbf8
+forum2
+ns14
+app2
+passport
+hd
+talk
+ex
+debian
+ct
+rc
+2012
+imap4
+blog2
+ce
+sk
+relay2
+green
+print
+geo
+multimedia
+iptv
+backup2
+webapps
+audio
+ro
+smtp4
+pg
+ldap2
+backend
+profile
+oldwww
+drive
+bill
+listas
+orders
+win
+mag
+apply
+bounce
+mta
+hp
+suporte
+dir
+pa
+sys
+mx0
+ems
+antivirus
+web8
+inside
+play
+nic
+welcome
+premium
+exam
+sub
+cz
+omega
+boutique
+pp
+management
+planet
+ww3
+orange
+c1
+zzb
+form
+ecommerce
+tmp
+plus
+openvpn
+fw1
+hk
+owncloud
+history
+clientes
+srv2
+img4
+open
+registration
+mp
+blackboard
+fc
+static3
+server4
+s6
+ecard
+dspace
+dns01
+md
+mcp
+ares
+spf
+kms
+intranet2
+accounts
+webapp
+ask
+rd
+www-dev
+gw2
+mall
+bg
+teste
+ldap1
+real
+m3
+wave
+movil
+portal2
+kids
+gw1
+ra
+tienda
+private
+po
+2013
+cdn4
+gps
+km
+ent
+tt
+ns21
+at
+athena
+cbf2
+webmail3
+mob
+matrix
+ns15
+send
+lb2
+pos
+2
+cl
+renew
+admissions
+am
+beta2
+gamma
+mx5
+portfolio
+contest
+box
+mg
+wwwold
+neptune
+mac
+pms
+traveler
+media2
+studio
+sw
+imp
+bs
+alfa
+cbf4
+servicedesk
+wmail
+video2
+switch
+sam
+sky
+ee
+widget
+reklama
+msn
+paris
+tms
+th
+vega
+trade
+intern
+ext
+oldsite
+learning
+group
+f1
+ns22
+ns20
+demo3
+bm
+dom
+pe
+annuaire
+portail
+graphics
+iris
+one
+robot
+ams
+s7
+foro
+gaia
+vpn3

tdagent/tools/get_domain_information.py

@@ -0,0 +1,334 @@
+import json
+import os
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+import gradio as gr
+import requests
+import urllib3
+from dns import message
+
+
+_DNS_SERVER  = "https://dns.google/dns-query" # can use others
+_DNS_RECORD_TYPES = [
+    "A",
+    "AAAA",
+    "CNAME",
+    "MX",
+    "NS",
+    "SOA",
+    "TXT",
+    "RP",
+    "LOC",
+    "CAA",
+    "SPF",
+    "SRV",
+    "NSEC",
+    "RRSIG",
+]
+
+_COMMON_SUBDOMAINS_TXT_PATH = Path("./subdomains/subdomains.txt")
+
+
+def get_geolocation(ip: str) -> dict[str, Any] | str:
+    """Get location information from an ip address.
+
+    Returns the following information on an ip address:
+        1. IPv4
+        2. city
+        4. country_code
+        5. country_name
+        6. latitude
+        7. longitude
+        8. postal
+        9. state
+
+    Example:
+    >>> from pprint import pprint
+    >>> pprint(get_location("103.100.104.0"))
+    ... {'IPv4': '103.100.104.0',
+        'city': None,
+        'country_code': 'NZ',
+        'country_name': 'New Zealand',
+        'latitude': -41,
+        'longitude': 174,
+        'postal': None,
+        'state': None}
+
+    Args:
+        ip: ip address
+
+    Returns:
+        Location information on the ip address.
+    """
+    try:
+        return requests.get(
+            f"https://geolocation-db.com/json/{ip}",
+            timeout=0.5,
+        ).json()
+    except Exception as e:  # noqa: BLE001
+        return str(e)
+
+
+def _request_dns_record(domain: str, record_type: str) -> str:
+    """Utility to build dns resolve requests that do not use port 53.
+
+    Args:
+        domain: domain to investigate
+        record_type: record type
+
+    Returns:
+        Information about the dns record type for the domain.
+    """
+    q = message.make_query(domain, record_type)
+    response = requests.post(
+        _DNS_SERVER,
+        headers={
+            "Content-Type": "application/dns-message",
+            "Accept": "application/dns-message",
+        },
+        data=q.to_wire(),
+        verify=True,
+        timeout=0.2,
+    )
+    dns_message = message.from_wire(response.content)
+    return [str(rdata) for rdata in dns_message.answer[0]] if dns_message.answer else []
+
+# see: https://thepythoncode.com/article/dns-enumeration-with-python
+# https://dnspython.readthedocs.io
+def enumerate_dns(domain_name: str) -> dict[str, Any] | None:
+    r"""Enumerates information about a specific domain's DNS configuration.
+
+    Information collected about the domain name:
+        1. A records: the IPv4 associated with the domain
+        2. AAAA records: the IPv6 associated with the domain
+        3. CAA records: used by owners to specify which Certificate Authorities
+            are authorized to issue SSL/TLS certificates for their domains.
+        4. CNAME records: alias of one name to another - the DNS lookup will
+            continue by retrying the lookup with the new name.
+        5. LOC records: geographic location associated with a domain name.
+        6. MX records: associated email servers to the domain.
+        7. NS records: DNS servers that are authoritative for a particular domain.
+            These may be use to inquire information about the domain.
+        8. SOA records: defines authoritative information about a DNS zone,
+            including zone transfers and cache expiration.
+        9. TXT records: used for domain verification and email security.
+        10. RP records: the responsible person for a domain.
+        11. SPF records: defines authorized email servers.
+        12. SRV records: specifies location of specific services
+            (port and host) for the domain.
+        14. NSEC records: proves non-existence of DNS records
+            and prevents zone enumeration.
+        15. RRSIG records: contains cryptographic signatures for DNSSEC-signed
+            records, providing authentication and integrity.
+
+    Example:
+    >>> from pprint import pprint
+    >>> pprint(enumerate_dns("youtube.com"))
+    ... {'A': 'youtube.com. 300 IN A 142.250.200.142',
+        'AAAA': 'youtube.com. 286 IN AAAA 2a00:1450:4003:80f::200e',
+        'CAA': 'youtube.com. 14352 IN CAA 0 issue "pki.goog"',
+        'CNAME': None,
+        'LOC': None,
+        'MX': 'youtube.com. 300 IN MX 0 smtp.google.com.',
+        'NS': 'youtube.com. 21600 IN NS ns4.google.com.\n'
+            'youtube.com. 21600 IN NS ns1.google.com.\n'
+            'youtube.com. 21600 IN NS ns2.google.com.\n'
+            'youtube.com. 21600 IN NS ns3.google.com.',
+        'NSEC': None,
+        'RP': None,
+        'RRSIG': None,
+        'SOA': 'youtube.com. 60 IN SOA ns1.google.com. dns-admin.google.com. '
+                '766113658 900 900 1800 60',
+        'SPF': None,
+        'SRV': None,
+        'TXT': 'youtube.com. 3586 IN TXT "v=spf1 include:google.com mx -all"\n'
+                'youtube.com. 3586 IN TXT '
+                '"facebook-domain-verification=64jdes7le4h7e7lfpi22rijygx58j1"\n'
+                'youtube.com. 3586 IN TXT '
+                '"google-site-verification=QtQWEwHWM8tHiJ4s-jJWzEQrD_fF3luPnpzNDH-Nw-w"'}
+
+    Args:
+        domain_name: domain name for which to
+            enumerate the DNS configuration.
+
+    Returns:
+        The domain's DNS configuration.
+    """
+    enumeration = {}
+    for record_type in _DNS_RECORD_TYPES:
+        try:
+            record = _request_dns_record(domain_name, record_type)
+            if record:
+                enumeration[record_type] = record
+        except Exception as e:  # noqa: BLE001, PERF203
+            enumeration[record_type] = str(e)
+    return enumeration if enumeration else None
+
+def resolve_subdomain_ipv4(domain: str) -> str | None:
+    """Resolve the IPv4 address of a domain.
+
+    Args:
+        domain: domain name
+
+    Returns:
+        The domain is returned provided
+            it was resolved. Otherwise nothing
+            is returned.
+    """
+    try:
+        _request_dns_record(domain, "A")
+        return domain  # noqa: TRY300
+    except Exception:  # noqa: BLE001
+        return None
+
+
+def scrap_subdomains_for_domain(domain_name: str) -> list[str]:
+    """Retrieves subdomains associated to a domain if any.
+
+    The information retrieved from a domain is its subdomains
+    provided they are the top 1000 subdomain prefixes as
+    indicated by https://github.com/rbsec/dnscan/tree/master
+
+    Importantly, it finds subdomains only if their prefixes
+    are along the top 1000 most common. Hence, it may not
+    yield all the subdomains associated to the domain.
+
+    Example:
+    >>> scrap_subdomains_for_domain("github.com")
+    ... ['www.github.com', 'smtp.github.com', 'ns1.github.com',
+        'ns2.github.com','autodiscover.github.com', 'test.github.com',
+        'blog.github.com', 'admin.github.com', 'support.github.com',
+        'docs.github.com', 'shop.github.com', 'wiki.github.com',
+        'api.github.com', 'live.github.com', 'help.github.com',
+        'jobs.github.com', 'services.github.com', 'de.github.com',
+        'cs.github.com', 'fr.github.com', 'ssh.github.com',
+        'partner.github.com', 'community.github.com',
+        'mailer.github.com', 'training.github.com', ...]
+
+    Args:
+        domain_name: domain name for which to retrieve a
+            list of subdomains
+
+    Returns:
+        List of subdomains if any.
+    """
+    try:
+        with open(_COMMON_SUBDOMAINS_TXT_PATH) as file:  # noqa: PTH123
+            subdomains = [line.strip() for line in file if line.strip()]
+    except FileNotFoundError:
+        return []
+
+    potential_subdomains = [f"{subdomain}.{domain_name}" for subdomain in subdomains]
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        results = executor.map(resolve_subdomain_ipv4, potential_subdomains)
+        return [domain for domain in results if domain]
+
+def retrieve_ioc_from_threatfox(potentially_ioc: str) -> str:
+    r"""Retrieves information about a potential IoC from ThreatFox.
+
+    It may be used to retrieve information of indicators of compromise
+    (IOCs) associated with malware, with the infosec community, AV
+    vendors and cyber threat intelligence providers.
+
+    Examples:
+    >>> retrieve_ioc_from_threatfox("139.180.203.104")
+    ... {
+    "query_status": "ok",
+    "data": [
+        {
+            "id": "12",
+            "ioc": "139.180.203.104:443",
+            "threat_type": "botnet_cc",
+            "threat_type_desc": "Indicator that identifies a botnet command&control...",
+            "ioc_type": "ip:port",
+            "ioc_type_desc": "ip:port combination that is used for botnet Command&...,
+            "malware": "win.cobalt_strike",
+            "malware_printable": "Cobalt Strike",
+            "malware_alias": "Agentemis,BEACON,CobaltStrike",
+            "malware_malpedia": "https:\/\/malpedia.caad.fkie.fraunhofer.de\/...",
+            "confidence_level": 75,
+            "first_seen": "2020-12-06 09:10:23 UTC",
+            "last_seen": null,
+            "reference": null,
+            "reporter": "abuse_ch",
+            "tags": null,
+            "malware_samples": [
+                {
+                    "time_stamp": "2021-03-23 08:18:06 UTC",
+                    "md5_hash": "5b7e82e051ade4b14d163eea2a17bf8b",
+                    "sha256_hash": "b325c92fa540edeb89b95dbfd4400c1cb33599c66859....",
+                    "malware_bazaar": "https:\/\/bazaar.abuse.ch\/sample\/b325c...\/"
+                },
+            ]
+
+        }
+    ]
+    }
+
+    Args:
+        potentially_ioc: this can be a url, a domain, a hash,
+            or any other type of IoC.
+
+    Returns:
+        Information of the input as an IoC: threat type, malware type andsamples,
+            confidence level, first/last seen dates, and more IoC information.
+    """
+    headers = {"Auth-Key": os.environ["THREATFOX_APIKEY"]}
+    pool = urllib3.HTTPSConnectionPool(
+        "threatfox-api.abuse.ch",
+        port=443,
+        maxsize=50,
+        headers=headers,
+    )
+    data = {
+        "query": "search_ioc",
+        "search_term": potentially_ioc,
+    }
+    json_data = json.dumps(data)
+    try:
+        response = pool.request("POST", "/api/v1/", body=json_data)
+        return response.data.decode("utf-8", "ignore")
+    except Exception as e:  # noqa: BLE001
+        return str(e)
+
+
+geo_location_tool = gr.Interface(
+    fn=get_geolocation,
+    inputs=["text"],
+    outputs=["text"],
+    title="Domain Associated Geolocation Finder",
+    description="Retrieves the geolocation associated to an input ip address",
+    theme="default",
+)
+
+dns_enumeration_tool = gr.Interface(
+    fn=enumerate_dns,
+    inputs=["text"],
+    outputs=["text"],
+    title="DNS record enumerator of domains",
+    description="Retrieves several dns record types for the input domain names",
+    theme="default",
+)
+
+scrap_subdomains_tool = gr.Interface(
+    fn=scrap_subdomains_for_domain,
+    inputs=["text"],
+    outputs=["text"],
+    title="Subdomains Extractor of domains",
+    description="Retrieves the subdomains for the input domain if they are common",
+    theme="default",
+)
+
+extractor_of_ioc_from_threatfox_tool = gr.Interface(
+    fn=retrieve_ioc_from_threatfox,
+    inputs=["text"],
+    outputs=["text"],
+    title="IoC information extractor associated to particular entities",
+    description=(
+        "If information as an Indicator of Compromise (IoC) exists"
+        "for the input url, domain or hash, it retrieves it"
+    ),
+    theme="default",
+)