/amass/datamgmtsrv.go

https://github.com/anshumanbh/Amass · Go · 427 lines · 389 code · 31 blank · 7 comment · 76 complexity · f1e2e9836a62d1eca5c6d3e1e7020856 MD5 · raw file

  1. // Copyright 2017 Jeff Foley. All rights reserved.
  2. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
  3. package amass
  4. import (
  5. "regexp"
  6. "strings"
  7. "time"
  8. "github.com/OWASP/Amass/amass/core"
  9. "github.com/OWASP/Amass/amass/handlers"
  10. "github.com/OWASP/Amass/amass/utils"
  11. "github.com/miekg/dns"
  12. )
  13. // DataManagerService is the Service that handles all data collected
  14. // within the architecture. This is achieved by watching all the RESOLVED events.
  15. type DataManagerService struct {
  16. core.BaseService
  17. Handlers []handlers.DataHandler
  18. domainFilter *utils.StringFilter
  19. }
  20. // NewDataManagerService returns he object initialized, but not yet started.
  21. func NewDataManagerService(config *core.Config, bus *core.EventBus) *DataManagerService {
  22. dms := &DataManagerService{domainFilter: utils.NewStringFilter()}
  23. dms.BaseService = *core.NewBaseService(dms, "Data Manager", config, bus)
  24. return dms
  25. }
  26. // OnStart implements the Service interface
  27. func (dms *DataManagerService) OnStart() error {
  28. dms.BaseService.OnStart()
  29. dms.Bus().Subscribe(core.NameResolvedTopic, dms.SendDNSRequest)
  30. go dms.processRequests()
  31. return nil
  32. }
  33. // AddDataHandler provides the Data Manager with another DataHandler.
  34. func (dms *DataManagerService) AddDataHandler(handler handlers.DataHandler) {
  35. dms.Handlers = append(dms.Handlers, handler)
  36. }
  37. func (dms *DataManagerService) processRequests() {
  38. for {
  39. select {
  40. case <-dms.PauseChan():
  41. <-dms.ResumeChan()
  42. case <-dms.Quit():
  43. return
  44. case req := <-dms.DNSRequestChan():
  45. dms.manageData(req)
  46. case <-dms.AddrRequestChan():
  47. case <-dms.ASNRequestChan():
  48. case <-dms.WhoisRequestChan():
  49. }
  50. }
  51. }
  52. func (dms *DataManagerService) manageData(req *core.DNSRequest) {
  53. req.Name = strings.ToLower(req.Name)
  54. req.Domain = strings.ToLower(req.Domain)
  55. dms.SetActive()
  56. dms.insertDomain(req.Domain)
  57. for i, r := range req.Records {
  58. req.Records[i].Name = strings.ToLower(r.Name)
  59. req.Records[i].Data = strings.ToLower(r.Data)
  60. switch uint16(r.Type) {
  61. case dns.TypeA:
  62. dms.insertA(req, i)
  63. case dns.TypeAAAA:
  64. dms.insertAAAA(req, i)
  65. case dns.TypeCNAME:
  66. dms.insertCNAME(req, i)
  67. case dns.TypePTR:
  68. dms.insertPTR(req, i)
  69. case dns.TypeSRV:
  70. dms.insertSRV(req, i)
  71. case dns.TypeNS:
  72. dms.insertNS(req, i)
  73. case dns.TypeMX:
  74. dms.insertMX(req, i)
  75. case dns.TypeTXT:
  76. dms.insertTXT(req, i)
  77. case dns.TypeSPF:
  78. dms.insertSPF(req, i)
  79. }
  80. }
  81. }
  82. func (dms *DataManagerService) checkDomain(domain string) bool {
  83. return dms.domainFilter.Duplicate(domain)
  84. }
  85. func (dms *DataManagerService) insertDomain(domain string) {
  86. domain = strings.ToLower(domain)
  87. if domain == "" || dms.checkDomain(domain) {
  88. return
  89. }
  90. for _, handler := range dms.Handlers {
  91. err := handler.Insert(&handlers.DataOptsParams{
  92. UUID: dms.Config().UUID.String(),
  93. Timestamp: time.Now().Format(time.RFC3339),
  94. Type: handlers.OptDomain,
  95. Domain: domain,
  96. Tag: core.DNS,
  97. Source: "Forward DNS",
  98. })
  99. if err != nil {
  100. dms.Config().Log.Printf("%s failed to insert domain: %v", handler, err)
  101. }
  102. }
  103. dms.Bus().Publish(core.NewNameTopic, &core.DNSRequest{
  104. Name: domain,
  105. Domain: domain,
  106. Tag: core.DNS,
  107. Source: "Forward DNS",
  108. })
  109. }
  110. func (dms *DataManagerService) insertCNAME(req *core.DNSRequest, recidx int) {
  111. target := core.RemoveLastDot(req.Records[recidx].Data)
  112. if target == "" {
  113. return
  114. }
  115. domain := strings.ToLower(core.SubdomainToDomain(target))
  116. if domain == "" {
  117. return
  118. }
  119. dms.insertDomain(domain)
  120. for _, handler := range dms.Handlers {
  121. err := handler.Insert(&handlers.DataOptsParams{
  122. UUID: dms.Config().UUID.String(),
  123. Timestamp: time.Now().Format(time.RFC3339),
  124. Type: handlers.OptCNAME,
  125. Name: req.Name,
  126. Domain: req.Domain,
  127. TargetName: target,
  128. TargetDomain: domain,
  129. Tag: req.Tag,
  130. Source: req.Source,
  131. })
  132. if err != nil {
  133. dms.Config().Log.Printf("%s failed to insert CNAME: %v", handler, err)
  134. }
  135. }
  136. dms.Bus().Publish(core.NewNameTopic, &core.DNSRequest{
  137. Name: target,
  138. Domain: domain,
  139. Tag: core.DNS,
  140. Source: "Forward DNS",
  141. })
  142. }
  143. func (dms *DataManagerService) insertA(req *core.DNSRequest, recidx int) {
  144. addr := strings.TrimSpace(req.Records[recidx].Data)
  145. if addr == "" {
  146. return
  147. }
  148. for _, handler := range dms.Handlers {
  149. err := handler.Insert(&handlers.DataOptsParams{
  150. UUID: dms.Config().UUID.String(),
  151. Timestamp: time.Now().Format(time.RFC3339),
  152. Type: handlers.OptA,
  153. Name: req.Name,
  154. Domain: req.Domain,
  155. Address: addr,
  156. Tag: req.Tag,
  157. Source: req.Source,
  158. })
  159. if err != nil {
  160. dms.Config().Log.Printf("%s failed to insert A record: %v", handler, err)
  161. }
  162. }
  163. dms.insertInfrastructure(addr)
  164. if dms.Config().IsDomainInScope(req.Name) {
  165. dms.Bus().Publish(core.NewAddrTopic, &core.AddrRequest{
  166. Address: addr,
  167. Domain: req.Domain,
  168. Tag: req.Tag,
  169. Source: req.Source,
  170. })
  171. }
  172. }
  173. func (dms *DataManagerService) insertAAAA(req *core.DNSRequest, recidx int) {
  174. addr := strings.TrimSpace(req.Records[recidx].Data)
  175. if addr == "" {
  176. return
  177. }
  178. for _, handler := range dms.Handlers {
  179. err := handler.Insert(&handlers.DataOptsParams{
  180. UUID: dms.Config().UUID.String(),
  181. Timestamp: time.Now().Format(time.RFC3339),
  182. Type: handlers.OptAAAA,
  183. Name: req.Name,
  184. Domain: req.Domain,
  185. Address: addr,
  186. Tag: req.Tag,
  187. Source: req.Source,
  188. })
  189. if err != nil {
  190. dms.Config().Log.Printf("%s failed to insert AAAA record: %v", handler, err)
  191. }
  192. }
  193. dms.insertInfrastructure(addr)
  194. if dms.Config().IsDomainInScope(req.Name) {
  195. dms.Bus().Publish(core.NewAddrTopic, &core.AddrRequest{
  196. Address: addr,
  197. Domain: req.Domain,
  198. Tag: req.Tag,
  199. Source: req.Source,
  200. })
  201. }
  202. }
  203. func (dms *DataManagerService) insertPTR(req *core.DNSRequest, recidx int) {
  204. target := core.RemoveLastDot(req.Records[recidx].Data)
  205. if target == "" {
  206. return
  207. }
  208. domain := strings.ToLower(dms.Config().WhichDomain(target))
  209. if domain == "" {
  210. return
  211. }
  212. dms.insertDomain(domain)
  213. for _, handler := range dms.Handlers {
  214. err := handler.Insert(&handlers.DataOptsParams{
  215. UUID: dms.Config().UUID.String(),
  216. Timestamp: time.Now().Format(time.RFC3339),
  217. Type: handlers.OptPTR,
  218. Name: req.Name,
  219. Domain: domain,
  220. TargetName: target,
  221. Tag: req.Tag,
  222. Source: req.Source,
  223. })
  224. if err != nil {
  225. dms.Config().Log.Printf("%s failed to insert PTR record: %v", handler, err)
  226. }
  227. }
  228. dms.Bus().Publish(core.NewNameTopic, &core.DNSRequest{
  229. Name: target,
  230. Domain: domain,
  231. Tag: core.DNS,
  232. Source: req.Source,
  233. })
  234. }
  235. func (dms *DataManagerService) insertSRV(req *core.DNSRequest, recidx int) {
  236. service := core.RemoveLastDot(req.Records[recidx].Name)
  237. target := core.RemoveLastDot(req.Records[recidx].Data)
  238. if target == "" || service == "" {
  239. return
  240. }
  241. for _, handler := range dms.Handlers {
  242. err := handler.Insert(&handlers.DataOptsParams{
  243. UUID: dms.Config().UUID.String(),
  244. Timestamp: time.Now().Format(time.RFC3339),
  245. Type: handlers.OptSRV,
  246. Name: req.Name,
  247. Domain: req.Domain,
  248. Service: service,
  249. TargetName: target,
  250. Tag: req.Tag,
  251. Source: req.Source,
  252. })
  253. if err != nil {
  254. dms.Config().Log.Printf("%s failed to insert SRV record: %v", handler, err)
  255. }
  256. }
  257. if domain := dms.Config().WhichDomain(target); domain != "" {
  258. dms.Bus().Publish(core.NewNameTopic, &core.DNSRequest{
  259. Name: target,
  260. Domain: domain,
  261. Tag: req.Tag,
  262. Source: req.Source,
  263. })
  264. }
  265. }
  266. func (dms *DataManagerService) insertNS(req *core.DNSRequest, recidx int) {
  267. pieces := strings.Split(req.Records[recidx].Data, ",")
  268. target := pieces[len(pieces)-1]
  269. if target == "" {
  270. return
  271. }
  272. domain := strings.ToLower(core.SubdomainToDomain(target))
  273. if domain == "" {
  274. return
  275. }
  276. dms.insertDomain(domain)
  277. for _, handler := range dms.Handlers {
  278. err := handler.Insert(&handlers.DataOptsParams{
  279. UUID: dms.Config().UUID.String(),
  280. Timestamp: time.Now().Format(time.RFC3339),
  281. Type: handlers.OptNS,
  282. Name: req.Name,
  283. Domain: req.Domain,
  284. TargetName: target,
  285. TargetDomain: domain,
  286. Tag: req.Tag,
  287. Source: req.Source,
  288. })
  289. if err != nil {
  290. dms.Config().Log.Printf("%s failed to insert NS record: %v", handler, err)
  291. }
  292. }
  293. if target != domain {
  294. dms.Bus().Publish(core.NewNameTopic, &core.DNSRequest{
  295. Name: target,
  296. Domain: domain,
  297. Tag: core.DNS,
  298. Source: "Forward DNS",
  299. })
  300. }
  301. }
  302. func (dms *DataManagerService) insertMX(req *core.DNSRequest, recidx int) {
  303. target := core.RemoveLastDot(req.Records[recidx].Data)
  304. if target == "" {
  305. return
  306. }
  307. domain := strings.ToLower(core.SubdomainToDomain(target))
  308. if domain == "" {
  309. return
  310. }
  311. dms.insertDomain(domain)
  312. for _, handler := range dms.Handlers {
  313. err := handler.Insert(&handlers.DataOptsParams{
  314. UUID: dms.Config().UUID.String(),
  315. Timestamp: time.Now().Format(time.RFC3339),
  316. Type: handlers.OptMX,
  317. Name: req.Name,
  318. Domain: req.Domain,
  319. TargetName: target,
  320. TargetDomain: domain,
  321. Tag: req.Tag,
  322. Source: req.Source,
  323. })
  324. if err != nil {
  325. dms.Config().Log.Printf("%s failed to insert MX record: %v", handler, err)
  326. }
  327. }
  328. if target != domain {
  329. dms.Bus().Publish(core.NewNameTopic, &core.DNSRequest{
  330. Name: target,
  331. Domain: domain,
  332. Tag: core.DNS,
  333. Source: "Forward DNS",
  334. })
  335. }
  336. }
  337. func (dms *DataManagerService) insertTXT(req *core.DNSRequest, recidx int) {
  338. if !dms.Config().IsDomainInScope(req.Name) {
  339. return
  340. }
  341. dms.findNamesAndAddresses(req.Records[recidx].Data, req.Domain)
  342. }
  343. func (dms *DataManagerService) insertSPF(req *core.DNSRequest, recidx int) {
  344. if !dms.Config().IsDomainInScope(req.Name) {
  345. return
  346. }
  347. dms.findNamesAndAddresses(req.Records[recidx].Data, req.Domain)
  348. }
  349. func (dms *DataManagerService) findNamesAndAddresses(data, domain string) {
  350. ipre := regexp.MustCompile(utils.IPv4RE)
  351. for _, ip := range ipre.FindAllString(data, -1) {
  352. dms.Bus().Publish(core.NewAddrTopic, &core.AddrRequest{
  353. Address: ip,
  354. Domain: domain,
  355. Tag: core.DNS,
  356. Source: "Forward DNS",
  357. })
  358. }
  359. subre := utils.AnySubdomainRegex()
  360. for _, name := range subre.FindAllString(data, -1) {
  361. if !dms.Config().IsDomainInScope(name) {
  362. continue
  363. }
  364. domain := strings.ToLower(dms.Config().WhichDomain(name))
  365. if domain == "" {
  366. continue
  367. }
  368. dms.Bus().Publish(core.NewNameTopic, &core.DNSRequest{
  369. Name: name,
  370. Domain: domain,
  371. Tag: core.DNS,
  372. Source: "Forward DNS",
  373. })
  374. }
  375. }
  376. func (dms *DataManagerService) insertInfrastructure(addr string) {
  377. asn, cidr, desc, err := IPRequest(addr, dms.Bus())
  378. if err != nil {
  379. dms.Config().Log.Printf("%s: %v", dms.String(), err)
  380. return
  381. }
  382. for _, handler := range dms.Handlers {
  383. err := handler.Insert(&handlers.DataOptsParams{
  384. UUID: dms.Config().UUID.String(),
  385. Timestamp: time.Now().Format(time.RFC3339),
  386. Type: handlers.OptInfrastructure,
  387. Address: addr,
  388. ASN: asn,
  389. CIDR: cidr.String(),
  390. Description: desc,
  391. })
  392. if err != nil {
  393. dms.Config().Log.Printf("%s: %s failed to insert infrastructure data: %v", dms.String(), handler, err)
  394. }
  395. }
  396. }