/app/Console/Commands/CrawlData.php

https://gitlab.com/alanmtk/sube-backend · PHP · 175 lines · 136 code · 36 blank · 3 comment · 1 complexity · 84f76b45ed45f2e4a199a5cd04587965 MD5 · raw file

  1. <?php
  2. namespace App\Console\Commands;
  3. use Illuminate\Console\Command;
  4. use App\Models\City;
  5. use App\Models\Province;
  6. use App\Models\Type;
  7. use App\Models\Venue;
  8. use \Curl\Curl;
  9. class CrawlData extends Command
  10. {
  11. protected $signature = 'crawl:data';
  12. protected $description = 'Crawls data from sube.gob.ar';
  13. public function __construct()
  14. {
  15. parent::__construct();
  16. }
  17. public function handle()
  18. {
  19. $this->prepareDatabase();
  20. $this->crawlProvinces();
  21. $this->crawlCities();
  22. $this->crawlTypes();
  23. $this->crawlVenues();
  24. }
  25. public function prepareDatabase()
  26. {
  27. $this->info('Prepearing the database');
  28. $this->call('migrate:refresh');
  29. $this->info('');
  30. }
  31. public function crawlProvinces()
  32. {
  33. $this->info('Processing provinces');
  34. $client = new Curl();
  35. $client->setOpt(CURLOPT_SSL_VERIFYPEER, false);
  36. $client->setOpt(CURLOPT_RETURNTRANSFER, true);
  37. $client->setOpt(CURLOPT_HTTPHEADER, array('Content-Type: application/json', 'Content-Length: 0'));
  38. $client->post('https://www.sube.gob.ar/MapasSUBE.aspx/getProvincias');
  39. $bar = $this->output->createProgressBar(count($client->response->d));
  40. foreach ($client->response->d as $province) {
  41. $tmp = json_decode($province);
  42. Province::create([
  43. 'code' => $tmp->Codigo,
  44. 'name' => trim(ucwords(mb_strtolower($tmp->Descripcion)))
  45. ]);
  46. $bar->advance();
  47. }
  48. $bar->finish();
  49. $this->info('');
  50. $this->info('');
  51. }
  52. public function crawlCities()
  53. {
  54. $provinces = Province::all();
  55. foreach ($provinces as $province) {
  56. /* TODO: Use Curl object implementation */
  57. $data = '{"provinciaId": "' . $province->code . '"}';
  58. $ch = curl_init('https://www.sube.gob.ar/MapasSUBE.aspx/getCiudades');
  59. curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
  60. curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json'));
  61. curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
  62. curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
  63. curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  64. $result = curl_exec($ch);
  65. $result = json_decode($result, true);
  66. $this->info('Processing cities for: ' . $province->name);
  67. $bar = $this->output->createProgressBar(count($result['d']));
  68. foreach ($result['d'] as $city) {
  69. $tmp = json_decode($city);
  70. City::create([
  71. 'name' => ucwords(mb_strtolower($tmp->Descripcion)),
  72. 'province_code' => $province->code
  73. ]);
  74. $bar->advance();
  75. }
  76. $bar->finish();
  77. $this->info('');
  78. $this->info('');
  79. }
  80. }
  81. public function crawlTypes()
  82. {
  83. $this->info('Processing types (only unique types will be stored)');
  84. /* TODO: Use Curl object implementation */
  85. $data = '{"tipo": -1}';
  86. $ch = curl_init('https://www.sube.gob.ar/MapasSUBE.aspx/getMarkerObjects');
  87. curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
  88. curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json', 'Content-Length: ' . strlen($data)));
  89. curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
  90. curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
  91. curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  92. $result = curl_exec($ch);
  93. $result = json_decode($result, true);
  94. $bar = $this->output->createProgressBar(count($result['d']));
  95. foreach ($result['d'] as $centro) {
  96. $tmp = json_decode($centro);
  97. if (!Type::where('name', '=', strip_tags($tmp->Descripcion))->exists()) {
  98. Type::create([
  99. 'name' => strip_tags($tmp->Descripcion),
  100. 'icon' => $tmp->UrlIcon]);
  101. }
  102. $bar->advance();
  103. }
  104. $bar->finish();
  105. $this->info('');
  106. $this->info(Type::count() . ' types have been processed');
  107. $this->info('');
  108. }
  109. public function crawlVenues()
  110. {
  111. $this->info('Processing venues');
  112. /* TODO: Use Curl object implementation */
  113. $data = '{"tipo": -1}';
  114. $ch = curl_init('https://www.sube.gob.ar/MapasSUBE.aspx/getMarkerObjects');
  115. curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
  116. curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json', 'Content-Length: ' . strlen($data)));
  117. curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
  118. curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
  119. curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  120. $result = curl_exec($ch);
  121. $result = json_decode($result, true);
  122. $bar = $this->output->createProgressBar(count($result['d']));
  123. foreach ($result['d'] as $venue) {
  124. $tmp = json_decode($venue);
  125. preg_match_all("#<b.*?>([^<]+)</b>#", $tmp->Direccion, $data);
  126. Venue::create([
  127. 'address' => isset($data[1][0]) ? ucwords(mb_strtolower($data[1][0])) : "",
  128. 'schedules' => isset($data[1][1]) ? $data[1][1] : "",
  129. 'postal_code' => $tmp->CP,
  130. 'latitude' => $tmp->sLatitud,
  131. 'longitude' => $tmp->sLongitud,
  132. 'type_id' => Type::where("name", strip_tags($tmp->Descripcion))->pluck('id')->first(),
  133. 'city_id' => City::where("name", ucwords(mb_strtolower($tmp->Localidad)))->pluck('id')->first()
  134. ]);
  135. $bar->advance();
  136. }
  137. $bar->finish();
  138. }
  139. }