/app/Console/Commands/CrawlData.php
https://gitlab.com/alanmtk/sube-backend · PHP · 175 lines · 136 code · 36 blank · 3 comment · 1 complexity · 84f76b45ed45f2e4a199a5cd04587965 MD5 · raw file
- <?php
- namespace App\Console\Commands;
- use Illuminate\Console\Command;
- use App\Models\City;
- use App\Models\Province;
- use App\Models\Type;
- use App\Models\Venue;
- use \Curl\Curl;
- class CrawlData extends Command
- {
- protected $signature = 'crawl:data';
- protected $description = 'Crawls data from sube.gob.ar';
- public function __construct()
- {
- parent::__construct();
- }
- public function handle()
- {
- $this->prepareDatabase();
- $this->crawlProvinces();
- $this->crawlCities();
- $this->crawlTypes();
- $this->crawlVenues();
- }
- public function prepareDatabase()
- {
- $this->info('Prepearing the database');
- $this->call('migrate:refresh');
- $this->info('');
- }
- public function crawlProvinces()
- {
- $this->info('Processing provinces');
- $client = new Curl();
- $client->setOpt(CURLOPT_SSL_VERIFYPEER, false);
- $client->setOpt(CURLOPT_RETURNTRANSFER, true);
- $client->setOpt(CURLOPT_HTTPHEADER, array('Content-Type: application/json', 'Content-Length: 0'));
- $client->post('https://www.sube.gob.ar/MapasSUBE.aspx/getProvincias');
- $bar = $this->output->createProgressBar(count($client->response->d));
- foreach ($client->response->d as $province) {
- $tmp = json_decode($province);
- Province::create([
- 'code' => $tmp->Codigo,
- 'name' => trim(ucwords(mb_strtolower($tmp->Descripcion)))
- ]);
- $bar->advance();
- }
- $bar->finish();
- $this->info('');
- $this->info('');
- }
- public function crawlCities()
- {
- $provinces = Province::all();
- foreach ($provinces as $province) {
- /* TODO: Use Curl object implementation */
- $data = '{"provinciaId": "' . $province->code . '"}';
- $ch = curl_init('https://www.sube.gob.ar/MapasSUBE.aspx/getCiudades');
- curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
- curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json'));
- curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
- curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- $result = curl_exec($ch);
- $result = json_decode($result, true);
- $this->info('Processing cities for: ' . $province->name);
- $bar = $this->output->createProgressBar(count($result['d']));
- foreach ($result['d'] as $city) {
- $tmp = json_decode($city);
- City::create([
- 'name' => ucwords(mb_strtolower($tmp->Descripcion)),
- 'province_code' => $province->code
- ]);
- $bar->advance();
- }
- $bar->finish();
- $this->info('');
- $this->info('');
- }
- }
- public function crawlTypes()
- {
- $this->info('Processing types (only unique types will be stored)');
- /* TODO: Use Curl object implementation */
- $data = '{"tipo": -1}';
- $ch = curl_init('https://www.sube.gob.ar/MapasSUBE.aspx/getMarkerObjects');
- curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
- curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json', 'Content-Length: ' . strlen($data)));
- curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
- curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- $result = curl_exec($ch);
- $result = json_decode($result, true);
- $bar = $this->output->createProgressBar(count($result['d']));
- foreach ($result['d'] as $centro) {
- $tmp = json_decode($centro);
- if (!Type::where('name', '=', strip_tags($tmp->Descripcion))->exists()) {
- Type::create([
- 'name' => strip_tags($tmp->Descripcion),
- 'icon' => $tmp->UrlIcon]);
- }
- $bar->advance();
- }
- $bar->finish();
- $this->info('');
- $this->info(Type::count() . ' types have been processed');
- $this->info('');
- }
- public function crawlVenues()
- {
- $this->info('Processing venues');
- /* TODO: Use Curl object implementation */
- $data = '{"tipo": -1}';
- $ch = curl_init('https://www.sube.gob.ar/MapasSUBE.aspx/getMarkerObjects');
- curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
- curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json', 'Content-Length: ' . strlen($data)));
- curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
- curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- $result = curl_exec($ch);
- $result = json_decode($result, true);
- $bar = $this->output->createProgressBar(count($result['d']));
- foreach ($result['d'] as $venue) {
- $tmp = json_decode($venue);
- preg_match_all("#<b.*?>([^<]+)</b>#", $tmp->Direccion, $data);
- Venue::create([
- 'address' => isset($data[1][0]) ? ucwords(mb_strtolower($data[1][0])) : "",
- 'schedules' => isset($data[1][1]) ? $data[1][1] : "",
- 'postal_code' => $tmp->CP,
- 'latitude' => $tmp->sLatitud,
- 'longitude' => $tmp->sLongitud,
- 'type_id' => Type::where("name", strip_tags($tmp->Descripcion))->pluck('id')->first(),
- 'city_id' => City::where("name", ucwords(mb_strtolower($tmp->Localidad)))->pluck('id')->first()
- ]);
- $bar->advance();
- }
- $bar->finish();
- }
- }