From 4f751868837a0f78423c927c6cd4aeb24bf37c00 Mon Sep 17 00:00:00 2001
From: Marvin Borner
Date: Wed, 19 Sep 2018 17:43:33 +0200
Subject: Finally fixed almost all bugs :zap: :fire:

---
 crawler/CrawlController.php | 57 ++++++++++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 27 deletions(-)

(limited to 'crawler/CrawlController.php')

diff --git a/crawler/CrawlController.php b/crawler/CrawlController.php
index 97edf25..53d5aac 100644
--- a/crawler/CrawlController.php
+++ b/crawler/CrawlController.php
@@ -1,4 +1,5 @@
 <?php
+header('Content-type: text/plain; charset=utf-8');
 /**
  * User: Marvin Borner
  * Date: 14/09/2018
@@ -12,48 +13,50 @@ require_once 'Algorithms.php';
 
 class CrawlController
 {
-    private static $currentlyCrawled;
+    public static $currentlyCrawled;
 
     public static function start($url = '')
     {
         set_time_limit(3600000);
-        error_reporting(E_ERROR | E_PARSE);
+
+        self::$currentlyCrawled = $url;
 
         while (true) {
-            self::$currentlyCrawled = $url;
-            self::crawl(self::$currentlyCrawled);
+            self::crawl(Algorithms::cleanUrl(self::$currentlyCrawled));
         }
     }
 
     private static function crawl($url)
     {
-        if (Database::alreadyCrawled(Algorithms::cleanUrl($url))) {
+        if ($url !== '' && Database::alreadyCrawled($url)) {
             Database::removeFromQueue(self::$currentlyCrawled);
             self::$currentlyCrawled = Database::getFromQueue('DESC');
         } else {
             $requestResponse = WebRequest::getContent($url);
-            self::$currentlyCrawled = $requestResponse[3];
-            if (preg_match('/2\d\d/', $requestResponse[1])) { // success
-                print 'Download Size: ' . $requestResponse[2];
-
-                $htmlPath = Algorithms::createPathFromHtml($requestResponse[0]);
-
-                $urlInfo = Algorithms::getUrlInfo($htmlPath);
-                Database::saveUrlData(self::$currentlyCrawled, $urlInfo);
-
-                $allLinks = Algorithms::getLinks($htmlPath);
-                Database::insertIntoQueue($allLinks);
-
-                Database::removeFromQueue(self::$currentlyCrawled);
-                self::$currentlyCrawled = Database::getFromQueue('DESC'); // set new from start
-                print "\e[96mFinished previous url - crawling: " . self::$currentlyCrawled . "\n";
-            } else {
-                print "\t\e[91mError " . $requestResponse[1] . ' ' . self::$currentlyCrawled . "\n";
-
-                Database::urlHasError(self::$currentlyCrawled); // prevents re-crawling of error url
-                Database::removeFromQueue(self::$currentlyCrawled);
-                self::$currentlyCrawled = Database::getFromQueue('ASC'); // set new from end
-                print "\e[91mFinished previous url with error - crawling: " . self::$currentlyCrawled . "\n";
+            if ($requestResponse) {
+                self::$currentlyCrawled = $requestResponse[3];
+                if (preg_match('/2\d\d/', $requestResponse[1])) { // success
+                    print 'Download Size: ' . $requestResponse[2];
+
+                    $htmlPath = Algorithms::createPathFromHtml($requestResponse[0]);
+
+                    $urlInfo = Algorithms::getUrlInfo($htmlPath);
+                    Database::saveUrlData(self::$currentlyCrawled, $urlInfo);
+
+                    $allLinks = Algorithms::getLinks($htmlPath);
+                    Database::insertIntoQueue($allLinks);
+
+                    Database::removeFromQueue(self::$currentlyCrawled);
+                    self::$currentlyCrawled = Database::getFromQueue('DESC'); // set new from start
+                    print "\e[96mFinished previous url - crawling: " . self::$currentlyCrawled . "\n";
+                } else {
+                    print "\t\e[91mError " . $requestResponse[1] . ' ' . self::$currentlyCrawled . "\n";
+
+                    Database::urlHasError(self::$currentlyCrawled); // prevents re-crawling of error url
+                    Database::removeFromQueue(self::$currentlyCrawled);
+                    self::$currentlyCrawled = Database::getFromQueue('ASC'); // set new from end
+                    print "\e[91mFinished previous url with error - crawling: " . self::$currentlyCrawled . "\n";
+                }
             }
         }
     }
-- 
cgit v1.2.3