103 |
- |
1 |
<?php
|
|
|
2 |
|
|
|
3 |
/***************************************************************************
|
|
|
4 |
* Copyright (C) 2009-2011 by Geo Varghese(www.seopanel.in) *
|
|
|
5 |
* sendtogeo@gmail.com *
|
|
|
6 |
* *
|
|
|
7 |
* This program is free software; you can redistribute it and/or modify *
|
|
|
8 |
* it under the terms of the GNU General Public License as published by *
|
|
|
9 |
* the Free Software Foundation; either version 2 of the License, or *
|
|
|
10 |
* (at your option) any later version. *
|
|
|
11 |
* *
|
|
|
12 |
* This program is distributed in the hope that it will be useful, *
|
|
|
13 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
|
|
|
14 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
|
|
|
15 |
* GNU General Public License for more details. *
|
|
|
16 |
* *
|
|
|
17 |
* You should have received a copy of the GNU General Public License *
|
|
|
18 |
* along with this program; if not, write to the *
|
|
|
19 |
* Free Software Foundation, Inc., *
|
|
|
20 |
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
|
|
|
21 |
***************************************************************************/
|
|
|
22 |
|
|
|
23 |
# class defines all crawl log functions
|
|
|
24 |
class CrawlLogController extends Controller {
|
|
|
25 |
|
|
|
26 |
/**
|
|
|
27 |
* the name of the crawl logging database tabel
|
|
|
28 |
*/
|
|
|
29 |
var $tablName = "crawl_log";
|
|
|
30 |
|
|
|
31 |
/**
|
|
|
32 |
* function to create crawl logs for report generation
|
|
|
33 |
* @param $crawlInfo The array contains all detaisl of crawl
|
|
|
34 |
*/
|
|
|
35 |
function createCrawlLog($crawlInfo) {
|
|
|
36 |
$dateTime = date('Y-m-d H:i:s');
|
|
|
37 |
$sql = "INSERT INTO ". $this->tablName ."(".implode(",", array_keys($crawlInfo)).", crawl_time)";
|
|
|
38 |
$sql .= " values ('".implode("','", array_values($crawlInfo))."', '$dateTime')";
|
|
|
39 |
$this->db->query($sql);
|
|
|
40 |
$logId = $this->db->getMaxId($this->tablName);
|
|
|
41 |
return $logId;
|
|
|
42 |
}
|
|
|
43 |
|
|
|
44 |
/**
|
|
|
45 |
* function to update crawl log
|
|
|
46 |
* @param int $logId The id of the crawl log needs to be updated
|
|
|
47 |
* @param Array $crawlInfo The array contains crawl log details needs to be updated
|
|
|
48 |
*/
|
|
|
49 |
function updateCrawlLog($logId, $crawlInfo) {
|
|
|
50 |
|
|
|
51 |
// if log id is not empty
|
|
|
52 |
if (!empty($logId)) {
|
|
|
53 |
|
|
|
54 |
$sql = "update $this->tablName set ";
|
|
|
55 |
|
|
|
56 |
// for each through the log values
|
|
|
57 |
foreach($crawlInfo as $key => $value) {
|
|
|
58 |
$sql .= "$key='". addslashes($value) ."',";
|
|
|
59 |
}
|
|
|
60 |
|
|
|
61 |
$sql = rtrim($sql, ",");
|
|
|
62 |
$sql .= " where id=$logId";
|
|
|
63 |
$this->db->query($sql);
|
|
|
64 |
}
|
|
|
65 |
|
|
|
66 |
}
|
|
|
67 |
|
|
|
68 |
/**
|
|
|
69 |
* Function to display crawl log details
|
|
|
70 |
* @param Array $info Contains all search details
|
|
|
71 |
*/
|
|
|
72 |
function listCrawlLog($info = ''){
|
|
|
73 |
|
|
|
74 |
$userId = isLoggedIn();
|
|
|
75 |
$sql = "select t.*, k.name keyword from $this->tablName t left join keywords k on t.ref_id=k.id where 1=1";
|
|
|
76 |
$conditions = "";
|
|
|
77 |
|
|
|
78 |
if (isset($info['status'])) {
|
|
|
79 |
if (($info['status'] == 'success') || ($info['status'] == 'fail')) {
|
|
|
80 |
$statVal = ($info['status']=='success') ? 1 : 0;
|
|
|
81 |
$conditions .= " and crawl_status=$statVal";
|
|
|
82 |
$urlParams .= "&status=".$info['status'];
|
|
|
83 |
}
|
|
|
84 |
} else {
|
|
|
85 |
$info['status'] = '';
|
|
|
86 |
}
|
|
|
87 |
|
|
|
88 |
$this->set('statVal', $info['status']);
|
|
|
89 |
|
|
|
90 |
if (empty($info['keyword'])) {
|
|
|
91 |
$info['keyword'] = '';
|
|
|
92 |
} else {
|
|
|
93 |
$info['keyword'] = urldecode($info['keyword']);
|
|
|
94 |
$searchKeyword = addslashes($info['keyword']);
|
|
|
95 |
$conditions .= " and (ref_id like '%$searchKeyword%' or subject like '%$searchKeyword%' or crawl_referer like '%$searchKeyword%'
|
|
|
96 |
or log_message like '%$searchKeyword%' or k.name like '%$searchKeyword%' or crawl_link like '%$searchKeyword%'
|
|
|
97 |
or crawl_cookie like '%$searchKeyword%' or crawl_post_fields like '%$searchKeyword%' or crawl_useragent like '%$searchKeyword%')";
|
|
|
98 |
$urlParams .= "&keyword=".urlencode($info['keyword']);
|
|
|
99 |
}
|
|
|
100 |
|
|
|
101 |
$this->set('keyword', $info['keyword']);
|
|
|
102 |
|
|
|
103 |
$crawlType = "";
|
|
|
104 |
if (!empty($info['crawl_type'])) {
|
|
|
105 |
$crawlType = $info['crawl_type'];
|
|
|
106 |
$conditions .= " and crawl_type='".addslashes($crawlType)."'";
|
|
|
107 |
$urlParams .= "&crawl_type=".$crawlType;
|
|
|
108 |
}
|
|
|
109 |
|
|
|
110 |
// find different crawl types
|
|
|
111 |
$crawlTypeSql = "select distinct crawl_type from $this->tablName";
|
|
|
112 |
$crawlTypeList = $this->db->select($crawlTypeSql);
|
|
|
113 |
$this->set('crawlTypeList', $crawlTypeList);
|
|
|
114 |
$this->set('crawlType', $crawlType);
|
|
|
115 |
|
|
|
116 |
$proxyId = "";
|
|
|
117 |
if (!empty($info['proxy_id'])) {
|
|
|
118 |
$proxyId = $info['proxy_id'];
|
|
|
119 |
$conditions .= " and proxy_id='".intval($proxyId)."'";
|
|
|
120 |
$urlParams .= "&proxy_id=".$proxyId;
|
|
|
121 |
}
|
|
|
122 |
|
|
|
123 |
// find different proxy used
|
|
|
124 |
$proxySql = "select distinct proxy_id, proxy, port from $this->tablName t, proxylist pl
|
|
|
125 |
where pl.id=t.proxy_id and t.proxy_id!=0";
|
|
|
126 |
$proxyList = $this->db->select($proxySql);
|
|
|
127 |
$this->set('proxyList', $proxyList);
|
|
|
128 |
$this->set('proxyId', $proxyId);
|
|
|
129 |
|
|
|
130 |
$seId = "";
|
|
|
131 |
$seController = New SearchEngineController();
|
|
|
132 |
$seList = $seController->__getAllSearchEngines();
|
|
|
133 |
$seNameList = array();
|
|
|
134 |
foreach ($seList as $seInfo) {
|
|
|
135 |
$seNameList[] = $seInfo['domain'];
|
|
|
136 |
}
|
|
|
137 |
|
|
|
138 |
if (!empty($info['se_id'])) {
|
|
|
139 |
$seId = intval($info['se_id']);
|
|
|
140 |
$conditions .= " and (subject='$seId' or subject in ('".implode(",", $seNameList)."'))";
|
|
|
141 |
$urlParams .= "&se_id=".$seId;
|
|
|
142 |
}
|
|
|
143 |
|
|
|
144 |
$this->set('seList', $seList);
|
|
|
145 |
$this->set('seId', $seId);
|
|
|
146 |
|
|
|
147 |
if (!empty ($info['from_time'])) {
|
|
|
148 |
$fromTime = strtotime($info['from_time'] . ' 00:00:00');
|
|
|
149 |
} else {
|
|
|
150 |
$fromTime = mktime(0, 0, 0, date('m'), date('d') - 30, date('Y'));
|
|
|
151 |
}
|
|
|
152 |
|
|
|
153 |
if (!empty ($info['to_time'])) {
|
|
|
154 |
$toTime = strtotime($info['to_time'] . ' 00:00:00');
|
|
|
155 |
} else {
|
|
|
156 |
$toTime = mktime(0, 0, 0, date('m'), date('d'), date('Y'));
|
|
|
157 |
}
|
|
|
158 |
|
|
|
159 |
$fromTimeLabel = date('Y-m-d', $fromTime);
|
|
|
160 |
$toTimeLabel = date('Y-m-d', $toTime);
|
|
|
161 |
$this->set('fromTime', $fromTimeLabel);
|
|
|
162 |
$this->set('toTime', $toTimeLabel);
|
|
|
163 |
$urlParams .= "&from_time=$fromTimeLabel&to_time=$toTimeLabel";
|
|
|
164 |
|
|
|
165 |
// sql created using param
|
|
|
166 |
$sql .= " $conditions and crawl_time >='$fromTimeLabel 00:00:00' and crawl_time<='$toTimeLabel 23:59:59' order by id DESC";
|
|
|
167 |
|
|
|
168 |
// pagination setup
|
|
|
169 |
$this->db->query($sql, true);
|
|
|
170 |
$this->paging->setDivClass('pagingdiv');
|
|
|
171 |
$this->paging->loadPaging($this->db->noRows, SP_PAGINGNO);
|
|
|
172 |
$pagingDiv = $this->paging->printPages('log.php', '', 'scriptDoLoad', 'content', $urlParams);
|
|
|
173 |
$this->set('pagingDiv', $pagingDiv);
|
|
|
174 |
$sql .= " limit ".$this->paging->start .",". $this->paging->per_page;
|
|
|
175 |
|
|
|
176 |
$logList = $this->db->select($sql);
|
|
|
177 |
$this->set('pageNo', $info['pageno']);
|
|
|
178 |
$this->set('list', $logList);
|
|
|
179 |
$this->set('urlParams', $urlParams);
|
|
|
180 |
$this->set('fromPopUp', $info['fromPopUp']);
|
|
|
181 |
$this->render('log/crawlloglist');
|
|
|
182 |
}
|
|
|
183 |
|
|
|
184 |
/**
|
|
|
185 |
* fucntion to delete log id
|
|
|
186 |
* @param int $logId The id of the log needs to be deleted
|
|
|
187 |
*/
|
|
|
188 |
function deleteCrawlLog($logId) {
|
|
|
189 |
$sql = "delete from crawl_log where id=".intval($logId);
|
|
|
190 |
$this->db->query($sql);
|
|
|
191 |
}
|
|
|
192 |
|
|
|
193 |
/**
|
|
|
194 |
* function to clear all logs saved in teh system
|
|
|
195 |
*/
|
|
|
196 |
function clearAllLog() {
|
|
|
197 |
$sql = "truncate crawl_log";
|
|
|
198 |
$this->db->query($sql);
|
|
|
199 |
}
|
|
|
200 |
|
|
|
201 |
/**
|
|
|
202 |
* function to get crawl log information
|
|
|
203 |
* @param int $logId The id of the log needs to be dispalyed
|
|
|
204 |
*/
|
|
|
205 |
function getCrawlLogInfo($logId) {
|
|
|
206 |
$sql = "select * from crawl_log where id=".intval($logId);
|
|
|
207 |
$logInfo = $this->db->select($sql, true);
|
|
|
208 |
return $logInfo;
|
|
|
209 |
}
|
|
|
210 |
|
|
|
211 |
/**
|
|
|
212 |
* fucntion to show crawl log details
|
|
|
213 |
* @param int $logId The id of the log needs to be dispalyed
|
|
|
214 |
*/
|
|
|
215 |
function showCrawlLogDetails($logId) {
|
|
|
216 |
$logInfo = $this->getCrawlLogInfo($logId);
|
|
|
217 |
$this->set('logInfo', $logInfo);
|
|
|
218 |
$this->render('log/showcrawllog');
|
|
|
219 |
}
|
|
|
220 |
|
|
|
221 |
/**
|
|
|
222 |
* function to clear logs before an interval
|
|
|
223 |
* @param int $daysBefore The days before the crawl logs to be deleted
|
|
|
224 |
*/
|
|
|
225 |
function clearCrawlLog($daysBefore) {
|
|
|
226 |
|
|
|
227 |
$dateBefore = mktime(0, 0, 0, date('m'), date('d') - $daysBefore, date('y'));
|
|
|
228 |
$dateBefore = date('Y-m-d H:i:s', $dateBefore);
|
|
|
229 |
$sql = "delete from crawl_log where crawl_time < '$dateBefore'";
|
|
|
230 |
$this->db->query($sql);
|
|
|
231 |
|
|
|
232 |
}
|
|
|
233 |
|
|
|
234 |
}
|
|
|
235 |
?>
|