| 91 |
- |
1 |
<?php
|
|
|
2 |
error_reporting(E_ALL);
|
|
|
3 |
|
|
|
4 |
// Get itunes listings
|
|
|
5 |
function get_amazon_scrape($query, $searchCondition) {
|
| 95 |
- |
6 |
$vendors = Vendors::getInstance();
|
|
|
7 |
$config = $vendors->getVendor(Vendors::AMAZON);
|
|
|
8 |
$numListings = $config['numListings'];
|
|
|
9 |
|
|
|
10 |
$needMatches = empty($_SESSION["discogs"]);
|
|
|
11 |
if ($needMatches) {
|
|
|
12 |
$_SESSION["discogs"] = startMatches();
|
|
|
13 |
}
|
|
|
14 |
|
| 91 |
- |
15 |
$arr = [];
|
|
|
16 |
$products = [];
|
| 95 |
- |
17 |
$cnt = 0;
|
| 91 |
- |
18 |
|
|
|
19 |
libxml_use_internal_errors(true);
|
| 99 |
- |
20 |
$html = getSearchCache("amazon_scrape", $query, $searchCondition);
|
|
|
21 |
if ($html === false) {
|
|
|
22 |
$html = getUrl("https://www.amazon.com/s?k=" . rawurlencode($query) . "&sf=qz&unfiltered=1&ref=nb_sb_noss");
|
|
|
23 |
|
|
|
24 |
saveSearchCache("amazon_scrape", $query, $searchCondition, $html);
|
|
|
25 |
}
|
|
|
26 |
|
| 91 |
- |
27 |
$dom = new DOMDocument;
|
|
|
28 |
$dom->loadHTML($html);
|
|
|
29 |
$xpath = new DOMXPath($dom);
|
|
|
30 |
$nodes = $xpath->query('//a/@href');
|
|
|
31 |
foreach($nodes as $href) {
|
|
|
32 |
if (strpos($href->nodeValue, "/gp/offer-listing/B") === 0) {
|
|
|
33 |
$products[] = $href->nodeValue;
|
|
|
34 |
}
|
|
|
35 |
}
|
|
|
36 |
|
|
|
37 |
foreach($products as $product) {
|
|
|
38 |
$url = "https://www.amazon.com" . $product . "&tag=uj024-20&language=en_US";
|
|
|
39 |
$asin = explode('/', $product)[3];
|
|
|
40 |
|
| 99 |
- |
41 |
$html = getSearchCache("amazon_scrape", $asin, "");
|
|
|
42 |
if ($html === false) {
|
|
|
43 |
$html = getUrl("https://www.amazon.com/dp/" . $asin, "Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/70.0");
|
|
|
44 |
saveSearchCache("amazon_scrape", $asin, "", $html);
|
|
|
45 |
}
|
| 91 |
- |
46 |
|
|
|
47 |
$dom = new DOMDocument;
|
|
|
48 |
$dom->loadHTML($html);
|
| 95 |
- |
49 |
$xpathPrd = new DOMXPath($dom);
|
| 91 |
- |
50 |
|
| 95 |
- |
51 |
$nodes = $xpathPrd->query('//table[@id="productDetailsTable"]//ul/li');
|
|
|
52 |
if ($nodes->length < 1) {
|
| 91 |
- |
53 |
continue;
|
|
|
54 |
}
|
| 95 |
- |
55 |
|
|
|
56 |
$format = "";
|
| 91 |
- |
57 |
foreach($nodes as $node) {
|
|
|
58 |
$str = trim($node->nodeValue);
|
| 95 |
- |
59 |
|
|
|
60 |
if (strpos($str, "Audio CD") === 0 ||
|
|
|
61 |
strpos($str, "Vinyl") === 0 ||
|
|
|
62 |
strpos($str, "Sheet") === 0 ||
|
|
|
63 |
strpos($str, "Hardcover") === 0 ||
|
|
|
64 |
strpos($str, "Paperback") === 0) {
|
|
|
65 |
$p = strpos($str, " (");
|
|
|
66 |
$format = ($p > 0 ? substr($str, 0, $p) : $str);
|
|
|
67 |
$releaseDate = ($p > 0 ? substr($str, $p+2, strlen($str) - $p - 3) : "");
|
|
|
68 |
}
|
|
|
69 |
|
| 91 |
- |
70 |
if (strpos($str, "Amazon Best Sellers Rank:") === 0) {
|
|
|
71 |
$p = strpos($str, " (");
|
|
|
72 |
$rank = substr($str, 11, $p-11);
|
|
|
73 |
}
|
|
|
74 |
}
|
|
|
75 |
|
|
|
76 |
if (strpos($format, "Audio CD") === 0 ||
|
|
|
77 |
strpos($format, "Vinyl") === 0 ||
|
| 95 |
- |
78 |
strpos($str, "Sheet") === 0 ||
|
| 91 |
- |
79 |
strpos($format, "Hardcover") === 0 ||
|
|
|
80 |
strpos($format, "Paperback") === 0) {
|
|
|
81 |
if (strpos($format, "Audio CD") !== false) {
|
|
|
82 |
$mediaType = "CD";
|
|
|
83 |
} else if (strpos($format, "Vinyl") !== false) {
|
|
|
84 |
$mediaType = "Record";
|
|
|
85 |
} else if (strpos($format, "Paperback") !== false ||
|
|
|
86 |
strpos($format, "Sheet") !== false ||
|
|
|
87 |
strpos($format, "Hardcover") !== false) {
|
|
|
88 |
$mediaType = "Book";
|
|
|
89 |
}
|
|
|
90 |
|
| 99 |
- |
91 |
$p = substr($product, 0, strpos($product, "/ref="));
|
|
|
92 |
$html = getSearchCache("amazon_scrape", $p, "");
|
|
|
93 |
if ($html === false) {
|
|
|
94 |
$str = "https://www.amazon.com" . $product;
|
|
|
95 |
$html = getUrl($str, "Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/70.0");
|
|
|
96 |
saveSearchCache("amazon_scrape", $p, "", $html);
|
|
|
97 |
}
|
| 91 |
- |
98 |
|
|
|
99 |
$dom = new DOMDocument;
|
|
|
100 |
$dom->loadHTML($html);
|
|
|
101 |
$xpath = new DOMXPath($dom);
|
|
|
102 |
|
|
|
103 |
$nodes = $xpath->query('//div[@id="olpProductImage"]//img');
|
| 97 |
- |
104 |
$pic = "";
|
|
|
105 |
if ($nodes->length > 0) {
|
|
|
106 |
$pic = $nodes->item(0)->getAttribute("src");
|
|
|
107 |
}
|
|
|
108 |
|
| 91 |
- |
109 |
$nodes = $xpath->query('//div[@id="olpProductDetails"]/h1');
|
|
|
110 |
$title = trim($nodes->item(0)->nodeValue);
|
| 95 |
- |
111 |
$fullTitle = $title;
|
| 91 |
- |
112 |
|
|
|
113 |
$nodes = $xpath->query('//div[@id="olpProductByline"]');
|
|
|
114 |
if ($nodes->length > 0) {
|
|
|
115 |
$artists = trim($nodes->item(0)->nodeValue);
|
| 95 |
- |
116 |
$artists = str_replace(" (Artist)", "", $artists);
|
|
|
117 |
if (strpos($artists, "~ ") === 0) {
|
|
|
118 |
$artists = substr($artists, 2);
|
|
|
119 |
}
|
|
|
120 |
$fullTitle = $title . " by " . $artists;
|
| 91 |
- |
121 |
}
|
|
|
122 |
|
| 95 |
- |
123 |
if (strpos($format, "Audio CD") === 0 ||
|
|
|
124 |
strpos($format, "Vinyl") === 0) {
|
|
|
125 |
if ($needMatches) {
|
|
|
126 |
addMatch_scrape($xpathPrd, ++$cnt, $title, $artists, $format, $releaseDate, $asin, $url, $pic);
|
|
|
127 |
}
|
|
|
128 |
}
|
|
|
129 |
|
| 91 |
- |
130 |
$listings = $xpath->query('//div[contains(concat(" ", normalize-space(@class), " "), " olpOffer ")]');
|
|
|
131 |
|
| 95 |
- |
132 |
$listingCnt = 0;
|
| 91 |
- |
133 |
foreach($listings as $listing) {
|
|
|
134 |
$nodes = $xpath->query('.//h3[contains(concat(" ", normalize-space(@class), " "), " olpSellerName ")]', $listing);
|
|
|
135 |
$str = trim($nodes->item(0)->nodeValue);
|
|
|
136 |
$sellerName = (empty($str) ? "Amazon" : $str);
|
|
|
137 |
$merchantName = "Amazon";
|
|
|
138 |
$feedbackPercent = -1;
|
|
|
139 |
$feedbackScore = -1;
|
|
|
140 |
|
|
|
141 |
if ($sellerName != "Amazon") {
|
|
|
142 |
$merchantName .= " Marketplace";
|
|
|
143 |
$nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpSellerColumn ")]//p', $listing);
|
|
|
144 |
if ($nodes->length > 0) {
|
|
|
145 |
$str = trim($nodes->item(0)->nodeValue);
|
|
|
146 |
$sellerrating = substr($str, 17);
|
|
|
147 |
$num = preg_match_all('/((?:[0-9]+,)*[0-9]+(?:\.[0-9]+)?)/', $sellerrating, $matches);
|
|
|
148 |
if ($num == 3) {
|
|
|
149 |
$feedbackPercent = (int)$matches[0][0];
|
|
|
150 |
$feedbackScore = (int)str_replace( ',', '', $matches[0][2]);
|
|
|
151 |
}
|
|
|
152 |
}
|
|
|
153 |
}
|
|
|
154 |
|
|
|
155 |
$nodes = $xpath->query('.//span[contains(concat(" ", normalize-space(@class), " "), " olpCondition ")]', $listing);
|
|
|
156 |
$str = trim($nodes->item(0)->nodeValue);
|
|
|
157 |
$pos = strpos($str, " - ");
|
|
|
158 |
if ($pos !== false) {
|
|
|
159 |
$condition = trim(substr($str, 0, $pos));
|
|
|
160 |
$detailCondition = trim(substr($str, $pos+3));
|
|
|
161 |
} else {
|
|
|
162 |
$condition = $str;
|
|
|
163 |
$detailCondition = $str;
|
|
|
164 |
}
|
|
|
165 |
if ($condition == "Collectible" || $condition == "Refurbished") {
|
|
|
166 |
$condition = 'Used';
|
|
|
167 |
}
|
|
|
168 |
|
|
|
169 |
$nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpConditionColumn ")]//div[contains(concat(" ", normalize-space(@class), " "), " comments ")]', $listing);
|
|
|
170 |
if ($nodes->length > 0) {
|
|
|
171 |
$conditionComment = trim($nodes->item(0)->nodeValue);
|
|
|
172 |
}
|
|
|
173 |
|
|
|
174 |
$nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpOfferPrice ")]', $listing);
|
|
|
175 |
$price = substr(trim($nodes->item(0)->nodeValue), 1);
|
|
|
176 |
$currency = 'USD';
|
|
|
177 |
|
|
|
178 |
$nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpShippingPrice ")]', $listing);
|
|
|
179 |
if ($nodes->length > 0) {
|
|
|
180 |
$shippingCost = substr(trim($nodes->item(0)->nodeValue), 1);
|
|
|
181 |
$shippingCurrency = 'USD';
|
|
|
182 |
$freeShippingCap = 0;
|
|
|
183 |
} else {
|
|
|
184 |
$shippingCost = 0.00;
|
|
|
185 |
$shippingCurrency = 'USD';
|
|
|
186 |
$nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//p[contains(concat(" ", normalize-space(@class), " "), " olpShippingInfo ")]', $listing);
|
|
|
187 |
$str= trim($nodes->item(0)->nodeValue);
|
|
|
188 |
if (strpos($str, "FREE Shipping") !== false) {
|
|
|
189 |
$freeShippingCap = 0.00;
|
|
|
190 |
}
|
|
|
191 |
if (strpos($str, "on orders over") !== false) {
|
|
|
192 |
$freeShippingCap = 25.00;
|
|
|
193 |
}
|
|
|
194 |
}
|
|
|
195 |
|
|
|
196 |
$country = 'US';
|
|
|
197 |
$nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpDeliveryColumn ")]//ul/li', $listing);
|
|
|
198 |
foreach($nodes as $node) {
|
|
|
199 |
$str = trim($node->nodeValue);
|
|
|
200 |
if (strpos($str, "Ships from") === 0) {
|
|
|
201 |
$p = strpos($str, ".");
|
|
|
202 |
$country = getCountryCode(substr($str, 11, $p-11));
|
|
|
203 |
}
|
|
|
204 |
}
|
|
|
205 |
|
|
|
206 |
$nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//i[contains(concat(" ", normalize-space(@class), " "), " a-icon-prime ")]', $listing);
|
|
|
207 |
if ($nodes->length > 0) {
|
|
|
208 |
$sellerName .= " Prime";
|
|
|
209 |
}
|
|
|
210 |
|
| 95 |
- |
211 |
if (++$listingCnt > $numListings) {
|
|
|
212 |
continue;
|
|
|
213 |
}
|
|
|
214 |
|
| 91 |
- |
215 |
$arr[] = array(
|
|
|
216 |
"Merchant" => $merchantName,
|
|
|
217 |
"Condition" => $condition,
|
| 95 |
- |
218 |
"Title" => $fullTitle,
|
| 91 |
- |
219 |
"Barcode" => "",
|
|
|
220 |
"BarcodeType" => "",
|
|
|
221 |
"Image" => $pic,
|
|
|
222 |
"URL" => $url,
|
|
|
223 |
"MediaType" => $mediaType,
|
|
|
224 |
"DetailCondition" => $detailCondition,
|
|
|
225 |
"Country" => $country,
|
|
|
226 |
"BestOffer" => false,
|
|
|
227 |
"TimeLeft" => 0,
|
|
|
228 |
"Price" => $price,
|
|
|
229 |
"Currency" => $currency,
|
|
|
230 |
"ListingType" => "Fixed",
|
|
|
231 |
"Location" => "US",
|
|
|
232 |
"Zip" => "",
|
|
|
233 |
"FeedbackScore" => $feedbackScore,
|
|
|
234 |
"FeedbackPercent" => $feedbackPercent,
|
|
|
235 |
"SellerName" => $sellerName,
|
|
|
236 |
"HandlingTime" => 1,
|
|
|
237 |
"ShippingCost" => $shippingCost,
|
|
|
238 |
"ShippingEstimated" => false,
|
|
|
239 |
"ShippingCurrency" => $shippingCurrency,
|
|
|
240 |
"FreeShippingCap" => $freeShippingCap,
|
|
|
241 |
"Show" => true
|
|
|
242 |
);
|
|
|
243 |
|
|
|
244 |
}
|
|
|
245 |
}
|
|
|
246 |
}
|
|
|
247 |
|
| 95 |
- |
248 |
if ($needMatches) {
|
|
|
249 |
if ($cnt = 0) {
|
|
|
250 |
$_SESSION["discogs"] = "";
|
|
|
251 |
} else {
|
|
|
252 |
$_SESSION["discogs"] .= endMatches();
|
|
|
253 |
}
|
|
|
254 |
}
|
|
|
255 |
|
| 91 |
- |
256 |
return ($arr);
|
| 93 |
- |
257 |
}
|
|
|
258 |
|
| 95 |
- |
259 |
function addMatch_scrape($xpath, $cnt, $title, $artists, $mediaType, $releaseDate, $asin, $url, $pic) {
|
|
|
260 |
$nodes = $xpath->query('//table[@id="productDetailsTable"]//ul/li');
|
|
|
261 |
if ($nodes->length < 1) {
|
|
|
262 |
return;
|
|
|
263 |
}
|
| 93 |
- |
264 |
|
| 95 |
- |
265 |
$runTime = "";
|
|
|
266 |
$noDiscs = "";
|
|
|
267 |
$label = "";
|
|
|
268 |
$edition = "";
|
|
|
269 |
$genre = "";
|
|
|
270 |
|
|
|
271 |
foreach($nodes as $node) {
|
|
|
272 |
$str = trim($node->nodeValue);
|
|
|
273 |
|
|
|
274 |
$p = strpos($str, "Run Time:");
|
|
|
275 |
if ($p === 0) {
|
|
|
276 |
$runTime = substr($str, 10);
|
| 93 |
- |
277 |
}
|
| 95 |
- |
278 |
|
|
|
279 |
$p = strpos($str, "Number of Discs:");
|
|
|
280 |
if ($p === 0) {
|
|
|
281 |
$noDiscs = substr($str, 17);
|
|
|
282 |
}
|
|
|
283 |
|
|
|
284 |
$p = strpos($str, "Label:");
|
|
|
285 |
if ($p === 0) {
|
|
|
286 |
$label = substr($str, 7);
|
|
|
287 |
}
|
|
|
288 |
|
|
|
289 |
$p = strpos($str, "Edition:");
|
|
|
290 |
if ($p === 0) {
|
|
|
291 |
$edition = substr($str, 9);
|
|
|
292 |
}
|
|
|
293 |
|
|
|
294 |
$p = strpos($str, "SPARS Code:");
|
|
|
295 |
if ($p === 0) {
|
|
|
296 |
$edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 12);
|
|
|
297 |
}
|
|
|
298 |
|
|
|
299 |
$p = strpos($str, "Format:");
|
|
|
300 |
if ($p === 0) {
|
|
|
301 |
$edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 8);
|
|
|
302 |
}
|
|
|
303 |
|
|
|
304 |
$p = strpos($str, "Performer:");
|
|
|
305 |
if ($p === 0) {
|
|
|
306 |
$artists = substr($str, 11);
|
|
|
307 |
}
|
|
|
308 |
|
|
|
309 |
$p = strpos($str, "Original Release Date:");
|
|
|
310 |
if ($p === 0) {
|
|
|
311 |
$releaseDate = substr($str, 23);
|
|
|
312 |
}
|
|
|
313 |
|
|
|
314 |
if (strpos($str, "Amazon Best Sellers Rank:") === 0) {
|
|
|
315 |
$pieces = explode("\n", $str);
|
|
|
316 |
$genres = [];
|
|
|
317 |
foreach($pieces as $piece) {
|
|
|
318 |
$piece = trim($piece);
|
|
|
319 |
$p1 = strpos($piece, "in ");
|
|
|
320 |
$p2 = strpos($piece, " (CDs & Vinyl)") ;
|
|
|
321 |
if ($p1 === 0 && $p2 > 0) {
|
|
|
322 |
$genres[] = substr($piece, 4, $p2 - 4);
|
|
|
323 |
}
|
|
|
324 |
}
|
|
|
325 |
$genre = join(", ", $genres);
|
|
|
326 |
}
|
| 93 |
- |
327 |
}
|
| 95 |
- |
328 |
|
|
|
329 |
$item = new SimpleXMLElement("<item></item>");
|
|
|
330 |
$item->addChild('ASIN', $asin);
|
|
|
331 |
$item->addChild('DetailPageURL', $url);
|
| 101 |
- |
332 |
$item->addChild('Images');
|
|
|
333 |
$item->{'Images'}->addChild('Primary');
|
|
|
334 |
$item->{'Images'}->{'Primary'}->addChild('Medium');
|
|
|
335 |
$item->{'Images'}->{'Primary'}->{'Medium'}->addChild('URL', $pic);
|
|
|
336 |
$item->{'Images'}->{'Primary'}->addChild('Large');
|
|
|
337 |
$item->{'Images'}->{'Primary'}->{'Large'}->addChild('URL', $pic);
|
| 95 |
- |
338 |
|
|
|
339 |
$nodes = $xpath->query('//table[@id="dmusic_tracklist_content"]//div[contains(concat(" ", normalize-space(@class), " "), " a-section ")]//a[contains(concat(" ", normalize-space(@class), " "), " TitleLink ")]');
|
|
|
340 |
if ($nodes->length > 0) {
|
|
|
341 |
$item->addChild('Tracks');
|
|
|
342 |
$item->Tracks->addChild('Disc', '1');
|
|
|
343 |
|
|
|
344 |
foreach($nodes as $node) {
|
|
|
345 |
$line = trim(preg_replace("/[\n\r]/","", $node->nodeValue));
|
|
|
346 |
$item->Tracks->Disc->addChild('Track', $line);
|
|
|
347 |
}
|
|
|
348 |
} else {
|
|
|
349 |
$nodes = $xpath->query('//div[@id="dmusic_tracklist_player"]//div[contains(concat(" ", normalize-space(@class), " "), " a-row ")]');
|
|
|
350 |
if ($nodes->length > 0) {
|
|
|
351 |
$item->addChild('Tracks');
|
|
|
352 |
$item->Tracks->addChild('Disc', '1');
|
|
|
353 |
|
|
|
354 |
foreach($nodes as $node) {
|
|
|
355 |
$line = trim($node->nodeValue);
|
|
|
356 |
if ($noDiscs == 1 && strpos($line, "Disc") === 0) {
|
|
|
357 |
continue;
|
|
|
358 |
}
|
|
|
359 |
$line = trim(preg_replace("/[\n\r]/","", $line));
|
|
|
360 |
if (bin2hex(substr($line, 0, 2)) == "c2a0") {
|
|
|
361 |
$line = trim(substr($line, 2));
|
|
|
362 |
}
|
|
|
363 |
$item->Tracks->Disc->addChild('Track', $line);
|
|
|
364 |
}
|
|
|
365 |
}
|
|
|
366 |
}
|
|
|
367 |
|
| 101 |
- |
368 |
$item->addChild('ItemInfo');
|
|
|
369 |
$item->ItemInfo->addChild('Title');
|
|
|
370 |
$item->ItemInfo->{'Title'}->addChild('DisplayValue', $title);
|
|
|
371 |
$item->ItemInfo->addChild('Artist', $artists);
|
|
|
372 |
$item->ItemInfo->addChild('ByLineInfo');
|
|
|
373 |
$item->ItemInfo->{'ByLineInfo'}->addChild('Manufacturer');
|
|
|
374 |
$item->ItemInfo->{'ByLineInfo'}->{'Manufacturer'}->addChild('DisplayValue', $label);
|
|
|
375 |
$item->ItemInfo->addChild('ContentInfo');
|
|
|
376 |
$item->ItemInfo->{'ContentInfo'}->addChild('ReleaseDate');
|
|
|
377 |
$item->ItemInfo->{'ContentInfo'}->{'ReleaseDate'}->addChild('DisplayValue', $releaseDate);
|
|
|
378 |
$item->ItemInfo->{'ContentInfo'}->addChild('UnitCount');
|
|
|
379 |
$item->ItemInfo->{'ContentInfo'}->{'UnitCount'}->addChild('DisplayValue', $noDiscs);
|
|
|
380 |
$item->ItemInfo->addChild('MediaType', $mediaType);
|
|
|
381 |
$item->ItemInfo->addChild('Edition', $edition);
|
|
|
382 |
$item->ItemInfo->addChild('Genre', $genre);
|
|
|
383 |
$item->ItemInfo->addChild('RunningTime', (int)$runTime);
|
| 95 |
- |
384 |
|
|
|
385 |
$_SESSION["discogs"] .= addMatch($item, $cnt, $mediaType);
|
|
|
386 |
}
|