Subversion Repositories cheapmusic

Rev

Rev 120 | Rev 127 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
91 - 1
<?php
2
error_reporting(E_ALL);
3
 
4
// Get itunes listings
5
function get_amazon_scrape($query, $searchCondition) {
95 - 6
    $vendors = Vendors::getInstance();
7
    $config = $vendors->getVendor(Vendors::AMAZON);
8
    $numListings = $config['numListings'];
9
 
10
    $needMatches = empty($_SESSION["discogs"]);
11
    if ($needMatches) {
12
        $_SESSION["discogs"] = startMatches();
13
    }
14
 
91 - 15
    $arr = [];
16
    $products = [];
95 - 17
    $cnt = 0;
120 - 18
    $wlAddArr = [];
19
    $wlSearchArr = [];
91 - 20
 
21
    libxml_use_internal_errors(true);
99 - 22
    $html = getSearchCache("amazon_scrape", $query, $searchCondition);
23
    if ($html === false) {
24
        $html = getUrl("https://www.amazon.com/s?k=" . rawurlencode($query) . "&sf=qz&unfiltered=1&ref=nb_sb_noss");
25
 
26
        saveSearchCache("amazon_scrape", $query, $searchCondition, $html);
27
    }
28
 
91 - 29
    $dom = new DOMDocument;
30
    $dom->loadHTML($html);
31
    $xpath = new DOMXPath($dom);
32
    $nodes = $xpath->query('//a/@href');
33
    foreach($nodes as $href) {
34
        if (strpos($href->nodeValue, "/gp/offer-listing/B") === 0) {
35
            $products[] = $href->nodeValue;
36
        }
37
    }
38
 
39
    foreach($products as $product) {
40
        $url = "https://www.amazon.com" . $product . "&tag=uj024-20&language=en_US";
41
        $asin = explode('/', $product)[3];
42
 
99 - 43
        $html = getSearchCache("amazon_scrape", $asin, "");
44
        if ($html === false) {
45
            $html = getUrl("https://www.amazon.com/dp/" . $asin, "Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/70.0");
46
            saveSearchCache("amazon_scrape", $asin, "", $html);
47
        }
91 - 48
 
49
        $dom = new DOMDocument;
50
        $dom->loadHTML($html);
95 - 51
        $xpathPrd = new DOMXPath($dom);
91 - 52
 
95 - 53
        $nodes = $xpathPrd->query('//table[@id="productDetailsTable"]//ul/li');
54
        if ($nodes->length < 1) {
91 - 55
            continue;
56
        }
95 - 57
 
58
        $format = "";
91 - 59
        foreach($nodes as $node) {
60
            $str = trim($node->nodeValue);
95 - 61
 
62
            if (strpos($str, "Audio CD") === 0 ||
63
                strpos($str, "Vinyl") === 0 ||
64
                strpos($str, "Sheet") === 0 ||
65
                strpos($str, "Hardcover") === 0 ||
66
                strpos($str, "Paperback") === 0) {
67
                $p = strpos($str, " (");
68
                $format = ($p > 0 ? substr($str, 0, $p) : $str);
69
                $releaseDate = ($p > 0 ? substr($str, $p+2, strlen($str) - $p - 3) : "");
70
            }
71
 
91 - 72
            if (strpos($str, "Amazon Best Sellers Rank:") === 0) {
73
                $p = strpos($str, " (");
74
                $rank = substr($str, 11, $p-11);
75
            }
76
        }
77
 
78
        if (strpos($format, "Audio CD") === 0 ||
79
            strpos($format, "Vinyl") === 0 ||
95 - 80
            strpos($str, "Sheet") === 0 ||
91 - 81
            strpos($format, "Hardcover") === 0 ||
82
            strpos($format, "Paperback") === 0) {
83
            if (strpos($format, "Audio CD") !== false) {
84
                $mediaType = "CD";
85
            } else if (strpos($format, "Vinyl") !== false) {
86
                $mediaType = "Record";
87
            } else if (strpos($format, "Paperback") !== false ||
88
                       strpos($format, "Sheet") !== false ||
89
                       strpos($format, "Hardcover") !== false) {
90
                $mediaType = "Book";
91
            }
92
 
99 - 93
            $p = substr($product, 0, strpos($product, "/ref="));
94
            $html = getSearchCache("amazon_scrape", $p, "");
95
            if ($html === false) {
96
                $str = "https://www.amazon.com" . $product;
97
                $html = getUrl($str, "Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/70.0");
98
                saveSearchCache("amazon_scrape", $p, "", $html);
99
            }
91 - 100
 
101
            $dom = new DOMDocument;
102
            $dom->loadHTML($html);
103
            $xpath = new DOMXPath($dom);
104
 
105
            $nodes = $xpath->query('//div[@id="olpProductImage"]//img');
97 - 106
            $pic = "";
107
            if ($nodes->length > 0) {
108
                $pic = $nodes->item(0)->getAttribute("src");
109
            }
110
 
91 - 111
            $nodes = $xpath->query('//div[@id="olpProductDetails"]/h1');
112
            $title = trim($nodes->item(0)->nodeValue);
95 - 113
            $fullTitle = $title;
91 - 114
 
115
            $nodes = $xpath->query('//div[@id="olpProductByline"]');
116
            if ($nodes->length > 0) {
117
                $artists = trim($nodes->item(0)->nodeValue);
95 - 118
                $artists = str_replace(" (Artist)", "", $artists);
119
                if (strpos($artists, "~ ") === 0) {
120
                    $artists = substr($artists, 2);
121
                }
122
                $fullTitle = $title . " by " . $artists;
91 - 123
            }
124
 
95 - 125
            if (strpos($format, "Audio CD") === 0 ||
126
                strpos($format, "Vinyl") === 0) {
127
                if ($needMatches) {
120 - 128
                    addMatch_scrape($xpathPrd, ++$cnt, $title, $artists, $format, $releaseDate, $asin, $url, $pic, $wlAddArr, $wlSearchArr);
95 - 129
                }
130
            }
131
 
91 - 132
            $listings = $xpath->query('//div[contains(concat(" ", normalize-space(@class), " "), " olpOffer ")]');
133
 
95 - 134
            $listingCnt = 0;
91 - 135
            foreach($listings as $listing) {
136
                $nodes = $xpath->query('.//h3[contains(concat(" ", normalize-space(@class), " "), " olpSellerName ")]', $listing);
137
                $str = trim($nodes->item(0)->nodeValue);
138
                $sellerName = (empty($str) ? "Amazon" : $str);
139
                $merchantName = "Amazon";
140
                $feedbackPercent = -1;
141
                $feedbackScore = -1;
142
 
143
                if ($sellerName != "Amazon") {
144
                    $merchantName .= " Marketplace";
145
                    $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpSellerColumn ")]//p', $listing);
146
                    if ($nodes->length > 0) {
147
                        $str = trim($nodes->item(0)->nodeValue);
148
                        $sellerrating = substr($str, 17);
149
                        $num = preg_match_all('/((?:[0-9]+,)*[0-9]+(?:\.[0-9]+)?)/', $sellerrating, $matches);
150
                        if ($num == 3) {
151
                            $feedbackPercent = (int)$matches[0][0];
152
                            $feedbackScore = (int)str_replace( ',', '', $matches[0][2]);
153
                        }
154
                    }
155
                }
156
 
157
                $nodes = $xpath->query('.//span[contains(concat(" ", normalize-space(@class), " "), " olpCondition ")]', $listing);
158
                $str = trim($nodes->item(0)->nodeValue);
159
                $pos = strpos($str, " - ");
160
                if ($pos !== false) {
161
                    $condition = trim(substr($str, 0, $pos));
162
                    $detailCondition = trim(substr($str, $pos+3));
163
                } else {
164
                    $condition = $str;
165
                    $detailCondition = $str;
166
                }
167
                if ($condition == "Collectible" || $condition == "Refurbished") {
168
                    $condition = 'Used';
169
                }
170
 
171
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpConditionColumn ")]//div[contains(concat(" ", normalize-space(@class), " "), " comments ")]', $listing);
172
                if ($nodes->length > 0) {
173
                    $conditionComment = trim($nodes->item(0)->nodeValue);
174
                }
175
 
176
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpOfferPrice ")]', $listing);
177
                $price = substr(trim($nodes->item(0)->nodeValue), 1);
178
                $currency = 'USD';
179
 
180
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpShippingPrice ")]', $listing);
181
                if ($nodes->length > 0) {
182
                    $shippingCost = substr(trim($nodes->item(0)->nodeValue), 1);
183
                    $shippingCurrency = 'USD';
184
                    $freeShippingCap = 0;
185
                } else {
186
                    $shippingCost = 0.00;
187
                    $shippingCurrency = 'USD';
188
                    $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//p[contains(concat(" ", normalize-space(@class), " "), " olpShippingInfo ")]', $listing);
189
                    $str= trim($nodes->item(0)->nodeValue);
190
                    if (strpos($str, "FREE Shipping") !== false) {
191
                        $freeShippingCap = 0.00;
192
                    }
193
                    if (strpos($str, "on orders over") !== false) {
194
                        $freeShippingCap = 25.00;
195
                    }
196
                }
197
 
198
                $country = 'US';
199
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpDeliveryColumn ")]//ul/li', $listing);
200
                foreach($nodes as $node) {
201
                    $str = trim($node->nodeValue);
202
                    if (strpos($str, "Ships from") === 0) {
203
                        $p = strpos($str, ".");
204
                        $country = getCountryCode(substr($str, 11, $p-11));
205
                    }
206
                }
207
 
208
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//i[contains(concat(" ", normalize-space(@class), " "), " a-icon-prime ")]', $listing);
209
                if ($nodes->length > 0) {
210
                    $sellerName .= " Prime";
211
                }
212
 
95 - 213
                if (++$listingCnt > $numListings) {
214
                    continue;
215
                }
216
 
91 - 217
                $arr[] = array(
218
                    "Merchant" => $merchantName,
219
                    "Condition" => $condition,
95 - 220
                    "Title" => $fullTitle,
91 - 221
                    "Barcode" => "",
222
                    "BarcodeType" => "",
223
                    "Image" => $pic,
224
                    "URL" => $url,
225
                    "MediaType" => $mediaType,
226
                    "DetailCondition" => $detailCondition,
227
                    "Country" => $country,
228
                    "BestOffer" => false,
229
                    "TimeLeft" => 0,
230
                    "Price" => $price,
231
                    "Currency" => $currency,
232
                    "ListingType" => "Fixed",
233
                    "Location" => "US",
234
                    "Zip" => "",
235
                    "FeedbackScore" => $feedbackScore,
236
                    "FeedbackPercent" => $feedbackPercent,
237
                    "SellerName" => $sellerName,
238
                    "HandlingTime" => 1,
239
                    "ShippingCost" => $shippingCost,
240
                    "ShippingEstimated" => false,
241
                    "ShippingCurrency" => $shippingCurrency,
242
                    "FreeShippingCap" => $freeShippingCap,
243
                    "Show" => true
244
                );
245
 
246
            }
247
        }
248
    }
249
 
95 - 250
    if ($needMatches) {
251
        if ($cnt = 0) {
252
            $_SESSION["discogs"] = "";
253
        } else {
121 - 254
            $_SESSION["discogs"] .= '<script nonce="xxxNONCExxx">';
120 - 255
            $_SESSION["discogs"] .= 'function addDiscogsEventsWait() {';
256
            $_SESSION["discogs"] .= 'document.addEventListener("DOMContentLoaded", function() {';
257
            $_SESSION["discogs"] .= '	addDiscogsEvents();';
258
            $_SESSION["discogs"] .= '});';
259
            $_SESSION["discogs"] .= '}';
260
 
261
            $_SESSION["discogs"] .= 'function addDiscogsEvents() {';
262
            foreach($wlAddArr as $k=>$v) {
263
                $_SESSION["discogs"] .= 'el = document.getElementById("' . $k . '");';
264
                $_SESSION["discogs"] .= 'if (el) document.getElementById("' . $k . '").addEventListener("click", function() {';
265
                $_SESSION["discogs"] .= $v;
266
                $_SESSION["discogs"] .= '	});';
267
            }
268
 
269
            foreach($wlSearchArr as $k=>$v) {
270
                $_SESSION["discogs"] .= 'el = document.getElementById("' . $k . '");';
271
                $_SESSION["discogs"] .= 'if (el) document.getElementById("' . $k . '").addEventListener("click", function() {';
272
                $_SESSION["discogs"] .= $v;
273
                $_SESSION["discogs"] .= '	});';
274
            }
275
            $_SESSION["discogs"] .= '}';
276
            $_SESSION["discogs"] .= 'addDiscogsEventsWait();';
277
            $_SESSION["discogs"] .= '</script>';
278
 
95 - 279
            $_SESSION["discogs"] .= endMatches();
280
        }
281
    }
282
 
91 - 283
    return ($arr);
93 - 284
}
285
 
120 - 286
function addMatch_scrape($xpath, $cnt, $title, $artists, $mediaType, $releaseDate, $asin, $url, $pic, &$wlAddArr, &$wlSearchArr) {
95 - 287
    $nodes = $xpath->query('//table[@id="productDetailsTable"]//ul/li');
288
    if ($nodes->length < 1) {
289
        return;
290
    }
93 - 291
 
95 - 292
    $runTime = "";
293
    $noDiscs = "";
294
    $label = "";
295
    $edition = "";
296
    $genre = "";
297
 
298
    foreach($nodes as $node) {
299
        $str = trim($node->nodeValue);
300
 
301
        $p = strpos($str, "Run Time:");
302
        if ($p === 0) {
303
            $runTime = substr($str, 10);
93 - 304
        }
95 - 305
 
306
        $p = strpos($str, "Number of Discs:");
307
        if ($p === 0) {
308
            $noDiscs = substr($str, 17);
309
        }
310
 
311
        $p = strpos($str, "Label:");
312
        if ($p === 0) {
313
            $label = substr($str, 7);
314
        }
315
 
316
        $p = strpos($str, "Edition:");
317
        if ($p === 0) {
318
            $edition = substr($str, 9);
319
        }
320
 
321
        $p = strpos($str, "SPARS Code:");
322
        if ($p === 0) {
323
            $edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 12);
324
        }
325
 
326
        $p = strpos($str, "Format:");
327
        if ($p === 0) {
328
            $edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 8);
329
        }
330
 
331
        $p = strpos($str, "Performer:");
332
        if ($p === 0) {
333
            $artists = substr($str, 11);
334
        }
335
 
336
        $p = strpos($str, "Original Release Date:");
337
        if ($p === 0) {
338
            $releaseDate = substr($str, 23);
339
        }
340
 
341
        if (strpos($str, "Amazon Best Sellers Rank:") === 0) {
342
            $pieces = explode("\n", $str);
343
            $genres = [];
344
            foreach($pieces as $piece) {
345
                $piece = trim($piece);
346
                $p1 = strpos($piece, "in ");
347
                $p2 = strpos($piece, " (CDs & Vinyl)") ;
348
                if ($p1 === 0 && $p2 > 0) {
349
                    $genres[] = substr($piece, 4, $p2 - 4);
350
                }
351
            }
352
            $genre = join(", ", $genres);
353
        }
93 - 354
    }
95 - 355
 
356
    $item = new SimpleXMLElement("<item></item>");
357
    $item->addChild('ASIN', $asin);
358
    $item->addChild('DetailPageURL', $url);
101 - 359
    $item->addChild('Images');
360
    $item->{'Images'}->addChild('Primary');
361
    $item->{'Images'}->{'Primary'}->addChild('Medium');
362
    $item->{'Images'}->{'Primary'}->{'Medium'}->addChild('URL', $pic);
363
    $item->{'Images'}->{'Primary'}->addChild('Large');
364
    $item->{'Images'}->{'Primary'}->{'Large'}->addChild('URL', $pic);
95 - 365
 
366
    $nodes = $xpath->query('//table[@id="dmusic_tracklist_content"]//div[contains(concat(" ", normalize-space(@class), " "), " a-section ")]//a[contains(concat(" ", normalize-space(@class), " "), " TitleLink ")]');
367
    if ($nodes->length > 0) {
368
        $item->addChild('Tracks');
369
        $item->Tracks->addChild('Disc', '1');
370
 
371
        foreach($nodes as $node) {
372
            $line = trim(preg_replace("/[\n\r]/","", $node->nodeValue));
373
            $item->Tracks->Disc->addChild('Track', $line);
374
        }
375
    } else {
376
        $nodes = $xpath->query('//div[@id="dmusic_tracklist_player"]//div[contains(concat(" ", normalize-space(@class), " "), " a-row ")]');
377
        if ($nodes->length > 0) {
378
            $item->addChild('Tracks');
379
            $item->Tracks->addChild('Disc', '1');
380
 
381
            foreach($nodes as $node) {
382
                $line = trim($node->nodeValue);
383
                if ($noDiscs == 1 && strpos($line, "Disc") === 0) {
384
                    continue;
385
                }
386
                $line = trim(preg_replace("/[\n\r]/","", $line));
387
                if (bin2hex(substr($line, 0, 2)) == "c2a0") {
388
                    $line = trim(substr($line, 2));
389
                }
390
                $item->Tracks->Disc->addChild('Track', $line);
391
            }
392
        }
393
    }
394
 
101 - 395
    $item->addChild('ItemInfo');
396
    $item->ItemInfo->addChild('Title');
397
    $item->ItemInfo->{'Title'}->addChild('DisplayValue', $title);
398
    $item->ItemInfo->addChild('Artist', $artists);
399
    $item->ItemInfo->addChild('ByLineInfo');
400
    $item->ItemInfo->{'ByLineInfo'}->addChild('Manufacturer');
401
    $item->ItemInfo->{'ByLineInfo'}->{'Manufacturer'}->addChild('DisplayValue', $label);
402
    $item->ItemInfo->addChild('ContentInfo');
403
    $item->ItemInfo->{'ContentInfo'}->addChild('ReleaseDate');
404
    $item->ItemInfo->{'ContentInfo'}->{'ReleaseDate'}->addChild('DisplayValue', $releaseDate);
405
    $item->ItemInfo->{'ContentInfo'}->addChild('UnitCount');
406
    $item->ItemInfo->{'ContentInfo'}->{'UnitCount'}->addChild('DisplayValue', $noDiscs);
407
    $item->ItemInfo->addChild('MediaType', $mediaType);
408
    $item->ItemInfo->addChild('Edition', $edition);
409
    $item->ItemInfo->addChild('Genre', $genre);
410
    $item->ItemInfo->addChild('RunningTime', (int)$runTime);
95 - 411
 
120 - 412
    $_SESSION["discogs"] .= addMatch($item, $cnt, $mediaType, $wlAddArr, $wlSearchArr);
95 - 413
}