Subversion Repositories cheapmusic

Rev

Rev 141 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
91 - 1
<?php
2
error_reporting(E_ALL);
3
 
129 - 4
// Get Amazon Listings from webpage scrapes
91 - 5
function get_amazon_scrape($query, $searchCondition) {
129 - 6
    $userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0";
136 - 7
    $affiliateTag = "tag=uj024-20&language=en_US";
129 - 8
    $urlDomain = "https://www.amazon.com";
136 - 9
    $productFragment = "/dp/";
10
    $productListingFragment = "/gp/offer-listing/";
95 - 11
    $vendors = Vendors::getInstance();
12
    $config = $vendors->getVendor(Vendors::AMAZON);
136 - 13
    $numResults = $config['numResults'];
95 - 14
    $numListings = $config['numListings'];
136 - 15
    $config = $vendors->getVendor(Vendors::DISCOGS);
16
    $maxMasterCount = $config['maxMasters'];
95 - 17
 
18
    $needMatches = empty($_SESSION["discogs"]);
19
    if ($needMatches) {
127 - 20
        $xh = new Html;
21
        $xh->init($_SESSION["htmlIndent"]);
22
        startMatches($xh);
95 - 23
    }
24
 
91 - 25
    $arr = [];
26
    $products = [];
95 - 27
    $cnt = 0;
91 - 28
 
29
    libxml_use_internal_errors(true);
99 - 30
    $html = getSearchCache("amazon_scrape", $query, $searchCondition);
31
    if ($html === false) {
129 - 32
        $html = getUrl($urlDomain . "/s?k=" . rawurlencode($query) . "&i=popular&sf=qz&unfiltered=1&ref=nb_sb_noss");
132 - 33
        $dom = new DOMDocument;
34
        $dom->loadHTML($html);
35
        $xpath = new DOMXPath($dom);
136 - 36
        $nodes = $xpath->query('//span[contains(@cel_widget_id,"SEARCH_RESULTS")]//a[contains(@href, "/dp/B") and (contains(text(), "Audio CD") or contains(text(), "Vinyl"))]/@href');
37
        $cnt = 0;
132 - 38
        foreach($nodes as $href) {
136 - 39
            $res = explode('/', $href->nodeValue);
40
            if ($cnt < $numResults && $res[2] == "dp" && $res[3][0] == "B") {
41
                if (!in_array($res[3], $products)) {
42
                    $products[] = $res[3];
43
                    ++$cnt;
44
                }
132 - 45
            }
46
        }
136 - 47
        saveSearchCache("amazon_scrape", $query, $searchCondition, join("|", $products));
132 - 48
    } else {
49
        $products = explode("|", $html);
99 - 50
    }
51
 
129 - 52
    $urls = [];
53
    $htmls_cache = [];
131 - 54
    $links = [];
136 - 55
    foreach($products as $asin) {
56
        $fragment = $productFragment . $asin;
57
        $url = $urlDomain . $fragment;
58
        $linkFragments[$url] = $fragment;
129 - 59
        $links[$url] = $asin;
136 - 60
        $htmls_cache[$url] = getSearchCache("amazon_scrape", $fragment, "");
129 - 61
        if ($htmls_cache[$url] === false) {
62
            unset($htmls_cache[$url]);
136 - 63
            $urls[$fragment] = $url;
129 - 64
        }
91 - 65
 
136 - 66
        $fragment = $productListingFragment . $asin;
67
        $url = $urlDomain . $fragment;
68
        $linkFragments[$url] = $fragment;
69
        $links[$url] = $asin;
70
        $htmls_cache[$url] = getSearchCache("amazon_scrape", $fragment, "");
129 - 71
        if ($htmls_cache[$url] === false) {
72
            unset($htmls_cache[$url]);
136 - 73
            $urls[$fragment] = $url;
99 - 74
        }
129 - 75
    }
91 - 76
 
129 - 77
    $htmls = [];
78
    if (count($urls) > 0) {
79
        $htmls = getMultiUrl($urls, $userAgent);
80
    }
81
 
82
    foreach($htmls as $key => &$html) {
132 - 83
        $dom = new DOMDocument;
136 - 84
        $dom->loadHTML('<?xml encoding="UTF-8">' . $html);
85
        $xpath = new DOMXPath($dom);
132 - 86
 
136 - 87
        $dels = array('script', 'style', 'link');
88
        foreach($dels as $del) {
89
            $nodes = $xpath->query('//' . $del);
90
            foreach($nodes as $node) {
91
                $node->parentNode->removeChild($node);
92
            }
93
        }
94
 
95
        $html = '<?xml encoding="UTF-8">';
96
 
132 - 97
        if (strpos($key, "offer-listing") > 0) {
136 - 98
            $nodes = $xpath->query('//div[@id="olpProduct"]');
132 - 99
            $html .= $dom->saveHTML($nodes[0]);
136 - 100
 
101
            $nodes = $xpath->query('//div[contains(concat(" ", normalize-space(@class), " "), " olpOffer ")]');
102
            if ($nodes->length > 0) {
103
                for ($i = 0; $i < $nodes->length && $i < $numListings; $i++) {
104
                    $html .= $dom->saveHTML($nodes[$i]);
105
                }
106
            }
132 - 107
        } else {
136 - 108
            $nodes = $xpath->query('//table[@id="productDetailsTable"]');
109
            $html .= $dom->saveHTML($nodes[0]);
110
            $nodes = $xpath->query('//div[@id="dmusicTracklist_feature_div"]');
111
            if ($nodes->length > 0) {
112
                $html .= $dom->saveHTML($nodes[0]);
113
            }
132 - 114
        }
115
 
116
        $html = preg_replace('/^[ \t]*[\r\n]+/m', '', $html);
136 - 117
        saveSearchCache("amazon_scrape", $linkFragments[$key], "", $html);
129 - 118
    }
119
 
120
    foreach($htmls_cache as $key => $html) {
121
        $htmls[$key] = $html;
122
    }
123
    unset($htmls_cache);
137 - 124
 
136 - 125
    $cnt = 0;
137 - 126
    $processedProducts = [];
136 - 127
    foreach($products as $asin) {
128
        $productUrl = $urlDomain . $productFragment . $asin;
129
        $offerUrl = $urlDomain . $productListingFragment . $asin;
129 - 130
 
137 - 131
        $domPrd = new DOMDocument;
132
        $domPrd->loadHTML($htmls[$productUrl]);
133
        $xpathPrd = new DOMXPath($domPrd);
91 - 134
 
95 - 135
        $nodes = $xpathPrd->query('//table[@id="productDetailsTable"]//ul/li');
136
        if ($nodes->length < 1) {
91 - 137
            continue;
138
        }
95 - 139
 
140
        $format = "";
91 - 141
        foreach($nodes as $node) {
142
            $str = trim($node->nodeValue);
95 - 143
            if (strpos($str, "Audio CD") === 0 ||
144
                strpos($str, "Vinyl") === 0 ||
145
                strpos($str, "Sheet") === 0 ||
129 - 146
                strpos($str, "MP3 Music") === 0 ||
95 - 147
                strpos($str, "Hardcover") === 0 ||
148
                strpos($str, "Paperback") === 0) {
149
                $p = strpos($str, " (");
150
                $format = ($p > 0 ? substr($str, 0, $p) : $str);
151
                $releaseDate = ($p > 0 ? substr($str, $p+2, strlen($str) - $p - 3) : "");
152
            }
91 - 153
        }
154
 
155
        if (strpos($format, "Audio CD") === 0 ||
156
            strpos($format, "Vinyl") === 0 ||
129 - 157
            strpos($format, "Sheet") === 0 ||
91 - 158
            strpos($format, "Hardcover") === 0 ||
159
            strpos($format, "Paperback") === 0) {
129 - 160
 
91 - 161
            if (strpos($format, "Audio CD") !== false) {
162
                $mediaType = "CD";
163
            } else if (strpos($format, "Vinyl") !== false) {
164
                $mediaType = "Record";
165
            } else if (strpos($format, "Paperback") !== false ||
166
                       strpos($format, "Sheet") !== false ||
167
                       strpos($format, "Hardcover") !== false) {
168
                $mediaType = "Book";
169
            }
170
 
171
            $dom = new DOMDocument;
136 - 172
            $dom->loadHTML($htmls[$offerUrl]);
91 - 173
            $xpath = new DOMXPath($dom);
174
 
175
            $nodes = $xpath->query('//div[@id="olpProductImage"]//img');
97 - 176
            $pic = "";
177
            if ($nodes->length > 0) {
178
                $pic = $nodes->item(0)->getAttribute("src");
179
            }
132 - 180
 
91 - 181
            $nodes = $xpath->query('//div[@id="olpProductDetails"]/h1');
129 - 182
            $div = $nodes->item(0);
183
            $title = "- / -";
184
            if (is_object($div)) {
185
                $title = trim($nodes->item(0)->nodeValue);
186
            }
95 - 187
            $fullTitle = $title;
91 - 188
 
136 - 189
            $artists = "";
91 - 190
            $nodes = $xpath->query('//div[@id="olpProductByline"]');
191
            if ($nodes->length > 0) {
192
                $artists = trim($nodes->item(0)->nodeValue);
95 - 193
                $artists = str_replace(" (Artist)", "", $artists);
194
                if (strpos($artists, "~ ") === 0) {
195
                    $artists = substr($artists, 2);
196
                }
129 - 197
                if (!empty($artists)) {
198
                    $fullTitle = $title . " by " . $artists;
199
                }
91 - 200
            }
201
 
136 - 202
            if (strpos($format, "Audio CD") === 0 || strpos($format, "Vinyl") === 0) {
137 - 203
                if ($needMatches && $cnt < $maxMasterCount) {
204
                    $releaseTitle = strtolower($title . " by " . $artists);
205
                    $releaseYear = 0;
206
                    if (($timestamp = strtotime($releaseDate)) !== false) {
207
                        $releaseYear = date("Y", $timestamp);
208
                    }
209
 
210
                    if (!skipDuplicateProduct($releaseTitle, $releaseYear, $processedProducts)) {
211
                        ++$cnt;
141 - 212
                        addMatch_scrape($xh, $xpathPrd, $cnt, $title, $artists, $format, $releaseDate, $asin, $productUrl, $pic);
137 - 213
                        $processedProducts[] = array( "title" => $releaseTitle, "year" => $releaseYear );
214
                    }
95 - 215
                }
216
            }
217
 
136 - 218
            $listings = $xpath->query('//div[contains(@class, "olpOffer")]');
91 - 219
 
136 - 220
            $listingCntUsed = 0;
221
            $listingCntNew = 0;
91 - 222
            foreach($listings as $listing) {
223
                $nodes = $xpath->query('.//h3[contains(concat(" ", normalize-space(@class), " "), " olpSellerName ")]', $listing);
224
                $str = trim($nodes->item(0)->nodeValue);
225
                $sellerName = (empty($str) ? "Amazon" : $str);
226
                $merchantName = "Amazon";
227
                $feedbackPercent = -1;
228
                $feedbackScore = -1;
229
 
230
                if ($sellerName != "Amazon") {
231
                    $merchantName .= " Marketplace";
232
                    $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpSellerColumn ")]//p', $listing);
233
                    if ($nodes->length > 0) {
234
                        $str = trim($nodes->item(0)->nodeValue);
235
                        $sellerrating = substr($str, 17);
236
                        $num = preg_match_all('/((?:[0-9]+,)*[0-9]+(?:\.[0-9]+)?)/', $sellerrating, $matches);
237
                        if ($num == 3) {
238
                            $feedbackPercent = (int)$matches[0][0];
239
                            $feedbackScore = (int)str_replace( ',', '', $matches[0][2]);
240
                        }
241
                    }
242
                }
243
 
244
                $nodes = $xpath->query('.//span[contains(concat(" ", normalize-space(@class), " "), " olpCondition ")]', $listing);
245
                $str = trim($nodes->item(0)->nodeValue);
246
                $pos = strpos($str, " - ");
247
                if ($pos !== false) {
248
                    $condition = trim(substr($str, 0, $pos));
249
                    $detailCondition = trim(substr($str, $pos+3));
250
                } else {
251
                    $condition = $str;
252
                    $detailCondition = $str;
253
                }
254
                if ($condition == "Collectible" || $condition == "Refurbished") {
255
                    $condition = 'Used';
256
                }
257
 
258
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpConditionColumn ")]//div[contains(concat(" ", normalize-space(@class), " "), " comments ")]', $listing);
259
                if ($nodes->length > 0) {
260
                    $conditionComment = trim($nodes->item(0)->nodeValue);
261
                }
262
 
263
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpOfferPrice ")]', $listing);
264
                $price = substr(trim($nodes->item(0)->nodeValue), 1);
265
                $currency = 'USD';
266
 
267
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpShippingPrice ")]', $listing);
268
                if ($nodes->length > 0) {
269
                    $shippingCost = substr(trim($nodes->item(0)->nodeValue), 1);
270
                    $shippingCurrency = 'USD';
271
                    $freeShippingCap = 0;
272
                } else {
273
                    $shippingCost = 0.00;
274
                    $shippingCurrency = 'USD';
275
                    $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//p[contains(concat(" ", normalize-space(@class), " "), " olpShippingInfo ")]', $listing);
276
                    $str= trim($nodes->item(0)->nodeValue);
277
                    if (strpos($str, "FREE Shipping") !== false) {
278
                        $freeShippingCap = 0.00;
279
                    }
280
                    if (strpos($str, "on orders over") !== false) {
281
                        $freeShippingCap = 25.00;
282
                    }
283
                }
284
 
285
                $country = 'US';
286
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpDeliveryColumn ")]//ul/li', $listing);
287
                foreach($nodes as $node) {
288
                    $str = trim($node->nodeValue);
289
                    if (strpos($str, "Ships from") === 0) {
290
                        $p = strpos($str, ".");
291
                        $country = getCountryCode(substr($str, 11, $p-11));
292
                    }
293
                }
294
 
295
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//i[contains(concat(" ", normalize-space(@class), " "), " a-icon-prime ")]', $listing);
296
                if ($nodes->length > 0) {
297
                    $sellerName .= " Prime";
298
                }
299
 
136 - 300
                ($condition == 'New' ? ++$listingCntNew : ++$listingCntUsed);
301
 
302
                if (($condition == 'New' && $listingCntNew > $numListings) ||
303
                    ($condition == 'Used' && $listingCntUsed > $numListings)) {
95 - 304
                    continue;
305
                }
306
 
91 - 307
                $arr[] = array(
308
                    "Merchant" => $merchantName,
309
                    "Condition" => $condition,
95 - 310
                    "Title" => $fullTitle,
91 - 311
                    "Barcode" => "",
312
                    "BarcodeType" => "",
313
                    "Image" => $pic,
136 - 314
                    "URL" => $offerUrl . '?' . $affiliateTag,
91 - 315
                    "MediaType" => $mediaType,
316
                    "DetailCondition" => $detailCondition,
317
                    "Country" => $country,
318
                    "BestOffer" => false,
319
                    "TimeLeft" => 0,
320
                    "Price" => $price,
321
                    "Currency" => $currency,
322
                    "ListingType" => "Fixed",
323
                    "Location" => "US",
324
                    "Zip" => "",
325
                    "FeedbackScore" => $feedbackScore,
326
                    "FeedbackPercent" => $feedbackPercent,
327
                    "SellerName" => $sellerName,
328
                    "HandlingTime" => 1,
329
                    "ShippingCost" => $shippingCost,
330
                    "ShippingEstimated" => false,
331
                    "ShippingCurrency" => $shippingCurrency,
332
                    "FreeShippingCap" => $freeShippingCap,
143 - 333
                    "Show" => true,
334
                    "Details" => ""
91 - 335
                );
336
            }
337
        }
338
    }
339
 
95 - 340
    if ($needMatches) {
136 - 341
        if ($cnt == 0) {
95 - 342
            $_SESSION["discogs"] = "";
343
        } else {
130 - 344
            endMatches($xh);
127 - 345
            $_SESSION["discogs"] = $xh->flush();
346
            //error_log(print_r($_SESSION["discogs"], 1));
95 - 347
        }
348
    }
349
 
91 - 350
    return ($arr);
93 - 351
}
352
 
130 - 353
function addMatch_scrape(&$xh, $xpath, $cnt, $title, $artists, $mediaType, $releaseDate, $asin, $url, $pic) {
95 - 354
    $nodes = $xpath->query('//table[@id="productDetailsTable"]//ul/li');
355
    if ($nodes->length < 1) {
356
        return;
357
    }
93 - 358
 
95 - 359
    $runTime = "";
360
    $noDiscs = "";
361
    $label = "";
362
    $edition = "";
363
    $genre = "";
364
 
365
    foreach($nodes as $node) {
366
        $str = trim($node->nodeValue);
132 - 367
 
95 - 368
        $p = strpos($str, "Run Time:");
369
        if ($p === 0) {
370
            $runTime = substr($str, 10);
93 - 371
        }
132 - 372
 
95 - 373
        $p = strpos($str, "Number of Discs:");
374
        if ($p === 0) {
375
            $noDiscs = substr($str, 17);
376
        }
132 - 377
 
95 - 378
        $p = strpos($str, "Label:");
379
        if ($p === 0) {
380
            $label = substr($str, 7);
381
        }
132 - 382
 
95 - 383
        $p = strpos($str, "Edition:");
384
        if ($p === 0) {
385
            $edition = substr($str, 9);
386
        }
132 - 387
 
95 - 388
        $p = strpos($str, "SPARS Code:");
389
        if ($p === 0) {
390
            $edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 12);
391
        }
132 - 392
 
95 - 393
        $p = strpos($str, "Format:");
394
        if ($p === 0) {
395
            $edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 8);
396
        }
397
 
398
        $p = strpos($str, "Performer:");
399
        if ($p === 0) {
400
            $artists = substr($str, 11);
401
        }
402
 
403
        $p = strpos($str, "Original Release Date:");
404
        if ($p === 0) {
405
            $releaseDate = substr($str, 23);
406
        }
407
 
408
        if (strpos($str, "Amazon Best Sellers Rank:") === 0) {
409
            $pieces = explode("\n", $str);
410
            $genres = [];
411
            foreach($pieces as $piece) {
412
                $piece = trim($piece);
413
                $p1 = strpos($piece, "inĀ ");
414
                $p2 = strpos($piece, " (CDs & Vinyl)") ;
415
                if ($p1 === 0 && $p2 > 0) {
416
                    $genres[] = substr($piece, 4, $p2 - 4);
417
                }
418
            }
419
            $genre = join(", ", $genres);
420
        }
93 - 421
    }
95 - 422
 
423
    $item = new SimpleXMLElement("<item></item>");
424
    $item->addChild('ASIN', $asin);
425
    $item->addChild('DetailPageURL', $url);
101 - 426
    $item->addChild('Images');
427
    $item->{'Images'}->addChild('Primary');
428
    $item->{'Images'}->{'Primary'}->addChild('Medium');
429
    $item->{'Images'}->{'Primary'}->{'Medium'}->addChild('URL', $pic);
430
    $item->{'Images'}->{'Primary'}->addChild('Large');
431
    $item->{'Images'}->{'Primary'}->{'Large'}->addChild('URL', $pic);
95 - 432
 
433
    $nodes = $xpath->query('//table[@id="dmusic_tracklist_content"]//div[contains(concat(" ", normalize-space(@class), " "), " a-section ")]//a[contains(concat(" ", normalize-space(@class), " "), " TitleLink ")]');
434
    if ($nodes->length > 0) {
435
        $item->addChild('Tracks');
436
        $item->Tracks->addChild('Disc', '1');
437
 
438
        foreach($nodes as $node) {
439
            $line = trim(preg_replace("/[\n\r]/","", $node->nodeValue));
136 - 440
            $item->Tracks->Disc->addChild('Track', htmlspecialchars($line, ENT_QUOTES));
95 - 441
        }
442
    } else {
132 - 443
        $nodes = $xpath->query('//div[@id="dmusic_tracklist_player"]//div[contains(concat(" ", normalize-space(@class), " "), " a-row ")]');
95 - 444
        if ($nodes->length > 0) {
445
            $item->addChild('Tracks');
446
            $item->Tracks->addChild('Disc', '1');
447
 
448
            foreach($nodes as $node) {
449
                $line = trim($node->nodeValue);
450
                if ($noDiscs == 1 && strpos($line, "Disc") === 0) {
451
                    continue;
452
                }
136 - 453
                $line = trim(preg_replace("/[\n\r]/", "", $line));
454
                $item->Tracks->Disc->addChild('Track', htmlspecialchars($line, ENT_QUOTES));
95 - 455
            }
456
        }
457
    }
458
 
101 - 459
    $item->addChild('ItemInfo');
460
    $item->ItemInfo->addChild('Title');
136 - 461
    $item->ItemInfo->{'Title'}->addChild('DisplayValue', htmlspecialchars($title, ENT_QUOTES));
462
    $item->ItemInfo->addChild('Artist', htmlspecialchars($artists, ENT_QUOTES));
101 - 463
    $item->ItemInfo->addChild('ByLineInfo');
464
    $item->ItemInfo->{'ByLineInfo'}->addChild('Manufacturer');
136 - 465
    $item->ItemInfo->{'ByLineInfo'}->{'Manufacturer'}->addChild('DisplayValue', htmlspecialchars($label, ENT_QUOTES));
101 - 466
    $item->ItemInfo->addChild('ContentInfo');
467
    $item->ItemInfo->{'ContentInfo'}->addChild('ReleaseDate');
468
    $item->ItemInfo->{'ContentInfo'}->{'ReleaseDate'}->addChild('DisplayValue', $releaseDate);
469
    $item->ItemInfo->{'ContentInfo'}->addChild('UnitCount');
470
    $item->ItemInfo->{'ContentInfo'}->{'UnitCount'}->addChild('DisplayValue', $noDiscs);
136 - 471
    $item->ItemInfo->addChild('MediaType', htmlspecialchars($mediaType, ENT_QUOTES));
472
    $item->ItemInfo->addChild('Edition', htmlspecialchars($edition, ENT_QUOTES));
473
    $item->ItemInfo->addChild('Genre', htmlspecialchars($genre, ENT_QUOTES));
101 - 474
    $item->ItemInfo->addChild('RunningTime', (int)$runTime);
95 - 475
 
130 - 476
    $_SESSION["discogs"] .= addMatch($xh, $item, $cnt, $mediaType);
95 - 477
}
137 - 478
 
479
function skipDuplicateProduct($title, $year, $processed) {
480
    foreach ($processed as $v) {
481
        $title = preg_replace("/[\[\(][^)]+[\]\)]/", "", $title);
482
        $sim = similar_text($title, $v['title'], $perc);
483
        if ($perc > 85.00) {
484
/* too many false positives
485
            if ($year > 0 && $v["year"] > 0 && $year != $v["year"]) {
486
                continue;
487
            }
488
*/
489
//error_log($title . " === " . $v['title']);
490
//error_log("similarity: $sim (" . number_format($perc, 2) . "%)");
491
            return true;
492
        }
493
    }
494
 
495
    return false;
143 - 496
}