Subversion Repositories cheapmusic

Rev

Rev 121 | Rev 129 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
91 - 1
<?php
2
error_reporting(E_ALL);
3
 
4
// Get itunes listings
5
function get_amazon_scrape($query, $searchCondition) {
95 - 6
    $vendors = Vendors::getInstance();
7
    $config = $vendors->getVendor(Vendors::AMAZON);
8
    $numListings = $config['numListings'];
9
 
10
    $needMatches = empty($_SESSION["discogs"]);
11
    if ($needMatches) {
127 - 12
        $xh = new Html;
13
        $xh->init($_SESSION["htmlIndent"]);
14
        startMatches($xh);
95 - 15
    }
16
 
91 - 17
    $arr = [];
18
    $products = [];
95 - 19
    $cnt = 0;
120 - 20
    $wlAddArr = [];
21
    $wlSearchArr = [];
91 - 22
 
23
    libxml_use_internal_errors(true);
99 - 24
    $html = getSearchCache("amazon_scrape", $query, $searchCondition);
25
    if ($html === false) {
26
        $html = getUrl("https://www.amazon.com/s?k=" . rawurlencode($query) . "&sf=qz&unfiltered=1&ref=nb_sb_noss");
27
 
28
        saveSearchCache("amazon_scrape", $query, $searchCondition, $html);
29
    }
30
 
91 - 31
    $dom = new DOMDocument;
32
    $dom->loadHTML($html);
33
    $xpath = new DOMXPath($dom);
34
    $nodes = $xpath->query('//a/@href');
35
    foreach($nodes as $href) {
36
        if (strpos($href->nodeValue, "/gp/offer-listing/B") === 0) {
37
            $products[] = $href->nodeValue;
38
        }
39
    }
40
 
41
    foreach($products as $product) {
42
        $url = "https://www.amazon.com" . $product . "&tag=uj024-20&language=en_US";
43
        $asin = explode('/', $product)[3];
44
 
99 - 45
        $html = getSearchCache("amazon_scrape", $asin, "");
46
        if ($html === false) {
47
            $html = getUrl("https://www.amazon.com/dp/" . $asin, "Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/70.0");
48
            saveSearchCache("amazon_scrape", $asin, "", $html);
49
        }
91 - 50
 
51
        $dom = new DOMDocument;
52
        $dom->loadHTML($html);
95 - 53
        $xpathPrd = new DOMXPath($dom);
91 - 54
 
95 - 55
        $nodes = $xpathPrd->query('//table[@id="productDetailsTable"]//ul/li');
56
        if ($nodes->length < 1) {
91 - 57
            continue;
58
        }
95 - 59
 
60
        $format = "";
91 - 61
        foreach($nodes as $node) {
62
            $str = trim($node->nodeValue);
95 - 63
 
64
            if (strpos($str, "Audio CD") === 0 ||
65
                strpos($str, "Vinyl") === 0 ||
66
                strpos($str, "Sheet") === 0 ||
67
                strpos($str, "Hardcover") === 0 ||
68
                strpos($str, "Paperback") === 0) {
69
                $p = strpos($str, " (");
70
                $format = ($p > 0 ? substr($str, 0, $p) : $str);
71
                $releaseDate = ($p > 0 ? substr($str, $p+2, strlen($str) - $p - 3) : "");
72
            }
73
 
91 - 74
            if (strpos($str, "Amazon Best Sellers Rank:") === 0) {
75
                $p = strpos($str, " (");
76
                $rank = substr($str, 11, $p-11);
77
            }
78
        }
79
 
80
        if (strpos($format, "Audio CD") === 0 ||
81
            strpos($format, "Vinyl") === 0 ||
95 - 82
            strpos($str, "Sheet") === 0 ||
91 - 83
            strpos($format, "Hardcover") === 0 ||
84
            strpos($format, "Paperback") === 0) {
85
            if (strpos($format, "Audio CD") !== false) {
86
                $mediaType = "CD";
87
            } else if (strpos($format, "Vinyl") !== false) {
88
                $mediaType = "Record";
89
            } else if (strpos($format, "Paperback") !== false ||
90
                       strpos($format, "Sheet") !== false ||
91
                       strpos($format, "Hardcover") !== false) {
92
                $mediaType = "Book";
93
            }
94
 
99 - 95
            $p = substr($product, 0, strpos($product, "/ref="));
96
            $html = getSearchCache("amazon_scrape", $p, "");
97
            if ($html === false) {
98
                $str = "https://www.amazon.com" . $product;
99
                $html = getUrl($str, "Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/70.0");
100
                saveSearchCache("amazon_scrape", $p, "", $html);
101
            }
91 - 102
 
103
            $dom = new DOMDocument;
104
            $dom->loadHTML($html);
105
            $xpath = new DOMXPath($dom);
106
 
107
            $nodes = $xpath->query('//div[@id="olpProductImage"]//img');
97 - 108
            $pic = "";
109
            if ($nodes->length > 0) {
110
                $pic = $nodes->item(0)->getAttribute("src");
111
            }
112
 
91 - 113
            $nodes = $xpath->query('//div[@id="olpProductDetails"]/h1');
114
            $title = trim($nodes->item(0)->nodeValue);
95 - 115
            $fullTitle = $title;
91 - 116
 
117
            $nodes = $xpath->query('//div[@id="olpProductByline"]');
118
            if ($nodes->length > 0) {
119
                $artists = trim($nodes->item(0)->nodeValue);
95 - 120
                $artists = str_replace(" (Artist)", "", $artists);
121
                if (strpos($artists, "~ ") === 0) {
122
                    $artists = substr($artists, 2);
123
                }
124
                $fullTitle = $title . " by " . $artists;
91 - 125
            }
126
 
95 - 127
            if (strpos($format, "Audio CD") === 0 ||
128
                strpos($format, "Vinyl") === 0) {
129
                if ($needMatches) {
127 - 130
                    addMatch_scrape($xh, $xpathPrd, ++$cnt, $title, $artists, $format, $releaseDate, $asin, $url, $pic, $wlAddArr, $wlSearchArr);
95 - 131
                }
132
            }
133
 
91 - 134
            $listings = $xpath->query('//div[contains(concat(" ", normalize-space(@class), " "), " olpOffer ")]');
135
 
95 - 136
            $listingCnt = 0;
91 - 137
            foreach($listings as $listing) {
138
                $nodes = $xpath->query('.//h3[contains(concat(" ", normalize-space(@class), " "), " olpSellerName ")]', $listing);
139
                $str = trim($nodes->item(0)->nodeValue);
140
                $sellerName = (empty($str) ? "Amazon" : $str);
141
                $merchantName = "Amazon";
142
                $feedbackPercent = -1;
143
                $feedbackScore = -1;
144
 
145
                if ($sellerName != "Amazon") {
146
                    $merchantName .= " Marketplace";
147
                    $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpSellerColumn ")]//p', $listing);
148
                    if ($nodes->length > 0) {
149
                        $str = trim($nodes->item(0)->nodeValue);
150
                        $sellerrating = substr($str, 17);
151
                        $num = preg_match_all('/((?:[0-9]+,)*[0-9]+(?:\.[0-9]+)?)/', $sellerrating, $matches);
152
                        if ($num == 3) {
153
                            $feedbackPercent = (int)$matches[0][0];
154
                            $feedbackScore = (int)str_replace( ',', '', $matches[0][2]);
155
                        }
156
                    }
157
                }
158
 
159
                $nodes = $xpath->query('.//span[contains(concat(" ", normalize-space(@class), " "), " olpCondition ")]', $listing);
160
                $str = trim($nodes->item(0)->nodeValue);
161
                $pos = strpos($str, " - ");
162
                if ($pos !== false) {
163
                    $condition = trim(substr($str, 0, $pos));
164
                    $detailCondition = trim(substr($str, $pos+3));
165
                } else {
166
                    $condition = $str;
167
                    $detailCondition = $str;
168
                }
169
                if ($condition == "Collectible" || $condition == "Refurbished") {
170
                    $condition = 'Used';
171
                }
172
 
173
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpConditionColumn ")]//div[contains(concat(" ", normalize-space(@class), " "), " comments ")]', $listing);
174
                if ($nodes->length > 0) {
175
                    $conditionComment = trim($nodes->item(0)->nodeValue);
176
                }
177
 
178
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpOfferPrice ")]', $listing);
179
                $price = substr(trim($nodes->item(0)->nodeValue), 1);
180
                $currency = 'USD';
181
 
182
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpShippingPrice ")]', $listing);
183
                if ($nodes->length > 0) {
184
                    $shippingCost = substr(trim($nodes->item(0)->nodeValue), 1);
185
                    $shippingCurrency = 'USD';
186
                    $freeShippingCap = 0;
187
                } else {
188
                    $shippingCost = 0.00;
189
                    $shippingCurrency = 'USD';
190
                    $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//p[contains(concat(" ", normalize-space(@class), " "), " olpShippingInfo ")]', $listing);
191
                    $str= trim($nodes->item(0)->nodeValue);
192
                    if (strpos($str, "FREE Shipping") !== false) {
193
                        $freeShippingCap = 0.00;
194
                    }
195
                    if (strpos($str, "on orders over") !== false) {
196
                        $freeShippingCap = 25.00;
197
                    }
198
                }
199
 
200
                $country = 'US';
201
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpDeliveryColumn ")]//ul/li', $listing);
202
                foreach($nodes as $node) {
203
                    $str = trim($node->nodeValue);
204
                    if (strpos($str, "Ships from") === 0) {
205
                        $p = strpos($str, ".");
206
                        $country = getCountryCode(substr($str, 11, $p-11));
207
                    }
208
                }
209
 
210
                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//i[contains(concat(" ", normalize-space(@class), " "), " a-icon-prime ")]', $listing);
211
                if ($nodes->length > 0) {
212
                    $sellerName .= " Prime";
213
                }
214
 
95 - 215
                if (++$listingCnt > $numListings) {
216
                    continue;
217
                }
218
 
91 - 219
                $arr[] = array(
220
                    "Merchant" => $merchantName,
221
                    "Condition" => $condition,
95 - 222
                    "Title" => $fullTitle,
91 - 223
                    "Barcode" => "",
224
                    "BarcodeType" => "",
225
                    "Image" => $pic,
226
                    "URL" => $url,
227
                    "MediaType" => $mediaType,
228
                    "DetailCondition" => $detailCondition,
229
                    "Country" => $country,
230
                    "BestOffer" => false,
231
                    "TimeLeft" => 0,
232
                    "Price" => $price,
233
                    "Currency" => $currency,
234
                    "ListingType" => "Fixed",
235
                    "Location" => "US",
236
                    "Zip" => "",
237
                    "FeedbackScore" => $feedbackScore,
238
                    "FeedbackPercent" => $feedbackPercent,
239
                    "SellerName" => $sellerName,
240
                    "HandlingTime" => 1,
241
                    "ShippingCost" => $shippingCost,
242
                    "ShippingEstimated" => false,
243
                    "ShippingCurrency" => $shippingCurrency,
244
                    "FreeShippingCap" => $freeShippingCap,
245
                    "Show" => true
246
                );
247
 
248
            }
249
        }
250
    }
251
 
95 - 252
    if ($needMatches) {
253
        if ($cnt = 0) {
254
            $_SESSION["discogs"] = "";
255
        } else {
127 - 256
            endMatches($xh, $wlAddArr, $wlSearchArr);
120 - 257
 
127 - 258
            $_SESSION["discogs"] = $xh->flush();
259
            //error_log(print_r($_SESSION["discogs"], 1));
95 - 260
        }
261
    }
262
 
91 - 263
    return ($arr);
93 - 264
}
265
 
127 - 266
function addMatch_scrape(&$xh, $xpath, $cnt, $title, $artists, $mediaType, $releaseDate, $asin, $url, $pic, &$wlAddArr, &$wlSearchArr) {
95 - 267
    $nodes = $xpath->query('//table[@id="productDetailsTable"]//ul/li');
268
    if ($nodes->length < 1) {
269
        return;
270
    }
93 - 271
 
95 - 272
    $runTime = "";
273
    $noDiscs = "";
274
    $label = "";
275
    $edition = "";
276
    $genre = "";
277
 
278
    foreach($nodes as $node) {
279
        $str = trim($node->nodeValue);
280
 
281
        $p = strpos($str, "Run Time:");
282
        if ($p === 0) {
283
            $runTime = substr($str, 10);
93 - 284
        }
95 - 285
 
286
        $p = strpos($str, "Number of Discs:");
287
        if ($p === 0) {
288
            $noDiscs = substr($str, 17);
289
        }
290
 
291
        $p = strpos($str, "Label:");
292
        if ($p === 0) {
293
            $label = substr($str, 7);
294
        }
295
 
296
        $p = strpos($str, "Edition:");
297
        if ($p === 0) {
298
            $edition = substr($str, 9);
299
        }
300
 
301
        $p = strpos($str, "SPARS Code:");
302
        if ($p === 0) {
303
            $edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 12);
304
        }
305
 
306
        $p = strpos($str, "Format:");
307
        if ($p === 0) {
308
            $edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 8);
309
        }
310
 
311
        $p = strpos($str, "Performer:");
312
        if ($p === 0) {
313
            $artists = substr($str, 11);
314
        }
315
 
316
        $p = strpos($str, "Original Release Date:");
317
        if ($p === 0) {
318
            $releaseDate = substr($str, 23);
319
        }
320
 
321
        if (strpos($str, "Amazon Best Sellers Rank:") === 0) {
322
            $pieces = explode("\n", $str);
323
            $genres = [];
324
            foreach($pieces as $piece) {
325
                $piece = trim($piece);
326
                $p1 = strpos($piece, "in ");
327
                $p2 = strpos($piece, " (CDs & Vinyl)") ;
328
                if ($p1 === 0 && $p2 > 0) {
329
                    $genres[] = substr($piece, 4, $p2 - 4);
330
                }
331
            }
332
            $genre = join(", ", $genres);
333
        }
93 - 334
    }
95 - 335
 
336
    $item = new SimpleXMLElement("<item></item>");
337
    $item->addChild('ASIN', $asin);
338
    $item->addChild('DetailPageURL', $url);
101 - 339
    $item->addChild('Images');
340
    $item->{'Images'}->addChild('Primary');
341
    $item->{'Images'}->{'Primary'}->addChild('Medium');
342
    $item->{'Images'}->{'Primary'}->{'Medium'}->addChild('URL', $pic);
343
    $item->{'Images'}->{'Primary'}->addChild('Large');
344
    $item->{'Images'}->{'Primary'}->{'Large'}->addChild('URL', $pic);
95 - 345
 
346
    $nodes = $xpath->query('//table[@id="dmusic_tracklist_content"]//div[contains(concat(" ", normalize-space(@class), " "), " a-section ")]//a[contains(concat(" ", normalize-space(@class), " "), " TitleLink ")]');
347
    if ($nodes->length > 0) {
348
        $item->addChild('Tracks');
349
        $item->Tracks->addChild('Disc', '1');
350
 
351
        foreach($nodes as $node) {
352
            $line = trim(preg_replace("/[\n\r]/","", $node->nodeValue));
353
            $item->Tracks->Disc->addChild('Track', $line);
354
        }
355
    } else {
356
        $nodes = $xpath->query('//div[@id="dmusic_tracklist_player"]//div[contains(concat(" ", normalize-space(@class), " "), " a-row ")]');
357
        if ($nodes->length > 0) {
358
            $item->addChild('Tracks');
359
            $item->Tracks->addChild('Disc', '1');
360
 
361
            foreach($nodes as $node) {
362
                $line = trim($node->nodeValue);
363
                if ($noDiscs == 1 && strpos($line, "Disc") === 0) {
364
                    continue;
365
                }
366
                $line = trim(preg_replace("/[\n\r]/","", $line));
367
                if (bin2hex(substr($line, 0, 2)) == "c2a0") {
368
                    $line = trim(substr($line, 2));
369
                }
370
                $item->Tracks->Disc->addChild('Track', $line);
371
            }
372
        }
373
    }
374
 
101 - 375
    $item->addChild('ItemInfo');
376
    $item->ItemInfo->addChild('Title');
377
    $item->ItemInfo->{'Title'}->addChild('DisplayValue', $title);
378
    $item->ItemInfo->addChild('Artist', $artists);
379
    $item->ItemInfo->addChild('ByLineInfo');
380
    $item->ItemInfo->{'ByLineInfo'}->addChild('Manufacturer');
381
    $item->ItemInfo->{'ByLineInfo'}->{'Manufacturer'}->addChild('DisplayValue', $label);
382
    $item->ItemInfo->addChild('ContentInfo');
383
    $item->ItemInfo->{'ContentInfo'}->addChild('ReleaseDate');
384
    $item->ItemInfo->{'ContentInfo'}->{'ReleaseDate'}->addChild('DisplayValue', $releaseDate);
385
    $item->ItemInfo->{'ContentInfo'}->addChild('UnitCount');
386
    $item->ItemInfo->{'ContentInfo'}->{'UnitCount'}->addChild('DisplayValue', $noDiscs);
387
    $item->ItemInfo->addChild('MediaType', $mediaType);
388
    $item->ItemInfo->addChild('Edition', $edition);
389
    $item->ItemInfo->addChild('Genre', $genre);
390
    $item->ItemInfo->addChild('RunningTime', (int)$runTime);
95 - 391
 
127 - 392
    $_SESSION["discogs"] .= addMatch($xh, $item, $cnt, $mediaType, $wlAddArr, $wlSearchArr);
95 - 393
}