Subversion Repositories cheapmusic

<?php

error_reporting(E_ALL);

// Get Amazon Listings from webpage scrapes

function get_amazon_scrape($query, $searchCondition) {

    $userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0";

    $affiliateTag = "tag=uj024-20&language=en_US";

    $urlDomain = "https://www.amazon.com";

    $productFragment = "/dp/";

    $productListingFragment = "/gp/offer-listing/";

    $vendors = Vendors::getInstance();

    $config = $vendors->getVendor(Vendors::AMAZON);

    $numResults = $config['numResults'];

    $numListings = $config['numListings'];

    $config = $vendors->getVendor(Vendors::DISCOGS);

    $maxMasterCount = $config['maxMasters'];

    $needMatches = empty($_SESSION["discogs"]);

    if ($needMatches) {

        $xh = new Html;

        $xh->init($_SESSION["htmlIndent"]);

        startMatches($xh);

    }

    $arr = [];

    $products = [];

    $cnt = 0;

    libxml_use_internal_errors(true);

    $html = getSearchCache("amazon_scrape", $query, $searchCondition);

    if ($html === false) {

        $html = getUrl($urlDomain . "/s?k=" . rawurlencode($query) . "&i=popular&sf=qz&unfiltered=1&ref=nb_sb_noss");

        $dom = new DOMDocument;

        $dom->loadHTML($html);

        $xpath = new DOMXPath($dom);

        $nodes = $xpath->query('//span[contains(@cel_widget_id,"SEARCH_RESULTS")]//a[contains(@href, "/dp/B") and (contains(text(), "Audio CD") or contains(text(), "Vinyl"))]/@href');

        $cnt = 0;

        foreach($nodes as $href) {

            $res = explode('/', $href->nodeValue);

            if ($cnt < $numResults && $res[2] == "dp" && $res[3][0] == "B") {

                if (!in_array($res[3], $products)) {

                    $products[] = $res[3];

                    ++$cnt;

                }

            }

        }

        saveSearchCache("amazon_scrape", $query, $searchCondition, join("|", $products));

    } else {

        $products = explode("|", $html);

    }

    $urls = [];

    $htmls_cache = [];

    $links = [];

    foreach($products as $asin) {

        $fragment = $productFragment . $asin;

        $url = $urlDomain . $fragment;

        $linkFragments[$url] = $fragment;

        $links[$url] = $asin;

        $htmls_cache[$url] = getSearchCache("amazon_scrape", $fragment, "");

        if ($htmls_cache[$url] === false) {

            unset($htmls_cache[$url]);

            $urls[$fragment] = $url;

        }

        $fragment = $productListingFragment . $asin;

        $url = $urlDomain . $fragment;

        $linkFragments[$url] = $fragment;

        $links[$url] = $asin;

        $htmls_cache[$url] = getSearchCache("amazon_scrape", $fragment, "");

        if ($htmls_cache[$url] === false) {

            unset($htmls_cache[$url]);

            $urls[$fragment] = $url;

        }

    }

    $htmls = [];

    if (count($urls) > 0) {

        $htmls = getMultiUrl($urls, $userAgent);

    }

    foreach($htmls as $key => &$html) {

        $dom = new DOMDocument;

        $dom->loadHTML('<?xml encoding="UTF-8">' . $html);

        $xpath = new DOMXPath($dom);

        $dels = array('script', 'style', 'link');

        foreach($dels as $del) {

            $nodes = $xpath->query('//' . $del);

            foreach($nodes as $node) {

                $node->parentNode->removeChild($node);

            }

        }

        $html = '<?xml encoding="UTF-8">';

        if (strpos($key, "offer-listing") > 0) {

            $nodes = $xpath->query('//div[@id="olpProduct"]');

            $html .= $dom->saveHTML($nodes[0]);

            $nodes = $xpath->query('//div[contains(concat(" ", normalize-space(@class), " "), " olpOffer ")]');

            if ($nodes->length > 0) {

                for ($i = 0; $i < $nodes->length && $i < $numListings; $i++) {

                    $html .= $dom->saveHTML($nodes[$i]);

                }

            }

        } else {

            $nodes = $xpath->query('//table[@id="productDetailsTable"]');

            $html .= $dom->saveHTML($nodes[0]);

            $nodes = $xpath->query('//div[@id="dmusicTracklist_feature_div"]');

            if ($nodes->length > 0) {

                $html .= $dom->saveHTML($nodes[0]);

            }

        }

        $html = preg_replace('/^[ \t]*[\r\n]+/m', '', $html);

        saveSearchCache("amazon_scrape", $linkFragments[$key], "", $html);

    }

    foreach($htmls_cache as $key => $html) {

        $htmls[$key] = $html;

    }

    unset($htmls_cache);

    $cnt = 0;

    $processedProducts = [];

    foreach($products as $asin) {

        $productUrl = $urlDomain . $productFragment . $asin;

        $offerUrl = $urlDomain . $productListingFragment . $asin;

        $domPrd = new DOMDocument;

        $domPrd->loadHTML($htmls[$productUrl]);

        $xpathPrd = new DOMXPath($domPrd);

        $nodes = $xpathPrd->query('//table[@id="productDetailsTable"]//ul/li');

        if ($nodes->length < 1) {

            continue;

        }

        $format = "";

        foreach($nodes as $node) {

            $str = trim($node->nodeValue);

            if (strpos($str, "Audio CD") === 0 ||

                strpos($str, "Vinyl") === 0 ||

                strpos($str, "Sheet") === 0 ||

                strpos($str, "MP3 Music") === 0 ||

                strpos($str, "Hardcover") === 0 ||

                strpos($str, "Paperback") === 0) {

                $p = strpos($str, " (");

                $format = ($p > 0 ? substr($str, 0, $p) : $str);

                $releaseDate = ($p > 0 ? substr($str, $p+2, strlen($str) - $p - 3) : "");

            }

        }

        if (strpos($format, "Audio CD") === 0 ||

            strpos($format, "Vinyl") === 0 ||

            strpos($format, "Sheet") === 0 ||

            strpos($format, "Hardcover") === 0 ||

            strpos($format, "Paperback") === 0) {

            if (strpos($format, "Audio CD") !== false) {

                $mediaType = "CD";

            } else if (strpos($format, "Vinyl") !== false) {

                $mediaType = "Record";

            } else if (strpos($format, "Paperback") !== false ||

                       strpos($format, "Sheet") !== false ||

                       strpos($format, "Hardcover") !== false) {

                $mediaType = "Book";

            }

            $dom = new DOMDocument;

            $dom->loadHTML($htmls[$offerUrl]);

            $xpath = new DOMXPath($dom);

            $nodes = $xpath->query('//div[@id="olpProductImage"]//img');

            $pic = "";

            if ($nodes->length > 0) {

                $pic = $nodes->item(0)->getAttribute("src");

            }

            $nodes = $xpath->query('//div[@id="olpProductDetails"]/h1');

            $div = $nodes->item(0);

            $title = "- / -";

            if (is_object($div)) {

                $title = trim($nodes->item(0)->nodeValue);

            }

            $fullTitle = $title;

            $artists = "";

            $nodes = $xpath->query('//div[@id="olpProductByline"]');

            if ($nodes->length > 0) {

                $artists = trim($nodes->item(0)->nodeValue);

                $artists = str_replace(" (Artist)", "", $artists);

                if (strpos($artists, "~ ") === 0) {

                    $artists = substr($artists, 2);

                }

                if (!empty($artists)) {

                    $fullTitle = $title . " by " . $artists;

                }

            }

            if (strpos($format, "Audio CD") === 0 || strpos($format, "Vinyl") === 0) {

                if ($needMatches && $cnt < $maxMasterCount) {

                    $releaseTitle = strtolower($title . " by " . $artists);

                    $releaseYear = 0;

                    if (($timestamp = strtotime($releaseDate)) !== false) {

                        $releaseYear = date("Y", $timestamp);

                    }

                    if (!skipDuplicateProduct($releaseTitle, $releaseYear, $processedProducts)) {

                        ++$cnt;

                        addMatch_scrape($xh, $xpathPrd, $cnt, $title, $artists, $format, $releaseDate, $asin, $productUrl, $pic);

                        $processedProducts[] = array( "title" => $releaseTitle, "year" => $releaseYear );

                    }

                }

            }

            $listings = $xpath->query('//div[contains(@class, "olpOffer")]');

            $listingCntUsed = 0;

            $listingCntNew = 0;

            foreach($listings as $listing) {

                $nodes = $xpath->query('.//h3[contains(concat(" ", normalize-space(@class), " "), " olpSellerName ")]', $listing);

                $str = trim($nodes->item(0)->nodeValue);

                $sellerName = (empty($str) ? "Amazon" : $str);

                $merchantName = "Amazon";

                $feedbackPercent = -1;

                $feedbackScore = -1;

                if ($sellerName != "Amazon") {

                    $merchantName .= " Marketplace";

                    $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpSellerColumn ")]//p', $listing);

                    if ($nodes->length > 0) {

                        $str = trim($nodes->item(0)->nodeValue);

                        $sellerrating = substr($str, 17);

                        $num = preg_match_all('/((?:[0-9]+,)*[0-9]+(?:\.[0-9]+)?)/', $sellerrating, $matches);

                        if ($num == 3) {

                            $feedbackPercent = (int)$matches[0][0];

                            $feedbackScore = (int)str_replace( ',', '', $matches[0][2]);

                        }

                    }

                }

                $nodes = $xpath->query('.//span[contains(concat(" ", normalize-space(@class), " "), " olpCondition ")]', $listing);

                $str = trim($nodes->item(0)->nodeValue);

                $pos = strpos($str, " - ");

                if ($pos !== false) {

                    $condition = trim(substr($str, 0, $pos));

                    $detailCondition = trim(substr($str, $pos+3));

                } else {

                    $condition = $str;

                    $detailCondition = $str;

                }

                if ($condition == "Collectible" || $condition == "Refurbished") {

                    $condition = 'Used';

                }

                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpConditionColumn ")]//div[contains(concat(" ", normalize-space(@class), " "), " comments ")]', $listing);

                if ($nodes->length > 0) {

                    $conditionComment = trim($nodes->item(0)->nodeValue);

                }

                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpOfferPrice ")]', $listing);

                $price = substr(trim($nodes->item(0)->nodeValue), 1);

                $currency = 'USD';

                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//span[contains(concat(" ", normalize-space(@class), " "), " olpShippingPrice ")]', $listing);

                if ($nodes->length > 0) {

                    $shippingCost = substr(trim($nodes->item(0)->nodeValue), 1);

                    $shippingCurrency = 'USD';

                    $freeShippingCap = 0;

                } else {

                    $shippingCost = 0.00;

                    $shippingCurrency = 'USD';

                    $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//p[contains(concat(" ", normalize-space(@class), " "), " olpShippingInfo ")]', $listing);

                    $str= trim($nodes->item(0)->nodeValue);

                    if (strpos($str, "FREE Shipping") !== false) {

                        $freeShippingCap = 0.00;

                    }

                    if (strpos($str, "on orders over") !== false) {

                        $freeShippingCap = 25.00;

                    }

                }

                $country = 'US';

                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpDeliveryColumn ")]//ul/li', $listing);

                foreach($nodes as $node) {

                    $str = trim($node->nodeValue);

                    if (strpos($str, "Ships from") === 0) {

                        $p = strpos($str, ".");

                        $country = getCountryCode(substr($str, 11, $p-11));

                    }

                }

                $nodes = $xpath->query('.//div[contains(concat(" ", normalize-space(@class), " "), " olpPriceColumn ")]//i[contains(concat(" ", normalize-space(@class), " "), " a-icon-prime ")]', $listing);

                if ($nodes->length > 0) {

                    $sellerName .= " Prime";

                }

                ($condition == 'New' ? ++$listingCntNew : ++$listingCntUsed);

                if (($condition == 'New' && $listingCntNew > $numListings) ||

                    ($condition == 'Used' && $listingCntUsed > $numListings)) {

                    continue;

                }

                $arr[] = array(

                    "Merchant" => $merchantName,

                    "Condition" => $condition,

                    "Title" => $fullTitle,

                    "Barcode" => "",

                    "BarcodeType" => "",

                    "Image" => $pic,

                    "URL" => $offerUrl . '?' . $affiliateTag,

                    "MediaType" => $mediaType,

                    "DetailCondition" => $detailCondition,

                    "Country" => $country,

                    "BestOffer" => false,

                    "TimeLeft" => 0,

                    "Price" => $price,

                    "Currency" => $currency,

                    "ListingType" => "Fixed",

                    "Location" => "US",

                    "Zip" => "",

                    "FeedbackScore" => $feedbackScore,

                    "FeedbackPercent" => $feedbackPercent,

                    "SellerName" => $sellerName,

                    "HandlingTime" => 1,

                    "ShippingCost" => $shippingCost,

                    "ShippingEstimated" => false,

                    "ShippingCurrency" => $shippingCurrency,

                    "FreeShippingCap" => $freeShippingCap ?? 0,

                    "Show" => true,

                    "Details" => ""

                );

            }

        }

    }

    if ($needMatches) {

        if ($cnt == 0) {

            $_SESSION["discogs"] = "";

        } else {

            endMatches($xh);

            $_SESSION["discogs"] = $xh->flush();

            //error_log(print_r($_SESSION["discogs"], 1));

        }

    }

    return ($arr);

}

function addMatch_scrape(&$xh, $xpath, $cnt, $title, $artists, $mediaType, $releaseDate, $asin, $url, $pic) {

    $nodes = $xpath->query('//table[@id="productDetailsTable"]//ul/li');

    if ($nodes->length < 1) {

        return;

    }

    $runTime = "";

    $noDiscs = "";

    $label = "";

    $edition = "";

    $genre = "";

    foreach($nodes as $node) {

        $str = trim($node->nodeValue);

        $p = strpos($str, "Run Time:");

        if ($p === 0) {

            $runTime = substr($str, 10);

        }

        $p = strpos($str, "Number of Discs:");

        if ($p === 0) {

            $noDiscs = substr($str, 17);

        }

        $p = strpos($str, "Label:");

        if ($p === 0) {

            $label = substr($str, 7);

        }

        $p = strpos($str, "Edition:");

        if ($p === 0) {

            $edition = substr($str, 9);

        }

        $p = strpos($str, "SPARS Code:");

        if ($p === 0) {

            $edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 12);

        }

        $p = strpos($str, "Format:");

        if ($p === 0) {

            $edition = (strlen($edition) > 0 ? ", " : "") . substr($str, 8);

        }

        $p = strpos($str, "Performer:");

        if ($p === 0) {

            $artists = substr($str, 11);

        }

        $p = strpos($str, "Original Release Date:");

        if ($p === 0) {

            $releaseDate = substr($str, 23);

        }

        if (strpos($str, "Amazon Best Sellers Rank:") === 0) {

            $pieces = explode("\n", $str);

            $genres = [];

            foreach($pieces as $piece) {

                $piece = trim($piece);

                $p1 = strpos($piece, "in ");

                $p2 = strpos($piece, " (CDs & Vinyl)") ;

                if ($p1 === 0 && $p2 > 0) {

                    $genres[] = substr($piece, 4, $p2 - 4);

                }

            }

            $genre = join(", ", $genres);

        }

    }

    $item = new SimpleXMLElement("<item></item>");

    $item->addChild('ASIN', $asin);

    $item->addChild('DetailPageURL', $url);

    $item->addChild('Images');

    $item->{'Images'}->addChild('Primary');

    $item->{'Images'}->{'Primary'}->addChild('Medium');

    $item->{'Images'}->{'Primary'}->{'Medium'}->addChild('URL', $pic);

    $item->{'Images'}->{'Primary'}->addChild('Large');

    $item->{'Images'}->{'Primary'}->{'Large'}->addChild('URL', $pic);

    $nodes = $xpath->query('//table[@id="dmusic_tracklist_content"]//div[contains(concat(" ", normalize-space(@class), " "), " a-section ")]//a[contains(concat(" ", normalize-space(@class), " "), " TitleLink ")]');

    if ($nodes->length > 0) {

        $item->addChild('Tracks');

        $item->Tracks->addChild('Disc', '1');

        foreach($nodes as $node) {

            $line = trim(preg_replace("/[\n\r]/","", $node->nodeValue));

            $item->Tracks->Disc->addChild('Track', htmlspecialchars($line, ENT_QUOTES));

        }

    } else {

        $nodes = $xpath->query('//div[@id="dmusic_tracklist_player"]//div[contains(concat(" ", normalize-space(@class), " "), " a-row ")]');

        if ($nodes->length > 0) {

            $item->addChild('Tracks');

            $item->Tracks->addChild('Disc', '1');

            foreach($nodes as $node) {

                $line = trim($node->nodeValue);

                if ($noDiscs == 1 && strpos($line, "Disc") === 0) {

                    continue;

                }

                $line = trim(preg_replace("/[\n\r]/", "", $line));

                $item->Tracks->Disc->addChild('Track', htmlspecialchars($line, ENT_QUOTES));

            }

        }

    }

    $item->addChild('ItemInfo');

    $item->ItemInfo->addChild('Title');

    $item->ItemInfo->{'Title'}->addChild('DisplayValue', htmlspecialchars($title, ENT_QUOTES));

    $item->ItemInfo->addChild('Artist', htmlspecialchars($artists, ENT_QUOTES));

    $item->ItemInfo->addChild('ByLineInfo');

    $item->ItemInfo->{'ByLineInfo'}->addChild('Manufacturer');

    $item->ItemInfo->{'ByLineInfo'}->{'Manufacturer'}->addChild('DisplayValue', htmlspecialchars($label, ENT_QUOTES));

    $item->ItemInfo->addChild('ContentInfo');

    $item->ItemInfo->{'ContentInfo'}->addChild('ReleaseDate');

    $item->ItemInfo->{'ContentInfo'}->{'ReleaseDate'}->addChild('DisplayValue', $releaseDate);

    $item->ItemInfo->{'ContentInfo'}->addChild('UnitCount');

    $item->ItemInfo->{'ContentInfo'}->{'UnitCount'}->addChild('DisplayValue', $noDiscs);

    $item->ItemInfo->addChild('MediaType', htmlspecialchars($mediaType, ENT_QUOTES));

    $item->ItemInfo->addChild('Edition', htmlspecialchars($edition, ENT_QUOTES));

    $item->ItemInfo->addChild('Genre', htmlspecialchars($genre, ENT_QUOTES));

    $item->ItemInfo->addChild('RunningTime', (int)$runTime);

    $_SESSION["discogs"] .= addMatch($xh, $item, $cnt, $mediaType);

}

function skipDuplicateProduct($title, $year, $processed) {

    foreach ($processed as $v) {

        $title = preg_replace("/[\[\(][^)]+[\]\)]/", "", $title);

        $sim = similar_text($title, $v['title'], $perc);

        if ($perc > 85.00) {

/* too many false positives

            if ($year > 0 && $v["year"] > 0 && $year != $v["year"]) {

                continue;

            }

*/

//error_log($title . " === " . $v['title']);

//error_log("similarity: $sim (" . number_format($perc, 2) . "%)");

            return true;

        }

    }

    return false;

}