import java.net.URL; import java.util.Map; import groovy.json.JsonSlurper import org.serviio.library.metadata.InvalidMetadataException import org.serviio.library.online.ContentURLContainer import org.serviio.library.online.PreferredQuality import org.serviio.library.online.WebResourceContainer import org.serviio.library.online.WebResourceItem import org.serviio.library.online.WebResourceUrlExtractor import org.serviio.util.HttpClient import org.serviio.util.ObjectValidator import org.serviio.util.UnicodeReader import org.serviio.util.ZipUtils import org.xml.sax.InputSource import com.sun.syndication.feed.synd.SyndEntry import com.sun.syndication.feed.synd.SyndFeed import com.sun.syndication.feed.synd.SyndLink import com.sun.syndication.io.FeedException import com.sun.syndication.io.SyndFeedInput class VevoBeta extends WebResourceUrlExtractor implements Logger { final int VERSION = 20 final PROTOCOL = '(https?://)' final ADDRESS = '(www.vevo.com)' final END_OR_END_WITH_A_SLASH = '(/?$)' final BEGIN_PARAMS = '(\\?)' final PARAM = '(\\w+=[^#]*)' final PARAMS = BEGIN_PARAMS + '((' + PARAM + '&)*' + PARAM + '?)' final END_WITH_PARAMS = PARAMS + '$' final END_OR_END_WITH_PARAMS = '(' + END_OR_END_WITH_A_SLASH + '|' + END_WITH_PARAMS + ')' final VALID_URL = '^' + PROTOCOL + ADDRESS + END_OR_END_WITH_PARAMS String getExtractorName() { return getClass().getName() } int getVersion() { return VERSION } boolean extractorMatches(URL resourceUrl) { return resourceUrl ==~ VALID_URL } WebResourceContainer extractItems(URL resourceUrl, int maxItemsToRetrieve) { debug("Extracting items from: " + resourceUrl) def urlHelper = new URLHelper(resourceUrl) maxItemsToRetrieve = (urlHelper.params["max"] == null)? maxItemsToRetrieve : urlHelper.params["max"].toInteger() def items = new ArrayList() VevoSongsExtractor.extractSongs(urlHelper.params["artist"], urlHelper.params["genre"], urlHelper.params["order"], maxItemsToRetrieve, this).each{ def songURL = "" if(it.youTubeId != null) { songURL = new URL("http://www.youtube.com/watch?v=" + it.youTubeId + "&feature=youtube_gdata") } else { debug("YouTube ID not found for: " + it.toString()) def songFeed = "http://gdata.youtube.com/feeds/api/videos?q=vevo+" songFeed += it.toString("urlSafeString", "+", "+", "+", "+") songFeed += "&start-index=1&max-results=1&v=2" Map links = FeedLinksExtractor.extractFirstEntryLink(new URL(songFeed), this) songURL = links.alternate != null ? links.alternate : links.default } def additionalInfo = new HashMap() additionalInfo.put("songURL", songURL) items.add(new WebResourceItem(title: it.toString(), releaseDate: null, additionalInfo: additionalInfo)) debug("Item extracted: " + songURL) } return new WebResourceContainer(title: "Vevo Top Videos", thumbnailUrl: "http://cdn.androidpolice.com/wp-content/uploads/2013/03/nexusae0_vevo.png", items: items) } ContentURLContainer extractUrl(WebResourceItem item, PreferredQuality requestedQuality) { def link = item.getAdditionalInfo().get("songURL") Map links = ['alternate': link] return new YouTube().extractUrl(links, requestedQuality) } public void debug(String text) { log(text) //println text } } interface Logger { void debug(String text) } class PrintLnLogger implements Logger { void debug(String text) { println text } } class FeedLinksExtractor { static Map extractFirstEntryLink(URL feedUrl, Logger logger) throws FileNotFoundException, IllegalArgumentException, IOException, InvalidMetadataException { logger?.debug("Parsing feed " + feedUrl.toString()) try { SyndFeed syndfeed = parseFeedStream(feedUrl, logger) return extractLinks((SyndEntry) syndfeed.getEntries().get(0)) } catch (FeedException e) { throw new InvalidMetadataException("Error during feed parsing, provided URL probably doesn't point to a valid RSS/Atom feed. Message: " + e.getMessage()) } } static Map extractLinks(SyndEntry entry) throws MalformedURLException { Map links = new HashMap<>() entry.getLinks() List syndLinks = entry.getLinks() if ((syndLinks != null) && (syndLinks.size() > 0)) for (SyndLink syndLink : syndLinks) { String key = syndLink.getRel() if (ObjectValidator.isEmpty(key)) { key = syndLink.getType() } if (ObjectValidator.isEmpty(key)) { key = "default" } links.put(key, new URL(syndLink.getHref())) } else if (ObjectValidator.isNotEmpty(entry.getLink())) links.put("default", new URL(entry.getLink())) return links } static SyndFeed parseFeedStream(URL feedUrl, Logger logger) throws FileNotFoundException, IOException, IllegalArgumentException, FeedException { SyndFeedInput input = new SyndFeedInput() byte[] feedBytes = HttpClient.retrieveBinaryFileFromURL(feedUrl.toString()) InputStream feedStream = new ByteArrayInputStream(feedBytes) try { return input.build(new InputSource(feedStream)) } catch (FeedException e) { try { logger?.debug("Feed failed parsing " + e.getMessage() + ", trying BOM detection") feedStream.reset() return input.build(new UnicodeReader(feedStream, "UTF-8")) } catch (FeedException e1) { try { logger?.debug("BOM Feed failed parsing " + e1.getMessage() + ", trying unzipping it") feedStream.reset() InputStream unzipped = ZipUtils.unGzipSingleFile(feedStream) try { return input.build(new InputSource(unzipped)) } catch (FeedException e2) { unzipped.reset() return input.build(new UnicodeReader(unzipped, "UTF-8")) } } catch (IOException e2) { } } throw e } } } class URLSafeString { String string String urlSafeString String toString() { return string; } } class Song { URLSafeString title List artistsMain List artistsFeatured String youTubeId String toString(attr, aMSep, aFSep, tSep, fSep) { def strArtistsMain = "" artistsMain.eachWithIndex{it, i -> strArtistsMain += it[attr] strArtistsMain += (i < artistsMain.size() - 1)? aMSep : "" } def strArtistsFeatured = "" artistsFeatured.eachWithIndex{it, i -> strArtistsFeatured += it[attr] strArtistsFeatured += (i < artistsFeatured.size() - 1)? aFSep : "" } def strSong = title[attr] + tSep + strArtistsMain strSong += (strArtistsFeatured.length() > 0)? (fSep + strArtistsFeatured) : "" strSong } public String toString() { toString("string", " & ", ", ", " - ", " ft. ") } } class VevoSongsExtractor { static List extractSongs(String artist, String genre, String order, int maxItemsToExtract, Logger logger) { logger?.debug("Retrieving JSON") maxItemsToExtract = (maxItemsToExtract < 0)? 100 : maxItemsToExtract genre = (genre == null)? "all-genres" : genre order = (order == null)? "mostviewedtoday" : order //http://api.vevo.com/mobile/v1/artist/selena-gomez/videos.json?offset=0&max=20 def playListUrl = "http://api.vevo.com/mobile/v1/" playListUrl += (artist == null)? "video/list.jsonp" : ("artist/" + artist + "/videos.json") playListUrl += "?order=" + order + "&genres=" + genre + "&offset=0&max=" + maxItemsToExtract def playListJson = new URL(playListUrl).getText() playListJson = (artist == null)? playListJson.substring(3,playListJson.size()) : playListJson logger?.debug("Extracting songs from: " + playListUrl) def jsonSlurper = new JsonSlurper() def parsedData = jsonSlurper.parseText(playListJson) def songs = new ArrayList() parsedData.result.eachWithIndex{ song, i -> def artistsMain = new ArrayList() def artistsFeatured = new ArrayList() song.artists_main.each { artistMain -> artistsMain += new URLSafeString(string: artistMain.name, urlSafeString: artistMain.url_safename) } song.artists_featured.each { artistFeatured -> artistsFeatured += new URLSafeString(string: artistFeatured.name, urlSafeString: artistFeatured.url_safename) } //http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=USH5V1321697 def songUrl = new URL("http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=" + song.isrc) def youTubeId = jsonSlurper.parseText(songUrl.getText())?.video?.videoVersions[0]?.id youTubeId = (youTubeId?.length() == 11)? youTubeId : null URLSafeString title = new URLSafeString(string: song.title, urlSafeString: song.url_safe_title) songs += new Song(title: title, artistsMain: artistsMain, artistsFeatured: artistsFeatured, youTubeId: youTubeId) logger?.debug("Song extracted: " + songs.get(i).toString()) } songs } } class URLHelper { String address Map params URLHelper(URL url) { address = "" params = new HashMap() def list = url.toString().tokenize("?") if(list.size() < 1) return address = list[0] if(list.size() < 2) return list[1].tokenize("&").each{ def keyValue = it.tokenize("=") params[keyValue[0]] = keyValue[1] } } } //def testURLMatches() { // assert new Vevo().extractorMatches(new URL("http://www.vevo.com")) // assert ! new Vevo().extractorMatches(new URL("http://www.vev.com")) // assert new Vevo().extractorMatches(new URL("https://www.vevo.com")) // assert new Vevo().extractorMatches(new URL("https://www.vevo.com/")) // assert new Vevo().extractorMatches(new URL("https://www.vevo.com?")) // assert new Vevo().extractorMatches(new URL("https://www.vevo.com?v=")) // assert new Vevo().extractorMatches(new URL("https://www.vevo.com?v=2")) // assert new Vevo().extractorMatches(new URL("https://www.vevo.com?v=2&")) // assert new Vevo().extractorMatches(new URL("https://www.vevo.com?v=")) // assert new Vevo().extractorMatches(new URL("https://www.vevo.com?v=2&X=4")) // assert ! new Vevo().extractorMatches(new URL("https://www.vevo.com?v=2&X=4#")) //} // //def testVevoSongsExtractor() { // VevoSongsExtractor.extractSongs(null, null, 10, new PrintLnLogger()) //} // //def testWebResourceUrlExtractor() { // //new Vevo().extractItems(new URL("http://www.vevo.com?max=100"), -1); // new Vevo().extractItems(new URL("http://www.vevo.com?genre=all-genres&order=mostviewedthisweek&max=30"), -1); // //new Vevo().extractItems(new URL("http://www.vevo.com?max=6&artist=coldplay&order=mostrecent"), -1); //} // //def testFeedLinksExtractor() { // FeedLinksExtractor.extractFirstEntryLink(new URL("http://gdata.youtube.com/feeds/api/videos?v=2&q=n-D1EB74Ckg"), new PrintLnLogger()) //} // //testWebResourceUrlExtractor()