import org.apache.commons.lang.StringEscapeUtils import org.serviio.library.metadata.* import org.serviio.library.online.* import org.serviio.util.* /** * Content URL extractor plugin for ZDF-Mediathek * * @author OAR * * Version : 3.0 */ class ZDF extends WebResourceUrlExtractor { final EXTRACTOR_NAME = 'ZDF-Mediathek' final EXTRACTOR_VERSION = 3 final EXTRACTOR_TIMEOUT = 40 final USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36' final URL_BASE = 'http://www.zdf.de' final URL_METABASE = URL_BASE + '/ZDFmediathek/xmlservice/v2/web/beitragsDetails?ak=web&id=' final URL_ASSETBASE = URL_BASE + '/ptmd/vod/mediathek/' final DATE_FORMAT = 'dd.MM.yyyy HH:mm' final KEY_VIDEO = 'VIDEO' final KEY_THUMB = 'THUMB' final KEY_EXPIRE = 'EXPIRE' final KEY_ASSET_HIGH = 'ASSET_H' final KEY_ASSET_MED = 'ASSET_M' final KEY_ASSET_LOW = 'ASSET_L' final VALID_FEED_URL = '^http[s]?://www\\.zdf\\.de/.+' final THEME_TITLE = '(?ms).*?([^<]*)' final THEME_PAGING = '(?ms)\\s*.*? wriCache = new HashMap() String[] assetHigh = [VIDEO_HIGH1, VIDEO_HIGH2] String[] assetMed = [VIDEO_MED1, VIDEO_MED2] String[] assetLow = [VIDEO_LOW1, VIDEO_LOW2] @Override String getExtractorName() { return EXTRACTOR_NAME } @Override int getVersion() { return EXTRACTOR_VERSION } @Override boolean extractorMatches(URL feedUrl) { return feedUrl ==~ VALID_FEED_URL } @Override int getExtractItemsTimeout() { return EXTRACTOR_TIMEOUT } @Override WebResourceContainer extractItems(URL resourceUrl, int maxItemsToRetrieve) { // check wriCache for outdated entries def now = System.currentTimeMillis() def nowDate = new Date(now) def List toDelete = [] wriCache.each { def itemExpireDate = Date.parse(DATE_FORMAT, it.value.getAdditionalInfo().get(KEY_EXPIRE)) if (itemExpireDate < nowDate) toDelete.add(it.key) } toDelete.each { wriCache.remove(it) } def expireDate = new Date(now + getOnlineFeedExpiryInterval()*3600000) //def expireDate = new Date(now + 48*3600000) // debug def expire = expireDate.format(DATE_FORMAT) def webPage def matcher def title def lastReq = '' def entries = new HashMap() def requestURL = resourceUrl def tryRequest = true while (tryRequest) { try { webPage = openURL(requestURL, USER_AGENT) } catch (ex) { log(ex.toString()) return null } // theme title if (title == null) { matcher = webPage =~ THEME_TITLE title = StringEscapeUtils.unescapeHtml(matcher[0][1].toString()) } // items matcher = webPage =~ THEME_ITEMS_ID for (def i = 0; i < matcher.getCount() && entries.size() < maxItemsToRetrieve; i++) { def id = matcher[i][1].toString() if (wriCache.containsKey(id)) { if (!entries.containsKey(id)) entries.put(id, wriCache.get(id)) } else { def item = buildItem(id, expire) if (item != null) { wriCache.put(id, item) entries.put(id, item) } } } //check for next page tryRequest = false if (entries.size() < maxItemsToRetrieve) { matcher = webPage =~ THEME_PAGING if(matcher.find()) { def nextReq = StringEscapeUtils.unescapeHtml(matcher[0][1].toString()) if(lastReq != nextReq) { lastReq = nextReq requestURL = new URL(URL_BASE + nextReq) tryRequest = true } } } } // container def container = new WebResourceContainer() container.setTitle(title) container.setThumbnailUrl(THEME_ICON) container.setItems(new ArrayList(entries.values())) return container } @Override ContentURLContainer extractUrl(WebResourceItem item, PreferredQuality requestedQuality) { def info = item.getAdditionalInfo() def matchArr def mapKey def videoURL switch (requestedQuality) { case PreferredQuality.HIGH: matchArr = assetHigh mapKey = KEY_ASSET_HIGH break case PreferredQuality.MEDIUM: matchArr = assetMed mapKey = KEY_ASSET_MED break default: matchArr = assetLow mapKey = KEY_ASSET_LOW break } if (info.containsKey(mapKey)) videoURL = info.get(mapKey) else { def link = info.get(KEY_VIDEO) def asset try { asset = openURL(new URL(link), USER_AGENT) } catch (ex) { log(ex.toString()) return null } videoURL = getVideoUrl(asset, matchArr) if (videoURL == null) return null info.put(mapKey, videoURL) } def container = new ContentURLContainer() container.setContentUrl(videoURL) container.setThumbnailUrl(info.get(KEY_THUMB)) container.setFileType(MediaFileType.VIDEO) def expire = info.get(KEY_EXPIRE) container.setExpiresOn(Date.parse(DATE_FORMAT, expire)) container.setCacheKey(item.getCacheKey() + mapKey) container.setLive(false) container.setUserAgent(USER_AGENT) return container } private WebResourceItem buildItem(String id, String expire) { def itemMeta try { itemMeta = openURL(new URL(URL_METABASE + id), USER_AGENT) } catch (ex) { log(ex.toString()) return null } // metadata def matcher = itemMeta =~ ITEM_TITLE def title = matcher[0][1].toString() matcher = itemMeta =~ ITEM_BASENAME def base = matcher[0][1].toString() matcher = itemMeta =~ ITEM_STREAMVERSION def version = '' if (matcher.find()) version = '/' + matcher[0][1].toString() def assetURL = URL_ASSETBASE + base + version matcher = itemMeta =~ ITEM_THUMB def thumb = matcher[0][1].toString() def map = new HashMap() map.put(KEY_VIDEO, assetURL) map.put(KEY_THUMB, thumb) map.put(KEY_EXPIRE, expire) def item = new WebResourceItem() item.setTitle(title) item.setAdditionalInfo(map) matcher = itemMeta =~ ITEM_RELEASE if (matcher.find()) { def release = matcher[0][1].toString() item.setReleaseDate(Date.parse(DATE_FORMAT, release)) } item.setCacheKey(ITEM_CACHEPREFIX + id) return item } private String getVideoUrl(String asset, String[] check) { def matcher def videoURL for (def test in check) { matcher = asset =~ test if (matcher.find()) { videoURL = matcher[0][1].toString() break } } return videoURL } static void main(args) { // this is just to test ZDF extractor = new ZDF(); URL videoLink = new URL("http://www.zdf.de/ZDFmediathek/suche?flash=off&sucheText=terra-x"); println "Name : " + extractor.getExtractorName(); println "Version : " + extractor.getVersion(); println "TestMatch : " + extractor.extractorMatches(videoLink); WebResourceContainer container = extractor.extractItems(videoLink, 50); for (def i = 0; i < container.getItems().size(); i++) { extractor.extractUrl(container.getItems()[i], PreferredQuality.HIGH); println "**** HIGH ****" } extractor.extractUrl(container.getItems()[0], PreferredQuality.MEDIUM);println "**** MEDIUM ****" extractor.extractUrl(container.getItems()[0], PreferredQuality.LOW);println "**** LOW ****" // Test Cache container = extractor.extractItems(videoLink, 10); extractor.extractUrl(container.getItems()[0], PreferredQuality.HIGH);println "**** HIGH ****" } }