Radio Liberty.js

{
	"translatorID": "b1c90b99-2e1a-4374-a03b-92e45f1afc55",
	"label": "Radio Free Europe / Radio Liberty",
	"creator": "Avram Lyon",
	"target": "^https?://(www\\.rferl\\.org/|www\\.azatliq\\.org/|www\\.azattyq\\.org/|rus\\.azattyq\\.org/|da\\.azadiradio\\.org/|pa\\.azadiradio\\.org/|www\\.azattyk\\.org/|www\\.ozodi\\.org/|www\\.ozodlik\\.org/|www\\.evropaelire\\.org/|www\\.slobodnaevropa\\.org/|www\\.makdenes\\.org/|www\\.iraqhurr\\.org/|www\\.radiofarda\\.com/|www\\.azatutyun\\.am/|www\\.azadliq\\.org/|www\\.svaboda\\.org/|www\\.svoboda\\.org/|www\\.tavisupleba\\.org/|www\\.azathabar\\.com/|www\\.svobodanews\\.ru/|www\\.europalibera\\.org/|www\\.radiosvoboda\\.org/)",
	"minVersion": "2.1.9",
	"maxVersion": "",
	"priority": 100,
	"inRepository": true,
	"translatorType": 4,
	"browserSupport": "gcsbv",
	"lastUpdated": "2014-04-04 10:15:37"
}

/*
   Radio Liberty Translator
   Copyright (C) 2009-2011 Avram Lyon, ajlyon@gmail.com

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 This translator works on articles posted on the websites of Radio Free Europe / Radio Liberty.
 It imports the basic metadata the site provides, from normal article pages and from search 
 result pages.

 The translator tries to work on all of the languages of RFE/RL; they should all work.
 
 Editions:
	English:	http://www.rferl.org/
	Tatar/Bashkir:	http://www.azatliq.org/
	Kazakh:		http://www.azattyq.org/	(Kazakh)
			http://rus.azattyq.org/	(Russian)
	Afghan:		http://da.azadiradio.org/ (Dari)
			http://pa.azadiradio.org/ (Pashto)
	Kirghiz:	http://www.azattyk.org/
	Tajik:		http://www.ozodi.org/
	Uzbek:		http://www.ozodlik.org/
	Albanian:	http://www.evropaelire.org/
	Bosnian/Montenegrin/Serbian:
			http://www.slobodnaevropa.org/
	Macedonian:	http://www.makdenes.org/
	Iraqi Arabic:	http://www.iraqhurr.org/
	Farsi:		http://www.radiofarda.com/
	Armenian:	http://www.azatutyun.am/
	Azerbaijani:	http://www.azadliq.org/
	Belarus:	http://www.svaboda.org/
	Georgian:	http://www.tavisupleba.org/
	Turkmen:	http://www.azathabar.com/
	Russian:	http://www.svobodanews.ru/ and svoboda.org
	Moldovan:	http://www.europalibera.org/ (Romanian)
	Ukrainian:	http://www.radiosvoboda.org/
 
 This translator does not yet attempt to work with the video files that Radio Liberty
 hosts and produces; work with them must be left for a future revision.

 It does try to save linked audio files for stories-- still nothing
 for video content.

 Another future improvement would be the facility to import from the front page and subject
 pages. This is not yet possible.

 Some of the services use non-standard ways of marking authorship, for example, the Pashto edition
 places the author at the bottom of the article, but there is no clear way to scrape that
 information and the translator does not load it.
*/

var item;
function detectWeb(doc, url){
	if (url.match(/\/content\/|\/archive\/news|\/archive\/ru_news_zone/)) {
		// The translator uses this type because RFE/RL generally has a place of publication
		// and a Section; both are specific to newspaperArticle.
		return "newspaperArticle";
	} else if (url.match(/\/search\/\?k=.+/)){
		return "multiple";
	}
}

function doWeb(doc, url){
	
	var articles = new Array();
	if (detectWeb(doc, url) == "multiple") {
		var results = doc.evaluate('//div[@class="searchResultItem"]', doc, null, XPathResult.ANY_TYPE, null);
		var items = new Array();
		var result;
		while(result = results.iterateNext()) {
			var link = doc.evaluate('./a[@class="resultLink"]', result, null, XPathResult.ANY_TYPE, null).iterateNext();
			var title = link.textContent;
			var url = link.href;
			items[url] = title;
		}
		Zotero.selectItems(items, function (items) {
			if (!items) {
				return true;
			}
			for (var i in items) {
				articles.push(i);
			}
			Zotero.Utilities.processDocuments(articles, scrape);	
		});
	} else {
		scrape(doc, url);
	}
	
function scrape(doc, url){
		item = new Zotero.Item("newspaperArticle");
		item.title = Zotero.Utilities.trimInternal(
			doc.evaluate('//h1', doc, null, XPathResult.ANY_TYPE, null).iterateNext().textContent
		);
		
		var author = doc.evaluate('//div[@id="article"]//div[@class="author"]', doc, null, XPathResult.ANY_TYPE, null);
		if ((author = author.iterateNext()) !== null) {
			author = author.textContent;
			// Sometimes we have "By Author"
			if(author.substr(0, 3).toLowerCase() == "by ") {
				author = author.substr(3);
			}
			var cleaned = Zotero.Utilities.cleanAuthor(author, "author");
			// If we have only one name, set the author to one-name mode
			if (cleaned.firstName == "") {
				cleaned["fieldMode"] = true;
			} else {
				// We can check for all lower-case and capitalize if necessary
				// All-uppercase is handled by cleanAuthor
				cleaned.firstName = (cleaned.firstName == cleaned.firstName.toLowerCase()) ?
					Zotero.Utilities.capitalizeTitle(cleaned.firstName, true) : cleaned.firstName;
				cleaned.lastName = (cleaned.lastName == cleaned.lastName.toLowerCase()) ?
					Zotero.Utilities.capitalizeTitle(cleaned.lastName, true) : cleaned.lastName;
			}
			item.creators.push(cleaned);
		}
		// The section should _always_ be present
		item.section = ZU.xpathText(doc, '//div[@id="article" or contains(@class, "middle_content")]/h2');

		// This exposes a limitation of Zotero's date handling; the Afghan services
		// use the Hijri calendar, and mixed sorting looks funny-- I'd like to be able
		// to mark such dates to be handled appropriately
		var date = doc.evaluate('//div[@id="article"]//p[@class="article_date"]', doc, null, XPathResult.ANY_TYPE, null);
		if ((date = date.iterateNext()) !== null) {
			// sometimes not present
			item.date = Zotero.Utilities.trimInternal(date.textContent);
		}

		// We can also try to derive the location-- if the byline can be parsed
		// Here, we assume that the byline uses all-caps for the location
		// TODO Use more general all-caps character class, since this excludes special
		// 	characters that may occur in city names.
		//	This all-caps class is borrowed from utilities.js and augmented by
		//	the basic Cyrillic capital letters.
		var textnode = doc.evaluate('//div[@id="article"]//div[@class="zoomMe"]', doc, null, XPathResult.ANY_TYPE, null).iterateNext();
		if (textnode) {
			var text = textnode.textContent;
			hits = text.match(/([A-ZА-Я \u0400-\u042f]+) \((.*)\) --/);
			if (!hits) {
				hits = text.match(/([A-ZА-Я \u0400-\u042f]+) --/);
			}
			if (hits) {
				var place = Zotero.Utilities.capitalizeTitle(hits[1], true);
				item.place = place;
				// We add the wire service as an author; it would be nice to have a field for it
				item.creators.push({lastName : hits[2], creatorType:"author", fieldMode:true});
			}
		}

		item.url = url;
		item.publicationTitle = doc.evaluate('//h2[@id="header_logo_anchor" or @id="header_logo"]//span', doc, null, XPathResult.ANY_TYPE, null).iterateNext().textContent.trim();

		// Language map:
		var map = {
			"www.rferl.org" : "English",
			"www.azatliq.org" : "Tatar/Bashkir",
			"www.azattyq.org" : "Kazakh",
			"rus.azattyq.org" : "Russian",
			"da.azadiradio.org" : "Dari",
			"pa.azadiradio.org" : "Pashto",
			"www.azattyk.org" : "Kirghiz",
			"www.ozodi.org" : "Tajik",
			"www.ozodlik.org" : "Uzbek",
			"www.evropaelire.org" : "Albanian",
			"www.slobodnaevropa.org" : "Bosnian/Montenegrin/Serbian",
			"www.makdenes.org" : "Macedonian",
			"www.iraqhurr.org" : "Iraqi Arabic",
			"www.radiofarda.com" : "Farsi",
			"www.azatutyun.am" : "Armenian",
			"www.azadliq.org" : "Azerbaijani",
			"www.svaboda.org" : "Belarussian",
			"www.tavisupleba.org" : "Georgian",
			"www.azathabar.com" : "Turkmen",
			"www.svobodanews.ru" : "Russian",
			"www.svoboda.org"	 : "Russian",
			"www.europalibera.org" : "Romanian",
			"www.radiosvoboda.org" : "Ukrainian"
		}
		domain = doc.location.href.match(/https?:\/\/([^/]+)/);
		item.language = map[domain[1]];

		/* The printable version doesn't save nicely, unfortunately.
		// Make printable URL for better saving
		var printurl = url.replace(/(.*)\/.*\/(.*\.html)/,"$1/articleprintview/$2");
		item.attachments.push({url:printurl, title:"RFE/RL Snapshot", mimeType:"text/html"});
		*/
		item.attachments.push({url:url, title: (item.publicationTitle + " Snapshot"), mimeType:"text/html"});

		var listenLink = doc.evaluate('//li[@class="listenlink"]/a', doc, null, XPathResult.ANY_TYPE, null).iterateNext();
		if (listenLink) {
				Zotero.Utilities.doGet(listenLink.href, addAudio, null);
		} else item.complete();

	}
}

function addAudio(text) {
	// http://realaudio.rferl.org/TB/2011/03/29/20110329-183936-TB-clip.mp3
	var audio = text.match(/https?:\/\/(realaudio|audioarchive)\.rferl\.org[^"]*\.mp3/);
	if (audio) item.attachments.push({url:audio[0], mimeType:"application/octet-stream", title:"RFE/RL Audio"})
	item.complete();
}

/** BEGIN TEST CASES **/
var testCases = [
	{
		"type": "web",
		"url": "http://www.azatliq.org/content/article/24281041.html",
		"items": [
			{
				"itemType": "newspaperArticle",
				"creators": [
					{
						"firstName": "Гүзәл",
						"lastName": "Мәхмүтова",
						"creatorType": "author"
					}
				],
				"notes": [],
				"tags": [],
				"seeAlso": [],
				"attachments": [
					{
						"url": false,
						"title": " Азатлык Радиосы  Snapshot",
						"mimeType": "text/html"
					},
					{
						"url": false,
						"mimeType": "application/octet-stream",
						"title": "RFE/RL Audio"
					}
				],
				"title": "Татар яшьләре татарлыкны сакларга тырыша",
				"section": "татарстан",
				"date": "29.07.2011",
				"url": "http://www.azatliq.org/content/article/24281041.html",
				"publicationTitle": "Азатлык Радиосы",
				"language": "Tatar/Bashkir",
				"libraryCatalog": "Radio Free Europe / Radio Liberty"
			}
		]
	},
	{
		"type": "web",
		"url": "http://www.svoboda.org/content/news/24382010.html",
		"items": [
			{
				"itemType": "newspaperArticle",
				"creators": [],
				"notes": [],
				"tags": [],
				"seeAlso": [],
				"attachments": [
					{
						"title": "Радио Свобода Snapshot",
						"mimeType": "text/html"
					}
				],
				"title": "Партия \"Яблоко\" перевела свою предвыборную программу на 18 языков",
				"section": "Новости",
				"date": "Опубликовано 05.11.2011 06:49",
				"url": "http://www.svoboda.org/content/news/24382010.html",
				"publicationTitle": "Радио Свобода",
				"language": "Russian",
				"libraryCatalog": "Radio Free Europe / Radio Liberty",
				"accessDate": "CURRENT_TIMESTAMP"
			}
		]
	}
]
/** END TEST CASES **/