Skip to content

Commit

Permalink
DTIC and ACM small updates. ACM should now work with connectors; Othe…
Browse files Browse the repository at this point in the history 
…rwise, lots of test-update for new clean author function and other nits.
  • Loading branch information
@adam3smith
adam3smith committed Mar 10, 2012
1 parent 47f4f55 commit 1d60337
Show file tree
Hide file tree
Showing 10 changed files with 143 additions and 164 deletions.
84 changes: 32 additions & 52 deletions ACM.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"inRepository": true,
"translatorType": 4,
"browserSupport": "g",
"lastUpdated": "2012-02-21 00:15:19"
"lastUpdated": "2012-03-09 19:51:25"
}

/*
Expand All @@ -32,35 +32,25 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.


function detectWeb(doc, url) {
var nsResolver = getNsResolver(doc);


if (url.indexOf("/results.cfm") != -1) {
//Zotero.debug("Multiple items detected");
return "multiple";
} else if (url.indexOf("/citation.cfm") != -1) {
//Zotero.debug("Single item detected");
return getArticleType(doc, url, nsResolver);
return getArticleType(doc, url);

}
}



function doWeb(doc, url) {
var namespace = doc.documentElement.namespaceURI;
var nsResolver = namespace ?
function (prefix) {
if (prefix == 'x') return namespace;
else return null;
} : null;

var URIs = new Array();
var URIs = new Array();
var items = new Object();
if (detectWeb(doc, url) == "multiple") {

var xpath = '//tr/td/a[@target="_self"]';
var articles = doc.evaluate(xpath, doc, nsResolver, XPathResult.ANY_TYPE, null);
var articles = doc.evaluate(xpath, doc, null, XPathResult.ANY_TYPE, null);
var next_art = articles.iterateNext();
while (next_art) {
items[next_art.href] = next_art.textContent;
Expand All @@ -72,6 +62,8 @@ function doWeb(doc, url) {
return true;
}
for (var i in items) {
i = i.replace(/\&preflayout\=(tabs|flat)/, "") + "&preflayout=flat"
//Z.debug(i)
URIs.push(i);
}
Zotero.Utilities.processDocuments(URIs, scrape, function () {
Expand All @@ -81,17 +73,16 @@ function doWeb(doc, url) {
Zotero.wait();
});
} else {
URIs.push(url);
Zotero.Utilities.processDocuments(URIs, scrape, function () {
Zotero.done();
});
Zotero.wait();
var newURL;
newURL = url.replace(/\&preflayout\=(tabs|flat)/, "") + "&preflayout=flat"
//Z.debug(newURL);
scrape(doc, newURL);
}
}
//get abstract where possible - this fails frequently

function scrape(doc) {
var xpath = '//div/div[@style="display:inline"]|//div[@id="abstract"]//div';
var xpath = '//div/div[@style="display:inline"]';
var abs = getText(xpath, doc);

//get genric URL
Expand Down Expand Up @@ -152,8 +143,8 @@ if (incomplete.length > 0) incomplete[0].complete();
* @return a string with either "multiple", "journalArticle", "conferencePaper", or "book" in it, depending on the type of document
*/

function getArticleType(doc, url, nsResolver) {
//var toc = doc.evaluate(tocX, doc, nsResolver, XPathResult.ANY_TYPE, null).iterateNext();
function getArticleType(doc, url) {
//var toc = doc.evaluate(tocX, doc, null, XPathResult.ANY_TYPE, null).iterateNext();
if (url.indexOf("results.cfm") != -1) {
//Zotero.debug("Type: multiple");
return "multiple";
Expand All @@ -162,8 +153,8 @@ function getArticleType(doc, url, nsResolver) {
var conference = getText('//meta[@name="citation_conference"]/@content', doc);
var journal = getText('//meta[@name="citation_journal_title"]/@content', doc);
//Zotero.debug(conference);
if (conference.indexOf(" ") != -1) return "conferencePaper";
else if (journal.indexOf(" ") != -1) return "journalArticle";
if (journal.indexOf(" ") != -1) return "journalArticle";
else if (conference.indexOf(" ") != -1) return "conferencePaper";
else return "book";

}
Expand All @@ -176,8 +167,8 @@ function getArticleType(doc, url, nsResolver) {
* @return the text in the defined node or "Unable to scrape text" if the node was not found or if there was no text content
*/

function getText(pathString, doc, nsResolver) {
var path = doc.evaluate(pathString, doc, nsResolver, XPathResult.ANY_TYPE, null);
function getText(pathString, doc) {
var path = doc.evaluate(pathString, doc, null, XPathResult.ANY_TYPE, null);
var node = path.iterateNext();

if (node == null || node.textContent == undefined || node.textContent == null) {
Expand All @@ -188,21 +179,7 @@ function getText(pathString, doc, nsResolver) {
return node.textContent;
}

/**
* Get a function for returning the namespace of a given document given its prefix
* @param nsResolver the namespace resolver function
*/

function getNsResolver(doc) {
var namespace = doc.documentElement.namespaceURI;
var nsResolver = namespace ?
function (prefix) {
if (prefix == 'x') return namespace;
else return null;
} : null;

return nsResolver;
} /** BEGIN TEST CASES **/
/** BEGIN TEST CASES **/
var testCases = [
{
"type": "web",
Expand Down Expand Up @@ -253,7 +230,7 @@ var testCases = [
},
{
"type": "web",
"url": "http://dl.acm.org/citation.cfm?id=1717186&coll=DL&dl=GUIDE&CFID=48452129&CFTOKEN=58065473",
"url": "http://dl.acm.org/citation.cfm?id=1717186&coll=DL&dl=GUIDE",
"items": [
{
"itemType": "book",
Expand All @@ -265,7 +242,9 @@ var testCases = [
}
],
"notes": [],
"tags": [],
"tags": [
""
],
"seeAlso": [],
"attachments": [
{
Expand All @@ -286,7 +265,7 @@ var testCases = [
},
{
"type": "web",
"url": "http://dl.acm.org/citation.cfm?id=254650.257486&coll=DL&dl=GUIDE&CFID=48452129&CFTOKEN=58065473",
"url": "http://dl.acm.org/citation.cfm?id=254650.257486&coll=DL&dl=GUIDE",
"items": [
{
"itemType": "journalArticle",
Expand Down Expand Up @@ -342,7 +321,7 @@ var testCases = [
"url": "http://dl.acm.org/citation.cfm?id=258948.258973&coll=DL&dl=ACM",
"items": [
{
"itemType": "conferencePaper",
"itemType": "journalArticle",
"creators": [
{
"firstName": "Conal",
Expand All @@ -368,13 +347,14 @@ var testCases = [
}
],
"title": "Functional reactive animation",
"publicationTitle": "Proceedings of the second ACM SIGPLAN international conference on Functional programming",
"series": "ICFP '97",
"date": "1997",
"ISBN": "0-89791-918-1",
"publicationTitle": "SIGPLAN Not.",
"volume": "32",
"issue": "8",
"date": "August 1997",
"ISSN": "0362-1340",
"pages": "263–273",
"url": "http://doi.acm.org/10.1145/258948.258973",
"DOI": "10.1145/258948.258973",
"url": "http://doi.acm.org/10.1145/258949.258973",
"DOI": "10.1145/258949.258973",
"publisher": "ACM",
"place": "New York, NY, USA",
"libraryCatalog": "ACM Digital Library",
Expand All @@ -383,4 +363,4 @@ var testCases = [
]
}
]
/** END TEST CASES **/
/** END TEST CASES **/
5 changes: 3 additions & 2 deletions ASCE.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@
"inRepository": true,
"translatorType": 4,
"browserSupport": "gcs",
"lastUpdated": "2012-03-01 20:19:57"
"lastUpdated": "2012-03-09 23:51:18"
}

/* FW LINE 55:0c9acb56c864 */ function flatten(c){var b=new Array();for(var d in c){var e=c[d];if(e instanceof Array){b=b.concat(flatten(e))}else{b.push(e)}}return b}var FW={_scrapers:new Array()};FW._Base=function(){this.callHook=function(b,c,e,a){if(typeof this["hooks"]==="object"){var d=this["hooks"][b];if(typeof d==="function"){d(c,e,a)}}};this.evaluateThing=function(f,e,c){var b=typeof f;if(b==="object"){if(f instanceof Array){var d=this.evaluateThing;var a=f.map(function(g){return d(g,e,c)});return flatten(a)}else{return f.evaluate(e,c)}}else{if(b==="function"){return f(e,c)}else{return f}}}};FW.Scraper=function(a){FW._scrapers.push(new FW._Scraper(a))};FW._Scraper=function(a){for(x in a){this[x]=a[x]}this._singleFieldNames=["abstractNote","applicationNumber","archive","archiveLocation","artworkMedium","artworkSize","assignee","audioFileType","audioRecordingType","billNumber","blogTitle","bookTitle","callNumber","caseName","code","codeNumber","codePages","codeVolume","committee","company","conferenceName","country","court","date","dateDecided","dateEnacted","dictionaryTitle","distributor","docketNumber","documentNumber","DOI","edition","encyclopediaTitle","episodeNumber","extra","filingDate","firstPage","forumTitle","genre","history","institution","interviewMedium","ISBN","ISSN","issue","issueDate","issuingAuthority","journalAbbreviation","label","language","legalStatus","legislativeBody","letterType","libraryCatalog","manuscriptType","mapType","medium","meetingName","nameOfAct","network","number","numberOfVolumes","numPages","pages","patentNumber","place","postType","presentationType","priorityNumbers","proceedingsTitle","programTitle","programmingLanguage","publicLawNumber","publicationTitle","publisher","references","reportNumber","reportType","reporter","reporterVolume","rights","runningTime","scale","section","series","seriesNumber","seriesText","seriesTitle","session","shortTitle","studio","subject","system","thesisType","title","type","university","url","version","videoRecordingType","volume","websiteTitle","websiteType"];this._makeAttachments=function(p,b,g,t){if(g instanceof Array){g.forEach(function(k){this._makeAttachments(p,b,k,t)},this)}else{if(typeof g==="object"){var o=g.urls||g.url;var m=g.types||g.type;var f=g.titles||g.title;var q=g.snapshots||g.snapshot;var j=this.evaluateThing(o,p,b);var n=this.evaluateThing(f,p,b);var s=this.evaluateThing(m,p,b);var d=this.evaluateThing(q,p,b);if(!(j instanceof Array)){j=[j]}for(var l in j){var c=j[l];var h;var e;var r;if(s instanceof Array){h=s[l]}else{h=s}if(n instanceof Array){e=n[l]}else{e=n}if(d instanceof Array){r=d[l]}else{r=d}t.attachments.push({url:c,title:e,type:h,snapshot:r})}}}};if(this.itemTrans!==undefined){this.makeItems=this.itemTrans.makeItems}else{this.makeItems=function(o,b,m,c,l){var q=new Zotero.Item(this.itemType);q.url=b;for(var h in this._singleFieldNames){var n=this._singleFieldNames[h];if(this[n]){var g=this.evaluateThing(this[n],o,b);if(g instanceof Array){q[n]=g[0]}else{q[n]=g}}}var r=["creators","tags"];for(var f in r){var p=r[f];var d=this.evaluateThing(this[p],o,b);if(d){for(var e in d){q[p].push(d[e])}}}this._makeAttachments(o,b,this["attachments"],q);c(q,this,o,b);l([q])}}};FW._Scraper.prototype=new FW._Base;FW.MultiScraper=function(a){FW._scrapers.push(new FW._MultiScraper(a))};FW._MultiScraper=function(a){for(x in a){this[x]=a[x]}this._mkSelectItems=function(e,d){var b=new Object;for(var c in e){b[d[c]]=e[c]}return b};this._selectItems=function(d,c,e){var b=new Array();Zotero.selectItems(this._mkSelectItems(d,c),function(f){for(var g in f){b.push(g)}e(b)})};this._mkAttachments=function(g,d,f){var b=this.evaluateThing(this["attachments"],g,d);var c=new Object();if(b){for(var e in f){c[f[e]]=b[e]}}return c};this._makeChoices=function(f,p,c,d,h){if(f instanceof Array){f.forEach(function(k){this._makeTitlesUrls(k,p,c,d,h)},this)}else{if(typeof f==="object"){var m=f.urls||f.url;var e=f.titles||f.title;var n=this.evaluateThing(m,p,c);var j=this.evaluateThing(e,p,c);var l=(j instanceof Array);if(!(n instanceof Array)){n=[n]}for(var g in n){var b=n[g];var o;if(l){o=j[g]}else{o=j}h.push(b);d.push(o)}}}};this.makeItems=function(j,b,g,c,f){if(this.beforeFilter){var k=this.beforeFilter(j,b);if(k!=b){this.makeItems(j,k,g,c,f);return}}var e=[];var h=[];this._makeChoices(this["choices"],j,b,e,h);var d=this._mkAttachments(j,b,h);this._selectItems(e,h,function(m){if(!m){f([])}else{var l=[];var n=this.itemTrans;Zotero.Utilities.processDocuments(m,function(q){var p=q.documentURI;var o=n;if(o===undefined){o=FW.getScraper(q,p)}if(o===undefined){}else{o.makeItems(q,p,d[p],function(r){l.push(r);c(r,o,q,p)},function(){})}},function(){f(l)})}})}};FW._MultiScraper.prototype=new FW._Base;FW.DelegateTranslator=function(a){return new FW._DelegateTranslator(a)};FW._DelegateTranslator=function(a){for(x in a){this[x]=a[x]}this._translator=Zotero.loadTranslator(this.translatorType);this._translator.setTranslator(this.translatorId);this.makeItems=function(g,d,b,f,c){var e;Zotero.Utilities.HTTP.doGet(d,function(h){this._translator.setHandler("itemDone",function(k,j){e=j;if(b){j.attachments=b}});if(this.preProcess){h=this.preProcess(h)}this._translator.setString(h);this._translator.translate();f(e)},function(){c([e])})}};FW.DelegateTranslator.prototype=new FW._Scraper;FW._StringMagic=function(){this._filters=new Array();this.addFilter=function(a){this._filters.push(a);return this};this.split=function(a){return this.addFilter(function(b){return b.split(a).filter(function(c){return(c!="")})})};this.replace=function(c,b,a){return this.addFilter(function(d){if(d.match(c)){return d.replace(c,b,a)}else{return d}})};this.prepend=function(a){return this.replace(/^/,a)};this.append=function(a){return this.replace(/$/,a)};this.remove=function(b,a){return this.replace(b,"",a)};this.trim=function(){return this.addFilter(function(a){return Zotero.Utilities.trim(a)})};this.trimInternal=function(){return this.addFilter(function(a){return Zotero.Utilities.trimInternal(a)})};this.match=function(a,b){if(!b){b=0}return this.addFilter(function(d){var c=d.match(a);if(c===undefined||c===null){return undefined}else{return c[b]}})};this.cleanAuthor=function(b,a){return this.addFilter(function(c){return Zotero.Utilities.cleanAuthor(c,b,a)})};this.key=function(a){return this.addFilter(function(b){return b[a]})};this.capitalizeTitle=function(){if(arguments.length>0&&arguments[0]==true){return this.addFilter(function(a){return Zotero.Utilities.capitalizeTitle(a,true)})}else{return this.addFilter(function(a){return Zotero.Utilities.capitalizeTitle(a)})}};this.unescapeHTML=function(){return this.addFilter(function(a){return Zotero.Utilities.unescapeHTML(a)})};this.unescape=function(){return this.addFilter(function(a){return unescape(a)})};this._applyFilters=function(c,e){for(i in this._filters){c=flatten(c);c=c.filter(function(a){return((a!==undefined)&&(a!==null))});for(var d=0;d<c.length;d++){try{if((c[d]===undefined)||(c[d]===null)){continue}else{c[d]=this._filters[i](c[d],e)}}catch(b){c[d]=undefined;Zotero.debug("Caught exception "+b+"on filter: "+this._filters[i])}}c=c.filter(function(a){return((a!==undefined)&&(a!==null))})}return c}};FW.PageText=function(){return new FW._PageText()};FW._PageText=function(){this._filters=new Array();this.evaluate=function(c){var b=[c.documentElement.innerHTML];b=this._applyFilters(b,c);if(b.length==0){return false}else{return b}}};FW._PageText.prototype=new FW._StringMagic();FW.Url=function(){return new FW._Url()};FW._Url=function(){this._filters=new Array();this.evaluate=function(d,c){var b=[c];b=this._applyFilters(b,d);if(b.length==0){return false}else{return b}}};FW._Url.prototype=new FW._StringMagic();FW.Xpath=function(a){return new FW._Xpath(a)};FW._Xpath=function(a){this._xpath=a;this._filters=new Array();this.text=function(){var b=function(c){if(typeof c==="object"&&c.textContent){return c.textContent}else{return c}};this.addFilter(b);return this};this.sub=function(b){var c=function(f,e){var d=e.evaluate(b,f,null,XPathResult.ANY_TYPE,null);if(d){return d.iterateNext()}else{return undefined}};this.addFilter(c);return this};this.evaluate=function(f){var e=f.evaluate(this._xpath,f,null,XPathResult.ANY_TYPE,null);var d=e.resultType;var c=new Array();if(d==XPathResult.STRING_TYPE){c.push(e.stringValue)}else{if(d==XPathResult.ORDERED_NODE_ITERATOR_TYPE||d==XPathResult.UNORDERED_NODE_ITERATOR_TYPE){var b;while((b=e.iterateNext())){c.push(b)}}}c=this._applyFilters(c,f);if(c.length==0){return false}else{return c}}};FW._Xpath.prototype=new FW._StringMagic();FW.detectWeb=function(e,b){for(var c in FW._scrapers){var d=FW._scrapers[c];var f=d.evaluateThing(d.itemType,e,b);var a=d.evaluateThing(d.detect,e,b);if(a.length>0&&a[0]){return f}}return undefined};FW.getScraper=function(b,a){var c=FW.detectWeb(b,a);return FW._scrapers.filter(function(d){return(d.evaluateThing(d.itemType,b,a)==c)&&(d.evaluateThing(d.detect,b,a))})[0]};FW.doWeb=function(c,a){var b=FW.getScraper(c,a);b.makeItems(c,a,[],function(f,e,g,d){e.callHook("scraperDone",f,g,d);if(!f.title){f.title=""}f.complete()},function(){Zotero.done()});Zotero.wait()};



/*
ASCE Translator
Copyright (C) 2011 Sebastian Karcher
Expand Down Expand Up @@ -99,7 +100,7 @@ var testCases = [
"creatorType": "author"
},
{
"firstName": "Upendra P",
"firstName": "Upendra P.",
"lastName": "Poudel",
"creatorType": "author"
}
Expand Down
15 changes: 5 additions & 10 deletions AfroEuropa.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
"minVersion": "1.0.0b4.r5",
"maxVersion": "",
"priority": 100,
"browserSupport": "gcs",
"inRepository": true,
"translatorType": 4,
"lastUpdated": "2012-01-01 01:42:16"
"browserSupport": "gcs",
"lastUpdated": "2012-03-09 19:53:34"
}

function detectWeb(doc, url) {
Expand All @@ -27,11 +27,6 @@ function makeExport(site, str) {
}

function doWeb(doc, url) {
var n = doc.documentElement.namespaceURI;
var ns = n ? function(prefix) {
if (prefix == 'x') return n; else return null;
} : null;

var site = url.match(/^http:\/\/([^/]*\/)+index\.php\/[^/]*\//)[0];
var arts = new Array();
if (detectWeb(doc, url) == "multiple") {
Expand All @@ -44,11 +39,11 @@ function doWeb(doc, url) {
var linkx = './td[2]/a[1]';
}
var items = new Object();
var results = doc.evaluate(xpath, doc, ns, XPathResult.ANY_TYPE, null);
var results = doc.evaluate(xpath, doc, null, XPathResult.ANY_TYPE, null);
var result;
while (result = results.iterateNext()) {
var title = Zotero.Utilities.trimInternal(doc.evaluate(titlex, result, ns, XPathResult.ANY_TYPE, null).iterateNext().textContent);
var link = doc.evaluate(linkx, result, ns, XPathResult.ANY_TYPE, null).iterateNext().href;
var title = Zotero.Utilities.trimInternal(doc.evaluate(titlex, result, null, XPathResult.ANY_TYPE, null).iterateNext().textContent);
var link = doc.evaluate(linkx, result, null, XPathResult.ANY_TYPE, null).iterateNext().href;
items[makeExport(site, link)] = title;
}
items = Zotero.selectItems(items);
Expand Down
39 changes: 24 additions & 15 deletions DTIC.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"inRepository": true,
"translatorType": 4,
"browserSupport": "g",
"lastUpdated": "2011-10-28 11:30:27"
"lastUpdated": "2012-03-09 23:59:56"
}

function detectWeb(doc, url) {
Expand All @@ -36,18 +36,28 @@ function doWeb(doc, url) {
while( link = links.iterateNext(), title = titles.iterateNext()){
items[link.textContent.replace(/&metadataPrefix=html/, "&metadataPrefix=oai_dc")] = title.textContent;
}
items = Zotero.selectItems(items);
if(!items) {
return true;
}
for (url in items) {
newURIs.push(url);
Zotero.debug(url);
}

Zotero.selectItems(items, function (items) {
if (!items) {
return true;
}
for (var url in items) {
newURIs.push(url);
}
Zotero.Utilities.processDocuments(newURIs, scrape, function () {
Zotero.done();
});
Zotero.wait();
});


} else {
newURIs.push(url.replace(/&metadataPrefix=html/, "&metadataPrefix=oai_dc"))
newURIs = url.replace(/&metadataPrefix=html/, "&metadataPrefix=oai_dc");
scrape(doc, newURIs);
}
}

function scrape(doc, newURIs){
// ripped the arXiv.org code, hence the funny var names.
Zotero.Utilities.HTTP.doGet(newURIs, function(text) {
var newItem = new Zotero.Item("report");
Expand All @@ -68,7 +78,7 @@ function doWeb(doc, url) {
newItem.title = title;
}
Zotero.debug("article title: " + title);
var type = "";
var type = "author";
if(citation.dc_creator.length()) {
var authors = citation.dc_creator;
for(var j=0; j<authors.length(); j++) {
Expand Down Expand Up @@ -122,9 +132,8 @@ function doWeb(doc, url) {
newItem.notes = new Array();
}
newItem.complete();
}, function() {Zotero.done();}, null);
Zotero.wait();
}/** BEGIN TEST CASES **/
})
}/** BEGIN TEST CASES **/
var testCases = [
{
"type": "web",
Expand All @@ -134,7 +143,7 @@ var testCases = [
"itemType": "report",
"creators": [
{
"firstName": "Dennis M",
"firstName": "Dennis M.",
"lastName": "DeCoste",
"creatorType": ""
}
Expand Down
6 changes: 3 additions & 3 deletions DeGruyter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"inRepository": true,
"translatorType": 4,
"browserSupport": "gcs",
"lastUpdated": "2012-03-08 07:30:46"
"lastUpdated": "2012-03-10 00:00:32"
}

/*
Expand Down Expand Up @@ -92,7 +92,7 @@ var testCases = [
"itemType": "journalArticle",
"creators": [
{
"firstName": "James E",
"firstName": "James E.",
"lastName": "Campbell",
"creatorType": "author"
}
Expand Down Expand Up @@ -176,4 +176,4 @@ var testCases = [
"items": "multiple"
}
]
/** END TEST CASES **/
/** END TEST CASES **/
Loading

0 comments on commit 1d60337

Please sign in to comment.