Skip to content

Commit

Permalink
Rewrite of ASCE using framework, by Sebastian
Browse files Browse the repository at this point in the history
  • Loading branch information
@avram
avram committed Oct 13, 2011
1 parent f1a8dc5 commit d449668
Showing 1 changed file with 132 additions and 68 deletions.
200 changes: 132 additions & 68 deletions ASCE.js
View file
Original file line number Diff line number Diff line change
@@ -1,76 +1,140 @@
{
"translatorID":"303bdfc5-11b8-4107-bca1-63ca97701a0f",
"translatorType":4,
"label":"ASCE",
"creator":"Michael Berkowitz",
"target":"^http://ascelibrary.aip.org/.+",
"minVersion":"1.0.0b3.r1",
"maxVersion":"",
"priority":100,
"inRepository":true,
"lastUpdated":"2007-09-06 19:30:00"
"translatorID": "303bdfc5-11b8-4107-bca1-63ca97701a0f",
"label": "ASCE",
"creator": "Sebastian Karcher",
"target": "^https?://(www\\.)?ascelibrary\\.org/",
"minVersion": "2.1.9",
"maxVersion": "",
"priority": 100,
"inRepository": true,
"translatorType": 4,
"browserSupport": "gcs",
"lastUpdated": "2011-10-12 16:11:43"
}

function detectWeb(doc, url) {
if (doc.evaluate('//div[@id="sr-content-wrap"]//div[@class="sr-right"]/p[@class="sr-art-title"]/a', doc, null, XPathResult.ANY_TYPE, null).iterateNext()) {
return "multiple";
} else {
return "journalArticle";
}
}
/* FW LINE 51:6e1628381879 */ function flatten(c){var b=new Array();for(var d in c){var e=c[d];if(e instanceof Array){b=b.concat(flatten(e))}else{b.push(e)}}return b}var FW={_scrapers:new Array()};FW._Base=function(){this.callHook=function(b,c,e,a){if(typeof this["hooks"]==="object"){var d=this["hooks"][b];if(typeof d==="function"){d(c,e,a)}}};this.evaluateThing=function(f,e,c){var b=typeof f;if(b==="string"){return f}else{if(b==="object"){if(f instanceof Array){var d=this.evaluateThing;var a=f.map(function(g){return d(g,e,c)});return flatten(a)}else{return f.evaluate(e,c)}}else{if(b==="function"){return f(e,c)}else{return undefined}}}}};FW.Scraper=function(a){FW._scrapers.push(new FW._Scraper(a))};FW._Scraper=function(a){for(x in a){this[x]=a[x]}this._singleFieldNames=["abstractNote","applicationNumber","archive","archiveLocation","artworkMedium","artworkSize","assignee","audioFileType","audioRecordingType","billNumber","blogTitle","bookTitle","callNumber","caseName","code","codeNumber","codePages","codeVolume","committee","company","conferenceName","country","court","date","dateDecided","dateEnacted","dictionaryTitle","distributor","docketNumber","documentNumber","DOI","edition","encyclopediaTitle","episodeNumber","extra","filingDate","firstPage","forumTitle","genre","history","institution","interviewMedium","ISBN","ISSN","issue","issueDate","issuingAuthority","journalAbbreviation","label","language","legalStatus","legislativeBody","letterType","libraryCatalog","manuscriptType","mapType","medium","meetingName","nameOfAct","network","number","numberOfVolumes","numPages","pages","patentNumber","place","postType","presentationType","priorityNumbers","proceedingsTitle","programTitle","programmingLanguage","publicLawNumber","publicationTitle","publisher","references","reportNumber","reportType","reporter","reporterVolume","rights","runningTime","scale","section","series","seriesNumber","seriesText","seriesTitle","session","shortTitle","studio","subject","system","thesisType","title","type","university","url","version","videoRecordingType","volume","websiteTitle","websiteType"];this._makeAttachments=function(q,b,f,s){if(f instanceof Array){f.forEach(function(k){this._makeAttachments(q,b,k,s)},this)}else{if(typeof f==="object"){var p=f.urls||f.url;var m=f.types||f.type;var e=f.titles||f.title;var h=this.evaluateThing(p,q,b);var o=this.evaluateThing(e,q,b);var r=this.evaluateThing(m,q,b);var l=(r instanceof Array);var n=(o instanceof Array);if(!(h instanceof Array)){h=[h]}for(var j in h){var c=h[j];var g;var d;if(l){g=r[j]}else{g=r}if(n){d=o[j]}else{d=o}s.attachments.push({url:c,title:d,type:g})}}}};if(this.itemTrans!==undefined){this.makeItems=this.itemTrans.makeItems}else{this.makeItems=function(o,b,m,c,l){var q=new Zotero.Item(this.itemType);q.url=b;for(var h in this._singleFieldNames){var n=this._singleFieldNames[h];if(this[n]){var g=this.evaluateThing(this[n],o,b);if(g instanceof Array){q[n]=g[0]}else{q[n]=g}}}var r=["creators","tags"];for(var f in r){var p=r[f];var d=this.evaluateThing(this[p],o,b);if(d){for(var e in d){q[p].push(d[e])}}}this._makeAttachments(o,b,this["attachments"],q);c(q,this,o,b);l([q])}}};FW._Scraper.prototype=new FW._Base;FW.MultiScraper=function(a){FW._scrapers.push(new FW._MultiScraper(a))};FW._MultiScraper=function(a){for(x in a){this[x]=a[x]}this._mkSelectItems=function(e,d){var b=new Object;for(var c in e){b[d[c]]=e[c]}return b};this._selectItems=function(d,c,e){var b=new Array();Zotero.selectItems(this._mkSelectItems(d,c),function(f){for(var g in f){b.push(g)}e(b)})};this._mkAttachments=function(g,d,f){var b=this.evaluateThing(this["attachments"],g,d);var c=new Object();if(b){for(var e in f){c[f[e]]=b[e]}}return c};this._makeChoices=function(f,p,c,d,h){if(f instanceof Array){f.forEach(function(k){this._makeTitlesUrls(k,p,c,d,h)},this)}else{if(typeof f==="object"){var m=f.urls||f.url;var e=f.titles||f.title;var n=this.evaluateThing(m,p,c);var j=this.evaluateThing(e,p,c);var l=(j instanceof Array);if(!(n instanceof Array)){n=[n]}for(var g in n){var b=n[g];var o;if(l){o=j[g]}else{o=j}h.push(b);d.push(o)}}}};this.makeItems=function(j,b,g,c,f){if(this.beforeFilter){var k=this.beforeFilter(j,b);if(k!=b){this.makeItems(j,k,g,c,f);return}}var e=[];var h=[];this._makeChoices(this["choices"],j,b,e,h);var d=this._mkAttachments(j,b,h);this._selectItems(e,h,function(m){if(!m){f([])}else{var l=[];var n=this.itemTrans;Zotero.Utilities.processDocuments(m,function(q){var p=q.documentURI;var o=n;if(o===undefined){o=FW.getScraper(q,p)}if(o===undefined){}else{o.makeItems(q,p,d[p],function(r){l.push(r);c(r,o,q,p)},function(){})}},function(){f(l)})}})}};FW._MultiScraper.prototype=new FW._Base;FW.DelegateTranslator=function(a){return new FW._DelegateTranslator(a)};FW._DelegateTranslator=function(a){for(x in a){this[x]=a[x]}this._translator=Zotero.loadTranslator(this.translatorType);this._translator.setTranslator(this.translatorId);this.makeItems=function(g,d,b,f,c){var e;Zotero.Utilities.HTTP.doGet(d,function(h){this._translator.setHandler("itemDone",function(k,j){e=j;if(b){j.attachments=b}});if(this.preProcess){h=this.preProcess(h)}this._translator.setString(h);this._translator.translate();f(e)},function(){c([e])})}};FW.DelegateTranslator.prototype=new FW._Scraper;FW._StringMagic=function(){this._filters=new Array();this.addFilter=function(a){this._filters.push(a);return this};this.split=function(a){return this.addFilter(function(b){return b.split(a).filter(function(c){return(c!="")})})};this.replace=function(c,b,a){return this.addFilter(function(d){if(d.match(c)){return d.replace(c,b,a)}else{return d}})};this.prepend=function(a){return this.replace(/^/,a)};this.append=function(a){return this.replace(/$/,a)};this.remove=function(b,a){return this.replace(b,"",a)};this.trim=function(){return this.addFilter(function(a){return Zotero.Utilities.trim(a)})};this.trimInternal=function(){return this.addFilter(function(a){return Zotero.Utilities.trimInternal(a)})};this.match=function(a,b){if(!b){b=0}return this.addFilter(function(d){var c=d.match(a);if(c===undefined||c===null){return undefined}else{return c[b]}})};this.cleanAuthor=function(b,a){return this.addFilter(function(c){return Zotero.Utilities.cleanAuthor(c,b,a)})};this.key=function(a){return this.addFilter(function(b){return b[a]})};this.capitalizeTitle=function(){return this.addFilter(function(a){return Zotero.Utilities.capitalizeTitle(a)})};this.unescapeHTML=function(){return this.addFilter(function(a){return Zotero.Utilities.unescapeHTML(a)})};this.unescape=function(){return this.addFilter(function(a){return unescape(a)})};this._applyFilters=function(c,e){for(i in this._filters){c=flatten(c);c=c.filter(function(a){return((a!==undefined)&&(a!==null))});for(var d=0;d<c.length;d++){try{if((c[d]===undefined)||(c[d]===null)){continue}else{c[d]=this._filters[i](c[d],e)}}catch(b){c[d]=undefined;Zotero.debug("Caught exception "+b+"on filter: "+this._filters[i])}}c=c.filter(function(a){return((a!==undefined)&&(a!==null))})}return c}};FW.PageText=function(){return new FW._PageText()};FW._PageText=function(){this._filters=new Array();this.evaluate=function(c){var b=[c.documentElement.innerHTML];b=this._applyFilters(b,c);if(b.length==0){return false}else{return b}}};FW._PageText.prototype=new FW._StringMagic();FW.Url=function(){return new FW._Url()};FW._Url=function(){this._filters=new Array();this.evaluate=function(d,c){var b=[c];b=this._applyFilters(b,d);if(b.length==0){return false}else{return b}}};FW._Url.prototype=new FW._StringMagic();FW.Xpath=function(a){return new FW._Xpath(a)};FW._Xpath=function(a){this._xpath=a;this._filters=new Array();this.text=function(){var b=function(c){if(typeof c==="object"&&c.textContent){return c.textContent}else{return c}};this.addFilter(b);return this};this.sub=function(b){var c=function(f,e){var d=e.evaluate(b,f,null,XPathResult.ANY_TYPE,null);if(d){return d.iterateNext()}else{return undefined}};this.addFilter(c);return this};this.evaluate=function(f){var e=f.evaluate(this._xpath,f,null,XPathResult.ANY_TYPE,null);var d=e.resultType;var c=new Array();if(d==XPathResult.STRING_TYPE){c.push(e.stringValue)}else{if(d==XPathResult.ORDERED_NODE_ITERATOR_TYPE||d==XPathResult.UNORDERED_NODE_ITERATOR_TYPE){var b;while((b=e.iterateNext())){c.push(b)}}}c=this._applyFilters(c,f);if(c.length==0){return false}else{return c}}};FW._Xpath.prototype=new FW._StringMagic();FW.detectWeb=function(e,b){for(var c in FW._scrapers){var d=FW._scrapers[c];var f=d.evaluateThing(d.itemType,e,b);var a=d.evaluateThing(d.detect,e,b);if(a.length>0&&a[0]){return f}}return undefined};FW.getScraper=function(b,a){var c=FW.detectWeb(b,a);return FW._scrapers.filter(function(d){return(d.evaluateThing(d.itemType,b,a)==c)&&(d.evaluateThing(d.detect,b,a))})[0]};FW.doWeb=function(c,a){var b=FW.getScraper(c,a);b.makeItems(c,a,[],function(f,e,g,d){e.callHook("scraperDone",f,g,d);if(!f.title){f.title=""}f.complete()},function(){Zotero.done()});Zotero.wait()};
/*
ASCE Translator
Copyright (C) 2011 Sebastian Karcher
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

function getRIS(doc, url) {
var newx = '//div[@id="sci-art-options-box"]//input[@name="SelectCheck"]';
var key = doc.evaluate(newx, doc, null, XPathResult.ANY_TYPE, null).iterateNext().value;
Zotero.debug(key);
var citation = 'http://ascelibrary.aip.org/getabs/servlet/GetCitation?source=scitation&PrefType=ARTICLE&PrefAction=Add+Selected&SelectCheck=' + key + '&fn=open_refworks&downloadcitation=+Go+';
Zotero.Utilities.HTTP.doGet(citation, function(text) {
var translator = Zotero.loadTranslator("import");
text = text.replace(/RT/, "TY");
text = text.replace(/VO/, "VL");
text = text.replace(/LK/, "UR");
text = text.replace(/YR/, "PY");
Zotero.debug(text);
translator.setTranslator("32d59d2d-b65a-4da4-b0a3-bdd3cfb979e7");
translator.setString(text.replace(/([A-Z][A-Z\d]\s)/g, "$1 - "));
translator.setHandler("itemDone", function(obj, item) {
item.attachments = [
{url:item.url, title:"ASCE Snapshot", mimeType:"text/html"},
{url:"http://ascelibrary.aip.org/getpdf/servlet/GetPDFServlet?filetype=pdf&id=" + key + "&idtype=cvips&prog=search", title:"EAS Full Text PDF", mimeType:"application/pdf"}
];
//item.itemType = "journalArticle";
item.complete();
});
translator.translate();
Zotero.wait();
Zotero.done();
});
}

function doWeb(doc, url) {
var articles = new Array();
var items = new Object();
var xpath = '//div[@class="sr-right"]/p[@class="sr-art-title"]/a';
if (doc.evaluate(xpath, doc, null, XPathResult.ANY_TYPE, null).iterateNext()) {
var titles = doc.evaluate(xpath, doc, null, XPathResult.ANY_TYPE, null);
while (new_title = titles.iterateNext()) {
items[new_title.href] = new_title.textContent;
}

items = Zotero.selectItems(items);

for (var i in items) {
articles.push(i)
}
} else {
var newx = '//div[@id="sci-art-options-box"]//input[@name="SelectCheck"]';
var stuff = doc.evaluate(newx, doc, null, XPathResult.ANY_TYPE, null).iterateNext().value;
Zotero.debug(stuff);
articles.push(url);
}

Zotero.debug(articles);
Zotero.Utilities.processDocuments(articles, getRIS, function() {Zotero.done});
Zotero.wait();


function detectWeb(doc, url) { return FW.detectWeb(doc, url); }
function doWeb(doc, url) { return FW.doWeb(doc, url); }

/** Articles */
FW.Scraper({
itemType : 'journalArticle',
detect : FW.Xpath('//meta[@name="citation_journal_title"]'),
title : FW.Xpath('//meta[@name="citation_title"]').key("content").text().trim(),

attachments : [{ url: FW.Xpath('//meta[@name="citation_pdf_url"]').key("content").text().trim(),
title: "ASCE Journal PDF",
type: "application/pdf" }],

creators : FW.Xpath('//meta[@name="citation_author"]').key("content").text().cleanAuthor("author", useComma="true"),
date : FW.Xpath('//meta[@name="citation_date"]').key("content").text(),
issue : FW.Xpath('//meta[@name="citation_issue"]').key("content").text(),
volume : FW.Xpath('//meta[@name="citation_volume"]').key("content").text(),
pages : FW.Xpath('concat(//meta[@name="citation_firstpage"]/@content, "-", //meta[@name="citation_lastpage"]/@content)').remove(/-$|^-/),
ISSN : FW.Xpath('//meta[@name="citation_issn"]').key("content").text(),
abstractNote: FW.Xpath('//meta[@name="description"]').key("content").text(),
journalAbbreviation : FW.Xpath('//meta[@name="citation_journal_abbrev"]').key("content").text(),
DOI : FW.Xpath('//meta[@name="citation_doi"]').key("content").text().remove(/^doi\:/),
language : FW.Xpath('//meta[@name="citation_language"]').key("content").text(),
tags : FW.Xpath('//meta[@name="citation_keywords"]').key("content").text().split(/;/),
publicationTitle : FW.Xpath('//meta[@name="citation_journal_title"]').key("content").text()
});


FW.MultiScraper({
itemType : 'multiple',
detect : FW.Xpath('//p[@class="dbttitle"]/a'),
choices : {
titles : FW.Xpath('//p[@class="dbttitle"]/a').text().trim(),
urls : FW.Xpath('//p[@class="dbttitle"]/a').key("href")
}
});

/** BEGIN TEST CASES **/
var testCases = [
{
"type": "web",
"url": "http://ascelibrary.org/search?key=ASCE_ALL&societykey=ASCE&coden=JENMDT&q=test&displayid=ASCE&sortby=newestdate&faceted=faceted&sortby=newestdate&CP_Style=true&alias=%2Femo&searchzone=2",
"items": "multiple"
},
{
"type": "web",
"url": "http://ascelibrary.org/emo/resource/1/jenmdt/v137/i7/p475_s1",
"items": [
{
"itemType": "journalArticle",
"creators": [
{
"firstName": "Jian",
"lastName": "Ye",
"creatorType": "author"
},
{
"firstName": "Gongkang",
"lastName": "Fu",
"creatorType": "author"
},
{
"firstName": "Upendra P",
"lastName": "Poudel",
"creatorType": "author"
}
],
"notes": [],
"tags": [
[
"Photogrammetry",
"Digital techniques",
"Deformation",
"Measurement",
"Signal processing"
]
],
"seeAlso": [],
"attachments": [
{
"url": "http://link.aip.org/link/?JENMDT/137/475/1/pdf",
"title": "ASCE Journal PDF",
"type": "application/pdf"
}
],
"url": "http://ascelibrary.org/emo/resource/1/jenmdt/v137/i7/p475_s1",
"abstractNote": "This paper proposes a close-range digital photogrammetric system based on edge detection for structural deformation measurement. Different from traditional photogrammetric applications using discrete points, continuous edges in digital images are used here as the controlling feature in this new system. This makes it possible to acquire spatially intensive information. The system uses several digital images of the structure, taken from a number of different stations before and after the deformation is induced. Then, an image-matching algorithm based on the coplanarity condition developed here is applied to establish spatial relationships of the interested edges, identified by using a new high-precision method. These relationships are subsequently employed to acquire full-field deformation measurement of the structure. A series of experiments was conducted in the laboratory to investigate the capability of the new photogrammetric system. Results show that the system is highly accurate and suitable for structural deformation measurement. It offers noncontact, full-field, and spatially intensive measurement, in contrast with conventional contact and point-measurement.",
"date": "2011-01-20",
"DOI": "10.1061/(ASCE)EM.1943-7889.0000251",
"ISSN": "07339399",
"issue": "7",
"language": "en",
"pages": "475",
"publicationTitle": "Journal of Engineering Mechanics",
"title": "Edge-Based Close-Range Digital Photogrammetry for Structural Deformation Measurement",
"volume": "137",
"libraryCatalog": "ASCE",
"accessDate": "CURRENT_TIMESTAMP"
}
]
}
]
/** END TEST CASES **/

0 comments on commit d449668

Please sign in to comment.