diff --git a/Boston Review.js b/Boston Review.js index eed191b738b..2013140cadb 100644 --- a/Boston Review.js +++ b/Boston Review.js @@ -3,112 +3,184 @@ "label": "Boston Review", "creator": "Sebastian Karcher", "target": "^https?://(www\\.)?bostonreview\\.net/", - "minVersion": "2.1.9", + "minVersion": "6.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2015-08-09 19:58:48" + "lastUpdated": "2023-08-16 01:46:35" } -/* FW LINE 59:b820c6d */ function flatten(t){var e=new Array;for(var i in t){var r=t[i];r instanceof Array?e=e.concat(flatten(r)):e.push(r)}return e}var FW={_scrapers:new Array};FW._Base=function(){this.callHook=function(t,e,i,r){if("object"==typeof this.hooks){var n=this.hooks[t];"function"==typeof n&&n(e,i,r)}},this.evaluateThing=function(t,e,i){var r=typeof t;if("object"===r){if(t instanceof Array){var n=this.evaluateThing,a=t.map(function(t){return n(t,e,i)});return flatten(a)}return t.evaluate(e,i)}return"function"===r?t(e,i):t},this.makeItems=function(t,e,i,r,n){n()}},FW.Scraper=function(t){FW._scrapers.push(new FW._Scraper(t))},FW._Scraper=function(t){for(x in t)this[x]=t[x];this._singleFieldNames=["abstractNote","applicationNumber","archive","archiveLocation","artworkMedium","artworkSize","assignee","audioFileType","audioRecordingType","billNumber","blogTitle","bookTitle","callNumber","caseName","code","codeNumber","codePages","codeVolume","committee","company","conferenceName","country","court","date","dateDecided","dateEnacted","dictionaryTitle","distributor","docketNumber","documentNumber","DOI","edition","encyclopediaTitle","episodeNumber","extra","filingDate","firstPage","forumTitle","genre","history","institution","interviewMedium","ISBN","ISSN","issue","issueDate","issuingAuthority","journalAbbreviation","label","language","legalStatus","legislativeBody","letterType","libraryCatalog","manuscriptType","mapType","medium","meetingName","nameOfAct","network","number","numberOfVolumes","numPages","pages","patentNumber","place","postType","presentationType","priorityNumbers","proceedingsTitle","programTitle","programmingLanguage","publicLawNumber","publicationTitle","publisher","references","reportNumber","reportType","reporter","reporterVolume","rights","runningTime","scale","section","series","seriesNumber","seriesText","seriesTitle","session","shortTitle","studio","subject","system","thesisType","title","type","university","url","version","videoRecordingType","volume","websiteTitle","websiteType"],this._makeAttachments=function(t,e,i,r){if(i instanceof Array)i.forEach(function(i){this._makeAttachments(t,e,i,r)},this);else if("object"==typeof i){var n=i.urls||i.url,a=i.types||i.type,s=i.titles||i.title,o=i.snapshots||i.snapshot,u=this.evaluateThing(n,t,e),l=this.evaluateThing(s,t,e),c=this.evaluateThing(a,t,e),h=this.evaluateThing(o,t,e);u instanceof Array||(u=[u]);for(var f in u){var p,m,v,d=u[f];p=c instanceof Array?c[f]:c,m=l instanceof Array?l[f]:l,v=h instanceof Array?h[f]:h,r.attachments.push({url:d,title:m,mimeType:p,snapshot:v})}}},this.makeItems=function(t,e,i,r,n){var a=new Zotero.Item(this.itemType);a.url=e;for(var s in this._singleFieldNames){var o=this._singleFieldNames[s];if(this[o]){var u=this.evaluateThing(this[o],t,e);u instanceof Array?a[o]=u[0]:a[o]=u}}var l=["creators","tags"];for(var c in l){var h=l[c],f=this.evaluateThing(this[h],t,e);if(f)for(var p in f)a[h].push(f[p])}this._makeAttachments(t,e,this.attachments,a),r(a,this,t,e),n()}},FW._Scraper.prototype=new FW._Base,FW.MultiScraper=function(t){FW._scrapers.push(new FW._MultiScraper(t))},FW._MultiScraper=function(t){for(x in t)this[x]=t[x];this._mkSelectItems=function(t,e){var i=new Object;for(var r in t)i[e[r]]=t[r];return i},this._selectItems=function(t,e,i){var r=new Array;Zotero.selectItems(this._mkSelectItems(t,e),function(t){for(var e in t)r.push(e);i(r)})},this._mkAttachments=function(t,e,i){var r=this.evaluateThing(this.attachments,t,e),n=new Object;if(r)for(var a in i)n[i[a]]=r[a];return n},this._makeChoices=function(t,e,i,r,n){if(t instanceof Array)t.forEach(function(t){this._makeTitlesUrls(t,e,i,r,n)},this);else if("object"==typeof t){var a=t.urls||t.url,s=t.titles||t.title,o=this.evaluateThing(a,e,i),u=this.evaluateThing(s,e,i),l=u instanceof Array;o instanceof Array||(o=[o]);for(var c in o){var h,f=o[c];h=l?u[c]:u,n.push(f),r.push(h)}}},this.makeItems=function(t,e,i,r,n){if(this.beforeFilter){var a=this.beforeFilter(t,e);if(a!=e)return void this.makeItems(t,a,i,r,n)}var s=[],o=[];this._makeChoices(this.choices,t,e,s,o);var u=this._mkAttachments(t,e,o),l=this.itemTrans;this._selectItems(s,o,function(t){if(t){var e=function(t){var e=t.documentURI,i=l;void 0===i&&(i=FW.getScraper(t,e)),void 0===i||i.makeItems(t,e,u[e],r,function(){})};Zotero.Utilities.processDocuments(t,e,n)}else n()})}},FW._MultiScraper.prototype=new FW._Base,FW.WebDelegateTranslator=function(t){return new FW._WebDelegateTranslator(t)},FW._WebDelegateTranslator=function(t){for(x in t)this[x]=t[x];this.makeItems=function(t,e,i,r,n){var a=this,s=Zotero.loadTranslator("web");s.setHandler("itemDone",function(i,n){r(n,a,t,e)}),s.setDocument(t),this.translatorId?(s.setTranslator(this.translatorId),s.translate()):(s.setHandler("translators",function(t,e){e.length&&(s.setTranslator(e[0]),s.translate())}),s.getTranslators()),n()}},FW._WebDelegateTranslator.prototype=new FW._Base,FW._StringMagic=function(){this._filters=new Array,this.addFilter=function(t){return this._filters.push(t),this},this.split=function(t){return this.addFilter(function(e){return e.split(t).filter(function(t){return""!=t})})},this.replace=function(t,e,i){return this.addFilter(function(r){return r.match(t)?r.replace(t,e,i):r})},this.prepend=function(t){return this.replace(/^/,t)},this.append=function(t){return this.replace(/$/,t)},this.remove=function(t,e){return this.replace(t,"",e)},this.trim=function(){return this.addFilter(function(t){return Zotero.Utilities.trim(t)})},this.trimInternal=function(){return this.addFilter(function(t){return Zotero.Utilities.trimInternal(t)})},this.match=function(t,e){return e||(e=0),this.addFilter(function(i){var r=i.match(t);return void 0===r||null===r?void 0:r[e]})},this.cleanAuthor=function(t,e){return this.addFilter(function(i){return Zotero.Utilities.cleanAuthor(i,t,e)})},this.key=function(t){return this.addFilter(function(e){return e[t]})},this.capitalizeTitle=function(){return this.addFilter(function(t){return Zotero.Utilities.capitalizeTitle(t)})},this.unescapeHTML=function(){return this.addFilter(function(t){return Zotero.Utilities.unescapeHTML(t)})},this.unescape=function(){return this.addFilter(function(t){return unescape(t)})},this._applyFilters=function(t,e){for(i in this._filters){t=flatten(t),t=t.filter(function(t){return void 0!==t&&null!==t});for(var r=0;r0&&a[0])return n}},FW.getScraper=function(t,e){var i=FW.detectWeb(t,e);return FW._scrapers.filter(function(r){return r.evaluateThing(r.itemType,t,e)==i&&r.evaluateThing(r.detect,t,e)})[0]},FW.doWeb=function(t,e){var i=FW.getScraper(t,e);i.makeItems(t,e,[],function(t,e,i,r){e.callHook("scraperDone",t,i,r),t.title||(t.title=""),t.complete()},function(){Zotero.done()}),Zotero.wait()}; -function detectWeb(doc, url) { return FW.detectWeb(doc, url); } -function doWeb(doc, url) { return FW.doWeb(doc, url); } - - - /* ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2012/2013 Sebastian Karcher - + + Copyright © 2012/2013 Sebastian Karcher, Zoë C. Ma, and contributors + This file is part of Zotero. - + Zotero is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Zotero is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. - + You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . - + along with Zotero. If not, see . + ***** END LICENSE BLOCK ***** */ -/**ToC **/ -FW.MultiScraper({ -itemType : 'multiple', -detect : FW.Xpath('//h6/a'), -choices : { - titles : FW.Xpath('//h6/a|//h2/a').text().trim(), - urls : FW.Xpath('//h6/a|//h2/a').key("href") + +function detectWeb(doc, url) { + let pageURL = new URL(url); + // Some content may not be in the print edition, but the web publication + // is a magazine nonetheless + if (/^\/(articles|forum(_response)?|us)\/.+/.test(pageURL.pathname)) { + return 'magazineArticle'; + } + else if (getSearchResults(doc, true)) { + return 'multiple'; + } + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + let outerSelectors = [ + "[data-elementor-type='search-results']", // search page + "[data-elementor-post-type='issue']", // issue TOC + "[data-elementor-post-type='special-project']", // special project + ]; + // Titles are in an "h3 a" inside the outer container element + // The computed selector looks like "container1 h3 a, container2 h3 a ..." + let selectors = outerSelectors.map(s => s + " h3 a").join(", "); + // Legacy issue pages; although many of the links are broken + selectors += ", [data-elementor-post-type='elementor_library'] h6 a"; + var rows = doc.querySelectorAll(selectors); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; } -}); - -/**Search Results**/ -FW.MultiScraper({ -itemType : 'multiple', -detect : FW.Xpath('//li[@class="search-result"]//h3/a'), -choices : { - titles : FW.Xpath('//h3/a').text().trim(), - urls : FW.Xpath('//h3/a').key("href") + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } } -}); - -/** Articles */ -FW.Scraper({ -itemType : 'magazineArticle', -detect : FW.Xpath('//div[@class="title"]/h1'), -title : FW.Xpath('//div[@class="title"]/h1').text().trim(), -attachments : [{ url: FW.Url(), - title: "Boston Review Snapshot", - type: "text/html" }], -creators : FW.Xpath('//div[@class="article-author"]//div[@class="author-name"]'+ - '|//div[contains(@class, "field-name-field-author")]//a').text().split(/ and /).cleanAuthor("author"), -date : FW.Xpath('//span[@class="date-display-single"]/@content').text(), -ISSN : "0734-2306", -abstractNote : FW.Xpath('//div[contains(@class, "field-name-field-subhead")]').text(), -publicationTitle : "Boston Review", -language : "en-US" -}); - +async function scrape(doc, url = doc.location.href) { + let translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + + translator.setHandler('itemDone', (_obj, item) => { + item.publicationTitle = item.libraryCatalog = "Boston Review"; + let ldInfo = text(doc, "script[type='application/ld+json']"); + let date; + if (ldInfo) { + // Get schema.org metadata for the web page from JSON-LD + let pageInfo + = (JSON.parse(ldInfo)["@graph"] || []) + .filter(x => x["@type"] === "WebPage")[0]; + date = pageInfo && pageInfo.datePublished; + } + if (!date) { + // Only as fallback; despite the itemprop value, this doesn't + // always appear to be "date modified"; rather, it's the original + // publication date + date = text(doc, ".elementor-post-info [itemprop='dateModified']"); + } + if (date) { + item.date = date; + } + + // Remove suffix " - Boston Review" in title + item.title = item.title.replace(/\s+-\s+Boston Review\s*$/, ""); + + // NOTE: the href property match takes care of both /author and + // /author-custom paths + let authors = doc.querySelectorAll("h2 a[href^='https://www.bostonreview.net/author']"); + for (let author of authors) { + let authorName = ZU.trimInternal(author.textContent.trim()); + item.creators.push(ZU.cleanAuthor(authorName, "author")); + } + + for (let tag of doc.querySelectorAll("a[href^='https://www.bostonreview.net/tag/']")) { + item.tags.push(ZU.trimInternal(tag.textContent)); + } + + // NOTE that in general there's no sure way to determine whether an + // article belongs to a print issue (hence no volume/issue numbers and + // no ISSN which is for the print publication). If you can help, please + // contribute! + + item.complete(); + }); + + let em = await translator.getTranslatorObject(); + em.itemType = 'magazineArticle'; + await em.doWeb(doc, url); +} /** BEGIN TEST CASES **/ var testCases = [ { "type": "web", - "url": "http://www.bostonreview.net/forum/promoting-social-mobility/rethinking-family-life-robin-west", + "url": "https://www.bostonreview.net/forum/can-global-brands-create-just-supply-chains-richard-locke", "items": [ { "itemType": "magazineArticle", - "title": "Promoting Social Mobility", + "title": "Can Global Brands Create Just Supply Chains?", "creators": [ { - "firstName": "Robin", - "lastName": "West", + "firstName": "Richard M.", + "lastName": "Locke", "creatorType": "author" } ], - "date": "2012-09-01T00:00:00-04:00", - "ISSN": "0734-2306", + "date": "2013-05-21T10:22:49+00:00", + "abstractNote": "When Jia Jingchuan, a 27-year-old electronics worker in Suzhou, China, sought compensation for the chemical poisoning he suffered at work, he appealed", "language": "en-US", "libraryCatalog": "Boston Review", "publicationTitle": "Boston Review", - "url": "http://www.bostonreview.net/forum/promoting-social-mobility/rethinking-family-life-robin-west", + "url": "https://www.bostonreview.net/forum/can-global-brands-create-just-supply-chains-richard-locke/", "attachments": [ { - "title": "Boston Review Snapshot", + "title": "Snapshot", "mimeType": "text/html" } ], - "tags": [], + "tags": [ + { + "tag": "Global" + }, + { + "tag": "Human Rights" + }, + { + "tag": "Labor" + } + ], "notes": [], "seeAlso": [] } @@ -116,31 +188,44 @@ var testCases = [ }, { "type": "web", - "url": "http://www.bostonreview.net/forum/can-global-brands-create-just-supply-chains-richard-locke", + "url": "https://www.bostonreview.net/articles/government-loansharking/", "items": [ { "itemType": "magazineArticle", - "title": "Can Global Brands Create Just Supply Chains?", + "title": "Government Loansharking", "creators": [ { - "firstName": "Richard M.", - "lastName": "Locke", + "firstName": "Malcolm", + "lastName": "Harris", "creatorType": "author" } ], - "date": "2013-05-21T00:00:00-04:00", - "ISSN": "0734-2306", + "date": "2013-06-07T12:00:00+00:00", + "abstractNote": "Last November when I first wrote about student loans for Boston Review, the Department of Education estimated it would be pulling in around $25 billion in", "language": "en-US", "libraryCatalog": "Boston Review", "publicationTitle": "Boston Review", - "url": "http://www.bostonreview.net/forum/can-global-brands-create-just-supply-chains-richard-locke", + "url": "https://www.bostonreview.net/articles/government-loansharking/", "attachments": [ { - "title": "Boston Review Snapshot", + "title": "Snapshot", "mimeType": "text/html" } ], - "tags": [], + "tags": [ + { + "tag": "Economy" + }, + { + "tag": "Education" + }, + { + "tag": "Politics" + }, + { + "tag": "U.S." + } + ], "notes": [], "seeAlso": [] } @@ -148,32 +233,59 @@ var testCases = [ }, { "type": "web", - "url": "http://www.bostonreview.net/us/government-loansharking", + "url": "https://bostonreview.net/issue/september-october-2012/", + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.bostonreview.net/?s=labor", + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.bostonreview.net/articles/astra-taylor-wolson-interview/", "items": [ { "itemType": "magazineArticle", - "title": "Government Loansharking", + "title": "Beyond the Neoliberal University", "creators": [ { - "firstName": "Malcolm", - "lastName": "Harris", + "firstName": "Astra", + "lastName": "Taylor", + "creatorType": "author" + }, + { + "firstName": "Todd", + "lastName": "Wolfson", "creatorType": "author" } ], - "date": "2013-06-07T00:00:00-04:00", - "ISSN": "0734-2306", - "abstractNote": "An Update on the Student-Debt Crisis", + "date": "2020-08-04T12:00:00+00:00", + "abstractNote": "Astra Taylor talks with Rutgers faculty union president Todd Wolfson about organizing academic communities in the age of COVID-19.", "language": "en-US", "libraryCatalog": "Boston Review", "publicationTitle": "Boston Review", - "url": "http://www.bostonreview.net/us/government-loansharking", + "url": "https://www.bostonreview.net/articles/astra-taylor-wolson-interview/", "attachments": [ { - "title": "Boston Review Snapshot", + "title": "Snapshot", "mimeType": "text/html" } ], - "tags": [], + "tags": [ + { + "tag": "COVID-19" + }, + { + "tag": "Education" + }, + { + "tag": "Interview" + }, + { + "tag": "Labor" + } + ], "notes": [], "seeAlso": [] } @@ -181,13 +293,8 @@ var testCases = [ }, { "type": "web", - "url": "http://bostonreview.net/septemberoctober-2012", - "items": "multiple" - }, - { - "type": "web", - "url": "http://www.bostonreview.net/search/node/labor", + "url": "https://www.bostonreview.net/special_project/opportunity-after-neoliberalism/", "items": "multiple" } ] -/** END TEST CASES **/ \ No newline at end of file +/** END TEST CASES **/