Merge pull request codeforamerica#322 from tdooner/add-logo-url

Add `logo_url` to Organization API response

migrations/versions/2864e71a466e_add_logo_url_to_organization.py

@@ -0,0 +1,26 @@
+"""Add Logo URL to Organization
+
+Revision ID: 2864e71a466e
+Revises: a5abdf9487c
+Create Date: 2018-01-25 15:59:31.973635
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '2864e71a466e'
+down_revision = 'a5abdf9487c'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('organization', sa.Column('logo_url', sa.Unicode(), nullable=True))
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('organization', 'logo_url')
+    ### end Alembic commands ###

models.py

@@ -98,14 +98,15 @@ class Organization(db.Model):
     keep = db.Column(db.Boolean())
     tsv_body = db.Column(TSVectorType())
     id = db.Column(db.Unicode())
+    logo_url = db.Column(db.Unicode())
 
     # Relationships
     # can contain events, stories, projects (these relationships are defined in the child objects)
 
     def __init__(self, name, website=None, events_url=None, members_count=None,
                  rss=None, projects_list_url=None, type=None, city=None,
                  latitude=None, longitude=None, last_updated=time.time(),
-                 tags=[], social_profiles={}):
+                 tags=[], social_profiles={}, logo_url=None):
         self.name = name
         self.website = website
         self.events_url = events_url
@@ -122,6 +123,7 @@ def __init__(self, name, website=None, events_url=None, members_count=None,
         self.started_on = unicode(date.today())
         self.id = safe_name(raw_name(name))
         self.members_count = members_count
+        self.logo_url = logo_url
 
     def current_events(self):
         '''

run_update.py

@@ -16,7 +16,6 @@
 from dateutil.tz import tzoffset
 import feedparser
 import json
-from psycopg2 import connect, extras
 from raven import Client as SentryClient
 from requests import get, exceptions
 
@@ -44,6 +43,7 @@
 # TODO: use a Meetup client library with pagination
 MEETUP_API_URL = "https://api.meetup.com/2/events?status=past,upcoming&format=json&group_urlname={group_urlname}&key={key}&desc=true&page=200"
 MEETUP_COUNT_API_URL = "https://api.meetup.com/2/groups?group_urlname={group_urlname}&key={key}"
+GITHUB_USER_API_URL = 'https://api.github.com/users/{username}'
 GITHUB_USER_REPOS_API_URL = 'https://api.github.com/users/{username}/repos'
 GITHUB_REPOS_API_URL = 'https://api.github.com/repos{repo_path}'
 GITHUB_ISSUES_API_URL = 'https://api.github.com/repos{repo_path}/issues'
@@ -321,6 +321,14 @@ def get_adjoined_json_lists(response, headers=None):
     return result, status_code
 
 
+def parse_github_user(url):
+    ''' given a URL, returns the github username or None if it is not a Github URL '''
+    _, host, path, _, _, _ = urlparse(url)
+    matched = match(r'(/orgs)?/(?P<name>[^/]+)/?$', path)
+    if host in ('www.github.com', 'github.com') and matched:
+        return matched.group('name')
+
+
 def get_projects(organization):
     '''
         Get a list of projects from CSV, TSV, JSON, or Github URL.
@@ -334,10 +342,9 @@ def get_projects(organization):
     # If projects_list is a GitHub organization
     # Use the GitHub auth to request all the included repos.
     # Follow next page links
-    _, host, path, _, _, _ = urlparse(organization.projects_list_url)
-    matched = match(r'(/orgs)?/(?P<name>[^/]+)/?$', path)
-    if host in ('www.github.com', 'github.com') and matched:
-        projects_url = GITHUB_USER_REPOS_API_URL.format(username=matched.group('name'))
+    github_username = parse_github_user(organization.projects_list_url)
+    if github_username:
+        projects_url = GITHUB_USER_REPOS_API_URL.format(username=github_username)
 
         try:
             got = get_github_api(projects_url)
@@ -1199,6 +1206,33 @@ def update_attendance(session, organization_name, attendance_dict):
     return existing_attendance
 
 
+def get_logo(org_info):
+    '''
+    get an organization's logo, looking first at 'logo_url' in the JSON and
+    then Github (project lists url)
+    '''
+    # allow specifying a logo_url in the json file
+    if 'logo_url' in org_info:
+        return org_info['logo_url']
+
+    if 'projects_list_url' not in org_info:
+        return None
+
+    github_username = parse_github_user(org_info['projects_list_url'])
+    if github_username:
+        # NOTE: This uses the /users/:id API endpoint to handle both cases
+        # where the brigade's profile is a single user account or an
+        # organizational account.
+        request_url = GITHUB_USER_API_URL.format(username=github_username)
+        got = get_github_api(request_url)
+        try:
+            github_response = got.json()
+            return github_response['avatar_url']
+        except ValueError:
+            logger.error("Malformed GitHub JSON fetching organization URL for " + github_username)
+            return
+
+
 def main(org_name=None, org_sources=None):
     ''' Update the API's database
     '''
@@ -1243,13 +1277,17 @@ def main(org_name=None, org_sources=None):
             db.session.execute(db.update(Project, values={'keep': False}).where(Project.organization_name == org_info['name']))
             db.session.execute(db.update(Organization, values={'keep': False}).where(Organization.name == org_info['name']))
 
+            # ORGANIZATION INFO
             # Save or update the organization
+            org_info.update({'logo_url': get_logo(org_info)})
+
             organization = save_organization_info(db.session, org_info)
             organization_names.add(organization.name)
 
             # commit the organization and the false keeps
             db.session.commit()
 
+
             # STORIES
             if organization.rss or organization.website:
                 logging.info(u"Gathering all of {}'s stories.".format(organization.name))

test/updater/test_run_update.py

@@ -91,6 +91,9 @@ def response_content(self, url, request):
             elif self.results_state == 'after':
                 return response(200, '''\n'''.join(project_lines[0:2]), {'content-type': 'text/csv; charset=UTF-8'})
 
+        # json of user description
+        elif url.geturl() == 'https://api.github.com/users/codeforamerica':
+            return response(200, '''{ "login": "codeforamerica", "id": 337792, "avatar_url": "https://avatars2.githubusercontent.com/u/337792?v=4", "gravatar_id": "", "url": "https://api.github.com/users/codeforamerica", "html_url": "https://github.com/codeforamerica", "followers_url": "https://api.github.com/users/codeforamerica/followers", "following_url": "https://api.github.com/users/codeforamerica/following{/other_user}", "gists_url": "https://api.github.com/users/codeforamerica/gists{/gist_id}", "starred_url": "https://api.github.com/users/codeforamerica/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/codeforamerica/subscriptions", "organizations_url": "https://api.github.com/users/codeforamerica/orgs", "repos_url": "https://api.github.com/users/codeforamerica/repos", "events_url": "https://api.github.com/users/codeforamerica/events{/privacy}", "received_events_url": "https://api.github.com/users/codeforamerica/received_events", "type": "Organization", "site_admin": false, "name": "Code for America", "company": null, "blog": "http://codeforamerica.org", "location": null, "email": "[email protected]", "hireable": null, "bio": null, "public_repos": 659, "public_gists": 0, "followers": 0, "following": 0, "created_at": "2010-07-19T19:41:04Z", "updated_at": "2017-09-05T10:22:41Z" }''')
         # json of project descriptions
         elif url.geturl() == 'https://api.github.com/users/codeforamerica/repos':
             return response(200, '''[{ "id": 10515516, "name": "cityvoice", "owner": { "login": "codeforamerica", "avatar_url": "https://avatars.githubusercontent.com/u/337792", "html_url": "https://github.com/codeforamerica", "type": "Organization"}, "html_url": "https://github.com/codeforamerica/cityvoice", "description": "A place-based call-in system for gathering and sharing community feedback",  "url": "https://api.github.com/repos/codeforamerica/cityvoice", "contributors_url": "https://api.github.com/repos/codeforamerica/cityvoice/contributors", "created_at": "2013-06-06T00:12:30Z", "updated_at": "2014-02-21T20:43:16Z", "pushed_at": "2014-02-21T20:43:16Z", "homepage": "http://www.cityvoiceapp.com/", "stargazers_count": 10, "watchers_count": 10, "language": "Ruby", "forks_count": 12, "open_issues": 37, "languages_url": "https://api.github.com/repos/codeforamerica/cityvoice/languages" }]''', headers=dict(Link='<https://api.github.com/user/337792/repos?page=2>; rel="next", <https://api.github.com/user/337792/repos?page=2>; rel="last"'))
@@ -1656,6 +1659,17 @@ def test_commit_status(self):
         cityvoice = self.db.session.query(Project).filter(filter).first()
         self.assertEqual("success", cityvoice.commit_status)
 
+    def test_logo_fetching(self):
+        """ Test grabbing the organization logo """
+        with HTTMock(self.response_content):
+            import run_update
+            run_update.main(org_sources=run_update.TEST_ORG_SOURCES_FILENAME)
+
+        from app import Organization
+        filter = Organization.name == u'Code for America (2)'
+        cfa = self.db.session.query(Organization).filter(filter).first()
+        self.assertEqual("https://avatars2.githubusercontent.com/u/337792?v=4", cfa.logo_url)
+
 
 if __name__ == '__main__':
     unittest.main()