Split up OAuth services into own path, allow for extension of classes

.gitignore

@@ -7,6 +7,8 @@
 .idea
 .vagrant
 .tox
+.rope_project/
+.ropeproject/
 _build
 cnames
 bower_components/
@@ -35,4 +37,3 @@ whoosh_index
 xml_output
 public_cnames
 public_symlinks
-.rope_project/

readthedocs/core/management/commands/import_github.py

@@ -1,7 +1,7 @@
 from django.core.management.base import BaseCommand
 from django.contrib.auth.models import User
 
-from readthedocs.oauth.utils import GitHubService
+from readthedocs.oauth.services import GitHubService
 
 
 class Command(BaseCommand):

readthedocs/oauth/services/__init__.py

@@ -0,0 +1,13 @@
+"""Conditional classes for OAuth services"""
+
+from django.utils.module_loading import import_by_path
+from django.conf import settings
+
+GitHubService = import_by_path(
+    getattr(settings, 'OAUTH_GITHUB_SERVICE',
+            'readthedocs.oauth.services.github.GitHubService'))
+BitbucketService = import_by_path(
+    getattr(settings, 'OAUTH_BITBUCKET_SERVICE',
+            'readthedocs.oauth.services.bitbucket.BitbucketService'))
+
+registry = [GitHubService, BitbucketService]

readthedocs/oauth/services/base.py

@@ -0,0 +1,141 @@
+"""OAuth utility functions"""
+
+import logging
+from datetime import datetime
+
+from django.conf import settings
+from requests_oauthlib import OAuth2Session
+from allauth.socialaccount.models import SocialAccount
+
+
+DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
+
+log = logging.getLogger(__name__)
+
+
+class Service(object):
+
+    """Service mapping for local accounts
+
+    :param user: User to use in token lookup and session creation
+    :param account: :py:cls:`SocialAccount` instance for user
+    """
+
+    adapter = None
+    url_pattern = None
+
+    def __init__(self, user, account):
+        self.session = None
+        self.user = user
+        self.account = account
+
+    @classmethod
+    def for_user(cls, user):
+        """Create instance if user has an account for the provider"""
+        try:
+            account = SocialAccount.objects.get(
+                user=user,
+                provider=cls.adapter.provider_id
+            )
+            return cls(user=user, account=account)
+        except SocialAccount.DoesNotExist:
+            return None
+
+    def get_adapter(self):
+        return self.adapter
+
+    @property
+    def provider_id(self):
+        return self.get_adapter().provider_id
+
+    def get_session(self):
+        if self.session is None:
+            self.create_session()
+        return self.session
+
+    def create_session(self):
+        """Create OAuth session for user
+
+        This configures the OAuth session based on the :py:cls:`SocialToken`
+        attributes. If there is an ``expires_at``, treat the session as an auto
+        renewing token. Some providers expire tokens after as little as 2
+        hours.
+        """
+        token = self.account.socialtoken_set.first()
+        if token is None:
+            return None
+
+        token_config = {
+            'access_token': str(token.token),
+            'token_type': 'bearer',
+        }
+        if token.expires_at is not None:
+            token_expires = (token.expires_at - datetime.now()).total_seconds()
+            token_config.update({
+                'refresh_token': str(token.token_secret),
+                'expires_in': token_expires,
+            })
+
+        self.session = OAuth2Session(
+            client_id=token.app.client_id,
+            token=token_config,
+            auto_refresh_kwargs={
+                'client_id': token.app.client_id,
+                'client_secret': token.app.secret,
+            },
+            auto_refresh_url=self.get_adapter().access_token_url,
+            token_updater=self.token_updater(token)
+        )
+
+        return self.session or None
+
+    def token_updater(self, token):
+        """Update token given data from OAuth response
+
+        Expect the following response into the closure::
+
+            {
+                u'token_type': u'bearer',
+                u'scopes': u'webhook repository team account',
+                u'refresh_token': u'...',
+                u'access_token': u'...',
+                u'expires_in': 3600,
+                u'expires_at': 1449218652.558185
+            }
+        """
+
+        def _updater(data):
+            token.token = data['access_token']
+            token.expires_at = datetime.fromtimestamp(data['expires_at'])
+            token.save()
+            log.info('Updated token %s:', token)
+
+        return _updater
+
+    def sync(self, sync):
+        raise NotImplementedError
+
+    def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
+                          organization=None):
+        raise NotImplementedError
+
+    def create_organization(self, fields):
+        raise NotImplementedError
+
+    def setup_webhook(self, project):
+        raise NotImplementedError
+
+    @classmethod
+    def is_project_service(cls, project):
+        """Determine if this is the service the project is using
+
+        .. note::
+            This should be deprecated in favor of attaching the
+            :py:cls:`RemoteRepository` to the project instance. This is a slight
+            improvement on the legacy check for webhooks
+        """
+        # TODO Replace this check by keying project to remote repos
+        return (
+            cls.url_pattern is not None and
+            cls.url_pattern.search(project.repo) is not None
+        )

readthedocs/oauth/services/bitbucket.py

@@ -0,0 +1,181 @@
+"""OAuth utility functions"""
+
+import logging
+import json
+import re
+
+from django.conf import settings
+from requests.exceptions import RequestException
+from allauth.socialaccount.providers.bitbucket_oauth2.views import (
+    BitbucketOAuth2Adapter)
+
+from readthedocs.builds import utils as build_utils
+
+from ..models import RemoteOrganization, RemoteRepository
+from .base import Service
+
+
+DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public')
+
+log = logging.getLogger(__name__)
+
+
+class BitbucketService(Service):
+
+    """Provider service for Bitbucket"""
+
+    adapter = BitbucketOAuth2Adapter
+    # TODO replace this with a less naive check
+    url_pattern = re.compile(r'bitbucket.org\/')
+
+    def sync(self, sync):
+        """Import from Bitbucket"""
+        if sync:
+            self.sync_repositories()
+            self.sync_teams()
+
+    def sync_repositories(self):
+        # Get user repos
+        try:
+            repos = self.paginate(
+                'https://bitbucket.org/api/2.0/repositories/?role=member')
+            for repo in repos:
+                self.create_repository(repo)
+        except (TypeError, ValueError) as e:
+            log.error('Error syncing Bitbucket repositories: %s',
+                      str(e), exc_info=True)
+            raise Exception('Could not sync your Bitbucket repositories, '
+                            'try reconnecting your account')
+
+        # Because privileges aren't returned with repository data, run query
+        # again for repositories that user has admin role for, and update
+        # existing repositories.
+        try:
+            resp = self.paginate(
+                'https://bitbucket.org/api/2.0/repositories/?role=admin')
+            repos = (
+                RemoteRepository.objects
+                .filter(users=self.user,
+                        full_name__in=[r['full_name'] for r in resp],
+                        account=self.account)
+            )
+            for repo in repos:
+                repo.admin = True
+                repo.save()
+        except (TypeError, ValueError):
+            pass
+
+    def sync_teams(self):
+        """Sync Bitbucket teams and team repositories for user token"""
+        try:
+            teams = self.paginate(
+                'https://api.bitbucket.org/2.0/teams/?role=member'
+            )
+            for team in teams:
+                org = self.create_organization(team)
+                repos = self.paginate(team['links']['repositories']['href'])
+                for repo in repos:
+                    self.create_repository(repo, organization=org)
+        except ValueError as e:
+            log.error('Error syncing Bitbucket organizations: %s',
+                      str(e), exc_info=True)
+            raise Exception('Could not sync your Bitbucket team repositories, '
+                            'try reconnecting your account')
+
+    def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL,
+                          organization=None):
+        """Update or create a repository from Bitbucket
+
+        This looks up existing repositories based on the full repository name,
+        that is the username and repository name.
+
+        .. note::
+            The :py:data:`admin` property is not set during creation, as
+            permissions are not part of the returned repository data from
+            Bitbucket.
+        """
+        if (fields['is_private'] is True and privacy == 'private' or
+                fields['is_private'] is False and privacy == 'public'):
+            repo, _ = RemoteRepository.objects.get_or_create(
+                full_name=fields['full_name'],
+                account=self.account,
+            )
+            if repo.organization and repo.organization != organization:
+                log.debug('Not importing %s because mismatched orgs' %
+                          fields['name'])
+                return None
+            else:
+                repo.organization = organization
+            repo.users.add(self.user)
+            repo.name = fields['name']
+            repo.description = fields['description']
+            repo.private = fields['is_private']
+            repo.clone_url = fields['links']['clone'][0]['href']
+            repo.ssh_url = fields['links']['clone'][1]['href']
+            if repo.private:
+                repo.clone_url = repo.ssh_url
+            repo.html_url = fields['links']['html']['href']
+            repo.vcs = fields['scm']
+            repo.account = self.account
+
+            avatar_url = fields['links']['avatar']['href'] or ''
+            repo.avatar_url = re.sub(r'\/16\/$', r'/32/', avatar_url)
+
+            repo.json = json.dumps(fields)
+            repo.save()
+            return repo
+        else:
+            log.debug('Not importing %s because mismatched type' %
+                      fields['name'])
+
+    def create_organization(self, fields):
+        organization, _ = RemoteOrganization.objects.get_or_create(
+            slug=fields.get('username'),
+            account=self.account,
+        )
+        organization.name = fields.get('display_name')
+        organization.email = fields.get('email')
+        organization.avatar_url = fields['links']['avatar']['href']
+        organization.html_url = fields['links']['html']['href']
+        organization.json = json.dumps(fields)
+        organization.account = self.account
+        organization.users.add(self.user)
+        organization.save()
+        return organization
+
+    def paginate(self, url):
+        """Combines results from Bitbucket pagination
+
+        :param url: start url to get the data from.
+        """
+        resp = self.get_session().get(url)
+        data = resp.json()
+        results = data.get('values', [])
+        next_url = data.get('next')
+        if next_url:
+            results.extend(self.paginate(next_url))
+        return results
+
+    def setup_webhook(self, project):
+        session = self.get_session()
+        owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo)
+        data = {
+            'type': 'POST',
+            'url': 'https://{domain}/bitbucket'.format(domain=settings.PRODUCTION_DOMAIN),
+        }
+        try:
+            resp = session.post(
+                'https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/services'.format(
+                    owner=owner, repo=repo
+                ),
+                data=data,
+            )
+            if resp.status_code == 200:
+                log.info('Created Bitbucket webhook: project={project}'
+                         .format(project=project))
+                return True
+        except RequestException:
+            pass
+        else:
+            log.exception('Bitbucket webhook creation failed', exc_info=True)
+            return False