Suppoerted Qiniu; Bugfix of upload_file.

img2url/main.py

@@ -10,7 +10,9 @@
 from img2url.metadata import VERSION
 
 from img2url.config import locate_config, load_config
+
 from img2url.remotes.github import GitHubConfig, GitHubOperation
+from img2url.remotes.qiniu import QiniuConfig, QiniuOperation
 
 
 DOC = '''
@@ -26,6 +28,7 @@
 
 REGISTERED_REMOTES = {
     'github': (GitHubConfig, GitHubOperation),
+    'qiniu': (QiniuConfig, QiniuOperation),
 }
 
 
@@ -47,31 +50,6 @@ def get_doc_type(args):
     return 'original'
 
 
-# def conditional_upload_file(path, config):
-#
-#     fname, _, sha = load_file(path)
-#
-#     # load remote files.
-#     exists_files = list_repo(config)
-#
-#     for remote_fname, remote_sha in exists_files:
-#         if sha != remote_sha and fname != remote_fname:
-#             continue
-#
-#         # case 1, file already exists in remote.
-#         if sha == remote_sha:
-#             return remote_fname
-#
-#         # case 2, filename conflicts, treat it as update.
-#         if fname == remote_fname:
-#             rep = update_file(path, config, pre_sha=remote_sha)
-#             return extract_filename(rep)
-#
-#     # case 3, file not exists.
-#     rep = create_file(path, config)
-#     return extract_filename(rep)
-
-
 def upload_file(fpath, fields, RemoteConfig, RemoteOperation):
 
     config = RemoteConfig(fields)
@@ -80,29 +58,25 @@ def upload_file(fpath, fields, RemoteConfig, RemoteOperation):
     ret_fname = None
 
     # load remote files.
-    exists_files = operator.list_repo()
-
-    for remote_fname, remote_fhash in exists_files:
+    exists_files = operator.list_remote()
 
-        if operator.fhash != remote_fhash and operator.fname != remote_fname:
-            # not this one.
-            continue
+    fname2fhash = {n: h for n, h in exists_files}
+    fhash2fname = {h: n for n, h in exists_files}
 
-        if operator.fhash == remote_fhash:
-            # case 1, file already exists in remote.
-            ret_fname = remote_fname
-            break
+    if operator.fhash in fhash2fname:
+        # case 1, file already exists in remote.
+        ret_fname = fhash2fname[operator.fhash]
 
-        if operator.fname == remote_fname:
-            # case 2, filename conflicts, treat it as update.
-            ret_fname = operator.update_file(old_fhash=remote_fhash)
-            break
+    elif operator.fname in fname2fhash:
+        # case 2, filename conflicts, treat it as update.
+        ret_fname = operator.update_file(old_fhash=fname2fhash[operator.fname])
 
-    # case 3, file not exists.
-    ret_fname = operator.create_file()
+    else:
+        # case 3, file not exists.
+        ret_fname = operator.create_file()
 
     # return corresponding url.
-    return ret_fname, operator.resource_url(ret_fname)
+    return ret_fname, operator.resource_url(ret_fname, operator.fhash)
 
 
 def entry_point():

img2url/remotes/base.py

@@ -68,19 +68,23 @@ def __init__(self, config, fpath):
         self._setup_file(fpath)
         self.init()
 
+    def get_config(self, key):
+        return self.config.fields.get(key, None)
+
     # fpath: ab
     def _setup_file(self, fpath):
         fpath = expanduser(fpath)
         if not exists(fpath):
             # TODO(huntzhan): enhance logging.
             raise RuntimeError('FATAL: {0} not exists'.format(fpath))
 
-        # fpath:     abspath of file.
-        # fname:     filename of file.
-        # fbasename: filename without ext.
-        # fext:      ext of file.
-        # fdata:     binary data of file.
-        # fhash:     hash value of file, defined by user.
+        # fpath:           abspath of file.
+        # fname:           filename of file.
+        # fbasename:       filename without ext.
+        # fext:            ext of file.
+        # fdata:           binary data of file.
+        # fhash:           hash value of file, defined by user.
+        # fname_with_hash: fname + '-' + fhash + fext
         self.fpath = fpath
         self.fname = basename(fpath)
         self.fbasename, self.ext = splitext(self.fname)
@@ -89,6 +93,11 @@ def _setup_file(self, fpath):
             self.fdata = fin.read()
         self.fhash = self.generate_file_hash(self.fdata)
 
+        # mainly for update.
+        self.fname_with_hash = '{0}[{1}]{2}'.format(
+            self.fbasename, self.fhash, self.ext,
+        )
+
     #############################################
     # subclass may override following methods.  #
     #############################################

img2url/remotes/github.py

@@ -24,12 +24,16 @@ class GitHubConfig(Configuration):
 
     FIELDS = [
 
-        ('token', REQUIRED_FIELD),
-        ('user', REQUIRED_FIELD),
-        ('repo', REQUIRED_FIELD),
+        ('github_token', REQUIRED_FIELD),
+        ('github_user', REQUIRED_FIELD),
+        ('github_repo', REQUIRED_FIELD),
+
+        ('github_branch', 'master'),
+        ('github_path', ''),
+
+        ('github_commiter_name', AUTHORS[0]),
+        ('github_commiter_email', EMAILS[0]),
 
-        ('branch', 'master'),
-        ('path', ''),
         (
             'message_template_create',
             '{filename} created by img2url at {time}.',
@@ -38,19 +42,22 @@ class GitHubConfig(Configuration):
             'message_template_update',
             '{filename} updated by img2url at {time}.',
         ),
-
-        ('commiter_name', AUTHORS[0]),
-        ('commiter_email', EMAILS[0]),
-
         ('proxies', None),
     ]
 
-    def postprocessing(self):
+    def postprocess_fields(self):
+        # TODO(huntzhan): fix that later.
+        # fix compatible issue.
+        for key, value in list(self.fields.items()):
+            if key.startswith('github_'):
+                key = key[7:]
+                self.fields[key] = value
+
         # 1. if path is empty, remain empty.
         # 2. otherwise, path is ended with '/' but not starts with '/'.
-        self.field['path'] = self.field['path'].strip('/')
-        if self.field['path']:
-            self.field['path'] += '/'
+        self.fields['path'] = self.fields['path'].strip('/')
+        if self.fields['path']:
+            self.fields['path'] += '/'
 
 
 # return: (filename, base64 encoded bytes, sha)

img2url/remotes/qiniu.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+from __future__ import (
+    division, absolute_import, print_function, unicode_literals,
+)
+from builtins import *                  # noqa
+from future.builtins.disabled import *  # noqa
+
+from cStringIO import StringIO
+
+from .base import REQUIRED_FIELD, Configuration, OperationPackage
+
+from qiniu import Auth, BucketManager, put_file
+from qiniu.utils import etag_stream
+
+
+class QiniuConfig(Configuration):
+
+    FIELDS = [
+        ('qiniu_access_key', REQUIRED_FIELD),
+        ('qiniu_secret_key', REQUIRED_FIELD),
+        ('qiniu_bucket', REQUIRED_FIELD),
+        ('qiniu_base_url', REQUIRED_FIELD),
+    ]
+
+    def postprocess_fields(self):
+        base_url = self.fields['qiniu_base_url']
+        if not base_url.endswith('/'):
+            base_url += '/'
+        if not base_url.startswith('http'):
+            base_url = 'http://' + base_url
+        self.fields['qiniu_base_url'] = base_url
+
+
+class QiniuOperation(OperationPackage):
+
+    def init(self):
+        self.q = Auth(
+            self.get_config('qiniu_access_key'),
+            self.get_config('qiniu_secret_key'),
+        )
+        self.bucket = BucketManager(self.q)
+
+    def token(self, filename):
+        return self.q.upload_token(
+            self.get_config('qiniu_bucket'),
+            filename,
+        )
+
+    def generate_file_hash(self, data):
+        fhash = etag_stream(StringIO(data))
+        if isinstance(fhash, bytes):
+            fhash = fhash.decode('ascii')
+        return fhash
+
+    def list_remote(self):
+        rep = self.bucket.list(self.get_config('qiniu_bucket'))
+        assert rep[1]
+        ret = []
+        for element in rep[0]['items']:
+            ret.append(
+                (element['key'], element['hash']),
+            )
+        return ret
+
+    def create_file(self):
+        token = self.token(self.fname)
+        put_file(token, self.fname, self.fpath)
+        return self.fname
+
+    def update_file(self, old_fhash):
+        token = self.token(self.fname_with_hash)
+        put_file(token, self.fname_with_hash, self.fpath)
+        return self.fname_with_hash
+
+    def resource_url(self, fname, hash_tag):
+        return self.get_config('qiniu_base_url') + fname

requirements.txt

@@ -5,3 +5,4 @@ PyYAML
 requests>=2.10.0
 pysocks
 docopt
+qiniu

tests/test_usage.py → tests/test_github.py

@@ -40,18 +40,18 @@ def token():
 
 
 CONFIG_PATH = tmpfile('''
-token: {0}
-user: img2url-testing
-repo: img2url-testing-travisci
+github_token: {0}
+github_user: img2url-testing
+github_repo: img2url-testing-travisci
 '''.format(token()))
 
 
 def test_config():
     GitHubConfig(load_config(CONFIG_PATH))
 
     bad_path = tmpfile('''
-user: img2url-testing
-repo: img2url-testing-travisci
+github_user: img2url-testing
+github_repo: img2url-testing-travisci
 ''')
     with pytest.raises(RuntimeError):
         GitHubConfig(load_config(bad_path))
@@ -69,10 +69,10 @@ def test_create_and_update():
 
 def test_branch():
     CONFIG_PATH_WITH_BRANCH = tmpfile('''
-token: {0}
-user: img2url-testing
-repo: img2url-testing-travisci
-branch: branch-test
+github_token: {0}
+github_user: img2url-testing
+github_repo: img2url-testing-travisci
+github_branch: branch-test
 '''.format(token()))
 
     path = tmpfile(random_str(10))
@@ -85,10 +85,10 @@ def test_branch():
 
 def test_path():
     CONFIG_PATH_WITH_PATH = tmpfile('''
-token: {0}
-user: img2url-testing
-repo: img2url-testing-travisci
-path: this-is/random-nested-path-{1}/
+github_token: {0}
+github_user: img2url-testing
+github_repo: img2url-testing-travisci
+github_path: this-is/random-nested-path-{1}/
 '''.format(token(), random_str(10)))
 
     path = tmpfile(random_str(10))