Incremental Linting (trailofbits#911)

* flake

* start removing * imports

wildcard imports make it so we can't easily reason about what is
available in scope and limit the utility of linting tools

* a wide variety of pep8 related changes

Also adds a tox.ini entry for flake8 experimentation right now.

* fixes

manticore/__init__.py

@@ -1,3 +1,3 @@
-from .manticore import Manticore
-from .models import variadic
-from .utils.helpers import issymbolic
+from .manticore import Manticore  # noqa
+from .models import variadic  # noqa
+from .utils.helpers import issymbolic  # noqa

manticore/__main__.py

@@ -121,7 +121,7 @@ def main():
 
     env = {key: val for key, val in map(lambda env: env[0].split('='), args.env)}
 
-    m = Manticore(args.argv[0], argv=args.argv[1:], env=env, entry_symbol=args.entrysymbol, workspace_url=args.workspace,  policy=args.policy, disasm=args.disasm, concrete_start=args.data)
+    m = Manticore(args.argv[0], argv=args.argv[1:], env=env, entry_symbol=args.entrysymbol, workspace_url=args.workspace, policy=args.policy, disasm=args.disasm, concrete_start=args.data)
 
     # Fixme(felipe) remove this, move to plugin
     m.coverage_file = args.coverage

manticore/binary/__init__.py

@@ -14,6 +14,8 @@
 and common API.  interpreters? linkers? linked DLLs?
 
 '''
+from elftools.elf.elffile import ELFFile
+import StringIO
 
 
 class Binary(object):
@@ -40,10 +42,6 @@ def threads(self):
         pass
 
 
-from elftools.elf.elffile import ELFFile
-import StringIO
-
-
 class CGCElf(Binary):
 
     @staticmethod

manticore/core/cpu/abstractcpu.py

@@ -6,14 +6,11 @@
 from functools import wraps
 from itertools import islice, imap
 
-import capstone as cs
 import unicorn
 
 from .disasm import init_disassembler
-from ..smtlib import Expression, Bool, BitVec, Array, Operators, Constant
-from ..memory import (
-    ConcretizeMemory, InvalidMemoryAccess, MemoryException, FileMap, AnonMap
-)
+from ..smtlib import BitVec, Operators, Constant
+from ..memory import ConcretizeMemory, InvalidMemoryAccess
 from ...utils.helpers import issymbolic
 from ...utils.emulate import UnicornEmulator
 from ...utils.event import Eventful
@@ -28,6 +25,7 @@
 class CpuException(Exception):
     ''' Base cpu exception '''
 
+
 class DecodeException(CpuException):
     '''
     Raised when trying to decode an unknown or invalid instruction '''

manticore/core/cpu/x86.py

@@ -7,13 +7,12 @@
 
 from .abstractcpu import (
     Abi, SyscallAbi, Cpu, RegisterFile, Operand, instruction,
-    ConcretizeRegister, ConcretizeRegister, ConcretizeArgument, Interruption,
-    Syscall, DivideByZeroError
+    ConcretizeRegister, Interruption, Syscall, DivideByZeroError
 )
 
 
 from ..smtlib import Operators, BitVec, Bool, BitVecConstant, operator, visitors
-from ..memory import MemoryException, ConcretizeMemory
+from ..memory import ConcretizeMemory
 from ...utils.helpers import issymbolic
 from functools import reduce
 
@@ -607,6 +606,7 @@ def read(self, name):
     def sizeof(self, reg):
         return self._table[reg].size
 
+
 # Operand Wrapper
 class AMD64Operand(Operand):
     ''' This class deals with capstone X86 operands '''

manticore/core/executor.py

@@ -6,7 +6,7 @@
 
 from ..utils.nointerrupt import WithKeyboardInterruptAs
 from ..utils.event import Eventful
-from .smtlib import solver, Z3Solver, Expression, SolverException
+from .smtlib import Z3Solver, Expression, SolverException
 from .state import Concretize, TerminateState
 
 from .workspace import Workspace

manticore/core/parser/parser.py

@@ -1,7 +1,7 @@
 # Minimal INTEL assembler expression calculator
 import ply.yacc as yacc
 import copy
-from ..smtlib import Operators, Bool
+from ..smtlib import Operators
 # Lexer
 # ------------------------------------------------------------
 # calclex.py

manticore/core/plugin.py

@@ -371,10 +371,6 @@ def will_start_run_callback(self, state):
     def did_finish_run_callback(self):
         logger.info('did_finish_run')
 
-    def did_enqueue_state_callback(self, state_id, state):
-        ''' state was just got enqueued in the executor procesing list'''
-        logger.info('did_enqueue_state %r %r', state_id, state)
-
     def will_fork_state_callback(self, parent_state, expression, solutions, policy):
         logger.info('will_fork_state %r %r %r %r', parent_state, expression, solutions, policy)
 

manticore/core/smtlib/__init__.py

@@ -1,8 +1,8 @@
-from __future__ import absolute_import
-from .expression import Expression, Bool, BitVec, Array
-from .constraints import ConstraintSet
-from .solver import *
-from . import operators as Operators
+from __future__ import absolute_import  # noqa
+from .expression import Expression, Bool, BitVec, Array  # noqa
+from .constraints import ConstraintSet  # noqa
+from .solver import *  # noqa
+from . import operators as Operators  # noqa
 
 
 import logging

manticore/core/smtlib/constraints.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 from .expression import BitVecVariable, BoolVariable, ArrayVariable, Array, Bool, BitVec, BoolConstant, ArrayProxy, BoolEq, Variable, Constant
-from .visitors import GetDeclarations, TranslatorSmtlib, PrettyPrinter, pretty_print, translate_to_smtlib, get_depth, get_variables, simplify, replace
+from .visitors import GetDeclarations, TranslatorSmtlib, get_variables, simplify, replace
 import logging
 
 logger = logging.getLogger(__name__)

manticore/core/smtlib/expression.py

@@ -720,10 +720,6 @@ def index_max(self):
     def value_bits(self):
         return self._array.value_bits
 
-    @property
-    def value_bits(self):
-        return self._array.value_bits
-
     @property
     def taint(self):
         return self._array.taint
@@ -807,7 +803,7 @@ def store(self, index, value):
             index = self.cast_index(index)
         if not isinstance(value, Expression):
             value = self.cast_value(value)
-        from manticore.core.smtlib.visitors import simplify, translate_to_smtlib
+        from manticore.core.smtlib.visitors import simplify
         index = simplify(index)
         if isinstance(index, Constant):
             self._concrete_cache[index.value] = value

manticore/core/smtlib/operators.py

@@ -1,6 +1,8 @@
 from __future__ import absolute_import
-from .expression import *
-from ...utils.helpers import issymbolic, istainted
+from .expression import (
+    BitVec, BitVecExtract, BitVecSignExtend, BitVecZeroExtend, BitVecConstant, BitVecConcat, Bool, BitVecITE, BoolConstant, BoolITE
+)
+from ...utils.helpers import issymbolic
 import math
 
 
@@ -219,16 +221,6 @@ def UDIV(dividend, divisor):
     return dividend / divisor
 
 
-def UREM(a, b):
-    if isinstance(a, BitVec):
-        return a.urem(b)
-    if isinstance(b, BitVec):
-        return b.rurem(a)
-    if a < 0 or b < 0:
-        raise "azaraza"
-    return a % b
-
-
 def SDIV(a, b):
     if isinstance(a, BitVec):
         return a / b

manticore/core/smtlib/solver.py

@@ -22,7 +22,7 @@
 import re
 import time
 from .visitors import *
-from ...utils.helpers import issymbolic, istainted, memoized
+from ...utils.helpers import issymbolic
 import collections
 
 logger = logging.getLogger(__name__)

manticore/core/smtlib/visitors.py

@@ -141,12 +141,6 @@ def result(self):
         return self.variables
 
 
-def get_variables(expression):
-    visitor = GetDeclarations()
-    visitor.visit(expression)
-    return visitor.result
-
-
 class GetDepth(Visitor):
     ''' Simple visitor to collect all variables in an expression or set of
         expressions

manticore/core/state.py

@@ -1,7 +1,7 @@
 import copy
 import logging
 
-from .smtlib import solver, Bool, ArrayProxy, Array
+from .smtlib import solver, Bool
 from ..utils.helpers import issymbolic
 from ..utils.event import Eventful
 

manticore/ethereum.py

@@ -4,11 +4,12 @@
 import os
 from . import Manticore
 from .manticore import ManticoreError
-from .core.smtlib import ConstraintSet, Operators, solver, issymbolic, istainted, Array, Expression, Constant, operators
+from .core.smtlib import ConstraintSet, Operators, solver, Constant, operators
 from .core.smtlib.visitors import simplify
 from .core.plugin import FilterFunctions
 from .platforms import evm
 from .core.state import State
+from .utils.helpers import istainted, issymbolic
 import tempfile
 from subprocess import Popen, PIPE, check_output
 from multiprocessing import Process, Queue
@@ -36,12 +37,6 @@ def __init__(self, lib_names):
         self.lib_names = lib_names
 
 
-class DependencyError(EthereumError):
-    def __init__(self, lib_names):
-        super(DependencyError, self).__init__("You must pre-load and provide libraries addresses{ libname:address, ...} for %r" % lib_names)
-        self.lib_names = lib_names
-
-
 class NoAliveStates(EthereumError):
     pass
 
@@ -454,7 +449,7 @@ def _parse_size(num):
     @staticmethod
     def get_uint(data, nbytes, offset):
         """
-        Read a `nbytes` bytes long big endian unsigned integer from `data` starting at `offset` 
+        Read a `nbytes` bytes long big endian unsigned integer from `data` starting at `offset`
 
         :param data: sliceable buffer; symbolic buffer of Eth ABI encoded data
         :param offset: byte offset
@@ -673,7 +668,7 @@ class ManticoreEVM(Manticore):
             #Initialize user and contracts
             user_account = m.create_account(balance=1000)
             contract_account = m.solidity_create_contract(source_code, owner=user_account, balance=0)
-            contract_account.set(12345, value=100) 
+            contract_account.set(12345, value=100)
 
             seth.report()
             print seth.coverage(contract_account)
@@ -803,7 +798,7 @@ def _compile(source_code, contract_name, libraries=None):
 
             :param source_code: solidity source as either a string or a file handle
             :param contract_name: a string with the name of the contract to analyze
-            :param libraries: an itemizable of pairs (library_name, address) 
+            :param libraries: an itemizable of pairs (library_name, address)
             :return: name, source_code, bytecode, srcmap, srcmap_runtime, hashes
             :return: name, source_code, bytecode, runtime, srcmap, srcmap_runtime, hashes, abi, warnings
         """

manticore/manticore.py

@@ -44,7 +44,7 @@ def make_binja(program, disasm, argv, env, symbolic_files, concrete_start=''):
     def _check_disassembler_present(disasm):
         if is_binja_disassembler(disasm):
             try:
-                import binaryninja
+                import binaryninja  # noqa
             except ImportError:
                 err = ("BinaryNinja not found! You MUST own a BinaryNinja version"
                        " that supports GUI-less processing for this option"
@@ -72,7 +72,7 @@ def make_decree(program, concrete_start='', **kwargs):
     if concrete_start != '':
         logger.info('Starting with concrete input: {}'.format(concrete_start))
     platform.input.transmit(concrete_start)
-    platform.input.transmit(initial_state.symbolicate_buffer('+'*14, label='RECEIVE'))
+    platform.input.transmit(initial_state.symbolicate_buffer('+' * 14, label='RECEIVE'))
     return initial_state
 
 
@@ -380,10 +380,6 @@ def verbosity(level):
         """
         log.set_verbosity(level)
 
-    @property
-    def running(self):
-        return self._executor._running.value
-
     @property
     def running(self):
         return self._executor.running
@@ -502,7 +498,6 @@ def apply_model_hooks(self, path):
 
         # Imported straight from __main__.py; this will be re-written once the new
         # event code is in place.
-        from .core import cpu
         import importlib
         from . import platforms
 
@@ -708,8 +703,6 @@ def coverage_file(self, path):
         self._coverage_file = path
 
     def _did_finish_run_callback(self):
-        _shared_context = self.context
-
         with self._output.save_stream('command.sh') as f:
             f.write(' '.join(sys.argv))
 

manticore/platforms/decree.py

@@ -2,7 +2,7 @@
 from . import cgcrandom
 # TODO use cpu factory
 from ..core.cpu.x86 import I386Cpu
-from ..core.cpu.abstractcpu import Interruption, Syscall, ConcretizeRegister
+from ..core.cpu.abstractcpu import Interruption, ConcretizeRegister
 from ..core.memory import SMemory32
 from ..core.smtlib import *
 from ..core.executor import TerminateState

manticore/platforms/evm.py

@@ -1,16 +1,14 @@
 ''' Symbolic EVM implementation based on the yellow paper: http://gavwood.com/paper.pdf '''
-import time
 import random
 import copy
 import inspect
 from functools import wraps
 from ..utils.helpers import issymbolic, memoized
 from ..platforms.platform import *
-from ..core.smtlib import solver, TooManySolutions, Expression, Bool, BitVec, Array, Operators, Constant, BitVecConstant, ConstraintSet, SolverException
-from ..core.state import ForkState, TerminateState
-from ..utils.event import Eventful
-from ..core.smtlib.visitors import pretty_print, translate_to_smtlib, simplify
+from ..core.smtlib import solver, BitVec, Array, Operators, Constant
 from ..core.state import Concretize, TerminateState
+from ..utils.event import Eventful
+from ..core.smtlib.visitors import simplify
 import logging
 import sys
 from collections import namedtuple
@@ -356,11 +354,6 @@ def writes_to_stack(self):
             ''' True if the instruction writes to the stack '''
             return self.pushes > 0
 
-        @property
-        def reads_from_memory(self):
-            ''' True if the instruction reads from memory '''
-            return self.semantics in ('MLOAD', 'CREATE', 'CALL', 'CALLCODE', 'RETURN', 'DELEGATECALL', 'REVERT')
-
         @property
         def writes_to_memory(self):
             ''' True if the instruction writes to memory '''
@@ -2206,10 +2199,6 @@ def __str__(self):
     def logs(self):
         return self._logs
 
-    @property
-    def deleted_accounts(self):
-        return self._deleted_accounts
-
     @property
     def constraints(self):
         return self._constraints

manticore/platforms/linux.py

@@ -15,7 +15,6 @@
 
 from ..core.cpu.abstractcpu import Interruption, Syscall, ConcretizeArgument
 from ..core.cpu.cpufactory import CpuFactory
-from ..core.cpu.binja import BinjaCpu
 from ..core.memory import SMemory32, SMemory64, Memory32, Memory64
 from ..core.smtlib import Operators, ConstraintSet, SolverException, solver
 from ..core.cpu.arm import *

manticore/platforms/platform.py

@@ -1,6 +1,6 @@
-
 from manticore.utils.event import Eventful
 
+
 class OSException(Exception):
     pass
 

manticore/utils/emulate.py

@@ -1,6 +1,6 @@
 import logging
 
-from ..core.memory import MemoryException, FileMap, AnonMap
+from ..core.memory import MemoryException
 
 from .helpers import issymbolic
 ######################################################################