Skip to content

Commit

Permalink
internal refactor
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 327536370
  • Loading branch information
tensorflower-gardener authored and tflite-support-robot committed Aug 20, 2020
1 parent 39610a5 commit 03ba6dd
Show file tree
Hide file tree
Showing 31 changed files with 710 additions and 92 deletions.
34 changes: 34 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ workspace(name = "org_tensorflow_lite_support")

load("@bazel_tools//tools/build_defs/repo:java.bzl", "java_import_external")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@//third_party/py:python_configure.bzl", "python_configure")

http_archive(
name = "io_bazel_rules_closure",
Expand Down Expand Up @@ -220,6 +221,37 @@ http_archive(
build_file = "@//third_party:icu.BUILD",
)

http_archive(
name = "gemmlowp",
sha256 = "6678b484d929f2d0d3229d8ac4e3b815a950c86bb9f17851471d143f6d4f7834",
strip_prefix = "gemmlowp-12fed0cd7cfcd9e169bf1925bc3a7a58725fdcc3",
urls = [
"http://mirror.tensorflow.org/github.com/google/gemmlowp/archive/12fed0cd7cfcd9e169bf1925bc3a7a58725fdcc3.zip",
"https://github.com/google/gemmlowp/archive/12fed0cd7cfcd9e169bf1925bc3a7a58725fdcc3.zip",
],
)

http_archive(
name = "fft2d",
build_file = "@//third_party/fft2d:fft2d.BUILD",
sha256 = "5f4dabc2ae21e1f537425d58a49cdca1c49ea11db0d6271e2a4b27e9697548eb",
strip_prefix = "OouraFFT-1.0",
urls = [
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/petewarden/OouraFFT/archive/v1.0.tar.gz",
"https://github.com/petewarden/OouraFFT/archive/v1.0.tar.gz",
],
)

http_archive(
name = "darts_clone",
build_file = "@//third_party:darts_clone.BUILD",
sha256 = "c97f55d05c98da6fcaf7f9ecc6a6dc6bc5b18b8564465f77abff8879d446491c",
strip_prefix = "darts-clone-e40ce4627526985a7767444b6ed6893ab6ff8983",
urls = [
"https://github.com/s-yata/darts-clone/archive/e40ce4627526985a7767444b6ed6893ab6ff8983.zip",
],
)

# AutoValue 1.6+ shades Guava, Auto Common, and JavaPoet. That's OK
# because none of these jars become runtime dependencies.
java_import_external(
Expand Down Expand Up @@ -318,3 +350,5 @@ android_configure(name="local_config_android")
load("@local_config_android//:android.bzl", "android_workspace")
android_workspace()

python_configure(name = "local_config_python")

2 changes: 1 addition & 1 deletion tensorflow_lite_support/codegen/python/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pybind_extension(
deps = [
"//tensorflow_lite_support/codegen:android_java_generator",
"//tensorflow_lite_support/codegen:code_generator",
"//third_party/python_runtime:headers",
"@local_config_python//:python_headers",
"@pybind11",
],
)
Expand Down
134 changes: 115 additions & 19 deletions tensorflow_lite_support/custom_ops/kernel/sentencepiece/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,7 @@

load("@flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
load(
"//tensorflow_lite_support/custom_ops/kernel/sentencepiece:native.bzl",
"micore_tf_copts",
"micore_tf_deps",
)
load(":native.bzl", "micore_tf_copts", "micore_tf_deps")
load("@org_tensorflow//tensorflow:tensorflow.bzl", "pybind_extension")

package(
Expand Down Expand Up @@ -63,7 +59,9 @@ cc_library(
],
deps = [
":encoder_config",
"//third_party/darts_clone",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@darts_clone",
],
)

Expand All @@ -90,9 +88,9 @@ cc_library(
deps = [
":double_array_trie_builder",
":encoder_config",
"//third_party/sentencepiece/src:sentencepiece_model_cc_proto",
"//tensorflow_lite_support/cc/port:statusor",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_sentencepiece//src:sentencepiece_model_cc_proto",
],
)

Expand Down Expand Up @@ -131,6 +129,19 @@ cc_library(
alwayslink = 1,
)

cc_binary(
name = "sentencepiece_tokenizer_op.so",
srcs = [
"sentencepiece_tokenizer_op.cc",
],
copts = micore_tf_copts(),
linkshared = 1,
deps = [
":sentencepiece_tokenizer_h",
":optimized_encoder",
] + micore_tf_deps(),
)

cc_library(
name = "sentencepiece_tokenizer_tflite",
srcs = ["sentencepiece_tokenizer_tflite.cc"],
Expand All @@ -150,13 +161,6 @@ cc_library(
],
)

tf_gen_op_wrapper_py(
name = "gen_sentencepiece_tokenizer_op",
out = "gen_sentencepiece_tokenizer_op.py",
op_whitelist = ["TFSentencepieceTokenizeOp"],
deps = [":sentencepiece_tokenizer_op"],
)

cc_test(
name = "optimized_encoder_test",
srcs = [
Expand All @@ -170,9 +174,11 @@ cc_test(
":encoder_config",
":model_converter",
":optimized_encoder",
"//third_party/sentencepiece/src:sentencepiece_cc_proto",
"//third_party/sentencepiece/src:sentencepiece_processor",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:str_format",
"@com_google_googletest//:gtest_main",
"@com_google_sentencepiece//src:sentencepiece_cc_proto",
"@com_google_sentencepiece//src:sentencepiece_processor",
"@org_tensorflow//tensorflow/core:lib",
],
)
Expand All @@ -199,9 +205,99 @@ pybind_extension(
module_name = "pywrap_tflite_registerer",
deps = [
":py_tflite_registerer",
"//third_party/pybind11",
"//third_party/python_runtime:headers",
"@local_config_python//:python_headers",
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
"@pybind11",
],
)

pybind_extension(
name = "pywrap_model_converter",
srcs = ["pywrap_model_converter.cc"],
hdrs = ["model_converter.h"],
additional_exported_symbols = [
"ConvertSentencepieceModel",
"GetVocabularySize",
],
copts = ["-fexceptions"],
features = ["-use_header_modules"],
module_name = "pywrap_model_converter",
deps = [
":model_converter",
"@com_google_absl//absl/status",
"@local_config_python//:python_headers",
"@pybind11",
],
)

config_setting(
name = "armeabi_v7a_and_fastbuild",
values = {
"cpu": "armeabi-v7a",
"compilation_mode": "fastbuild",
},
visibility = ["//visibility:public"],
)

config_setting(
name = "armeabi_v7a_and_dbg",
values = {
"cpu": "armeabi-v7a",
"compilation_mode": "dbg",
},
visibility = ["//visibility:public"],
)

config_setting(
name = "android",
values = {"crosstool_top": "//external:android/crosstool"},
visibility = ["//visibility:public"],
)

config_setting(
name = "macos_i386",
values = {
"apple_platform_type": "macos",
"cpu": "darwin",
},
visibility = ["//visibility:public"],
)

config_setting(
name = "macos_x86_64",
values = {
"apple_platform_type": "macos",
"cpu": "darwin_x86_64",
},
visibility = ["//visibility:public"],
)

alias(
name = "macos",
actual = select({
":macos_i386": ":macos_i386",
":macos_x86_64": ":macos_x86_64",
"//conditions:default": ":macos_i386", # Arbitrarily chosen from above.
}),
visibility = ["//visibility:public"],
)

config_setting(
name = "ios",
values = {
"crosstool_top": "@bazel_tools//tools/cpp:toolchain",
"apple_platform_type": "ios",
},
visibility = ["//visibility:public"],
)

alias(
name = "apple",
actual = select({
":macos": ":macos",
":ios": ":ios",
"//conditions:default": ":ios", # Arbitrarily chosen from above.
}),
visibility = ["//visibility:public"],
)
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ limitations under the License.

#include <algorithm>

#include "third_party/darts_clone/include/darts.h"
#include "absl/memory/memory.h"
#include "include/darts.h"

namespace tflite {
namespace support {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ limitations under the License.

#include "tensorflow_lite_support/custom_ops/kernel/sentencepiece/double_array_trie_builder.h"
#include "tensorflow_lite_support/custom_ops/kernel/sentencepiece/encoder_config_generated.h"
#include "src/sentencepiece_model.proto.h"
#include "absl/status/status.h"
#include "src/sentencepiece_model.pb.h"

namespace tflite {
namespace support {
namespace ops {
Expand Down Expand Up @@ -50,7 +50,7 @@ DecodePrecompiledCharsmap(
std::vector<int8_t>(normalized_ptr, normalized_ptr + normalized_size));
}

absl::StatusOr<std::string> ConvertSentencepieceModelToFlatBuffer(
tflite::support::StatusOr<std::string> ConvertSentencepieceModelToFlatBuffer(
const std::string& model_config_str, int encoding_offset) {
::sentencepiece::ModelProto model_config;
if (!model_config.ParseFromString(model_config_str)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ limitations under the License.
#define THIRD_PARTY_TENSORFLOW_LITE_SUPPORT_CUSTOM_OPS_KERNEL_SENTENCEPIECE_MODEL_CONVERTER_H_
#include <string>

#include "absl/status/statusor.h"
#include "tensorflow_lite_support/cc/port/statusor.h"

namespace tflite {
namespace support {
namespace ops {
// Converts Sentencepiece configuration to flatbuffer format.
// encoding_offset is used by some encoders that combine different encodings.
absl::StatusOr<std::string> ConvertSentencepieceModelToFlatBuffer(
tflite::support::StatusOr<std::string> ConvertSentencepieceModelToFlatBuffer(
const std::string& model_config_str, int encoding_offset = 0);

// The functions that are provided for the Python wrapper.
Expand Down
86 changes: 86 additions & 0 deletions tensorflow_lite_support/custom_ops/kernel/sentencepiece/native.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
"""Build definitions supporting platform-independent native build."""

load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_copts", "tf_opts_nortti_if_android")
load("@bazel_skylib//lib:selects.bzl", "selects")

def micore_if(android, ios = [], default = []):
"""Helper to create a select.
Args:
android: what to return if compiling for Android.
ios: what to return if compiling for iOS.
default: what to return otherwise.
Returns:
the `android` list for Android compilation and the
`default` list otherwise.
"""
return select({
":android": android,
":apple": ios,
"//conditions:default": default,
})

def micore_tf_copts():
"""C options for Tensorflow builds.
Returns:
a list of copts which must be used by each cc_library which
refers to Tensorflow. Enables the library to compile both for
Android and for Linux.
"""
return tf_copts(android_optimization_level_override = None) + tf_opts_nortti_if_android() + [
"-Wno-narrowing",
"-Wno-sign-compare",
"-Wno-overloaded-virtual",
] + micore_if(
android = [
# Set a define so Tensorflow's register_types.h
# adopts to support a rich set of types, to be pruned by
# selective registration.
"-DSUPPORT_SELECTIVE_REGISTRATION",
# Selective registration uses constexprs with recursive
# string comparisons; that can lead to compiler errors, so
# we increase the constexpr recursion depth.
"-fconstexpr-depth=1024",
],
) + selects.with_or({
# If building for armeabi-v7a, and if compilation_mode is 'fastbuild'
# or 'dbg' then forcefully add -Oz to the list compiler options.
# Without it, some TF dependencies can't build (b/112286436). If
# compilation_mode is 'opt' then rely on the toolchain default.
(
":armeabi_v7a_and_fastbuild",
":armeabi_v7a_and_dbg",
): ["-Oz"],
"//conditions:default": [],
})

def micore_tf_deps():
"""Dependencies for Tensorflow builds.
Returns:
list of dependencies which must be used by each cc_library
which refers to Tensorflow. Enables the library to compile both for
Android and for Linux. Use this macro instead of directly
declaring dependencies on Tensorflow.
"""
return micore_if(
android = [
# Link to library which does not contain any ops.
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
"@gemmlowp//:eight_bit_int_gemm",
"@fft2d//:fft2d",
],
ios = [
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
"@gemmlowp//:eight_bit_int_gemm",
"@fft2d//:fft2d",
],
default = [
# Standard references for Tensorflow when building for Linux. We use
# an indirection via the alias targets below, to facilitate whitelisting
# these deps in the mobile license presubmit checks.
"@local_config_tf//:libtensorflow_framework",
"@local_config_tf//:tf_header_lib",
],
)
Loading

0 comments on commit 03ba6dd

Please sign in to comment.